- developers - lists.mariadb.org

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2706: Fix for bug 39200.
by noreply＠launchpad.net 02 Jun '09

02 Jun '09

------------------------------------------------------------ revno: 2706 committer: Michael Widenius <monty(a)mysql.com> branch nick: mysql-5.1-maria-lp timestamp: Tue 2009-06-02 12:58:27 +0300 message: Fix for bug 39200. modified: mysql-test/suite/maria/r/maria3.result mysql-test/suite/maria/t/maria3.test sql/sql_table.cc === modified file 'mysql-test/suite/maria/r/maria3.result' --- mysql-test/suite/maria/r/maria3.result 2009-02-19 09:01:25 +0000 +++ mysql-test/suite/maria/r/maria3.result 2009-06-02 09:58:27 +0000 @@ -123,7 +123,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL -) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=PAGE TRANSACTIONAL=0 +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=DYNAMIC TRANSACTIONAL=0 alter table t1 row_format=DYNAMIC; show create table t1; Table Create Table @@ -151,6 +151,48 @@ `a` int(11) DEFAULT NULL ) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=DYNAMIC drop table t1; +create table t1 (a int) transactional=0 row_format=FIXED; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=FIXED TRANSACTIONAL=0 +alter table t1 transactional=1; +Warnings: +Note 1478 Row format set to PAGE because of TRANSACTIONAL=1 option +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=PAGE TRANSACTIONAL=1 +alter table t1 transactional=0; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=FIXED TRANSACTIONAL=0 +drop table t1; +create table t1 (a int) transactional=0 row_format=FIXED; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=FIXED TRANSACTIONAL=0 +alter table t1 transactional=1; +Warnings: +Note 1478 Row format set to PAGE because of TRANSACTIONAL=1 option +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=PAGE TRANSACTIONAL=1 +alter table t1 transactional=0; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MARIA DEFAULT CHARSET=latin1 PAGE_CHECKSUM=0 ROW_FORMAT=FIXED TRANSACTIONAL=0 +drop table t1; create table `t1` ( t1_name varchar(255) default null, t1_id int(10) unsigned not null auto_increment, === modified file 'mysql-test/suite/maria/t/maria3.test' --- mysql-test/suite/maria/t/maria3.test 2009-02-19 09:01:25 +0000 +++ mysql-test/suite/maria/t/maria3.test 2009-06-02 09:58:27 +0000 @@ -109,6 +109,30 @@ show create table t1; drop table t1; +# +# MySQL Bug#39200: optimize table does not recognize ROW_FORMAT=COMPRESSED +# + +create table t1 (a int) transactional=0 row_format=FIXED; +show create table t1; +alter table t1 transactional=1; +show create table t1; +alter table t1 transactional=0; +show create table t1; +drop table t1; + +# +# MySQL Bug#39200: optimize table does not recognize ROW_FORMAT=COMPRESSED +# + +create table t1 (a int) transactional=0 row_format=FIXED; +show create table t1; +alter table t1 transactional=1; +show create table t1; +alter table t1 transactional=0; +show create table t1; +drop table t1; + # CHECK TABLE was reporting # "Size of datafile is: 0 Should be: 16384" # === modified file 'sql/sql_table.cc' --- sql/sql_table.cc 2009-04-25 10:05:32 +0000 +++ sql/sql_table.cc 2009-06-02 09:58:27 +0000 @@ -5323,7 +5323,7 @@ create_info->used_fields & HA_CREATE_USED_ENGINE || create_info->used_fields & HA_CREATE_USED_CHARSET || create_info->used_fields & HA_CREATE_USED_DEFAULT_CHARSET || - create_info->used_fields & HA_CREATE_USED_ROW_FORMAT || + (table->s->row_type != create_info->row_type) || create_info->used_fields & HA_CREATE_USED_PAGE_CHECKSUM || create_info->used_fields & HA_CREATE_USED_TRANSACTIONAL || create_info->used_fields & HA_CREATE_USED_PACK_KEYS || @@ -6305,7 +6305,10 @@ } if (create_info->row_type == ROW_TYPE_NOT_USED) + { create_info->row_type= table->s->row_type; + create_info->used_fields |= HA_CREATE_USED_ROW_FORMAT; + } DBUG_PRINT("info", ("old type: %s new type: %s", ha_resolve_storage_engine_name(old_db_type), -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] New (by Monty): MERGE tables for Maria (28)
by worklog-noreply＠askmonty.org 02 Jun '09

02 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: MERGE tables for Maria CREATION DATE..: Tue, 02 Jun 2009, 01:39 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 28 (http://askmonty.org/worklog/?tid=28) VERSION........: Server-5.2 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 80 (hours remain) ORIG. ESTIMATE.: 80 PROGRESS NOTES: DESCRIPTION: Make a version of MERGE tables for Maria. This is needed to make Maria a full MyISAM replacement. (And yes, Partition is not a replacement for MERGE table) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 02 Jun 2009, 00:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.23548 2009-06-02 00:54:13.000000000 +0300 +++ /tmp/wklog.17.new.23548 2009-06-02 00:54:13.000000000 +0300 @@ -128,3 +128,10 @@ - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. + +* Table elimination is performed after constant table detection (but before + the range analysis). Constant tables are technically different from + eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). + Considering we've already done the join_read_const_table() call, is there any + real difference between constant table and eliminated one? If there is, should + we mark const tables also as eliminated? -=-=(Psergey - Mon, 01 Jun 2009, 20:46)=-=- Low Level Design modified. --- /tmp/wklog.17.old.17448 2009-06-01 20:46:40.000000000 +0300 +++ /tmp/wklog.17.new.17448 2009-06-01 20:46:40.000000000 +0300 @@ -122,3 +122,9 @@ always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. + +* Should eliminated tables be shown in EXPLAIN EXTENDED? + - If we just ignore the question, they will be shown + - this is what happens for constant tables, too. + - I don't see how showing them could be of any use. They only make it + harder to read the rewritten query. -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 ------------------------------------------------------------ -=-=(View All Progress Notes, 18 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. * Should eliminated tables be shown in EXPLAIN EXTENDED? - If we just ignore the question, they will be shown - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. * Table elimination is performed after constant table detection (but before the range analysis). Constant tables are technically different from eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). Considering we've already done the join_read_const_table() call, is there any real difference between constant table and eliminated one? If there is, should we mark const tables also as eliminated? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 02 Jun 2009, 00:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.23548 2009-06-02 00:54:13.000000000 +0300 +++ /tmp/wklog.17.new.23548 2009-06-02 00:54:13.000000000 +0300 @@ -128,3 +128,10 @@ - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. + +* Table elimination is performed after constant table detection (but before + the range analysis). Constant tables are technically different from + eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). + Considering we've already done the join_read_const_table() call, is there any + real difference between constant table and eliminated one? If there is, should + we mark const tables also as eliminated? -=-=(Psergey - Mon, 01 Jun 2009, 20:46)=-=- Low Level Design modified. --- /tmp/wklog.17.old.17448 2009-06-01 20:46:40.000000000 +0300 +++ /tmp/wklog.17.new.17448 2009-06-01 20:46:40.000000000 +0300 @@ -122,3 +122,9 @@ always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. + +* Should eliminated tables be shown in EXPLAIN EXTENDED? + - If we just ignore the question, they will be shown + - this is what happens for constant tables, too. + - I don't see how showing them could be of any use. They only make it + harder to read the rewritten query. -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 ------------------------------------------------------------ -=-=(View All Progress Notes, 18 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. * Should eliminated tables be shown in EXPLAIN EXTENDED? - If we just ignore the question, they will be shown - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. * Table elimination is performed after constant table detection (but before the range analysis). Constant tables are technically different from eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). Considering we've already done the join_read_const_table() call, is there any real difference between constant table and eliminated one? If there is, should we mark const tables also as eliminated? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: 9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Category updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-RawIdeaBin -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Version updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Server-9.x +9.x -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: 9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Category updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-RawIdeaBin -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Version updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Server-9.x +9.x -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 01 Jun 2009, 20:46)=-=- Low Level Design modified. --- /tmp/wklog.17.old.17448 2009-06-01 20:46:40.000000000 +0300 +++ /tmp/wklog.17.new.17448 2009-06-01 20:46:40.000000000 +0300 @@ -122,3 +122,9 @@ always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. + +* Should eliminated tables be shown in EXPLAIN EXTENDED? + - If we just ignore the question, they will be shown + - this is what happens for constant tables, too. + - I don't see how showing them could be of any use. They only make it + harder to read the rewritten query. -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint ------------------------------------------------------------ -=-=(View All Progress Notes, 17 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. * Should eliminated tables be shown in EXPLAIN EXTENDED? - If we just ignore the question, they will be shown - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 01 Jun 2009, 20:46)=-=- Low Level Design modified. --- /tmp/wklog.17.old.17448 2009-06-01 20:46:40.000000000 +0300 +++ /tmp/wklog.17.new.17448 2009-06-01 20:46:40.000000000 +0300 @@ -122,3 +122,9 @@ always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. + +* Should eliminated tables be shown in EXPLAIN EXTENDED? + - If we just ignore the question, they will be shown + - this is what happens for constant tables, too. + - I don't see how showing them could be of any use. They only make it + harder to read the rewritten query. -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint ------------------------------------------------------------ -=-=(View All Progress Notes, 17 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. * Should eliminated tables be shown in EXPLAIN EXTENDED? - If we just ignore the question, they will be shown - this is what happens for constant tables, too. - I don't see how showing them could be of any use. They only make it harder to read the rewritten query. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Bzr merge order
by Kristian Nielsen 01 Jun '09

01 Jun '09

I noticed a difference between Bitkeeper and Bzr merges. In Bitkeeper, merge parents are essentially unordered. The two parents of a merge are completely equal, and there is no way from the Bitkeeper to history to say whether one parent was merged into the other or vise versa. The actual order in which merge parents are listed is based on some deterministic algorithm (most recent commit date if I recall correctly). In particular, this means that in Bitkeeper, the result of a pull of A into B is exactly identical to a pull of B into A. This makes it impossible to establish the history of pushes into main trees from just the Bitkeeper history. In Bzr, merge parents _are_ ordered. Merging B into A gives a different result from merging A into B. When merging B into A (cd A; bzr merge B), A is the primary or left-hand parent. The resulting tree has one extra top-level merge revision following the tip of A, with all merged revisions of B appearing as sub-revisions of this. This means that if one merges A into B giving C1, and later merges B into A giving C2, then C1 and C2 are considered diverged and need another merge changeset to be consolidated (in Bitkeeper C1 and C2 are identical and need no further merge). What this means is that in bzr is _is_ actually possible to see the history of what was merged into what, at least in some sense. Unfortunately, it tends to be backwards with usual MySQL working style. This is to make a branch at some point X, make a patch, commit and get review. Then when pushing, the main tree has gotten additional pushes by others and is now at revision Y > X. So one does a bzr merge of the main tree into the local clone, followed by a push to the main tree. Now the local patch becomes the main merge parent, and all other pushes between X and Y become sub-revisions with their revision numbers renamed. Which eventually makes primary/secondary merge parent relationships more or less random. So since I did not take part in the transition to bzr within Sun, I just wanted to ask if this is something that was discussed, and if so if there were any conclusions? As I understand, the Drizzle people make sure that whenever they merge, they merge into a copy of their trunk, so that all past pushes are visible as the line of primary/left-most merge parents in the main tree. This works better for them, as they follow the model of a single/few merge captains merging in other peoples trees upon merge requests. My personal opinion is that while it would be nice to have the push history available like the Drizzle people do, the bzr support for this is not good enough, without a very tight control on who can push to main trees that we do not want. What one can do is when pushing, first swith to an up-to-date fresh clone of the main tree, pull from the local branch with own changes, then push that to the main tree. Maybe this is a good way, even though it cannot be enforced? Just wanted to hear if there are any other opinions on this? - Kristian.

3 8

[Maria-developers] Buildbot questions
by Sergey Petrunya 01 Jun '09

01 Jun '09

Hi! I've got several questions for our instance of buildbot: Grid display (http://askmonty.org/buildbot/grid) is very nice, but it would be even nicer if one could display only pushes into one particular branch, like in pushbuild. Is it difficult to achieve? If not, can I request it from somebody or will need to spend the effort myself? Can we have buildbot automatically re-build things, say, daily when no other load is present? I've re-ran maria-5.1 tests on my machine several times and it failed all three times with different number of failing tests, which is an indication of precence of random failures. (In Sun/MySQL I used to push fixes that touch only comments in order to trigger re-builds and catch random failures) Can we have buildbot also process other branches? maria trees are public- facing trees, so I suppose that means we should not break them too often, and one way to do that is to have staging trees where features could be tested before they are pushed into the main tree. One could argue that this could be achieved by autopush, but autopush only uses one platform, doesn't save/keep track of test run results, etc. My experience at Sun/MySQL shows that pushbuild (and I suppose buildbot) over a separate branch is much better. It runs different tests on different platforms, pedantically records test results (where everyone can see them), etc etc. Is it possible to set up buildbot to run several configurations, but one after the other? Looking at the buildbot status page, I see that - there are not too many hosts - they are idle most of the time. Suppose I want to use my 32/64 box to build 32 and 64 bit, debug and release, using gcc and sunstudio. That's 2^3 = 8 combinations, but I don't want to have 8 build bots running in parallel (I dont think the box has enough RAM for that)? p.s. I myself have almost repaired my desktop computer, so we can expect to have a dual 32/64-bit buildbot slave to be added as soon as I finish setting things up (sometime this week). BR Sergey -- Sergey Petrunia, Software Developer Monty Program AB, http://askmonty.org Blog: http://s.petrunia.net/blog

4 3

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 ------------------------------------------------------------ -=-=(View All Progress Notes, 16 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 01 Jun '09

01 Jun '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 01 Jun 2009, 12:49)=-=- Low Level Design modified. --- /tmp/wklog.17.old.32202 2009-06-01 12:49:15.000000000 +0300 +++ /tmp/wklog.17.new.32202 2009-06-01 12:49:15.000000000 +0300 @@ -8,7 +8,7 @@ 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved - +7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner @@ -116,3 +116,9 @@ * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? +* Aggregate functions report they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + + always. If we want table elimination to work in presence of grouping, need + to devise some other way of analyzing aggregate functions. -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 ------------------------------------------------------------ -=-=(View All Progress Notes, 16 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved 7. Additional issues </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? * Aggregate functions report they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. If we want table elimination to work in presence of grouping, need to devise some other way of analyzing aggregate functions. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 28 May '09

28 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned ------------------------------------------------------------ -=-=(View All Progress Notes, 15 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 28 May '09

28 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 29 May 2009, 00:45)=-=- Low Level Design modified. --- /tmp/wklog.17.old.1348 2009-05-29 00:45:21.000000000 +0300 +++ /tmp/wklog.17.new.1348 2009-05-29 00:45:21.000000000 +0300 @@ -111,3 +111,8 @@ referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. +7. Additional issues +-------------------- +* We remove ON clauses within semi-join nests. If these clauses contain + subqueries, they probably should be gone from EXPLAIN output also? + -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned ------------------------------------------------------------ -=-=(View All Progress Notes, 15 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. 7. Additional issues -------------------- * We remove ON clauses within semi-join nests. If these clauses contain subqueries, they probably should be gone from EXPLAIN output also? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] bzr commit into mariadb 6.0, with Maria 1.5:maria/6.0 branch (knielsen:3040)
by knielsen＠knielsen-hq.org 28 May '09

28 May '09

#At file:///space/x/bzr2git/bzr-repo/tmp-6.0/ 3040 knielsen(a)knielsen-hq.org 2009-05-28 42 modified: README === modified file 'README' --- a/README 2008-03-26 10:15:03 +0000 +++ b/README 2009-05-28 13:49:29 +0000 @@ -33,3 +33,4 @@ file. IMPORTANT: Bug or error reports should be sent to http://bugs.mysql.com. +

1 0

[Maria-developers] [Merge] lp:~maria-captains/maria/slow-extended-patch into lp:maria
by Vadim Tkachenko 28 May '09

28 May '09

Vadim Tkachenko has proposed merging lp:~maria-captains/maria/slow-extended-patch into lp:maria. Requested reviews: Maria-captains (maria-captains) Merge contains patch http://www.percona.com/docs/wiki/patches:microslow_innodb (not InnoDB part, all InnoDB related things will be included in XtraDB) -- https://code.launchpad.net/~maria-captains/maria/slow-extended-patch/+merge… Your team Maria developers is subscribed to branch lp:maria.

1 0

[Maria-developers] Updated (by Monty): Remove all warnings (25)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Remove all warnings CREATION DATE..: Tue, 26 May 2009, 19:22 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-BackLog TASK ID........: 25 (http://askmonty.org/worklog/?tid=25) VERSION........: Server-5.1 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 20 (hours remain) ORIG. ESTIMATE.: 20 PROGRESS NOTES: -=-=(Monty - Wed, 27 May 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.25.old.10351 2009-05-27 15:47:46.000000000 +0300 +++ /tmp/wklog.25.new.10351 2009-05-27 15:47:46.000000000 +0300 @@ -1 +1,9 @@ +Most of the warnings comes from the included readline libary. + +One of the first tasks should be to update ours (based on 5.2) to the latest +readline version (6.0). After that: +- Update it with our changes to the readline (mostly about getting rid of warnings) +- Fix so that there is no warnings from readline +- Update MariaDB with the new fixed library +- Send a patch of the fixed warnings to the current maintainer of readline DESCRIPTION: Remove all compiler warnings from MariaDB: - Remove all current warnings - Add flag 'abort on compiler error - Add flags to get more compiler errors Drizzle has already done this, so we should be able to get hints from their code for some of the fixes and the compiler warning flags HIGH-LEVEL SPECIFICATION: Most of the warnings comes from the included readline libary. One of the first tasks should be to update ours (based on 5.2) to the latest readline version (6.0). After that: - Update it with our changes to the readline (mostly about getting rid of warnings) - Fix so that there is no warnings from readline - Update MariaDB with the new fixed library - Send a patch of the fixed warnings to the current maintainer of readline ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Monty): Change strlen() to LEX_STRING or String (27)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change strlen() to LEX_STRING or String CREATION DATE..: Wed, 27 May 2009, 15:45 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-BackLog TASK ID........: 27 (http://askmonty.org/worklog/?tid=27) VERSION........: Server-5.2 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 100 (hours remain) ORIG. ESTIMATE.: 100 PROGRESS NOTES: -=-=(Monty - Wed, 27 May 2009, 15:46)=-=- High Level Description modified. --- /tmp/wklog.27.old.10318 2009-05-27 15:46:46.000000000 +0300 +++ /tmp/wklog.27.new.10318 2009-05-27 15:46:46.000000000 +0300 @@ -8,3 +8,5 @@ - Faster, smaller code - Safer for strings/character sets that contains \0 - Allows us to change c_ptr() to ptr() in some cases + +This is same task as the WL#2894 at MySQL. DESCRIPTION: Remove all strlen() from MariaDB. The objects we are using strlen() on should be changed to either LEX_STRING or String. Reasons for doing this cleanup: - Faster, smaller code - Safer for strings/character sets that contains \0 - Allows us to change c_ptr() to ptr() in some cases This is same task as the WL#2894 at MySQL. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Monty): Change strlen() to LEX_STRING or String (27)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change strlen() to LEX_STRING or String CREATION DATE..: Wed, 27 May 2009, 15:45 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-BackLog TASK ID........: 27 (http://askmonty.org/worklog/?tid=27) VERSION........: Server-5.2 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 100 (hours remain) ORIG. ESTIMATE.: 100 PROGRESS NOTES: DESCRIPTION: Remove all strlen() from MariaDB. The objects we are using strlen() on should be changed to either LEX_STRING or String. Reasons for doing this cleanup: - Faster, smaller code - Safer for strings/character sets that contains \0 - Allows us to change c_ptr() to ptr() in some cases ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Better choice between range and index_merge/intersection options (26)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Better choice between range and index_merge/intersection options CREATION DATE..: Wed, 27 May 2009, 15:20 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 26 (http://askmonty.org/worklog/?tid=26) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: The optimizer does a cost-based choice between possible range and index_merge/intersection scans. There are some issues with it: - index_merge/intersection gets chosen even when there is a single multi-part index that covers all keys. Measurements show that this is a poor choice. - The picked index_merge/intersection can use a redundant set of indexes: it will be intersect(idx1, ..., idxN) where all columns in idxN are covered by other used indexes. This WL is to fix these limitations. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Better choice between range and index_merge/intersection options (26)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Better choice between range and index_merge/intersection options CREATION DATE..: Wed, 27 May 2009, 15:20 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 26 (http://askmonty.org/worklog/?tid=26) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: The optimizer does a cost-based choice between possible range and index_merge/intersection scans. There are some issues with it: - index_merge/intersection gets chosen even when there is a single multi-part index that covers all keys. Measurements show that this is a poor choice. - The picked index_merge/intersection can use a redundant set of indexes: it will be intersect(idx1, ..., idxN) where all columns in idxN are covered by other used indexes. This WL is to fix these limitations. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: 9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Category updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-RawIdeaBin -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Version updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Server-9.x +9.x -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: 9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Category updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-RawIdeaBin -=-=(Guest - Wed, 27 May 2009, 14:41)=-=- Version updated. --- /tmp/wklog.24.old.8414 2009-05-27 14:41:43.000000000 +0300 +++ /tmp/wklog.24.new.8414 2009-05-27 14:41:43.000000000 +0300 @@ -1 +1 @@ -Server-9.x +9.x -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 27 May '09

27 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Version updated. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -Benchmarks-3.0 +Server-9.x -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to ------------------------------------------------------------ -=-=(View All Progress Notes, 14 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 21:52)=-=- Low Level Design modified. --- /tmp/wklog.17.old.14120 2009-05-26 21:52:06.000000000 +0300 +++ /tmp/wklog.17.new.14120 2009-05-26 21:52:06.000000000 +0300 @@ -1,11 +1,14 @@ <contents> 1. Conditions for removal +1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface -5. Todo, issues to resolve -5.1 To resolve -5.2 Resolved +5. Tests and benchmarks +6. Todo, issues to resolve +6.1 To resolve +6.2 Resolved + </contents> It's not really about elimination of tables, it's about elimination of inner @@ -29,6 +32,18 @@ GROUP BY and HAVING do not refer to the inner tables of the outer join nest. +1.1 Quick check if there are candidates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start to enumerate join nests, here is a quick way to check if +there *can be* something to be removed: + + if ((tables used in select_list | + tables used in group/order by UNION | + tables used in where) != bitmap_of_all_tables) + { + attempt table elimination; + } + 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) @@ -56,22 +71,24 @@ ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. + (Note ^^ utility of the above questioned ^, as table elimination can never + be worse than no elimination. We're leaning towards not adding the flag) -* With EXPLAIN, there are two options: - - Show removed tables in a way similar to const tables, with some - indication that they are removed. - - Do not show them altogether. -(the second one seems to be better? We're targeting a situation with VIEWs, -where the user would not care about what tables were added into his query -and then discarded from it?) +* EXPLAIN will not show the removed tables at all. This will allow to check + if tables were removed, and also will behave nicely with anchor model and + VIEWs: stuff that user doesn't care about just won't be there. + +5. Tests and benchmarks +----------------------- +Should create a benchmark in sql-bench which checks if the dbms has table +elimination. +TODO elaborate -5. Todo, issues to resolve +6. Todo, issues to resolve -------------------------- -5.1 To resolve +6.1 To resolve ~~~~~~~~~~~~~~ -- See EXPLAIN question in section #4. - - Re-check how this works with equality propagation. - Relationship with prepared statements. @@ -87,7 +104,7 @@ that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. -5.2 Resolved +6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to ------------------------------------------------------------ -=-=(View All Progress Notes, 14 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 1.1 Quick check if there are candidates 2. Removal operation properties 3. Removal operation 4. User interface 5. Tests and benchmarks 6. Todo, issues to resolve 6.1 To resolve 6.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 1.1 Quick check if there are candidates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start to enumerate join nests, here is a quick way to check if there *can be* something to be removed: if ((tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_tables) { attempt table elimination; } 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. (Note ^^ utility of the above questioned ^, as table elimination can never be worse than no elimination. We're leaning towards not adding the flag) * EXPLAIN will not show the removed tables at all. This will allow to check if tables were removed, and also will behave nicely with anchor model and VIEWs: stuff that user doesn't care about just won't be there. 5. Tests and benchmarks ----------------------- Should create a benchmark in sql-bench which checks if the dbms has table elimination. TODO elaborate 6. Todo, issues to resolve -------------------------- 6.1 To resolve ~~~~~~~~~~~~~~ - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 6.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Monty): Remove all warnings (25)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Remove all warnings CREATION DATE..: Tue, 26 May 2009, 19:22 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-BackLog TASK ID........: 25 (http://askmonty.org/worklog/?tid=25) VERSION........: Server-5.1 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 20 (hours remain) ORIG. ESTIMATE.: 20 PROGRESS NOTES: DESCRIPTION: Remove all compiler warnings from MariaDB: - Remove all current warnings - Add flag 'abort on compiler error - Add flags to get more compiler errors Drizzle has already done this, so we should be able to get hints from their code for some of the fixes and the compiler warning flags ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: non-ROR intersection (21)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: non-ROR intersection CREATION DATE..: Thu, 21 May 2009, 21:32 SUPERVISOR.....: Knielsen IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 21 (http://askmonty.org/worklog/?tid=21) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 14:04)=-=- High-Level Specification modified. --- /tmp/wklog.21.old.1802 2009-05-26 14:04:57.000000000 +0300 +++ /tmp/wklog.21.new.1802 2009-05-26 14:04:57.000000000 +0300 @@ -1,4 +1,3 @@ - <contents> 1. Execution 1.1 Temptable @@ -30,6 +29,8 @@ 1.1 Temptable ------------- +[ This is our strategy of choice at the moment] + Use a temporary heap-grow-out-to-myisam table with a primary key: create table temp_table ( @@ -168,3 +169,8 @@ a subset of columns covered by all other indexes. = (TODO any other rules?) +- Correlation across selectivities. If there is a condition + + "cond(key1) AND cond(key2) AND ... AND cond(keyN)", + + can we consider satisfaction of AND-parts to be independent? -=-=(Psergey - Thu, 21 May 2009, 21:33)=-=- High-Level Specification modified. --- /tmp/wklog.21.old.25705 2009-05-21 21:33:02.000000000 +0300 +++ /tmp/wklog.21.new.25705 2009-05-21 21:33:02.000000000 +0300 @@ -1 +1,170 @@ +<contents> +1. Execution +1.1 Temptable +1.1.1 Improvement +1.2 Produce/merge sorted streams +1.3 Extend Unique class to handle intersection +1.4 Strategies that do not seem to be useful +1.4.1 Remove matches after having produced an ordered stream +1.4.2 Sparse rowid bitmaps +2. Optimization + +</contents> + +1. Execution +============ + +The primary task is to find means to compute an intersection of N unordered +streams. Besides general memory/cpu cost of computation, we consider: + +- whether the produced rowid stream is ordered. If it is, it can be piped + into index_merge/intersect (as opposed to sort-intersect) + +- whether the strategy can take advantage of the fact that some input streams + are already rowid-ordered + +- startup cost (cost of producing the first output record) + +We see the following possible strategies: + +1.1 Temptable +------------- +Use a temporary heap-grow-out-to-myisam table with a primary key: + +create table temp_table ( + rowid binary($rowid_size), + count n, + primary key(rowid); +); + +Then use this algorithm: + + i1= {index with the least E(#records)}; + + for each record R in range_scan(i1) + temp_table.insert(R.rowid, count=1); + + for each index idx except i1 + { + for each R record in scan(idx) // (INNER-LOOP) + { + if (temp_table has R) + temptable[R].count++; + } + } + + // The following loop can do ordered or unordered scan + // if we want it to be ordered scan, we probably better arrange so that + // 'count' column is part of the index. + for each record R in temp_table + { + if (R.count == number_of_streams) + emit(R.rowid); + } + +The algorithm has an option to emit an ordered rowid stream. + +In the above form, the cost to produce the first record is high. It's easy to +adjust the algorithm to make it low - we'll need to just start scanning all +indexes at once, and finish as soon as we got a full match, i.e. the + + temptable[R].count++ + +operation resulted in the counter being equal to the number of merged scans. + +1.1.1 Improvement +~~~~~~~~~~~~~~~~~ +When running INNER-LOOP, we could count how many times we've done the +"count++" operation. If it has been done #records-in-temptable times, that +means that all further records will not have matches and we can finish the +scan, i.e. break out of the INNER-LOOP. + +1.2 Produce/merge sorted streams +-------------------------------- +For each of the merged scan, use filesort-like action to end up with an +ordered stream of rowids. Then merge the ordered streams. + +By filesort-like action we mean + - Run over index, collect rowids in a buffer. + - When the buffer is full, sort it and dump into a temporary file. +After the above we'll end up with a number of sorted buffers on disk. We can +use mergebuff() function (it is part of filesort's functions) to produce one +ordered sequence (i.e. array, which may be partially on disk) of rowids. + +Merging of ordered streams with help of priority queue is already implemented +in QUICK_ROR_INTERSECT_SELECT. We'll need to substitute the + + child_quick->get_next() + +call with a call to read rowid from an ordered sequence. + +1.3 Extend Unique class to handle intersection +---------------------------------------------- +There is no point to use Unique object as a device that accumulates rowids of +a single scan then produces them in sorted order. One could do the same faster +with accumulating an array of rowids and then sorting it. + +It's possible to use Unique object to collect/merge data from all scans though. +The idea is as follows: + +- Unique should store <rowid, n_scans> pairs +- Duplicates are pairs with the same rowid +- Unique should try to avoid creating duplicates: + - don't add a duplicate into the in-memory part, instead combine two elements + together by adding their n_scans elements. + - combine duplicates when it sees them in Unique.get() call +- The data we get from Unique.get() should be filtered, all records that have + n_scans != number_of_scans_being_merged should be discarded. + +If we're lucky to have started and finished a scan on some index (denote it +as S) without flushing the Unique in the process, then: +- there is no point in adding any new records into the Unique because their + absence in the Unique means that they don't have match in S and hence will + not get into the result of intersection. +- we need to only update the counters to be able to tell if the elements that + are already in the Unique will have matches in all scans. + +1.4 Strategies that do not seem to be useful +-------------------------------------------- + +keeping them here so we don't consider them over and over + +1.4.1 Remove matches after having produced an ordered stream +------------------------------------------------------------ +We can dump everything into a rowid stream and get it sorted. Then we read it, +and if we see a rowid repeated $n_merged_scans times, it belongs to the +intersection (pass to output), otherwise it doesn't (skip). +This doesn't have any advantages over the produce/merge sorted streams +approach. + +1.4.2 Sparse rowid bitmaps +-------------------------- +Use Falcon-style rowid bitmaps. The problem with that is that Falcon's +bitmaps assume there will always be enough memory to accommodate them. + +PostgreSQL makes bitmaps "loose" when they exceed certain size by remembering +disk pages, not ids of individual records. It's hard for us to do something +similar because our rowids are opaque entities whose meaning depends on the +storage engines. + +This seems to require too much change to be worth it. + +2. Optimization +=============== + +SEL_TREE objects already represent intersections. The problems with +optimizations are: + +- Cost formula(s) +- When N keys/conditions are present: + + "cond(key1) AND cond(key2) AND ... AND cond(keyN)", + + somehow avoid considering (2^n - n) possible options. + +- Avoid producing (or even considering) apparently suboptimal plans: + = Don't generate a merge of indexes (I_1, ... I_n) where columns of I_n are + a subset of columns covered by all other indexes. + = (TODO any other rules?) + DESCRIPTION: At the moment index_merge supports intersection only for rowid-ordered streams. This translates into a limitation that index_merge/intersect can only be constructed for equality conditions (t.keypart1=const1 AND t.keypart2=const2 AND ... ) and the equalities should cover all index components. This WL entry is to lift this limitation by developing algorithms that do intersection on non-ROR scans. HIGH-LEVEL SPECIFICATION: <contents> 1. Execution 1.1 Temptable 1.1.1 Improvement 1.2 Produce/merge sorted streams 1.3 Extend Unique class to handle intersection 1.4 Strategies that do not seem to be useful 1.4.1 Remove matches after having produced an ordered stream 1.4.2 Sparse rowid bitmaps 2. Optimization </contents> 1. Execution ============ The primary task is to find means to compute an intersection of N unordered streams. Besides general memory/cpu cost of computation, we consider: - whether the produced rowid stream is ordered. If it is, it can be piped into index_merge/intersect (as opposed to sort-intersect) - whether the strategy can take advantage of the fact that some input streams are already rowid-ordered - startup cost (cost of producing the first output record) We see the following possible strategies: 1.1 Temptable ------------- [ This is our strategy of choice at the moment] Use a temporary heap-grow-out-to-myisam table with a primary key: create table temp_table ( rowid binary($rowid_size), count n, primary key(rowid); ); Then use this algorithm: i1= {index with the least E(#records)}; for each record R in range_scan(i1) temp_table.insert(R.rowid, count=1); for each index idx except i1 { for each R record in scan(idx) // (INNER-LOOP) { if (temp_table has R) temptable[R].count++; } } // The following loop can do ordered or unordered scan // if we want it to be ordered scan, we probably better arrange so that // 'count' column is part of the index. for each record R in temp_table { if (R.count == number_of_streams) emit(R.rowid); } The algorithm has an option to emit an ordered rowid stream. In the above form, the cost to produce the first record is high. It's easy to adjust the algorithm to make it low - we'll need to just start scanning all indexes at once, and finish as soon as we got a full match, i.e. the temptable[R].count++ operation resulted in the counter being equal to the number of merged scans. 1.1.1 Improvement ~~~~~~~~~~~~~~~~~ When running INNER-LOOP, we could count how many times we've done the "count++" operation. If it has been done #records-in-temptable times, that means that all further records will not have matches and we can finish the scan, i.e. break out of the INNER-LOOP. 1.2 Produce/merge sorted streams -------------------------------- For each of the merged scan, use filesort-like action to end up with an ordered stream of rowids. Then merge the ordered streams. By filesort-like action we mean - Run over index, collect rowids in a buffer. - When the buffer is full, sort it and dump into a temporary file. After the above we'll end up with a number of sorted buffers on disk. We can use mergebuff() function (it is part of filesort's functions) to produce one ordered sequence (i.e. array, which may be partially on disk) of rowids. Merging of ordered streams with help of priority queue is already implemented in QUICK_ROR_INTERSECT_SELECT. We'll need to substitute the child_quick->get_next() call with a call to read rowid from an ordered sequence. 1.3 Extend Unique class to handle intersection ---------------------------------------------- There is no point to use Unique object as a device that accumulates rowids of a single scan then produces them in sorted order. One could do the same faster with accumulating an array of rowids and then sorting it. It's possible to use Unique object to collect/merge data from all scans though. The idea is as follows: - Unique should store <rowid, n_scans> pairs - Duplicates are pairs with the same rowid - Unique should try to avoid creating duplicates: - don't add a duplicate into the in-memory part, instead combine two elements together by adding their n_scans elements. - combine duplicates when it sees them in Unique.get() call - The data we get from Unique.get() should be filtered, all records that have n_scans != number_of_scans_being_merged should be discarded. If we're lucky to have started and finished a scan on some index (denote it as S) without flushing the Unique in the process, then: - there is no point in adding any new records into the Unique because their absence in the Unique means that they don't have match in S and hence will not get into the result of intersection. - we need to only update the counters to be able to tell if the elements that are already in the Unique will have matches in all scans. 1.4 Strategies that do not seem to be useful -------------------------------------------- keeping them here so we don't consider them over and over 1.4.1 Remove matches after having produced an ordered stream ------------------------------------------------------------ We can dump everything into a rowid stream and get it sorted. Then we read it, and if we see a rowid repeated $n_merged_scans times, it belongs to the intersection (pass to output), otherwise it doesn't (skip). This doesn't have any advantages over the produce/merge sorted streams approach. 1.4.2 Sparse rowid bitmaps -------------------------- Use Falcon-style rowid bitmaps. The problem with that is that Falcon's bitmaps assume there will always be enough memory to accommodate them. PostgreSQL makes bitmaps "loose" when they exceed certain size by remembering disk pages, not ids of individual records. It's hard for us to do something similar because our rowids are opaque entities whose meaning depends on the storage engines. This seems to require too much change to be worth it. 2. Optimization =============== SEL_TREE objects already represent intersections. The problems with optimizations are: - Cost formula(s) - When N keys/conditions are present: "cond(key1) AND cond(key2) AND ... AND cond(keyN)", somehow avoid considering (2^n - n) possible options. - Avoid producing (or even considering) apparently suboptimal plans: = Don't generate a merge of indexes (I_1, ... I_n) where columns of I_n are a subset of columns covered by all other indexes. = (TODO any other rules?) - Correlation across selectivities. If there is a condition "cond(key1) AND cond(key2) AND ... AND cond(keyN)", can we consider satisfaction of AND-parts to be independent? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: non-ROR intersection (21)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: non-ROR intersection CREATION DATE..: Thu, 21 May 2009, 21:32 SUPERVISOR.....: Knielsen IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 21 (http://askmonty.org/worklog/?tid=21) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 14:04)=-=- High-Level Specification modified. --- /tmp/wklog.21.old.1802 2009-05-26 14:04:57.000000000 +0300 +++ /tmp/wklog.21.new.1802 2009-05-26 14:04:57.000000000 +0300 @@ -1,4 +1,3 @@ - <contents> 1. Execution 1.1 Temptable @@ -30,6 +29,8 @@ 1.1 Temptable ------------- +[ This is our strategy of choice at the moment] + Use a temporary heap-grow-out-to-myisam table with a primary key: create table temp_table ( @@ -168,3 +169,8 @@ a subset of columns covered by all other indexes. = (TODO any other rules?) +- Correlation across selectivities. If there is a condition + + "cond(key1) AND cond(key2) AND ... AND cond(keyN)", + + can we consider satisfaction of AND-parts to be independent? -=-=(Psergey - Thu, 21 May 2009, 21:33)=-=- High-Level Specification modified. --- /tmp/wklog.21.old.25705 2009-05-21 21:33:02.000000000 +0300 +++ /tmp/wklog.21.new.25705 2009-05-21 21:33:02.000000000 +0300 @@ -1 +1,170 @@ +<contents> +1. Execution +1.1 Temptable +1.1.1 Improvement +1.2 Produce/merge sorted streams +1.3 Extend Unique class to handle intersection +1.4 Strategies that do not seem to be useful +1.4.1 Remove matches after having produced an ordered stream +1.4.2 Sparse rowid bitmaps +2. Optimization + +</contents> + +1. Execution +============ + +The primary task is to find means to compute an intersection of N unordered +streams. Besides general memory/cpu cost of computation, we consider: + +- whether the produced rowid stream is ordered. If it is, it can be piped + into index_merge/intersect (as opposed to sort-intersect) + +- whether the strategy can take advantage of the fact that some input streams + are already rowid-ordered + +- startup cost (cost of producing the first output record) + +We see the following possible strategies: + +1.1 Temptable +------------- +Use a temporary heap-grow-out-to-myisam table with a primary key: + +create table temp_table ( + rowid binary($rowid_size), + count n, + primary key(rowid); +); + +Then use this algorithm: + + i1= {index with the least E(#records)}; + + for each record R in range_scan(i1) + temp_table.insert(R.rowid, count=1); + + for each index idx except i1 + { + for each R record in scan(idx) // (INNER-LOOP) + { + if (temp_table has R) + temptable[R].count++; + } + } + + // The following loop can do ordered or unordered scan + // if we want it to be ordered scan, we probably better arrange so that + // 'count' column is part of the index. + for each record R in temp_table + { + if (R.count == number_of_streams) + emit(R.rowid); + } + +The algorithm has an option to emit an ordered rowid stream. + +In the above form, the cost to produce the first record is high. It's easy to +adjust the algorithm to make it low - we'll need to just start scanning all +indexes at once, and finish as soon as we got a full match, i.e. the + + temptable[R].count++ + +operation resulted in the counter being equal to the number of merged scans. + +1.1.1 Improvement +~~~~~~~~~~~~~~~~~ +When running INNER-LOOP, we could count how many times we've done the +"count++" operation. If it has been done #records-in-temptable times, that +means that all further records will not have matches and we can finish the +scan, i.e. break out of the INNER-LOOP. + +1.2 Produce/merge sorted streams +-------------------------------- +For each of the merged scan, use filesort-like action to end up with an +ordered stream of rowids. Then merge the ordered streams. + +By filesort-like action we mean + - Run over index, collect rowids in a buffer. + - When the buffer is full, sort it and dump into a temporary file. +After the above we'll end up with a number of sorted buffers on disk. We can +use mergebuff() function (it is part of filesort's functions) to produce one +ordered sequence (i.e. array, which may be partially on disk) of rowids. + +Merging of ordered streams with help of priority queue is already implemented +in QUICK_ROR_INTERSECT_SELECT. We'll need to substitute the + + child_quick->get_next() + +call with a call to read rowid from an ordered sequence. + +1.3 Extend Unique class to handle intersection +---------------------------------------------- +There is no point to use Unique object as a device that accumulates rowids of +a single scan then produces them in sorted order. One could do the same faster +with accumulating an array of rowids and then sorting it. + +It's possible to use Unique object to collect/merge data from all scans though. +The idea is as follows: + +- Unique should store <rowid, n_scans> pairs +- Duplicates are pairs with the same rowid +- Unique should try to avoid creating duplicates: + - don't add a duplicate into the in-memory part, instead combine two elements + together by adding their n_scans elements. + - combine duplicates when it sees them in Unique.get() call +- The data we get from Unique.get() should be filtered, all records that have + n_scans != number_of_scans_being_merged should be discarded. + +If we're lucky to have started and finished a scan on some index (denote it +as S) without flushing the Unique in the process, then: +- there is no point in adding any new records into the Unique because their + absence in the Unique means that they don't have match in S and hence will + not get into the result of intersection. +- we need to only update the counters to be able to tell if the elements that + are already in the Unique will have matches in all scans. + +1.4 Strategies that do not seem to be useful +-------------------------------------------- + +keeping them here so we don't consider them over and over + +1.4.1 Remove matches after having produced an ordered stream +------------------------------------------------------------ +We can dump everything into a rowid stream and get it sorted. Then we read it, +and if we see a rowid repeated $n_merged_scans times, it belongs to the +intersection (pass to output), otherwise it doesn't (skip). +This doesn't have any advantages over the produce/merge sorted streams +approach. + +1.4.2 Sparse rowid bitmaps +-------------------------- +Use Falcon-style rowid bitmaps. The problem with that is that Falcon's +bitmaps assume there will always be enough memory to accommodate them. + +PostgreSQL makes bitmaps "loose" when they exceed certain size by remembering +disk pages, not ids of individual records. It's hard for us to do something +similar because our rowids are opaque entities whose meaning depends on the +storage engines. + +This seems to require too much change to be worth it. + +2. Optimization +=============== + +SEL_TREE objects already represent intersections. The problems with +optimizations are: + +- Cost formula(s) +- When N keys/conditions are present: + + "cond(key1) AND cond(key2) AND ... AND cond(keyN)", + + somehow avoid considering (2^n - n) possible options. + +- Avoid producing (or even considering) apparently suboptimal plans: + = Don't generate a merge of indexes (I_1, ... I_n) where columns of I_n are + a subset of columns covered by all other indexes. + = (TODO any other rules?) + DESCRIPTION: At the moment index_merge supports intersection only for rowid-ordered streams. This translates into a limitation that index_merge/intersect can only be constructed for equality conditions (t.keypart1=const1 AND t.keypart2=const2 AND ... ) and the equalities should cover all index components. This WL entry is to lift this limitation by developing algorithms that do intersection on non-ROR scans. HIGH-LEVEL SPECIFICATION: <contents> 1. Execution 1.1 Temptable 1.1.1 Improvement 1.2 Produce/merge sorted streams 1.3 Extend Unique class to handle intersection 1.4 Strategies that do not seem to be useful 1.4.1 Remove matches after having produced an ordered stream 1.4.2 Sparse rowid bitmaps 2. Optimization </contents> 1. Execution ============ The primary task is to find means to compute an intersection of N unordered streams. Besides general memory/cpu cost of computation, we consider: - whether the produced rowid stream is ordered. If it is, it can be piped into index_merge/intersect (as opposed to sort-intersect) - whether the strategy can take advantage of the fact that some input streams are already rowid-ordered - startup cost (cost of producing the first output record) We see the following possible strategies: 1.1 Temptable ------------- [ This is our strategy of choice at the moment] Use a temporary heap-grow-out-to-myisam table with a primary key: create table temp_table ( rowid binary($rowid_size), count n, primary key(rowid); ); Then use this algorithm: i1= {index with the least E(#records)}; for each record R in range_scan(i1) temp_table.insert(R.rowid, count=1); for each index idx except i1 { for each R record in scan(idx) // (INNER-LOOP) { if (temp_table has R) temptable[R].count++; } } // The following loop can do ordered or unordered scan // if we want it to be ordered scan, we probably better arrange so that // 'count' column is part of the index. for each record R in temp_table { if (R.count == number_of_streams) emit(R.rowid); } The algorithm has an option to emit an ordered rowid stream. In the above form, the cost to produce the first record is high. It's easy to adjust the algorithm to make it low - we'll need to just start scanning all indexes at once, and finish as soon as we got a full match, i.e. the temptable[R].count++ operation resulted in the counter being equal to the number of merged scans. 1.1.1 Improvement ~~~~~~~~~~~~~~~~~ When running INNER-LOOP, we could count how many times we've done the "count++" operation. If it has been done #records-in-temptable times, that means that all further records will not have matches and we can finish the scan, i.e. break out of the INNER-LOOP. 1.2 Produce/merge sorted streams -------------------------------- For each of the merged scan, use filesort-like action to end up with an ordered stream of rowids. Then merge the ordered streams. By filesort-like action we mean - Run over index, collect rowids in a buffer. - When the buffer is full, sort it and dump into a temporary file. After the above we'll end up with a number of sorted buffers on disk. We can use mergebuff() function (it is part of filesort's functions) to produce one ordered sequence (i.e. array, which may be partially on disk) of rowids. Merging of ordered streams with help of priority queue is already implemented in QUICK_ROR_INTERSECT_SELECT. We'll need to substitute the child_quick->get_next() call with a call to read rowid from an ordered sequence. 1.3 Extend Unique class to handle intersection ---------------------------------------------- There is no point to use Unique object as a device that accumulates rowids of a single scan then produces them in sorted order. One could do the same faster with accumulating an array of rowids and then sorting it. It's possible to use Unique object to collect/merge data from all scans though. The idea is as follows: - Unique should store <rowid, n_scans> pairs - Duplicates are pairs with the same rowid - Unique should try to avoid creating duplicates: - don't add a duplicate into the in-memory part, instead combine two elements together by adding their n_scans elements. - combine duplicates when it sees them in Unique.get() call - The data we get from Unique.get() should be filtered, all records that have n_scans != number_of_scans_being_merged should be discarded. If we're lucky to have started and finished a scan on some index (denote it as S) without flushing the Unique in the process, then: - there is no point in adding any new records into the Unique because their absence in the Unique means that they don't have match in S and hence will not get into the result of intersection. - we need to only update the counters to be able to tell if the elements that are already in the Unique will have matches in all scans. 1.4 Strategies that do not seem to be useful -------------------------------------------- keeping them here so we don't consider them over and over 1.4.1 Remove matches after having produced an ordered stream ------------------------------------------------------------ We can dump everything into a rowid stream and get it sorted. Then we read it, and if we see a rowid repeated $n_merged_scans times, it belongs to the intersection (pass to output), otherwise it doesn't (skip). This doesn't have any advantages over the produce/merge sorted streams approach. 1.4.2 Sparse rowid bitmaps -------------------------- Use Falcon-style rowid bitmaps. The problem with that is that Falcon's bitmaps assume there will always be enough memory to accommodate them. PostgreSQL makes bitmaps "loose" when they exceed certain size by remembering disk pages, not ids of individual records. It's hard for us to do something similar because our rowids are opaque entities whose meaning depends on the storage engines. This seems to require too much change to be worth it. 2. Optimization =============== SEL_TREE objects already represent intersections. The problems with optimizations are: - Cost formula(s) - When N keys/conditions are present: "cond(key1) AND cond(key2) AND ... AND cond(keyN)", somehow avoid considering (2^n - n) possible options. - Avoid producing (or even considering) apparently suboptimal plans: = Don't generate a merge of indexes (I_1, ... I_n) where columns of I_n are a subset of columns covered by all other indexes. = (TODO any other rules?) - Correlation across selectivities. If there is a condition "cond(key1) AND cond(key2) AND ... AND cond(keyN)", can we consider satisfaction of AND-parts to be independent? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge optimizer: dont discard index_merge union strategies when range is available (24)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge optimizer: dont discard index_merge union strategies when range is available CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge optimizer: dont discard index_merge union strategies when range is available (24)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge optimizer: dont discard index_merge union strategies when range is available CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): index_merge optimizer: dont discard index_merge union strategies when range is available (24)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge optimizer: dont discard index_merge union strategies when range is available CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): index_merge optimizer: dont discard index_merge union strategies when range is available (24)
by worklog-noreply＠askmonty.org 26 May '09

26 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge optimizer: dont discard index_merge union strategies when range is available CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Client-BackLog TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2705: Fix ps_not_windows test for -all-static build which cannot load plugins.
by noreply＠launchpad.net 22 May '09

22 May '09

------------------------------------------------------------ revno: 2705 committer: knielsen(a)knielsen-hq.org branch nick: work-5.1-buildbot timestamp: Fri 2009-05-22 19:53:25 +0200 message: Fix ps_not_windows test for -all-static build which cannot load plugins. added: mysql-test/include/have_dynamic_loading.inc modified: mysql-test/t/ps_not_windows.test === added file 'mysql-test/include/have_dynamic_loading.inc' --- mysql-test/include/have_dynamic_loading.inc 1970-01-01 00:00:00 +0000 +++ mysql-test/include/have_dynamic_loading.inc 2009-05-22 17:53:25 +0000 @@ -0,0 +1,4 @@ +-- require r/have_dynamic_loading.require +disable_query_log; +show variables like 'have_dynamic_loading'; +enable_query_log; === modified file 'mysql-test/t/ps_not_windows.test' --- mysql-test/t/ps_not_windows.test 2007-01-02 23:31:12 +0000 +++ mysql-test/t/ps_not_windows.test 2009-05-22 17:53:25 +0000 @@ -2,6 +2,8 @@ --source include/not_embedded.inc # Non-windows specific ps tests. --source include/not_windows.inc +# Requires ability to install plugins. +--source include/have_dynamic_loading.inc # # Bug #20665: All commands supported in Stored Procedures should work in -- Main stable branch https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2705)
by knielsen＠knielsen-hq.org 22 May '09

22 May '09

#At lp:maria 2705 knielsen(a)knielsen-hq.org 2009-05-22 Fix ps_not_windows test for -all-static build which cannot load plugins. added: mysql-test/include/have_dynamic_loading.inc modified: mysql-test/t/ps_not_windows.test === added file 'mysql-test/include/have_dynamic_loading.inc' --- a/mysql-test/include/have_dynamic_loading.inc 1970-01-01 00:00:00 +0000 +++ b/mysql-test/include/have_dynamic_loading.inc 2009-05-22 17:53:25 +0000 @@ -0,0 +1,4 @@ +-- require r/have_dynamic_loading.require +disable_query_log; +show variables like 'have_dynamic_loading'; +enable_query_log; === modified file 'mysql-test/t/ps_not_windows.test' --- a/mysql-test/t/ps_not_windows.test 2007-01-02 23:31:12 +0000 +++ b/mysql-test/t/ps_not_windows.test 2009-05-22 17:53:25 +0000 @@ -2,6 +2,8 @@ --source include/not_embedded.inc # Non-windows specific ps tests. --source include/not_windows.inc +# Requires ability to install plugins. +--source include/have_dynamic_loading.inc # # Bug #20665: All commands supported in Stored Procedures should work in

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-BackLog TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-5.2 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:51)=-=- Version updated. --- /tmp/wklog.20.old.31642 2009-05-22 17:51:21.000000000 +0300 +++ /tmp/wklog.20.new.31642 2009-05-22 17:51:21.000000000 +0300 @@ -1 +1 @@ -Server-9.x +Server-5.2 -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Category updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Server-BackLog -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Status updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-BackLog TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-5.2 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:51)=-=- Version updated. --- /tmp/wklog.20.old.31642 2009-05-22 17:51:21.000000000 +0300 +++ /tmp/wklog.20.new.31642 2009-05-22 17:51:21.000000000 +0300 @@ -1 +1 @@ -Server-9.x +Server-5.2 -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Category updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Server-BackLog -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Status updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-BackLog TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-9.x STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Category updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Server-BackLog -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Status updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-BackLog TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-9.x STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Category updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Server-BackLog -=-=(Guest - Fri, 22 May 2009, 17:50)=-=- Status updated. --- /tmp/wklog.20.old.31598 2009-05-22 17:50:49.000000000 +0300 +++ /tmp/wklog.20.new.31598 2009-05-22 17:50:49.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table Elimination: remove the facts table (20)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table Elimination: remove the facts table CREATION DATE..: Thu, 21 May 2009, 01:07 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 20 (http://askmonty.org/worklog/?tid=20) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:29)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.31038 2009-05-22 17:29:58.000000000 +0300 +++ /tmp/wklog.20.new.31038 2009-05-22 17:29:58.000000000 +0300 @@ -68,7 +68,7 @@ where ACBDT_Birthdate < '1950-01-01' -At this point, it's possible to conclude that table anchor can be removed from +At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( @@ -80,6 +80,18 @@ mean actors that have NULL/no names), but we're not interested in those as it is an inner join. +The result is (notice the changed select): + +select + actor_birthday.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor_birthdate + left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + How to remove ------------- * Find inner joins, such that -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.28489 2009-05-21 22:54:52.000000000 +0300 +++ /tmp/wklog.20.new.28489 2009-05-21 22:54:52.000000000 +0300 @@ -11,7 +11,7 @@ -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( - AC_ID primary key, + AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), @@ -21,7 +21,7 @@ -- Another physical attribute: actor birthdate -- create table actor_birthdate ( - AC_ID primary key, + AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) -=-=(Psergey - Thu, 21 May 2009, 01:07)=-=- High-Level Specification modified. --- /tmp/wklog.20.old.26317 2009-05-21 01:07:55.000000000 +0300 +++ /tmp/wklog.20.new.26317 2009-05-21 01:07:55.000000000 +0300 @@ -1 +1,91 @@ +Consider the following schema: + +-- +-- The physical anchor table +-- +create table actor ( + AC_ID int primary key +); + +-- +-- A physical attribute: actor name (non-historized for simplicity) +-- +create table actor_name ( + AC_ID primary key, + ACNAM_Name varchar(20), + index(ACNAM_Name), + foreign key (AC_ID) references actor(AC_ID), +); + +-- +-- Another physical attribute: actor birthdate +-- +create table actor_birthdate ( + AC_ID primary key, + ACBDT_Birthdate datetime, + index(ACBDT_Birthdate), + foreign key (AC_ID) references actor(AC_ID) +); + +-- +-- The knot view: actor with its attributes +-- +create view actor_attributes as +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate + ... (other attributes follow) +from + actor + left join actor_name on (actor_name.AC_ID = actor.AC_ID) + left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + ... other attributes follow ... +; + +-- The query: names of senior actors +select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' + +Query processing +---------------- +The following steps will occur +* The VIEW will be resolved with algorithm=merge +* Outer-to-inner join conversion will make use of the + ACBDT_Birthdate < '1950-01-01' condition and change one outer join into + inner. +* Table elimination variant#1 will remove all unused attributes. + +And we'll end up with: + +select + actor.AC_ID, + actor_name.ACNAM_Name, + actor_birthdate.ACBDT_Birthdate +from + actor + join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) + left join actor_name on (actor_name.AC_ID = actor.AC_ID) +where + ACBDT_Birthdate < '1950-01-01' + +At this point, it's possible to conclude that table anchor can be removed from +the query plan: anchor model specifies that actor_name has a foreign key: + + create table actor_name ( + ... + foreign key (AC_ID) references actor(AC_ID), + +This means that each record in actor_name has exactly one match in table actor. +There may be records in `actor` that have no matches in `actor_name` (they +mean actors that have NULL/no names), but we're not interested in those as it +is an inner join. + +How to remove +------------- +* Find inner joins, such that + - some table T can be accessed using eq_ref access method. + - some other table T2 has Foreign Key that refers to table T. + - there are no references to table T anywhere in the query, except for + references to T.primary_key, which can be substituted for columns of T2. + DESCRIPTION: In certain cases it is possible to eliminate table when one is running inner joins and there is a foreign key relationship between the tables. HIGH-LEVEL SPECIFICATION: Consider the following schema: -- -- The physical anchor table -- create table actor ( AC_ID int primary key ); -- -- A physical attribute: actor name (non-historized for simplicity) -- create table actor_name ( AC_ID int primary key, ACNAM_Name varchar(20), index(ACNAM_Name), foreign key (AC_ID) references actor(AC_ID), ); -- -- Another physical attribute: actor birthdate -- create table actor_birthdate ( AC_ID int primary key, ACBDT_Birthdate datetime, index(ACBDT_Birthdate), foreign key (AC_ID) references actor(AC_ID) ); -- -- The knot view: actor with its attributes -- create view actor_attributes as select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate ... (other attributes follow) from actor left join actor_name on (actor_name.AC_ID = actor.AC_ID) left join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) ... other attributes follow ... ; -- The query: names of senior actors select ACNAM_Name from actor_attributes where ACBDT_Birthdate < '1950-01-01' Query processing ---------------- The following steps will occur * The VIEW will be resolved with algorithm=merge * Outer-to-inner join conversion will make use of the ACBDT_Birthdate < '1950-01-01' condition and change one outer join into inner. * Table elimination variant#1 will remove all unused attributes. And we'll end up with: select actor.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor join actor_birthdate on (actor_birthdate.AC_ID = actor.AC_ID) left join actor_name on (actor_name.AC_ID = actor.AC_ID) where ACBDT_Birthdate < '1950-01-01' At this point, it's possible to conclude that table actor can be removed from the query plan: anchor model specifies that actor_name has a foreign key: create table actor_name ( ... foreign key (AC_ID) references actor(AC_ID), This means that each record in actor_name has exactly one match in table actor. There may be records in `actor` that have no matches in `actor_name` (they mean actors that have NULL/no names), but we're not interested in those as it is an inner join. The result is (notice the changed select): select actor_birthday.AC_ID, actor_name.ACNAM_Name, actor_birthdate.ACBDT_Birthdate from actor_birthdate left join actor_name on (actor_name.AC_ID = actor_birthday.AC_ID) where ACBDT_Birthdate < '1950-01-01' How to remove ------------- * Find inner joins, such that - some table T can be accessed using eq_ref access method. - some other table T2 has Foreign Key that refers to table T. - there are no references to table T anywhere in the query, except for references to T.primary_key, which can be substituted for columns of T2. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative ------------------------------------------------------------ -=-=(View All Progress Notes, 13 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:23)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.30851 2009-05-22 17:23:38.000000000 +0300 +++ /tmp/wklog.17.new.30851 2009-05-22 17:23:38.000000000 +0300 @@ -6,7 +6,7 @@ elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the -execution plan when it is unneccessary to include them. This can, of +execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: @@ -22,30 +22,26 @@ When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will -contain a NULL value. +still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If -A -contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know -what -the result will look like is to actually touch both tables during +what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly -as -many rows as there are in tableA, since joining with tableB cannot +as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is -unneccessary. We can remove the whole join operation from the execution +unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination -in -the case described above. Let us look at a more advanced query, where +in the case described above. Let us look at a more advanced query, where Oracle fails. select -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative ------------------------------------------------------------ -=-=(View All Progress Notes, 13 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. ------------------------------------------------------------ -=-=(View All Progress Notes, 12 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Server-5.1 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30176 2009-05-22 17:00:35.000000000 +0300 +++ /tmp/wklog.17.new.30176 2009-05-22 17:00:35.000000000 +0300 @@ -1 +1 @@ -Maria-2.0 +Server-5.1 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. ------------------------------------------------------------ -=-=(View All Progress Notes, 12 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Maria-2.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. -=-=(Guest - Tue, 19 May 2009, 18:27)=-=- Title modified. --- /tmp/wklog.17.old.22023 2009-05-19 18:27:41.000000000 +0300 +++ /tmp/wklog.17.new.22023 2009-05-19 18:27:41.000000000 +0300 @@ -1 +1 @@ -Table ellimination +Table elimination ------------------------------------------------------------ -=-=(View All Progress Notes, 11 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: Maria-2.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -Maria-Sprint +Server-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -2.0 +9.x -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30162 2009-05-22 17:00:28.000000000 +0300 +++ /tmp/wklog.17.new.30162 2009-05-22 17:00:28.000000000 +0300 @@ -1 +1 @@ -9.x +Maria-2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. -=-=(Guest - Tue, 19 May 2009, 18:27)=-=- Title modified. --- /tmp/wklog.17.old.22023 2009-05-19 18:27:41.000000000 +0300 +++ /tmp/wklog.17.new.22023 2009-05-19 18:27:41.000000000 +0300 @@ -1 +1 @@ -Table ellimination +Table elimination ------------------------------------------------------------ -=-=(View All Progress Notes, 11 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Maria-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: 2.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. -=-=(Guest - Tue, 19 May 2009, 18:27)=-=- Title modified. --- /tmp/wklog.17.old.22023 2009-05-19 18:27:41.000000000 +0300 +++ /tmp/wklog.17.new.22023 2009-05-19 18:27:41.000000000 +0300 @@ -1 +1 @@ -Table ellimination +Table elimination -=-=(Monty - Sun, 10 May 2009, 19:58)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.5423 2009-05-10 19:58:06.000000000 +0300 +++ /tmp/wklog.17.new.5423 2009-05-10 19:58:06.000000000 +0300 @@ -1 +1,90 @@ +Here is an extended explanation of table elimination. + +Table elimination is a feature found in some modern query optimizers, of +which Microsoft SQL Server 2005/2008 seems to have the most advanced +implementation. Oracle 11g has also been confirmed to use table +elimination but not to the same extent. + +Basically, what table elimination does, is to remove tables from the +execution plan when it is unneccessary to include them. This can, of +course, only happen if the right circumstances arise. Let us for example +look at the following query: + +select + A.colA +from + tableA A +left outer join + tableB B +on + B.id = A.id; + +When using A as the left table we ensure that the query will return at +least as many rows as there are in that table. For rows where the join +condition (B.id = A.id) is not met the selected column (A.colA) will +contain a NULL value. + +However, the result set could actually contain more rows than what is +found in tableA if there are duplicates of the column B.id in tableB. If +A +contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +then two rows will match in the join condition. The only way to know +what +the result will look like is to actually touch both tables during +execution. + +Instead, let's say that tableB contains rows that make it possible to +place a unique constraint on the column B.id, for example and often the +case a primary key. In this situation we know that we will get exactly +as +many rows as there are in tableA, since joining with tableB cannot +introduce any duplicates. If further, as in the example query, we do not +select any columns from tableB, touching that table during execution is +unneccessary. We can remove the whole join operation from the execution +plan. + +Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination +in +the case described above. Let us look at a more advanced query, where +Oracle fails. + +select + A.colA +from + tableA A +left outer join + tableB B +on + B.id = A.id +and + B.fromDate = ( + select + max(sub.fromDate) + from + tableB sub + where + sub.id = A.id + ); + +In this example we have added another join condition, which ensures +that we only pick the matching row from tableB having the latest +fromDate. In this case tableB will contain duplicates of the column +B.id, so in order to ensure uniqueness the primary key has to contain +the fromDate column as well. In other words the primary key of tableB +is (B.id, B.fromDate). + +Furthermore, since the subselect ensures that we only pick the latest +B.fromDate for a given B.id we know that at most one row will match +the join condition. We will again have the situation where joining +with tableB cannot affect the number of rows in the result set. Since +we do not select any columns from tableB, the whole join operation can +be eliminated from the execution plan. + +SQL Server 2005/2008 will deploy table elimination in this situation as +well. We have not found a way to make Oracle 11g use it for this type of +query. Queries like these arise in two situations. Either when you have +denormalized model consisting of a fact table with several related +dimension tables, or when you have a highly normalized model where each +attribute is stored in its own table. The example with the subselect is +common whenever you store historized/versioned data. DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Maria-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: 2.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Category updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-RawIdeaBin +Maria-Sprint -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Version updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Server-5.2 +2.0 -=-=(Guest - Fri, 22 May 2009, 17:00)=-=- Status updated. --- /tmp/wklog.17.old.30122 2009-05-22 17:00:02.000000000 +0300 +++ /tmp/wklog.17.new.30122 2009-05-22 17:00:02.000000000 +0300 @@ -1 +1 @@ -Un-Assigned +Assigned -=-=(Guest - Thu, 21 May 2009, 22:54)=-=- Low Level Design modified. --- /tmp/wklog.17.old.28468 2009-05-21 22:54:28.000000000 +0300 +++ /tmp/wklog.17.new.28468 2009-05-21 22:54:28.000000000 +0300 @@ -77,7 +77,7 @@ - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicablity by removing [MARK1] as that can change during + to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to -=-=(Psergey - Tue, 19 May 2009, 20:20)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27126 2009-05-19 20:20:06.000000000 +0300 +++ /tmp/wklog.17.new.27126 2009-05-19 20:20:06.000000000 +0300 @@ -48,6 +48,10 @@ another outer join nest, that shouldn't prevent us from moving away the eliminated tables. +* Update join->table_count and all-join-tables bitmap. + +* That's it. Nothing else? + 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative -=-=(Psergey - Tue, 19 May 2009, 20:18)=-=- Low Level Design modified. --- /tmp/wklog.17.old.27035 2009-05-19 20:18:27.000000000 +0300 +++ /tmp/wklog.17.new.27035 2009-05-19 20:18:27.000000000 +0300 @@ -1 +1,92 @@ +<contents> +1. Conditions for removal +2. Removal operation properties +3. Removal operation +4. User interface +5. Todo, issues to resolve +5.1 To resolve +5.2 Resolved +</contents> + +It's not really about elimination of tables, it's about elimination of inner +sides of outer joins. + +1. Conditions for removal +------------------------- +We can eliminate an inner side of outer join if: +1. For each record combination of outer tables, it will always produce + exactly one record. +2. There are no references to columns of the inner tables anywhere else in + the query. + +#1 means that every table inside the outer join nest is: + - is a constant table: + = because it can be accessed via eq_ref(const) access, or + = it is a zero-rows or one-row MyISAM-like table [MARK1] + - has an eq_ref access method candidate. + +#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, + GROUP BY and HAVING do not refer to the inner tables of the outer join + nest. + +2. Removal operation properties +------------------------------- +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + +3. Removal operation +-------------------- +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to front like it is done with const + tables, with exception that if eliminated outer join nest was within + another outer join nest, that shouldn't prevent us from moving away the + eliminated tables. + +4. User interface +----------------- +* We'll add an @@optimizer switch flag for table elimination. Tentative + name: 'table_elimination'. + +* With EXPLAIN, there are two options: + - Show removed tables in a way similar to const tables, with some + indication that they are removed. + - Do not show them altogether. +(the second one seems to be better? We're targeting a situation with VIEWs, +where the user would not care about what tables were added into his query +and then discarded from it?) + +5. Todo, issues to resolve +-------------------------- + +5.1 To resolve +~~~~~~~~~~~~~~ +- See EXPLAIN question in section #4. + +- Re-check how this works with equality propagation. + +- Relationship with prepared statements. + On one hand, it's natural to desire to make table elimination a + once-per-statement operation, like outer->inner join conversion. We'll have + to limit the applicablity by removing [MARK1] as that can change during + lifetime of the statement. + + The other option is to do table elimination every time. This will require to + rework operation [MARK2] to be undoable. + + I'm leaning towards doing the former. With anchor modeling, it is unlikely + that we'll meet outer joins which have N inner tables of which some are 1-row + MyISAM tables that do not have primary key. + +5.2 Resolved +~~~~~~~~~~~~ +- outer->inner join conversion is not a problem for table elimination. + We make outer->inner conversions based on predicates in WHERE. If the WHERE + referred to an inner table (requirement for OJ->IJ conversion) then table + elimination would not be applicable anyway. -=-=(Guest - Tue, 19 May 2009, 18:27)=-=- Title modified. --- /tmp/wklog.17.old.22023 2009-05-19 18:27:41.000000000 +0300 +++ /tmp/wklog.17.new.22023 2009-05-19 18:27:41.000000000 +0300 @@ -1 +1 @@ -Table ellimination +Table elimination -=-=(Monty - Sun, 10 May 2009, 19:58)=-=- High-Level Specification modified. --- /tmp/wklog.17.old.5423 2009-05-10 19:58:06.000000000 +0300 +++ /tmp/wklog.17.new.5423 2009-05-10 19:58:06.000000000 +0300 @@ -1 +1,90 @@ +Here is an extended explanation of table elimination. + +Table elimination is a feature found in some modern query optimizers, of +which Microsoft SQL Server 2005/2008 seems to have the most advanced +implementation. Oracle 11g has also been confirmed to use table +elimination but not to the same extent. + +Basically, what table elimination does, is to remove tables from the +execution plan when it is unneccessary to include them. This can, of +course, only happen if the right circumstances arise. Let us for example +look at the following query: + +select + A.colA +from + tableA A +left outer join + tableB B +on + B.id = A.id; + +When using A as the left table we ensure that the query will return at +least as many rows as there are in that table. For rows where the join +condition (B.id = A.id) is not met the selected column (A.colA) will +contain a NULL value. + +However, the result set could actually contain more rows than what is +found in tableA if there are duplicates of the column B.id in tableB. If +A +contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] +then two rows will match in the join condition. The only way to know +what +the result will look like is to actually touch both tables during +execution. + +Instead, let's say that tableB contains rows that make it possible to +place a unique constraint on the column B.id, for example and often the +case a primary key. In this situation we know that we will get exactly +as +many rows as there are in tableA, since joining with tableB cannot +introduce any duplicates. If further, as in the example query, we do not +select any columns from tableB, touching that table during execution is +unneccessary. We can remove the whole join operation from the execution +plan. + +Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination +in +the case described above. Let us look at a more advanced query, where +Oracle fails. + +select + A.colA +from + tableA A +left outer join + tableB B +on + B.id = A.id +and + B.fromDate = ( + select + max(sub.fromDate) + from + tableB sub + where + sub.id = A.id + ); + +In this example we have added another join condition, which ensures +that we only pick the matching row from tableB having the latest +fromDate. In this case tableB will contain duplicates of the column +B.id, so in order to ensure uniqueness the primary key has to contain +the fromDate column as well. In other words the primary key of tableB +is (B.id, B.fromDate). + +Furthermore, since the subselect ensures that we only pick the latest +B.fromDate for a given B.id we know that at most one row will match +the join condition. We will again have the situation where joining +with tableB cannot affect the number of rows in the result set. Since +we do not select any columns from tableB, the whole join operation can +be eliminated from the execution plan. + +SQL Server 2005/2008 will deploy table elimination in this situation as +well. We have not found a way to make Oracle 11g use it for this type of +query. Queries like these arise in two situations. Either when you have +denormalized model consisting of a fact table with several related +dimension tables, or when you have a highly normalized model where each +attribute is stored in its own table. The example with the subselect is +common whenever you store historized/versioned data. DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unneccessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will contain a NULL value. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unneccessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: <contents> 1. Conditions for removal 2. Removal operation properties 3. Removal operation 4. User interface 5. Todo, issues to resolve 5.1 To resolve 5.2 Resolved </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Conditions for removal ------------------------- We can eliminate an inner side of outer join if: 1. For each record combination of outer tables, it will always produce exactly one record. 2. There are no references to columns of the inner tables anywhere else in the query. #1 means that every table inside the outer join nest is: - is a constant table: = because it can be accessed via eq_ref(const) access, or = it is a zero-rows or one-row MyISAM-like table [MARK1] - has an eq_ref access method candidate. #2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, GROUP BY and HAVING do not refer to the inner tables of the outer join nest. 2. Removal operation properties ------------------------------- * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 3. Removal operation -------------------- * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to front like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. * That's it. Nothing else? 4. User interface ----------------- * We'll add an @@optimizer switch flag for table elimination. Tentative name: 'table_elimination'. * With EXPLAIN, there are two options: - Show removed tables in a way similar to const tables, with some indication that they are removed. - Do not show them altogether. (the second one seems to be better? We're targeting a situation with VIEWs, where the user would not care about what tables were added into his query and then discarded from it?) 5. Todo, issues to resolve -------------------------- 5.1 To resolve ~~~~~~~~~~~~~~ - See EXPLAIN question in section #4. - Re-check how this works with equality propagation. - Relationship with prepared statements. On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. I'm leaning towards doing the former. With anchor modeling, it is unlikely that we'll meet outer joins which have N inner tables of which some are 1-row MyISAM tables that do not have primary key. 5.2 Resolved ~~~~~~~~~~~~ - outer->inner join conversion is not a problem for table elimination. We make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (requirement for OJ->IJ conversion) then table elimination would not be applicable anyway. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2704)
by knielsen＠knielsen-hq.org 22 May '09

22 May '09

#At lp:maria 2704 knielsen(a)knielsen-hq.org 2009-05-22 After-merge fixes for problems seen in buildbot after merging MySQL-5.1.35. - Version number. - Valgrind false alarms in libz. - New variant of suppression for Valgrind warning in dlclose(). - Fix double free() in plugin init error case. modified: configure.in include/my_sys.h mysql-test/valgrind.supp mysys/my_compress.c sql/handler.cc sql/item_strfunc.cc storage/archive/azio.c per-file messages: configure.in Fix version number. We should reset the maria variant back to `1' when the MySQL version number increases. include/my_sys.h Fix false alarms in Valgrind for zlib. Apply same fix as for archive storage handler also to the cases of compression in the client protocol, and to the compression SQL function. mysql-test/valgrind.supp A new variant of the dlclose() suppression is needed now. mysys/my_compress.c Fix false alarms in Valgrind for zlib. Apply same fix as for archive storage handler also to the cases of compression in the client protocol, and to the compression SQL function. sql/handler.cc Fix a double free() in error case for plugin initialisation. sql/item_strfunc.cc Fix false alarms in Valgrind for zlib. Apply same fix as for archive storage handler also to the cases of compression in the client protocol, and to the compression SQL function. === modified file 'configure.in' --- a/configure.in 2009-04-25 10:05:32 +0000 +++ b/configure.in 2009-05-22 12:38:50 +0000 @@ -9,8 +9,11 @@ AC_CANONICAL_SYSTEM # remember to also update version.c in ndb # # When changing major version number please also check switch statement -# in mysqlbinlog::check_master_version(). -AM_INIT_AUTOMAKE(mysql, 5.1.35-maria-beta2) +# in mysqlbinlog.cc / check_master_version(). +# +# When merging new MySQL releases, update the version number to match the +# MySQL version number, but reset the maria subrelease (-beta1). +AM_INIT_AUTOMAKE(mysql, 5.1.35-maria-beta1) AM_CONFIG_HEADER([include/config.h:config.h.in]) PROTOCOL_VERSION=10 === modified file 'include/my_sys.h' --- a/include/my_sys.h 2009-04-25 10:05:32 +0000 +++ b/include/my_sys.h 2009-05-22 12:38:50 +0000 @@ -879,6 +879,10 @@ extern my_bool my_compress(uchar *, size extern my_bool my_uncompress(uchar *, size_t , size_t *); extern uchar *my_compress_alloc(const uchar *packet, size_t *len, size_t *complen); +extern void *my_az_allocator(void *dummy, unsigned int items, unsigned int size); +extern void my_az_free(void *dummy, void *address); +extern int my_compress_buffer(uchar *dest, size_t *destLen, + const uchar *source, size_t sourceLen); extern int packfrm(uchar *, size_t, uchar **, size_t *); extern int unpackfrm(uchar **, size_t *, const uchar *); === modified file 'mysql-test/valgrind.supp' --- a/mysql-test/valgrind.supp 2009-04-08 16:55:26 +0000 +++ b/mysql-test/valgrind.supp 2009-05-22 12:38:50 +0000 @@ -401,6 +401,20 @@ } { + dlclose memory loss from plugin variant 3 + Memcheck:Leak + fun:malloc + fun:_dl_scope_free + fun:_dl_close_worker + fun:_dl_close + fun:_dl_catch_error + fun:_dlerror_run + fun:dlclose + fun:_Z15free_plugin_memP12st_plugin_dl + fun:_Z13plugin_dl_delPK19st_mysql_lex_string +} + +{ dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 32/64 bit Memcheck:Leak fun:*alloc === modified file 'mysys/my_compress.c' --- a/mysys/my_compress.c 2009-04-25 10:05:32 +0000 +++ b/mysys/my_compress.c 2009-05-22 12:38:50 +0000 @@ -57,19 +57,85 @@ my_bool my_compress(uchar *packet, size_ } +/* + Valgrind normally gives false alarms for zlib operations, in the form of + "conditional jump depends on uninitialised values" etc. The reason is + explained in the zlib FAQ (http://www.zlib.net/zlib_faq.html#faq36) + + "That is intentional for performance reasons, and the output of deflate + is not affected." + + Also discussed on a blog + (http://www.sirena.org.uk/log/2006/02/19/zlib-generating-valgrind-warnings/) + + "...loop unrolling in the zlib library causes the mentioned + “Conditional jump or move depends on uninitialised value(s)” + warnings. These are safe since the results of the comparison are + subsequently ignored..." + + "the results of the calculations are discarded by bounds checking done + after the loop exits" + + Fix by initializing the memory allocated by zlib when running under Valgrind. + + This fix is safe, since such memory is only used internally by zlib, so we + will not hide any bugs in mysql this way. +*/ +void *my_az_allocator(void *dummy, unsigned int items, unsigned int size) +{ + return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); +} + +void my_az_free(void *dummy, void *address) +{ + my_free(address, MYF(MY_ALLOW_ZERO_PTR)); +} + +/* + This works like zlib compress(), but using custom memory allocators to work + better with my_malloc leak detection and Valgrind. +*/ +int my_compress_buffer(uchar *dest, size_t *destLen, + const uchar *source, size_t sourceLen) +{ + z_stream stream; + int err; + + stream.next_in = (Bytef*)source; + stream.avail_in = (uInt)sourceLen; + stream.next_out = (Bytef*)dest; + stream.avail_out = (uInt)*destLen; + if ((size_t)stream.avail_out != *destLen) + return Z_BUF_ERROR; + + stream.zalloc = (alloc_func)my_az_allocator; + stream.zfree = (free_func)my_az_free; + stream.opaque = (voidpf)0; + + err = deflateInit(&stream, Z_DEFAULT_COMPRESSION); + if (err != Z_OK) return err; + + err = deflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) { + deflateEnd(&stream); + return err == Z_OK ? Z_BUF_ERROR : err; + } + *destLen = stream.total_out; + + err = deflateEnd(&stream); + return err; +} + uchar *my_compress_alloc(const uchar *packet, size_t *len, size_t *complen) { uchar *compbuf; - uLongf tmp_complen; int res; *complen= *len * 120 / 100 + 12; if (!(compbuf= (uchar *) my_malloc(*complen, MYF(MY_WME)))) return 0; /* Not enough memory */ - tmp_complen= (uLongf) *complen; - res= compress((Bytef*) compbuf, &tmp_complen, (Bytef*) packet, (uLong) *len); - *complen= tmp_complen; + res= my_compress_buffer(compbuf, complen, packet, *len); if (res != Z_OK) { === modified file 'sql/handler.cc' --- a/sql/handler.cc 2009-04-25 10:05:32 +0000 +++ b/sql/handler.cc 2009-05-22 12:38:50 +0000 @@ -432,9 +432,6 @@ int ha_initialize_handlerton(st_plugin_i { sql_print_error("Plugin '%s' init function returned error.", plugin->name.str); - /* Free data, so that we don't refer to it in ha_finalize_handlerton */ - my_free(hton, MYF(0)); - plugin->data= 0; goto err; } === modified file 'sql/item_strfunc.cc' --- a/sql/item_strfunc.cc 2009-04-25 10:05:32 +0000 +++ b/sql/item_strfunc.cc 2009-05-22 12:38:50 +0000 @@ -3243,7 +3243,7 @@ longlong Item_func_crc32::val_int() String *Item_func_compress::val_str(String *str) { int err= Z_OK, code; - ulong new_size; + size_t new_size; String *res; Byte *body; char *tmp, *last_char; @@ -3279,8 +3279,8 @@ String *Item_func_compress::val_str(Stri body= ((Byte*)buffer.ptr()) + 4; // As far as we have checked res->is_empty() we can use ptr() - if ((err= compress(body, &new_size, - (const Bytef*)res->ptr(), res->length())) != Z_OK) + if ((err= my_compress_buffer(body, &new_size, (const uchar *)res->ptr(), + res->length())) != Z_OK) { code= err==Z_MEM_ERROR ? ER_ZLIB_Z_MEM_ERROR : ER_ZLIB_Z_BUF_ERROR; push_warning(current_thd,MYSQL_ERROR::WARN_LEVEL_ERROR,code,ER(code)); === modified file 'storage/archive/azio.c' --- a/storage/archive/azio.c 2009-05-19 09:28:05 +0000 +++ b/storage/archive/azio.c 2009-05-22 12:38:50 +0000 @@ -16,6 +16,8 @@ #include <stdio.h> #include <string.h> +#include "my_sys.h" + static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ static int const az_magic[3] = {0xfe, 0x03, 0x01}; /* az magic header */ @@ -37,40 +39,6 @@ void putLong(File file, uLong x); uLong getLong(azio_stream *s); void read_header(azio_stream *s, unsigned char *buffer); -/* - Valgrind normally gives false alarms for zlib operations, in the form of - "conditional jump depends on uninitialised values" etc. The reason is - explained in the zlib FAQ (http://www.zlib.net/zlib_faq.html#faq36) - - "That is intentional for performance reasons, and the output of deflate - is not affected." - - Also discussed on a blog - (http://www.sirena.org.uk/log/2006/02/19/zlib-generating-valgrind-warnings/) - - "...loop unrolling in the zlib library causes the mentioned - “Conditional jump or move depends on uninitialised value(s)” - warnings. These are safe since the results of the comparison are - subsequently ignored..." - - "the results of the calculations are discarded by bounds checking done - after the loop exits" - - Fix by initializing the memory allocated by zlib when running under Valgrind. - - This fix is safe, since such memory is only used internally by zlib, so we - will not hide any bugs in mysql this way. -*/ -static void *az_allocator(void *dummy, uInt items, uInt size) -{ - return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); -} - -static void az_free(void *dummy, void *address) -{ - my_free(address, MYF(MY_ALLOW_ZERO_PTR)); -} - /* =========================================================================== Opens a gzip (.gz) file for reading or writing. The mode parameter is as in fopen ("rb" or "wb"). The file is given either by file descriptor @@ -86,8 +54,8 @@ int az_open (azio_stream *s, const char int level = Z_DEFAULT_COMPRESSION; /* compression level */ int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - s->stream.zalloc = az_allocator; - s->stream.zfree = az_free; + s->stream.zalloc = my_az_allocator; + s->stream.zfree = my_az_free; s->stream.opaque = (voidpf)0; memset(s->inbuf, 0, AZ_BUFSIZE_READ); memset(s->outbuf, 0, AZ_BUFSIZE_WRITE);

1 0

[Maria-developers] Updated (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Knielsen - Fri, 22 May 2009, 14:31)=-=- Low Level Design modified. --- /tmp/wklog.23.old.24587 2009-05-22 14:31:52.000000000 +0300 +++ /tmp/wklog.23.new.24587 2009-05-22 14:31:52.000000000 +0300 @@ -1 +1,7 @@ +Two places where we call into libz, and where checking for defined parameters +would be good: + + - mysys/my_compress.c + + - sql/item_strfunc.cc (Item_func_compress). -=-=(Guest - Fri, 22 May 2009, 12:04)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.18061 2009-05-22 12:04:05.000000000 +0300 +++ /tmp/wklog.23.new.18061 2009-05-22 12:04:05.000000000 +0300 @@ -26,3 +26,5 @@ initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. + +See also a patch here: http://bugs.mysql.com/bug.php?id=44582 -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. See also a patch here: http://bugs.mysql.com/bug.php?id=44582 LOW-LEVEL DESIGN: Two places where we call into libz, and where checking for defined parameters would be good: - mysys/my_compress.c - sql/item_strfunc.cc (Item_func_compress). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Knielsen - Fri, 22 May 2009, 14:31)=-=- Low Level Design modified. --- /tmp/wklog.23.old.24587 2009-05-22 14:31:52.000000000 +0300 +++ /tmp/wklog.23.new.24587 2009-05-22 14:31:52.000000000 +0300 @@ -1 +1,7 @@ +Two places where we call into libz, and where checking for defined parameters +would be good: + + - mysys/my_compress.c + + - sql/item_strfunc.cc (Item_func_compress). -=-=(Guest - Fri, 22 May 2009, 12:04)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.18061 2009-05-22 12:04:05.000000000 +0300 +++ /tmp/wklog.23.new.18061 2009-05-22 12:04:05.000000000 +0300 @@ -26,3 +26,5 @@ initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. + +See also a patch here: http://bugs.mysql.com/bug.php?id=44582 -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. See also a patch here: http://bugs.mysql.com/bug.php?id=44582 LOW-LEVEL DESIGN: Two places where we call into libz, and where checking for defined parameters would be good: - mysys/my_compress.c - sql/item_strfunc.cc (Item_func_compress). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 12:04)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.18061 2009-05-22 12:04:05.000000000 +0300 +++ /tmp/wklog.23.new.18061 2009-05-22 12:04:05.000000000 +0300 @@ -26,3 +26,5 @@ initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. + +See also a patch here: http://bugs.mysql.com/bug.php?id=44582 -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. See also a patch here: http://bugs.mysql.com/bug.php?id=44582 ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Guest - Fri, 22 May 2009, 12:04)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.18061 2009-05-22 12:04:05.000000000 +0300 +++ /tmp/wklog.23.new.18061 2009-05-22 12:04:05.000000000 +0300 @@ -26,3 +26,5 @@ initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. + +See also a patch here: http://bugs.mysql.com/bug.php?id=44582 -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. See also a patch here: http://bugs.mysql.com/bug.php?id=44582 ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: -=-=(Knielsen - Fri, 22 May 2009, 11:52)=-=- High-Level Specification modified. --- /tmp/wklog.23.old.17628 2009-05-22 11:52:33.000000000 +0300 +++ /tmp/wklog.23.new.17628 2009-05-22 11:52:33.000000000 +0300 @@ -1 +1,28 @@ +With custom memory allocators, using the Valgrind APIs we can tell Valgrind when +a memory block is allocated (so that data read from memory is marked as undefined +instead of being defined or not at random depending on prior use); and when a +memory block is freed (so that use after freeing can be reported as an error). +In some cases cheking for leaks may also be appropriate. + +Another possibility is to add an explicit check for whether memory is defined. + +One place this would be useful is when calling libz. Due to the design of that +library, Valgrind produces lots of false alarms about using undefined values +(I think the issue is that it runs a few bytes off of initialized memory to +reduce boundary checks in each loop iteration, then after the loop has checks to +avoid using the undefined part of the result). This means we have lots of libz +Valgrind suppressions and continue to add more as new warnings surface. So we +might easily miss a real problem in this area. This could be improved by adding +explicit checks at the call to libz functions that the passed memory is properly +defined. + +Another use is to improve debugging. It is often the case when debugging a +warning about using un-initialised memory that the detection happens long after +the real problem, the un-initialized value being passed along through the code +for a long time before being detected. This makes debugging the problem slow. + +By adding in strategic places code that asserts that a specific value must be +initialised, it is possible to detect problems earlier, speeding up debugging. +Such code can be added in more places over time as development and debugging +goes on. DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). HIGH-LEVEL SPECIFICATION: With custom memory allocators, using the Valgrind APIs we can tell Valgrind when a memory block is allocated (so that data read from memory is marked as undefined instead of being defined or not at random depending on prior use); and when a memory block is freed (so that use after freeing can be reported as an error). In some cases cheking for leaks may also be appropriate. Another possibility is to add an explicit check for whether memory is defined. One place this would be useful is when calling libz. Due to the design of that library, Valgrind produces lots of false alarms about using undefined values (I think the issue is that it runs a few bytes off of initialized memory to reduce boundary checks in each loop iteration, then after the loop has checks to avoid using the undefined part of the result). This means we have lots of libz Valgrind suppressions and continue to add more as new warnings surface. So we might easily miss a real problem in this area. This could be improved by adding explicit checks at the call to libz functions that the passed memory is properly defined. Another use is to improve debugging. It is often the case when debugging a warning about using un-initialised memory that the detection happens long after the real problem, the un-initialized value being passed along through the code for a long time before being detected. This makes debugging the problem slow. By adding in strategic places code that asserts that a specific value must be initialised, it is possible to detect problems earlier, speeding up debugging. Such code can be added in more places over time as development and debugging goes on. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Newly Added (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Knielsen): Using the Valgrind API in mysqld (23)
by worklog-noreply＠askmonty.org 22 May '09

22 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Using the Valgrind API in mysqld CREATION DATE..: Fri, 22 May 2009, 11:43 SUPERVISOR.....: Monty IMPLEMENTOR....: Knielsen COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 23 (http://askmonty.org/worklog/?tid=23) VERSION........: Benchmarks-3.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 40 (hours remain) ORIG. ESTIMATE.: 40 PROGRESS NOTES: DESCRIPTION: Valgrind (the memcheck tool) has some very useful APIs that can be used in mysqld when testing with Valgrind to improve testing and/or debugging: file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.clientreqs file:///usr/share/doc/valgrind/html/mc-manual.html#mc-manual.mempools This worklog is about adding configure checks and headers to allow to use these in a way that continues to work on machines where the Valgrind headers or functionality is missing. It also includes adding some basic Valgrind enhancements: - Adding Valgrind annotations to custom memory allocators so that Valgrind can detect leaks, use-before-init, and use-after-free problems also for these allocators. - Adding checks for definedness in appropriate places (eg. when calling libz). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Implement mysql-test output parser for Buildbot (22)
by worklog-noreply＠askmonty.org 21 May '09

21 May '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Implement mysql-test output parser for Buildbot CREATION DATE..: Thu, 21 May 2009, 22:19 SUPERVISOR.....: Knielsen IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Other TASK ID........: 22 (http://askmonty.org/worklog/?tid=22) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 30 (hours remain) ORIG. ESTIMATE.: 30 PROGRESS NOTES: -=-=(Knielsen - Thu, 21 May 2009, 22:33)=-=- High Level Description modified. --- /tmp/wklog.22.old.27679 2009-05-21 22:33:35.000000000 +0300 +++ /tmp/wklog.22.new.27679 2009-05-21 22:33:35.000000000 +0300 @@ -9,3 +9,6 @@ http://djmitche.github.com/buildbot/docs/0.7.10/#Writing-New-BuildSteps +Later, once we get the infrastructure to write Buildbot results into a MySQL +database, we want to extend this to also insert into the database all test +failures and the mysqltest failure output (for cross-reference search). DESCRIPTION: Like in Pushbuild at MySQL AB, we want to have buildbot parse the output of mysql-test-run for test errors so we can display on the front page the name of any tests that failed. The parser can also count the number of tests completed so far so Buildbot can provide more accurate completion estimates. Buildbot already has support for plugging in such modules. See eg. http://djmitche.github.com/buildbot/docs/0.7.10/#Writing-New-BuildSteps Later, once we get the infrastructure to write Buildbot results into a MySQL database, we want to extend this to also insert into the database all test failures and the mysqltest failure output (for cross-reference search). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2703)
by knielsen＠knielsen-hq.org 20 May '09

20 May '09

#At lp:maria 2703 knielsen(a)knielsen-hq.org 2009-05-20 Fix accessing ulong enum option as uint, failing on 64-bit big-endian. modified: mysys/my_getopt.c === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-04-25 10:05:32 +0000 +++ b/mysys/my_getopt.c 2009-05-20 15:34:34 +0000 @@ -647,7 +647,7 @@ static int setval(const struct my_option return EXIT_OUT_OF_MEMORY; break; case GET_ENUM: - if (((*(int*)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) + if (((*(ulong *)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) return EXIT_ARGUMENT_INVALID; break; case GET_SET: @@ -983,7 +983,7 @@ static void init_one_value(const struct *((int*) variable)= (int) getopt_ll_limit_value((int) value, option, NULL); break; case GET_ENUM: - *((uint*) variable)= (uint) value; + *((ulong*) variable)= (uint) value; break; case GET_UINT: *((uint*) variable)= (uint) getopt_ull_limit_value((uint) value, option, NULL); @@ -1221,7 +1221,7 @@ void my_print_variables(const struct my_ } break; case GET_ENUM: - printf("%s\n", get_type(optp->typelib, *(uint*) value)); + printf("%s\n", get_type(optp->typelib, *(ulong*) value)); break; case GET_STR: case GET_STR_ALLOC: /* fall through */

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2699)
by knielsen＠knielsen-hq.org 20 May '09

20 May '09

#At lp:maria 2699 knielsen(a)knielsen-hq.org 2009-05-11 [merge] Merge Toby Thain's Solaris fixes. modified: BUILD/SETUP.sh BUILD/compile-solaris-amd64 BUILD/compile-solaris-amd64-debug* BUILD/compile-solaris-amd64-forte BUILD/compile-solaris-amd64-forte-debug* storage/innobase/include/univ.i storage/pbxt/src/Makefile.am storage/pbxt/src/lock_xt.h storage/pbxt/src/pbms.h === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-05-06 12:03:24 +0000 +++ b/BUILD/SETUP.sh 2009-05-11 15:31:30 +0000 @@ -80,7 +80,13 @@ path=`dirname $0` . "$path/check-cpu" export AM_MAKEFLAGS -AM_MAKEFLAGS="-j 6" +# Default to a parallel build, but only if AM_MAKEFLAGS is not set. +# (So buildbots can easily disable this behaviour if required.) +if test -z "$AM_MAKEFLAGS" +then + AM_MAKEFLAGS="-j 6" +fi + # SSL library to use.--with-ssl will select our bundled yaSSL # implementation of SSL. To use openSSl you will nee too point out === modified file 'BUILD/compile-solaris-amd64' --- a/BUILD/compile-solaris-amd64 2007-04-12 11:20:38 +0000 +++ b/BUILD/compile-solaris-amd64 2009-05-09 04:01:53 +0000 @@ -1,55 +1,32 @@ -#!/usr/bin/bash +#!/bin/sh -function _find_mysql_root () ( - while [ "x$PWD" != "x/" ]; do - # Check if some directories are present - if [ -d BUILD -a -d sql -a -d mysys ]; then - echo "$PWD" - return 0 - fi - cd .. - done - return 1 -) +# Build setup for Solaris 10 + +# Make the Sun Freeware packages use the bundled Perl, instead of their own: +# ln -s `which perl` /usr/local/bin +# Your $PATH needs to include (in this order): +# /usr/local/bin:/usr/sfw/bin:/usr/ccs/bin +# (For Sun Freeware, bundled GNU utilities, Solaris ar, etc.) +# +# Required packages from http://sunfreeware.com/indexintel10.html : +# (The GNU m4 bundled with Solaris is too old.) +# automake-1.10.2-sol10-x86-local.gz +# autoconf-2.63-sol10-x86-local.gz +# m4-1.4.12-sol10-x86-local.gz +# libsigsegv-2.6-sol10-x86-local.gz +# libtool-1.5.24-sol10-x86-local.gz +# ( how to install these packages: + # wget ftp://ftp.sunfreeware.com/pub/freeware/intel/10/automake-1.10.2-sol10-x86-l… + # gunzip automake-1.10.2-sol10-x86-local.gz + # pkgadd -d automake-1.10.2-sol10-x86-local +# ) -make -k clean || true -/bin/rm -f */.deps/*.P config.cache - path=`dirname $0` -. "$path/autorun.sh" - -warning_flags="-Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Wunused" -compiler_flags="-g -O3 -fno-omit-frame-pointer" +. "$path/SETUP.sh" +extra_flags="$amd64_cflags -D__sun -m64 -mtune=athlon64" +extra_configs="$amd64_configs $max_configs --with-libevent" -export CC CXX CFLAGS CXXFLAGS LDFLAGS LIBS -CC="gcc" -CXX="gcc" -CFLAGS="$warning_flags $compiler_flags" -CXXFLAGS="" -LDFLAGS="-O3 -g -static-libgcc" -LIBS=-lmtmalloc -root=$(_find_mysql_root) +LDFLAGS="-lmtmalloc -static-libgcc" +export LDFLAGS -$root/configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-berkeley-db \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-ndbcluster \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --with-embedded-server \ - --disable-shared +. "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-debug' (properties changed: -x to +x) --- a/BUILD/compile-solaris-amd64-debug 2007-11-10 10:03:07 +0000 +++ b/BUILD/compile-solaris-amd64-debug 2009-05-09 04:01:53 +0000 @@ -1,10 +1,11 @@ -#! /bin/sh +#!/bin/sh + path=`dirname $0` . "$path/SETUP.sh" -amd64_cflags="-m64 -mtune=athlon64" -extra_flags="$amd64_cflags $debug_cflags $max_cflags" -c_warnings="$c_warnings $debug_extra_warnings" -cxx_warnings="$cxx_warnings $debug_extra_warnings" -extra_configs="$amd64_configs $debug_configs $max_configs --enable-thread-safe-client" +extra_flags="$amd64_cflags -D__sun -m64 -mtune=athlon64 $debug_cflags" +extra_configs="$amd64_configs $debug_configs $max_configs --with-libevent" + +LDFLAGS="-lmtmalloc -static-libgcc" +export LDFLAGS . "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-forte' --- a/BUILD/compile-solaris-amd64-forte 2008-09-30 20:57:48 +0000 +++ b/BUILD/compile-solaris-amd64-forte 2009-05-09 04:01:53 +0000 @@ -1,53 +1,29 @@ -#! /bin/sh +#!/bin/sh + +# See file compile-solaris-amd64 for basic pre-requisites. + +# This build uses the Sun Studio compilers (cc, CC), available from: +# http://developers.sun.com/sunstudio/downloads/index.jsp +# Note that you may want to apply current patches, as the downloaded version +# is typically out of date. Download the PKG version if you intend to patch! + +# After installing, add /opt/SUNWspro/bin to your $PATH + -gmake -k clean || true -/bin/rm -f */.deps/*.P config.cache - path=`dirname $0` -. "$path/autorun.sh" +. "$path/SETUP.sh" -# For "optimal" code for this computer add -fast to EXTRA -# To compile 64 bit, add -m64 to EXTRA_64_BIT +extra_flags="-m64 -mt -D_FORTEC_ -xbuiltin=%all -xlibmil -xlibmopt -fns=no -xprefetch=auto -xprefetch_level=3" +extra_configs="$max_configs --with-libevent" -EXTRA_64_BIT="-m64" -EXTRA="-fast" +warnings="" +c_warnings="" +cxx_warnings="" +base_cxxflags="-noex" -# -# The following should not need to be touched -# - -export CC CXX CFLAGS CXXFLAGS LIBS -STD="-g -mt -D_FORTEC_ $EXTRA $EXTRA_64_BIT" -ASFLAGS="$EXTRA_64_BIT" -CC=cc-5.0 -CFLAGS="-Xa -xstrconst $STD" +CC=cc +CFLAGS="-xstrconst" CXX=CC -CXXFLAGS="-noex $STD" -LIBS=-lmtmalloc -./configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --enable-assembler - -# Not including: -# --with-ndbcluster -# --with-berkeley-db +LDFLAGS="-lmtmalloc" -gmake -j4 -test $? = 0 && make test +. "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-forte-debug' (properties changed: -x to +x) --- a/BUILD/compile-solaris-amd64-forte-debug 2008-09-30 20:57:48 +0000 +++ b/BUILD/compile-solaris-amd64-forte-debug 2009-05-09 04:01:53 +0000 @@ -1,54 +1,27 @@ -#! /bin/sh +#!/bin/sh -gmake -k clean || true -/bin/rm -f */.deps/*.P config.cache - path=`dirname $0` -. "$path/autorun.sh" +. "$path/SETUP.sh" -# To compile 64 bit, add -m64 to EXTRA_64_BIT -EXTRA_64_BIT="-m64" +# Take only #define options - the others are gcc specific. +# (real fix is for SETUP.sh not to put gcc specific options in $debug_cflags) +DEFS="" +for F in $debug_cflags ; do + expr "$F" : "^-D" && DEFS="$DEFS $F" +done +debug_cflags="-O0 -g $DEFS" + +extra_flags="-m64 -mt -D_FORTEC_ -xlibmopt -fns=no $debug_cflags" +extra_configs="$max_configs --with-libevent $debug_configs" + +warnings="" +c_warnings="" +cxx_warnings="" +base_cxxflags="-noex" -# For "optimal" code for this computer add -fast to EXTRA. Note that -# this causes problem with debugging the program since -fast implies -# -xO5. -EXTRA="" - -# -# The following should not need to be touched -# - -export CC CXX CFLAGS CXXFLAGS -STD="-g -mt -D_FORTEC_ $EXTRA $EXTRA_64_BIT $debug_cflags" -ASFLAGS="$EXTRA_64_BIT" -CC=cc-5.0 -CFLAGS="-Xa -xstrconst $STD" +CC=cc +CFLAGS="-xstrconst" CXX=CC -CXXFLAGS="-noex $STD" -./configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --with-debug \ - --enable-assembler - -# Not including: -# --with-ndbcluster -# --with-berkeley-db +LDFLAGS="-lmtmalloc" -gmake -j4 +. "$path/FINISH.sh" === modified file 'storage/innobase/include/univ.i' --- a/storage/innobase/include/univ.i 2009-05-06 12:03:24 +0000 +++ b/storage/innobase/include/univ.i 2009-05-11 15:31:30 +0000 @@ -55,10 +55,10 @@ of the 32-bit x86 assembler in mutex ope # define UNIV_CAN_USE_X86_ASSEMBLER # endif -/* We only try to do explicit inlining of functions with gcc and -Microsoft Visual C++ */ +/* Enable explicit inlining of functions only for compilers known to +support it. */ -# if !defined(__GNUC__) +# if !defined(__GNUC__) && !defined(__SUNPRO_C) # undef UNIV_MUST_NOT_INLINE /* Remove compiler warning */ # define UNIV_MUST_NOT_INLINE # endif === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-04-01 06:40:11 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-05-09 04:01:53 +0000 @@ -46,7 +46,7 @@ libpbxt_la_CFLAGS = $(AM_CFLAGS) -DMYSQ EXTRA_LIBRARIES = libpbxt.a noinst_LIBRARIES = libpbxt.a libpbxt_a_SOURCES = $(libpbxt_la_SOURCES) -libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) -Wno-long-long +libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) libpbxt_a_CFLAGS = $(AM_CFLAGS) -std=c99 EXTRA_DIST = CMakeLists.txt === modified file 'storage/pbxt/src/lock_xt.h' --- a/storage/pbxt/src/lock_xt.h 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/lock_xt.h 2009-05-09 04:01:53 +0000 @@ -208,9 +208,9 @@ inline void xt_atomic_dec2(volatile xtWo #elif defined(__GNUC__) __sync_fetch_and_sub(mptr, 1); #elif defined(XT_SPL_SOLARIS_LIB) - val1 = atomic_dec_16_nv(mptr); + atomic_dec_16_nv(mptr); #else - val1 = --(*mptr); + --(*mptr); #endif } === modified file 'storage/pbxt/src/pbms.h' --- a/storage/pbxt/src/pbms.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/pbms.h 2009-05-09 04:01:53 +0000 @@ -782,21 +782,28 @@ private: void deleteTempFiles() { - struct dirent entry; + struct dirent *entry; struct dirent *result; DIR *odir; int err; + size_t sz; char temp_file[100]; +#ifdef XT_SOLARIS + sz = sizeof(struct dirent) + pathconf("/tmp/", _PC_NAME_MAX); // Solaris, see readdir(3C) +#else + sz = sizeof(struct dirent); +#endif + entry = (struct dirent*)malloc(sz); if (!(odir = opendir("/tmp/"))) return; - err = readdir_r(odir, &entry, &result); + err = readdir_r(odir, entry, &result); while (!err && result) { const char **prefix = temp_prefix; while (*prefix) { - if (startsWith(entry.d_name, *prefix)) { - int pid = atoi(entry.d_name + strlen(*prefix)); + if (startsWith(entry->d_name, *prefix)) { + int pid = atoi(entry->d_name + strlen(*prefix)); /* If the process does not exist: */ if (kill(pid, 0) == -1 && errno == ESRCH) { @@ -807,9 +814,10 @@ private: prefix++; } - err = readdir_r(odir, &entry, &result); + err = readdir_r(odir, entry, &result); } closedir(odir); + free(entry); } }; #endif // PBMS_API

3 3

[Maria-developers] compile issue and the -lz flag
by Don Kehn 19 May '09

19 May '09

Hi all: when compiling on x64 systems (i.e. CentOS & Fedora core 10) and using the BUILD/compile-pentium64-max, I'm getting the -lz error "/usr/bin/ld: cannot find -lz" and when I simplify the ./comfigure line to a --without-zlib it works. So, after modifing the configure flags that results into: ./configure --prefix=/u02/local/mysql-5.1.34 --enable-assembler --with-extra-charsets=complex --enable-thread-safe-client --with-big-tables --with-readline --with-ssl --with-plugins=max --with-embedded-server --with-mysqld-ldflags=-all-static --with-client-ldflags=-all-static --without-zlib --enable-local-infile It still has problems with the -lz flag, so does anybody know what is bringing the -lz into the configure? Note: I've also tried --with-zlibdir=/usr/lib64, but same error. Thanks tons.

4 11

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2700)
by knielsen＠knielsen-hq.org 18 May '09

18 May '09

#At lp:maria 2700 knielsen(a)knielsen-hq.org 2009-05-18 Fix support-files/build-tags for newer bzr which do not default to --recursive on bzr ls. modified: support-files/build-tags === modified file 'support-files/build-tags' --- a/support-files/build-tags 2008-08-06 20:25:25 +0000 +++ b/support-files/build-tags 2009-05-18 07:29:14 +0000 @@ -4,7 +4,7 @@ rm -f TAGS filter='\.cc$\|\.c$\|\.h$\|\.yy$' list="find . -type f" -bzr root >/dev/null 2>/dev/null && list="bzr ls --kind=file --versioned" +bzr root >/dev/null 2>/dev/null && list="bzr ls --recursive --kind=file --versioned" $list |grep $filter |while read f; do

1 0

[Maria-developers] ptr_compare replace with memcmp and associated performance
by Stewart Smith 17 May '09

17 May '09

So i broke out mtaylor's patch that we were bumming around with at the UC to replace ptr_compare with a simple memcmp call. At the UC I benched that this patch actually caused a measurable performance regresssion. So what's the difference? (same benchmark and machine as in previous mail) MAX_FIELDS=64 with std::bitset read/write requests: 70000 (3374.50 per sec.) read/write requests: 70000 (3102.42 per sec.) read/write requests: 70000 (3113.47 per sec.) read/write requests: 70000 (3401.89 per sec.) read/write requests: 70000 (3164.61 per sec.) AVERAGE= 3231.37 With ptr_compare replaced with memcmp: read/write requests: 70000 (3090.33 per sec.) read/write requests: 70000 (3066.97 per sec.) read/write requests: 70000 (2954.93 per sec.) read/write requests: 70000 (2875.57 per sec.) read/write requests: 70000 (2953.99 per sec.) AVERAGE= 2988.35 With ptr_compare replaced with __builtin_memcmp: read/write requests: 70000 (3245.37 per sec.) read/write requests: 70000 (3001.91 per sec.) read/write requests: 70000 (3254.99 per sec.) read/write requests: 70000 (3177.13 per sec.) read/write requests: 70000 (3195.49 per sec.) AVERAGE= 3174.97 So, I thought that with just using the __builtin_memcmp automatically, we may be okay with merging this. However... I do have the following concerns: 1) I'm pretty sure that __builtin_memcmp is gcc only, and that SunStudio may have an issue (yay - wrappers!) 2) Is this perf difference going to be true on different platforms? 3) the various memcmp implementations do seem to be dependent on a few things for performance: alignment, size of data. For what we're seeing in sysbench, the parameters are things like this: "SELECT c from sbtest where id between 9942 and 10041 order by c" 0xc30780,0xc2c340,240 0xc2e560,0xc2c340,240 0xc30780,0xc2e560,240 0xc37390,0xc32f50,240 0xc35170,0xc32f50,240 0xc37390,0xc35170,240 0xc3dcc8,0xc39888,240 0xc3baa8,0xc2e560,240 0xc35170,0xc2e560,240 0xc3baa8,0xc35170,240 0xc2c340,0xc35170,240 0xc35170,0xc3dcc8,240 0xc2c618,0xc35170,240 0xc35170,0xc3d9f0,240 0xc2c8f0,0xc35170,240 0x7fbed85f10f0,0x7fbed85eccb0,240 and from the CREATE TABLE: `c` VARCHAR(120) NOT NULL COLLATE utf8_general_ci DEFAULT `` , So here it's large, and aligned. For some other micro-benchmarks I've clocked things looking much different.... so it's possibly query dependent as to what ends up being better (i.e. how much data we're comparing). (1st parameter is number of repetitions, 2nd is number of bytes to memcmp) For comparing equal values: stewart@willster:~/src/test/memcmp$ ./a.out 168435455 8 168435455 repetitions Testing memcmp ..... done, 7.432 seconds Testing builtin memcmp ....... done, 16.313 seconds Testing loop .... done, 6.144 seconds Testing loop32 .... done, 2.764 seconds Testing loop64 .... done, 2.128 seconds Testing no-op .... done, 1.488 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 16 168435455 repetitions Testing memcmp ..... done, 5.912 seconds Testing builtin memcmp ....... done, 26.418 seconds Testing loop .... done, 11.201 seconds Testing loop32 .... done, 3.980 seconds Testing loop64 .... done, 2.564 seconds Testing no-op .... done, 1.492 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 32 168435455 repetitions Testing memcmp ..... done, 6.828 seconds Testing builtin memcmp ....... done, 46.891 seconds Testing loop .... done, 21.473 seconds Testing loop32 .... done, 6.536 seconds Testing loop64 .... done, 3.804 seconds Testing no-op .... done, 1.476 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 64 168435455 repetitions Testing memcmp ..... done, 9.549 seconds Testing builtin memcmp ....... done, 87.513 seconds Testing loop .... done, 41.455 seconds Testing loop32 .... done, 11.669 seconds Testing loop64 .... done, 6.368 seconds Testing no-op .... done, 1.468 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 128 168435455 repetitions Testing memcmp ..... done, 15.081 seconds Testing builtin memcmp ....... done, 169.143 seconds Testing loop .... done, 86.397 seconds Testing loop32 .... done, 21.877 seconds Testing loop64 .... done, 11.445 seconds Testing no-op .... done, 1.488 seconds stewart@willster:~/src/test/memcmp$ gcc -O3 -fno-builtin memcmpbench.c stewart@willster:~/src/test/memcmp$ ./a.out 168435455 256 168435455 repetitions Testing memcmp ..... done, 26.134 seconds Testing loop64 .... done, 21.549 seconds Testing no-op .... done, 1.500 seconds Completely inequal values are all about the same: stewart@willster:~/src/test/memcmp$ ./a.out 168435455 8 168435455 repetitions Testing memcmp ..... done, 3.204 seconds Testing builtin memcmp ....... done, 13.945 seconds Testing loop .... done, 1.896 seconds Testing loop32 .... done, 2.092 seconds Testing loop64 .... done, 2.136 seconds Testing no-op .... done, 1.488 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 240 168435455 repetitions Testing memcmp ..... done, 5.520 seconds Testing builtin memcmp ....... done, 13.973 seconds Testing loop .... done, 1.912 seconds Testing loop32 .... done, 2.124 seconds Testing loop64 .... done, 2.104 seconds Testing no-op .... done, 1.468 seconds For half the same: stewart@willster:~/src/test/memcmp$ ./a.out 168435455 8 168435455 repetitions Testing memcmp ..... done, 7.340 seconds Testing builtin memcmp ....... done, 19.193 seconds Testing loop .... done, 4.212 seconds Testing loop32 .... done, 2.956 seconds Testing loop64 .... done, 2.112 seconds Testing no-op .... done, 1.476 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 32 168435455 repetitions Testing memcmp ..... done, 5.920 seconds Testing builtin memcmp ....... done, 34.526 seconds Testing loop .... done, 11.885 seconds Testing loop32 .... done, 4.444 seconds Testing loop64 .... done, 3.388 seconds Testing no-op .... done, 1.468 seconds stewart@willster:~/src/test/memcmp$ ./a.out 168435455 64 168435455 repetitions Testing memcmp ..... done, 6.948 seconds Testing builtin memcmp ....... done, 54.923 seconds Testing loop .... done, 22.081 seconds Testing loop32 .... done, 7.000 seconds Testing loop64 .... done, 4.444 seconds Testing no-op .... done, 1.488 seconds Is concurrency an issue here and not raw single threaded performance? Let's try with 64 threads (on the same 2 core box): 64 threads, __builtin_memcmp: read/write requests: 70000 (3184.20 per sec.) read/write requests: 70000 (3184.87 per sec.) read/write requests: 70000 (3166.83 per sec.) read/write requests: 70000 (3085.69 per sec.) AVERAGE=3155.39 64 threads, memcmp: read/write requests: 70000 (3218.07 per sec.) read/write requests: 70000 (3219.04 per sec.) read/write requests: 70000 (3222.48 per sec.) read/write requests: 70000 (3116.00 per sec.) AVERAGE=3193.89 64 threads, 32bit cmp loop: read/write requests: 70000 (3173.76 per sec.) read/write requests: 70000 (3156.23 per sec.) read/write requests: 70000 (3206.70 per sec.) read/write requests: 70000 (3247.61 per sec.) AVERAGE=3196.07 64 threads, 64bit cmp loop: read/write requests: 70000 (3210.02 per sec.) read/write requests: 70000 (3218.87 per sec.) read/write requests: 70000 (3225.98 per sec.) read/write requests: 70000 (3131.60 per sec.) AVERAGE= 3196.61 64 threads, baseline (using original ptr_compare): read/write requests: 70000 (3542.95 per sec.) read/write requests: 70000 (3556.15 per sec.) read/write requests: 70000 (3560.71 per sec.) read/write requests: 70000 (3476.80 per sec.) AVERAGE=3534.15 i.e. the old ptr_compare whoops the arse of any of the memcmp calls with higher concurrency. So, after this I can conclude: - memcmp is nothing if not consistent across various microbenchmarks - builtin_memcmp seems to help a bit at lower concurrency, not at all at higher and is seldom useful in micro benchmark. - the 64bit loop isn't so good at higher concurrency - the 64bit loop is favourable in microbenchmarks - memcmp is close to the 64bit loop performance (at most only 2x slower) Unrolling loops being the key? Testing loop .... done, 11.813 seconds Testing unrollloop .... done, 7.628 seconds Testing unrollloop32 .... done, 2.532 seconds Testing unrollloop64 .... done, 2.536 seconds Testing loop32 .... done, 4.432 seconds Testing loop64 .... done, 3.792 seconds Testing no-op .... done, 1.276 seconds (32 and 64 do 32,64 bit compares in an unrolled loop) Questions: - Why does my 64bit compare loop not equal performance of ptr_compare unrolled loop? (in a microbenchmark, it beats it). Concurrency again? So what should we do? Certainly a call to memcmp is easier to understand and keeps the code nice and simple, but we're talking up to a 10% speed hit here at higher concurrency (at least on my hardware). I wonder what the difference is on various other hardware setups. I'd be very interested to see what happens on sparc. I'd also like to see it micro-benchmarked - preferably something we could repeat in future (even in ./configure ? on server startup ?) I'm voting to keep the current ptr_compare code, and hope that somebody provides further explanation on top of what I've looked at here. Below is the patch, followed by the code i used for microbenchmarking (note the commented out 64bit loop i used for the above loop tests): === modified file 'drizzled/filesort.cc' --- drizzled/filesort.cc 2009-04-28 00:17:10 +0000 +++ drizzled/filesort.cc 2009-05-08 05:59:11 +0000 @@ -1135,7 +1135,7 @@ int merge_buffers(SORTPARAM *param, IO_C } else { - cmp= get_ptr_compare(sort_length); + cmp= (qsort2_cmp)ptr_compare; first_cmp_arg= (void*) &sort_length; } priority_queue<BUFFPEK *, vector<BUFFPEK *>, compare_functor > === modified file 'mysys/mf_sort.cc' --- mysys/mf_sort.cc 2009-04-17 21:01:47 +0000 +++ mysys/mf_sort.cc 2009-05-08 05:59:11 +0000 @@ -34,7 +34,7 @@ void my_string_ptr_sort(unsigned char *b { if (size && items) { - my_qsort2(base,items, sizeof(unsigned char*), get_ptr_compare(size), + my_qsort2(base,items, sizeof(unsigned char*), (qsort2_cmp)ptr_compare, (void*) &size); } } === modified file 'mysys/my_sys.h' --- mysys/my_sys.h 2009-04-27 22:05:43 +0000 +++ mysys/my_sys.h 2009-05-08 05:59:11 +0000 @@ -420,7 +420,12 @@ extern void my_qsort(void *base_ptr, siz qsort_cmp cmp); extern void my_qsort2(void *base_ptr, size_t total_elems, size_t size, qsort2_cmp cmp, void *cmp_argument); -extern qsort2_cmp get_ptr_compare(size_t); + +#if defined(__cplusplus) +extern "C" +#endif +int ptr_compare(size_t *compare_length, unsigned char **a, unsigned char **b); + void my_store_ptr(unsigned char *buff, size_t pack_length, my_off_t pos); my_off_t my_get_ptr(unsigned char *ptr, size_t pack_length); File create_temp_file(char *to, const char *dir, const char *pfx, === modified file 'mysys/ptr_cmp.cc' --- mysys/ptr_cmp.cc 2009-04-26 16:53:32 +0000 +++ mysys/ptr_cmp.cc 2009-05-08 05:59:11 +0000 @@ -21,137 +21,33 @@ #include "mysys/mysys_priv.h" #include "plugin/myisam/myisampack.h" - -static int ptr_compare(size_t *compare_length, unsigned char **a, unsigned char **b); -static int ptr_compare_0(size_t *compare_length, unsigned char **a, unsigned char **b); -static int ptr_compare_1(size_t *compare_length, unsigned char **a, unsigned char **b); -static int ptr_compare_2(size_t *compare_length, unsigned char **a, unsigned char **b); -static int ptr_compare_3(size_t *compare_length, unsigned char **a, unsigned char **b); - - /* Get a pointer to a optimal byte-compare function for a given size */ - -qsort2_cmp get_ptr_compare (size_t size) -{ - if (size < 4) - return (qsort2_cmp) ptr_compare; - switch (size & 3) { - case 0: return (qsort2_cmp) ptr_compare_0; - case 1: return (qsort2_cmp) ptr_compare_1; - case 2: return (qsort2_cmp) ptr_compare_2; - case 3: return (qsort2_cmp) ptr_compare_3; - } - return 0; /* Impossible */ -} - +#include <string.h> /* Compare to keys to see witch is smaller. - Loop unrolled to make it quick !! */ -#define cmp(N) if (first[N] != last[N]) return (int) first[N] - (int) last[N] - -static int ptr_compare(size_t *compare_length, unsigned char **a, unsigned char **b) -{ - register int length= *compare_length; - register unsigned char *first,*last; - - first= *a; last= *b; - while (--length) - { - if (*first++ != *last++) - return (int) first[-1] - (int) last[-1]; - } - return (int) first[0] - (int) last[0]; -} - - -static int ptr_compare_0(size_t *compare_length,unsigned char **a, unsigned char **b) +extern "C" +int ptr_compare(size_t *compare_length, unsigned char **a, unsigned char **b) { - register int length= *compare_length; - register unsigned char *first,*last; - - first= *a; last= *b; - loop: - cmp(0); - cmp(1); - cmp(2); - cmp(3); - if ((length-=4)) +/* size_t l= *compare_length / sizeof(uint64_t); + uint64_t *aa= (uint64_t*)*a; + uint64_t *bb= (uint64_t*)*b; + while(l--) { - first+=4; - last+=4; - goto loop; - } - return (0); -} - - -static int ptr_compare_1(size_t *compare_length,unsigned char **a, unsigned char **b) -{ - register int length= *compare_length-1; - register unsigned char *first,*last; - - first= *a+1; last= *b+1; - cmp(-1); - loop: - cmp(0); - cmp(1); - cmp(2); - cmp(3); - if ((length-=4)) - { - first+=4; - last+=4; - goto loop; - } - return (0); -} - -static int ptr_compare_2(size_t *compare_length,unsigned char **a, unsigned char **b) -{ - register int length= *compare_length-2; - register unsigned char *first,*last; - - first= *a +2 ; last= *b +2; - cmp(-2); - cmp(-1); - loop: - cmp(0); - cmp(1); - cmp(2); - cmp(3); - if ((length-=4)) - { - first+=4; - last+=4; - goto loop; + if(*aa != *bb) + { + if(*aa < *bb) + return -1; + else + return 1; + } + aa++, bb++; } - return (0); + return 0; +*/ return memcmp(*a, *b, *compare_length); } -static int ptr_compare_3(size_t *compare_length,unsigned char **a, unsigned char **b) -{ - register int length= *compare_length-3; - register unsigned char *first,*last; - - first= *a +3 ; last= *b +3; - cmp(-3); - cmp(-2); - cmp(-1); - loop: - cmp(0); - cmp(1); - cmp(2); - cmp(3); - if ((length-=4)) - { - first+=4; - last+=4; - goto loop; - } - return (0); -} void my_store_ptr(unsigned char *buff, size_t pack_length, my_off_t pos) { For those interested, i started with the code at http://gcc.gnu.org/ml/gcc/2002-10/msg01666.html and ended up with something like this: #include <sys/resource.h> #include <sys/time.h> #include <stdio.h> #include <assert.h> #include <stdlib.h> char s1[256]; char s2[256]; int cmpsize= 240; int speed_memcmp () { return memcmp (s1, s2, cmpsize); } int speed_bimemcmp () { return __builtin_memcmp (s1, s2, cmpsize); } int speed_loop () { int i=cmpsize; char *a= s1; char *b= s2; while(i--) { if(*a != *b) return -1; a++, b++; }; return 0; } int speed_loop32 () { int i=cmpsize/4; int *a= (int*)s1; int *b= (int*)s2; while(i--) { if(*a != *b) return -1; a++, b++; }; return 0; } int speed_loop64 () { int i=cmpsize/8; long long *a= (long long*)s1; long long *b= (long long*)s2; while(i--) { if(*a != *b) return -1; a++, b++; }; return 0; } int speed_null() { return 0; } double do_test (int repetitions, int (*test_function) ()) { struct rusage r1, r2; getrusage(RUSAGE_SELF, &r1); while (repetitions--) test_function (); getrusage(RUSAGE_SELF, &r2); return (r2.ru_utime.tv_sec - r1.ru_utime.tv_sec) + (r2.ru_utime.tv_usec - r1.ru_utime.tv_usec) / 1000000.0; } main(int argc, char **argv) { int repetitions = (argc == 1) ? 0x0fffffff : atoi(argv[1]); cmpsize = (argc == 2) ? 240 : atoi(argv[2]); memset(s1, 42, 256); memset(s2, 42, 256); memset(s2+(cmpsize/2), 55, cmpsize/2); printf ("%d repetitions\n\n", repetitions); printf ("Testing memcmp ....."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_memcmp)); printf ("Testing builtin memcmp ......."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_bimemcmp)); printf ("Testing loop ...."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_loop)); printf ("Testing loop32 ...."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_loop32)); printf ("Testing loop64 ...."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_loop64)); printf ("Testing no-op ...."); fflush (stdout); printf (" done, %10.3f seconds\n", do_test (repetitions, speed_null)); exit (0); } -- Stewart Smith

6 5

[Maria-developers] Cmake changes
by Stewart Smith 16 May '09

16 May '09

Antony mentioned on IRC about CMake changes that let you not use config.js and other nasty things when building on Win32 (and Win64). These are all just in the MySQL Cluster branch - namely 7.0. Who knows when this gets back into any mainline branch though... -- Stewart Smith

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2699)
by knielsen＠knielsen-hq.org 15 May '09

15 May '09

#At lp:maria 2699 knielsen(a)knielsen-hq.org 2009-05-15 test commit 1 modified: README === modified file 'README' --- a/README 2008-03-26 10:15:03 +0000 +++ b/README 2009-05-15 06:31:51 +0000 @@ -33,3 +33,4 @@ file. IMPORTANT: Bug or error reports should be sent to http://bugs.mysql.com. +

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2689)
by knielsen＠knielsen-hq.org 13 May '09

13 May '09

#At lp:maria 2689 knielsen(a)knielsen-hq.org 2009-03-31 Fix build error after last push with --with-debug=full due to SAFEMALLOC now being defined in my_config.h (as opposed to in CFLAGS before.) modified: mysys/my_malloc.c mysys/my_once.c mysys/my_realloc.c per-file messages: mysys/my_malloc.c Need to include my_global.h before messing with SAFEMALLOC, as now that macro may be re-defined in my_config.h, which is included from my_global.h mysys/my_once.c Need to include my_global.h before messing with SAFEMALLOC, as now that macro may be re-defined in my_config.h, which is included from my_global.h mysys/my_realloc.c Need to include my_global.h before messing with SAFEMALLOC, as now that macro may be re-defined in my_config.h, which is included from my_global.h === modified file 'mysys/my_malloc.c' --- a/mysys/my_malloc.c 2007-10-02 07:32:33 +0000 +++ b/mysys/my_malloc.c 2009-03-31 08:06:51 +0000 @@ -13,6 +13,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif === modified file 'mysys/my_once.c' --- a/mysys/my_once.c 2007-05-10 09:59:39 +0000 +++ b/mysys/my_once.c 2009-03-31 08:06:51 +0000 @@ -15,6 +15,9 @@ /* Not MT-SAFE */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif === modified file 'mysys/my_realloc.c' --- a/mysys/my_realloc.c 2008-04-28 16:24:05 +0000 +++ b/mysys/my_realloc.c 2009-03-31 08:06:51 +0000 @@ -13,6 +13,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif

8 29

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2698)
by knielsen＠knielsen-hq.org 10 May '09

10 May '09

#At lp:maria 2698 knielsen(a)knielsen-hq.org 2009-05-05 [merge] test merge. modified: BUILD/compile-solaris-amd64 BUILD/compile-solaris-amd64-debug* BUILD/compile-solaris-amd64-forte BUILD/compile-solaris-amd64-forte-debug* storage/innobase/include/univ.i storage/pbxt/src/Makefile.am storage/pbxt/src/lock_xt.h storage/pbxt/src/pbms.h === modified file 'BUILD/compile-solaris-amd64' --- a/BUILD/compile-solaris-amd64 2007-04-12 11:20:38 +0000 +++ b/BUILD/compile-solaris-amd64 2009-05-04 14:36:55 +0000 @@ -1,55 +1,34 @@ -#!/usr/bin/bash +#!/bin/sh -function _find_mysql_root () ( - while [ "x$PWD" != "x/" ]; do - # Check if some directories are present - if [ -d BUILD -a -d sql -a -d mysys ]; then - echo "$PWD" - return 0 - fi - cd .. - done - return 1 -) +# Pre-requisites for Solaris 10: +# +# setup: +# ln -s `which perl` /usr/local/bin +# your $PATH needs to include (in this order): +# /usr/local/bin:/usr/sfw/bin:/usr/ccs/bin +# +# packages from http://sunfreeware.com/indexintel10.html : +# automake-1.10.2-sol10-x86-local.gz +# autoconf-2.63-sol10-x86-local.gz +# m4-1.4.12-sol10-x86-local.gz +# libsigsegv-2.6-sol10-x86-local.gz +# libtool-1.5.24-sol10-x86-local.gz +# ( how to install these packages: + # wget ftp://ftp.sunfreeware.com/pub/freeware/intel/10/automake-1.10.2-sol10-x86-l… + # gunzip automake-1.10.2-sol10-x86-local.gz + # pkgadd -d automake-1.10.2-sol10-x86-local +# ) + +# +# if using --with-libevent, install that library from +# http://www.monkey.org/~provos/libevent/ -make -k clean || true -/bin/rm -f */.deps/*.P config.cache - path=`dirname $0` -. "$path/autorun.sh" - -warning_flags="-Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Wunused" -compiler_flags="-g -O3 -fno-omit-frame-pointer" +. "$path/SETUP.sh" +extra_flags="$amd64_cflags -D__sun -m64 -mtune=athlon64" +extra_configs="$amd64_configs $max_configs --with-libevent" -export CC CXX CFLAGS CXXFLAGS LDFLAGS LIBS -CC="gcc" -CXX="gcc" -CFLAGS="$warning_flags $compiler_flags" -CXXFLAGS="" -LDFLAGS="-O3 -g -static-libgcc" -LIBS=-lmtmalloc -root=$(_find_mysql_root) +LDFLAGS="-lmtmalloc -static-libgcc" +export LDFLAGS -$root/configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-berkeley-db \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-ndbcluster \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --with-embedded-server \ - --disable-shared +. "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-debug' (properties changed: -x to +x) --- a/BUILD/compile-solaris-amd64-debug 2007-11-10 10:03:07 +0000 +++ b/BUILD/compile-solaris-amd64-debug 2009-05-04 14:36:55 +0000 @@ -1,10 +1,11 @@ -#! /bin/sh +#!/bin/sh + path=`dirname $0` . "$path/SETUP.sh" -amd64_cflags="-m64 -mtune=athlon64" -extra_flags="$amd64_cflags $debug_cflags $max_cflags" -c_warnings="$c_warnings $debug_extra_warnings" -cxx_warnings="$cxx_warnings $debug_extra_warnings" -extra_configs="$amd64_configs $debug_configs $max_configs --enable-thread-safe-client" +extra_flags="$amd64_cflags -D__sun -m64 $debug_cflags" +extra_configs="$amd64_configs $debug_configs $max_configs --with-libevent" + +LDFLAGS="-lmtmalloc -static-libgcc" +export LDFLAGS . "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-forte' --- a/BUILD/compile-solaris-amd64-forte 2008-09-30 20:57:48 +0000 +++ b/BUILD/compile-solaris-amd64-forte 2009-05-04 15:48:27 +0000 @@ -1,53 +1,35 @@ -#! /bin/sh +#!/bin/sh -gmake -k clean || true -/bin/rm -f */.deps/*.P config.cache - -path=`dirname $0` -. "$path/autorun.sh" +# See file compile-solaris-amd64 for basic pre-requisites. + +# This build uses the Sun Studio compilers (cc, CC). +# http://developers.sun.com/sunstudio/downloads/index.jsp +# Note that you may want to apply current patches, as the downloaded version +# is typically out of date. Download the PKG version if you intend to patch! + +# After installing, add /opt/SUNWspro/bin to your $PATH -# For "optimal" code for this computer add -fast to EXTRA -# To compile 64 bit, add -m64 to EXTRA_64_BIT -EXTRA_64_BIT="-m64" -EXTRA="-fast" +path=`dirname $0` +. "$path/SETUP.sh" + +extra_flags="-m64 -mt -D_FORTEC_ -xbuiltin=%all -xlibmil -xlibmopt -fns=no -xprefetch=auto -xprefetch_level=3" +extra_configs="$max_configs --with-libevent" -# -# The following should not need to be touched -# - -export CC CXX CFLAGS CXXFLAGS LIBS -STD="-g -mt -D_FORTEC_ $EXTRA $EXTRA_64_BIT" -ASFLAGS="$EXTRA_64_BIT" -CC=cc-5.0 -CFLAGS="-Xa -xstrconst $STD" +# I want this for my buildbot, which has limited zone resources +#export AM_MAKEFLAGS +#AM_MAKEFLAGS="" + +export warnings c_warnings cxx_warnings base_cxxflags +warnings="" +c_warnings="" +cxx_warnings="" +base_cxxflags="-noex" + +export CC CXX CFLAGS CXXFLAGS LDFLAGS +CC=cc +CFLAGS="-xstrconst" CXX=CC -CXXFLAGS="-noex $STD" -LIBS=-lmtmalloc -./configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --enable-assembler - -# Not including: -# --with-ndbcluster -# --with-berkeley-db +LDFLAGS="-lmtmalloc" -gmake -j4 -test $? = 0 && make test +. "$path/FINISH.sh" === modified file 'BUILD/compile-solaris-amd64-forte-debug' (properties changed: -x to +x) --- a/BUILD/compile-solaris-amd64-forte-debug 2008-09-30 20:57:48 +0000 +++ b/BUILD/compile-solaris-amd64-forte-debug 2009-05-04 15:48:27 +0000 @@ -1,54 +1,28 @@ -#! /bin/sh +#!/bin/sh -gmake -k clean || true -/bin/rm -f */.deps/*.P config.cache - path=`dirname $0` -. "$path/autorun.sh" +. "$path/SETUP.sh" -# To compile 64 bit, add -m64 to EXTRA_64_BIT -EXTRA_64_BIT="-m64" - -# For "optimal" code for this computer add -fast to EXTRA. Note that -# this causes problem with debugging the program since -fast implies -# -xO5. -EXTRA="" - -# -# The following should not need to be touched -# - -export CC CXX CFLAGS CXXFLAGS -STD="-g -mt -D_FORTEC_ $EXTRA $EXTRA_64_BIT $debug_cflags" -ASFLAGS="$EXTRA_64_BIT" -CC=cc-5.0 -CFLAGS="-Xa -xstrconst $STD" +# take only #define options - the others are gcc specific +DEFS="" +for F in $debug_cflags ; do + expr "$F" : "^-D" && DEFS="$DEFS $F" +done +debug_cflags="-O0 -g $DEFS" + +extra_flags="-m64 -mt -D_FORTEC_ -xlibmopt -fns=no $debug_cflags" +extra_configs="$max_configs --with-libevent $debug_configs" + +export warnings c_warnings cxx_warnings base_cxxflags debug_cflags +warnings="" +c_warnings="" +cxx_warnings="" +base_cxxflags="-noex" + +export CC CXX CFLAGS CXXFLAGS LDFLAGS +CC=cc +CFLAGS="-xstrconst" CXX=CC -CXXFLAGS="-noex $STD" -./configure \ - --prefix=/usr/local/mysql \ - --localstatedir=/usr/local/mysql/data \ - --libexecdir=/usr/local/mysql/bin \ - --with-extra-charsets=complex \ - --enable-thread-safe-client \ - --enable-local-infile \ - --with-zlib-dir=bundled \ - --with-big-tables \ - --with-readline \ - --with-archive-storage-engine \ - --with-named-curses=-lcurses \ - --with-big-tables \ - --with-innodb \ - --with-example-storage-engine \ - --with-blackhole-storage-engine \ - --with-federated-storage-engine \ - --with-csv-storage-engine \ - --with-ssl \ - --with-debug \ - --enable-assembler - -# Not including: -# --with-ndbcluster -# --with-berkeley-db +LDFLAGS="-lmtmalloc" -gmake -j4 +. "$path/FINISH.sh" === modified file 'storage/innobase/include/univ.i' --- a/storage/innobase/include/univ.i 2008-03-27 01:40:45 +0000 +++ b/storage/innobase/include/univ.i 2009-05-04 14:54:10 +0000 @@ -58,7 +58,7 @@ of the 32-bit x86 assembler in mutex ope /* We only try to do explicit inlining of functions with gcc and Microsoft Visual C++ */ -# if !defined(__GNUC__) +# if !defined(__GNUC__) && !defined(__SUNPRO_C) # undef UNIV_MUST_NOT_INLINE /* Remove compiler warning */ # define UNIV_MUST_NOT_INLINE # endif === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-04-01 06:40:11 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-05-04 14:36:55 +0000 @@ -46,7 +46,7 @@ libpbxt_la_CFLAGS = $(AM_CFLAGS) -DMYSQ EXTRA_LIBRARIES = libpbxt.a noinst_LIBRARIES = libpbxt.a libpbxt_a_SOURCES = $(libpbxt_la_SOURCES) -libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) -Wno-long-long +libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) libpbxt_a_CFLAGS = $(AM_CFLAGS) -std=c99 EXTRA_DIST = CMakeLists.txt === modified file 'storage/pbxt/src/lock_xt.h' --- a/storage/pbxt/src/lock_xt.h 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/lock_xt.h 2009-05-04 14:36:55 +0000 @@ -208,9 +208,9 @@ inline void xt_atomic_dec2(volatile xtWo #elif defined(__GNUC__) __sync_fetch_and_sub(mptr, 1); #elif defined(XT_SPL_SOLARIS_LIB) - val1 = atomic_dec_16_nv(mptr); + atomic_dec_16_nv(mptr); #else - val1 = --(*mptr); + --(*mptr); #endif } === modified file 'storage/pbxt/src/pbms.h' --- a/storage/pbxt/src/pbms.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/pbms.h 2009-05-03 16:31:02 +0000 @@ -782,21 +782,28 @@ private: void deleteTempFiles() { - struct dirent entry; + struct dirent *entry; struct dirent *result; DIR *odir; int err; + size_t sz; char temp_file[100]; +#ifdef XT_SOLARIS + sz = sizeof(struct dirent) + pathconf("/tmp/", _PC_NAME_MAX); // Solaris, see readdir(3C) +#else + sz = sizeof(struct dirent); +#endif + entry = (struct dirent*)malloc(sz); if (!(odir = opendir("/tmp/"))) return; - err = readdir_r(odir, &entry, &result); + err = readdir_r(odir, entry, &result); while (!err && result) { const char **prefix = temp_prefix; while (*prefix) { - if (startsWith(entry.d_name, *prefix)) { - int pid = atoi(entry.d_name + strlen(*prefix)); + if (startsWith(entry->d_name, *prefix)) { + int pid = atoi(entry->d_name + strlen(*prefix)); /* If the process does not exist: */ if (kill(pid, 0) == -1 && errno == ESRCH) { @@ -807,9 +814,10 @@ private: prefix++; } - err = readdir_r(odir, &entry, &result); + err = readdir_r(odir, entry, &result); } closedir(odir); + free(entry); } }; #endif // PBMS_API

4 10

Re: [Maria-developers] SQL Parser
by Antony T Curtis 09 May '09

09 May '09

On 9 May 2009, at 10:22, Jim Idle wrote: > Antony T Curtis wrote: >> >> On 9 May 2009, at 00:35, Michael Widenius wrote: >> >>> >>> jim> Indeed. Obviously syntactical changes are just slog really and >>> jim> most of that is ensuring that there is enough coverage in >>> regression >>> jim> tests to exercise all the syntactic differences between T-SQL >>> and >>> jim> MySql, or rather, all the syntax of MySQL. A lot of >>> commonality but >>> jim> still a fair bit of work. What is the definitive language >>> reference >>> jim> for MySQL Marina? I can locate reference manuals and ANSI >>> specs of >>> jim> course, but if you have one particular document in mind, then >>> it would >>> jim> be useful to work off that so I can evaluate the syntactical >>> changes I >>> jim> would need to make. >>> >>> Unfortunately there is no other spec than the sql_yacc.yy file that >>> comes with the MySQL source code. >> >> I have a tool somewhere which can build .dot files from the .yy >> files in the same way how ANTLRworks draws its syntax diagrams.... >> Copies of the tool exist somewhere on the old MySQL internal >> repositories but I am sure I have a backup copy somewhere. > Yes, I added this ability to ANTLR C runtime too. You can see the > effect here: > > http://www.temporal-wave.com/index.php?option=com_psrrun&view=psrrun&Itemid… What I mean was... Feed in the sql_yacc.yy file and output a human understandable syntax diagram. > >> >>> >>> jim> >>> jim> I would like to do this in two steps: >>> jim> >>> jim> a) Replace the mysql_yacc.yy file, with as few changes as >>> possible in >>> jim> the rest of the MySQL source. >>> jim> >>> jim> >>> jim> ANTLR is essentially the same gestalt as bison: one or more >>> source files define the grammar and contain actions in C, >>> triggered at the parse points; These grammar definition files are >>> converted into C source by invoking an external program (in the >>> case of ANTLR it is a Java program) and the resulting .c/.h files >>> are compiled with the rest of the code. >>> jim> >>> jim> I wrote the C output to be machine independent. The idea >>> being that you can generate the .c files on one machine, check >>> that source code in and use it on any other machine without re- >>> generating. This can be useful when targeting systems without Java >>> machines (or ones that don't work very well). So, like many other >>> systems you can have a 'build completely from scratch' option and >>> a 'use pre-generated files option'. >>> jim> >>> jim> In the main then, I suspect that the bigger changes to the >>> MySQL base would be build changes. I say this based upon the >>> assumption that any new parser should build the same AST for a >>> query as the yacc based version does. This means the new parser >>> makes the same calls as the old one and thus the code external to >>> the parser would probably require little, if anything in the way >>> of change (but see b below). >>> >>> That sounds very good. >> >> Not possible to build the same AST as the yacc/bison based parser >> because there is no AST. >> The execution structures are directly built during the parsing phase. > Sure - for AST, you can read execution structures - there didn't > really seem to be any nomenclature but when I discussed this with > some people at Sun a while back, they used the terms AST and tree, > so I just copied them :-) So I think I would use the AST I already > build, to generate the execution structures there currently. Having > an AST means that there is a structure that can be later used for > optimization (should that be any better than optimization the > current execution structures). For a few of us, AST != execution structures .... mostly because the execution structures are very much not "abstract" and were dictated by historical and implementation issues. I have argued years ago that a good AST would allow better optimization later cheaply. For example, right now, there is a lot of work to try to simplify the Item trees for boolean expressions. This work would be better done with ASTs without wasting as much memory. > >> >>> Agree >> >> You will have to implement an all-new AST which then can be used to >> 'compile' the execution structures. >> (this was a minor sticking part of an earlier attempt to replace >> the parser - there were objections to building an AST and adding an >> additional stage to the execution) > Yeah - I just wrote that before I read this paragraph ;-) As I > already have this structure, it can be shown to pretty much zero > overhead compared to query optimization and execution time. A good > AST mostly simplifies rather than complicates things of course. > Whether my current AST needs shaping to optimally generate MySQL > remains to be seen, but it is easy to change if needed. ++Agree The only tricky issues is to try to generate exactly the same structures but in some instances later, there will be scope for actually creating simpler structures than what MySQL currently generates. > >> >>> n more generic structures. That should be easy to define. >>> >>> The MySQL code doesn't rely on bison structures at all (as far as >>> I know). >> >> AFAIK, MySQL does not use any Bison internals as such. > Yes - I could not see any reliance myself. >> A past project from many years ago, I put into MySQL a PCCTS (pre- >> ANTLR) LL parser, basically keeping MySQL's custom lexer but it >> generated Token objects instead suitable for the PCCTS generated >> parser. The project was never completed because integration of >> Stored Procedures became the objective for MySQL 5.0 instead of >> implementation of a new parser. >> >> I have asked Stewart Smith to see if he can help make public that >> old source repository as a Bazaar repository as people may be able >> to learn from it how to replace the parser without major pain. > Sure - any prior work obviously helps, but I think that the lexing > is already good. I might make changes to keyword detection to become > MySql friendly; T-SQL is a historically a bit unsure exactly when > keywords can be used. This would reduce the DFA sizes for the parser. >> >>> >>> Did you get time to do the analyse? >>> >>> jim> >>> jim> A few other things that may be of note: >>> jim> >>> jim> 1) The ANTLR generated code is free threading so long as any >>> action code that you place withing the parser is also free >>> threading. >>> >>> In other words, same as with bison >> >> The ANTLR runtime does, of course, require Java. > This, this would be my C runtime as Java is way too slow for this. > There is no Java required at runtime, only for developers to > generate the C. >> So developers who wish to extend the grammar do have to have the >> Java Runtime installed as well as the ANTLR jars. > For grammar yes as the ANTLR tool runs in Java. In practice this > isn't really a problem I think? Yeah. When I wrote "ANTLR runtime", I meant "ANTLR compiler". > >> >> I too am interested to help if at all possible. > One thing that this would need would be a good regression testing > suite. Right now I have 1300 .sql files that prove the lexer/parser/ > ast, but there would need to be more of these and runtime testing > and so on. There will have to be proper unit tests to show that a certain AST builds the correct required MySQL execution structures (Item trees, JOIN, TABLE_LIST etc etc). The best part of such an effort would be "At last! a formal specification of how the execution structures should look like". This would be exciting and cool if we can (at last) have a sane parser... especially as LL parsers always generate much nicer error messages for the users. Regards, Antony.

1 0

Re: [Maria-developers] SQL Parser
by Michael Widenius 09 May '09

09 May '09

Hi! I fixed the maria-developers address so that everyone else can follow and participate in the discussion. jim> jim> Monty, jim> jim> jim> jim> My T-SQL parser is now complete and a number of people are using jim> jim> the Java, C# and C version. Would you like to have a look at it? jim> jim> Monty> Yes, I would be intrested to know more about this! jim> jim> OK - I have my demo online now. If you visit http://www.temporal-wave.com jim> jim> and take the Online Demos->T-SQL menu option you can enter T-SQL jim> batches and see the parse results. Over the next day or so, I am jim> adding demos of my C# and VB.Net parsers, so if the site goes up and jim> down then just wait a few minutes and try again :-) Once I have jim> evaluated some of the things you mention below then I will let you jim> have a scan at the source code. ok. I will take a look as soon as I have got through the emails that has batched up during the last 3 weeks. jim> jim> I think that I could steal a lot of common syntax from this for a jim> new MySQL parser. There are 1100 regression tests culled from the jim> T-SQL language spec and user examples; on my Linux system the parse jim> time, including building and walking the AST is: jim> jim> jim> jim> jimi(50_64)-clean: wc -l *.sql jim> jim> ... jim> jim> 12923 total jim> jim> jim> jim> jimi(50_64)-clean: time tsqlc . >out jim> jim> 0.15user 0.19system 0:00.34elapsed 100%CPU (0avgtext+0avgdata jim> 0maxresident)k jim> jim> 0inputs+232outputs (0major+102772minor)pagefaults 0swaps jim> jim> jim> Which includes the time taken to read each of the 1100 files from jim> the file system and to print a message for each saying the parse was jim> successful of course. jim> jim> Anyway, if you want to take a look at the code, just let me know :-) jim> jim> Monty> The big question is how much work it would be to jim> replace the MySQLparser with your T-SQL parser. jim> Indeed. Obviously syntactical changes are just slog really and jim> most of that is ensuring that there is enough coverage in regression jim> tests to exercise all the syntactic differences between T-SQL and jim> MySql, or rather, all the syntax of MySQL. A lot of commonality but jim> still a fair bit of work. What is the definitive language reference jim> for MySQL Marina? I can locate reference manuals and ANSI specs of jim> course, but if you have one particular document in mind, then it would jim> be useful to work off that so I can evaluate the syntactical changes I jim> would need to make. Unfortunately there is no other spec than the sql_yacc.yy file that comes with the MySQL source code. jim> jim> I would like to do this in two steps: jim> jim> a) Replace the mysql_yacc.yy file, with as few changes as possible in jim> the rest of the MySQL source. jim> jim> jim> ANTLR is essentially the same gestalt as bison: one or more source files define the grammar and contain actions in C, triggered at the parse points; These grammar definition files are converted into C source by invoking an external program (in the case of ANTLR it is a Java program) and the resulting .c/.h files are compiled with the rest of the code. jim> jim> I wrote the C output to be machine independent. The idea being that you can generate the .c files on one machine, check that source code in and use it on any other machine without re-generating. This can be useful when targeting systems without Java machines (or ones that don't work very well). So, like many other systems you can have a 'build completely from scratch' option and a 'use pre-generated files option'. jim> jim> In the main then, I suspect that the bigger changes to the MySQL base would be build changes. I say this based upon the assumption that any new parser should build the same AST for a query as the yacc based version does. This means the new parser makes the same calls as the old one and thus the code external to the parser would probably require little, if anything in the way of change (but see b below). That sounds very good. jim> b) Optimze MySQL to be even faster, with the new parser at a base. jim> jim> Obviously there are two aspects, being the speed of the parser itself, which I don't think will be an issue, and then what kind of transformations are done on the resulting AST for optimization purposes. If we use exactly the same AST as MySQL does now, then improving MySQL performance is the same problem domain as it is now. However, perhaps there is opportunity for a new parser to provide more information about the query that would aid optimization. Possibilities here include modifications to the AST, external information structures that current code can use when walking the AST, or even a completely new AST. The latter would probably have quite an effect on the front end of the parse->AST->execution phase of course, so I would think that the first step is to try to achieve your step a with the existing AST as output. Agree. jim> Of course maintaining and changing an ANTLR based grammar is somewhat easier than changing bison/yacc, if for no other reason than ANTLR doesn't need much in the way of code assists for correct parsing, whereas Bison quite often does. jim> The a) option is quite important to be able to do easy merges with the jim> main MySQL tree. jim> jim> I have time this week to take a look at this. Mechanically I know it is easy enough, but I need to look at the calls that are made from the current yacc/bison parser and see if there is anything in there that would be awkward. The only things I could think of would be if the existing code relies on internal structures of bison rather than more generic structures. That should be easy to define. The MySQL code doesn't rely on bison structures at all (as far as I know). Did you get time to do the analyse? jim> jim> A few other things that may be of note: jim> jim> 1) The ANTLR generated code is free threading so long as any action code that you place withing the parser is also free threading. In other words, same as with bison jim> 2) The only system calls it uses is {m|c|re}alloc/free, which the C runtime component can re-direct (say to the MySQL overrides). For completely free threading, those calls also need to be free threading and so they are the only limit to threading of course. Sounds perfect. I would *really* like to see a new parser for MariaDB. Anything that we can do to help get this done ? Regards, Monty

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2698)
by knielsen＠knielsen-hq.org 06 May '09

06 May '09

#At lp:maria 2698 knielsen(a)knielsen-hq.org 2009-05-06 We are now using Valgrind rather than purify, and have for quite some time. Consequently, rename HAVE_purify to HAVE_valgrind, and related changes. Leave some comments about purify when not clear that they apply also to Valgrind. Fix redundant IF_VALGRIND declaration. Misc. small fixes: - Fixes for pool-of-threads patch. - Fixes for push of PBXT storage engine. - mysql-test-run.pl fix. - Fix build problem in compile-pentium64-max. modified: BUILD/SETUP.sh BUILD/build_mccge.sh BUILD/compile-dist BUILD/compile-pentium64-max BUILD/compile-solaris-sparc-purify include/m_string.h include/my_global.h mysql-test/Makefile.am mysql-test/lib/mtr_unique.pm mysql-test/t/pool_of_threads.cnf mysys/mf_qsort.c mysys/my_alloc.c mysys/my_init.c mysys/my_rnd.c mysys/safemalloc.c scripts/mysql_config.pl.in scripts/mysql_config.sh sql/field_conv.cc sql/filesort.cc sql/ha_partition.cc sql/hostname.cc sql/item_timefunc.cc sql/log_event.cc sql/log_event_old.cc sql/my_decimal.h sql/mysqld.cc sql/opt_range.cc sql/opt_range.h sql/records.cc sql/rpl_rli.cc sql/rpl_rli.h sql/set_var.cc sql/slave.cc sql/sql_base.cc sql/sql_binlog.cc sql/sql_class.cc sql/sql_list.h sql/sql_load.cc sql/sql_select.cc sql/table.cc storage/archive/azio.c storage/innobase/buf/buf0buf.c storage/innobase/include/univ.i storage/innobase/srv/srv0start.c storage/maria/ha_maria.cc storage/maria/ma_blockrec.c storage/maria/ma_check.c storage/maria/ma_loghandler.c storage/maria/ma_packrec.c storage/maria/ma_page.c storage/maria/ma_pagecrc.c storage/maria/ma_search.c storage/myisam/mi_check.c storage/myisam/mi_page.c storage/myisam/mi_search.c storage/myisammrg/ha_myisammrg.cc strings/bcmp.c strings/decimal.c strings/strmake.c per-file messages: BUILD/SETUP.sh Rename purify -> valgrind. BUILD/build_mccge.sh Rename purify -> valgrind. BUILD/compile-dist Fix that PBXT was missing in source tarball after `BUILD/compile-dist && make dist` BUILD/compile-pentium64-max Fix a build problem with BUILD/compile-pentium64-max on CentOS/Fedora Core 10 amd64. On these systems, there is libz.so but no libz.a. Finding libz.so, ./configure decides to use system zlib. But since BUILD/compile-pentium64-max builds a fully static binary with -all-static, the link of mysqld fails due to missing libz.a. Fix by using bundled zlib in the build script. BUILD/compile-solaris-sparc-purify Rename purify -> valgrind. include/m_string.h Rename purify -> valgrind. include/my_global.h Rename purify -> valgrind. mysql-test/Makefile.am Fix that PBXT test suite was missing from `make dist` source tarball. mysql-test/lib/mtr_unique.pm Better fix to avoid races when chmod'ing the semaphore file. (Though using chmod 666 shared files in /tmp/ is still not a very good solution). mysql-test/t/pool_of_threads.cnf Fix that test case pool_of_threads fails if run on mysqld with no --with-libevent support. mysys/mf_qsort.c Rename purify -> valgrind. mysys/my_alloc.c Rename purify -> valgrind. mysys/my_init.c Rename purify -> valgrind. mysys/my_rnd.c Rename purify -> valgrind. mysys/safemalloc.c Rename purify -> valgrind. scripts/mysql_config.pl.in Rename purify -> valgrind. scripts/mysql_config.sh Rename purify -> valgrind. sql/field_conv.cc Rename purify -> valgrind. sql/filesort.cc Rename purify -> valgrind. sql/ha_partition.cc Rename purify -> valgrind. sql/hostname.cc Rename purify -> valgrind. sql/item_timefunc.cc Rename purify -> valgrind. sql/log_event.cc Rename purify -> valgrind. sql/log_event_old.cc Rename purify -> valgrind. sql/my_decimal.h Rename purify -> valgrind. sql/mysqld.cc Rename purify -> valgrind. Fix redundant IF_VALGRIND declaration. sql/opt_range.cc Rename purify -> valgrind. sql/opt_range.h Rename purify -> valgrind. sql/records.cc Rename purify -> valgrind. sql/rpl_rli.cc Rename purify -> valgrind. sql/rpl_rli.h Rename purify -> valgrind. sql/set_var.cc Fix missing static declaration on pool_of_threads. sql/slave.cc Rename purify -> valgrind. sql/sql_base.cc Rename purify -> valgrind. sql/sql_binlog.cc Rename purify -> valgrind. sql/sql_class.cc Rename purify -> valgrind. sql/sql_list.h Rename purify -> valgrind. sql/sql_load.cc Rename purify -> valgrind. sql/sql_select.cc Rename purify -> valgrind. sql/table.cc Rename purify -> valgrind. storage/archive/azio.c Rename purify -> valgrind. storage/innobase/buf/buf0buf.c Rename purify -> valgrind. storage/innobase/include/univ.i Rename purify -> valgrind. storage/innobase/srv/srv0start.c Rename purify -> valgrind. storage/maria/ha_maria.cc Rename purify -> valgrind. storage/maria/ma_blockrec.c Rename purify -> valgrind. storage/maria/ma_check.c Rename purify -> valgrind. storage/maria/ma_loghandler.c Rename purify -> valgrind. storage/maria/ma_packrec.c Rename purify -> valgrind. storage/maria/ma_page.c Rename purify -> valgrind. storage/maria/ma_pagecrc.c Rename purify -> valgrind. storage/maria/ma_search.c Rename purify -> valgrind. storage/myisam/mi_check.c Rename purify -> valgrind. storage/myisam/mi_page.c Rename purify -> valgrind. storage/myisam/mi_search.c Rename purify -> valgrind. storage/myisammrg/ha_myisammrg.cc Rename purify -> valgrind. strings/bcmp.c Rename purify -> valgrind. strings/decimal.c Rename purify -> valgrind. strings/strmake.c Rename purify -> valgrind. === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-03-22 12:16:09 +0000 +++ b/BUILD/SETUP.sh 2009-05-06 12:03:24 +0000 @@ -119,7 +119,7 @@ fi # Set flags for various build configurations. # Used in -valgrind builds -valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_purify " +valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_valgrind " valgrind_flags="$valgrind_flags -DMYSQL_SERVER_SUFFIX=-valgrind-max" # # Used in -debug builds === modified file 'BUILD/build_mccge.sh' --- a/BUILD/build_mccge.sh 2008-02-07 23:12:34 +0000 +++ b/BUILD/build_mccge.sh 2009-05-06 12:03:24 +0000 @@ -938,7 +938,7 @@ set_up_ccache() set_valgrind_flags() { if test "x$valgrind_flag" = "xyes" ; then - loc_valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_purify " + loc_valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_valgrind " loc_valgrind_flags="$loc_valgrind_flags -DMYSQL_SERVER_SUFFIX=-valgrind-max" compiler_flags="$compiler_flags $loc_valgrind_flags" fi === modified file 'BUILD/compile-dist' --- a/BUILD/compile-dist 2009-02-19 09:01:25 +0000 +++ b/BUILD/compile-dist 2009-05-06 12:03:24 +0000 @@ -60,8 +60,6 @@ fi # Make sure to enable all features that affect "make dist" # Remember that configure restricts the man pages to the configured features ! ./configure \ - --with-maria-storage-engine \ - --with-embedded-server \ - --with-ndbcluster -$gmake - + --with-plugins=max-no-ndb \ + --with-embedded-server +$gmake -j4 === modified file 'BUILD/compile-pentium64-max' --- a/BUILD/compile-pentium64-max 2007-04-11 12:12:00 +0000 +++ b/BUILD/compile-pentium64-max 2009-05-06 12:03:24 +0000 @@ -4,7 +4,10 @@ path=`dirname $0` . "$path/SETUP.sh" extra_flags="$pentium64_cflags $fast_cflags" -extra_configs="$pentium_configs $max_configs $static_link" +# On CentOS/Fedora Core 10 amd64, there is system libz.so but not +# libz.a, so need to use bundled zlib when building static +# binary. Hence we use --with-zlib-dir=bundled +extra_configs="$pentium_configs $max_configs $static_link --with-zlib-dir=bundled" CC="$CC --pipe" strip=yes === modified file 'BUILD/compile-solaris-sparc-purify' --- a/BUILD/compile-solaris-sparc-purify 2007-02-26 15:12:40 +0000 +++ b/BUILD/compile-solaris-sparc-purify 2009-05-06 12:03:24 +0000 @@ -37,7 +37,7 @@ make -k maintainer-clean || true path=`dirname $0` . "$path/autorun.sh" -CFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wimplicit-int -Wparentheses -Wsign-compare -Wwrite-strings -Wunused -DHAVE_purify -DEXTRA_DEBUG -O2" CXX=gcc CXXLD=g++ CXXFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Woverloaded-virtual -Wsign-promo -Wreorder -Wctor-dtor-privacy -Wnon-virtual-dtor -felide-constructors -fno-exceptions -fno-rtti -DHAVE_purify -DEXTRA_DEBUG -O2" ./configure --prefix=/usr/local/mysql --enable-assembler --with-extra-charsets=complex --enable-thread-safe-client --with-embedded-server --with-innodb $EXTRA_CONFIG_FLAGS +CFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wimplicit-int -Wparentheses -Wsign-compare -Wwrite-strings -Wunused -DHAVE_valgrind -DEXTRA_DEBUG -O2" CXX=gcc CXXLD=g++ CXXFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Woverloaded-virtual -Wsign-promo -Wreorder -Wctor-dtor-privacy -Wnon-virtual-dtor -felide-constructors -fno-exceptions -fno-rtti -DHAVE_valgrind -DEXTRA_DEBUG -O2" ./configure --prefix=/usr/local/mysql --enable-assembler --with-extra-charsets=complex --enable-thread-safe-client --with-embedded-server --with-innodb $EXTRA_CONFIG_FLAGS make -j 4 === modified file 'include/m_string.h' --- a/include/m_string.h 2008-06-26 15:10:11 +0000 +++ b/include/m_string.h 2009-05-06 12:03:24 +0000 @@ -108,7 +108,7 @@ extern const double log_10[309]; #define memcpy_fixed(A,B,C) memcpy((A),(B),(C)) #endif -#if (!defined(USE_BMOVE512) || defined(HAVE_purify)) && !defined(bmove512) +#if (!defined(USE_BMOVE512) || defined(HAVE_valgrind)) && !defined(bmove512) #define bmove512(A,B,C) memcpy(A,B,C) #endif @@ -125,14 +125,14 @@ extern void bzero(uchar * dst,size_t len #if !defined(bcmp) && !defined(HAVE_BCMP) extern size_t bcmp(const uchar *s1,const uchar *s2,size_t len); #endif -#ifdef HAVE_purify +#ifdef HAVE_valgrind extern size_t my_bcmp(const uchar *s1,const uchar *s2,size_t len); #undef bcmp #define bcmp(A,B,C) my_bcmp((A),(B),(C)) -#define bzero_if_purify(A,B) bzero(A,B) +#define bzero_if_valgrind(A,B) bzero(A,B) #else -#define bzero_if_purify(A,B) -#endif /* HAVE_purify */ +#define bzero_if_valgrind(A,B) +#endif /* HAVE_valgrind */ #if defined(_lint) || defined(FORCE_INIT_OF_VARS) #define LINT_INIT_STRUCT(var) bzero(&var, sizeof(var)) /* No uninitialize-warning */ === modified file 'include/my_global.h' --- a/include/my_global.h 2009-04-08 16:55:26 +0000 +++ b/include/my_global.h 2009-05-06 12:03:24 +0000 @@ -563,16 +563,16 @@ int __void__; #define LINT_INIT(var) #endif -#if defined(_lint) || defined(FORCE_INIT_OF_VARS) || defined(HAVE_purify) -#define PURIFY_OR_LINT_INIT(var) var=0 +#if defined(_lint) || defined(FORCE_INIT_OF_VARS) || defined(HAVE_valgrind) +#define VALGRIND_OR_LINT_INIT(var) var=0 #else -#define PURIFY_OR_LINT_INIT(var) +#define VALGRIND_OR_LINT_INIT(var) #endif -#ifdef HAVE_purify -#define IF_PURIFY(A,B) (A) +#ifdef HAVE_valgrind +#define IF_VALGRIND(A,B) (A) #else -#define IF_PURIFY(A,B) (B) +#define IF_VALGRIND(A,B) (B) #endif #if !defined(HAVE_UINT) @@ -1174,7 +1174,7 @@ typedef char bool; /* Ordinary boolean ((uint32) (uchar) (A)[0]))) #define sint4korr(A) (*((const long *) (A))) #define uint2korr(A) (*((const uint16 *) (A))) -#if defined(HAVE_purify) && !defined(_WIN32) +#if defined(HAVE_valgrind) && !defined(_WIN32) #define uint3korr(A) (uint32) (((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ (((uint32) ((uchar) (A)[2])) << 16)) @@ -1186,7 +1186,7 @@ typedef char bool; /* Ordinary boolean It means, that you have to provide enough allocated space ! */ #define uint3korr(A) (long) (*((const unsigned int *) (A)) & 0xFFFFFF) -#endif /* HAVE_purify && !_WIN32 */ +#endif /* HAVE_valgrind && !_WIN32 */ #define uint4korr(A) (*((const uint32 *) (A))) #define uint5korr(A) ((ulonglong)(((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ === modified file 'mysql-test/Makefile.am' --- a/mysql-test/Makefile.am 2009-02-13 12:25:41 +0000 +++ b/mysql-test/Makefile.am 2009-05-06 12:03:24 +0000 @@ -98,7 +98,8 @@ TEST_DIRS = t r include std_data std_dat suite/stress/include suite/stress/t suite/stress/r \ suite/ndb suite/ndb/t suite/ndb/r \ suite/rpl_ndb suite/rpl_ndb/t suite/rpl_ndb/r \ - suite/parts suite/parts/t suite/parts/r suite/parts/inc + suite/parts suite/parts/t suite/parts/r suite/parts/inc \ + suite/pbxt/t suite/pbxt/r # Used by dist-hook and install-data-local to copy all # test files into either dist or install directory === modified file 'mysql-test/lib/mtr_unique.pm' --- a/mysql-test/lib/mtr_unique.pm 2009-04-08 16:55:26 +0000 +++ b/mysql-test/lib/mtr_unique.pm 2009-05-06 12:03:24 +0000 @@ -62,14 +62,14 @@ sub mtr_get_unique_id($$) { die 'lock file is a symbolic link'; } + my $save_umask= umask(0); open SEM, ">", "$file.sem" or die "can't write to $file.sem"; - chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; if(! -e $file) { open FILE, ">", $file or die "can't create $file"; close FILE; } - chmod 0777, $file; + umask($save_umask); msg("HAVE THE LOCK"); @@ -135,8 +135,8 @@ sub mtr_release_unique_id($) { die 'lock file is a symbolic link'; } + my $save_umask= umask(0); open SEM, ">", "$file.sem" or die "can't write to $file.sem"; - chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; msg("HAVE THE LOCK"); @@ -149,7 +149,7 @@ sub mtr_release_unique_id($) { open FILE, ">", $file or die "can't create $file"; close FILE; } - chmod 0777, "$file.sem"; + umask($save_umask); open FILE, "+<", $file or die "can't open $file"; #select undef,undef,undef,0.2; seek FILE, 0, 0; === modified file 'mysql-test/t/pool_of_threads.cnf' --- a/mysql-test/t/pool_of_threads.cnf 2009-03-18 15:46:32 +0000 +++ b/mysql-test/t/pool_of_threads.cnf 2009-05-06 12:03:24 +0000 @@ -2,8 +2,8 @@ [mysqld.1] test-ignore-wrong-options -thread-handling= pool-of-threads -thread_pool_size= 2 +loose-thread-handling= pool-of-threads +loose-thread_pool_size= 2 extra-port= @mysqld.1.#extra-port extra-max-connections=1 === modified file 'mysys/mf_qsort.c' --- a/mysys/mf_qsort.c 2007-10-17 16:08:58 +0000 +++ b/mysys/mf_qsort.c 2009-05-06 12:03:24 +0000 @@ -108,7 +108,7 @@ qsort_t my_qsort(void *base_ptr, size_t low = (char*) base_ptr; high = low+ size * (count - 1); stack_ptr = stack + 1; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* The first element in the stack will be accessed for the last POP */ stack[0].low=stack[0].high=0; #endif === modified file 'mysys/my_alloc.c' --- a/mysys/my_alloc.c 2009-01-15 18:11:25 +0000 +++ b/mysys/my_alloc.c 2009-05-06 12:03:24 +0000 @@ -56,7 +56,7 @@ void init_alloc_root(MEM_ROOT *mem_root, mem_root->block_num= 4; /* We shift this with >>2 */ mem_root->first_block_usage= 0; -#if !(defined(HAVE_purify) && defined(EXTRA_DEBUG)) +#if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { if ((mem_root->free= mem_root->pre_alloc= @@ -96,7 +96,7 @@ void reset_root_defaults(MEM_ROOT *mem_r DBUG_ASSERT(alloc_root_inited(mem_root)); mem_root->block_size= block_size - ALLOC_ROOT_MIN_BLOCK_SIZE; -#if !(defined(HAVE_purify) && defined(EXTRA_DEBUG)) +#if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { size_t size= pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)); @@ -147,7 +147,7 @@ void reset_root_defaults(MEM_ROOT *mem_r void *alloc_root(MEM_ROOT *mem_root, size_t length) { -#if defined(HAVE_purify) && defined(EXTRA_DEBUG) +#if defined(HAVE_valgrind) && defined(EXTRA_DEBUG) reg1 USED_MEM *next; DBUG_ENTER("alloc_root"); DBUG_PRINT("enter",("root: 0x%lx", (long) mem_root)); === modified file 'mysys/my_init.c' --- a/mysys/my_init.c 2009-02-19 09:01:25 +0000 +++ b/mysys/my_init.c 2009-05-06 12:03:24 +0000 @@ -170,7 +170,7 @@ void my_end(int infoflag) { #ifdef HAVE_GETRUSAGE struct rusage rus; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Purify assumes that rus is uninitialized after getrusage call */ bzero((char*) &rus, sizeof(rus)); #endif === modified file 'mysys/my_rnd.c' --- a/mysys/my_rnd.c 2007-12-14 08:47:00 +0000 +++ b/mysys/my_rnd.c 2009-05-06 12:03:24 +0000 @@ -26,7 +26,7 @@ void my_rnd_init(struct my_rnd_struct *rand_st, ulong seed1, ulong seed2) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero((char*) rand_st,sizeof(*rand_st)); /* Avoid UMC varnings */ #endif rand_st->max_value= 0x3FFFFFFFL; === modified file 'mysys/safemalloc.c' --- a/mysys/safemalloc.c 2008-10-10 15:28:41 +0000 +++ b/mysys/safemalloc.c 2009-05-06 12:03:24 +0000 @@ -304,7 +304,7 @@ void _myfree(void *ptr, const char *file sf_malloc_count--; pthread_mutex_unlock(&THR_LOCK_malloc); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Mark this data as free'ed */ if (!sf_malloc_quick) bfill(ptr, irem->datasize, (pchar) FREE_VAL); === modified file 'scripts/mysql_config.pl.in' --- a/scripts/mysql_config.pl.in 2008-11-10 21:12:15 +0000 +++ b/scripts/mysql_config.pl.in 2009-05-06 12:03:24 +0000 @@ -44,7 +44,7 @@ use strict; my @exclude_cflags = qw/DDBUG_OFF DSAFEMALLOC USAFEMALLOC DSAFE_MUTEX DPEDANTIC_SAFEMALLOC DUNIV_MUST_NOT_INLINE DFORCE_INIT_OF_VARS - DEXTRA_DEBUG DHAVE_purify O O[0-9] xO[0-9] W[-A-Za-z]* + DEXTRA_DEBUG DHAVE_valgrind O O[0-9] xO[0-9] W[-A-Za-z]* Xa xstrconst xc99=none unroll2 ip mp restrict/; === modified file 'scripts/mysql_config.sh' --- a/scripts/mysql_config.sh 2008-11-10 21:12:15 +0000 +++ b/scripts/mysql_config.sh 2009-05-06 12:03:24 +0000 @@ -129,7 +129,7 @@ include="-I$pkgincludedir" # to make --cflags usable for HP C++ (aCC) for remove in DDBUG_OFF DSAFEMALLOC USAFEMALLOC DSAFE_MUTEX \ DPEDANTIC_SAFEMALLOC DUNIV_MUST_NOT_INLINE DFORCE_INIT_OF_VARS \ - DEXTRA_DEBUG DHAVE_purify O 'O[0-9]' 'xO[0-9]' 'W[-A-Za-z]*' \ + DEXTRA_DEBUG DHAVE_valgrind O 'O[0-9]' 'xO[0-9]' 'W[-A-Za-z]*' \ 'mtune=[-A-Za-z0-9]*' 'mcpu=[-A-Za-z0-9]*' 'march=[-A-Za-z0-9]*' \ Xa xstrconst "xc99=none" AC99 \ unroll2 ip mp restrict === modified file 'sql/field_conv.cc' --- a/sql/field_conv.cc 2009-01-31 21:22:44 +0000 +++ b/sql/field_conv.cc 2009-05-06 12:03:24 +0000 @@ -773,7 +773,7 @@ int field_conv(Field *to,Field *from) ((Field_varstring*)from)->length_bytes == ((Field_varstring*)to)->length_bytes)) { // Identical fields -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* This may happen if one does 'UPDATE ... SET x=x' */ if (to->ptr != from->ptr) #endif === modified file 'sql/filesort.cc' --- a/sql/filesort.cc 2009-02-19 09:01:25 +0000 +++ b/sql/filesort.cc 2009-05-06 12:03:24 +0000 @@ -953,13 +953,13 @@ static void make_sortkey(register SORTPA if (addonf->null_bit && field->is_null()) { nulls[addonf->null_offset]|= addonf->null_bit; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(to, addonf->length); #endif } else { -#ifdef HAVE_purify +#ifdef HAVE_valgrind uchar *end= field->pack(to, field->ptr); uint length= (uint) ((to + addonf->length) - end); DBUG_ASSERT((int) length >= 0); === modified file 'sql/ha_partition.cc' --- a/sql/ha_partition.cc 2009-02-19 09:01:25 +0000 +++ b/sql/ha_partition.cc 2009-05-06 12:03:24 +0000 @@ -3548,10 +3548,10 @@ void ha_partition::position(const uchar (ref_length - PARTITION_BYTES_IN_POS)); #ifdef SUPPORTING_PARTITION_OVER_DIFFERENT_ENGINES -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(ref + PARTITION_BYTES_IN_POS + ref_length, max_ref_length-ref_length); -#endif /* HAVE_purify */ +#endif /* HAVE_valgrind */ #endif DBUG_VOID_RETURN; } === modified file 'sql/hostname.cc' --- a/sql/hostname.cc 2007-10-11 17:29:09 +0000 +++ b/sql/hostname.cc 2009-05-06 12:03:24 +0000 @@ -175,7 +175,7 @@ char * ip_to_hostname(struct in_addr *in char buff[GETHOSTBYADDR_BUFF_SIZE],buff2[GETHOSTBYNAME_BUFF_SIZE]; int tmp_errno; struct hostent tmp_hostent, tmp_hostent2; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(buff,sizeof(buff)); // Bug in purify #endif if (!(hp=gethostbyaddr_r((char*) in,sizeof(*in), === modified file 'sql/item_timefunc.cc' --- a/sql/item_timefunc.cc 2009-02-15 10:58:34 +0000 +++ b/sql/item_timefunc.cc 2009-05-06 12:03:24 +0000 @@ -290,8 +290,8 @@ static bool extract_date_time(DATE_TIME_ LINT_INIT(strict_week_number); /* Remove valgrind varnings when using gcc 3.3 and -O1 */ - PURIFY_OR_LINT_INIT(strict_week_number_year_type); - PURIFY_OR_LINT_INIT(sunday_first_n_first_week_non_iso); + VALGRIND_OR_LINT_INIT(strict_week_number_year_type); + VALGRIND_OR_LINT_INIT(sunday_first_n_first_week_non_iso); if (!sub_pattern_end) bzero((char*) l_time, sizeof(*l_time)); === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2009-02-19 09:01:25 +0000 +++ b/sql/log_event.cc 2009-05-06 12:03:24 +0000 @@ -6990,7 +6990,7 @@ int Rows_log_event::do_add_row_data(ucha Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("row_data", row_data, min(length, 32)); #endif @@ -7771,7 +7771,7 @@ Table_map_log_event::Table_map_log_event Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("event buffer", (uchar*) buf, event_len); #endif @@ -8674,7 +8674,7 @@ int Rows_log_event::find_row(const Relay Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("key data", m_key, table->key_info->key_length); #endif @@ -8704,7 +8704,7 @@ int Rows_log_event::find_row(const Relay Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_PRINT("info",("found first matching record")); DBUG_DUMP("record[0]", table->record[0], table->s->reclength); #endif @@ -9067,7 +9067,7 @@ Update_rows_log_event::do_exec_row(const Now we have the right row to update. The old row (the one we're looking for) is in record[1] and the new row is in record[0]. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Don't print debug messages when running valgrind since they can trigger false warnings. === modified file 'sql/log_event_old.cc' --- a/sql/log_event_old.cc 2009-01-31 21:22:44 +0000 +++ b/sql/log_event_old.cc 2009-05-06 12:03:24 +0000 @@ -724,7 +724,7 @@ static int find_and_fetch_row(TABLE *tab Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("table->record[0]", table->record[0], table->s->reclength); DBUG_DUMP("table->record[1]", table->record[1], table->s->reclength); #endif @@ -750,7 +750,7 @@ static int find_and_fetch_row(TABLE *tab Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("table->record[0]", table->record[0], table->s->reclength); DBUG_DUMP("table->record[1]", table->record[1], table->s->reclength); #endif @@ -1404,7 +1404,7 @@ int Old_rows_log_event::do_add_row_data( Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("row_data", row_data, min(length, 32)); #endif @@ -2353,7 +2353,7 @@ int Old_rows_log_event::find_row(const R Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("key data", m_key, table->key_info->key_length); #endif @@ -2383,7 +2383,7 @@ int Old_rows_log_event::find_row(const R Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_PRINT("info",("found first matching record")); DBUG_DUMP("record[0]", table->record[0], table->s->reclength); #endif @@ -2878,7 +2878,7 @@ Update_rows_log_event_old::do_exec_row(c Now we have the right row to update. The old row (the one we're looking for) is in record[1] and the new row is in record[0]. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Don't print debug messages when running valgrind since they can trigger false warnings. === modified file 'sql/my_decimal.h' --- a/sql/my_decimal.h 2008-05-20 07:38:17 +0000 +++ b/sql/my_decimal.h 2009-05-06 12:03:24 +0000 @@ -99,7 +99,7 @@ public: { len= DECIMAL_BUFF_LENGTH; buf= buffer; -#if !defined (HAVE_purify) && !defined(DBUG_OFF) +#if !defined (HAVE_valgrind) && !defined(DBUG_OFF) /* Set buffer to 'random' value to find wrong buffer usage */ for (uint i= 0; i < DECIMAL_BUFF_LENGTH; i++) buffer[i]= i; === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-04-08 21:16:10 +0000 +++ b/sql/mysqld.cc 2009-05-06 12:03:24 +0000 @@ -57,12 +57,6 @@ #define mysqld_charset &my_charset_latin1 -#ifdef HAVE_purify -#define IF_PURIFY(A,B) (A) -#else -#define IF_PURIFY(A,B) (B) -#endif - #if SIZEOF_CHARP == 4 #define MAX_MEM_TABLE_SIZE ~(ulong) 0 #else @@ -76,9 +70,9 @@ char pstack_file_name[80]; #endif /* __linux__ */ -/* We have HAVE_purify below as this speeds up the shutdown of MySQL */ +/* We have HAVE_valgrind below as this speeds up the shutdown of MySQL */ -#if defined(HAVE_DEC_3_2_THREADS) || defined(SIGNALS_DONT_BREAK_READ) || defined(HAVE_purify) && defined(__linux__) +#if defined(HAVE_DEC_3_2_THREADS) || defined(SIGNALS_DONT_BREAK_READ) || defined(HAVE_valgrind) && defined(__linux__) #define HAVE_CLOSE_SERVER_SOCK 1 #endif @@ -6520,7 +6514,7 @@ log and this option does nothing anymore purify. These are not suppressed: instead we disable symlinks option if compiled with valgrind support. */ - IF_PURIFY(0,1), 0, 0, 0, 0, 0}, + IF_VALGRIND(0,1), 0, 0, 0, 0, 0}, {"sysdate-is-now", OPT_SYSDATE_IS_NOW, "Non-default option to alias SYSDATE() to NOW() to make it safe-replicable. Since 5.0, SYSDATE() returns a `dynamic' value different for different invocations, even within the same statement.", (uchar**) &global_system_variables.sysdate_is_now, @@ -6561,7 +6555,7 @@ log and this option does nothing anymore 0, 0, 0, 0, 0}, {"use-symbolic-links", 's', "Enable symbolic link support. Deprecated option; use --symbolic-links instead.", (uchar**) &my_use_symdir, (uchar**) &my_use_symdir, 0, GET_BOOL, NO_ARG, - IF_PURIFY(0,1), 0, 0, 0, 0, 0}, + IF_VALGRIND(0,1), 0, 0, 0, 0, 0}, {"user", 'u', "Run mysqld daemon as user.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Used with --help option for detailed help", @@ -7661,7 +7655,7 @@ static int mysql_init_variables(void) /* Things reset to zero */ opt_skip_slave_start= opt_reckless_slave = 0; mysql_home[0]= pidfile_name[0]= log_error_file[0]= 0; -#if defined(HAVE_REALPATH) && !defined(HAVE_purify) && !defined(HAVE_BROKEN_REALPATH) +#if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) /* We can only test for sub paths if my_symlink.c is using realpath */ myisam_test_invalid_symlink= test_if_data_home_dir; #endif === modified file 'sql/opt_range.cc' --- a/sql/opt_range.cc 2009-02-19 09:01:25 +0000 +++ b/sql/opt_range.cc 2009-05-06 12:03:24 +0000 @@ -8347,7 +8347,7 @@ int QUICK_RANGE_SELECT::reset() multi_range_buff->buffer= mrange_buff; multi_range_buff->buffer_end= mrange_buff + mrange_bufsiz; multi_range_buff->end_of_used_area= mrange_buff; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* We need this until ndb will use the buffer efficiently (Now ndb stores complete row in here, instead of only the used fields === modified file 'sql/opt_range.h' --- a/sql/opt_range.h 2008-08-25 17:18:22 +0000 +++ b/sql/opt_range.h 2009-05-06 12:03:24 +0000 @@ -44,7 +44,7 @@ class QUICK_RANGE :public Sql_alloc { uint16 min_length,max_length,flag; key_part_map min_keypart_map, // bitmap of used keyparts in min_key max_keypart_map; // bitmap of used keyparts in max_key -#ifdef HAVE_purify +#ifdef HAVE_valgrind uint16 dummy; /* Avoid warnings on 'flag' */ #endif QUICK_RANGE(); /* Full range */ @@ -61,7 +61,7 @@ class QUICK_RANGE :public Sql_alloc { min_keypart_map(min_keypart_map_arg), max_keypart_map(max_keypart_map_arg) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind dummy=0; #endif } === modified file 'sql/records.cc' --- a/sql/records.cc 2008-07-17 18:26:55 +0000 +++ b/sql/records.cc 2009-05-06 12:03:24 +0000 @@ -520,7 +520,7 @@ static int init_rr_cache(THD *thd, READ_ info->struct_length+1, MYF(0)))) DBUG_RETURN(1); -#ifdef HAVE_purify +#ifdef HAVE_valgrind // Avoid warnings in qsort bzero(info->cache,rec_cache_size+info->cache_records* info->struct_length+1); #endif === modified file 'sql/rpl_rli.cc' --- a/sql/rpl_rli.cc 2009-02-19 09:01:25 +0000 +++ b/sql/rpl_rli.cc 2009-05-06 12:03:24 +0000 @@ -34,7 +34,7 @@ Relay_log_info::Relay_log_info() no_storage(FALSE), replicate_same_server_id(::replicate_same_server_id), info_fd(-1), cur_log_fd(-1), save_temporary_tables(0), cur_log_old_open_count(0), group_relay_log_pos(0), event_relay_log_pos(0), -#if HAVE_purify +#if HAVE_valgrind is_fake(FALSE), #endif group_master_log_pos(0), log_space_total(0), ignore_log_space_limit(0), === modified file 'sql/rpl_rli.h' --- a/sql/rpl_rli.h 2008-02-27 17:46:06 +0000 +++ b/sql/rpl_rli.h 2009-05-06 12:03:24 +0000 @@ -154,7 +154,7 @@ public: ulonglong event_relay_log_pos; ulonglong future_event_relay_log_pos; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bool is_fake; /* Mark that this is a fake relay log info structure */ #endif === modified file 'sql/set_var.cc' --- a/sql/set_var.cc 2009-04-08 16:55:26 +0000 +++ b/sql/set_var.cc 2009-05-06 12:03:24 +0000 @@ -642,8 +642,8 @@ static sys_var_long_ptr sys_table_lock_w static sys_var_long_ptr sys_thread_cache_size(&vars, "thread_cache_size", &thread_cache_size); #if HAVE_POOL_OF_THREADS == 1 -sys_var_long_ptr sys_thread_pool_size(&vars, "thread_pool_size", - &thread_pool_size); +static sys_var_long_ptr sys_thread_pool_size(&vars, "thread_pool_size", + &thread_pool_size); #endif static sys_var_thd_enum sys_tx_isolation(&vars, "tx_isolation", &SV::tx_isolation, === modified file 'sql/slave.cc' --- a/sql/slave.cc 2009-02-19 09:01:25 +0000 +++ b/sql/slave.cc 2009-05-06 12:03:24 +0000 @@ -2014,7 +2014,7 @@ int apply_event_and_update_pos(Log_event if (exec_res == 0) { int error= ev->update_pos(rli); -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (!rli->is_fake) #endif { === modified file 'sql/sql_base.cc' --- a/sql/sql_base.cc 2009-02-19 09:01:25 +0000 +++ b/sql/sql_base.cc 2009-05-06 12:03:24 +0000 @@ -3000,7 +3000,7 @@ TABLE *open_table(THD *thd, TABLE_LIST * table->pos_in_table_list= table_list; table_list->updatable= 1; // It is not derived table nor non-updatable VIEW table->clear_column_bitmaps(); -#if !defined(DBUG_OFF) && !defined(HAVE_purify) +#if !defined(DBUG_OFF) && !defined(HAVE_valgrind) /* Fill record with random values to find bugs where we access fields without first reading them. @@ -7363,7 +7363,7 @@ int setup_wild(THD *thd, TABLE_LIST *tab /* make * substituting permanent */ SELECT_LEX *select_lex= thd->lex->current_select; select_lex->with_wild= 0; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (&select_lex->item_list != &fields) // Avoid warning #endif select_lex->item_list= fields; === modified file 'sql/sql_binlog.cc' --- a/sql/sql_binlog.cc 2009-01-09 12:49:24 +0000 +++ b/sql/sql_binlog.cc 2009-05-06 12:03:24 +0000 @@ -59,7 +59,7 @@ void mysql_client_binlog_statement(THD* if (!thd->rli_fake) { thd->rli_fake= new Relay_log_info; -#ifdef HAVE_purify +#ifdef HAVE_valgrind thd->rli_fake->is_fake= TRUE; #endif have_fd_event= FALSE; @@ -95,7 +95,7 @@ void mysql_client_binlog_statement(THD* char const *endptr= 0; int bytes_decoded= base64_decode(strptr, coded_len, buf, &endptr); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* This debug printout should not be used for valgrind builds since it will read from unassigned memory. @@ -188,7 +188,7 @@ void mysql_client_binlog_statement(THD* bufptr += event_len; DBUG_PRINT("info",("ev->get_type_code()=%d", ev->get_type_code())); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* This debug printout should not be used for valgrind builds since it will read from unassigned memory. === modified file 'sql/sql_class.cc' --- a/sql/sql_class.cc 2009-03-12 22:27:35 +0000 +++ b/sql/sql_class.cc 2009-05-06 12:03:24 +0000 @@ -3480,7 +3480,7 @@ int THD::binlog_update_row(TABLE* table, Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("before_record", before_record, table->s->reclength); DBUG_DUMP("after_record", after_record, table->s->reclength); DBUG_DUMP("before_row", before_row, before_size); === modified file 'sql/sql_list.h' --- a/sql/sql_list.h 2009-01-15 18:11:25 +0000 +++ b/sql/sql_list.h 2009-05-06 12:03:24 +0000 @@ -43,7 +43,7 @@ public: static void operator delete[](void *ptr, MEM_ROOT *mem_root) { /* never called */ } static void operator delete[](void *ptr, size_t size) { TRASH(ptr, size); } -#ifdef HAVE_purify +#ifdef HAVE_valgrind bool dummy; inline Sql_alloc() :dummy(0) {} inline ~Sql_alloc() {} === modified file 'sql/sql_load.cc' --- a/sql/sql_load.cc 2009-01-31 21:22:44 +0000 +++ b/sql/sql_load.cc 2009-05-06 12:03:24 +0000 @@ -581,7 +581,7 @@ read_fixed_length(THD *thd, COPY_INFO &i } it.rewind(); uchar *pos=read_info.row_start; -#ifdef HAVE_purify +#ifdef HAVE_valgrind read_info.row_end[0]=0; #endif === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-04-08 16:55:26 +0000 +++ b/sql/sql_select.cc 2009-05-06 12:03:24 +0000 @@ -1818,7 +1818,7 @@ JOIN::exec() curr_join->having= curr_join->tmp_having= 0; // Allready done /* Change sum_fields reference to calculated fields in tmp_table */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif curr_join->all_fields= *curr_all_fields; @@ -1840,7 +1840,7 @@ JOIN::exec() fields_list.elements, all_fields)) DBUG_VOID_RETURN; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif { @@ -1994,7 +1994,7 @@ JOIN::exec() tmp_fields_list2, tmp_all_fields2, fields_list.elements, tmp_all_fields1)) DBUG_VOID_RETURN; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Some GCCs use memcpy() for struct assignment, even for x=x. GCC bug 19410: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=19410 @@ -2059,7 +2059,7 @@ JOIN::exec() tmp_table_param.save_copy_field= curr_join->tmp_table_param.copy_field; tmp_table_param.save_copy_field_end= curr_join->tmp_table_param.copy_field_end; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif { @@ -3848,7 +3848,7 @@ update_ref_and_keys(THD *thd, DYNAMIC_AR continue; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Valgrind complains about overlapped memcpy when save_pos==use. */ if (save_pos != use) #endif @@ -15138,7 +15138,7 @@ setup_copy_fields(THD *thd, TMP_TABLE_PA DBUG_ASSERT (param->field_count > (uint) (copy - copy_start)); copy->set(tmp, item->result_field); item->result_field->move_field(copy->to_ptr,copy->to_null_ptr,1); -#ifdef HAVE_purify +#ifdef HAVE_valgrind copy->to_ptr[copy->from_length]= 0; #endif copy++; === modified file 'sql/table.cc' --- a/sql/table.cc 2009-02-19 09:01:25 +0000 +++ b/sql/table.cc 2009-05-06 12:03:24 +0000 @@ -1694,7 +1694,7 @@ int open_table_from_share(THD *thd, TABL outparam->record[1]= outparam->record[0]; // Safety } -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* We need this because when we read var-length rows, we are not updating bytes after end of varchar @@ -1916,7 +1916,7 @@ partititon_err: } } -#if defined(HAVE_purify) && !defined(DBUG_OFF) +#if defined(HAVE_valgrind) && !defined(DBUG_OFF) bzero((char*) bitmaps, bitmap_size*3); #endif === modified file 'storage/archive/azio.c' --- a/storage/archive/azio.c 2009-04-08 16:55:26 +0000 +++ b/storage/archive/azio.c 2009-05-06 12:03:24 +0000 @@ -63,7 +63,7 @@ void read_header(azio_stream *s, unsigne */ static void *az_allocator(void *dummy, uInt items, uInt size) { - return my_malloc((size_t)items*(size_t)size, IF_PURIFY(MY_ZEROFILL, MYF(0))); + return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } static void az_free(void *dummy, void *address) === modified file 'storage/innobase/buf/buf0buf.c' --- a/storage/innobase/buf/buf0buf.c 2008-08-20 00:37:41 +0000 +++ b/storage/innobase/buf/buf0buf.c 2009-05-06 12:03:24 +0000 @@ -756,7 +756,7 @@ buf_pool_init( /* Wipe contents of frame to eliminate a Purify warning */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind memset(block->frame, '\0', UNIV_PAGE_SIZE); #endif if (srv_use_awe) { === modified file 'storage/innobase/include/univ.i' --- a/storage/innobase/include/univ.i 2008-03-27 01:40:45 +0000 +++ b/storage/innobase/include/univ.i 2009-05-06 12:03:24 +0000 @@ -107,7 +107,7 @@ operations (very slow); also UNIV_DEBUG #define UNIV_BTR_DEBUG /* check B-tree links */ #define UNIV_LIGHT_MEM_DEBUG /* light memory debugging */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* The following sets all new allocated memory to zero before use: this can be used to eliminate unnecessary Purify warnings, but note that it also masks many bugs Purify could detect. For detailed Purify analysis it === modified file 'storage/innobase/srv/srv0start.c' --- a/storage/innobase/srv/srv0start.c 2008-12-14 19:21:24 +0000 +++ b/storage/innobase/srv/srv0start.c 2009-05-06 12:03:24 +0000 @@ -104,7 +104,7 @@ static char* srv_monitor_file_name; /* Avoid warnings when using purify */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind static int inno_bcmp(register const char *s1, register const char *s2, register uint len) { === modified file 'storage/maria/ha_maria.cc' --- a/storage/maria/ha_maria.cc 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ha_maria.cc 2009-05-06 12:03:24 +0000 @@ -3166,7 +3166,7 @@ static int ha_maria_init(void *p) ma_checkpoint_init(checkpoint_interval); maria_multi_threaded= maria_in_ha_maria= TRUE; -#if defined(HAVE_REALPATH) && !defined(HAVE_purify) && !defined(HAVE_BROKEN_REALPATH) +#if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) /* We can only test for sub paths if my_symlink.c is using realpath */ maria_test_invalid_symlink= test_if_data_home_dir; #endif === modified file 'storage/maria/ma_blockrec.c' --- a/storage/maria/ma_blockrec.c 2009-04-08 08:47:30 +0000 +++ b/storage/maria/ma_blockrec.c 2009-05-06 12:03:24 +0000 @@ -1594,7 +1594,7 @@ static void make_empty_page(MARIA_HA *in bzero(buff, PAGE_HEADER_SIZE); -#if !defined(DONT_ZERO_PAGE_BLOCKS) || defined(HAVE_purify) +#if !defined(DONT_ZERO_PAGE_BLOCKS) || defined(HAVE_valgrind) /* We zero the rest of the block to avoid getting old memory information to disk and to allow the file to be compressed better if archived. === modified file 'storage/maria/ma_check.c' --- a/storage/maria/ma_check.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_check.c 2009-05-06 12:03:24 +0000 @@ -4522,7 +4522,7 @@ static int sort_key_read(MARIA_SORT_PARA sort_param->current_filepos, 0); sort_param->real_key_length= int_key.data_length + int_key.ref_length; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); #endif @@ -4567,7 +4567,7 @@ static int sort_maria_ft_key_read(MARIA_ sort_param->current_filepos); sort_param->real_key_length= int_key.data_length + int_key.ref_length; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (sort_param->key_length > sort_param->real_key_length) bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); @@ -5104,7 +5104,7 @@ static int sort_get_next_record(MARIA_SO llstr(sort_param->pos,llbuff)); continue; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(sort_param->rec_buff + block_info.rec_len, share->base.extra_rec_buff_size); #endif === modified file 'storage/maria/ma_loghandler.c' --- a/storage/maria/ma_loghandler.c 2009-03-17 19:05:01 +0000 +++ b/storage/maria/ma_loghandler.c 2009-05-06 12:03:24 +0000 @@ -6194,12 +6194,12 @@ my_bool translog_write_record(LSN *lsn, { uint i; uint len= 0; -#ifdef HAVE_purify +#ifdef HAVE_valgrind ha_checksum checksum= 0; #endif for (i= TRANSLOG_INTERNAL_PARTS; i < part_no; i++) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Find unitialized bytes early */ checksum+= my_checksum(checksum, parts_data[i].str, parts_data[i].length); === modified file 'storage/maria/ma_packrec.c' --- a/storage/maria/ma_packrec.c 2009-01-09 04:23:25 +0000 +++ b/storage/maria/ma_packrec.c 2009-05-06 12:03:24 +0000 @@ -256,7 +256,7 @@ static my_bool _ma_read_pack_info(MARIA_ (uint) (share->pack.header_length-sizeof(header)), MYF(MY_NABP))) goto err2; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Zero bytes accessed by fill_buffer */ bzero(disk_cache + (share->pack.header_length-sizeof(header)), share->base.extra_rec_buff_size); === modified file 'storage/maria/ma_page.c' --- a/storage/maria/ma_page.c 2008-09-01 17:31:40 +0000 +++ b/storage/maria/ma_page.c 2009-05-06 12:03:24 +0000 @@ -200,7 +200,7 @@ my_bool _ma_write_keypage(MARIA_PAGE *pa /* Verify that keynr is correct */ DBUG_ASSERT(_ma_get_keynr(share, buff) == page->keyinfo->key_nr); -#if defined(EXTRA_DEBUG) && defined(HAVE_purify) && defined(NOT_ANYMORE) +#if defined(EXTRA_DEBUG) && defined(HAVE_valgrind) && defined(NOT_ANYMORE) { /* This is here to catch uninitialized bytes */ uint length= page->size; === modified file 'storage/maria/ma_pagecrc.c' --- a/storage/maria/ma_pagecrc.c 2008-03-04 11:58:21 +0000 +++ b/storage/maria/ma_pagecrc.c 2009-05-06 12:03:24 +0000 @@ -318,7 +318,7 @@ my_bool maria_page_filler_set_none(uchar __attribute__((unused)), uchar *data_ptr __attribute__((unused))) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind int4store_aligned(page + ((MARIA_SHARE *)data_ptr)->block_size - CRC_SIZE, 0); #endif === modified file 'storage/maria/ma_search.c' --- a/storage/maria/ma_search.c 2009-01-12 11:12:00 +0000 +++ b/storage/maria/ma_search.c 2009-05-06 12:03:24 +0000 @@ -2172,7 +2172,7 @@ int _ma_calc_bin_pack_key_length(const M s_temp->totlength= key_length= (int_key->data_length + int_key->ref_length+ nod_flag); -#ifdef HAVE_purify +#ifdef HAVE_valgrind s_temp->n_length= s_temp->n_ref_length=0; /* For valgrind */ #endif s_temp->key=key; === modified file 'storage/myisam/mi_check.c' --- a/storage/myisam/mi_check.c 2009-02-19 09:01:25 +0000 +++ b/storage/myisam/mi_check.c 2009-05-06 12:03:24 +0000 @@ -3147,7 +3147,7 @@ static int sort_key_read(MI_SORT_PARAM * (info->s->rec_reflength+ _mi_make_key(info, sort_param->key, (uchar*) key, sort_param->record, sort_param->filepos)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); #endif @@ -3187,7 +3187,7 @@ static int sort_ft_key_read(MI_SORT_PARA sort_param->real_key_length=(info->s->rec_reflength+ _ft_make_key(info, sort_param->key, key, wptr++, sort_param->filepos)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (sort_param->key_length > sort_param->real_key_length) bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); === modified file 'storage/myisam/mi_page.c' --- a/storage/myisam/mi_page.c 2008-08-28 12:43:44 +0000 +++ b/storage/myisam/mi_page.c 2009-05-06 12:03:24 +0000 @@ -86,7 +86,7 @@ int _mi_write_keypage(register MI_INFO * if ((length=keyinfo->block_length) > IO_SIZE*2 && info->state->key_file_length != page+length) length= ((mi_getint(buff)+IO_SIZE-1) & (uint) ~(IO_SIZE-1)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind { length=mi_getint(buff); bzero((uchar*) buff+length,keyinfo->block_length-length); === modified file 'storage/myisam/mi_search.c' --- a/storage/myisam/mi_search.c 2008-08-28 18:52:23 +0000 +++ b/storage/myisam/mi_search.c 2009-05-06 12:03:24 +0000 @@ -1749,7 +1749,7 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF * uint length,key_length,ref_length; s_temp->totlength=key_length=_mi_keylength(keyinfo,key)+nod_flag; -#ifdef HAVE_purify +#ifdef HAVE_valgrind s_temp->n_length= s_temp->n_ref_length=0; /* For valgrind */ #endif s_temp->key=key; === modified file 'storage/myisammrg/ha_myisammrg.cc' --- a/storage/myisammrg/ha_myisammrg.cc 2008-12-04 00:36:55 +0000 +++ b/storage/myisammrg/ha_myisammrg.cc 2009-05-06 12:03:24 +0000 @@ -837,7 +837,7 @@ int ha_myisammrg::info(uint flag) { if (table->s->key_parts && mrg_info.rec_per_key) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* valgrind may be unhappy about it, because optimizer may access values between file->keys and table->key_parts, that will be uninitialized. === modified file 'strings/bcmp.c' --- a/strings/bcmp.c 2007-05-10 09:59:39 +0000 +++ b/strings/bcmp.c 2009-05-06 12:03:24 +0000 @@ -24,7 +24,7 @@ #include <my_global.h> #include "m_string.h" -#ifdef HAVE_purify +#ifdef HAVE_valgrind #undef bcmp #undef HAVE_BCMP #endif @@ -50,7 +50,7 @@ uint len; /* 0 <= len <= 65535 */ #else -#ifndef HAVE_purify +#ifndef HAVE_valgrind size_t bcmp(register const uchar *s1,register const uchar *s2, register size_t len) #else === modified file 'strings/decimal.c' --- a/strings/decimal.c 2008-08-15 19:49:43 +0000 +++ b/strings/decimal.c 2009-05-06 12:03:24 +0000 @@ -144,7 +144,7 @@ static double scaler1[]= { 1.0, 10.0, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9 }; -#ifdef HAVE_purify +#ifdef HAVE_valgrind #define sanity(d) DBUG_ASSERT((d)->len > 0) #else #define sanity(d) DBUG_ASSERT((d)->len >0 && ((d)->buf[0] | \ === modified file 'strings/strmake.c' --- a/strings/strmake.c 2008-04-28 16:24:05 +0000 +++ b/strings/strmake.c 2009-05-06 12:03:24 +0000 @@ -51,11 +51,11 @@ char *strmake(register char *dst, regist */ if (length) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind dst[length-1]= 'Z'; #else bfill(dst, length-1, (int) 'Z'); -#endif /* HAVE_purify */ +#endif /* HAVE_valgrind */ } #endif /* EXTRA_DEBUG */ return dst-1;

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2699)
by knielsen＠knielsen-hq.org 06 May '09

06 May '09

#At lp:maria 2699 knielsen(a)knielsen-hq.org 2009-04-27 We are now using Valgrind rather than purify, and have for quite some time. Consequently, rename HAVE_purify to HAVE_valgrind, and related changes. Leave some comments about purify when not clear that they apply also to Valgrind. Fix redundant IF_VALGRIND declaration. modified: BUILD/SETUP.sh BUILD/build_mccge.sh BUILD/compile-solaris-sparc-purify include/m_string.h include/my_global.h mysys/mf_qsort.c mysys/my_alloc.c mysys/my_init.c mysys/my_rnd.c mysys/safemalloc.c scripts/mysql_config.pl.in scripts/mysql_config.sh sql/field_conv.cc sql/filesort.cc sql/ha_partition.cc sql/hostname.cc sql/item_timefunc.cc sql/log_event.cc sql/log_event_old.cc sql/my_decimal.h sql/mysqld.cc sql/opt_range.cc sql/opt_range.h sql/records.cc sql/rpl_rli.cc sql/rpl_rli.h sql/slave.cc sql/sql_base.cc sql/sql_binlog.cc sql/sql_class.cc sql/sql_list.h sql/sql_load.cc sql/sql_select.cc sql/table.cc storage/archive/azio.c storage/innobase/buf/buf0buf.c storage/innobase/include/univ.i storage/innobase/srv/srv0start.c storage/maria/ha_maria.cc storage/maria/ma_blockrec.c storage/maria/ma_check.c storage/maria/ma_loghandler.c storage/maria/ma_packrec.c storage/maria/ma_page.c storage/maria/ma_pagecrc.c storage/maria/ma_search.c storage/myisam/mi_check.c storage/myisam/mi_page.c storage/myisam/mi_search.c storage/myisammrg/ha_myisammrg.cc strings/bcmp.c strings/decimal.c strings/strmake.c === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-03-22 12:16:09 +0000 +++ b/BUILD/SETUP.sh 2009-04-27 18:01:20 +0000 @@ -119,7 +119,7 @@ fi # Set flags for various build configurations. # Used in -valgrind builds -valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_purify " +valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_valgrind " valgrind_flags="$valgrind_flags -DMYSQL_SERVER_SUFFIX=-valgrind-max" # # Used in -debug builds === modified file 'BUILD/build_mccge.sh' --- a/BUILD/build_mccge.sh 2008-02-07 23:12:34 +0000 +++ b/BUILD/build_mccge.sh 2009-04-27 18:01:20 +0000 @@ -938,7 +938,7 @@ set_up_ccache() set_valgrind_flags() { if test "x$valgrind_flag" = "xyes" ; then - loc_valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_purify " + loc_valgrind_flags="-USAFEMALLOC -UFORCE_INIT_OF_VARS -DHAVE_valgrind " loc_valgrind_flags="$loc_valgrind_flags -DMYSQL_SERVER_SUFFIX=-valgrind-max" compiler_flags="$compiler_flags $loc_valgrind_flags" fi === modified file 'BUILD/compile-solaris-sparc-purify' --- a/BUILD/compile-solaris-sparc-purify 2007-02-26 15:12:40 +0000 +++ b/BUILD/compile-solaris-sparc-purify 2009-04-27 18:01:20 +0000 @@ -37,7 +37,7 @@ make -k maintainer-clean || true path=`dirname $0` . "$path/autorun.sh" -CFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wimplicit-int -Wparentheses -Wsign-compare -Wwrite-strings -Wunused -DHAVE_purify -DEXTRA_DEBUG -O2" CXX=gcc CXXLD=g++ CXXFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Woverloaded-virtual -Wsign-promo -Wreorder -Wctor-dtor-privacy -Wnon-virtual-dtor -felide-constructors -fno-exceptions -fno-rtti -DHAVE_purify -DEXTRA_DEBUG -O2" ./configure --prefix=/usr/local/mysql --enable-assembler --with-extra-charsets=complex --enable-thread-safe-client --with-embedded-server --with-innodb $EXTRA_CONFIG_FLAGS +CFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wimplicit-int -Wparentheses -Wsign-compare -Wwrite-strings -Wunused -DHAVE_valgrind -DEXTRA_DEBUG -O2" CXX=gcc CXXLD=g++ CXXFLAGS="-g -Wimplicit -Wreturn-type -Wswitch -Wtrigraphs -Wcomment -W -Wchar-subscripts -Wformat -Wparentheses -Wsign-compare -Wwrite-strings -Woverloaded-virtual -Wsign-promo -Wreorder -Wctor-dtor-privacy -Wnon-virtual-dtor -felide-constructors -fno-exceptions -fno-rtti -DHAVE_valgrind -DEXTRA_DEBUG -O2" ./configure --prefix=/usr/local/mysql --enable-assembler --with-extra-charsets=complex --enable-thread-safe-client --with-embedded-server --with-innodb $EXTRA_CONFIG_FLAGS make -j 4 === modified file 'include/m_string.h' --- a/include/m_string.h 2008-06-26 15:10:11 +0000 +++ b/include/m_string.h 2009-04-27 18:01:20 +0000 @@ -108,7 +108,7 @@ extern const double log_10[309]; #define memcpy_fixed(A,B,C) memcpy((A),(B),(C)) #endif -#if (!defined(USE_BMOVE512) || defined(HAVE_purify)) && !defined(bmove512) +#if (!defined(USE_BMOVE512) || defined(HAVE_valgrind)) && !defined(bmove512) #define bmove512(A,B,C) memcpy(A,B,C) #endif @@ -125,14 +125,14 @@ extern void bzero(uchar * dst,size_t len #if !defined(bcmp) && !defined(HAVE_BCMP) extern size_t bcmp(const uchar *s1,const uchar *s2,size_t len); #endif -#ifdef HAVE_purify +#ifdef HAVE_valgrind extern size_t my_bcmp(const uchar *s1,const uchar *s2,size_t len); #undef bcmp #define bcmp(A,B,C) my_bcmp((A),(B),(C)) -#define bzero_if_purify(A,B) bzero(A,B) +#define bzero_if_valgrind(A,B) bzero(A,B) #else -#define bzero_if_purify(A,B) -#endif /* HAVE_purify */ +#define bzero_if_valgrind(A,B) +#endif /* HAVE_valgrind */ #if defined(_lint) || defined(FORCE_INIT_OF_VARS) #define LINT_INIT_STRUCT(var) bzero(&var, sizeof(var)) /* No uninitialize-warning */ === modified file 'include/my_global.h' --- a/include/my_global.h 2009-04-08 16:55:26 +0000 +++ b/include/my_global.h 2009-04-27 18:01:20 +0000 @@ -563,16 +563,16 @@ int __void__; #define LINT_INIT(var) #endif -#if defined(_lint) || defined(FORCE_INIT_OF_VARS) || defined(HAVE_purify) -#define PURIFY_OR_LINT_INIT(var) var=0 +#if defined(_lint) || defined(FORCE_INIT_OF_VARS) || defined(HAVE_valgrind) +#define VALGRIND_OR_LINT_INIT(var) var=0 #else -#define PURIFY_OR_LINT_INIT(var) +#define VALGRIND_OR_LINT_INIT(var) #endif -#ifdef HAVE_purify -#define IF_PURIFY(A,B) (A) +#ifdef HAVE_valgrind +#define IF_VALGRIND(A,B) (A) #else -#define IF_PURIFY(A,B) (B) +#define IF_VALGRIND(A,B) (B) #endif #if !defined(HAVE_UINT) @@ -1174,7 +1174,7 @@ typedef char bool; /* Ordinary boolean ((uint32) (uchar) (A)[0]))) #define sint4korr(A) (*((const long *) (A))) #define uint2korr(A) (*((const uint16 *) (A))) -#if defined(HAVE_purify) && !defined(_WIN32) +#if defined(HAVE_valgrind) && !defined(_WIN32) #define uint3korr(A) (uint32) (((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ (((uint32) ((uchar) (A)[2])) << 16)) @@ -1186,7 +1186,7 @@ typedef char bool; /* Ordinary boolean It means, that you have to provide enough allocated space ! */ #define uint3korr(A) (long) (*((const unsigned int *) (A)) & 0xFFFFFF) -#endif /* HAVE_purify && !_WIN32 */ +#endif /* HAVE_valgrind && !_WIN32 */ #define uint4korr(A) (*((const uint32 *) (A))) #define uint5korr(A) ((ulonglong)(((uint32) ((uchar) (A)[0])) +\ (((uint32) ((uchar) (A)[1])) << 8) +\ === modified file 'mysys/mf_qsort.c' --- a/mysys/mf_qsort.c 2007-10-17 16:08:58 +0000 +++ b/mysys/mf_qsort.c 2009-04-27 18:01:20 +0000 @@ -108,7 +108,7 @@ qsort_t my_qsort(void *base_ptr, size_t low = (char*) base_ptr; high = low+ size * (count - 1); stack_ptr = stack + 1; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* The first element in the stack will be accessed for the last POP */ stack[0].low=stack[0].high=0; #endif === modified file 'mysys/my_alloc.c' --- a/mysys/my_alloc.c 2009-01-15 18:11:25 +0000 +++ b/mysys/my_alloc.c 2009-04-27 18:01:20 +0000 @@ -56,7 +56,7 @@ void init_alloc_root(MEM_ROOT *mem_root, mem_root->block_num= 4; /* We shift this with >>2 */ mem_root->first_block_usage= 0; -#if !(defined(HAVE_purify) && defined(EXTRA_DEBUG)) +#if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { if ((mem_root->free= mem_root->pre_alloc= @@ -96,7 +96,7 @@ void reset_root_defaults(MEM_ROOT *mem_r DBUG_ASSERT(alloc_root_inited(mem_root)); mem_root->block_size= block_size - ALLOC_ROOT_MIN_BLOCK_SIZE; -#if !(defined(HAVE_purify) && defined(EXTRA_DEBUG)) +#if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { size_t size= pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)); @@ -147,7 +147,7 @@ void reset_root_defaults(MEM_ROOT *mem_r void *alloc_root(MEM_ROOT *mem_root, size_t length) { -#if defined(HAVE_purify) && defined(EXTRA_DEBUG) +#if defined(HAVE_valgrind) && defined(EXTRA_DEBUG) reg1 USED_MEM *next; DBUG_ENTER("alloc_root"); DBUG_PRINT("enter",("root: 0x%lx", (long) mem_root)); === modified file 'mysys/my_init.c' --- a/mysys/my_init.c 2009-02-19 09:01:25 +0000 +++ b/mysys/my_init.c 2009-04-27 18:01:20 +0000 @@ -170,7 +170,7 @@ void my_end(int infoflag) { #ifdef HAVE_GETRUSAGE struct rusage rus; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Purify assumes that rus is uninitialized after getrusage call */ bzero((char*) &rus, sizeof(rus)); #endif === modified file 'mysys/my_rnd.c' --- a/mysys/my_rnd.c 2007-12-14 08:47:00 +0000 +++ b/mysys/my_rnd.c 2009-04-27 18:01:20 +0000 @@ -26,7 +26,7 @@ void my_rnd_init(struct my_rnd_struct *rand_st, ulong seed1, ulong seed2) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero((char*) rand_st,sizeof(*rand_st)); /* Avoid UMC varnings */ #endif rand_st->max_value= 0x3FFFFFFFL; === modified file 'mysys/safemalloc.c' --- a/mysys/safemalloc.c 2008-10-10 15:28:41 +0000 +++ b/mysys/safemalloc.c 2009-04-27 18:01:20 +0000 @@ -304,7 +304,7 @@ void _myfree(void *ptr, const char *file sf_malloc_count--; pthread_mutex_unlock(&THR_LOCK_malloc); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Mark this data as free'ed */ if (!sf_malloc_quick) bfill(ptr, irem->datasize, (pchar) FREE_VAL); === modified file 'scripts/mysql_config.pl.in' --- a/scripts/mysql_config.pl.in 2008-11-10 21:12:15 +0000 +++ b/scripts/mysql_config.pl.in 2009-04-27 18:01:20 +0000 @@ -44,7 +44,7 @@ use strict; my @exclude_cflags = qw/DDBUG_OFF DSAFEMALLOC USAFEMALLOC DSAFE_MUTEX DPEDANTIC_SAFEMALLOC DUNIV_MUST_NOT_INLINE DFORCE_INIT_OF_VARS - DEXTRA_DEBUG DHAVE_purify O O[0-9] xO[0-9] W[-A-Za-z]* + DEXTRA_DEBUG DHAVE_valgrind O O[0-9] xO[0-9] W[-A-Za-z]* Xa xstrconst xc99=none unroll2 ip mp restrict/; === modified file 'scripts/mysql_config.sh' --- a/scripts/mysql_config.sh 2008-11-10 21:12:15 +0000 +++ b/scripts/mysql_config.sh 2009-04-27 18:01:20 +0000 @@ -129,7 +129,7 @@ include="-I$pkgincludedir" # to make --cflags usable for HP C++ (aCC) for remove in DDBUG_OFF DSAFEMALLOC USAFEMALLOC DSAFE_MUTEX \ DPEDANTIC_SAFEMALLOC DUNIV_MUST_NOT_INLINE DFORCE_INIT_OF_VARS \ - DEXTRA_DEBUG DHAVE_purify O 'O[0-9]' 'xO[0-9]' 'W[-A-Za-z]*' \ + DEXTRA_DEBUG DHAVE_valgrind O 'O[0-9]' 'xO[0-9]' 'W[-A-Za-z]*' \ 'mtune=[-A-Za-z0-9]*' 'mcpu=[-A-Za-z0-9]*' 'march=[-A-Za-z0-9]*' \ Xa xstrconst "xc99=none" AC99 \ unroll2 ip mp restrict === modified file 'sql/field_conv.cc' --- a/sql/field_conv.cc 2009-01-31 21:22:44 +0000 +++ b/sql/field_conv.cc 2009-04-27 18:01:20 +0000 @@ -773,7 +773,7 @@ int field_conv(Field *to,Field *from) ((Field_varstring*)from)->length_bytes == ((Field_varstring*)to)->length_bytes)) { // Identical fields -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* This may happen if one does 'UPDATE ... SET x=x' */ if (to->ptr != from->ptr) #endif === modified file 'sql/filesort.cc' --- a/sql/filesort.cc 2009-02-19 09:01:25 +0000 +++ b/sql/filesort.cc 2009-04-27 18:01:20 +0000 @@ -953,13 +953,13 @@ static void make_sortkey(register SORTPA if (addonf->null_bit && field->is_null()) { nulls[addonf->null_offset]|= addonf->null_bit; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(to, addonf->length); #endif } else { -#ifdef HAVE_purify +#ifdef HAVE_valgrind uchar *end= field->pack(to, field->ptr); uint length= (uint) ((to + addonf->length) - end); DBUG_ASSERT((int) length >= 0); === modified file 'sql/ha_partition.cc' --- a/sql/ha_partition.cc 2009-02-19 09:01:25 +0000 +++ b/sql/ha_partition.cc 2009-04-27 18:01:20 +0000 @@ -3548,10 +3548,10 @@ void ha_partition::position(const uchar (ref_length - PARTITION_BYTES_IN_POS)); #ifdef SUPPORTING_PARTITION_OVER_DIFFERENT_ENGINES -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(ref + PARTITION_BYTES_IN_POS + ref_length, max_ref_length-ref_length); -#endif /* HAVE_purify */ +#endif /* HAVE_valgrind */ #endif DBUG_VOID_RETURN; } === modified file 'sql/hostname.cc' --- a/sql/hostname.cc 2007-10-11 17:29:09 +0000 +++ b/sql/hostname.cc 2009-04-27 18:01:20 +0000 @@ -175,7 +175,7 @@ char * ip_to_hostname(struct in_addr *in char buff[GETHOSTBYADDR_BUFF_SIZE],buff2[GETHOSTBYNAME_BUFF_SIZE]; int tmp_errno; struct hostent tmp_hostent, tmp_hostent2; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(buff,sizeof(buff)); // Bug in purify #endif if (!(hp=gethostbyaddr_r((char*) in,sizeof(*in), === modified file 'sql/item_timefunc.cc' --- a/sql/item_timefunc.cc 2009-02-15 10:58:34 +0000 +++ b/sql/item_timefunc.cc 2009-04-27 18:01:20 +0000 @@ -290,8 +290,8 @@ static bool extract_date_time(DATE_TIME_ LINT_INIT(strict_week_number); /* Remove valgrind varnings when using gcc 3.3 and -O1 */ - PURIFY_OR_LINT_INIT(strict_week_number_year_type); - PURIFY_OR_LINT_INIT(sunday_first_n_first_week_non_iso); + VALGRIND_OR_LINT_INIT(strict_week_number_year_type); + VALGRIND_OR_LINT_INIT(sunday_first_n_first_week_non_iso); if (!sub_pattern_end) bzero((char*) l_time, sizeof(*l_time)); === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2009-02-19 09:01:25 +0000 +++ b/sql/log_event.cc 2009-04-27 18:01:20 +0000 @@ -6990,7 +6990,7 @@ int Rows_log_event::do_add_row_data(ucha Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("row_data", row_data, min(length, 32)); #endif @@ -7771,7 +7771,7 @@ Table_map_log_event::Table_map_log_event Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("event buffer", (uchar*) buf, event_len); #endif @@ -8674,7 +8674,7 @@ int Rows_log_event::find_row(const Relay Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("key data", m_key, table->key_info->key_length); #endif @@ -8704,7 +8704,7 @@ int Rows_log_event::find_row(const Relay Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_PRINT("info",("found first matching record")); DBUG_DUMP("record[0]", table->record[0], table->s->reclength); #endif @@ -9067,7 +9067,7 @@ Update_rows_log_event::do_exec_row(const Now we have the right row to update. The old row (the one we're looking for) is in record[1] and the new row is in record[0]. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Don't print debug messages when running valgrind since they can trigger false warnings. === modified file 'sql/log_event_old.cc' --- a/sql/log_event_old.cc 2009-01-31 21:22:44 +0000 +++ b/sql/log_event_old.cc 2009-04-27 18:01:20 +0000 @@ -724,7 +724,7 @@ static int find_and_fetch_row(TABLE *tab Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("table->record[0]", table->record[0], table->s->reclength); DBUG_DUMP("table->record[1]", table->record[1], table->s->reclength); #endif @@ -750,7 +750,7 @@ static int find_and_fetch_row(TABLE *tab Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("table->record[0]", table->record[0], table->s->reclength); DBUG_DUMP("table->record[1]", table->record[1], table->s->reclength); #endif @@ -1404,7 +1404,7 @@ int Old_rows_log_event::do_add_row_data( Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("row_data", row_data, min(length, 32)); #endif @@ -2353,7 +2353,7 @@ int Old_rows_log_event::find_row(const R Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("key data", m_key, table->key_info->key_length); #endif @@ -2383,7 +2383,7 @@ int Old_rows_log_event::find_row(const R Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_PRINT("info",("found first matching record")); DBUG_DUMP("record[0]", table->record[0], table->s->reclength); #endif @@ -2878,7 +2878,7 @@ Update_rows_log_event_old::do_exec_row(c Now we have the right row to update. The old row (the one we're looking for) is in record[1] and the new row is in record[0]. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* Don't print debug messages when running valgrind since they can trigger false warnings. === modified file 'sql/my_decimal.h' --- a/sql/my_decimal.h 2008-05-20 07:38:17 +0000 +++ b/sql/my_decimal.h 2009-04-27 18:01:20 +0000 @@ -99,7 +99,7 @@ public: { len= DECIMAL_BUFF_LENGTH; buf= buffer; -#if !defined (HAVE_purify) && !defined(DBUG_OFF) +#if !defined (HAVE_valgrind) && !defined(DBUG_OFF) /* Set buffer to 'random' value to find wrong buffer usage */ for (uint i= 0; i < DECIMAL_BUFF_LENGTH; i++) buffer[i]= i; === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-04-08 21:16:10 +0000 +++ b/sql/mysqld.cc 2009-04-27 18:01:20 +0000 @@ -57,12 +57,6 @@ #define mysqld_charset &my_charset_latin1 -#ifdef HAVE_purify -#define IF_PURIFY(A,B) (A) -#else -#define IF_PURIFY(A,B) (B) -#endif - #if SIZEOF_CHARP == 4 #define MAX_MEM_TABLE_SIZE ~(ulong) 0 #else @@ -76,9 +70,9 @@ char pstack_file_name[80]; #endif /* __linux__ */ -/* We have HAVE_purify below as this speeds up the shutdown of MySQL */ +/* We have HAVE_valgrind below as this speeds up the shutdown of MySQL */ -#if defined(HAVE_DEC_3_2_THREADS) || defined(SIGNALS_DONT_BREAK_READ) || defined(HAVE_purify) && defined(__linux__) +#if defined(HAVE_DEC_3_2_THREADS) || defined(SIGNALS_DONT_BREAK_READ) || defined(HAVE_valgrind) && defined(__linux__) #define HAVE_CLOSE_SERVER_SOCK 1 #endif @@ -6520,7 +6514,7 @@ log and this option does nothing anymore purify. These are not suppressed: instead we disable symlinks option if compiled with valgrind support. */ - IF_PURIFY(0,1), 0, 0, 0, 0, 0}, + IF_VALGRIND(0,1), 0, 0, 0, 0, 0}, {"sysdate-is-now", OPT_SYSDATE_IS_NOW, "Non-default option to alias SYSDATE() to NOW() to make it safe-replicable. Since 5.0, SYSDATE() returns a `dynamic' value different for different invocations, even within the same statement.", (uchar**) &global_system_variables.sysdate_is_now, @@ -6561,7 +6555,7 @@ log and this option does nothing anymore 0, 0, 0, 0, 0}, {"use-symbolic-links", 's', "Enable symbolic link support. Deprecated option; use --symbolic-links instead.", (uchar**) &my_use_symdir, (uchar**) &my_use_symdir, 0, GET_BOOL, NO_ARG, - IF_PURIFY(0,1), 0, 0, 0, 0, 0}, + IF_VALGRIND(0,1), 0, 0, 0, 0, 0}, {"user", 'u', "Run mysqld daemon as user.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Used with --help option for detailed help", @@ -7661,7 +7655,7 @@ static int mysql_init_variables(void) /* Things reset to zero */ opt_skip_slave_start= opt_reckless_slave = 0; mysql_home[0]= pidfile_name[0]= log_error_file[0]= 0; -#if defined(HAVE_REALPATH) && !defined(HAVE_purify) && !defined(HAVE_BROKEN_REALPATH) +#if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) /* We can only test for sub paths if my_symlink.c is using realpath */ myisam_test_invalid_symlink= test_if_data_home_dir; #endif === modified file 'sql/opt_range.cc' --- a/sql/opt_range.cc 2009-02-19 09:01:25 +0000 +++ b/sql/opt_range.cc 2009-04-27 18:01:20 +0000 @@ -8347,7 +8347,7 @@ int QUICK_RANGE_SELECT::reset() multi_range_buff->buffer= mrange_buff; multi_range_buff->buffer_end= mrange_buff + mrange_bufsiz; multi_range_buff->end_of_used_area= mrange_buff; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* We need this until ndb will use the buffer efficiently (Now ndb stores complete row in here, instead of only the used fields === modified file 'sql/opt_range.h' --- a/sql/opt_range.h 2008-08-25 17:18:22 +0000 +++ b/sql/opt_range.h 2009-04-27 18:01:20 +0000 @@ -44,7 +44,7 @@ class QUICK_RANGE :public Sql_alloc { uint16 min_length,max_length,flag; key_part_map min_keypart_map, // bitmap of used keyparts in min_key max_keypart_map; // bitmap of used keyparts in max_key -#ifdef HAVE_purify +#ifdef HAVE_valgrind uint16 dummy; /* Avoid warnings on 'flag' */ #endif QUICK_RANGE(); /* Full range */ @@ -61,7 +61,7 @@ class QUICK_RANGE :public Sql_alloc { min_keypart_map(min_keypart_map_arg), max_keypart_map(max_keypart_map_arg) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind dummy=0; #endif } === modified file 'sql/records.cc' --- a/sql/records.cc 2008-07-17 18:26:55 +0000 +++ b/sql/records.cc 2009-04-27 18:01:20 +0000 @@ -520,7 +520,7 @@ static int init_rr_cache(THD *thd, READ_ info->struct_length+1, MYF(0)))) DBUG_RETURN(1); -#ifdef HAVE_purify +#ifdef HAVE_valgrind // Avoid warnings in qsort bzero(info->cache,rec_cache_size+info->cache_records* info->struct_length+1); #endif === modified file 'sql/rpl_rli.cc' --- a/sql/rpl_rli.cc 2009-02-19 09:01:25 +0000 +++ b/sql/rpl_rli.cc 2009-04-27 18:01:20 +0000 @@ -34,7 +34,7 @@ Relay_log_info::Relay_log_info() no_storage(FALSE), replicate_same_server_id(::replicate_same_server_id), info_fd(-1), cur_log_fd(-1), save_temporary_tables(0), cur_log_old_open_count(0), group_relay_log_pos(0), event_relay_log_pos(0), -#if HAVE_purify +#if HAVE_valgrind is_fake(FALSE), #endif group_master_log_pos(0), log_space_total(0), ignore_log_space_limit(0), === modified file 'sql/rpl_rli.h' --- a/sql/rpl_rli.h 2008-02-27 17:46:06 +0000 +++ b/sql/rpl_rli.h 2009-04-27 18:01:20 +0000 @@ -154,7 +154,7 @@ public: ulonglong event_relay_log_pos; ulonglong future_event_relay_log_pos; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bool is_fake; /* Mark that this is a fake relay log info structure */ #endif === modified file 'sql/slave.cc' --- a/sql/slave.cc 2009-02-19 09:01:25 +0000 +++ b/sql/slave.cc 2009-04-27 18:01:20 +0000 @@ -2014,7 +2014,7 @@ int apply_event_and_update_pos(Log_event if (exec_res == 0) { int error= ev->update_pos(rli); -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (!rli->is_fake) #endif { === modified file 'sql/sql_base.cc' --- a/sql/sql_base.cc 2009-02-19 09:01:25 +0000 +++ b/sql/sql_base.cc 2009-04-27 18:01:20 +0000 @@ -3000,7 +3000,7 @@ TABLE *open_table(THD *thd, TABLE_LIST * table->pos_in_table_list= table_list; table_list->updatable= 1; // It is not derived table nor non-updatable VIEW table->clear_column_bitmaps(); -#if !defined(DBUG_OFF) && !defined(HAVE_purify) +#if !defined(DBUG_OFF) && !defined(HAVE_valgrind) /* Fill record with random values to find bugs where we access fields without first reading them. @@ -7363,7 +7363,7 @@ int setup_wild(THD *thd, TABLE_LIST *tab /* make * substituting permanent */ SELECT_LEX *select_lex= thd->lex->current_select; select_lex->with_wild= 0; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (&select_lex->item_list != &fields) // Avoid warning #endif select_lex->item_list= fields; === modified file 'sql/sql_binlog.cc' --- a/sql/sql_binlog.cc 2009-01-09 12:49:24 +0000 +++ b/sql/sql_binlog.cc 2009-04-27 18:01:20 +0000 @@ -59,7 +59,7 @@ void mysql_client_binlog_statement(THD* if (!thd->rli_fake) { thd->rli_fake= new Relay_log_info; -#ifdef HAVE_purify +#ifdef HAVE_valgrind thd->rli_fake->is_fake= TRUE; #endif have_fd_event= FALSE; @@ -95,7 +95,7 @@ void mysql_client_binlog_statement(THD* char const *endptr= 0; int bytes_decoded= base64_decode(strptr, coded_len, buf, &endptr); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* This debug printout should not be used for valgrind builds since it will read from unassigned memory. @@ -188,7 +188,7 @@ void mysql_client_binlog_statement(THD* bufptr += event_len; DBUG_PRINT("info",("ev->get_type_code()=%d", ev->get_type_code())); -#ifndef HAVE_purify +#ifndef HAVE_valgrind /* This debug printout should not be used for valgrind builds since it will read from unassigned memory. === modified file 'sql/sql_class.cc' --- a/sql/sql_class.cc 2009-03-12 22:27:35 +0000 +++ b/sql/sql_class.cc 2009-04-27 18:01:20 +0000 @@ -3480,7 +3480,7 @@ int THD::binlog_update_row(TABLE* table, Don't print debug messages when running valgrind since they can trigger false warnings. */ -#ifndef HAVE_purify +#ifndef HAVE_valgrind DBUG_DUMP("before_record", before_record, table->s->reclength); DBUG_DUMP("after_record", after_record, table->s->reclength); DBUG_DUMP("before_row", before_row, before_size); === modified file 'sql/sql_list.h' --- a/sql/sql_list.h 2009-01-15 18:11:25 +0000 +++ b/sql/sql_list.h 2009-04-27 18:01:20 +0000 @@ -43,7 +43,7 @@ public: static void operator delete[](void *ptr, MEM_ROOT *mem_root) { /* never called */ } static void operator delete[](void *ptr, size_t size) { TRASH(ptr, size); } -#ifdef HAVE_purify +#ifdef HAVE_valgrind bool dummy; inline Sql_alloc() :dummy(0) {} inline ~Sql_alloc() {} === modified file 'sql/sql_load.cc' --- a/sql/sql_load.cc 2009-01-31 21:22:44 +0000 +++ b/sql/sql_load.cc 2009-04-27 18:01:20 +0000 @@ -581,7 +581,7 @@ read_fixed_length(THD *thd, COPY_INFO &i } it.rewind(); uchar *pos=read_info.row_start; -#ifdef HAVE_purify +#ifdef HAVE_valgrind read_info.row_end[0]=0; #endif === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-04-08 16:55:26 +0000 +++ b/sql/sql_select.cc 2009-04-27 18:01:20 +0000 @@ -1818,7 +1818,7 @@ JOIN::exec() curr_join->having= curr_join->tmp_having= 0; // Allready done /* Change sum_fields reference to calculated fields in tmp_table */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif curr_join->all_fields= *curr_all_fields; @@ -1840,7 +1840,7 @@ JOIN::exec() fields_list.elements, all_fields)) DBUG_VOID_RETURN; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif { @@ -1994,7 +1994,7 @@ JOIN::exec() tmp_fields_list2, tmp_all_fields2, fields_list.elements, tmp_all_fields1)) DBUG_VOID_RETURN; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Some GCCs use memcpy() for struct assignment, even for x=x. GCC bug 19410: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=19410 @@ -2059,7 +2059,7 @@ JOIN::exec() tmp_table_param.save_copy_field= curr_join->tmp_table_param.copy_field; tmp_table_param.save_copy_field_end= curr_join->tmp_table_param.copy_field_end; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (curr_join != this) #endif { @@ -3848,7 +3848,7 @@ update_ref_and_keys(THD *thd, DYNAMIC_AR continue; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Valgrind complains about overlapped memcpy when save_pos==use. */ if (save_pos != use) #endif @@ -15138,7 +15138,7 @@ setup_copy_fields(THD *thd, TMP_TABLE_PA DBUG_ASSERT (param->field_count > (uint) (copy - copy_start)); copy->set(tmp, item->result_field); item->result_field->move_field(copy->to_ptr,copy->to_null_ptr,1); -#ifdef HAVE_purify +#ifdef HAVE_valgrind copy->to_ptr[copy->from_length]= 0; #endif copy++; === modified file 'sql/table.cc' --- a/sql/table.cc 2009-02-19 09:01:25 +0000 +++ b/sql/table.cc 2009-04-27 18:01:20 +0000 @@ -1694,7 +1694,7 @@ int open_table_from_share(THD *thd, TABL outparam->record[1]= outparam->record[0]; // Safety } -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* We need this because when we read var-length rows, we are not updating bytes after end of varchar @@ -1916,7 +1916,7 @@ partititon_err: } } -#if defined(HAVE_purify) && !defined(DBUG_OFF) +#if defined(HAVE_valgrind) && !defined(DBUG_OFF) bzero((char*) bitmaps, bitmap_size*3); #endif === modified file 'storage/archive/azio.c' --- a/storage/archive/azio.c 2009-04-08 16:55:26 +0000 +++ b/storage/archive/azio.c 2009-04-27 18:01:20 +0000 @@ -63,7 +63,7 @@ void read_header(azio_stream *s, unsigne */ static void *az_allocator(void *dummy, uInt items, uInt size) { - return my_malloc((size_t)items*(size_t)size, IF_PURIFY(MY_ZEROFILL, MYF(0))); + return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } static void az_free(void *dummy, void *address) === modified file 'storage/innobase/buf/buf0buf.c' --- a/storage/innobase/buf/buf0buf.c 2008-08-20 00:37:41 +0000 +++ b/storage/innobase/buf/buf0buf.c 2009-04-27 18:01:20 +0000 @@ -756,7 +756,7 @@ buf_pool_init( /* Wipe contents of frame to eliminate a Purify warning */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind memset(block->frame, '\0', UNIV_PAGE_SIZE); #endif if (srv_use_awe) { === modified file 'storage/innobase/include/univ.i' --- a/storage/innobase/include/univ.i 2008-03-27 01:40:45 +0000 +++ b/storage/innobase/include/univ.i 2009-04-27 18:01:20 +0000 @@ -107,7 +107,7 @@ operations (very slow); also UNIV_DEBUG #define UNIV_BTR_DEBUG /* check B-tree links */ #define UNIV_LIGHT_MEM_DEBUG /* light memory debugging */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* The following sets all new allocated memory to zero before use: this can be used to eliminate unnecessary Purify warnings, but note that it also masks many bugs Purify could detect. For detailed Purify analysis it === modified file 'storage/innobase/srv/srv0start.c' --- a/storage/innobase/srv/srv0start.c 2008-12-14 19:21:24 +0000 +++ b/storage/innobase/srv/srv0start.c 2009-04-27 18:01:20 +0000 @@ -104,7 +104,7 @@ static char* srv_monitor_file_name; /* Avoid warnings when using purify */ -#ifdef HAVE_purify +#ifdef HAVE_valgrind static int inno_bcmp(register const char *s1, register const char *s2, register uint len) { === modified file 'storage/maria/ha_maria.cc' --- a/storage/maria/ha_maria.cc 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ha_maria.cc 2009-04-27 18:01:20 +0000 @@ -3166,7 +3166,7 @@ static int ha_maria_init(void *p) ma_checkpoint_init(checkpoint_interval); maria_multi_threaded= maria_in_ha_maria= TRUE; -#if defined(HAVE_REALPATH) && !defined(HAVE_purify) && !defined(HAVE_BROKEN_REALPATH) +#if defined(HAVE_REALPATH) && !defined(HAVE_valgrind) && !defined(HAVE_BROKEN_REALPATH) /* We can only test for sub paths if my_symlink.c is using realpath */ maria_test_invalid_symlink= test_if_data_home_dir; #endif === modified file 'storage/maria/ma_blockrec.c' --- a/storage/maria/ma_blockrec.c 2009-04-08 08:47:30 +0000 +++ b/storage/maria/ma_blockrec.c 2009-04-27 18:01:20 +0000 @@ -1594,7 +1594,7 @@ static void make_empty_page(MARIA_HA *in bzero(buff, PAGE_HEADER_SIZE); -#if !defined(DONT_ZERO_PAGE_BLOCKS) || defined(HAVE_purify) +#if !defined(DONT_ZERO_PAGE_BLOCKS) || defined(HAVE_valgrind) /* We zero the rest of the block to avoid getting old memory information to disk and to allow the file to be compressed better if archived. === modified file 'storage/maria/ma_check.c' --- a/storage/maria/ma_check.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_check.c 2009-04-27 18:01:20 +0000 @@ -4522,7 +4522,7 @@ static int sort_key_read(MARIA_SORT_PARA sort_param->current_filepos, 0); sort_param->real_key_length= int_key.data_length + int_key.ref_length; -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); #endif @@ -4567,7 +4567,7 @@ static int sort_maria_ft_key_read(MARIA_ sort_param->current_filepos); sort_param->real_key_length= int_key.data_length + int_key.ref_length; -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (sort_param->key_length > sort_param->real_key_length) bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); @@ -5104,7 +5104,7 @@ static int sort_get_next_record(MARIA_SO llstr(sort_param->pos,llbuff)); continue; } -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(sort_param->rec_buff + block_info.rec_len, share->base.extra_rec_buff_size); #endif === modified file 'storage/maria/ma_loghandler.c' --- a/storage/maria/ma_loghandler.c 2009-03-17 19:05:01 +0000 +++ b/storage/maria/ma_loghandler.c 2009-04-27 18:01:20 +0000 @@ -6194,12 +6194,12 @@ my_bool translog_write_record(LSN *lsn, { uint i; uint len= 0; -#ifdef HAVE_purify +#ifdef HAVE_valgrind ha_checksum checksum= 0; #endif for (i= TRANSLOG_INTERNAL_PARTS; i < part_no; i++) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Find unitialized bytes early */ checksum+= my_checksum(checksum, parts_data[i].str, parts_data[i].length); === modified file 'storage/maria/ma_packrec.c' --- a/storage/maria/ma_packrec.c 2009-01-09 04:23:25 +0000 +++ b/storage/maria/ma_packrec.c 2009-04-27 18:01:20 +0000 @@ -256,7 +256,7 @@ static my_bool _ma_read_pack_info(MARIA_ (uint) (share->pack.header_length-sizeof(header)), MYF(MY_NABP))) goto err2; -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* Zero bytes accessed by fill_buffer */ bzero(disk_cache + (share->pack.header_length-sizeof(header)), share->base.extra_rec_buff_size); === modified file 'storage/maria/ma_page.c' --- a/storage/maria/ma_page.c 2008-09-01 17:31:40 +0000 +++ b/storage/maria/ma_page.c 2009-04-27 18:01:20 +0000 @@ -200,7 +200,7 @@ my_bool _ma_write_keypage(MARIA_PAGE *pa /* Verify that keynr is correct */ DBUG_ASSERT(_ma_get_keynr(share, buff) == page->keyinfo->key_nr); -#if defined(EXTRA_DEBUG) && defined(HAVE_purify) && defined(NOT_ANYMORE) +#if defined(EXTRA_DEBUG) && defined(HAVE_valgrind) && defined(NOT_ANYMORE) { /* This is here to catch uninitialized bytes */ uint length= page->size; === modified file 'storage/maria/ma_pagecrc.c' --- a/storage/maria/ma_pagecrc.c 2008-03-04 11:58:21 +0000 +++ b/storage/maria/ma_pagecrc.c 2009-04-27 18:01:20 +0000 @@ -318,7 +318,7 @@ my_bool maria_page_filler_set_none(uchar __attribute__((unused)), uchar *data_ptr __attribute__((unused))) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind int4store_aligned(page + ((MARIA_SHARE *)data_ptr)->block_size - CRC_SIZE, 0); #endif === modified file 'storage/maria/ma_search.c' --- a/storage/maria/ma_search.c 2009-01-12 11:12:00 +0000 +++ b/storage/maria/ma_search.c 2009-04-27 18:01:20 +0000 @@ -2172,7 +2172,7 @@ int _ma_calc_bin_pack_key_length(const M s_temp->totlength= key_length= (int_key->data_length + int_key->ref_length+ nod_flag); -#ifdef HAVE_purify +#ifdef HAVE_valgrind s_temp->n_length= s_temp->n_ref_length=0; /* For valgrind */ #endif s_temp->key=key; === modified file 'storage/myisam/mi_check.c' --- a/storage/myisam/mi_check.c 2009-02-19 09:01:25 +0000 +++ b/storage/myisam/mi_check.c 2009-04-27 18:01:20 +0000 @@ -3147,7 +3147,7 @@ static int sort_key_read(MI_SORT_PARAM * (info->s->rec_reflength+ _mi_make_key(info, sort_param->key, (uchar*) key, sort_param->record, sort_param->filepos)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); #endif @@ -3187,7 +3187,7 @@ static int sort_ft_key_read(MI_SORT_PARA sort_param->real_key_length=(info->s->rec_reflength+ _ft_make_key(info, sort_param->key, key, wptr++, sort_param->filepos)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind if (sort_param->key_length > sort_param->real_key_length) bzero(key+sort_param->real_key_length, (sort_param->key_length-sort_param->real_key_length)); === modified file 'storage/myisam/mi_page.c' --- a/storage/myisam/mi_page.c 2008-08-28 12:43:44 +0000 +++ b/storage/myisam/mi_page.c 2009-04-27 18:01:20 +0000 @@ -86,7 +86,7 @@ int _mi_write_keypage(register MI_INFO * if ((length=keyinfo->block_length) > IO_SIZE*2 && info->state->key_file_length != page+length) length= ((mi_getint(buff)+IO_SIZE-1) & (uint) ~(IO_SIZE-1)); -#ifdef HAVE_purify +#ifdef HAVE_valgrind { length=mi_getint(buff); bzero((uchar*) buff+length,keyinfo->block_length-length); === modified file 'storage/myisam/mi_search.c' --- a/storage/myisam/mi_search.c 2008-08-28 18:52:23 +0000 +++ b/storage/myisam/mi_search.c 2009-04-27 18:01:20 +0000 @@ -1749,7 +1749,7 @@ _mi_calc_bin_pack_key_length(MI_KEYDEF * uint length,key_length,ref_length; s_temp->totlength=key_length=_mi_keylength(keyinfo,key)+nod_flag; -#ifdef HAVE_purify +#ifdef HAVE_valgrind s_temp->n_length= s_temp->n_ref_length=0; /* For valgrind */ #endif s_temp->key=key; === modified file 'storage/myisammrg/ha_myisammrg.cc' --- a/storage/myisammrg/ha_myisammrg.cc 2008-12-04 00:36:55 +0000 +++ b/storage/myisammrg/ha_myisammrg.cc 2009-04-27 18:01:20 +0000 @@ -837,7 +837,7 @@ int ha_myisammrg::info(uint flag) { if (table->s->key_parts && mrg_info.rec_per_key) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind /* valgrind may be unhappy about it, because optimizer may access values between file->keys and table->key_parts, that will be uninitialized. === modified file 'strings/bcmp.c' --- a/strings/bcmp.c 2007-05-10 09:59:39 +0000 +++ b/strings/bcmp.c 2009-04-27 18:01:20 +0000 @@ -24,7 +24,7 @@ #include <my_global.h> #include "m_string.h" -#ifdef HAVE_purify +#ifdef HAVE_valgrind #undef bcmp #undef HAVE_BCMP #endif @@ -50,7 +50,7 @@ uint len; /* 0 <= len <= 65535 */ #else -#ifndef HAVE_purify +#ifndef HAVE_valgrind size_t bcmp(register const uchar *s1,register const uchar *s2, register size_t len) #else === modified file 'strings/decimal.c' --- a/strings/decimal.c 2008-08-15 19:49:43 +0000 +++ b/strings/decimal.c 2009-04-27 18:01:20 +0000 @@ -144,7 +144,7 @@ static double scaler1[]= { 1.0, 10.0, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9 }; -#ifdef HAVE_purify +#ifdef HAVE_valgrind #define sanity(d) DBUG_ASSERT((d)->len > 0) #else #define sanity(d) DBUG_ASSERT((d)->len >0 && ((d)->buf[0] | \ === modified file 'strings/strmake.c' --- a/strings/strmake.c 2008-04-28 16:24:05 +0000 +++ b/strings/strmake.c 2009-04-27 18:01:20 +0000 @@ -51,11 +51,11 @@ char *strmake(register char *dst, regist */ if (length) { -#ifdef HAVE_purify +#ifdef HAVE_valgrind dst[length-1]= 'Z'; #else bfill(dst, length-1, (int) 'Z'); -#endif /* HAVE_purify */ +#endif /* HAVE_valgrind */ } #endif /* EXTRA_DEBUG */ return dst-1;

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2698)
by knielsen＠knielsen-hq.org 06 May '09

06 May '09

#At lp:maria 2698 knielsen(a)knielsen-hq.org 2009-04-27 Misc. small fixes: - Fixes for pool-of-threads patch. - Fixes for push of PBXT storage engine. - mysql-test-run.pl fix. modified: BUILD/compile-dist mysql-test/Makefile.am mysql-test/lib/mtr_unique.pm mysql-test/t/pool_of_threads.cnf sql/set_var.cc per-file messages: BUILD/compile-dist Fix that PBXT was missing in source tarball after `BUILD/compile-dist && make dist` mysql-test/Makefile.am Fix that PBXT test suite was missing from `make dist` source tarball. mysql-test/lib/mtr_unique.pm Better fix to avoid races when chmod'ing the semaphore file. (Though using chmod 666 shared files in /tmp/ is still not a very good solution). mysql-test/t/pool_of_threads.cnf Fix that test case pool_of_threads fails if run on mysqld with no --with-libevent support. sql/set_var.cc Fix missing static declaration on pool_of_threads. === modified file 'BUILD/compile-dist' --- a/BUILD/compile-dist 2009-02-19 09:01:25 +0000 +++ b/BUILD/compile-dist 2009-04-27 13:17:47 +0000 @@ -60,8 +60,6 @@ fi # Make sure to enable all features that affect "make dist" # Remember that configure restricts the man pages to the configured features ! ./configure \ - --with-maria-storage-engine \ - --with-embedded-server \ - --with-ndbcluster -$gmake - + --with-plugins=max-no-ndb \ + --with-embedded-server +$gmake -j4 === modified file 'mysql-test/Makefile.am' --- a/mysql-test/Makefile.am 2009-02-13 12:25:41 +0000 +++ b/mysql-test/Makefile.am 2009-04-27 13:17:47 +0000 @@ -98,7 +98,8 @@ TEST_DIRS = t r include std_data std_dat suite/stress/include suite/stress/t suite/stress/r \ suite/ndb suite/ndb/t suite/ndb/r \ suite/rpl_ndb suite/rpl_ndb/t suite/rpl_ndb/r \ - suite/parts suite/parts/t suite/parts/r suite/parts/inc + suite/parts suite/parts/t suite/parts/r suite/parts/inc \ + suite/pbxt/t suite/pbxt/r # Used by dist-hook and install-data-local to copy all # test files into either dist or install directory === modified file 'mysql-test/lib/mtr_unique.pm' --- a/mysql-test/lib/mtr_unique.pm 2009-04-08 16:55:26 +0000 +++ b/mysql-test/lib/mtr_unique.pm 2009-04-27 13:17:47 +0000 @@ -62,14 +62,14 @@ sub mtr_get_unique_id($$) { die 'lock file is a symbolic link'; } + my $save_umask= umask(0); open SEM, ">", "$file.sem" or die "can't write to $file.sem"; - chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; if(! -e $file) { open FILE, ">", $file or die "can't create $file"; close FILE; } - chmod 0777, $file; + umask($save_umask); msg("HAVE THE LOCK"); @@ -135,8 +135,8 @@ sub mtr_release_unique_id($) { die 'lock file is a symbolic link'; } + my $save_umask= umask(0); open SEM, ">", "$file.sem" or die "can't write to $file.sem"; - chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; msg("HAVE THE LOCK"); @@ -149,7 +149,7 @@ sub mtr_release_unique_id($) { open FILE, ">", $file or die "can't create $file"; close FILE; } - chmod 0777, "$file.sem"; + umask($save_umask); open FILE, "+<", $file or die "can't open $file"; #select undef,undef,undef,0.2; seek FILE, 0, 0; === modified file 'mysql-test/t/pool_of_threads.cnf' --- a/mysql-test/t/pool_of_threads.cnf 2009-03-18 15:46:32 +0000 +++ b/mysql-test/t/pool_of_threads.cnf 2009-04-27 13:17:47 +0000 @@ -2,8 +2,8 @@ [mysqld.1] test-ignore-wrong-options -thread-handling= pool-of-threads -thread_pool_size= 2 +loose-thread-handling= pool-of-threads +loose-thread_pool_size= 2 extra-port= @mysqld.1.#extra-port extra-max-connections=1 === modified file 'sql/set_var.cc' --- a/sql/set_var.cc 2009-04-08 16:55:26 +0000 +++ b/sql/set_var.cc 2009-04-27 13:17:47 +0000 @@ -642,7 +642,7 @@ static sys_var_long_ptr sys_table_lock_w static sys_var_long_ptr sys_thread_cache_size(&vars, "thread_cache_size", &thread_cache_size); #if HAVE_POOL_OF_THREADS == 1 -sys_var_long_ptr sys_thread_pool_size(&vars, "thread_pool_size", +static sys_var_long_ptr sys_thread_pool_size(&vars, "thread_pool_size", &thread_pool_size); #endif static sys_var_thd_enum sys_tx_isolation(&vars, "tx_isolation",

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2698)
by knielsen＠knielsen-hq.org 06 May '09

06 May '09

#At lp:maria 2698 knielsen(a)knielsen-hq.org 2009-05-06 Fix a build problem with BUILD/compile-pentium64-max on CentOS/Fedora Core 10 amd64. On these systems, there is libz.so but no libz.a. Finding libz.so, ./configure decides to use system zlib. But since BUILD/compile-pentium64-max builds a fully static binary with -all-static, the link of mysqld fails due to missing libz.a. Fix by using bundled zlib in the build script. modified: BUILD/compile-pentium64-max === modified file 'BUILD/compile-pentium64-max' --- a/BUILD/compile-pentium64-max 2007-04-11 12:12:00 +0000 +++ b/BUILD/compile-pentium64-max 2009-05-06 09:51:46 +0000 @@ -4,7 +4,10 @@ path=`dirname $0` . "$path/SETUP.sh" extra_flags="$pentium64_cflags $fast_cflags" -extra_configs="$pentium_configs $max_configs $static_link" +# On CentOS/Fedora Core 10 amd64, there is system libz.so but not +# libz.a, so need to use bundled zlib when building static +# binary. Hence we use --with-zlib-dir=bundled +extra_configs="$pentium_configs $max_configs $static_link --with-zlib-dir=bundled" CC="$CC --pipe" strip=yes

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2702)
by knielsen＠knielsen-hq.org 05 May '09

05 May '09

#At lp:maria 2702 knielsen(a)knielsen-hq.org 2009-05-05 [merge] test merge 2

1 0

Re: [Maria-developers] main.pool_of_threads fail - Solaris 10
by Adam M. Dutko 27 Apr '09

27 Apr '09

Toby, You might also want to check this page out: http://askmonty.org/wiki/index.php/MariaDB_versus_MySQL -Adam

1 0

[Maria-developers] main.pool_of_threads fail - Solaris 10
by Toby Thain 27 Apr '09

27 Apr '09

Hi list, I am setting up a buildbot for Solaris 10/amd64 (later SPARC) and I see this test failure on Sun cc (only). gcc builds pass all tests. (Although a patch is required.) Any clues? --Toby bzr branch lp:maria ... BUILD/compile-solaris-amd64-forte && cd mysql-test/ && ./mysql-test-run ... main.connect [ pass ] 16399 worker[1] mysql-test-run: WARNING: Process [mysqld.1 - pid: 495, winpid: 495, exit: 1792] died main.pool_of_threads [ fail ] Test ended at 2009-04-26 15:18:41 090426 22:17:50 [Note] /home/buildbot/maria/sql/mysqld: Normal shutdown 090426 22:17:50 [Note] Event Scheduler: Purging the queue. 0 events 090426 22:17:53 InnoDB: Starting shutdown... 090426 22:17:53 InnoDB: Shutdown completed; log sequence number 0 1943684 090426 22:17:53 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete 090426 22:17:54 InnoDB: Started; log sequence number 0 1943684 090426 22:17:54 [Warning] /home/buildbot/maria/sql/mysqld: unknown variable 'loose-debug-sync-timeout=300' 090426 22:17:54 [Note] Event Scheduler: Loaded 0 events 090426 22:17:54 [Note] /home/buildbot/maria/sql/mysqld: ready for connections. Version: '5.1.32-maria-beta2-log' socket: '/home/buildbot/maria/ mysql-test/var/tmp/mysqld.1.sock' port: 12500 Source distribution CURRENT_TEST: rpl.rpl_truncate_3innodb 090426 22:18:01 [Note] /home/buildbot/maria/sql/mysqld: Normal shutdown 090426 22:18:01 [Note] Event Scheduler: Purging the queue. 0 events 090426 22:18:03 InnoDB: Starting shutdown... 090426 22:18:05 InnoDB: Shutdown completed; log sequence number 0 1968339 090426 22:18:05 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete CURRENT_TEST: federated.federated 090426 22:18:05 [Note] Plugin 'InnoDB' disabled by command line option 090426 22:18:05 [Warning] /home/buildbot/maria/sql/mysqld: unknown variable 'loose-debug-sync-timeout=300' 090426 22:18:05 [Note] Event Scheduler: Loaded 0 events 090426 22:18:05 [Note] /home/buildbot/maria/sql/mysqld: ready for connections. Version: '5.1.32-maria-beta2-log' socket: '/home/buildbot/maria/ mysql-test/var/tmp/mysqld.1.sock' port: 12500 Source distribution CURRENT_TEST: federated.federated_archive CURRENT_TEST: federated.federated_bug_13118 CURRENT_TEST: federated.federated_bug_25714 CURRENT_TEST: federated.federated_server 090426 22:18:17 [Note] /home/buildbot/maria/sql/mysqld: Normal shutdown 090426 22:18:17 [Note] Event Scheduler: Purging the queue. 0 events 090426 22:18:17 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete CURRENT_TEST: federated.federated_innodb InnoDB: The first specified data file ./ibdata1 did not exist: InnoDB: a new database to be created! 090426 22:18:18 InnoDB: Setting file ./ibdata1 size to 10 MB InnoDB: Database physically writes the file full: wait... 090426 22:18:18 InnoDB: Log file ./ib_logfile0 did not exist: new to be created InnoDB: Setting log file ./ib_logfile0 size to 5 MB InnoDB: Database physically writes the file full: wait... 090426 22:18:18 InnoDB: Log file ./ib_logfile1 did not exist: new to be created InnoDB: Setting log file ./ib_logfile1 size to 5 MB InnoDB: Database physically writes the file full: wait... InnoDB: Doublewrite buffer not found: creating new InnoDB: Doublewrite buffer created InnoDB: Creating foreign key constraint system tables InnoDB: Foreign key constraint system tables created 090426 22:18:20 InnoDB: Started; log sequence number 0 0 090426 22:18:20 [Warning] /home/buildbot/maria/sql/mysqld: unknown variable 'loose-debug-sync-timeout=300' 090426 22:18:20 [Note] Event Scheduler: Loaded 0 events 090426 22:18:20 [Note] /home/buildbot/maria/sql/mysqld: ready for connections. Version: '5.1.32-maria-beta2-log' socket: '/home/buildbot/maria/ mysql-test/var/tmp/mysqld.1.sock' port: 12500 Source distribution 090426 22:18:20 [Note] /home/buildbot/maria/sql/mysqld: Normal shutdown 090426 22:18:20 [Note] Event Scheduler: Purging the queue. 0 events 090426 22:18:21 InnoDB: Starting shutdown... 090426 22:18:23 InnoDB: Shutdown completed; log sequence number 0 46409 090426 22:18:23 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete CURRENT_TEST: main.connect 090426 22:18:23 [Note] Plugin 'InnoDB' disabled by command line option 090426 22:18:23 [Warning] /home/buildbot/maria/sql/mysqld: unknown variable 'loose-debug-sync-timeout=300' 090426 22:18:23 [Note] Event Scheduler: Loaded 0 events 090426 22:18:23 [Note] /home/buildbot/maria/sql/mysqld: ready for connections. Version: '5.1.32-maria-beta2-log' socket: '/home/buildbot/maria/ mysql-test/var/tmp/mysqld.1.sock' port: 12500 Source distribution 090426 22:18:29 [Note] Event Scheduler: scheduler thread started with id 21 090426 22:18:40 [Note] Event Scheduler: Killing the scheduler thread, thread id 21 090426 22:18:40 [Note] Event Scheduler: Waiting for the scheduler thread to reply 090426 22:18:40 [Note] Event Scheduler: Stopped 090426 22:18:40 [Note] Event Scheduler: scheduler thread started with id 77 090426 22:18:40 [Note] Event Scheduler: Killing the scheduler thread, thread id 77 090426 22:18:40 [Note] Event Scheduler: Waiting for the scheduler thread to reply 090426 22:18:40 [Note] Event Scheduler: Stopped 090426 22:18:40 [Note] /home/buildbot/maria/sql/mysqld: Normal shutdown 090426 22:18:40 [Note] Event Scheduler: Purging the queue. 0 events 090426 22:18:40 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete CURRENT_TEST: main.pool_of_threads Unknown option to thread_handling: pool-of-threads Alternatives are: 'one-thread-per-connection','no-threads' 090426 22:18:40 [Note] Plugin 'InnoDB' disabled by command line option 090426 22:18:40 [ERROR] /home/buildbot/maria/sql/mysqld: unknown variable 'thread_pool_size=2' 090426 22:18:40 [ERROR] Aborting 090426 22:18:40 [Note] /home/buildbot/maria/sql/mysqld: Shutdown complete Failed to start mysqld.1 - saving '/home/buildbot/maria/mysql-test/var/log/ main.pool_of_threads/' to '/home/buildbot/maria/mysql-test/var/log/ main.pool_of_threads/' Only 863 of 866 completed. mysql-test-run: *** ERROR: Not all tests completed *** Error code 1 make: Fatal error: Command failed for target `test-ns'

2 1

[Maria-developers] order by x limit y optimization
by Tonu Samuel 26 Apr '09

26 Apr '09

Hi! Just was looking at processlist and thinking "why he cannot optimize such a easy case". I think there is a possible BOOST place (this is common query in many apps): mysql> select count(*) from url; +----------+ | count(*) | +----------+ | 17945916 | +----------+ 1 row in set (0.00 sec) mysql> mysql> show full processlist; +------+------+-----------+-----------+---------+------+----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Id | User | Host | db | Command | Time | State | Info | +------+------+-----------+-----------+---------+------+----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 78 | root | localhost | eeindexer | Query | 141 | Sorting result | INSERT INTO udm_url_tmp SELECT url.rec_id FROM url WHERE next_index_time<=1240742878 ORDER BY hops LIMIT 256 ..... So, it takes HUGE time to find out 256 rows with least hops. How I would make it manually (MySQL engine has internally better ways to do it): mysql> select min(hops) from url; +-----------+ | min(hops) | +-----------+ | 0 | +-----------+ 1 row in set (0.12 sec) mysql> select count(hops) from url where hops=0; +-------------+ | count(hops) | +-------------+ | 1 | +-------------+ 1 row in set (0.00 sec) mysql> select count(hops) from url where hops=1; +-------------+ | count(hops) | +-------------+ | 92 | +-------------+ 1 row in set (0.00 sec) mysql> select count(hops) from url where hops=2; +-------------+ | count(hops) | +-------------+ | 9790 | +-------------+ 1 row in set (0.00 sec) mysql> mysql> SELECT url.rec_id FROM url WHERE hops in (0,1,2,3) ORDER BY hops LIMIT 256; +--------+ | rec_id | +--------+ | 1 | | 82 | | 121 | | 55 | | 63 | | 39 | | 65 | | 100 | | 129 | | 85 | | 25 | | 11 | | 12 | | 54 | | 3 | | 101 | | 7 | | 50 | | 76 | | 77 | | 10 | | 24 | | 14 | | 2 | | 57 | | 46 | | 44 | | 17 | | 122 | | 135 | | 8 | | 51 | | 104 | | 124 | | 35 | | 56 | | 116 | | 62 | | 41 | | 78 | | 126 | | 15 | | 26 | | 40 | | 90 | | 80 | | 13 | | 36 | | 45 | | 70 | | 60 | | 117 | | 47 | | 114 | | 132 | | 19 | | 96 | | 118 | | 74 | | 9 | | 120 | | 95 | | 23 | | 136 | | 71 | | 130 | | 83 | | 33 | | 111 | | 125 | | 81 | | 43 | | 97 | | 28 | | 73 | | 109 | | 98 | | 105 | | 133 | | 128 | | 106 | | 4 | | 102 | | 127 | | 5 | | 30 | | 115 | | 123 | | 27 | | 6 | | 113 | | 93 | | 48 | | 10047 | | 55273 | | 7120 | | 21448 | | 16872 | | 17985 | | 9862 | | 20979 | | 45717 | | 491 | | 16943 | | 18183 | | 1753 | | 21176 | | 16486 | | 7578 | | 2329 | | 9824 | | 1239 | | 400 | | 6470 | | 8052 | | 2752 | | 1258 | | 16039 | | 593 | | 49887 | | 5884 | | 7119 | | 3011 | | 7819 | | 14710 | | 244 | | 6881 | | 16518 | | 240 | | 16445 | | 49988 | | 62181 | | 61227 | | 61651 | | 18118 | | 9951 | | 54964 | | 61197 | | 16361 | | 21452 | | 15361 | | 2840 | | 16479 | | 17006 | | 2768 | | 7917 | | 61198 | | 21556 | | 2650 | | 2525 | | 55022 | | 54911 | | 55672 | | 15326 | | 14763 | | 15493 | | 55156 | | 21435 | | 49959 | | 18242 | | 1887 | | 6437 | | 19407 | | 7593 | | 2355 | | 16908 | | 16102 | | 1275 | | 57868 | | 15943 | | 7977 | | 16079 | | 14712 | | 49874 | | 17269 | | 55851 | | 57783 | | 3442 | | 18085 | | 50913 | | 4067 | | 19402 | | 2785 | | 7920 | | 16456 | | 19143 | | 15141 | | 54895 | | 3578 | | 2905 | | 16623 | | 16802 | | 6472 | | 5894 | | 7004 | | 14711 | | 2545 | | 19184 | | 2994 | | 3297 | | 7993 | | 10034 | | 51738 | | 16854 | | 7007 | | 7583 | | 15967 | | 3720 | | 54940 | | 15299 | | 2343 | | 9896 | | 7016 | | 19668 | | 16128 | | 21624 | | 7529 | | 3521 | | 16602 | | 16089 | | 54821 | | 51589 | | 61828 | | 19690 | | 3608 | | 17946 | | 61146 | | 2265 | | 9809 | | 2509 | | 54729 | | 2426 | | 21617 | | 7543 | | 1841 | | 3700 | | 21391 | | 21442 | | 55155 | | 7532 | | 3490 | | 7821 | | 1891 | | 9876 | | 3677 | | 10019 | | 2862 | | 16186 | | 16299 | | 16528 | | 50641 | | 5926 | | 55135 | | 16800 | | 54734 | | 19375 | +--------+ 256 rows in set (0.09 sec) mysql> So why not to look on index cardinality, limit size (let say, less than 1% of table), min or max value of field ordered by and then exclude big part of table to avoid unnecessary work? Tõnu

1 0

[Maria-developers] Failures in mysql_client_test
by Kristian Nielsen 24 Apr '09

24 Apr '09

Hi Igor, I have been trying to figure out the problems you see in the mysql_client_test test case in the MariaDB 5.1 tree. It has been tricky due to not being able to reproduce. Patrick Crews was able to reproduce though, and was helpful enough to file this bug with some additional info: https://bugs.launchpad.net/maria/+bug/363181 >From this information, I pushed a few minor fixes to the Launchpad tree (two missing DBUG_RETURN statements). After this push, Patrick can no longer reproduce the failure. So, while my fixes were done in the area of the code where the problem seems to occur, I don't understand why two missing DBUG_RETURN would cause the error we saw... Maybe you can try again to see if you can reproduce the bug in the latest tree? If you cannot, then we may have to consider the problem fixed for now. If you can still reproduce, it would be very helpful if you could reproduce it with dbug enabled: perl mysql-test-run.pl --debug mysql_client_test and collect all the *.trace and *.err files from mysql-test/var/ and attach them to the bug or email them. It would be good to get progress on this annoying problem. - Kristian.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2697)
by knielsen＠knielsen-hq.org 23 Apr '09

23 Apr '09

#At lp:maria 2697 knielsen(a)knielsen-hq.org 2009-04-23 Bump maria version following release. modified: configure.in per-file messages: configure.in Bump maria version following release. === modified file 'configure.in' --- a/configure.in 2009-03-22 12:16:09 +0000 +++ b/configure.in 2009-04-23 13:06:16 +0000 @@ -10,7 +10,7 @@ AC_CANONICAL_SYSTEM # # When changing major version number please also check switch statement # in mysqlbinlog::check_master_version(). -AM_INIT_AUTOMAKE(mysql, 5.1.32-maria-beta) +AM_INIT_AUTOMAKE(mysql, 5.1.32-maria-beta2) AM_CONFIG_HEADER([include/config.h:config.h.in]) PROTOCOL_VERSION=10

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2696)
by knielsen＠knielsen-hq.org 21 Apr '09

21 Apr '09

#At lp:maria 2696 knielsen(a)knielsen-hq.org 2009-04-21 [merge] merged from trunk. modified: sql/mysqld.cc sql/sql_string.cc sql/strfunc.cc === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-04-08 16:55:26 +0000 +++ b/sql/mysqld.cc 2009-04-08 21:16:10 +0000 @@ -4975,11 +4975,11 @@ inline void kill_broken_server() #if !defined(__NETWARE__) unix_sock == INVALID_SOCKET || #endif - (!opt_disable_networking && - (base_ip_sock == INVALID_SOCKET || extra_ip_sock != INVALID_SOCKET))) + (!opt_disable_networking && base_ip_sock == INVALID_SOCKET)) { select_thread_in_use = 0; /* The following call will never return */ + DBUG_PRINT("general", ("killing server because socket is closed")); kill_server(IF_NETWARE(MYSQL_KILL_SIGNAL, (void*) MYSQL_KILL_SIGNAL)); } } === modified file 'sql/sql_string.cc' --- a/sql/sql_string.cc 2008-02-29 13:56:50 +0000 +++ b/sql/sql_string.cc 2009-04-08 20:08:38 +0000 @@ -65,10 +65,10 @@ bool String::real_alloc(uint32 arg_lengt bool String::realloc(uint32 alloc_length) { - uint32 len=ALIGN_SIZE(alloc_length+1); - if (Alloced_length < len) + if (Alloced_length <= alloc_length) { char *new_ptr; + uint32 len= ALIGN_SIZE(alloc_length+1); if (alloced) { if ((new_ptr= (char*) my_realloc(Ptr,len,MYF(MY_WME)))) === modified file 'sql/strfunc.cc' --- a/sql/strfunc.cc 2007-08-13 13:11:25 +0000 +++ b/sql/strfunc.cc 2009-04-08 20:08:38 +0000 @@ -38,7 +38,8 @@ static const char field_separator=','; -ulonglong find_set(TYPELIB *lib, const char *str, uint length, CHARSET_INFO *cs, +ulonglong find_set(TYPELIB *lib, const char *str, uint length, + CHARSET_INFO *cs, char **err_pos, uint *err_len, bool *set_warning) { CHARSET_INFO *strip= cs ? cs : &my_charset_latin1;

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2695)
by knielsen＠knielsen-hq.org 21 Apr '09

21 Apr '09

#At lp:maria 2695 knielsen(a)knielsen-hq.org 2009-04-21 Fix two missing DBUG_RETURN. modified: libmysql/libmysql.c tests/mysql_client_test.c === modified file 'libmysql/libmysql.c' --- a/libmysql/libmysql.c 2008-12-10 09:02:25 +0000 +++ b/libmysql/libmysql.c 2009-04-21 10:24:51 +0000 @@ -4634,7 +4634,7 @@ int STDCALL mysql_stmt_fetch_column(MYSQ if ((int) stmt->state < (int) MYSQL_STMT_FETCH_DONE) { set_stmt_error(stmt, CR_NO_DATA, unknown_sqlstate, NULL); - return 1; + DBUG_RETURN(1); } if (column >= stmt->field_count) { === modified file 'tests/mysql_client_test.c' --- a/tests/mysql_client_test.c 2009-02-19 09:01:25 +0000 +++ b/tests/mysql_client_test.c 2009-04-21 10:24:51 +0000 @@ -16295,6 +16295,8 @@ static void test_bug27876() rc= mysql_query(mysql, "set names default"); myquery(rc); + + DBUG_VOID_RETURN; }

1 0

[Maria-developers] MariaDB source and binaries tarballs made available
by Kristian Nielsen 20 Apr '09

20 Apr '09

As the MySQL Users's Conference is starting up, Monty and I wanted to have tarballs with source and binaries available to make it easier for interested people to try it out. The plan for future binaries is to have Ourdelta publish them from MariaDB sources, however due to lack of time this was not possible to set up before the conference. So to at least have something, I have built a source tarball from the latest bzr repository of MariaDB 5.1, as well as generic binary packages for 32- and 64-bit x86 Linux. I made the tarballs available here: http://askmonty.org/wiki/index.php/MariaDB:Download I took some care to get the build options correct and to check that everything works ok. However, I cannot check everything; notably I do not have access to anything but 32- and 64-bit x86 Ubuntu 8.04 systems to test on. I checked that the test suite passes, that the major storage engines work (MyISAM, Maria, InnoDB, PBXT), that the major features are there (partitioning, ssl, pool-of-threads), and that the binary tarballs can be installed and run. Any additional help with testing the packages for possible problems with be much appreciated, of course. The packages are currently hosted on a 5Mbit uplink I believe, so I will next work to get mirroring set up with the help of those in the community that have already offered to help with this. - Kristian.

1 0

Re: [Maria-developers] Maria binary builds
by Kristian Nielsen 19 Apr '09

19 Apr '09

Michael Widenius <michael.widenius(a)gmail.com> writes: > Hi! > >>>>>> "Kristian" == Kristian Nielsen <knielsen(a)knielsen-hq.org> writes: > > <cut> > > Kristian> I am still not sure whether generic (.tar.gz) binaries should be build > Kristian> statically (and to what extend). > > The benefit of building them staticly are: Unfortunately, the issue is more complex than just this. A fully static binary will not work at all in a number of scenarios. A fully static binary is obtained by linking everything statically, including libc. However, linking libc statically creates a number of problems if mysqld tries to load additional .so files when it is run. The additional .so files then use a different version of libc than the one mysqld code uses, which does not work. This gives problem loading plugins (like UDFs), and on many systems causes problems even without loading any plugins, as libc loads .so files itself (notably for NSS, domain name resolution). There are a number of bugs on this on bugs.mysql.com. Currently, I'm linking libc dynamically to avoid these problems (I believe/hope that glibc binary compatibility has become fairly good by now). I'm linking libz statically (bundled version) as well as libgcc as this shouldn't cause similar problems. And compiling with `gcc` instead of `g++` to avoid linking libstc++ at all. All mariadb code is linked statically as well, of course. I checked, and this seems to be the same that is done in the Sun/MySQL generic binaries these days. - Kristian.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (monty:2680)
by Michael Widenius 14 Apr '09

14 Apr '09

#At lp:maria based on revid:monty@askmonty.org-20090309141344-wriztb0fcn00syim 2680 Michael Widenius 2009-03-13 Added "pool-of-threads" handling (with libevent) This is a backport of code from MySQL 6.0 with cleanups and extensions The following new options are supported configure options: --with-libevent ; Enable use of libevent, which is needed for pool of threads mysqld options: --thread-handling=pool-of-threads ; Use a pool of threads to handle queries --thread-pool-size=# ; Define how many threads should be created to handle all queries --extra-port=# ; Extra tcp port that uses the old one-thread-per-connection method --extra-max-connections=# ; Number of connections to accept to 'extra-port' --test-ignore-wrong-options ; Ignore setting an enum value to a wrong option (for mysql-test-run) added: config/ac-macros/libevent.m4 config/ac-macros/libevent_configure.m4 extra/libevent/ extra/libevent/CMakeLists.txt extra/libevent/Makefile.am extra/libevent/README extra/libevent/WIN32-Code/ extra/libevent/WIN32-Code/config.h extra/libevent/WIN32-Code/misc.c extra/libevent/WIN32-Code/misc.h extra/libevent/WIN32-Code/tree.h extra/libevent/WIN32-Code/win32.c extra/libevent/buffer.c extra/libevent/compat/ extra/libevent/compat/sys/ extra/libevent/compat/sys/_time.h extra/libevent/compat/sys/queue.h extra/libevent/compat/sys/tree.h extra/libevent/devpoll.c extra/libevent/epoll.c extra/libevent/epoll_sub.c extra/libevent/evbuffer.c extra/libevent/evdns.c extra/libevent/evdns.h extra/libevent/event-config.h extra/libevent/event-internal.h extra/libevent/event.c extra/libevent/event.h extra/libevent/event_tagging.c extra/libevent/evhttp.h extra/libevent/evport.c extra/libevent/evrpc-internal.h extra/libevent/evrpc.c extra/libevent/evrpc.h extra/libevent/evsignal.h extra/libevent/evutil.c extra/libevent/evutil.h extra/libevent/http-internal.h extra/libevent/http.c extra/libevent/kqueue.c extra/libevent/log.c extra/libevent/log.h extra/libevent/min_heap.h extra/libevent/poll.c extra/libevent/select.c extra/libevent/signal.c extra/libevent/strlcpy-internal.h extra/libevent/strlcpy.c mysql-test/include/have_pool_of_threads.inc mysql-test/r/have_pool_of_threads.require mysql-test/r/pool_of_threads.result mysql-test/t/pool_of_threads-master.opt mysql-test/t/pool_of_threads.test modified: BUILD/SETUP.sh CMakeLists.txt Makefile.am configure.in dbug/dbug.c extra/Makefile.am include/config-win.h include/my_dbug.h include/mysql.h.pp include/typelib.h include/violite.h libmysqld/Makefile.am mysql-test/mysql-test-run.pl mysql-test/r/crash_commit_before.result mysql-test/r/subselect_debug.result mysql-test/t/crash_commit_before.test mysql-test/t/merge-big.test mysql-test/t/subselect_debug.test mysys/typelib.c sql/Makefile.am sql/handler.cc sql/mysql_priv.h sql/mysqld.cc sql/scheduler.cc sql/scheduler.h sql/set_var.cc sql/sql_class.cc sql/sql_class.h sql/sql_connect.cc vio/viosocket.c per-file messages: BUILD/SETUP.sh Added libevents (and thus pool-of-threads) to max builds CMakeLists.txt Added libevent Makefile.am Added libevents config/ac-macros/libevent.m4 Libevent code for configure config/ac-macros/libevent_configure.m4 Libevent code for configure configure.in Added libevents dbug/dbug.c Added _db_is_pushed(); Needed for pool-of-threads code extra/Makefile.am Added libevents extra/libevent Libevent initial code extra/libevent/CMakeLists.txt Libevent initial code extra/libevent/Makefile.am Libevent initial code extra/libevent/README Libevent initial code extra/libevent/WIN32-Code Libevent initial code extra/libevent/WIN32-Code/config.h Libevent initial code extra/libevent/WIN32-Code/misc.c Libevent initial code extra/libevent/WIN32-Code/misc.h Libevent initial code extra/libevent/WIN32-Code/tree.h Libevent initial code extra/libevent/WIN32-Code/win32.c Libevent initial code extra/libevent/buffer.c Libevent initial code extra/libevent/compat Libevent initial code extra/libevent/compat/sys Libevent initial code extra/libevent/compat/sys/_time.h Libevent initial code extra/libevent/compat/sys/queue.h Libevent initial code extra/libevent/compat/sys/tree.h Libevent initial code extra/libevent/devpoll.c Libevent initial code extra/libevent/epoll.c Libevent initial code extra/libevent/epoll_sub.c Libevent initial code extra/libevent/evbuffer.c Libevent initial code extra/libevent/evdns.c Libevent initial code extra/libevent/evdns.h Libevent initial code extra/libevent/event-config.h Libevent initial code extra/libevent/event-internal.h Libevent initial code extra/libevent/event.c Libevent initial code extra/libevent/event.h Libevent initial code extra/libevent/event_tagging.c Libevent initial code extra/libevent/evhttp.h Libevent initial code extra/libevent/evport.c Libevent initial code extra/libevent/evrpc-internal.h Libevent initial code extra/libevent/evrpc.c Libevent initial code extra/libevent/evrpc.h Libevent initial code extra/libevent/evsignal.h Libevent initial code extra/libevent/evutil.c Libevent initial code extra/libevent/evutil.h Libevent initial code extra/libevent/http-internal.h Libevent initial code extra/libevent/http.c Libevent initial code extra/libevent/kqueue.c Libevent initial code extra/libevent/log.c Libevent initial code extra/libevent/log.h Libevent initial code extra/libevent/min_heap.h Libevent initial code extra/libevent/poll.c Libevent initial code extra/libevent/select.c Libevent initial code extra/libevent/signal.c Libevent initial code extra/libevent/strlcpy-internal.h Libevent initial code extra/libevent/strlcpy.c Libevent initial code include/config-win.h Libevent support include/my_dbug.h ADded _db_is_pushed include/mysql.h.pp Update to handle new prototypes include/typelib.h Split find_type_or_exit() into two functions include/violite.h Added vio_is_pending() libmysqld/Makefile.am Added libevent mysql-test/include/have_pool_of_threads.inc Added test for pool-of-threads mysql-test/mysql-test-run.pl Don't abort based on time and don't retry test cases when run under --gdb or --debug mysql-test/r/crash_commit_before.result USE GLOBAL for debug variable mysql-test/r/have_pool_of_threads.require Added test for pool-of-threads mysql-test/r/pool_of_threads.result Added test for pool-of-threads mysql-test/r/subselect_debug.result USE GLOBAL for debug variable mysql-test/t/crash_commit_before.test USE GLOBAL for debug variable mysql-test/t/merge-big.test USE GLOBAL for debug variable mysql-test/t/pool_of_threads-master.opt Added test for pool-of-threads mysql-test/t/pool_of_threads.test Added test for pool-of-threads mysys/typelib.c Split find_type_or_exit() into find_type_with_warning() sql/Makefile.am Added libevent sql/handler.cc Indentation fix. Fixed memory loss bug Fixed crash on exit when handler plugin failed sql/mysql_priv.h Added extra_max_connections and mysqld_extra_port Added extern functions from sql_connect.cc sql/mysqld.cc Added support for new mysqld options Added code for 'extra-port' and 'extra-max-connections' Split some functions into smaller pieces to be able to reuse code Added code for test-ignore-wrong-options sql/scheduler.cc Updated schduler code from MySQL 6.0 sql/scheduler.h Updated schduler code from MySQL 6.0 sql/set_var.cc Added support for changing "extra_max_connections" sql/sql_class.cc Iniitalize thread schduler options in THD sql/sql_class.h Added to extra_port and scheduler to 'THD' sql/sql_connect.cc Use thd->schduler to check number of connections and terminate connection Made some local functions global (for scheduler.cc) vio/viosocket.c Added 'vio_pending', needed for scheduler..c === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-01-31 21:22:44 +0000 +++ b/BUILD/SETUP.sh 2009-03-12 22:27:35 +0000 @@ -166,8 +166,8 @@ local_infile_configs="--enable-local-inf max_no_embedded_configs="$SSL_LIBRARY --with-plugins=max" -max_no_ndb_configs="$SSL_LIBRARY --with-plugins=max-no-ndb --with-embedded-server" -max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server" +max_no_ndb_configs="$SSL_LIBRARY --with-plugins=max-no-ndb --with-embedded-server --with-libevent" +max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server -with-libevent" # Disable NDB in maria max builds max_configs=$max_no_ndb_configs === modified file 'CMakeLists.txt' --- a/CMakeLists.txt 2009-02-19 09:01:25 +0000 +++ b/CMakeLists.txt 2009-03-12 22:27:35 +0000 @@ -235,6 +235,7 @@ ADD_SUBDIRECTORY(scripts) ADD_SUBDIRECTORY(zlib) ADD_SUBDIRECTORY(extra/yassl) ADD_SUBDIRECTORY(extra/yassl/taocrypt) +ADD_SUBDIRECTORY(extra/libevent) ADD_SUBDIRECTORY(extra) ADD_SUBDIRECTORY(storage/heap) ADD_SUBDIRECTORY(storage/myisam) === modified file 'Makefile.am' --- a/Makefile.am 2009-02-19 09:01:25 +0000 +++ b/Makefile.am 2009-03-12 22:27:35 +0000 @@ -19,7 +19,9 @@ AUTOMAKE_OPTIONS = foreign # These are built from source in the Docs directory EXTRA_DIST = INSTALL-SOURCE INSTALL-WIN-SOURCE \ - README COPYING EXCEPTIONS-CLIENT CMakeLists.txt + README COPYING EXCEPTIONS-CLIENT \ + CMakeLists.txt \ + config/ac-macros/libevent_configure.m4 SUBDIRS = . include @docs_dirs@ @zlib_dir@ \ @readline_topdir@ sql-common scripts \ @@ -51,7 +53,6 @@ bin-dist: all # Remove BK's "SCCS" subdirectories from source distribution # Create initial database files for Windows installations and check them. dist-hook: - rm -rf `find $(distdir) -type d -name SCCS -print` mkdir -p $(distdir)/win scripts/mysql_install_db --no-defaults --cross-bootstrap \ --builddir=$(top_builddir) \ @@ -97,7 +98,7 @@ test-nr: test-pr: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) $(mem) --ps-protocol --mysqld=--binlog-format=row + @PERL@ ./mysql-test-run.pl $(force) $(mem) --ps-protocol --mysqld=--binlog-format=row #@libevent_test_option@ test-ns: cd mysql-test ; \ === added file 'config/ac-macros/libevent.m4' --- a/config/ac-macros/libevent.m4 1970-01-01 00:00:00 +0000 +++ b/config/ac-macros/libevent.m4 2009-03-12 22:27:35 +0000 @@ -0,0 +1,55 @@ +dnl --------------------------------------------------------------------------- +dnl Macro: MYSQL_USE_BUNDLED_LIBEVENT +dnl +dnl SYNOPSIS +dnl MYSQL_USE_BUNDLED_LIBEVENT() +dnl +dnl DESCRIPTION +dnl Add defines so libevent is built and linked with +dnl --------------------------------------------------------------------------- +AC_DEFUN([MYSQL_USE_BUNDLED_LIBEVENT], [ + + libevent_dir="libevent" + AC_SUBST([libevent_dir]) + + libevent_libs="\$(top_builddir)/extra/libevent/libevent.a" + libevent_includes="-I\$(top_builddir)/extra/libevent" + libevent_test_option="--mysqld=--thread-handling=pool-of-threads" + AC_SUBST(libevent_libs) + AC_SUBST(libevent_includes) + AC_SUBST(libevent_test_option) + + AC_DEFINE([HAVE_LIBEVENT], [1], [If we want to use libevent and have connection pooling]) + AC_MSG_RESULT([using bundled libevent]) + + dnl Get the upstream file with the original libevent configure macros. + dnl Use builtin include for this, to work around path problems in old versions of aclocal. + builtin([include],[config/ac-macros/libevent_configure.m4]) +]) + + +dnl ------------------------------------------------------------------------ +dnl Macro: MYSQL_CHECK_LIBEVENT +dnl +dnl SYNOPSIS +dnl MYSQL_CHECK_LIBEVENT +dnl +dnl ------------------------------------------------------------------------ +AC_DEFUN([MYSQL_CHECK_LIBEVENT], [ + + AC_CONFIG_FILES(extra/libevent/Makefile) + + AC_MSG_CHECKING(for libevent) + AC_ARG_WITH([libevent], + [ --with-libevent use libevent and have connection pooling], + [with_libevent=$withval], + [with_libevent=no] + ) + + if test "$with_libevent" != "no"; then + MYSQL_USE_BUNDLED_LIBEVENT + else + AC_MSG_RESULT([disabled]) + fi + AM_CONDITIONAL([HAVE_LIBEVENT], [ test "$with_libevent" != "no" ]) +]) === added file 'config/ac-macros/libevent_configure.m4' --- a/config/ac-macros/libevent_configure.m4 1970-01-01 00:00:00 +0000 +++ b/config/ac-macros/libevent_configure.m4 2009-03-12 22:27:35 +0000 @@ -0,0 +1,324 @@ +dnl Checks for libraries. +AC_CHECK_LIB(socket, socket) +AC_CHECK_LIB(resolv, inet_aton) +AC_CHECK_LIB(rt, clock_gettime) +AC_CHECK_LIB(nsl, inet_ntoa) + +dnl Checks for header files. +AC_HEADER_STDC +AC_CHECK_HEADERS(fcntl.h stdarg.h inttypes.h stdint.h poll.h signal.h unistd.h sys/epoll.h sys/time.h sys/queue.h sys/event.h sys/param.h sys/ioctl.h sys/select.h sys/devpoll.h port.h netinet/in6.h sys/socket.h) +if test "x$ac_cv_header_sys_queue_h" = "xyes"; then + AC_MSG_CHECKING(for TAILQ_FOREACH in sys/queue.h) + AC_EGREP_CPP(yes, +[ +#include <sys/queue.h> +#ifdef TAILQ_FOREACH + yes +#endif +], [AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_TAILQFOREACH, 1, + [Define if TAILQ_FOREACH is defined in <sys/queue.h>])], + [AC_MSG_RESULT(no)] + ) +fi + +if test "x$ac_cv_header_sys_time_h" = "xyes"; then + AC_MSG_CHECKING(for timeradd in sys/time.h) + AC_EGREP_CPP(yes, +[ +#include <sys/time.h> +#ifdef timeradd + yes +#endif +], [ AC_DEFINE(HAVE_TIMERADD, 1, + [Define if timeradd is defined in <sys/time.h>]) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)] +) +fi + +if test "x$ac_cv_header_sys_time_h" = "xyes"; then + AC_MSG_CHECKING(for timercmp in sys/time.h) + AC_EGREP_CPP(yes, +[ +#include <sys/time.h> +#ifdef timercmp + yes +#endif +], [ AC_DEFINE(HAVE_TIMERCMP, 1, + [Define if timercmp is defined in <sys/time.h>]) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)] +) +fi + +if test "x$ac_cv_header_sys_time_h" = "xyes"; then + AC_MSG_CHECKING(for timerclear in sys/time.h) + AC_EGREP_CPP(yes, +[ +#include <sys/time.h> +#ifdef timerclear + yes +#endif +], [ AC_DEFINE(HAVE_TIMERCLEAR, 1, + [Define if timerclear is defined in <sys/time.h>]) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)] +) +fi + +if test "x$ac_cv_header_sys_time_h" = "xyes"; then + AC_MSG_CHECKING(for timerisset in sys/time.h) + AC_EGREP_CPP(yes, +[ +#include <sys/time.h> +#ifdef timerisset + yes +#endif +], [ AC_DEFINE(HAVE_TIMERISSET, 1, + [Define if timerisset is defined in <sys/time.h>]) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)] +) +fi + +dnl Checks for typedefs, structures, and compiler characteristics. +AC_C_CONST +AC_C_INLINE +AC_HEADER_TIME + +dnl Checks for library functions. +AC_CHECK_FUNCS(gettimeofday vasprintf fcntl clock_gettime strtok_r strsep getaddrinfo getnameinfo strlcpy inet_ntop signal sigaction strtoll) + +AC_CHECK_SIZEOF(long) + +if test "x$ac_cv_func_clock_gettime" = "xyes"; then + AC_DEFINE(DNS_USE_CPU_CLOCK_FOR_ID, 1, [Define if clock_gettime is available in libc]) +else + AC_DEFINE(DNS_USE_GETTIMEOFDAY_FOR_ID, 1, [Define is no secure id variant is available]) +fi + +AC_MSG_CHECKING(for F_SETFD in fcntl.h) +AC_EGREP_CPP(yes, +[ +#define _GNU_SOURCE +#include <fcntl.h> +#ifdef F_SETFD +yes +#endif +], [ AC_DEFINE(HAVE_SETFD, 1, + [Define if F_SETFD is defined in <fcntl.h>]) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no)] + ) + +needsignal=no +haveselect=no +AC_CHECK_FUNCS(select, [haveselect=yes]) +if test "x$haveselect" = "xyes" ; then + AC_LIBOBJ(select) + needsignal=yes +fi + +havepoll=no +AC_CHECK_FUNCS(poll, [havepoll=yes]) +if test "x$havepoll" = "xyes" ; then + AC_LIBOBJ(poll) + needsignal=yes +fi + +haveepoll=no +AC_CHECK_FUNCS(epoll_ctl, [haveepoll=yes]) +if test "x$haveepoll" = "xyes" ; then + AC_DEFINE(HAVE_EPOLL, 1, + [Define if your system supports the epoll system calls]) + AC_LIBOBJ(epoll) + needsignal=yes +fi + +havedevpoll=no +if test "x$ac_cv_header_sys_devpoll_h" = "xyes"; then + AC_DEFINE(HAVE_DEVPOLL, 1, + [Define if /dev/poll is available]) + AC_LIBOBJ(devpoll) +fi + +havekqueue=no +if test "x$ac_cv_header_sys_event_h" = "xyes"; then + AC_CHECK_FUNCS(kqueue, [havekqueue=yes]) + if test "x$havekqueue" = "xyes" ; then + AC_MSG_CHECKING(for working kqueue) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ +#include <sys/types.h> +#include <sys/time.h> +#include <sys/event.h> +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> + ]], + [[ + int kq; + int n; + int fd[2]; + struct kevent ev; + struct timespec ts; + char buf[8000]; + + if (pipe(fd) == -1) + exit(1); + if (fcntl(fd[1], F_SETFL, O_NONBLOCK) == -1) + exit(1); + + while ((n = write(fd[1], buf, sizeof(buf))) == sizeof(buf)) + ; + + if ((kq = kqueue()) == -1) + exit(1); + + ev.ident = fd[1]; + ev.filter = EVFILT_WRITE; + ev.flags = EV_ADD | EV_ENABLE; + n = kevent(kq, &ev, 1, NULL, 0, NULL); + if (n == -1) + exit(1); + + read(fd[0], buf, sizeof(buf)); + + ts.tv_sec = 0; + ts.tv_nsec = 0; + n = kevent(kq, NULL, 0, &ev, 1, &ts); + if (n == -1 || n == 0) + exit(1); + + exit(0); + ]] + )], + [AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_WORKING_KQUEUE, 1, + [Define if kqueue works correctly with pipes]) + AC_LIBOBJ(kqueue)], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(no)] + ) + fi +fi + +haveepollsyscall=no +if test "x$ac_cv_header_sys_epoll_h" = "xyes"; then + if test "x$haveepoll" = "xno" ; then + AC_MSG_CHECKING(for epoll system call) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ +#include <stdint.h> +#include <sys/param.h> +#include <sys/types.h> +#include <sys/syscall.h> +#include <sys/epoll.h> +#include <unistd.h> + +int +epoll_create(int size) +{ + return (syscall(__NR_epoll_create, size)); +} + ]], + [[ + int epfd; + + epfd = epoll_create(256); + exit (epfd == -1 ? 1 : 0); + ]] + )], + [AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_EPOLL, 1, + [Define if your system supports the epoll system calls]) + needsignal=yes + AC_LIBOBJ(epoll_sub) + AC_LIBOBJ(epoll)], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(no)] + ) + fi +fi + +haveeventports=no +AC_CHECK_FUNCS(port_create, [haveeventports=yes]) +if test "x$haveeventports" = "xyes" ; then + AC_DEFINE(HAVE_EVENT_PORTS, 1, + [Define if your system supports event ports]) + AC_LIBOBJ(evport) + needsignal=yes +fi +if test "x$bwin32" = "xtrue"; then + needsignal=yes +fi +if test "x$bwin32" = "xtrue"; then + needsignal=yes +fi +if test "x$needsignal" = "xyes" ; then + AC_LIBOBJ(signal) +fi + +AC_TYPE_PID_T +AC_TYPE_SIZE_T +AC_CHECK_TYPES([uint64_t, uint32_t, uint16_t, uint8_t],[],[], +[[#ifdef HAVE_STDINT_H +#include <stdint.h> +#elif defined(HAVE_INTTYPES_H) +#include <inttypes.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif]]) +AC_CHECK_SIZEOF(long long) +AC_CHECK_SIZEOF(long) +AC_CHECK_SIZEOF(int) +AC_CHECK_SIZEOF(short) +AC_CHECK_TYPES([struct in6_addr],[],[], +[[#ifdef WIN32 +#include <winsock2.h> +#else +#include <sys/types.h> +#include <netinet/in.h> +#include <sys/socket.h> +#endif +#ifdef HAVE_NETINET_IN6_H +#include <netinet/in6.h> +#endif]]) + +AC_MSG_CHECKING([for socklen_t]) +AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include <sys/types.h> + #include <sys/socket.h> + ]], + [[ + socklen_t x; + ]] + )], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_DEFINE(socklen_t, unsigned int, + [Define to unsigned int if you dont have it])] +) + +AC_MSG_CHECKING([whether our compiler supports __func__]) +AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([],[[const char *cp = __func__;]])], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_CHECKING([whether our compiler supports __FUNCTION__]) + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([],[[const char *cp = __FUNCTION__;]])], + [AC_MSG_RESULT([yes]) + AC_DEFINE(__func__, __FUNCTION__, + [Define to appropriate substitue if compiler doesnt have __func__])], + [AC_MSG_RESULT([no]) + AC_DEFINE(__func__, __FILE__, + [Define to appropriate substitue if compiler doesnt have __func__])] + )] +) === modified file 'configure.in' --- a/configure.in 2009-03-05 22:25:54 +0000 +++ b/configure.in 2009-03-12 22:27:35 +0000 @@ -55,6 +55,7 @@ sinclude(config/ac-macros/large_file.m4) sinclude(config/ac-macros/misc.m4) sinclude(config/ac-macros/readline.m4) sinclude(config/ac-macros/ssl.m4) +sinclude(config/ac-macros/libevent.m4) sinclude(config/ac-macros/zlib.m4) # Remember to add a directory sql/share/LANGUAGE @@ -2389,6 +2390,7 @@ MYSQL_CHECK_BIG_TABLES MYSQL_CHECK_MAX_INDEXES MYSQL_CHECK_VIO MYSQL_CHECK_SSL +MYSQL_CHECK_LIBEVENT #-------------------------------------------------------------------- # Declare our plugin modules === modified file 'dbug/dbug.c' --- a/dbug/dbug.c 2009-02-19 09:01:25 +0000 +++ b/dbug/dbug.c 2009-03-12 22:27:35 +0000 @@ -878,6 +878,18 @@ void _db_push_(const char *control) FixTraceFlags(old_fflags, cs); } +/** + Returns TRUE if session-local settings have been set. +*/ + +int _db_is_pushed_() +{ + CODE_STATE *cs= NULL; + get_code_state_or_return FALSE; + return (cs->stack != &init_settings); +} + + /* * FUNCTION * === modified file 'extra/Makefile.am' --- a/extra/Makefile.am 2008-11-21 14:21:50 +0000 +++ b/extra/Makefile.am 2009-03-12 22:27:35 +0000 @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2006 MySQL AB +# Copyright (C) 2000-2006 MySQL AB, 2009 Monty Program AB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,8 +24,8 @@ BUILT_SOURCES= $(top_builddir)/include/ pkginclude_HEADERS= $(BUILT_SOURCES) EXTRA_PROGRAMS = comp_err DISTCLEANFILES = $(BUILT_SOURCES) $(EXTRA_PROGRAMS) -SUBDIRS = @yassl_dir@ -DIST_SUBDIRS = yassl +SUBDIRS = @yassl_dir@ @libevent_dir@ +DIST_SUBDIRS = yassl libevent # This will build mysqld_error.h, mysqld_ername.h and sql_state.h # NOTE Built files should depend on their sources to avoid === added directory 'extra/libevent' === added file 'extra/libevent/CMakeLists.txt' --- a/extra/libevent/CMakeLists.txt 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/CMakeLists.txt 2009-03-12 22:27:35 +0000 @@ -0,0 +1,34 @@ +INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/extra/libevent + ${CMAKE_SOURCE_DIR}/extra/libevent/compat + ${CMAKE_SOURCE_DIR}/extra/libevent/WIN32-Code) + +IF(MSVC) + ADD_DEFINITIONS("-DWIN32 -DHAVE_CONFIG_H") +ENDIF(MSVC) + +SET(LIBEVENT_SOURCES + buffer.c + evbuffer.c + event.c + evutil.c + log.c + signal.c + strlcpy.c + WIN32-Code/win32.c + WIN32-Code/config.h + WIN32-Code/misc.c + WIN32-Code/misc.h + event-internal.h + event.h + evsignal.h + evutil.h + log.h + min_heap.h + strlcpy-internal.h + ) + +CONFIGURE_FILE(WIN32-Code/config.h ${CMAKE_SOURCE_DIR}/extra/libevent/event-config.h COPYONLY) +IF(NOT SOURCE_SUBLIBS) + ADD_LIBRARY(libevent ${LIBEVENT_SOURCES}) +ENDIF(NOT SOURCE_SUBLIBS) === added file 'extra/libevent/Makefile.am' --- a/extra/libevent/Makefile.am 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/Makefile.am 2009-03-12 22:27:35 +0000 @@ -0,0 +1,39 @@ +AUTOMAKE_OPTIONS = foreign no-dependencies + +EXTRA_DIST = README compat/sys \ + buffer.c epoll.c evbuffer.c event.c evport.c evutil.c kqueue.c poll.c \ + signal.c devpoll.c epoll_sub.c evdns.c event_tagging.c evrpc.c http.c \ + log.c select.c strlcpy.c \ + evdns.h event.h evrpc-internal.h evsignal.h http-internal.h log.h \ + min_heap.h event-internal.h evhttp.h evrpc.h evutil.h strlcpy-internal.h \ + WIN32-Code/misc.c WIN32-Code/misc.h WIN32-Code/config.h WIN32-Code/tree.h \ + WIN32-Code/win32.c CMakeLists.txt + + +DISTCLEANFILES = event-config.h + +noinst_LIBRARIES = libevent.a + +libevent_a_SOURCES = event.c buffer.c evbuffer.c log.c evutil.c \ + select.c poll.c epoll.c epoll_sub.c devpoll.c kqueue.c \ + evport.c signal.c + +include_HEADERS = event.h evutil.h event-config.h + +BUILT_SOURCES = event-config.h + +event-config.h: $(top_srcdir)/include/config.h + echo '/* event-config.h' > $@ + echo ' * Generated by autoconf; post-processed by libevent.' >> $@ + echo ' * Do not edit this file.' >> $@ + echo ' * Do not rely on macros in this file existing in later versions.'>> $@ + echo ' */' >> $@ + echo '#ifndef _EVENT_CONFIG_H_' >> $@ + echo '#define _EVENT_CONFIG_H_' >> $@ + + sed -e 's/#define /#define _EVENT_/' \ + -e 's/#undef /#undef _EVENT_/' \ + -e 's/#ifndef /#ifndef _EVENT_/' < $(top_srcdir)/include/config.h >> $@ + echo "#endif" >> $@ + +AM_CPPFLAGS = -Icompat -I$(top_srcdir)/include === added file 'extra/libevent/README' --- a/extra/libevent/README 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/README 2009-03-12 22:27:35 +0000 @@ -0,0 +1,57 @@ +To build libevent, type + +$ ./configure && make + + (If you got libevent from the subversion repository, you will + first need to run the included "autogen.sh" script in order to + generate the configure script.) + +Install as root via + +# make install + +You can run the regression tests by + +$ make verify + +Before, reporting any problems, please run the regression tests. + +To enable the low-level tracing build the library as: + +CFLAGS=-DUSE_DEBUG ./configure [...] + +Acknowledgements: +----------------- + +The following people have helped with suggestions, ideas, code or +fixing bugs: + + Alejo + Weston Andros Adamson + William Ahern + Stas Bekman + Andrew Danforth + Mike Davis + Shie Erlich + Alexander von Gernler + Artur Grabowski + Aaron Hopkins + Claudio Jeker + Scott Lamb + Adam Langley + Philip Lewis + David Libenzi + Nick Mathewson + Andrey Matveev + Richard Nyberg + Jon Oberheide + Phil Oleson + Dave Pacheco + Tassilo von Parseval + Pierre Phaneuf + Jon Poland + Bert JW Regeer + Dug Song + Taral + +If I have forgotten your name, please contact me. === added directory 'extra/libevent/WIN32-Code' === added file 'extra/libevent/WIN32-Code/config.h' --- a/extra/libevent/WIN32-Code/config.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/WIN32-Code/config.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,247 @@ +/* config.h. Generated by configure. */ +/* config.h.in. Generated from configure.in by autoheader. */ + +/* Define if clock_gettime is available in libc */ +/* #undef DNS_USE_CPU_CLOCK_FOR_ID */ + +/* Define if no secure id variant is available */ +#define DNS_USE_FTIME_FOR_ID 1 + +/* Define if no secure id variant is available */ +/* #define DNS_USE_GETTIMEOFDAY_FOR_ID 1 */ + +/* Define to 1 if you have the `clock_gettime' function. */ +/* #undef HAVE_CLOCK_GETTIME */ + +/* Define if /dev/poll is available */ +/* #undef HAVE_DEVPOLL */ + +/* Define to 1 if you have the <dlfcn.h> header file. */ +/* #undef HAVE_DLFCN_H */ + +/* Define if your system supports the epoll system calls */ +/* #undef HAVE_EPOLL */ + +/* Define to 1 if you have the `epoll_ctl' function. */ +/* #undef HAVE_EPOLL_CTL */ + +/* Define if your system supports event ports */ +/* #undef HAVE_EVENT_PORTS */ + +/* Define to 1 if you have the `fcntl' function. */ +/* #undef HAVE_FCNTL */ + +/* Define to 1 if you have the <fcntl.h> header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the `getaddrinfo' function. */ +/* #undef HAVE_GETADDRINFO */ + +/* Define to 1 if you have the `getnameinfo' function. */ +/* #undef HAVE_GETNAMEINFO */ + +/* Define to 1 if you have the `gettimeofday' function. */ +/* #define HAVE_GETTIMEOFDAY 1 */ + +/* Define to 1 if you have the `inet_ntop' function. */ +/* #undef HAVE_INET_NTOP */ + +/* Define to 1 if you have the <inttypes.h> header file. */ +/* #undef HAVE_INTTYPES_H 1 */ + +/* Define to 1 if you have the `kqueue' function. */ +/* #undef HAVE_KQUEUE */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `resolv' library (-lresolv). */ +/* #undef HAVE_LIBRESOLV */ + +/* Define to 1 if you have the `rt' library (-lrt). */ +/* #undef HAVE_LIBRT */ + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define to 1 if you have the <memory.h> header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the <netinet/in6.h> header file. */ +/* #undef HAVE_NETINET_IN6_H */ + +/* Define to 1 if you have the `poll' function. */ +/* #undef HAVE_POLL */ + +/* Define to 1 if you have the <poll.h> header file. */ +/* #undef HAVE_POLL_H */ + +/* Define to 1 if you have the `port_create' function. */ +/* #undef HAVE_PORT_CREATE */ + +/* Define to 1 if you have the <port.h> header file. */ +/* #undef HAVE_PORT_H */ + +/* Define to 1 if you have the `select' function. */ +/* #undef HAVE_SELECT */ + +/* Define if F_SETFD is defined in <fcntl.h> */ +/* #undef HAVE_SETFD */ + +/* Define to 1 if you have the `sigaction' function. */ +/* #undef HAVE_SIGACTION */ + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the <signal.h> header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if you have the <stdarg.h> header file. */ +#define HAVE_STDARG_H 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +/* #define HAVE_STDINT_H 1 */ + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the `strsep' function. */ +/* #undef HAVE_STRSEP */ + +/* Define to 1 if you have the `strtok_r' function. */ +/* #undef HAVE_STRTOK_R */ + +/* Define to 1 if the system has the type `struct in6_addr'. */ +#define HAVE_STRUCT_IN6_ADDR 1 + +/* Define to 1 if you have the <sys/devpoll.h> header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the <sys/epoll.h> header file. */ +/* #undef HAVE_SYS_EPOLL_H */ + +/* Define to 1 if you have the <sys/event.h> header file. */ +/* #undef HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the <sys/ioctl.h> header file. */ +/* #undef HAVE_SYS_IOCTL_H */ + +/* Define to 1 if you have the <sys/queue.h> header file. */ +/* #undef HAVE_SYS_QUEUE_H */ + +/* Define to 1 if you have the <sys/select.h> header file. */ +/* #undef HAVE_SYS_SELECT_H */ + +/* Define to 1 if you have the <sys/socket.h> header file. */ +/* #undef HAVE_SYS_SOCKET_H */ + +/* Define to 1 if you have the <sys/stat.h> header file. */ +/* #define HAVE_SYS_STAT_H 1 */ + +/* Define to 1 if you have the <sys/time.h> header file. */ +/* #define HAVE_SYS_TIME_H 1 */ + +/* Define to 1 if you have the <sys/types.h> header file. */ +/* #define HAVE_SYS_TYPES_H 1 */ + +/* Define if TAILQ_FOREACH is defined in <sys/queue.h> */ +/* #undef HAVE_TAILQFOREACH */ + +/* Define if timeradd is defined in <sys/time.h> */ +/* #undef HAVE_TIMERADD */ + +/* Define if timerclear is defined in <sys/time.h> */ +/* #define HAVE_TIMERCLEAR 1 */ + +/* Define if timercmp is defined in <sys/time.h> */ +#define HAVE_TIMERCMP 1 + +/* Define if timerisset is defined in <sys/time.h> */ +#define HAVE_TIMERISSET 1 + +/* Define to 1 if you have the <unistd.h> header file. */ +/* #define HAVE_UNISTD_H 1 */ + +/* Define to 1 if you have the `vasprintf' function. */ +/* #undef HAVE_VASPRINTF */ + +/* Define if kqueue works correctly with pipes */ +/* #undef HAVE_WORKING_KQUEUE */ + +/* Name of package */ +#ifndef PACKAGE +#define PACKAGE "libevent" +#endif + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ +#define TIME_WITH_SYS_TIME 1 + +/* Version number of package */ +#define VERSION "1.3.99-trunk" + +#ifndef __func__ +/* Define to appropriate substitue if compiler doesnt have __func__ */ +#if defined(_MSC_VER) && _MSC_VER < 1300 +#define __func__ "??" +#else +#define __func__ __FUNCTION__ +#endif +#endif + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +#define inline __inline +#endif + +/* Define to `int' if <sys/types.h> does not define. */ +/* #undef pid_t */ + +/* Define to `unsigned' if <sys/types.h> does not define. */ +/* #undef size_t */ + +/* Define to unsigned int if you dont have it */ +#define socklen_t unsigned int + +/* Define to `unsigned short' if <sys/types.h> does not define. */ +#define uint16_t unsigned short + +/* Define to `unsigned int' if <sys/types.h> does not define. */ +#define uint32_t unsigned int + +/* Define to `unsigned long long' if <sys/types.h> does not define. */ +#define uint64_t __uint64_t + +/* Define to `unsigned char' if <sys/types.h> does not define. */ +#define uint8_t unsigned char === added file 'extra/libevent/WIN32-Code/misc.c' --- a/extra/libevent/WIN32-Code/misc.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/WIN32-Code/misc.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,38 @@ +#include <stdio.h> +#include <string.h> +#include <windows.h> +#include <sys/timeb.h> +#include <time.h> + +#ifdef __GNUC__ +/*our prototypes for timeval and timezone are in here, just in case the above + headers don't have them*/ +#include "misc.h" +#endif + +/**************************************************************************** + * + * Function: gettimeofday(struct timeval *, struct timezone *) + * + * Purpose: Get current time of day. + * + * Arguments: tv => Place to store the curent time of day. + * tz => Ignored. + * + * Returns: 0 => Success. + * + ****************************************************************************/ + +#ifndef HAVE_GETTIMEOFDAY +int gettimeofday(struct timeval *tv, struct timezone *tz) { + struct _timeb tb; + + if(tv == NULL) + return -1; + + _ftime(&tb); + tv->tv_sec = (long) tb.time; + tv->tv_usec = ((int) tb.millitm) * 1000; + return 0; +} +#endif === added file 'extra/libevent/WIN32-Code/misc.h' --- a/extra/libevent/WIN32-Code/misc.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/WIN32-Code/misc.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,11 @@ +#ifndef MISC_H +#define MISC_H + +struct timezone; +struct timeval; + +#ifndef HAVE_GETTIMEOFDAY +int gettimeofday(struct timeval *,struct timezone *); +#endif + +#endif === added file 'extra/libevent/WIN32-Code/tree.h' --- a/extra/libevent/WIN32-Code/tree.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/WIN32-Code/tree.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,1354 @@ +/* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ +/* + * Copyright 2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_TREE_H_ +#define _SYS_TREE_H_ + +/* + * This file defines data structures for different types of trees: + * splay trees and red-black trees. + * + * A splay tree is a self-organizing data structure. Every operation + * on the tree causes a splay to happen. The splay moves the requested + * node to the root of the tree and partly rebalances it. + * + * This has the benefit that request locality causes faster lookups as + * the requested nodes move to the top of the tree. On the other hand, + * every lookup causes memory writes. + * + * The Balance Theorem bounds the total access time for m operations + * and n inserts on an initially empty tree as O((m + n)lg n). The + * amortized cost for a sequence of m accesses to a splay tree is O(lg n); + * + * A red-black tree is a binary search tree with the node color as an + * extra attribute. It fulfills a set of conditions: + * - every search path from the root to a leaf consists of the + * same number of black nodes, + * - each red node (except for the root) has a black parent, + * - each leaf node is black. + * + * Every operation on a red-black tree is bounded as O(lg n). + * The maximum height of a red-black tree is 2lg (n+1). + */ + +#define SPLAY_HEAD(name, type) \ +struct name { \ + struct type *sph_root; /* root of the tree */ \ +} + +#define SPLAY_INITIALIZER(root) \ + { NULL } + +#define SPLAY_INIT(root) do { \ + (root)->sph_root = NULL; \ +} while (0) + +#define SPLAY_ENTRY(type) \ +struct { \ + struct type *spe_left; /* left element */ \ + struct type *spe_right; /* right element */ \ +} + +#define SPLAY_LEFT(elm, field) (elm)->field.spe_left +#define SPLAY_RIGHT(elm, field) (elm)->field.spe_right +#define SPLAY_ROOT(head) (head)->sph_root +#define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) + +/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ +#define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_LINKLEFT(head, tmp, field) do { \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_LINKRIGHT(head, tmp, field) do { \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ + SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ + SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ +} while (0) + +/* Generates prototypes and inline functions */ + +#define SPLAY_PROTOTYPE(name, type, field, cmp) \ +void name##_SPLAY(struct name *, struct type *); \ +void name##_SPLAY_MINMAX(struct name *, int); \ +struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ +struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ + \ +/* Finds the node with the same key as elm */ \ +static __inline struct type * \ +name##_SPLAY_FIND(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) \ + return(NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) \ + return (head->sph_root); \ + return (NULL); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_NEXT(struct name *head, struct type *elm) \ +{ \ + name##_SPLAY(head, elm); \ + if (SPLAY_RIGHT(elm, field) != NULL) { \ + elm = SPLAY_RIGHT(elm, field); \ + while (SPLAY_LEFT(elm, field) != NULL) { \ + elm = SPLAY_LEFT(elm, field); \ + } \ + } else \ + elm = NULL; \ + return (elm); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_MIN_MAX(struct name *head, int val) \ +{ \ + name##_SPLAY_MINMAX(head, val); \ + return (SPLAY_ROOT(head)); \ +} + +/* Main splay operation. + * Moves node close to the key of elm to top + */ +#define SPLAY_GENERATE(name, type, field, cmp) \ +struct type * \ +name##_SPLAY_INSERT(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) { \ + SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ + } else { \ + int __comp; \ + name##_SPLAY(head, elm); \ + __comp = (cmp)(elm, (head)->sph_root); \ + if(__comp < 0) { \ + SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ + SPLAY_RIGHT(elm, field) = (head)->sph_root; \ + SPLAY_LEFT((head)->sph_root, field) = NULL; \ + } else if (__comp > 0) { \ + SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT(elm, field) = (head)->sph_root; \ + SPLAY_RIGHT((head)->sph_root, field) = NULL; \ + } else \ + return ((head)->sph_root); \ + } \ + (head)->sph_root = (elm); \ + return (NULL); \ +} \ + \ +struct type * \ +name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *__tmp; \ + if (SPLAY_EMPTY(head)) \ + return (NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) { \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ + } else { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ + name##_SPLAY(head, elm); \ + SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ + } \ + return (elm); \ + } \ + return (NULL); \ +} \ + \ +void \ +name##_SPLAY(struct name *head, struct type *elm) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ + int __comp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while ((__comp = (cmp)(elm, (head)->sph_root))) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) > 0){ \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} \ + \ +/* Splay with either the minimum or the maximum element \ + * Used to find minimum or maximum element in tree. \ + */ \ +void name##_SPLAY_MINMAX(struct name *head, int __comp) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while (1) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp > 0) { \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} + +#define SPLAY_NEGINF -1 +#define SPLAY_INF 1 + +#define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) +#define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) +#define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) +#define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) +#define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) +#define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) + +#define SPLAY_FOREACH(x, name, head) \ + for ((x) = SPLAY_MIN(name, head); \ + (x) != NULL; \ + (x) = SPLAY_NEXT(name, head, x)) + +/* Macros that define a red-back tree */ +#define RB_HEAD(name, type) \ +struct name { \ + struct type *rbh_root; /* root of the tree */ \ +} + +#define RB_INITIALIZER(root) \ + { NULL } + +#define RB_INIT(root) do { \ + (root)->rbh_root = NULL; \ +} while (0) + +#define RB_BLACK 0 +#define RB_RED 1 +#define RB_ENTRY(type) \ +struct { \ + struct type *rbe_left; /* left element */ \ + struct type *rbe_right; /* right element */ \ + struct type *rbe_parent; /* parent element */ \ + int rbe_color; /* node color */ \ +} + +#define RB_LEFT(elm, field) (elm)->field.rbe_left +#define RB_RIGHT(elm, field) (elm)->field.rbe_right +#define RB_PARENT(elm, field) (elm)->field.rbe_parent +#define RB_COLOR(elm, field) (elm)->field.rbe_color +#define RB_ROOT(head) (head)->rbh_root +#define RB_EMPTY(head) (RB_ROOT(head) == NULL) + +#define RB_SET(elm, parent, field) do { \ + RB_PARENT(elm, field) = parent; \ + RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ + RB_COLOR(elm, field) = RB_RED; \ +} while (0) + +#define RB_SET_BLACKRED(black, red, field) do { \ + RB_COLOR(black, field) = RB_BLACK; \ + RB_COLOR(red, field) = RB_RED; \ +} while (0) + +#ifndef RB_AUGMENT +#define RB_AUGMENT(x) +#endif + +#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ + (tmp) = RB_RIGHT(elm, field); \ + if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field))) { \ + RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_LEFT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ + (tmp) = RB_LEFT(elm, field); \ + if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field))) { \ + RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_RIGHT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +/* Generates prototypes and inline functions */ +#define RB_PROTOTYPE(name, type, field, cmp) \ +void name##_RB_INSERT_COLOR(struct name *, struct type *); \ +void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ +struct type *name##_RB_REMOVE(struct name *, struct type *); \ +struct type *name##_RB_INSERT(struct name *, struct type *); \ +struct type *name##_RB_FIND(struct name *, struct type *); \ +struct type *name##_RB_NEXT(struct type *); \ +struct type *name##_RB_MINMAX(struct name *, int); \ + \ + +/* Main rb operation. + * Moves node close to the key of elm to top + */ +#define RB_GENERATE(name, type, field, cmp) \ +void \ +name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +{ \ + struct type *parent, *gparent, *tmp; \ + while ((parent = RB_PARENT(elm, field)) && \ + RB_COLOR(parent, field) == RB_RED) { \ + gparent = RB_PARENT(parent, field); \ + if (parent == RB_LEFT(gparent, field)) { \ + tmp = RB_RIGHT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_RIGHT(parent, field) == elm) { \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_RIGHT(head, gparent, tmp, field); \ + } else { \ + tmp = RB_LEFT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_LEFT(parent, field) == elm) { \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_LEFT(head, gparent, tmp, field); \ + } \ + } \ + RB_COLOR(head->rbh_root, field) = RB_BLACK; \ +} \ + \ +void \ +name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ +{ \ + struct type *tmp; \ + while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ + elm != RB_ROOT(head)) { \ + if (RB_LEFT(parent, field) == elm) { \ + tmp = RB_RIGHT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ + struct type *oleft; \ + if ((oleft = RB_LEFT(tmp, field)))\ + RB_COLOR(oleft, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_RIGHT(head, tmp, oleft, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_RIGHT(tmp, field)) \ + RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } else { \ + tmp = RB_LEFT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ + struct type *oright; \ + if ((oright = RB_RIGHT(tmp, field)))\ + RB_COLOR(oright, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_LEFT(head, tmp, oright, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_LEFT(tmp, field)) \ + RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } \ + } \ + if (elm) \ + RB_COLOR(elm, field) = RB_BLACK; \ +} \ + \ +struct type * \ +name##_RB_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *child, *parent, *old = elm; \ + int color; \ + if (RB_LEFT(elm, field) == NULL) \ + child = RB_RIGHT(elm, field); \ + else if (RB_RIGHT(elm, field) == NULL) \ + child = RB_LEFT(elm, field); \ + else { \ + struct type *left; \ + elm = RB_RIGHT(elm, field); \ + while ((left = RB_LEFT(elm, field))) \ + elm = left; \ + child = RB_RIGHT(elm, field); \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ + if (RB_PARENT(elm, field) == old) \ + parent = elm; \ + (elm)->field = (old)->field; \ + if (RB_PARENT(old, field)) { \ + if (RB_LEFT(RB_PARENT(old, field), field) == old)\ + RB_LEFT(RB_PARENT(old, field), field) = elm;\ + else \ + RB_RIGHT(RB_PARENT(old, field), field) = elm;\ + RB_AUGMENT(RB_PARENT(old, field)); \ + } else \ + RB_ROOT(head) = elm; \ + RB_PARENT(RB_LEFT(old, field), field) = elm; \ + if (RB_RIGHT(old, field)) \ + RB_PARENT(RB_RIGHT(old, field), field) = elm; \ + if (parent) { \ + left = parent; \ + do { \ + RB_AUGMENT(left); \ + } while ((left = RB_PARENT(left, field))); \ + } \ + goto color; \ + } \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ +color: \ + if (color == RB_BLACK) \ + name##_RB_REMOVE_COLOR(head, parent, child); \ + return (old); \ +} \ + \ +/* Inserts a node into the RB tree */ \ +struct type * \ +name##_RB_INSERT(struct name *head, struct type *elm) \ +{ \ + struct type *tmp; \ + struct type *parent = NULL; \ + int comp = 0; \ + tmp = RB_ROOT(head); \ + while (tmp) { \ + parent = tmp; \ + comp = (cmp)(elm, parent); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + RB_SET(elm, parent, field); \ + if (parent != NULL) { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = elm; \ + name##_RB_INSERT_COLOR(head, elm); \ + return (NULL); \ +} \ + \ +/* Finds the node with the same key as elm */ \ +struct type * \ +name##_RB_FIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (NULL); \ +} \ + \ +struct type * \ +name##_RB_NEXT(struct type *elm) \ +{ \ + if (RB_RIGHT(elm, field)) { \ + elm = RB_RIGHT(elm, field); \ + while (RB_LEFT(elm, field)) \ + elm = RB_LEFT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +struct type * \ +name##_RB_MINMAX(struct name *head, int val) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *parent = NULL; \ + while (tmp) { \ + parent = tmp; \ + if (val < 0) \ + tmp = RB_LEFT(tmp, field); \ + else \ + tmp = RB_RIGHT(tmp, field); \ + } \ + return (parent); \ +} + +#define RB_NEGINF -1 +#define RB_INF 1 + +#define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) +#define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) +#define RB_FIND(name, x, y) name##_RB_FIND(x, y) +#define RB_NEXT(name, x, y) name##_RB_NEXT(y) +#define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) +#define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) + +#define RB_FOREACH(x, name, head) \ + for ((x) = RB_MIN(name, head); \ + (x) != NULL; \ + (x) = name##_RB_NEXT(x)) + +#endif /* _SYS_TREE_H_ */ +/* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ +/* + * Copyright 2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_TREE_H_ +#define _SYS_TREE_H_ + +/* + * This file defines data structures for different types of trees: + * splay trees and red-black trees. + * + * A splay tree is a self-organizing data structure. Every operation + * on the tree causes a splay to happen. The splay moves the requested + * node to the root of the tree and partly rebalances it. + * + * This has the benefit that request locality causes faster lookups as + * the requested nodes move to the top of the tree. On the other hand, + * every lookup causes memory writes. + * + * The Balance Theorem bounds the total access time for m operations + * and n inserts on an initially empty tree as O((m + n)lg n). The + * amortized cost for a sequence of m accesses to a splay tree is O(lg n); + * + * A red-black tree is a binary search tree with the node color as an + * extra attribute. It fulfills a set of conditions: + * - every search path from the root to a leaf consists of the + * same number of black nodes, + * - each red node (except for the root) has a black parent, + * - each leaf node is black. + * + * Every operation on a red-black tree is bounded as O(lg n). + * The maximum height of a red-black tree is 2lg (n+1). + */ + +#define SPLAY_HEAD(name, type) \ +struct name { \ + struct type *sph_root; /* root of the tree */ \ +} + +#define SPLAY_INITIALIZER(root) \ + { NULL } + +#define SPLAY_INIT(root) do { \ + (root)->sph_root = NULL; \ +} while (0) + +#define SPLAY_ENTRY(type) \ +struct { \ + struct type *spe_left; /* left element */ \ + struct type *spe_right; /* right element */ \ +} + +#define SPLAY_LEFT(elm, field) (elm)->field.spe_left +#define SPLAY_RIGHT(elm, field) (elm)->field.spe_right +#define SPLAY_ROOT(head) (head)->sph_root +#define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) + +/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ +#define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_LINKLEFT(head, tmp, field) do { \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_LINKRIGHT(head, tmp, field) do { \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ + SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ + SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ +} while (0) + +/* Generates prototypes and inline functions */ + +#define SPLAY_PROTOTYPE(name, type, field, cmp) \ +void name##_SPLAY(struct name *, struct type *); \ +void name##_SPLAY_MINMAX(struct name *, int); \ +struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ +struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ + \ +/* Finds the node with the same key as elm */ \ +static __inline struct type * \ +name##_SPLAY_FIND(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) \ + return(NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) \ + return (head->sph_root); \ + return (NULL); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_NEXT(struct name *head, struct type *elm) \ +{ \ + name##_SPLAY(head, elm); \ + if (SPLAY_RIGHT(elm, field) != NULL) { \ + elm = SPLAY_RIGHT(elm, field); \ + while (SPLAY_LEFT(elm, field) != NULL) { \ + elm = SPLAY_LEFT(elm, field); \ + } \ + } else \ + elm = NULL; \ + return (elm); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_MIN_MAX(struct name *head, int val) \ +{ \ + name##_SPLAY_MINMAX(head, val); \ + return (SPLAY_ROOT(head)); \ +} + +/* Main splay operation. + * Moves node close to the key of elm to top + */ +#define SPLAY_GENERATE(name, type, field, cmp) \ +struct type * \ +name##_SPLAY_INSERT(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) { \ + SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ + } else { \ + int __comp; \ + name##_SPLAY(head, elm); \ + __comp = (cmp)(elm, (head)->sph_root); \ + if(__comp < 0) { \ + SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ + SPLAY_RIGHT(elm, field) = (head)->sph_root; \ + SPLAY_LEFT((head)->sph_root, field) = NULL; \ + } else if (__comp > 0) { \ + SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT(elm, field) = (head)->sph_root; \ + SPLAY_RIGHT((head)->sph_root, field) = NULL; \ + } else \ + return ((head)->sph_root); \ + } \ + (head)->sph_root = (elm); \ + return (NULL); \ +} \ + \ +struct type * \ +name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *__tmp; \ + if (SPLAY_EMPTY(head)) \ + return (NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) { \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ + } else { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ + name##_SPLAY(head, elm); \ + SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ + } \ + return (elm); \ + } \ + return (NULL); \ +} \ + \ +void \ +name##_SPLAY(struct name *head, struct type *elm) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ + int __comp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while ((__comp = (cmp)(elm, (head)->sph_root))) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) > 0){ \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} \ + \ +/* Splay with either the minimum or the maximum element \ + * Used to find minimum or maximum element in tree. \ + */ \ +void name##_SPLAY_MINMAX(struct name *head, int __comp) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while (1) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp > 0) { \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} + +#define SPLAY_NEGINF -1 +#define SPLAY_INF 1 + +#define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) +#define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) +#define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) +#define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) +#define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) +#define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) + +#define SPLAY_FOREACH(x, name, head) \ + for ((x) = SPLAY_MIN(name, head); \ + (x) != NULL; \ + (x) = SPLAY_NEXT(name, head, x)) + +/* Macros that define a red-back tree */ +#define RB_HEAD(name, type) \ +struct name { \ + struct type *rbh_root; /* root of the tree */ \ +} + +#define RB_INITIALIZER(root) \ + { NULL } + +#define RB_INIT(root) do { \ + (root)->rbh_root = NULL; \ +} while (0) + +#define RB_BLACK 0 +#define RB_RED 1 +#define RB_ENTRY(type) \ +struct { \ + struct type *rbe_left; /* left element */ \ + struct type *rbe_right; /* right element */ \ + struct type *rbe_parent; /* parent element */ \ + int rbe_color; /* node color */ \ +} + +#define RB_LEFT(elm, field) (elm)->field.rbe_left +#define RB_RIGHT(elm, field) (elm)->field.rbe_right +#define RB_PARENT(elm, field) (elm)->field.rbe_parent +#define RB_COLOR(elm, field) (elm)->field.rbe_color +#define RB_ROOT(head) (head)->rbh_root +#define RB_EMPTY(head) (RB_ROOT(head) == NULL) + +#define RB_SET(elm, parent, field) do { \ + RB_PARENT(elm, field) = parent; \ + RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ + RB_COLOR(elm, field) = RB_RED; \ +} while (0) + +#define RB_SET_BLACKRED(black, red, field) do { \ + RB_COLOR(black, field) = RB_BLACK; \ + RB_COLOR(red, field) = RB_RED; \ +} while (0) + +#ifndef RB_AUGMENT +#define RB_AUGMENT(x) +#endif + +#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ + (tmp) = RB_RIGHT(elm, field); \ + if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field))) { \ + RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_LEFT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ + (tmp) = RB_LEFT(elm, field); \ + if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field))) { \ + RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_RIGHT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +/* Generates prototypes and inline functions */ +#define RB_PROTOTYPE(name, type, field, cmp) \ +void name##_RB_INSERT_COLOR(struct name *, struct type *); \ +void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ +struct type *name##_RB_REMOVE(struct name *, struct type *); \ +struct type *name##_RB_INSERT(struct name *, struct type *); \ +struct type *name##_RB_FIND(struct name *, struct type *); \ +struct type *name##_RB_NEXT(struct type *); \ +struct type *name##_RB_MINMAX(struct name *, int); \ + \ + +/* Main rb operation. + * Moves node close to the key of elm to top + */ +#define RB_GENERATE(name, type, field, cmp) \ +void \ +name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +{ \ + struct type *parent, *gparent, *tmp; \ + while ((parent = RB_PARENT(elm, field)) && \ + RB_COLOR(parent, field) == RB_RED) { \ + gparent = RB_PARENT(parent, field); \ + if (parent == RB_LEFT(gparent, field)) { \ + tmp = RB_RIGHT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_RIGHT(parent, field) == elm) { \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_RIGHT(head, gparent, tmp, field); \ + } else { \ + tmp = RB_LEFT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_LEFT(parent, field) == elm) { \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_LEFT(head, gparent, tmp, field); \ + } \ + } \ + RB_COLOR(head->rbh_root, field) = RB_BLACK; \ +} \ + \ +void \ +name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ +{ \ + struct type *tmp; \ + while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ + elm != RB_ROOT(head)) { \ + if (RB_LEFT(parent, field) == elm) { \ + tmp = RB_RIGHT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ + struct type *oleft; \ + if ((oleft = RB_LEFT(tmp, field)))\ + RB_COLOR(oleft, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_RIGHT(head, tmp, oleft, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_RIGHT(tmp, field)) \ + RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } else { \ + tmp = RB_LEFT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ + struct type *oright; \ + if ((oright = RB_RIGHT(tmp, field)))\ + RB_COLOR(oright, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_LEFT(head, tmp, oright, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_LEFT(tmp, field)) \ + RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } \ + } \ + if (elm) \ + RB_COLOR(elm, field) = RB_BLACK; \ +} \ + \ +struct type * \ +name##_RB_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *child, *parent, *old = elm; \ + int color; \ + if (RB_LEFT(elm, field) == NULL) \ + child = RB_RIGHT(elm, field); \ + else if (RB_RIGHT(elm, field) == NULL) \ + child = RB_LEFT(elm, field); \ + else { \ + struct type *left; \ + elm = RB_RIGHT(elm, field); \ + while ((left = RB_LEFT(elm, field))) \ + elm = left; \ + child = RB_RIGHT(elm, field); \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ + if (RB_PARENT(elm, field) == old) \ + parent = elm; \ + (elm)->field = (old)->field; \ + if (RB_PARENT(old, field)) { \ + if (RB_LEFT(RB_PARENT(old, field), field) == old)\ + RB_LEFT(RB_PARENT(old, field), field) = elm;\ + else \ + RB_RIGHT(RB_PARENT(old, field), field) = elm;\ + RB_AUGMENT(RB_PARENT(old, field)); \ + } else \ + RB_ROOT(head) = elm; \ + RB_PARENT(RB_LEFT(old, field), field) = elm; \ + if (RB_RIGHT(old, field)) \ + RB_PARENT(RB_RIGHT(old, field), field) = elm; \ + if (parent) { \ + left = parent; \ + do { \ + RB_AUGMENT(left); \ + } while ((left = RB_PARENT(left, field))); \ + } \ + goto color; \ + } \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ +color: \ + if (color == RB_BLACK) \ + name##_RB_REMOVE_COLOR(head, parent, child); \ + return (old); \ +} \ + \ +/* Inserts a node into the RB tree */ \ +struct type * \ +name##_RB_INSERT(struct name *head, struct type *elm) \ +{ \ + struct type *tmp; \ + struct type *parent = NULL; \ + int comp = 0; \ + tmp = RB_ROOT(head); \ + while (tmp) { \ + parent = tmp; \ + comp = (cmp)(elm, parent); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + RB_SET(elm, parent, field); \ + if (parent != NULL) { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = elm; \ + name##_RB_INSERT_COLOR(head, elm); \ + return (NULL); \ +} \ + \ +/* Finds the node with the same key as elm */ \ +struct type * \ +name##_RB_FIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (NULL); \ +} \ + \ +struct type * \ +name##_RB_NEXT(struct type *elm) \ +{ \ + if (RB_RIGHT(elm, field)) { \ + elm = RB_RIGHT(elm, field); \ + while (RB_LEFT(elm, field)) \ + elm = RB_LEFT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +struct type * \ +name##_RB_MINMAX(struct name *head, int val) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *parent = NULL; \ + while (tmp) { \ + parent = tmp; \ + if (val < 0) \ + tmp = RB_LEFT(tmp, field); \ + else \ + tmp = RB_RIGHT(tmp, field); \ + } \ + return (parent); \ +} + +#define RB_NEGINF -1 +#define RB_INF 1 + +#define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) +#define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) +#define RB_FIND(name, x, y) name##_RB_FIND(x, y) +#define RB_NEXT(name, x, y) name##_RB_NEXT(y) +#define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) +#define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) + +#define RB_FOREACH(x, name, head) \ + for ((x) = RB_MIN(name, head); \ + (x) != NULL; \ + (x) = name##_RB_NEXT(x)) + +#endif /* _SYS_TREE_H_ */ === added file 'extra/libevent/WIN32-Code/win32.c' --- a/extra/libevent/WIN32-Code/win32.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/WIN32-Code/win32.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,471 @@ +/* + * Copyright 2000-2002 Niels Provos <provos(a)citi.umich.edu> + * Copyright 2003 Michael A. Davis <mike(a)datanerds.net> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef _MSC_VER +#include "./config.h" +#else +/* Avoid the windows/msvc thing. */ +#include "../config.h" +#endif + +#include <winsock2.h> +#include <windows.h> +#include <sys/types.h> +#include <sys/queue.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <assert.h> + +#define RB_AUGMENT(x) (void)(x) +#include "./tree.h" +#include "log.h" +#include "event.h" +#include "event-internal.h" + +#define XFREE(ptr) do { if (ptr) free(ptr); } while(0) + +extern struct event_list timequeue; +extern struct event_list addqueue; +#if 0 +extern struct event_list signalqueue; +#endif + +struct win_fd_set { + u_int fd_count; + SOCKET fd_array[1]; +}; + +int evsigcaught[NSIG]; +volatile sig_atomic_t signal_caught = 0; +/* MSDN says this is required to handle SIGFPE */ +volatile double SIGFPE_REQ = 0.0f; + +#if 0 +static void signal_handler(int sig); + +void signal_process(void); +int signal_recalc(void); +#endif + +struct event_entry { + RB_ENTRY(event_entry) node; + SOCKET sock; + int read_pos; + int write_pos; + struct event *read_event; + struct event *write_event; +}; + +static int +compare(struct event_entry *a, struct event_entry *b) +{ + if (a->sock < b->sock) + return -1; + else if (a->sock > b->sock) + return 1; + else + return 0; +} + +struct win32op { + int fd_setsz; + struct win_fd_set *readset_in; + struct win_fd_set *writeset_in; + struct win_fd_set *readset_out; + struct win_fd_set *writeset_out; + struct win_fd_set *exset_out; + RB_HEAD(event_map, event_entry) event_root; +}; + +RB_PROTOTYPE(event_map, event_entry, node, compare); +RB_GENERATE(event_map, event_entry, node, compare); + +void *win32_init (struct event_base *); +int win32_insert (void *, struct event *); +int win32_del (void *, struct event *); +int win32_dispatch (struct event_base *base, void *, struct timeval *); +void win32_dealloc (struct event_base *, void *); + +struct eventop win32ops = { + "win32", + win32_init, + win32_insert, + win32_del, + win32_dispatch, + win32_dealloc, + 0 +}; + +#define FD_SET_ALLOC_SIZE(n) ((sizeof(struct win_fd_set) + ((n)-1)*sizeof(SOCKET))) + +static int +realloc_fd_sets(struct win32op *op, size_t new_size) +{ + size_t size; + + assert(new_size >= op->readset_in->fd_count && + new_size >= op->writeset_in->fd_count); + assert(new_size >= 1); + + size = FD_SET_ALLOC_SIZE(new_size); + if (!(op->readset_in = realloc(op->readset_in, size))) + return (-1); + if (!(op->writeset_in = realloc(op->writeset_in, size))) + return (-1); + if (!(op->readset_out = realloc(op->readset_out, size))) + return (-1); + if (!(op->exset_out = realloc(op->exset_out, size))) + return (-1); + if (!(op->writeset_out = realloc(op->writeset_out, size))) + return (-1); + op->fd_setsz = (int)new_size; + return (0); +} + +static int +timeval_to_ms(struct timeval *tv) +{ + return ((tv->tv_sec * 1000) + (tv->tv_usec / 1000)); +} + +static struct event_entry* +get_event_entry(struct win32op *op, SOCKET s, int create) +{ + struct event_entry key, *val; + key.sock = s; + val = RB_FIND(event_map, &op->event_root, &key); + if (val || !create) + return val; + if (!(val = calloc(1, sizeof(struct event_entry)))) { + event_warn("%s: calloc", __func__); + return NULL; + } + val->sock = s; + val->read_pos = val->write_pos = -1; + RB_INSERT(event_map, &op->event_root, val); + return val; +} + +static int +do_fd_set(struct win32op *op, struct event_entry *ent, int read) +{ + SOCKET s = ent->sock; + struct win_fd_set *set = read ? op->readset_in : op->writeset_in; + if (read) { + if (ent->read_pos >= 0) + return (0); + } else { + if (ent->write_pos >= 0) + return (0); + } + if (set->fd_count == op->fd_setsz) { + if (realloc_fd_sets(op, op->fd_setsz*2)) + return (-1); + /* set pointer will have changed and needs reiniting! */ + set = read ? op->readset_in : op->writeset_in; + } + set->fd_array[set->fd_count] = s; + if (read) + ent->read_pos = set->fd_count; + else + ent->write_pos = set->fd_count; + return (set->fd_count++); +} + +static int +do_fd_clear(struct win32op *op, struct event_entry *ent, int read) +{ + int i; + struct win_fd_set *set = read ? op->readset_in : op->writeset_in; + if (read) { + i = ent->read_pos; + ent->read_pos = -1; + } else { + i = ent->write_pos; + ent->write_pos = -1; + } + if (i < 0) + return (0); + if (--set->fd_count != i) { + struct event_entry *ent2; + SOCKET s2; + s2 = set->fd_array[i] = set->fd_array[set->fd_count]; + ent2 = get_event_entry(op, s2, 0); + if (!ent) /* This indicates a bug. */ + return (0); + if (read) + ent2->read_pos = i; + else + ent2->write_pos = i; + } + return (0); +} + +#define NEVENT 64 +void * +win32_init(struct event_base *_base) +{ + struct win32op *winop; + size_t size; + if (!(winop = calloc(1, sizeof(struct win32op)))) + return NULL; + winop->fd_setsz = NEVENT; + size = FD_SET_ALLOC_SIZE(NEVENT); + if (!(winop->readset_in = malloc(size))) + goto err; + if (!(winop->writeset_in = malloc(size))) + goto err; + if (!(winop->readset_out = malloc(size))) + goto err; + if (!(winop->writeset_out = malloc(size))) + goto err; + if (!(winop->exset_out = malloc(size))) + goto err; + RB_INIT(&winop->event_root); + winop->readset_in->fd_count = winop->writeset_in->fd_count = 0; + winop->readset_out->fd_count = winop->writeset_out->fd_count + = winop->exset_out->fd_count = 0; + + evsignal_init(_base); + + return (winop); + err: + XFREE(winop->readset_in); + XFREE(winop->writeset_in); + XFREE(winop->readset_out); + XFREE(winop->writeset_out); + XFREE(winop->exset_out); + XFREE(winop); + return (NULL); +} + +int +win32_insert(void *op, struct event *ev) +{ + struct win32op *win32op = op; + struct event_entry *ent; + + if (ev->ev_events & EV_SIGNAL) { + return (evsignal_add(ev)); + } + if (!(ev->ev_events & (EV_READ|EV_WRITE))) + return (0); + ent = get_event_entry(win32op, ev->ev_fd, 1); + if (!ent) + return (-1); /* out of memory */ + + event_debug(("%s: adding event for %d", __func__, (int)ev->ev_fd)); + if (ev->ev_events & EV_READ) { + if (do_fd_set(win32op, ent, 1)<0) + return (-1); + ent->read_event = ev; + } + if (ev->ev_events & EV_WRITE) { + if (do_fd_set(win32op, ent, 0)<0) + return (-1); + ent->write_event = ev; + } + return (0); +} + +int +win32_del(void *op, struct event *ev) +{ + struct win32op *win32op = op; + struct event_entry *ent; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_del(ev)); + + if (!(ent = get_event_entry(win32op, ev->ev_fd, 0))) + return (-1); + event_debug(("%s: Removing event for %d", __func__, ev->ev_fd)); + if (ev == ent->read_event) { + do_fd_clear(win32op, ent, 1); + ent->read_event = NULL; + } + if (ev == ent->write_event) { + do_fd_clear(win32op, ent, 0); + ent->write_event = NULL; + } + if (!ent->read_event && !ent->write_event) { + RB_REMOVE(event_map, &win32op->event_root, ent); + free(ent); + } + + return 0; +} + +static void +fd_set_copy(struct win_fd_set *out, const struct win_fd_set *in) +{ + out->fd_count = in->fd_count; + memcpy(out->fd_array, in->fd_array, in->fd_count * (sizeof(SOCKET))); +} + +/* + static void dump_fd_set(struct win_fd_set *s) + { + unsigned int i; + printf("[ "); + for(i=0;i<s->fd_count;++i) + printf("%d ",(int)s->fd_array[i]); + printf("]\n"); + } +*/ + +int +win32_dispatch(struct event_base *base, void *op, + struct timeval *tv) +{ + struct win32op *win32op = op; + int res = 0; + u_int i; + int fd_count; + + fd_set_copy(win32op->readset_out, win32op->readset_in); + fd_set_copy(win32op->exset_out, win32op->readset_in); + fd_set_copy(win32op->writeset_out, win32op->writeset_in); + + fd_count = + (win32op->readset_out->fd_count > win32op->writeset_out->fd_count) ? + win32op->readset_out->fd_count : win32op->writeset_out->fd_count; + + if (!fd_count) { + /* Windows doesn't like you to call select() with no sockets */ + Sleep(timeval_to_ms(tv)); + evsignal_process(base); + return (0); + } + + res = select(fd_count, + (struct fd_set*)win32op->readset_out, + (struct fd_set*)win32op->writeset_out, + (struct fd_set*)win32op->exset_out, tv); + + event_debug(("%s: select returned %d", __func__, res)); + + if(res <= 0) { + evsignal_process(base); + return res; + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + for (i=0; i<win32op->readset_out->fd_count; ++i) { + struct event_entry *ent; + SOCKET s = win32op->readset_out->fd_array[i]; + if ((ent = get_event_entry(win32op, s, 0)) && ent->read_event) + event_active(ent->read_event, EV_READ, 1); + } + for (i=0; i<win32op->exset_out->fd_count; ++i) { + struct event_entry *ent; + SOCKET s = win32op->exset_out->fd_array[i]; + if ((ent = get_event_entry(win32op, s, 0)) && ent->read_event) + event_active(ent->read_event, EV_READ, 1); + } + for (i=0; i<win32op->writeset_out->fd_count; ++i) { + struct event_entry *ent; + SOCKET s = win32op->writeset_out->fd_array[i]; + if ((ent = get_event_entry(win32op, s, 0)) && ent->write_event) + event_active(ent->write_event, EV_WRITE, 1); + } + +#if 0 + if (signal_recalc() == -1) + return (-1); +#endif + + return (0); +} + +void +win32_dealloc(struct event_base *_base, void *arg) +{ + struct win32op *win32op = arg; + + evsignal_dealloc(_base); + if (win32op->readset_in) + free(win32op->readset_in); + if (win32op->writeset_in) + free(win32op->writeset_in); + if (win32op->readset_out) + free(win32op->readset_out); + if (win32op->writeset_out) + free(win32op->writeset_out); + if (win32op->exset_out) + free(win32op->exset_out); + /* XXXXX free the tree. */ + + memset(win32op, 0, sizeof(win32op)); + free(win32op); +} + +#if 0 +static void +signal_handler(int sig) +{ + evsigcaught[sig]++; + signal_caught = 1; +} + +int +signal_recalc(void) +{ + struct event *ev; + + /* Reinstall our signal handler. */ + TAILQ_FOREACH(ev, &signalqueue, ev_signal_next) { + if((int)signal(EVENT_SIGNAL(ev), signal_handler) == -1) + return (-1); + } + return (0); +} + +void +signal_process(void) +{ + struct event *ev; + short ncalls; + + TAILQ_FOREACH(ev, &signalqueue, ev_signal_next) { + ncalls = evsigcaught[EVENT_SIGNAL(ev)]; + if (ncalls) { + if (!(ev->ev_events & EV_PERSIST)) + event_del(ev); + event_active(ev, EV_SIGNAL, ncalls); + } + } + + memset(evsigcaught, 0, sizeof(evsigcaught)); + signal_caught = 0; +} +#endif + === added file 'extra/libevent/buffer.c' --- a/extra/libevent/buffer.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/buffer.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,455 @@ +/* + * Copyright (c) 2002, 2003 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +#ifdef HAVE_VASPRINTF +/* If we have vasprintf, we need to define this before we include stdio.h. */ +#define _GNU_SOURCE +#endif + +#include <sys/types.h> + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include <sys/ioctl.h> +#endif + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_STDARG_H +#include <stdarg.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include "event.h" +#include "config.h" + +struct evbuffer * +evbuffer_new(void) +{ + struct evbuffer *buffer; + + buffer = calloc(1, sizeof(struct evbuffer)); + + return (buffer); +} + +void +evbuffer_free(struct evbuffer *buffer) +{ + if (buffer->orig_buffer != NULL) + free(buffer->orig_buffer); + free(buffer); +} + +/* + * This is a destructive add. The data from one buffer moves into + * the other buffer. + */ + +#define SWAP(x,y) do { \ + (x)->buffer = (y)->buffer; \ + (x)->orig_buffer = (y)->orig_buffer; \ + (x)->misalign = (y)->misalign; \ + (x)->totallen = (y)->totallen; \ + (x)->off = (y)->off; \ +} while (0) + +int +evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf) +{ + int res; + + /* Short cut for better performance */ + if (outbuf->off == 0) { + struct evbuffer tmp; + size_t oldoff = inbuf->off; + + /* Swap them directly */ + SWAP(&tmp, outbuf); + SWAP(outbuf, inbuf); + SWAP(inbuf, &tmp); + + /* + * Optimization comes with a price; we need to notify the + * buffer if necessary of the changes. oldoff is the amount + * of data that we transfered from inbuf to outbuf + */ + if (inbuf->off != oldoff && inbuf->cb != NULL) + (*inbuf->cb)(inbuf, oldoff, inbuf->off, inbuf->cbarg); + if (oldoff && outbuf->cb != NULL) + (*outbuf->cb)(outbuf, 0, oldoff, outbuf->cbarg); + + return (0); + } + + res = evbuffer_add(outbuf, inbuf->buffer, inbuf->off); + if (res == 0) { + /* We drain the input buffer on success */ + evbuffer_drain(inbuf, inbuf->off); + } + + return (res); +} + +int +evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap) +{ + char *buffer; + size_t space; + size_t oldoff = buf->off; + int sz; + va_list aq; + + /* make sure that at least some space is available */ + evbuffer_expand(buf, 64); + for (;;) { + size_t used = buf->misalign + buf->off; + buffer = (char *)buf->buffer + buf->off; + assert(buf->totallen >= used); + space = buf->totallen - used; + +#ifndef va_copy +#define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list)) +#endif + va_copy(aq, ap); + +#ifdef WIN32 + sz = _vsnprintf(buffer, space - 1, fmt, aq); + buffer[space - 1] = '\0'; +#else + sz = vsnprintf(buffer, space, fmt, aq); +#endif + + va_end(aq); + + if (sz < 0) + return (-1); + if ((size_t)sz < space) { + buf->off += sz; + if (buf->cb != NULL) + (*buf->cb)(buf, oldoff, buf->off, buf->cbarg); + return (sz); + } + if (evbuffer_expand(buf, sz + 1) == -1) + return (-1); + + } + /* NOTREACHED */ +} + +int +evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...) +{ + int res = -1; + va_list ap; + + va_start(ap, fmt); + res = evbuffer_add_vprintf(buf, fmt, ap); + va_end(ap); + + return (res); +} + +/* Reads data from an event buffer and drains the bytes read */ + +int +evbuffer_remove(struct evbuffer *buf, void *data, size_t datlen) +{ + size_t nread = datlen; + if (nread >= buf->off) + nread = buf->off; + + memcpy(data, buf->buffer, nread); + evbuffer_drain(buf, nread); + + return (int)(nread); +} + +/* + * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'. + * The returned buffer needs to be freed by the called. + */ + +char * +evbuffer_readline(struct evbuffer *buffer) +{ + u_char *data = EVBUFFER_DATA(buffer); + size_t len = EVBUFFER_LENGTH(buffer); + char *line; + unsigned int i; + + for (i = 0; i < len; i++) { + if (data[i] == '\r' || data[i] == '\n') + break; + } + + if (i == len) + return (NULL); + + if ((line = malloc(i + 1)) == NULL) { + fprintf(stderr, "%s: out of memory\n", __func__); + evbuffer_drain(buffer, i); + return (NULL); + } + + memcpy(line, data, i); + line[i] = '\0'; + + /* + * Some protocols terminate a line with '\r\n', so check for + * that, too. + */ + if ( i < len - 1 ) { + char fch = data[i], sch = data[i+1]; + + /* Drain one more character if needed */ + if ( (sch == '\r' || sch == '\n') && sch != fch ) + i += 1; + } + + evbuffer_drain(buffer, i + 1); + + return (line); +} + +/* Adds data to an event buffer */ + +static void +evbuffer_align(struct evbuffer *buf) +{ + memmove(buf->orig_buffer, buf->buffer, buf->off); + buf->buffer = buf->orig_buffer; + buf->misalign = 0; +} + +/* Expands the available space in the event buffer to at least datlen */ + +int +evbuffer_expand(struct evbuffer *buf, size_t datlen) +{ + size_t need = buf->misalign + buf->off + datlen; + + /* If we can fit all the data, then we don't have to do anything */ + if (buf->totallen >= need) + return (0); + + /* + * If the misalignment fulfills our data needs, we just force an + * alignment to happen. Afterwards, we have enough space. + */ + if (buf->misalign >= datlen) { + evbuffer_align(buf); + } else { + void *newbuf; + size_t length = buf->totallen; + + if (length < 256) + length = 256; + while (length < need) + length <<= 1; + + if (buf->orig_buffer != buf->buffer) + evbuffer_align(buf); + if ((newbuf = realloc(buf->buffer, length)) == NULL) + return (-1); + + buf->orig_buffer = buf->buffer = newbuf; + buf->totallen = length; + } + + return (0); +} + +int +evbuffer_add(struct evbuffer *buf, const void *data, size_t datlen) +{ + size_t need = buf->misalign + buf->off + datlen; + size_t oldoff = buf->off; + + if (buf->totallen < need) { + if (evbuffer_expand(buf, datlen) == -1) + return (-1); + } + + memcpy(buf->buffer + buf->off, data, datlen); + buf->off += datlen; + + if (datlen && buf->cb != NULL) + (*buf->cb)(buf, oldoff, buf->off, buf->cbarg); + + return (0); +} + +void +evbuffer_drain(struct evbuffer *buf, size_t len) +{ + size_t oldoff = buf->off; + + if (len >= buf->off) { + buf->off = 0; + buf->buffer = buf->orig_buffer; + buf->misalign = 0; + goto done; + } + + buf->buffer += len; + buf->misalign += len; + + buf->off -= len; + + done: + /* Tell someone about changes in this buffer */ + if (buf->off != oldoff && buf->cb != NULL) + (*buf->cb)(buf, oldoff, buf->off, buf->cbarg); + +} + +/* + * Reads data from a file descriptor into a buffer. + */ + +#define EVBUFFER_MAX_READ 4096 + +int +evbuffer_read(struct evbuffer *buf, int fd, int howmuch) +{ + u_char *p; + size_t oldoff = buf->off; + int n = EVBUFFER_MAX_READ; + +#if defined(FIONREAD) +#ifdef WIN32 + long lng = (long)n; + if (ioctlsocket(fd, FIONREAD, &lng) == -1 || (n=lng) == 0) { +#else + if (ioctl(fd, FIONREAD, &n) == -1 || n == 0) { +#endif + n = EVBUFFER_MAX_READ; + } else if (n > EVBUFFER_MAX_READ && n > howmuch) { + /* + * It's possible that a lot of data is available for + * reading. We do not want to exhaust resources + * before the reader has a chance to do something + * about it. If the reader does not tell us how much + * data we should read, we artifically limit it. + */ + if ((size_t)n > (buf->totallen << 2)) + n = (int)(buf->totallen << 2); + if (n < EVBUFFER_MAX_READ) + n = EVBUFFER_MAX_READ; + } +#endif + if (howmuch < 0 || howmuch > n) + howmuch = n; + + /* If we don't have FIONREAD, we might waste some space here */ + if (evbuffer_expand(buf, howmuch) == -1) + return (-1); + + /* We can append new data at this point */ + p = buf->buffer + buf->off; + +#ifndef WIN32 + n = read(fd, p, howmuch); +#else + n = recv(fd, p, howmuch, 0); +#endif + if (n == -1) + return (-1); + if (n == 0) + return (0); + + buf->off += n; + + /* Tell someone about changes in this buffer */ + if (buf->off != oldoff && buf->cb != NULL) + (*buf->cb)(buf, oldoff, buf->off, buf->cbarg); + + return (n); +} + +int +evbuffer_write(struct evbuffer *buffer, int fd) +{ + int n; + +#ifndef WIN32 + n = write(fd, buffer->buffer, buffer->off); +#else + n = send(fd, buffer->buffer, (int)buffer->off, 0); +#endif + if (n == -1) + return (-1); + if (n == 0) + return (0); + evbuffer_drain(buffer, n); + + return (n); +} + +u_char * +evbuffer_find(struct evbuffer *buffer, const u_char *what, size_t len) +{ + u_char *search = buffer->buffer, *end = search + buffer->off; + u_char *p; + + while (search < end && + (p = memchr(search, *what, end - search)) != NULL) { + if (p + len > end) + break; + if (memcmp(p, what, len) == 0) + return (p); + search = p + 1; + } + + return (NULL); +} + +void evbuffer_setcb(struct evbuffer *buffer, + void (*cb)(struct evbuffer *, size_t, size_t, void *), + void *cbarg) +{ + buffer->cb = cb; + buffer->cbarg = cbarg; +} === added directory 'extra/libevent/compat' === added directory 'extra/libevent/compat/sys' === added file 'extra/libevent/compat/sys/_time.h' --- a/extra/libevent/compat/sys/_time.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/compat/sys/_time.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,163 @@ +/* $OpenBSD: time.h,v 1.11 2000/10/10 13:36:48 itojun Exp $ */ +/* $NetBSD: time.h,v 1.18 1996/04/23 10:29:33 mycroft Exp $ */ + +/* + * Copyright (c) 1982, 1986, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)time.h 8.2 (Berkeley) 7/10/94 + */ + +#ifndef _SYS_TIME_H_ +#define _SYS_TIME_H_ + +#include <sys/types.h> + +/* + * Structure returned by gettimeofday(2) system call, + * and used in other calls. + */ +struct timeval { + long tv_sec; /* seconds */ + long tv_usec; /* and microseconds */ +}; + +/* + * Structure defined by POSIX.1b to be like a timeval. + */ +struct timespec { + time_t tv_sec; /* seconds */ + long tv_nsec; /* and nanoseconds */ +}; + +#define TIMEVAL_TO_TIMESPEC(tv, ts) { \ + (ts)->tv_sec = (tv)->tv_sec; \ + (ts)->tv_nsec = (tv)->tv_usec * 1000; \ +} +#define TIMESPEC_TO_TIMEVAL(tv, ts) { \ + (tv)->tv_sec = (ts)->tv_sec; \ + (tv)->tv_usec = (ts)->tv_nsec / 1000; \ +} + +struct timezone { + int tz_minuteswest; /* minutes west of Greenwich */ + int tz_dsttime; /* type of dst correction */ +}; +#define DST_NONE 0 /* not on dst */ +#define DST_USA 1 /* USA style dst */ +#define DST_AUST 2 /* Australian style dst */ +#define DST_WET 3 /* Western European dst */ +#define DST_MET 4 /* Middle European dst */ +#define DST_EET 5 /* Eastern European dst */ +#define DST_CAN 6 /* Canada */ + +/* Operations on timevals. */ +#define timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_usec = 0 +#define timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_usec) +#define timercmp(tvp, uvp, cmp) \ + (((tvp)->tv_sec == (uvp)->tv_sec) ? \ + ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ + ((tvp)->tv_sec cmp (uvp)->tv_sec)) +#define timeradd(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ + if ((vvp)->tv_usec >= 1000000) { \ + (vvp)->tv_sec++; \ + (vvp)->tv_usec -= 1000000; \ + } \ + } while (0) +#define timersub(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ + if ((vvp)->tv_usec < 0) { \ + (vvp)->tv_sec--; \ + (vvp)->tv_usec += 1000000; \ + } \ + } while (0) + +/* Operations on timespecs. */ +#define timespecclear(tsp) (tsp)->tv_sec = (tsp)->tv_nsec = 0 +#define timespecisset(tsp) ((tsp)->tv_sec || (tsp)->tv_nsec) +#define timespeccmp(tsp, usp, cmp) \ + (((tsp)->tv_sec == (usp)->tv_sec) ? \ + ((tsp)->tv_nsec cmp (usp)->tv_nsec) : \ + ((tsp)->tv_sec cmp (usp)->tv_sec)) +#define timespecadd(tsp, usp, vsp) \ + do { \ + (vsp)->tv_sec = (tsp)->tv_sec + (usp)->tv_sec; \ + (vsp)->tv_nsec = (tsp)->tv_nsec + (usp)->tv_nsec; \ + if ((vsp)->tv_nsec >= 1000000000L) { \ + (vsp)->tv_sec++; \ + (vsp)->tv_nsec -= 1000000000L; \ + } \ + } while (0) +#define timespecsub(tsp, usp, vsp) \ + do { \ + (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec; \ + (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec; \ + if ((vsp)->tv_nsec < 0) { \ + (vsp)->tv_sec--; \ + (vsp)->tv_nsec += 1000000000L; \ + } \ + } while (0) + +/* + * Names of the interval timers, and structure + * defining a timer setting. + */ +#define ITIMER_REAL 0 +#define ITIMER_VIRTUAL 1 +#define ITIMER_PROF 2 + +struct itimerval { + struct timeval it_interval; /* timer interval */ + struct timeval it_value; /* current value */ +}; + +/* + * Getkerninfo clock information structure + */ +struct clockinfo { + int hz; /* clock frequency */ + int tick; /* micro-seconds per hz tick */ + int tickadj; /* clock skew rate for adjtime() */ + int stathz; /* statistics clock frequency */ + int profhz; /* profiling clock frequency */ +}; + +#define CLOCK_REALTIME 0 +#define CLOCK_VIRTUAL 1 +#define CLOCK_PROF 2 + +#define TIMER_RELTIME 0x0 /* relative timer */ +#define TIMER_ABSTIME 0x1 /* absolute timer */ + +/* --- stuff got cut here - niels --- */ + +#endif /* !_SYS_TIME_H_ */ === added file 'extra/libevent/compat/sys/queue.h' --- a/extra/libevent/compat/sys/queue.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/compat/sys/queue.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,488 @@ +/* $OpenBSD: queue.h,v 1.16 2000/09/07 19:47:59 art Exp $ */ +/* $NetBSD: queue.h,v 1.11 1996/05/16 05:17:14 mycroft Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + */ + +#ifndef _SYS_QUEUE_H_ +#define _SYS_QUEUE_H_ + +/* + * This file defines five types of data structures: singly-linked lists, + * lists, simple queues, tail queues, and circular queues. + * + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * A simple queue is headed by a pair of pointers, one the head of the + * list and the other to the tail of the list. The elements are singly + * linked to save space, so elements can only be removed from the + * head of the list. New elements can be added to the list before or after + * an existing element, at the head of the list, or at the end of the + * list. A simple queue may only be traversed in the forward direction. + * + * A tail queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or + * after an existing element, at the head of the list, or at the end of + * the list. A tail queue may be traversed in either direction. + * + * A circle queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or after + * an existing element, at the head of the list, or at the end of the list. + * A circle queue may be traversed in either direction, but has a more + * complex end of list detection. + * + * For details on the use of these macros, see the queue(3) manual page. + */ + +/* + * Singly-linked List definitions. + */ +#define SLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#ifndef WIN32 +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} +#endif + +/* + * Singly-linked List access methods. + */ +#define SLIST_FIRST(head) ((head)->slh_first) +#define SLIST_END(head) NULL +#define SLIST_EMPTY(head) (SLIST_FIRST(head) == SLIST_END(head)) +#define SLIST_NEXT(elm, field) ((elm)->field.sle_next) + +#define SLIST_FOREACH(var, head, field) \ + for((var) = SLIST_FIRST(head); \ + (var) != SLIST_END(head); \ + (var) = SLIST_NEXT(var, field)) + +/* + * Singly-linked List functions. + */ +#define SLIST_INIT(head) { \ + SLIST_FIRST(head) = SLIST_END(head); \ +} + +#define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ + (elm)->field.sle_next = (slistelm)->field.sle_next; \ + (slistelm)->field.sle_next = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.sle_next = (head)->slh_first; \ + (head)->slh_first = (elm); \ +} while (0) + +#define SLIST_REMOVE_HEAD(head, field) do { \ + (head)->slh_first = (head)->slh_first->field.sle_next; \ +} while (0) + +/* + * List definitions. + */ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +/* + * List access methods + */ +#define LIST_FIRST(head) ((head)->lh_first) +#define LIST_END(head) NULL +#define LIST_EMPTY(head) (LIST_FIRST(head) == LIST_END(head)) +#define LIST_NEXT(elm, field) ((elm)->field.le_next) + +#define LIST_FOREACH(var, head, field) \ + for((var) = LIST_FIRST(head); \ + (var)!= LIST_END(head); \ + (var) = LIST_NEXT(var, field)) + +/* + * List functions. + */ +#define LIST_INIT(head) do { \ + LIST_FIRST(head) = LIST_END(head); \ +} while (0) + +#define LIST_INSERT_AFTER(listelm, elm, field) do { \ + if (((elm)->field.le_next = (listelm)->field.le_next) != NULL) \ + (listelm)->field.le_next->field.le_prev = \ + &(elm)->field.le_next; \ + (listelm)->field.le_next = (elm); \ + (elm)->field.le_prev = &(listelm)->field.le_next; \ +} while (0) + +#define LIST_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + (elm)->field.le_next = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &(elm)->field.le_next; \ +} while (0) + +#define LIST_INSERT_HEAD(head, elm, field) do { \ + if (((elm)->field.le_next = (head)->lh_first) != NULL) \ + (head)->lh_first->field.le_prev = &(elm)->field.le_next;\ + (head)->lh_first = (elm); \ + (elm)->field.le_prev = &(head)->lh_first; \ +} while (0) + +#define LIST_REMOVE(elm, field) do { \ + if ((elm)->field.le_next != NULL) \ + (elm)->field.le_next->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = (elm)->field.le_next; \ +} while (0) + +#define LIST_REPLACE(elm, elm2, field) do { \ + if (((elm2)->field.le_next = (elm)->field.le_next) != NULL) \ + (elm2)->field.le_next->field.le_prev = \ + &(elm2)->field.le_next; \ + (elm2)->field.le_prev = (elm)->field.le_prev; \ + *(elm2)->field.le_prev = (elm2); \ +} while (0) + +/* + * Simple queue definitions. + */ +#define SIMPLEQ_HEAD(name, type) \ +struct name { \ + struct type *sqh_first; /* first element */ \ + struct type **sqh_last; /* addr of last next element */ \ +} + +#define SIMPLEQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).sqh_first } + +#define SIMPLEQ_ENTRY(type) \ +struct { \ + struct type *sqe_next; /* next element */ \ +} + +/* + * Simple queue access methods. + */ +#define SIMPLEQ_FIRST(head) ((head)->sqh_first) +#define SIMPLEQ_END(head) NULL +#define SIMPLEQ_EMPTY(head) (SIMPLEQ_FIRST(head) == SIMPLEQ_END(head)) +#define SIMPLEQ_NEXT(elm, field) ((elm)->field.sqe_next) + +#define SIMPLEQ_FOREACH(var, head, field) \ + for((var) = SIMPLEQ_FIRST(head); \ + (var) != SIMPLEQ_END(head); \ + (var) = SIMPLEQ_NEXT(var, field)) + +/* + * Simple queue functions. + */ +#define SIMPLEQ_INIT(head) do { \ + (head)->sqh_first = NULL; \ + (head)->sqh_last = &(head)->sqh_first; \ +} while (0) + +#define SIMPLEQ_INSERT_HEAD(head, elm, field) do { \ + if (((elm)->field.sqe_next = (head)->sqh_first) == NULL) \ + (head)->sqh_last = &(elm)->field.sqe_next; \ + (head)->sqh_first = (elm); \ +} while (0) + +#define SIMPLEQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.sqe_next = NULL; \ + *(head)->sqh_last = (elm); \ + (head)->sqh_last = &(elm)->field.sqe_next; \ +} while (0) + +#define SIMPLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ + if (((elm)->field.sqe_next = (listelm)->field.sqe_next) == NULL)\ + (head)->sqh_last = &(elm)->field.sqe_next; \ + (listelm)->field.sqe_next = (elm); \ +} while (0) + +#define SIMPLEQ_REMOVE_HEAD(head, elm, field) do { \ + if (((head)->sqh_first = (elm)->field.sqe_next) == NULL) \ + (head)->sqh_last = &(head)->sqh_first; \ +} while (0) + +/* + * Tail queue definitions. + */ +#define TAILQ_HEAD(name, type) \ +struct name { \ + struct type *tqh_first; /* first element */ \ + struct type **tqh_last; /* addr of last next element */ \ +} + +#define TAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).tqh_first } + +#define TAILQ_ENTRY(type) \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ +} + +/* + * tail queue access methods + */ +#define TAILQ_FIRST(head) ((head)->tqh_first) +#define TAILQ_END(head) NULL +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) +#define TAILQ_LAST(head, headname) \ + (*(((struct headname *)((head)->tqh_last))->tqh_last)) +/* XXX */ +#define TAILQ_PREV(elm, headname, field) \ + (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) +#define TAILQ_EMPTY(head) \ + (TAILQ_FIRST(head) == TAILQ_END(head)) + +#define TAILQ_FOREACH(var, head, field) \ + for((var) = TAILQ_FIRST(head); \ + (var) != TAILQ_END(head); \ + (var) = TAILQ_NEXT(var, field)) + +#define TAILQ_FOREACH_REVERSE(var, head, field, headname) \ + for((var) = TAILQ_LAST(head, headname); \ + (var) != TAILQ_END(head); \ + (var) = TAILQ_PREV(var, headname, field)) + +/* + * Tail queue functions. + */ +#define TAILQ_INIT(head) do { \ + (head)->tqh_first = NULL; \ + (head)->tqh_last = &(head)->tqh_first; \ +} while (0) + +#define TAILQ_INSERT_HEAD(head, elm, field) do { \ + if (((elm)->field.tqe_next = (head)->tqh_first) != NULL) \ + (head)->tqh_first->field.tqe_prev = \ + &(elm)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + (head)->tqh_first = (elm); \ + (elm)->field.tqe_prev = &(head)->tqh_first; \ +} while (0) + +#define TAILQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.tqe_next = NULL; \ + (elm)->field.tqe_prev = (head)->tqh_last; \ + *(head)->tqh_last = (elm); \ + (head)->tqh_last = &(elm)->field.tqe_next; \ +} while (0) + +#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ + if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\ + (elm)->field.tqe_next->field.tqe_prev = \ + &(elm)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm)->field.tqe_next; \ + (listelm)->field.tqe_next = (elm); \ + (elm)->field.tqe_prev = &(listelm)->field.tqe_next; \ +} while (0) + +#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ + (elm)->field.tqe_next = (listelm); \ + *(listelm)->field.tqe_prev = (elm); \ + (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ +} while (0) + +#define TAILQ_REMOVE(head, elm, field) do { \ + if (((elm)->field.tqe_next) != NULL) \ + (elm)->field.tqe_next->field.tqe_prev = \ + (elm)->field.tqe_prev; \ + else \ + (head)->tqh_last = (elm)->field.tqe_prev; \ + *(elm)->field.tqe_prev = (elm)->field.tqe_next; \ +} while (0) + +#define TAILQ_REPLACE(head, elm, elm2, field) do { \ + if (((elm2)->field.tqe_next = (elm)->field.tqe_next) != NULL) \ + (elm2)->field.tqe_next->field.tqe_prev = \ + &(elm2)->field.tqe_next; \ + else \ + (head)->tqh_last = &(elm2)->field.tqe_next; \ + (elm2)->field.tqe_prev = (elm)->field.tqe_prev; \ + *(elm2)->field.tqe_prev = (elm2); \ +} while (0) + +/* + * Circular queue definitions. + */ +#define CIRCLEQ_HEAD(name, type) \ +struct name { \ + struct type *cqh_first; /* first element */ \ + struct type *cqh_last; /* last element */ \ +} + +#define CIRCLEQ_HEAD_INITIALIZER(head) \ + { CIRCLEQ_END(&head), CIRCLEQ_END(&head) } + +#define CIRCLEQ_ENTRY(type) \ +struct { \ + struct type *cqe_next; /* next element */ \ + struct type *cqe_prev; /* previous element */ \ +} + +/* + * Circular queue access methods + */ +#define CIRCLEQ_FIRST(head) ((head)->cqh_first) +#define CIRCLEQ_LAST(head) ((head)->cqh_last) +#define CIRCLEQ_END(head) ((void *)(head)) +#define CIRCLEQ_NEXT(elm, field) ((elm)->field.cqe_next) +#define CIRCLEQ_PREV(elm, field) ((elm)->field.cqe_prev) +#define CIRCLEQ_EMPTY(head) \ + (CIRCLEQ_FIRST(head) == CIRCLEQ_END(head)) + +#define CIRCLEQ_FOREACH(var, head, field) \ + for((var) = CIRCLEQ_FIRST(head); \ + (var) != CIRCLEQ_END(head); \ + (var) = CIRCLEQ_NEXT(var, field)) + +#define CIRCLEQ_FOREACH_REVERSE(var, head, field) \ + for((var) = CIRCLEQ_LAST(head); \ + (var) != CIRCLEQ_END(head); \ + (var) = CIRCLEQ_PREV(var, field)) + +/* + * Circular queue functions. + */ +#define CIRCLEQ_INIT(head) do { \ + (head)->cqh_first = CIRCLEQ_END(head); \ + (head)->cqh_last = CIRCLEQ_END(head); \ +} while (0) + +#define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do { \ + (elm)->field.cqe_next = (listelm)->field.cqe_next; \ + (elm)->field.cqe_prev = (listelm); \ + if ((listelm)->field.cqe_next == CIRCLEQ_END(head)) \ + (head)->cqh_last = (elm); \ + else \ + (listelm)->field.cqe_next->field.cqe_prev = (elm); \ + (listelm)->field.cqe_next = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do { \ + (elm)->field.cqe_next = (listelm); \ + (elm)->field.cqe_prev = (listelm)->field.cqe_prev; \ + if ((listelm)->field.cqe_prev == CIRCLEQ_END(head)) \ + (head)->cqh_first = (elm); \ + else \ + (listelm)->field.cqe_prev->field.cqe_next = (elm); \ + (listelm)->field.cqe_prev = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.cqe_next = (head)->cqh_first; \ + (elm)->field.cqe_prev = CIRCLEQ_END(head); \ + if ((head)->cqh_last == CIRCLEQ_END(head)) \ + (head)->cqh_last = (elm); \ + else \ + (head)->cqh_first->field.cqe_prev = (elm); \ + (head)->cqh_first = (elm); \ +} while (0) + +#define CIRCLEQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.cqe_next = CIRCLEQ_END(head); \ + (elm)->field.cqe_prev = (head)->cqh_last; \ + if ((head)->cqh_first == CIRCLEQ_END(head)) \ + (head)->cqh_first = (elm); \ + else \ + (head)->cqh_last->field.cqe_next = (elm); \ + (head)->cqh_last = (elm); \ +} while (0) + +#define CIRCLEQ_REMOVE(head, elm, field) do { \ + if ((elm)->field.cqe_next == CIRCLEQ_END(head)) \ + (head)->cqh_last = (elm)->field.cqe_prev; \ + else \ + (elm)->field.cqe_next->field.cqe_prev = \ + (elm)->field.cqe_prev; \ + if ((elm)->field.cqe_prev == CIRCLEQ_END(head)) \ + (head)->cqh_first = (elm)->field.cqe_next; \ + else \ + (elm)->field.cqe_prev->field.cqe_next = \ + (elm)->field.cqe_next; \ +} while (0) + +#define CIRCLEQ_REPLACE(head, elm, elm2, field) do { \ + if (((elm2)->field.cqe_next = (elm)->field.cqe_next) == \ + CIRCLEQ_END(head)) \ + (head).cqh_last = (elm2); \ + else \ + (elm2)->field.cqe_next->field.cqe_prev = (elm2); \ + if (((elm2)->field.cqe_prev = (elm)->field.cqe_prev) == \ + CIRCLEQ_END(head)) \ + (head).cqh_first = (elm2); \ + else \ + (elm2)->field.cqe_prev->field.cqe_next = (elm2); \ +} while (0) + +#endif /* !_SYS_QUEUE_H_ */ === added file 'extra/libevent/compat/sys/tree.h' --- a/extra/libevent/compat/sys/tree.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/compat/sys/tree.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,677 @@ +/* $OpenBSD: tree.h,v 1.7 2002/10/17 21:51:54 art Exp $ */ +/* + * Copyright 2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_TREE_H_ +#define _SYS_TREE_H_ + +/* + * This file defines data structures for different types of trees: + * splay trees and red-black trees. + * + * A splay tree is a self-organizing data structure. Every operation + * on the tree causes a splay to happen. The splay moves the requested + * node to the root of the tree and partly rebalances it. + * + * This has the benefit that request locality causes faster lookups as + * the requested nodes move to the top of the tree. On the other hand, + * every lookup causes memory writes. + * + * The Balance Theorem bounds the total access time for m operations + * and n inserts on an initially empty tree as O((m + n)lg n). The + * amortized cost for a sequence of m accesses to a splay tree is O(lg n); + * + * A red-black tree is a binary search tree with the node color as an + * extra attribute. It fulfills a set of conditions: + * - every search path from the root to a leaf consists of the + * same number of black nodes, + * - each red node (except for the root) has a black parent, + * - each leaf node is black. + * + * Every operation on a red-black tree is bounded as O(lg n). + * The maximum height of a red-black tree is 2lg (n+1). + */ + +#define SPLAY_HEAD(name, type) \ +struct name { \ + struct type *sph_root; /* root of the tree */ \ +} + +#define SPLAY_INITIALIZER(root) \ + { NULL } + +#define SPLAY_INIT(root) do { \ + (root)->sph_root = NULL; \ +} while (0) + +#define SPLAY_ENTRY(type) \ +struct { \ + struct type *spe_left; /* left element */ \ + struct type *spe_right; /* right element */ \ +} + +#define SPLAY_LEFT(elm, field) (elm)->field.spe_left +#define SPLAY_RIGHT(elm, field) (elm)->field.spe_right +#define SPLAY_ROOT(head) (head)->sph_root +#define SPLAY_EMPTY(head) (SPLAY_ROOT(head) == NULL) + +/* SPLAY_ROTATE_{LEFT,RIGHT} expect that tmp hold SPLAY_{RIGHT,LEFT} */ +#define SPLAY_ROTATE_RIGHT(head, tmp, field) do { \ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(tmp, field); \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_ROTATE_LEFT(head, tmp, field) do { \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(tmp, field); \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + (head)->sph_root = tmp; \ +} while (0) + +#define SPLAY_LINKLEFT(head, tmp, field) do { \ + SPLAY_LEFT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_LINKRIGHT(head, tmp, field) do { \ + SPLAY_RIGHT(tmp, field) = (head)->sph_root; \ + tmp = (head)->sph_root; \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field); \ +} while (0) + +#define SPLAY_ASSEMBLE(head, node, left, right, field) do { \ + SPLAY_RIGHT(left, field) = SPLAY_LEFT((head)->sph_root, field); \ + SPLAY_LEFT(right, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT((head)->sph_root, field) = SPLAY_RIGHT(node, field); \ + SPLAY_RIGHT((head)->sph_root, field) = SPLAY_LEFT(node, field); \ +} while (0) + +/* Generates prototypes and inline functions */ + +#define SPLAY_PROTOTYPE(name, type, field, cmp) \ +void name##_SPLAY(struct name *, struct type *); \ +void name##_SPLAY_MINMAX(struct name *, int); \ +struct type *name##_SPLAY_INSERT(struct name *, struct type *); \ +struct type *name##_SPLAY_REMOVE(struct name *, struct type *); \ + \ +/* Finds the node with the same key as elm */ \ +static __inline struct type * \ +name##_SPLAY_FIND(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) \ + return(NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) \ + return (head->sph_root); \ + return (NULL); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_NEXT(struct name *head, struct type *elm) \ +{ \ + name##_SPLAY(head, elm); \ + if (SPLAY_RIGHT(elm, field) != NULL) { \ + elm = SPLAY_RIGHT(elm, field); \ + while (SPLAY_LEFT(elm, field) != NULL) { \ + elm = SPLAY_LEFT(elm, field); \ + } \ + } else \ + elm = NULL; \ + return (elm); \ +} \ + \ +static __inline struct type * \ +name##_SPLAY_MIN_MAX(struct name *head, int val) \ +{ \ + name##_SPLAY_MINMAX(head, val); \ + return (SPLAY_ROOT(head)); \ +} + +/* Main splay operation. + * Moves node close to the key of elm to top + */ +#define SPLAY_GENERATE(name, type, field, cmp) \ +struct type * \ +name##_SPLAY_INSERT(struct name *head, struct type *elm) \ +{ \ + if (SPLAY_EMPTY(head)) { \ + SPLAY_LEFT(elm, field) = SPLAY_RIGHT(elm, field) = NULL; \ + } else { \ + int __comp; \ + name##_SPLAY(head, elm); \ + __comp = (cmp)(elm, (head)->sph_root); \ + if(__comp < 0) { \ + SPLAY_LEFT(elm, field) = SPLAY_LEFT((head)->sph_root, field);\ + SPLAY_RIGHT(elm, field) = (head)->sph_root; \ + SPLAY_LEFT((head)->sph_root, field) = NULL; \ + } else if (__comp > 0) { \ + SPLAY_RIGHT(elm, field) = SPLAY_RIGHT((head)->sph_root, field);\ + SPLAY_LEFT(elm, field) = (head)->sph_root; \ + SPLAY_RIGHT((head)->sph_root, field) = NULL; \ + } else \ + return ((head)->sph_root); \ + } \ + (head)->sph_root = (elm); \ + return (NULL); \ +} \ + \ +struct type * \ +name##_SPLAY_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *__tmp; \ + if (SPLAY_EMPTY(head)) \ + return (NULL); \ + name##_SPLAY(head, elm); \ + if ((cmp)(elm, (head)->sph_root) == 0) { \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL) { \ + (head)->sph_root = SPLAY_RIGHT((head)->sph_root, field);\ + } else { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + (head)->sph_root = SPLAY_LEFT((head)->sph_root, field);\ + name##_SPLAY(head, elm); \ + SPLAY_RIGHT((head)->sph_root, field) = __tmp; \ + } \ + return (elm); \ + } \ + return (NULL); \ +} \ + \ +void \ +name##_SPLAY(struct name *head, struct type *elm) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ + int __comp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while ((__comp = (cmp)(elm, (head)->sph_root))) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if ((cmp)(elm, __tmp) > 0){ \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} \ + \ +/* Splay with either the minimum or the maximum element \ + * Used to find minimum or maximum element in tree. \ + */ \ +void name##_SPLAY_MINMAX(struct name *head, int __comp) \ +{ \ + struct type __node, *__left, *__right, *__tmp; \ +\ + SPLAY_LEFT(&__node, field) = SPLAY_RIGHT(&__node, field) = NULL;\ + __left = __right = &__node; \ +\ + while (1) { \ + if (__comp < 0) { \ + __tmp = SPLAY_LEFT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp < 0){ \ + SPLAY_ROTATE_RIGHT(head, __tmp, field); \ + if (SPLAY_LEFT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKLEFT(head, __right, field); \ + } else if (__comp > 0) { \ + __tmp = SPLAY_RIGHT((head)->sph_root, field); \ + if (__tmp == NULL) \ + break; \ + if (__comp > 0) { \ + SPLAY_ROTATE_LEFT(head, __tmp, field); \ + if (SPLAY_RIGHT((head)->sph_root, field) == NULL)\ + break; \ + } \ + SPLAY_LINKRIGHT(head, __left, field); \ + } \ + } \ + SPLAY_ASSEMBLE(head, &__node, __left, __right, field); \ +} + +#define SPLAY_NEGINF -1 +#define SPLAY_INF 1 + +#define SPLAY_INSERT(name, x, y) name##_SPLAY_INSERT(x, y) +#define SPLAY_REMOVE(name, x, y) name##_SPLAY_REMOVE(x, y) +#define SPLAY_FIND(name, x, y) name##_SPLAY_FIND(x, y) +#define SPLAY_NEXT(name, x, y) name##_SPLAY_NEXT(x, y) +#define SPLAY_MIN(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_NEGINF)) +#define SPLAY_MAX(name, x) (SPLAY_EMPTY(x) ? NULL \ + : name##_SPLAY_MIN_MAX(x, SPLAY_INF)) + +#define SPLAY_FOREACH(x, name, head) \ + for ((x) = SPLAY_MIN(name, head); \ + (x) != NULL; \ + (x) = SPLAY_NEXT(name, head, x)) + +/* Macros that define a red-back tree */ +#define RB_HEAD(name, type) \ +struct name { \ + struct type *rbh_root; /* root of the tree */ \ +} + +#define RB_INITIALIZER(root) \ + { NULL } + +#define RB_INIT(root) do { \ + (root)->rbh_root = NULL; \ +} while (0) + +#define RB_BLACK 0 +#define RB_RED 1 +#define RB_ENTRY(type) \ +struct { \ + struct type *rbe_left; /* left element */ \ + struct type *rbe_right; /* right element */ \ + struct type *rbe_parent; /* parent element */ \ + int rbe_color; /* node color */ \ +} + +#define RB_LEFT(elm, field) (elm)->field.rbe_left +#define RB_RIGHT(elm, field) (elm)->field.rbe_right +#define RB_PARENT(elm, field) (elm)->field.rbe_parent +#define RB_COLOR(elm, field) (elm)->field.rbe_color +#define RB_ROOT(head) (head)->rbh_root +#define RB_EMPTY(head) (RB_ROOT(head) == NULL) + +#define RB_SET(elm, parent, field) do { \ + RB_PARENT(elm, field) = parent; \ + RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \ + RB_COLOR(elm, field) = RB_RED; \ +} while (0) + +#define RB_SET_BLACKRED(black, red, field) do { \ + RB_COLOR(black, field) = RB_BLACK; \ + RB_COLOR(red, field) = RB_RED; \ +} while (0) + +#ifndef RB_AUGMENT +#define RB_AUGMENT(x) +#endif + +#define RB_ROTATE_LEFT(head, elm, tmp, field) do { \ + (tmp) = RB_RIGHT(elm, field); \ + if ((RB_RIGHT(elm, field) = RB_LEFT(tmp, field))) { \ + RB_PARENT(RB_LEFT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_LEFT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \ + (tmp) = RB_LEFT(elm, field); \ + if ((RB_LEFT(elm, field) = RB_RIGHT(tmp, field))) { \ + RB_PARENT(RB_RIGHT(tmp, field), field) = (elm); \ + } \ + RB_AUGMENT(elm); \ + if ((RB_PARENT(tmp, field) = RB_PARENT(elm, field))) { \ + if ((elm) == RB_LEFT(RB_PARENT(elm, field), field)) \ + RB_LEFT(RB_PARENT(elm, field), field) = (tmp); \ + else \ + RB_RIGHT(RB_PARENT(elm, field), field) = (tmp); \ + } else \ + (head)->rbh_root = (tmp); \ + RB_RIGHT(tmp, field) = (elm); \ + RB_PARENT(elm, field) = (tmp); \ + RB_AUGMENT(tmp); \ + if ((RB_PARENT(tmp, field))) \ + RB_AUGMENT(RB_PARENT(tmp, field)); \ +} while (0) + +/* Generates prototypes and inline functions */ +#define RB_PROTOTYPE(name, type, field, cmp) \ +void name##_RB_INSERT_COLOR(struct name *, struct type *); \ +void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\ +struct type *name##_RB_REMOVE(struct name *, struct type *); \ +struct type *name##_RB_INSERT(struct name *, struct type *); \ +struct type *name##_RB_FIND(struct name *, struct type *); \ +struct type *name##_RB_NEXT(struct type *); \ +struct type *name##_RB_MINMAX(struct name *, int); \ + \ + +/* Main rb operation. + * Moves node close to the key of elm to top + */ +#define RB_GENERATE(name, type, field, cmp) \ +void \ +name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \ +{ \ + struct type *parent, *gparent, *tmp; \ + while ((parent = RB_PARENT(elm, field)) && \ + RB_COLOR(parent, field) == RB_RED) { \ + gparent = RB_PARENT(parent, field); \ + if (parent == RB_LEFT(gparent, field)) { \ + tmp = RB_RIGHT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_RIGHT(parent, field) == elm) { \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_RIGHT(head, gparent, tmp, field); \ + } else { \ + tmp = RB_LEFT(gparent, field); \ + if (tmp && RB_COLOR(tmp, field) == RB_RED) { \ + RB_COLOR(tmp, field) = RB_BLACK; \ + RB_SET_BLACKRED(parent, gparent, field);\ + elm = gparent; \ + continue; \ + } \ + if (RB_LEFT(parent, field) == elm) { \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = parent; \ + parent = elm; \ + elm = tmp; \ + } \ + RB_SET_BLACKRED(parent, gparent, field); \ + RB_ROTATE_LEFT(head, gparent, tmp, field); \ + } \ + } \ + RB_COLOR(head->rbh_root, field) = RB_BLACK; \ +} \ + \ +void \ +name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \ +{ \ + struct type *tmp; \ + while ((elm == NULL || RB_COLOR(elm, field) == RB_BLACK) && \ + elm != RB_ROOT(head)) { \ + if (RB_LEFT(parent, field) == elm) { \ + tmp = RB_RIGHT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK) {\ + struct type *oleft; \ + if ((oleft = RB_LEFT(tmp, field)))\ + RB_COLOR(oleft, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_RIGHT(head, tmp, oleft, field);\ + tmp = RB_RIGHT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_RIGHT(tmp, field)) \ + RB_COLOR(RB_RIGHT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_LEFT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } else { \ + tmp = RB_LEFT(parent, field); \ + if (RB_COLOR(tmp, field) == RB_RED) { \ + RB_SET_BLACKRED(tmp, parent, field); \ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + if ((RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) &&\ + (RB_RIGHT(tmp, field) == NULL || \ + RB_COLOR(RB_RIGHT(tmp, field), field) == RB_BLACK)) {\ + RB_COLOR(tmp, field) = RB_RED; \ + elm = parent; \ + parent = RB_PARENT(elm, field); \ + } else { \ + if (RB_LEFT(tmp, field) == NULL || \ + RB_COLOR(RB_LEFT(tmp, field), field) == RB_BLACK) {\ + struct type *oright; \ + if ((oright = RB_RIGHT(tmp, field)))\ + RB_COLOR(oright, field) = RB_BLACK;\ + RB_COLOR(tmp, field) = RB_RED; \ + RB_ROTATE_LEFT(head, tmp, oright, field);\ + tmp = RB_LEFT(parent, field); \ + } \ + RB_COLOR(tmp, field) = RB_COLOR(parent, field);\ + RB_COLOR(parent, field) = RB_BLACK; \ + if (RB_LEFT(tmp, field)) \ + RB_COLOR(RB_LEFT(tmp, field), field) = RB_BLACK;\ + RB_ROTATE_RIGHT(head, parent, tmp, field);\ + elm = RB_ROOT(head); \ + break; \ + } \ + } \ + } \ + if (elm) \ + RB_COLOR(elm, field) = RB_BLACK; \ +} \ + \ +struct type * \ +name##_RB_REMOVE(struct name *head, struct type *elm) \ +{ \ + struct type *child, *parent, *old = elm; \ + int color; \ + if (RB_LEFT(elm, field) == NULL) \ + child = RB_RIGHT(elm, field); \ + else if (RB_RIGHT(elm, field) == NULL) \ + child = RB_LEFT(elm, field); \ + else { \ + struct type *left; \ + elm = RB_RIGHT(elm, field); \ + while ((left = RB_LEFT(elm, field))) \ + elm = left; \ + child = RB_RIGHT(elm, field); \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ + if (RB_PARENT(elm, field) == old) \ + parent = elm; \ + (elm)->field = (old)->field; \ + if (RB_PARENT(old, field)) { \ + if (RB_LEFT(RB_PARENT(old, field), field) == old)\ + RB_LEFT(RB_PARENT(old, field), field) = elm;\ + else \ + RB_RIGHT(RB_PARENT(old, field), field) = elm;\ + RB_AUGMENT(RB_PARENT(old, field)); \ + } else \ + RB_ROOT(head) = elm; \ + RB_PARENT(RB_LEFT(old, field), field) = elm; \ + if (RB_RIGHT(old, field)) \ + RB_PARENT(RB_RIGHT(old, field), field) = elm; \ + if (parent) { \ + left = parent; \ + do { \ + RB_AUGMENT(left); \ + } while ((left = RB_PARENT(left, field))); \ + } \ + goto color; \ + } \ + parent = RB_PARENT(elm, field); \ + color = RB_COLOR(elm, field); \ + if (child) \ + RB_PARENT(child, field) = parent; \ + if (parent) { \ + if (RB_LEFT(parent, field) == elm) \ + RB_LEFT(parent, field) = child; \ + else \ + RB_RIGHT(parent, field) = child; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = child; \ +color: \ + if (color == RB_BLACK) \ + name##_RB_REMOVE_COLOR(head, parent, child); \ + return (old); \ +} \ + \ +/* Inserts a node into the RB tree */ \ +struct type * \ +name##_RB_INSERT(struct name *head, struct type *elm) \ +{ \ + struct type *tmp; \ + struct type *parent = NULL; \ + int comp = 0; \ + tmp = RB_ROOT(head); \ + while (tmp) { \ + parent = tmp; \ + comp = (cmp)(elm, parent); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + RB_SET(elm, parent, field); \ + if (parent != NULL) { \ + if (comp < 0) \ + RB_LEFT(parent, field) = elm; \ + else \ + RB_RIGHT(parent, field) = elm; \ + RB_AUGMENT(parent); \ + } else \ + RB_ROOT(head) = elm; \ + name##_RB_INSERT_COLOR(head, elm); \ + return (NULL); \ +} \ + \ +/* Finds the node with the same key as elm */ \ +struct type * \ +name##_RB_FIND(struct name *head, struct type *elm) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + int comp; \ + while (tmp) { \ + comp = cmp(elm, tmp); \ + if (comp < 0) \ + tmp = RB_LEFT(tmp, field); \ + else if (comp > 0) \ + tmp = RB_RIGHT(tmp, field); \ + else \ + return (tmp); \ + } \ + return (NULL); \ +} \ + \ +struct type * \ +name##_RB_NEXT(struct type *elm) \ +{ \ + if (RB_RIGHT(elm, field)) { \ + elm = RB_RIGHT(elm, field); \ + while (RB_LEFT(elm, field)) \ + elm = RB_LEFT(elm, field); \ + } else { \ + if (RB_PARENT(elm, field) && \ + (elm == RB_LEFT(RB_PARENT(elm, field), field))) \ + elm = RB_PARENT(elm, field); \ + else { \ + while (RB_PARENT(elm, field) && \ + (elm == RB_RIGHT(RB_PARENT(elm, field), field)))\ + elm = RB_PARENT(elm, field); \ + elm = RB_PARENT(elm, field); \ + } \ + } \ + return (elm); \ +} \ + \ +struct type * \ +name##_RB_MINMAX(struct name *head, int val) \ +{ \ + struct type *tmp = RB_ROOT(head); \ + struct type *parent = NULL; \ + while (tmp) { \ + parent = tmp; \ + if (val < 0) \ + tmp = RB_LEFT(tmp, field); \ + else \ + tmp = RB_RIGHT(tmp, field); \ + } \ + return (parent); \ +} + +#define RB_NEGINF -1 +#define RB_INF 1 + +#define RB_INSERT(name, x, y) name##_RB_INSERT(x, y) +#define RB_REMOVE(name, x, y) name##_RB_REMOVE(x, y) +#define RB_FIND(name, x, y) name##_RB_FIND(x, y) +#define RB_NEXT(name, x, y) name##_RB_NEXT(y) +#define RB_MIN(name, x) name##_RB_MINMAX(x, RB_NEGINF) +#define RB_MAX(name, x) name##_RB_MINMAX(x, RB_INF) + +#define RB_FOREACH(x, name, head) \ + for ((x) = RB_MIN(name, head); \ + (x) != NULL; \ + (x) = name##_RB_NEXT(x)) + +#endif /* _SYS_TREE_H_ */ === added file 'extra/libevent/devpoll.c' --- a/extra/libevent/devpoll.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/devpoll.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,421 @@ +/* + * Copyright 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_DEVPOLL + +#include <sys/types.h> +#include <sys/resource.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <sys/queue.h> +#include <sys/devpoll.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <assert.h> + +#include "event.h" +#include "event-internal.h" +#include "evsignal.h" +#include "log.h" + +/* due to limitations in the devpoll interface, we need to keep track of + * all file descriptors outself. + */ +struct evdevpoll { + struct event *evread; + struct event *evwrite; +}; + +struct devpollop { + struct evdevpoll *fds; + int nfds; + struct pollfd *events; + int nevents; + int dpfd; + struct pollfd *changes; + int nchanges; +}; + +static void *devpoll_init (struct event_base *); +static int devpoll_add (void *, struct event *); +static int devpoll_del (void *, struct event *); +static int devpoll_dispatch (struct event_base *, void *, struct timeval *); +static void devpoll_dealloc (struct event_base *, void *); + +struct eventop devpollops = { + "devpoll", + devpoll_init, + devpoll_add, + devpoll_del, + devpoll_dispatch, + devpoll_dealloc, + 1 /* need reinit */ +}; + +#define NEVENT 32000 + +static int +devpoll_commit(struct devpollop *devpollop) +{ + /* + * Due to a bug in Solaris, we have to use pwrite with an offset of 0. + * Write is limited to 2GB of data, until it will fail. + */ + if (pwrite(devpollop->dpfd, devpollop->changes, + sizeof(struct pollfd) * devpollop->nchanges, 0) == -1) + return(-1); + + devpollop->nchanges = 0; + return(0); +} + +static int +devpoll_queue(struct devpollop *devpollop, int fd, int events) { + struct pollfd *pfd; + + if (devpollop->nchanges >= devpollop->nevents) { + /* + * Change buffer is full, must commit it to /dev/poll before + * adding more + */ + if (devpoll_commit(devpollop) != 0) + return(-1); + } + + pfd = &devpollop->changes[devpollop->nchanges++]; + pfd->fd = fd; + pfd->events = events; + pfd->revents = 0; + + return(0); +} + +static void * +devpoll_init(struct event_base *base) +{ + int dpfd, nfiles = NEVENT; + struct rlimit rl; + struct devpollop *devpollop; + + /* Disable devpoll when this environment variable is set */ + if (getenv("EVENT_NODEVPOLL")) + return (NULL); + + if (!(devpollop = calloc(1, sizeof(struct devpollop)))) + return (NULL); + + if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && + rl.rlim_cur != RLIM_INFINITY) + nfiles = rl.rlim_cur - 1; + + /* Initialize the kernel queue */ + if ((dpfd = open("/dev/poll", O_RDWR)) == -1) { + event_warn("open: /dev/poll"); + free(devpollop); + return (NULL); + } + + devpollop->dpfd = dpfd; + + /* Initialize fields */ + devpollop->events = calloc(nfiles, sizeof(struct pollfd)); + if (devpollop->events == NULL) { + free(devpollop); + close(dpfd); + return (NULL); + } + devpollop->nevents = nfiles; + + devpollop->fds = calloc(nfiles, sizeof(struct evdevpoll)); + if (devpollop->fds == NULL) { + free(devpollop->events); + free(devpollop); + close(dpfd); + return (NULL); + } + devpollop->nfds = nfiles; + + devpollop->changes = calloc(nfiles, sizeof(struct pollfd)); + if (devpollop->changes == NULL) { + free(devpollop->fds); + free(devpollop->events); + free(devpollop); + close(dpfd); + return (NULL); + } + + evsignal_init(base); + + return (devpollop); +} + +static int +devpoll_recalc(struct event_base *base, void *arg, int max) +{ + struct devpollop *devpollop = arg; + + if (max > devpollop->nfds) { + struct evdevpoll *fds; + int nfds; + + nfds = devpollop->nfds; + while (nfds < max) + nfds <<= 1; + + fds = realloc(devpollop->fds, nfds * sizeof(struct evdevpoll)); + if (fds == NULL) { + event_warn("realloc"); + return (-1); + } + devpollop->fds = fds; + memset(fds + devpollop->nfds, 0, + (nfds - devpollop->nfds) * sizeof(struct evdevpoll)); + devpollop->nfds = nfds; + } + + return (0); +} + +static int +devpoll_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + struct devpollop *devpollop = arg; + struct pollfd *events = devpollop->events; + struct dvpoll dvp; + struct evdevpoll *evdp; + int i, res, timeout = -1; + + if (devpollop->nchanges) + devpoll_commit(devpollop); + + if (tv != NULL) + timeout = tv->tv_sec * 1000 + (tv->tv_usec + 999) / 1000; + + dvp.dp_fds = devpollop->events; + dvp.dp_nfds = devpollop->nevents; + dvp.dp_timeout = timeout; + + res = ioctl(devpollop->dpfd, DP_POLL, &dvp); + + if (res == -1) { + if (errno != EINTR) { + event_warn("ioctl: DP_POLL"); + return (-1); + } + + evsignal_process(base); + return (0); + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + event_debug(("%s: devpoll_wait reports %d", __func__, res)); + + for (i = 0; i < res; i++) { + int which = 0; + int what = events[i].revents; + struct event *evread = NULL, *evwrite = NULL; + + assert(events[i].fd < devpollop->nfds); + evdp = &devpollop->fds[events[i].fd]; + + if (what & POLLHUP) + what |= POLLIN | POLLOUT; + else if (what & POLLERR) + what |= POLLIN | POLLOUT; + + if (what & POLLIN) { + evread = evdp->evread; + which |= EV_READ; + } + + if (what & POLLOUT) { + evwrite = evdp->evwrite; + which |= EV_WRITE; + } + + if (!which) + continue; + + if (evread != NULL && !(evread->ev_events & EV_PERSIST)) + event_del(evread); + if (evwrite != NULL && evwrite != evread && + !(evwrite->ev_events & EV_PERSIST)) + event_del(evwrite); + + if (evread != NULL) + event_active(evread, EV_READ, 1); + if (evwrite != NULL) + event_active(evwrite, EV_WRITE, 1); + } + + return (0); +} + + +static int +devpoll_add(void *arg, struct event *ev) +{ + struct devpollop *devpollop = arg; + struct evdevpoll *evdp; + int fd, events; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_add(ev)); + + fd = ev->ev_fd; + if (fd >= devpollop->nfds) { + /* Extend the file descriptor array as necessary */ + if (devpoll_recalc(ev->ev_base, devpollop, fd) == -1) + return (-1); + } + evdp = &devpollop->fds[fd]; + + /* + * It's not necessary to OR the existing read/write events that we + * are currently interested in with the new event we are adding. + * The /dev/poll driver ORs any new events with the existing events + * that it has cached for the fd. + */ + + events = 0; + if (ev->ev_events & EV_READ) { + if (evdp->evread && evdp->evread != ev) { + /* There is already a different read event registered */ + return(-1); + } + events |= POLLIN; + } + + if (ev->ev_events & EV_WRITE) { + if (evdp->evwrite && evdp->evwrite != ev) { + /* There is already a different write event registered */ + return(-1); + } + events |= POLLOUT; + } + + if (devpoll_queue(devpollop, fd, events) != 0) + return(-1); + + /* Update events responsible */ + if (ev->ev_events & EV_READ) + evdp->evread = ev; + if (ev->ev_events & EV_WRITE) + evdp->evwrite = ev; + + return (0); +} + +static int +devpoll_del(void *arg, struct event *ev) +{ + struct devpollop *devpollop = arg; + struct evdevpoll *evdp; + int fd, events; + int needwritedelete = 1, needreaddelete = 1; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_del(ev)); + + fd = ev->ev_fd; + if (fd >= devpollop->nfds) + return (0); + evdp = &devpollop->fds[fd]; + + events = 0; + if (ev->ev_events & EV_READ) + events |= POLLIN; + if (ev->ev_events & EV_WRITE) + events |= POLLOUT; + + /* + * The only way to remove an fd from the /dev/poll monitored set is + * to use POLLREMOVE by itself. This removes ALL events for the fd + * provided so if we care about two events and are only removing one + * we must re-add the other event after POLLREMOVE. + */ + + if (devpoll_queue(devpollop, fd, POLLREMOVE) != 0) + return(-1); + + if ((events & (POLLIN|POLLOUT)) != (POLLIN|POLLOUT)) { + /* + * We're not deleting all events, so we must resubmit the + * event that we are still interested in if one exists. + */ + + if ((events & POLLIN) && evdp->evwrite != NULL) { + /* Deleting read, still care about write */ + devpoll_queue(devpollop, fd, POLLOUT); + needwritedelete = 0; + } else if ((events & POLLOUT) && evdp->evread != NULL) { + /* Deleting write, still care about read */ + devpoll_queue(devpollop, fd, POLLIN); + needreaddelete = 0; + } + } + + if (needreaddelete) + evdp->evread = NULL; + if (needwritedelete) + evdp->evwrite = NULL; + + return (0); +} + +static void +devpoll_dealloc(struct event_base *base, void *arg) +{ + struct devpollop *devpollop = arg; + + evsignal_dealloc(base); + if (devpollop->fds) + free(devpollop->fds); + if (devpollop->events) + free(devpollop->events); + if (devpollop->changes) + free(devpollop->changes); + if (devpollop->dpfd >= 0) + close(devpollop->dpfd); + + memset(devpollop, 0, sizeof(struct devpollop)); + free(devpollop); +} + +#endif /* HAVE_DEVPOLL */ === added file 'extra/libevent/epoll.c' --- a/extra/libevent/epoll.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/epoll.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,359 @@ +/* + * Copyright 2000-2003 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_EPOLL + +#include <stdint.h> +#include <sys/types.h> +#include <sys/resource.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <sys/queue.h> +#include <sys/epoll.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#include "event.h" +#include "event-internal.h" +#include "evsignal.h" +#include "log.h" + +/* due to limitations in the epoll interface, we need to keep track of + * all file descriptors outself. + */ +struct evepoll { + struct event *evread; + struct event *evwrite; +}; + +struct epollop { + struct evepoll *fds; + int nfds; + struct epoll_event *events; + int nevents; + int epfd; +}; + +static void *epoll_init (struct event_base *); +static int epoll_add (void *, struct event *); +static int epoll_del (void *, struct event *); +static int epoll_dispatch (struct event_base *, void *, struct timeval *); +static void epoll_dealloc (struct event_base *, void *); + +struct eventop epollops = { + "epoll", + epoll_init, + epoll_add, + epoll_del, + epoll_dispatch, + epoll_dealloc, + 1 /* need reinit */ +}; + +#ifdef HAVE_SETFD +#define FD_CLOSEONEXEC(x) do { \ + if (fcntl(x, F_SETFD, 1) == -1) \ + event_warn("fcntl(%d, F_SETFD)", x); \ +} while (0) +#else +#define FD_CLOSEONEXEC(x) +#endif + +#define NEVENT 32000 + +static void * +epoll_init(struct event_base *base) +{ + int epfd, nfiles = NEVENT; + struct rlimit rl; + struct epollop *epollop; + + /* Disable epollueue when this environment variable is set */ + if (getenv("EVENT_NOEPOLL")) + return (NULL); + + if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && + rl.rlim_cur != RLIM_INFINITY) { + /* + * Solaris is somewhat retarded - it's important to drop + * backwards compatibility when making changes. So, don't + * dare to put rl.rlim_cur here. + */ + nfiles = rl.rlim_cur - 1; + } + + /* Initalize the kernel queue */ + + if ((epfd = epoll_create(nfiles)) == -1) { + event_warn("epoll_create"); + return (NULL); + } + + FD_CLOSEONEXEC(epfd); + + if (!(epollop = calloc(1, sizeof(struct epollop)))) + return (NULL); + + epollop->epfd = epfd; + + /* Initalize fields */ + epollop->events = malloc(nfiles * sizeof(struct epoll_event)); + if (epollop->events == NULL) { + free(epollop); + return (NULL); + } + epollop->nevents = nfiles; + + epollop->fds = calloc(nfiles, sizeof(struct evepoll)); + if (epollop->fds == NULL) { + free(epollop->events); + free(epollop); + return (NULL); + } + epollop->nfds = nfiles; + + evsignal_init(base); + + return (epollop); +} + +static int +epoll_recalc(struct event_base *base, void *arg, int max) +{ + struct epollop *epollop = arg; + + if (max > epollop->nfds) { + struct evepoll *fds; + int nfds; + + nfds = epollop->nfds; + while (nfds < max) + nfds <<= 1; + + fds = realloc(epollop->fds, nfds * sizeof(struct evepoll)); + if (fds == NULL) { + event_warn("realloc"); + return (-1); + } + epollop->fds = fds; + memset(fds + epollop->nfds, 0, + (nfds - epollop->nfds) * sizeof(struct evepoll)); + epollop->nfds = nfds; + } + + return (0); +} + +static int +epoll_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + struct epollop *epollop = arg; + struct epoll_event *events = epollop->events; + struct evepoll *evep; + int i, res, timeout = -1; + + if (tv != NULL) + timeout = tv->tv_sec * 1000 + (tv->tv_usec + 999) / 1000; + + res = epoll_wait(epollop->epfd, events, epollop->nevents, timeout); + + if (res == -1) { + if (errno != EINTR) { + event_warn("epoll_wait"); + return (-1); + } + + evsignal_process(base); + return (0); + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + event_debug(("%s: epoll_wait reports %d", __func__, res)); + + for (i = 0; i < res; i++) { + int what = events[i].events; + struct event *evread = NULL, *evwrite = NULL; + + evep = (struct evepoll *)events[i].data.ptr; + + if (what & (EPOLLHUP|EPOLLERR)) { + evread = evep->evread; + evwrite = evep->evwrite; + } else { + if (what & EPOLLIN) { + evread = evep->evread; + } + + if (what & EPOLLOUT) { + evwrite = evep->evwrite; + } + } + + if (!(evread||evwrite)) + continue; + + if (evread != NULL) + event_active(evread, EV_READ, 1); + if (evwrite != NULL) + event_active(evwrite, EV_WRITE, 1); + } + + return (0); +} + + +static int +epoll_add(void *arg, struct event *ev) +{ + struct epollop *epollop = arg; + struct epoll_event epev = {0, {0}}; + struct evepoll *evep; + int fd, op, events; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_add(ev)); + + fd = ev->ev_fd; + if (fd >= epollop->nfds) { + /* Extent the file descriptor array as necessary */ + if (epoll_recalc(ev->ev_base, epollop, fd) == -1) + return (-1); + } + evep = &epollop->fds[fd]; + op = EPOLL_CTL_ADD; + events = 0; + if (evep->evread != NULL) { + events |= EPOLLIN; + op = EPOLL_CTL_MOD; + } + if (evep->evwrite != NULL) { + events |= EPOLLOUT; + op = EPOLL_CTL_MOD; + } + + if (ev->ev_events & EV_READ) + events |= EPOLLIN; + if (ev->ev_events & EV_WRITE) + events |= EPOLLOUT; + + epev.data.ptr = evep; + epev.events = events; + if (epoll_ctl(epollop->epfd, op, ev->ev_fd, &epev) == -1) + return (-1); + + /* Update events responsible */ + if (ev->ev_events & EV_READ) + evep->evread = ev; + if (ev->ev_events & EV_WRITE) + evep->evwrite = ev; + + return (0); +} + +static int +epoll_del(void *arg, struct event *ev) +{ + struct epollop *epollop = arg; + struct epoll_event epev = {0, {0}}; + struct evepoll *evep; + int fd, events, op; + int needwritedelete = 1, needreaddelete = 1; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_del(ev)); + + fd = ev->ev_fd; + if (fd >= epollop->nfds) + return (0); + evep = &epollop->fds[fd]; + + op = EPOLL_CTL_DEL; + events = 0; + + if (ev->ev_events & EV_READ) + events |= EPOLLIN; + if (ev->ev_events & EV_WRITE) + events |= EPOLLOUT; + + if ((events & (EPOLLIN|EPOLLOUT)) != (EPOLLIN|EPOLLOUT)) { + if ((events & EPOLLIN) && evep->evwrite != NULL) { + needwritedelete = 0; + events = EPOLLOUT; + op = EPOLL_CTL_MOD; + } else if ((events & EPOLLOUT) && evep->evread != NULL) { + needreaddelete = 0; + events = EPOLLIN; + op = EPOLL_CTL_MOD; + } + } + + epev.events = events; + epev.data.ptr = evep; + + if (needreaddelete) + evep->evread = NULL; + if (needwritedelete) + evep->evwrite = NULL; + + if (epoll_ctl(epollop->epfd, op, fd, &epev) == -1) + return (-1); + + return (0); +} + +static void +epoll_dealloc(struct event_base *base, void *arg) +{ + struct epollop *epollop = arg; + + evsignal_dealloc(base); + if (epollop->fds) + free(epollop->fds); + if (epollop->events) + free(epollop->events); + if (epollop->epfd >= 0) + close(epollop->epfd); + + memset(epollop, 0, sizeof(struct epollop)); + free(epollop); +} + +#endif /* HAVE_EPOLL */ === added file 'extra/libevent/epoll_sub.c' --- a/extra/libevent/epoll_sub.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/epoll_sub.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,60 @@ +/* + * Copyright 2003 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_EPOLL + +#include <stdint.h> + +#include <sys/param.h> +#include <sys/types.h> +#include <sys/syscall.h> +#include <sys/epoll.h> +#include <unistd.h> + +int +epoll_create(int size) +{ + return (syscall(__NR_epoll_create, size)); +} + +int +epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) +{ + + return (syscall(__NR_epoll_ctl, epfd, op, fd, event)); +} + +int +epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) +{ + return (syscall(__NR_epoll_wait, epfd, events, maxevents, timeout)); +} + +#endif /* HAVE_EPOLL */ === added file 'extra/libevent/evbuffer.c' --- a/extra/libevent/evbuffer.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evbuffer.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2002-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/types.h> + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_STDARG_H +#include <stdarg.h> +#endif + +#ifdef WIN32 +#include <winsock2.h> +#endif + +#include "evutil.h" +#include "event.h" + +/* prototypes */ + +void bufferevent_setwatermark(struct bufferevent *, short, size_t, size_t); +void bufferevent_read_pressure_cb(struct evbuffer *, size_t, size_t, void *); + +static int +bufferevent_add(struct event *ev, int timeout) +{ + struct timeval tv, *ptv = NULL; + + if (timeout) { + evutil_timerclear(&tv); + tv.tv_sec = timeout; + ptv = &tv; + } + + return (event_add(ev, ptv)); +} + +/* + * This callback is executed when the size of the input buffer changes. + * We use it to apply back pressure on the reading side. + */ + +void +bufferevent_read_pressure_cb(struct evbuffer *buf, size_t old, size_t now, + void *arg) { + struct bufferevent *bufev = arg; + /* + * If we are below the watermark then reschedule reading if it's + * still enabled. + */ + if (bufev->wm_read.high == 0 || now < bufev->wm_read.high) { + evbuffer_setcb(buf, NULL, NULL); + + if (bufev->enabled & EV_READ) + bufferevent_add(&bufev->ev_read, bufev->timeout_read); + } +} + +static void +bufferevent_readcb(int fd, short event, void *arg) +{ + struct bufferevent *bufev = arg; + int res = 0; + short what = EVBUFFER_READ; + size_t len; + int howmuch = -1; + + if (event == EV_TIMEOUT) { + what |= EVBUFFER_TIMEOUT; + goto error; + } + + /* + * If we have a high watermark configured then we don't want to + * read more data than would make us reach the watermark. + */ + if (bufev->wm_read.high != 0) + howmuch = (int)bufev->wm_read.high; + + res = evbuffer_read(bufev->input, fd, howmuch); + if (res == -1) { + if (errno == EAGAIN || errno == EINTR) + goto reschedule; + /* error case */ + what |= EVBUFFER_ERROR; + } else if (res == 0) { + /* eof case */ + what |= EVBUFFER_EOF; + } + + if (res <= 0) + goto error; + + bufferevent_add(&bufev->ev_read, bufev->timeout_read); + + /* See if this callbacks meets the water marks */ + len = EVBUFFER_LENGTH(bufev->input); + if (bufev->wm_read.low != 0 && len < bufev->wm_read.low) + return; + if (bufev->wm_read.high != 0 && len > bufev->wm_read.high) { + struct evbuffer *buf = bufev->input; + event_del(&bufev->ev_read); + + /* Now schedule a callback for us */ + evbuffer_setcb(buf, bufferevent_read_pressure_cb, bufev); + return; + } + + /* Invoke the user callback - must always be called last */ + if (bufev->readcb != NULL) + (*bufev->readcb)(bufev, bufev->cbarg); + return; + + reschedule: + bufferevent_add(&bufev->ev_read, bufev->timeout_read); + return; + + error: + (*bufev->errorcb)(bufev, what, bufev->cbarg); +} + +static void +bufferevent_writecb(int fd, short event, void *arg) +{ + struct bufferevent *bufev = arg; + int res = 0; + short what = EVBUFFER_WRITE; + + if (event == EV_TIMEOUT) { + what |= EVBUFFER_TIMEOUT; + goto error; + } + + if (EVBUFFER_LENGTH(bufev->output)) { + res = evbuffer_write(bufev->output, fd); + if (res == -1) { +#ifndef WIN32 +/*todo. evbuffer uses WriteFile when WIN32 is set. WIN32 system calls do not + *set errno. thus this error checking is not portable*/ + if (errno == EAGAIN || + errno == EINTR || + errno == EINPROGRESS) + goto reschedule; + /* error case */ + what |= EVBUFFER_ERROR; + +#else + goto reschedule; +#endif + + } else if (res == 0) { + /* eof case */ + what |= EVBUFFER_EOF; + } + if (res <= 0) + goto error; + } + + if (EVBUFFER_LENGTH(bufev->output) != 0) + bufferevent_add(&bufev->ev_write, bufev->timeout_write); + + /* + * Invoke the user callback if our buffer is drained or below the + * low watermark. + */ + if (bufev->writecb != NULL && + EVBUFFER_LENGTH(bufev->output) <= bufev->wm_write.low) + (*bufev->writecb)(bufev, bufev->cbarg); + + return; + + reschedule: + if (EVBUFFER_LENGTH(bufev->output) != 0) + bufferevent_add(&bufev->ev_write, bufev->timeout_write); + return; + + error: + (*bufev->errorcb)(bufev, what, bufev->cbarg); +} + +/* + * Create a new buffered event object. + * + * The read callback is invoked whenever we read new data. + * The write callback is invoked whenever the output buffer is drained. + * The error callback is invoked on a write/read error or on EOF. + * + * Both read and write callbacks maybe NULL. The error callback is not + * allowed to be NULL and have to be provided always. + */ + +struct bufferevent * +bufferevent_new(int fd, evbuffercb readcb, evbuffercb writecb, + everrorcb errorcb, void *cbarg) +{ + struct bufferevent *bufev; + + if ((bufev = calloc(1, sizeof(struct bufferevent))) == NULL) + return (NULL); + + if ((bufev->input = evbuffer_new()) == NULL) { + free(bufev); + return (NULL); + } + + if ((bufev->output = evbuffer_new()) == NULL) { + evbuffer_free(bufev->input); + free(bufev); + return (NULL); + } + + event_set(&bufev->ev_read, fd, EV_READ, bufferevent_readcb, bufev); + event_set(&bufev->ev_write, fd, EV_WRITE, bufferevent_writecb, bufev); + + bufev->readcb = readcb; + bufev->writecb = writecb; + bufev->errorcb = errorcb; + + bufev->cbarg = cbarg; + + /* + * Set to EV_WRITE so that using bufferevent_write is going to + * trigger a callback. Reading needs to be explicitly enabled + * because otherwise no data will be available. + */ + bufev->enabled = EV_WRITE; + + return (bufev); +} + +int +bufferevent_priority_set(struct bufferevent *bufev, int priority) +{ + if (event_priority_set(&bufev->ev_read, priority) == -1) + return (-1); + if (event_priority_set(&bufev->ev_write, priority) == -1) + return (-1); + + return (0); +} + +/* Closing the file descriptor is the responsibility of the caller */ + +void +bufferevent_free(struct bufferevent *bufev) +{ + event_del(&bufev->ev_read); + event_del(&bufev->ev_write); + + evbuffer_free(bufev->input); + evbuffer_free(bufev->output); + + free(bufev); +} + +/* + * Returns 0 on success; + * -1 on failure. + */ + +int +bufferevent_write(struct bufferevent *bufev, const void *data, size_t size) +{ + int res; + + res = evbuffer_add(bufev->output, data, size); + + if (res == -1) + return (res); + + /* If everything is okay, we need to schedule a write */ + if (size > 0 && (bufev->enabled & EV_WRITE)) + bufferevent_add(&bufev->ev_write, bufev->timeout_write); + + return (res); +} + +int +bufferevent_write_buffer(struct bufferevent *bufev, struct evbuffer *buf) +{ + int res; + + res = bufferevent_write(bufev, buf->buffer, buf->off); + if (res != -1) + evbuffer_drain(buf, buf->off); + + return (res); +} + +size_t +bufferevent_read(struct bufferevent *bufev, void *data, size_t size) +{ + struct evbuffer *buf = bufev->input; + + if (buf->off < size) + size = buf->off; + + /* Copy the available data to the user buffer */ + memcpy(data, buf->buffer, size); + + if (size) + evbuffer_drain(buf, size); + + return (size); +} + +int +bufferevent_enable(struct bufferevent *bufev, short event) +{ + if (event & EV_READ) { + if (bufferevent_add(&bufev->ev_read, bufev->timeout_read) == -1) + return (-1); + } + if (event & EV_WRITE) { + if (bufferevent_add(&bufev->ev_write, bufev->timeout_write) == -1) + return (-1); + } + + bufev->enabled |= event; + return (0); +} + +int +bufferevent_disable(struct bufferevent *bufev, short event) +{ + if (event & EV_READ) { + if (event_del(&bufev->ev_read) == -1) + return (-1); + } + if (event & EV_WRITE) { + if (event_del(&bufev->ev_write) == -1) + return (-1); + } + + bufev->enabled &= ~event; + return (0); +} + +/* + * Sets the read and write timeout for a buffered event. + */ + +void +bufferevent_settimeout(struct bufferevent *bufev, + int timeout_read, int timeout_write) { + bufev->timeout_read = timeout_read; + bufev->timeout_write = timeout_write; +} + +/* + * Sets the water marks + */ + +void +bufferevent_setwatermark(struct bufferevent *bufev, short events, + size_t lowmark, size_t highmark) +{ + if (events & EV_READ) { + bufev->wm_read.low = lowmark; + bufev->wm_read.high = highmark; + } + + if (events & EV_WRITE) { + bufev->wm_write.low = lowmark; + bufev->wm_write.high = highmark; + } + + /* If the watermarks changed then see if we should call read again */ + bufferevent_read_pressure_cb(bufev->input, + 0, EVBUFFER_LENGTH(bufev->input), bufev); +} + +int +bufferevent_base_set(struct event_base *base, struct bufferevent *bufev) +{ + int res; + + res = event_base_set(base, &bufev->ev_read); + if (res == -1) + return (res); + + res = event_base_set(base, &bufev->ev_write); + return (res); +} === added file 'extra/libevent/evdns.c' --- a/extra/libevent/evdns.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evdns.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,3150 @@ +/* $Id: evdns.c 6979 2006-08-04 18:31:13Z nickm $ */ + +/* The original version of this module was written by Adam Langley; for + * a history of modifications, check out the subversion logs. + * + * When editing this module, try to keep it re-mergeable by Adam. Don't + * reformat the whitespace, add Tor dependencies, or so on. + * + * TODO: + * - Support IPv6 and PTR records. + * - Replace all externally visible magic numbers with #defined constants. + * - Write doccumentation for APIs of all external functions. + */ + +/* Async DNS Library + * Adam Langley <agl(a)imperialviolet.org> + * http://www.imperialviolet.org/eventdns.html + * Public Domain code + * + * This software is Public Domain. To view a copy of the public domain dedication, + * visit http://creativecommons.org/licenses/publicdomain/ or send a letter to + * Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA. + * + * I ask and expect, but do not require, that all derivative works contain an + * attribution similar to: + * Parts developed by Adam Langley <agl(a)imperialviolet.org> + * + * You may wish to replace the word "Parts" with something else depending on + * the amount of original code. + * + * (Derivative works does not include programs which link against, run or include + * the source verbatim in their source distributions) + * + * Version: 0.1b + */ + +#include <sys/types.h> +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#include "misc.h" +#endif + +#ifdef DNS_USE_FTIME_FOR_ID +#include <sys/timeb.h> +#endif + +#ifndef DNS_USE_CPU_CLOCK_FOR_ID +#ifndef DNS_USE_GETTIMEOFDAY_FOR_ID +#ifndef DNS_USE_OPENSSL_FOR_ID +#ifndef DNS_USE_FTIME_FOR_ID +#error Must configure at least one id generation method. +#error Please see the documentation. +#endif +#endif +#endif +#endif + +/* #define _POSIX_C_SOURCE 200507 */ +#define _GNU_SOURCE + +#ifdef DNS_USE_CPU_CLOCK_FOR_ID +#ifdef DNS_USE_OPENSSL_FOR_ID +#error Multiple id options selected +#endif +#ifdef DNS_USE_GETTIMEOFDAY_FOR_ID +#error Multiple id options selected +#endif +#include <time.h> +#endif + +#ifdef DNS_USE_OPENSSL_FOR_ID +#ifdef DNS_USE_GETTIMEOFDAY_FOR_ID +#error Multiple id options selected +#endif +#include <openssl/rand.h> +#endif + +#define _FORTIFY_SOURCE 3 + +#include <string.h> +#include <fcntl.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#ifdef HAVE_STDINT_H +#include <stdint.h> +#endif +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <assert.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#include <limits.h> +#include <sys/stat.h> +#include <ctype.h> +#include <stdio.h> +#include <stdarg.h> + +#include "evdns.h" +#include "evutil.h" +#include "log.h" +#ifdef WIN32 +#include <winsock2.h> +#include <windows.h> +#include <iphlpapi.h> +#include <io.h> +#else +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#endif + +#ifdef HAVE_NETINET_IN6_H +#include <netinet/in6.h> +#endif + +#define EVDNS_LOG_DEBUG 0 +#define EVDNS_LOG_WARN 1 + +#ifndef HOST_NAME_MAX +#define HOST_NAME_MAX 255 +#endif + +#include <stdio.h> + +#undef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) + +#ifdef __USE_ISOC99B +/* libevent doesn't work without this */ +typedef ev_uint8_t u_char; +typedef unsigned int uint; +#endif +#include <event.h> + +#define u64 ev_uint64_t +#define u32 ev_uint32_t +#define u16 ev_uint16_t +#define u8 ev_uint8_t + +#ifdef WIN32 +#define snprintf _snprintf +#define open _open +#define read _read +#define close _close +#define strdup _strdup +#endif + +#define MAX_ADDRS 32 /* maximum number of addresses from a single packet */ +/* which we bother recording */ + +#define TYPE_A EVDNS_TYPE_A +#define TYPE_CNAME 5 +#define TYPE_PTR EVDNS_TYPE_PTR +#define TYPE_AAAA EVDNS_TYPE_AAAA + +#define CLASS_INET EVDNS_CLASS_INET + +struct request { + u8 *request; /* the dns packet data */ + unsigned int request_len; + int reissue_count; + int tx_count; /* the number of times that this packet has been sent */ + unsigned int request_type; /* TYPE_PTR or TYPE_A */ + void *user_pointer; /* the pointer given to us for this request */ + evdns_callback_type user_callback; + struct nameserver *ns; /* the server which we last sent it */ + + /* elements used by the searching code */ + int search_index; + struct search_state *search_state; + char *search_origname; /* needs to be free()ed */ + int search_flags; + + /* these objects are kept in a circular list */ + struct request *next, *prev; + + struct event timeout_event; + + u16 trans_id; /* the transaction id */ + char request_appended; /* true if the request pointer is data which follows this struct */ + char transmit_me; /* needs to be transmitted */ +}; + +#ifndef HAVE_STRUCT_IN6_ADDR +struct in6_addr { + u8 s6_addr[16]; +}; +#endif + +struct reply { + unsigned int type; + unsigned int have_answer; + union { + struct { + u32 addrcount; + u32 addresses[MAX_ADDRS]; + } a; + struct { + u32 addrcount; + struct in6_addr addresses[MAX_ADDRS]; + } aaaa; + struct { + char name[HOST_NAME_MAX]; + } ptr; + } data; +}; + +struct nameserver { + int socket; /* a connected UDP socket */ + u32 address; + int failed_times; /* number of times which we have given this server a chance */ + int timedout; /* number of times in a row a request has timed out */ + struct event event; + /* these objects are kept in a circular list */ + struct nameserver *next, *prev; + struct event timeout_event; /* used to keep the timeout for */ + /* when we next probe this server. */ + /* Valid if state == 0 */ + char state; /* zero if we think that this server is down */ + char choked; /* true if we have an EAGAIN from this server's socket */ + char write_waiting; /* true if we are waiting for EV_WRITE events */ +}; + +static struct request *req_head = NULL, *req_waiting_head = NULL; +static struct nameserver *server_head = NULL; + +/* Represents a local port where we're listening for DNS requests. Right now, */ +/* only UDP is supported. */ +struct evdns_server_port { + int socket; /* socket we use to read queries and write replies. */ + int refcnt; /* reference count. */ + char choked; /* Are we currently blocked from writing? */ + char closing; /* Are we trying to close this port, pending writes? */ + evdns_request_callback_fn_type user_callback; /* Fn to handle requests */ + void *user_data; /* Opaque pointer passed to user_callback */ + struct event event; /* Read/write event */ + /* circular list of replies that we want to write. */ + struct server_request *pending_replies; +}; + +/* Represents part of a reply being built. (That is, a single RR.) */ +struct server_reply_item { + struct server_reply_item *next; /* next item in sequence. */ + char *name; /* name part of the RR */ + u16 type : 16; /* The RR type */ + u16 class : 16; /* The RR class (usually CLASS_INET) */ + u32 ttl; /* The RR TTL */ + char is_name; /* True iff data is a label */ + u16 datalen; /* Length of data; -1 if data is a label */ + void *data; /* The contents of the RR */ +}; + +/* Represents a request that we've received as a DNS server, and holds */ +/* the components of the reply as we're constructing it. */ +struct server_request { + /* Pointers to the next and previous entries on the list of replies */ + /* that we're waiting to write. Only set if we have tried to respond */ + /* and gotten EAGAIN. */ + struct server_request *next_pending; + struct server_request *prev_pending; + + u16 trans_id; /* Transaction id. */ + struct evdns_server_port *port; /* Which port received this request on? */ + struct sockaddr_storage addr; /* Where to send the response */ + socklen_t addrlen; /* length of addr */ + + int n_answer; /* how many answer RRs have been set? */ + int n_authority; /* how many authority RRs have been set? */ + int n_additional; /* how many additional RRs have been set? */ + + struct server_reply_item *answer; /* linked list of answer RRs */ + struct server_reply_item *authority; /* linked list of authority RRs */ + struct server_reply_item *additional; /* linked list of additional RRs */ + + /* Constructed response. Only set once we're ready to send a reply. */ + /* Once this is set, the RR fields are cleared, and no more should be set. */ + char *response; + size_t response_len; + + /* Caller-visible fields: flags, questions. */ + struct evdns_server_request base; +}; + +/* helper macro */ +#define OFFSET_OF(st, member) ((off_t) (((char*)&((st*)0)->member)-(char*)0)) + +/* Given a pointer to an evdns_server_request, get the corresponding */ +/* server_request. */ +#define TO_SERVER_REQUEST(base_ptr) \ + ((struct server_request*) \ + (((char*)(base_ptr) - OFFSET_OF(struct server_request, base)))) + +/* The number of good nameservers that we have */ +static int global_good_nameservers = 0; + +/* inflight requests are contained in the req_head list */ +/* and are actually going out across the network */ +static int global_requests_inflight = 0; +/* requests which aren't inflight are in the waiting list */ +/* and are counted here */ +static int global_requests_waiting = 0; + +static int global_max_requests_inflight = 64; + +static struct timeval global_timeout = {5, 0}; /* 5 seconds */ +static int global_max_reissues = 1; /* a reissue occurs when we get some errors from the server */ +static int global_max_retransmits = 3; /* number of times we'll retransmit a request which timed out */ +/* number of timeouts in a row before we consider this server to be down */ +static int global_max_nameserver_timeout = 3; + +/* These are the timeout values for nameservers. If we find a nameserver is down */ +/* we try to probe it at intervals as given below. Values are in seconds. */ +static const struct timeval global_nameserver_timeouts[] = {{10, 0}, {60, 0}, {300, 0}, {900, 0}, {3600, 0}}; +static const int global_nameserver_timeouts_length = sizeof(global_nameserver_timeouts)/sizeof(struct timeval); + +static struct nameserver *nameserver_pick(void); +static void evdns_request_insert(struct request *req, struct request **head); +static void nameserver_ready_callback(int fd, short events, void *arg); +static int evdns_transmit(void); +static int evdns_request_transmit(struct request *req); +static void nameserver_send_probe(struct nameserver *const ns); +static void search_request_finished(struct request *const); +static int search_try_next(struct request *const req); +static int search_request_new(int type, const char *const name, int flags, evdns_callback_type user_callback, void *user_arg); +static void evdns_requests_pump_waiting_queue(void); +static u16 transaction_id_pick(void); +static struct request *request_new(int type, const char *name, int flags, evdns_callback_type callback, void *ptr); +static void request_submit(struct request *const req); + +static int server_request_free(struct server_request *req); +static void server_request_free_answers(struct server_request *req); +static void server_port_free(struct evdns_server_port *port); +static void server_port_ready_callback(int fd, short events, void *arg); + +static int strtoint(const char *const str); + +#ifdef WIN32 +static int +last_error(int sock) +{ + int optval, optvallen=sizeof(optval); + int err = WSAGetLastError(); + if (err == WSAEWOULDBLOCK && sock >= 0) { + if (getsockopt(sock, SOL_SOCKET, SO_ERROR, (void*)&optval, + &optvallen)) + return err; + if (optval) + return optval; + } + return err; + +} +static int +error_is_eagain(int err) +{ + return err == EAGAIN || err == WSAEWOULDBLOCK; +} +static int +inet_aton(const char *c, struct in_addr *addr) +{ + ev_uint32_t r; + if (strcmp(c, "255.255.255.255") == 0) { + addr->s_addr = 0xffffffffu; + } else { + r = inet_addr(c); + if (r == INADDR_NONE) + return 0; + addr->s_addr = r; + } + return 1; +} +#else +#define last_error(sock) (errno) +#define error_is_eagain(err) ((err) == EAGAIN) +#endif +#define CLOSE_SOCKET(s) EVUTIL_CLOSESOCKET(s) + +#define ISSPACE(c) isspace((int)(unsigned char)(c)) +#define ISDIGIT(c) isdigit((int)(unsigned char)(c)) + +static const char * +debug_ntoa(u32 address) +{ + static char buf[32]; + u32 a = ntohl(address); + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + (int)(u8)((a>>24)&0xff), + (int)(u8)((a>>16)&0xff), + (int)(u8)((a>>8 )&0xff), + (int)(u8)((a )&0xff)); + return buf; +} + +static evdns_debug_log_fn_type evdns_log_fn = NULL; + +void +evdns_set_log_fn(evdns_debug_log_fn_type fn) +{ + evdns_log_fn = fn; +} + +#ifdef __GNUC__ +#define EVDNS_LOG_CHECK __attribute__ ((format(printf, 2, 3))) +#else +#define EVDNS_LOG_CHECK +#endif + +static void _evdns_log(int warn, const char *fmt, ...) EVDNS_LOG_CHECK; +static void +_evdns_log(int warn, const char *fmt, ...) +{ + va_list args; + static char buf[512]; + if (!evdns_log_fn) + return; + va_start(args,fmt); +#ifdef WIN32 + _vsnprintf(buf, sizeof(buf), fmt, args); +#else + vsnprintf(buf, sizeof(buf), fmt, args); +#endif + buf[sizeof(buf)-1] = '\0'; + evdns_log_fn(warn, buf); + va_end(args); +} + +#define log _evdns_log + +/* This walks the list of inflight requests to find the */ +/* one with a matching transaction id. Returns NULL on */ +/* failure */ +static struct request * +request_find_from_trans_id(u16 trans_id) { + struct request *req = req_head, *const started_at = req_head; + + if (req) { + do { + if (req->trans_id == trans_id) return req; + req = req->next; + } while (req != started_at); + } + + return NULL; +} + +/* a libevent callback function which is called when a nameserver */ +/* has gone down and we want to test if it has came back to life yet */ +static void +nameserver_prod_callback(int fd, short events, void *arg) { + struct nameserver *const ns = (struct nameserver *) arg; + (void)fd; + (void)events; + + nameserver_send_probe(ns); +} + +/* a libevent callback which is called when a nameserver probe (to see if */ +/* it has come back to life) times out. We increment the count of failed_times */ +/* and wait longer to send the next probe packet. */ +static void +nameserver_probe_failed(struct nameserver *const ns) { + const struct timeval * timeout; + (void) evtimer_del(&ns->timeout_event); + if (ns->state == 1) { + /* This can happen if the nameserver acts in a way which makes us mark */ + /* it as bad and then starts sending good replies. */ + return; + } + + timeout = + &global_nameserver_timeouts[MIN(ns->failed_times, + global_nameserver_timeouts_length - 1)]; + ns->failed_times++; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); + if (evtimer_add(&ns->timeout_event, (struct timeval *) timeout) < 0) { + log(EVDNS_LOG_WARN, + "Error from libevent when adding timer event for %s", + debug_ntoa(ns->address)); + /* ???? Do more? */ + } +} + +/* called when a nameserver has been deemed to have failed. For example, too */ +/* many packets have timed out etc */ +static void +nameserver_failed(struct nameserver *const ns, const char *msg) { + struct request *req, *started_at; + /* if this nameserver has already been marked as failed */ + /* then don't do anything */ + if (!ns->state) return; + + log(EVDNS_LOG_WARN, "Nameserver %s has failed: %s", + debug_ntoa(ns->address), msg); + global_good_nameservers--; + assert(global_good_nameservers >= 0); + if (global_good_nameservers == 0) { + log(EVDNS_LOG_WARN, "All nameservers have failed"); + } + + ns->state = 0; + ns->failed_times = 1; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); + if (evtimer_add(&ns->timeout_event, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { + log(EVDNS_LOG_WARN, + "Error from libevent when adding timer event for %s", + debug_ntoa(ns->address)); + /* ???? Do more? */ + } + + /* walk the list of inflight requests to see if any can be reassigned to */ + /* a different server. Requests in the waiting queue don't have a */ + /* nameserver assigned yet */ + + /* if we don't have *any* good nameservers then there's no point */ + /* trying to reassign requests to one */ + if (!global_good_nameservers) return; + + req = req_head; + started_at = req_head; + if (req) { + do { + if (req->tx_count == 0 && req->ns == ns) { + /* still waiting to go out, can be moved */ + /* to another server */ + req->ns = nameserver_pick(); + } + req = req->next; + } while (req != started_at); + } +} + +static void +nameserver_up(struct nameserver *const ns) { + if (ns->state) return; + log(EVDNS_LOG_WARN, "Nameserver %s is back up", + debug_ntoa(ns->address)); + evtimer_del(&ns->timeout_event); + ns->state = 1; + ns->failed_times = 0; + ns->timedout = 0; + global_good_nameservers++; +} + +static void +request_trans_id_set(struct request *const req, const u16 trans_id) { + req->trans_id = trans_id; + *((u16 *) req->request) = htons(trans_id); +} + +/* Called to remove a request from a list and dealloc it. */ +/* head is a pointer to the head of the list it should be */ +/* removed from or NULL if the request isn't in a list. */ +static void +request_finished(struct request *const req, struct request **head) { + if (head) { + if (req->next == req) { + /* only item in the list */ + *head = NULL; + } else { + req->next->prev = req->prev; + req->prev->next = req->next; + if (*head == req) *head = req->next; + } + } + + log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", + (unsigned long) req); + evtimer_del(&req->timeout_event); + + search_request_finished(req); + global_requests_inflight--; + + if (!req->request_appended) { + /* need to free the request data on it's own */ + free(req->request); + } else { + /* the request data is appended onto the header */ + /* so everything gets free()ed when we: */ + } + + free(req); + + evdns_requests_pump_waiting_queue(); +} + +/* This is called when a server returns a funny error code. */ +/* We try the request again with another server. */ +/* */ +/* return: */ +/* 0 ok */ +/* 1 failed/reissue is pointless */ +static int +request_reissue(struct request *req) { + const struct nameserver *const last_ns = req->ns; + /* the last nameserver should have been marked as failing */ + /* by the caller of this function, therefore pick will try */ + /* not to return it */ + req->ns = nameserver_pick(); + if (req->ns == last_ns) { + /* ... but pick did return it */ + /* not a lot of point in trying again with the */ + /* same server */ + return 1; + } + + req->reissue_count++; + req->tx_count = 0; + req->transmit_me = 1; + + return 0; +} + +/* this function looks for space on the inflight queue and promotes */ +/* requests from the waiting queue if it can. */ +static void +evdns_requests_pump_waiting_queue(void) { + while (global_requests_inflight < global_max_requests_inflight && + global_requests_waiting) { + struct request *req; + /* move a request from the waiting queue to the inflight queue */ + assert(req_waiting_head); + if (req_waiting_head->next == req_waiting_head) { + /* only one item in the queue */ + req = req_waiting_head; + req_waiting_head = NULL; + } else { + req = req_waiting_head; + req->next->prev = req->prev; + req->prev->next = req->next; + req_waiting_head = req->next; + } + + global_requests_waiting--; + global_requests_inflight++; + + req->ns = nameserver_pick(); + request_trans_id_set(req, transaction_id_pick()); + + evdns_request_insert(req, &req_head); + evdns_request_transmit(req); + evdns_transmit(); + } +} + +static void +reply_callback(struct request *const req, u32 ttl, u32 err, struct reply *reply) { + switch (req->request_type) { + case TYPE_A: + if (reply) + req->user_callback(DNS_ERR_NONE, DNS_IPv4_A, + reply->data.a.addrcount, ttl, + reply->data.a.addresses, + req->user_pointer); + else + req->user_callback(err, 0, 0, 0, NULL, req->user_pointer); + return; + case TYPE_PTR: + if (reply) { + char *name = reply->data.ptr.name; + req->user_callback(DNS_ERR_NONE, DNS_PTR, 1, ttl, + &name, req->user_pointer); + } else { + req->user_callback(err, 0, 0, 0, NULL, + req->user_pointer); + } + return; + case TYPE_AAAA: + if (reply) + req->user_callback(DNS_ERR_NONE, DNS_IPv6_AAAA, + reply->data.aaaa.addrcount, ttl, + reply->data.aaaa.addresses, + req->user_pointer); + else + req->user_callback(err, 0, 0, 0, NULL, req->user_pointer); + return; + } + assert(0); +} + +/* this processes a parsed reply packet */ +static void +reply_handle(struct request *const req, u16 flags, u32 ttl, struct reply *reply) { + int error; + static const int error_codes[] = {DNS_ERR_FORMAT, DNS_ERR_SERVERFAILED, DNS_ERR_NOTEXIST, DNS_ERR_NOTIMPL, DNS_ERR_REFUSED}; + + if (flags & 0x020f || !reply || !reply->have_answer) { + /* there was an error */ + if (flags & 0x0200) { + error = DNS_ERR_TRUNCATED; + } else { + u16 error_code = (flags & 0x000f) - 1; + if (error_code > 4) { + error = DNS_ERR_UNKNOWN; + } else { + error = error_codes[error_code]; + } + } + + switch(error) { + case DNS_ERR_NOTIMPL: + case DNS_ERR_REFUSED: + /* we regard these errors as marking a bad nameserver */ + if (req->reissue_count < global_max_reissues) { + char msg[64]; + snprintf(msg, sizeof(msg), "Bad response %d (%s)", + error, evdns_err_to_string(error)); + nameserver_failed(req->ns, msg); + if (!request_reissue(req)) return; + } + break; + case DNS_ERR_SERVERFAILED: + /* rcode 2 (servfailed) sometimes means "we are broken" and + * sometimes (with some binds) means "that request was very + * confusing." Treat this as a timeout, not a failure. + */ + log(EVDNS_LOG_DEBUG, "Got a SERVERFAILED from nameserver %s; " + "will allow the request to time out.", + debug_ntoa(req->ns->address)); + break; + default: + /* we got a good reply from the nameserver */ + nameserver_up(req->ns); + } + + if (req->search_state && req->request_type != TYPE_PTR) { + /* if we have a list of domains to search in, try the next one */ + if (!search_try_next(req)) { + /* a new request was issued so this request is finished and */ + /* the user callback will be made when that request (or a */ + /* child of it) finishes. */ + request_finished(req, &req_head); + return; + } + } + + /* all else failed. Pass the failure up */ + reply_callback(req, 0, error, NULL); + request_finished(req, &req_head); + } else { + /* all ok, tell the user */ + reply_callback(req, ttl, 0, reply); + nameserver_up(req->ns); + request_finished(req, &req_head); + } +} + +static int +name_parse(u8 *packet, int length, int *idx, char *name_out, int name_out_len) { + int name_end = -1; + int j = *idx; + int ptr_count = 0; +#define GET32(x) do { if (j + 4 > length) goto err; memcpy(&_t32, packet + j, 4); j += 4; x = ntohl(_t32); } while(0) +#define GET16(x) do { if (j + 2 > length) goto err; memcpy(&_t, packet + j, 2); j += 2; x = ntohs(_t); } while(0) +#define GET8(x) do { if (j >= length) goto err; x = packet[j++]; } while(0) + + char *cp = name_out; + const char *const end = name_out + name_out_len; + + /* Normally, names are a series of length prefixed strings terminated */ + /* with a length of 0 (the lengths are u8's < 63). */ + /* However, the length can start with a pair of 1 bits and that */ + /* means that the next 14 bits are a pointer within the current */ + /* packet. */ + + for(;;) { + u8 label_len; + if (j >= length) return -1; + GET8(label_len); + if (!label_len) break; + if (label_len & 0xc0) { + u8 ptr_low; + GET8(ptr_low); + if (name_end < 0) name_end = j; + j = (((int)label_len & 0x3f) << 8) + ptr_low; + /* Make sure that the target offset is in-bounds. */ + if (j < 0 || j >= length) return -1; + /* If we've jumped more times than there are characters in the + * message, we must have a loop. */ + if (++ptr_count > length) return -1; + continue; + } + if (label_len > 63) return -1; + if (cp != name_out) { + if (cp + 1 >= end) return -1; + *cp++ = '.'; + } + if (cp + label_len >= end) return -1; + memcpy(cp, packet + j, label_len); + cp += label_len; + j += label_len; + } + if (cp >= end) return -1; + *cp = '\0'; + if (name_end < 0) + *idx = j; + else + *idx = name_end; + return 0; + err: + return -1; +} + +/* parses a raw request from a nameserver */ +static int +reply_parse(u8 *packet, int length) { + int j = 0; /* index into packet */ + u16 _t; /* used by the macros */ + u32 _t32; /* used by the macros */ + char tmp_name[256]; /* used by the macros */ + + u16 trans_id, questions, answers, authority, additional, datalength; + u16 flags = 0; + u32 ttl, ttl_r = 0xffffffff; + struct reply reply; + struct request *req = NULL; + unsigned int i; + + GET16(trans_id); + GET16(flags); + GET16(questions); + GET16(answers); + GET16(authority); + GET16(additional); + (void) authority; /* suppress "unused variable" warnings. */ + (void) additional; /* suppress "unused variable" warnings. */ + + req = request_find_from_trans_id(trans_id); + if (!req) return -1; + + memset(&reply, 0, sizeof(reply)); + + /* If it's not an answer, it doesn't correspond to any request. */ + if (!(flags & 0x8000)) return -1; /* must be an answer */ + if (flags & 0x020f) { + /* there was an error */ + goto err; + } + /* if (!answers) return; */ /* must have an answer of some form */ + + /* This macro skips a name in the DNS reply. */ +#define SKIP_NAME \ + do { tmp_name[0] = '\0'; \ + if (name_parse(packet, length, &j, tmp_name, sizeof(tmp_name))<0) \ + goto err; \ + } while(0); + + reply.type = req->request_type; + + /* skip over each question in the reply */ + for (i = 0; i < questions; ++i) { + /* the question looks like + * <label:name><u16:type><u16:class> + */ + SKIP_NAME; + j += 4; + if (j > length) goto err; + } + + /* now we have the answer section which looks like + * <label:name><u16:type><u16:class><u32:ttl><u16:len><data...> + */ + + for (i = 0; i < answers; ++i) { + u16 type, class; + + SKIP_NAME; + GET16(type); + GET16(class); + GET32(ttl); + GET16(datalength); + + if (type == TYPE_A && class == CLASS_INET) { + int addrcount, addrtocopy; + if (req->request_type != TYPE_A) { + j += datalength; continue; + } + if ((datalength & 3) != 0) /* not an even number of As. */ + goto err; + addrcount = datalength >> 2; + addrtocopy = MIN(MAX_ADDRS - reply.data.a.addrcount, (unsigned)addrcount); + + ttl_r = MIN(ttl_r, ttl); + /* we only bother with the first four addresses. */ + if (j + 4*addrtocopy > length) goto err; + memcpy(&reply.data.a.addresses[reply.data.a.addrcount], + packet + j, 4*addrtocopy); + j += 4*addrtocopy; + reply.data.a.addrcount += addrtocopy; + reply.have_answer = 1; + if (reply.data.a.addrcount == MAX_ADDRS) break; + } else if (type == TYPE_PTR && class == CLASS_INET) { + if (req->request_type != TYPE_PTR) { + j += datalength; continue; + } + if (name_parse(packet, length, &j, reply.data.ptr.name, + sizeof(reply.data.ptr.name))<0) + goto err; + ttl_r = MIN(ttl_r, ttl); + reply.have_answer = 1; + break; + } else if (type == TYPE_AAAA && class == CLASS_INET) { + int addrcount, addrtocopy; + if (req->request_type != TYPE_AAAA) { + j += datalength; continue; + } + if ((datalength & 15) != 0) /* not an even number of AAAAs. */ + goto err; + addrcount = datalength >> 4; /* each address is 16 bytes long */ + addrtocopy = MIN(MAX_ADDRS - reply.data.aaaa.addrcount, (unsigned)addrcount); + ttl_r = MIN(ttl_r, ttl); + + /* we only bother with the first four addresses. */ + if (j + 16*addrtocopy > length) goto err; + memcpy(&reply.data.aaaa.addresses[reply.data.aaaa.addrcount], + packet + j, 16*addrtocopy); + reply.data.aaaa.addrcount += addrtocopy; + j += 16*addrtocopy; + reply.have_answer = 1; + if (reply.data.aaaa.addrcount == MAX_ADDRS) break; + } else { + /* skip over any other type of resource */ + j += datalength; + } + } + + reply_handle(req, flags, ttl_r, &reply); + return 0; + err: + if (req) + reply_handle(req, flags, 0, NULL); + return -1; +} + +/* Parse a raw request (packet,length) sent to a nameserver port (port) from */ +/* a DNS client (addr,addrlen), and if it's well-formed, call the corresponding */ +/* callback. */ +static int +request_parse(u8 *packet, int length, struct evdns_server_port *port, struct sockaddr *addr, socklen_t addrlen) +{ + int j = 0; /* index into packet */ + u16 _t; /* used by the macros */ + char tmp_name[256]; /* used by the macros */ + + int i; + u16 trans_id, flags, questions, answers, authority, additional; + struct server_request *server_req = NULL; + + /* Get the header fields */ + GET16(trans_id); + GET16(flags); + GET16(questions); + GET16(answers); + GET16(authority); + GET16(additional); + + if (flags & 0x8000) return -1; /* Must not be an answer. */ + flags &= 0x0110; /* Only RD and CD get preserved. */ + + server_req = malloc(sizeof(struct server_request)); + if (server_req == NULL) return -1; + memset(server_req, 0, sizeof(struct server_request)); + + server_req->trans_id = trans_id; + memcpy(&server_req->addr, addr, addrlen); + server_req->addrlen = addrlen; + + server_req->base.flags = flags; + server_req->base.nquestions = 0; + server_req->base.questions = malloc(sizeof(struct evdns_server_question *) * questions); + if (server_req->base.questions == NULL) + goto err; + + for (i = 0; i < questions; ++i) { + u16 type, class; + struct evdns_server_question *q; + int namelen; + if (name_parse(packet, length, &j, tmp_name, sizeof(tmp_name))<0) + goto err; + GET16(type); + GET16(class); + namelen = strlen(tmp_name); + q = malloc(sizeof(struct evdns_server_question) + namelen); + if (!q) + goto err; + q->type = type; + q->dns_question_class = class; + memcpy(q->name, tmp_name, namelen+1); + server_req->base.questions[server_req->base.nquestions++] = q; + } + + /* Ignore answers, authority, and additional. */ + + server_req->port = port; + port->refcnt++; + + /* Only standard queries are supported. */ + if (flags & 0x7800) { + evdns_server_request_respond(&(server_req->base), DNS_ERR_NOTIMPL); + return -1; + } + + port->user_callback(&(server_req->base), port->user_data); + + return 0; +err: + if (server_req) { + if (server_req->base.questions) { + for (i = 0; i < server_req->base.nquestions; ++i) + free(server_req->base.questions[i]); + free(server_req->base.questions); + } + free(server_req); + } + return -1; + +#undef SKIP_NAME +#undef GET32 +#undef GET16 +#undef GET8 +} + +static u16 +default_transaction_id_fn(void) +{ + u16 trans_id; +#ifdef DNS_USE_CPU_CLOCK_FOR_ID + struct timespec ts; +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) +#else + if (clock_gettime(CLOCK_REALTIME, &ts) == -1) +#endif + event_err(1, "clock_gettime"); + trans_id = ts.tv_nsec & 0xffff; +#endif + +#ifdef DNS_USE_FTIME_FOR_ID + struct _timeb tb; + _ftime(&tb); + trans_id = tb.millitm & 0xffff; +#endif + +#ifdef DNS_USE_GETTIMEOFDAY_FOR_ID + struct timeval tv; + gettimeofday(&tv, NULL); + trans_id = tv.tv_usec & 0xffff; +#endif + +#ifdef DNS_USE_OPENSSL_FOR_ID + if (RAND_pseudo_bytes((u8 *) &trans_id, 2) == -1) { + /* in the case that the RAND call fails we back */ + /* down to using gettimeofday. */ + /* + struct timeval tv; + gettimeofday(&tv, NULL); + trans_id = tv.tv_usec & 0xffff; + */ + abort(); + } +#endif + return trans_id; +} + +static ev_uint16_t (*trans_id_function)(void) = default_transaction_id_fn; + +void +evdns_set_transaction_id_fn(ev_uint16_t (*fn)(void)) +{ + if (fn) + trans_id_function = fn; + else + trans_id_function = default_transaction_id_fn; +} + +/* Try to choose a strong transaction id which isn't already in flight */ +static u16 +transaction_id_pick(void) { + for (;;) { + const struct request *req = req_head, *started_at; + u16 trans_id = trans_id_function(); + + if (trans_id == 0xffff) continue; + /* now check to see if that id is already inflight */ + req = started_at = req_head; + if (req) { + do { + if (req->trans_id == trans_id) break; + req = req->next; + } while (req != started_at); + } + /* we didn't find it, so this is a good id */ + if (req == started_at) return trans_id; + } +} + +/* choose a namesever to use. This function will try to ignore */ +/* nameservers which we think are down and load balance across the rest */ +/* by updating the server_head global each time. */ +static struct nameserver * +nameserver_pick(void) { + struct nameserver *started_at = server_head, *picked; + if (!server_head) return NULL; + + /* if we don't have any good nameservers then there's no */ + /* point in trying to find one. */ + if (!global_good_nameservers) { + server_head = server_head->next; + return server_head; + } + + /* remember that nameservers are in a circular list */ + for (;;) { + if (server_head->state) { + /* we think this server is currently good */ + picked = server_head; + server_head = server_head->next; + return picked; + } + + server_head = server_head->next; + if (server_head == started_at) { + /* all the nameservers seem to be down */ + /* so we just return this one and hope for the */ + /* best */ + assert(global_good_nameservers == 0); + picked = server_head; + server_head = server_head->next; + return picked; + } + } +} + +/* this is called when a namesever socket is ready for reading */ +static void +nameserver_read(struct nameserver *ns) { + u8 packet[1500]; + + for (;;) { + const int r = recv(ns->socket, packet, sizeof(packet), 0); + if (r < 0) { + int err = last_error(ns->socket); + if (error_is_eagain(err)) return; + nameserver_failed(ns, strerror(err)); + return; + } + ns->timedout = 0; + reply_parse(packet, r); + } +} + +/* Read a packet from a DNS client on a server port s, parse it, and */ +/* act accordingly. */ +static void +server_port_read(struct evdns_server_port *s) { + u8 packet[1500]; + struct sockaddr_storage addr; + socklen_t addrlen; + int r; + + for (;;) { + addrlen = sizeof(struct sockaddr_storage); + r = recvfrom(s->socket, packet, sizeof(packet), 0, + (struct sockaddr*) &addr, &addrlen); + if (r < 0) { + int err = last_error(s->socket); + if (error_is_eagain(err)) return; + log(EVDNS_LOG_WARN, "Error %s (%d) while reading request.", + strerror(err), err); + return; + } + request_parse(packet, r, s, (struct sockaddr*) &addr, addrlen); + } +} + +/* Try to write all pending replies on a given DNS server port. */ +static void +server_port_flush(struct evdns_server_port *port) +{ + while (port->pending_replies) { + struct server_request *req = port->pending_replies; + int r = sendto(port->socket, req->response, req->response_len, 0, + (struct sockaddr*) &req->addr, req->addrlen); + if (r < 0) { + int err = last_error(port->socket); + if (error_is_eagain(err)) + return; + log(EVDNS_LOG_WARN, "Error %s (%d) while writing response to port; dropping", strerror(err), err); + } + if (server_request_free(req)) { + /* we released the last reference to req->port. */ + return; + } + } + + /* We have no more pending requests; stop listening for 'writeable' events. */ + (void) event_del(&port->event); + event_set(&port->event, port->socket, EV_READ | EV_PERSIST, + server_port_ready_callback, port); + if (event_add(&port->event, NULL) < 0) { + log(EVDNS_LOG_WARN, "Error from libevent when adding event for DNS server."); + /* ???? Do more? */ + } +} + +/* set if we are waiting for the ability to write to this server. */ +/* if waiting is true then we ask libevent for EV_WRITE events, otherwise */ +/* we stop these events. */ +static void +nameserver_write_waiting(struct nameserver *ns, char waiting) { + if (ns->write_waiting == waiting) return; + + ns->write_waiting = waiting; + (void) event_del(&ns->event); + event_set(&ns->event, ns->socket, EV_READ | (waiting ? EV_WRITE : 0) | EV_PERSIST, + nameserver_ready_callback, ns); + if (event_add(&ns->event, NULL) < 0) { + log(EVDNS_LOG_WARN, "Error from libevent when adding event for %s", + debug_ntoa(ns->address)); + /* ???? Do more? */ + } +} + +/* a callback function. Called by libevent when the kernel says that */ +/* a nameserver socket is ready for writing or reading */ +static void +nameserver_ready_callback(int fd, short events, void *arg) { + struct nameserver *ns = (struct nameserver *) arg; + (void)fd; + + if (events & EV_WRITE) { + ns->choked = 0; + if (!evdns_transmit()) { + nameserver_write_waiting(ns, 0); + } + } + if (events & EV_READ) { + nameserver_read(ns); + } +} + +/* a callback function. Called by libevent when the kernel says that */ +/* a server socket is ready for writing or reading. */ +static void +server_port_ready_callback(int fd, short events, void *arg) { + struct evdns_server_port *port = (struct evdns_server_port *) arg; + (void) fd; + + if (events & EV_WRITE) { + port->choked = 0; + server_port_flush(port); + } + if (events & EV_READ) { + server_port_read(port); + } +} + +/* This is an inefficient representation; only use it via the dnslabel_table_* + * functions, so that is can be safely replaced with something smarter later. */ +#define MAX_LABELS 128 +/* Structures used to implement name compression */ +struct dnslabel_entry { char *v; off_t pos; }; +struct dnslabel_table { + int n_labels; /* number of current entries */ + /* map from name to position in message */ + struct dnslabel_entry labels[MAX_LABELS]; +}; + +/* Initialize dnslabel_table. */ +static void +dnslabel_table_init(struct dnslabel_table *table) +{ + table->n_labels = 0; +} + +/* Free all storage held by table, but not the table itself. */ +static void +dnslabel_clear(struct dnslabel_table *table) +{ + int i; + for (i = 0; i < table->n_labels; ++i) + free(table->labels[i].v); + table->n_labels = 0; +} + +/* return the position of the label in the current message, or -1 if the label */ +/* hasn't been used yet. */ +static int +dnslabel_table_get_pos(const struct dnslabel_table *table, const char *label) +{ + int i; + for (i = 0; i < table->n_labels; ++i) { + if (!strcmp(label, table->labels[i].v)) + return table->labels[i].pos; + } + return -1; +} + +/* remember that we've used the label at position pos */ +static int +dnslabel_table_add(struct dnslabel_table *table, const char *label, off_t pos) +{ + char *v; + int p; + if (table->n_labels == MAX_LABELS) + return (-1); + v = strdup(label); + if (v == NULL) + return (-1); + p = table->n_labels++; + table->labels[p].v = v; + table->labels[p].pos = pos; + + return (0); +} + +/* Converts a string to a length-prefixed set of DNS labels, starting */ +/* at buf[j]. name and buf must not overlap. name_len should be the length */ +/* of name. table is optional, and is used for compression. */ +/* */ +/* Input: abc.def */ +/* Output: <3>abc<3>def<0> */ +/* */ +/* Returns the first index after the encoded name, or negative on error. */ +/* -1 label was > 63 bytes */ +/* -2 name too long to fit in buffer. */ +/* */ +static off_t +dnsname_to_labels(u8 *const buf, size_t buf_len, off_t j, + const char *name, const int name_len, + struct dnslabel_table *table) { + const char *end = name + name_len; + int ref = 0; + u16 _t; + +#define APPEND16(x) do { \ + if (j + 2 > (off_t)buf_len) \ + goto overflow; \ + _t = htons(x); \ + memcpy(buf + j, &_t, 2); \ + j += 2; \ + } while (0) +#define APPEND32(x) do { \ + if (j + 4 > (off_t)buf_len) \ + goto overflow; \ + _t32 = htonl(x); \ + memcpy(buf + j, &_t32, 4); \ + j += 4; \ + } while (0) + + if (name_len > 255) return -2; + + for (;;) { + const char *const start = name; + if (table && (ref = dnslabel_table_get_pos(table, name)) >= 0) { + APPEND16(ref | 0xc000); + return j; + } + name = strchr(name, '.'); + if (!name) { + const unsigned int label_len = end - start; + if (label_len > 63) return -1; + if ((size_t)(j+label_len+1) > buf_len) return -2; + if (table) dnslabel_table_add(table, start, j); + buf[j++] = label_len; + + memcpy(buf + j, start, end - start); + j += end - start; + break; + } else { + /* append length of the label. */ + const unsigned int label_len = name - start; + if (label_len > 63) return -1; + if ((size_t)(j+label_len+1) > buf_len) return -2; + if (table) dnslabel_table_add(table, start, j); + buf[j++] = label_len; + + memcpy(buf + j, start, name - start); + j += name - start; + /* hop over the '.' */ + name++; + } + } + + /* the labels must be terminated by a 0. */ + /* It's possible that the name ended in a . */ + /* in which case the zero is already there */ + if (!j || buf[j-1]) buf[j++] = 0; + return j; + overflow: + return (-2); +} + +/* Finds the length of a dns request for a DNS name of the given */ +/* length. The actual request may be smaller than the value returned */ +/* here */ +static int +evdns_request_len(const int name_len) { + return 96 + /* length of the DNS standard header */ + name_len + 2 + + 4; /* space for the resource type */ +} + +/* build a dns request packet into buf. buf should be at least as long */ +/* as evdns_request_len told you it should be. */ +/* */ +/* Returns the amount of space used. Negative on error. */ +static int +evdns_request_data_build(const char *const name, const int name_len, + const u16 trans_id, const u16 type, const u16 class, + u8 *const buf, size_t buf_len) { + off_t j = 0; /* current offset into buf */ + u16 _t; /* used by the macros */ + + APPEND16(trans_id); + APPEND16(0x0100); /* standard query, recusion needed */ + APPEND16(1); /* one question */ + APPEND16(0); /* no answers */ + APPEND16(0); /* no authority */ + APPEND16(0); /* no additional */ + + j = dnsname_to_labels(buf, buf_len, j, name, name_len, NULL); + if (j < 0) { + return (int)j; + } + + APPEND16(type); + APPEND16(class); + + return (int)j; + overflow: + return (-1); +} + +/* exported function */ +struct evdns_server_port * +evdns_add_server_port(int socket, int is_tcp, evdns_request_callback_fn_type cb, void *user_data) +{ + struct evdns_server_port *port; + if (!(port = malloc(sizeof(struct evdns_server_port)))) + return NULL; + memset(port, 0, sizeof(struct evdns_server_port)); + + assert(!is_tcp); /* TCP sockets not yet implemented */ + port->socket = socket; + port->refcnt = 1; + port->choked = 0; + port->closing = 0; + port->user_callback = cb; + port->user_data = user_data; + port->pending_replies = NULL; + + event_set(&port->event, port->socket, EV_READ | EV_PERSIST, + server_port_ready_callback, port); + event_add(&port->event, NULL); /* check return. */ + return port; +} + +/* exported function */ +void +evdns_close_server_port(struct evdns_server_port *port) +{ + if (--port->refcnt == 0) + server_port_free(port); + port->closing = 1; +} + +/* exported function */ +int +evdns_server_request_add_reply(struct evdns_server_request *_req, int section, const char *name, int type, int class, int ttl, int datalen, int is_name, const char *data) +{ + struct server_request *req = TO_SERVER_REQUEST(_req); + struct server_reply_item **itemp, *item; + int *countp; + + if (req->response) /* have we already answered? */ + return (-1); + + switch (section) { + case EVDNS_ANSWER_SECTION: + itemp = &req->answer; + countp = &req->n_answer; + break; + case EVDNS_AUTHORITY_SECTION: + itemp = &req->authority; + countp = &req->n_authority; + break; + case EVDNS_ADDITIONAL_SECTION: + itemp = &req->additional; + countp = &req->n_additional; + break; + default: + return (-1); + } + while (*itemp) { + itemp = &((*itemp)->next); + } + item = malloc(sizeof(struct server_reply_item)); + if (!item) + return -1; + item->next = NULL; + if (!(item->name = strdup(name))) { + free(item); + return -1; + } + item->type = type; + item->dns_question_class = class; + item->ttl = ttl; + item->is_name = is_name != 0; + item->datalen = 0; + item->data = NULL; + if (data) { + if (item->is_name) { + if (!(item->data = strdup(data))) { + free(item->name); + free(item); + return -1; + } + item->datalen = (u16)-1; + } else { + if (!(item->data = malloc(datalen))) { + free(item->name); + free(item); + return -1; + } + item->datalen = datalen; + memcpy(item->data, data, datalen); + } + } + + *itemp = item; + ++(*countp); + return 0; +} + +/* exported function */ +int +evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +{ + return evdns_server_request_add_reply( + req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, + ttl, n*4, 0, addrs); +} + +/* exported function */ +int +evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +{ + return evdns_server_request_add_reply( + req, EVDNS_ANSWER_SECTION, name, TYPE_AAAA, CLASS_INET, + ttl, n*16, 0, addrs); +} + +/* exported function */ +int +evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl) +{ + u32 a; + char buf[32]; + assert(in || inaddr_name); + assert(!(in && inaddr_name)); + if (in) { + a = ntohl(in->s_addr); + snprintf(buf, sizeof(buf), "%d.%d.%d.%d.in-addr.arpa", + (int)(u8)((a )&0xff), + (int)(u8)((a>>8 )&0xff), + (int)(u8)((a>>16)&0xff), + (int)(u8)((a>>24)&0xff)); + inaddr_name = buf; + } + return evdns_server_request_add_reply( + req, EVDNS_ANSWER_SECTION, inaddr_name, TYPE_PTR, CLASS_INET, + ttl, -1, 1, hostname); +} + +/* exported function */ +int +evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl) +{ + return evdns_server_request_add_reply( + req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, + ttl, -1, 1, cname); +} + + +static int +evdns_server_request_format_response(struct server_request *req, int err) +{ + unsigned char buf[1500]; + size_t buf_len = sizeof(buf); + off_t j = 0, r; + u16 _t; + u32 _t32; + int i; + u16 flags; + struct dnslabel_table table; + + if (err < 0 || err > 15) return -1; + + /* Set response bit and error code; copy OPCODE and RD fields from + * question; copy RA and AA if set by caller. */ + flags = req->base.flags; + flags |= (0x8000 | err); + + dnslabel_table_init(&table); + APPEND16(req->trans_id); + APPEND16(flags); + APPEND16(req->base.nquestions); + APPEND16(req->n_answer); + APPEND16(req->n_authority); + APPEND16(req->n_additional); + + /* Add questions. */ + for (i=0; i < req->base.nquestions; ++i) { + const char *s = req->base.questions[i]->name; + j = dnsname_to_labels(buf, buf_len, j, s, strlen(s), &table); + if (j < 0) { + dnslabel_clear(&table); + return (int) j; + } + APPEND16(req->base.questions[i]->type); + APPEND16(req->base.questions[i]->dns_question_class); + } + + /* Add answer, authority, and additional sections. */ + for (i=0; i<3; ++i) { + struct server_reply_item *item; + if (i==0) + item = req->answer; + else if (i==1) + item = req->authority; + else + item = req->additional; + while (item) { + r = dnsname_to_labels(buf, buf_len, j, item->name, strlen(item->name), &table); + if (r < 0) + goto overflow; + j = r; + + APPEND16(item->type); + APPEND16(item->dns_question_class); + APPEND32(item->ttl); + if (item->is_name) { + off_t len_idx = j, name_start; + j += 2; + name_start = j; + r = dnsname_to_labels(buf, buf_len, j, item->data, strlen(item->data), &table); + if (r < 0) + goto overflow; + j = r; + _t = htons( (short) (j-name_start) ); + memcpy(buf+len_idx, &_t, 2); + } else { + APPEND16(item->datalen); + if (j+item->datalen > (off_t)buf_len) + goto overflow; + memcpy(buf+j, item->data, item->datalen); + j += item->datalen; + } + item = item->next; + } + } + + if (j > 512) { +overflow: + j = 512; + buf[3] |= 0x02; /* set the truncated bit. */ + } + + req->response_len = j; + + if (!(req->response = malloc(req->response_len))) { + server_request_free_answers(req); + dnslabel_clear(&table); + return (-1); + } + memcpy(req->response, buf, req->response_len); + server_request_free_answers(req); + dnslabel_clear(&table); + return (0); +} + +/* exported function */ +int +evdns_server_request_respond(struct evdns_server_request *_req, int err) +{ + struct server_request *req = TO_SERVER_REQUEST(_req); + struct evdns_server_port *port = req->port; + int r; + if (!req->response) { + if ((r = evdns_server_request_format_response(req, err))<0) + return r; + } + + r = sendto(port->socket, req->response, req->response_len, 0, + (struct sockaddr*) &req->addr, req->addrlen); + if (r<0) { + int sock_err = last_error(port->socket); + if (! error_is_eagain(sock_err)) + return -1; + + if (port->pending_replies) { + req->prev_pending = port->pending_replies->prev_pending; + req->next_pending = port->pending_replies; + req->prev_pending->next_pending = + req->next_pending->prev_pending = req; + } else { + req->prev_pending = req->next_pending = req; + port->pending_replies = req; + port->choked = 1; + + (void) event_del(&port->event); + event_set(&port->event, port->socket, (port->closing?0:EV_READ) | EV_WRITE | EV_PERSIST, server_port_ready_callback, port); + + if (event_add(&port->event, NULL) < 0) { + log(EVDNS_LOG_WARN, "Error from libevent when adding event for DNS server"); + } + + } + + return 1; + } + if (server_request_free(req)) + return 0; + + if (port->pending_replies) + server_port_flush(port); + + return 0; +} + +/* Free all storage held by RRs in req. */ +static void +server_request_free_answers(struct server_request *req) +{ + struct server_reply_item *victim, *next, **list; + int i; + for (i = 0; i < 3; ++i) { + if (i==0) + list = &req->answer; + else if (i==1) + list = &req->authority; + else + list = &req->additional; + + victim = *list; + while (victim) { + next = victim->next; + free(victim->name); + if (victim->data) + free(victim->data); + free(victim); + victim = next; + } + *list = NULL; + } +} + +/* Free all storage held by req, and remove links to it. */ +/* return true iff we just wound up freeing the server_port. */ +static int +server_request_free(struct server_request *req) +{ + int i, rc=1; + if (req->base.questions) { + for (i = 0; i < req->base.nquestions; ++i) + free(req->base.questions[i]); + free(req->base.questions); + } + + if (req->port) { + if (req->port->pending_replies == req) { + if (req->next_pending) + req->port->pending_replies = req->next_pending; + else + req->port->pending_replies = NULL; + } + rc = --req->port->refcnt; + } + + if (req->response) { + free(req->response); + } + + server_request_free_answers(req); + + if (req->next_pending && req->next_pending != req) { + req->next_pending->prev_pending = req->prev_pending; + req->prev_pending->next_pending = req->next_pending; + } + + if (rc == 0) { + server_port_free(req->port); + free(req); + return (1); + } + free(req); + return (0); +} + +/* Free all storage held by an evdns_server_port. Only called when */ +static void +server_port_free(struct evdns_server_port *port) +{ + assert(port); + assert(!port->refcnt); + assert(!port->pending_replies); + if (port->socket > 0) { + CLOSE_SOCKET(port->socket); + port->socket = -1; + } + (void) event_del(&port->event); + /* XXXX actually free the port? -NM */ +} + +/* exported function */ +int +evdns_server_request_drop(struct evdns_server_request *_req) +{ + struct server_request *req = TO_SERVER_REQUEST(_req); + server_request_free(req); + return 0; +} + +/* exported function */ +int +evdns_server_request_get_requesting_addr(struct evdns_server_request *_req, struct sockaddr *sa, int addr_len) +{ + struct server_request *req = TO_SERVER_REQUEST(_req); + if (addr_len < (int)req->addrlen) + return -1; + memcpy(sa, &(req->addr), req->addrlen); + return req->addrlen; +} + +#undef APPEND16 +#undef APPEND32 + +/* this is a libevent callback function which is called when a request */ +/* has timed out. */ +static void +evdns_request_timeout_callback(int fd, short events, void *arg) { + struct request *const req = (struct request *) arg; + (void) fd; + (void) events; + + log(EVDNS_LOG_DEBUG, "Request %lx timed out", (unsigned long) arg); + + req->ns->timedout++; + if (req->ns->timedout > global_max_nameserver_timeout) { + req->ns->timedout = 0; + nameserver_failed(req->ns, "request timed out."); + } + + (void) evtimer_del(&req->timeout_event); + if (req->tx_count >= global_max_retransmits) { + /* this request has failed */ + reply_callback(req, 0, DNS_ERR_TIMEOUT, NULL); + request_finished(req, &req_head); + } else { + /* retransmit it */ + evdns_request_transmit(req); + } +} + +/* try to send a request to a given server. */ +/* */ +/* return: */ +/* 0 ok */ +/* 1 temporary failure */ +/* 2 other failure */ +static int +evdns_request_transmit_to(struct request *req, struct nameserver *server) { + const int r = send(server->socket, req->request, req->request_len, 0); + if (r < 0) { + int err = last_error(server->socket); + if (error_is_eagain(err)) return 1; + nameserver_failed(req->ns, strerror(err)); + return 2; + } else if (r != (int)req->request_len) { + return 1; /* short write */ + } else { + return 0; + } +} + +/* try to send a request, updating the fields of the request */ +/* as needed */ +/* */ +/* return: */ +/* 0 ok */ +/* 1 failed */ +static int +evdns_request_transmit(struct request *req) { + int retcode = 0, r; + + /* if we fail to send this packet then this flag marks it */ + /* for evdns_transmit */ + req->transmit_me = 1; + if (req->trans_id == 0xffff) abort(); + + if (req->ns->choked) { + /* don't bother trying to write to a socket */ + /* which we have had EAGAIN from */ + return 1; + } + + r = evdns_request_transmit_to(req, req->ns); + switch (r) { + case 1: + /* temp failure */ + req->ns->choked = 1; + nameserver_write_waiting(req->ns, 1); + return 1; + case 2: + /* failed in some other way */ + retcode = 1; + /* fall through */ + default: + /* all ok */ + log(EVDNS_LOG_DEBUG, + "Setting timeout for request %lx", (unsigned long) req); + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (evtimer_add(&req->timeout_event, &global_timeout) < 0) { + log(EVDNS_LOG_WARN, + "Error from libevent when adding timer for request %lx", + (unsigned long) req); + /* ???? Do more? */ + } + req->tx_count++; + req->transmit_me = 0; + return retcode; + } +} + +static void +nameserver_probe_callback(int result, char type, int count, int ttl, void *addresses, void *arg) { + struct nameserver *const ns = (struct nameserver *) arg; + (void) type; + (void) count; + (void) ttl; + (void) addresses; + + if (result == DNS_ERR_NONE || result == DNS_ERR_NOTEXIST) { + /* this is a good reply */ + nameserver_up(ns); + } else nameserver_probe_failed(ns); +} + +static void +nameserver_send_probe(struct nameserver *const ns) { + struct request *req; + /* here we need to send a probe to a given nameserver */ + /* in the hope that it is up now. */ + + log(EVDNS_LOG_DEBUG, "Sending probe to %s", debug_ntoa(ns->address)); + + req = request_new(TYPE_A, "www.google.com", DNS_QUERY_NO_SEARCH, nameserver_probe_callback, ns); + if (!req) return; + /* we force this into the inflight queue no matter what */ + request_trans_id_set(req, transaction_id_pick()); + req->ns = ns; + request_submit(req); +} + +/* returns: */ +/* 0 didn't try to transmit anything */ +/* 1 tried to transmit something */ +static int +evdns_transmit(void) { + char did_try_to_transmit = 0; + + if (req_head) { + struct request *const started_at = req_head, *req = req_head; + /* first transmit all the requests which are currently waiting */ + do { + if (req->transmit_me) { + did_try_to_transmit = 1; + evdns_request_transmit(req); + } + + req = req->next; + } while (req != started_at); + } + + return did_try_to_transmit; +} + +/* exported function */ +int +evdns_count_nameservers(void) +{ + const struct nameserver *server = server_head; + int n = 0; + if (!server) + return 0; + do { + ++n; + server = server->next; + } while (server != server_head); + return n; +} + +/* exported function */ +int +evdns_clear_nameservers_and_suspend(void) +{ + struct nameserver *server = server_head, *started_at = server_head; + struct request *req = req_head, *req_started_at = req_head; + + if (!server) + return 0; + while (1) { + struct nameserver *next = server->next; + (void) event_del(&server->event); + if (evtimer_initialized(&server->timeout_event)) + (void) evtimer_del(&server->timeout_event); + if (server->socket >= 0) + CLOSE_SOCKET(server->socket); + free(server); + if (next == started_at) + break; + server = next; + } + server_head = NULL; + global_good_nameservers = 0; + + while (req) { + struct request *next = req->next; + req->tx_count = req->reissue_count = 0; + req->ns = NULL; + /* ???? What to do about searches? */ + (void) evtimer_del(&req->timeout_event); + req->trans_id = 0; + req->transmit_me = 0; + + global_requests_waiting++; + evdns_request_insert(req, &req_waiting_head); + /* We want to insert these suspended elements at the front of + * the waiting queue, since they were pending before any of + * the waiting entries were added. This is a circular list, + * so we can just shift the start back by one.*/ + req_waiting_head = req_waiting_head->prev; + + if (next == req_started_at) + break; + req = next; + } + req_head = NULL; + global_requests_inflight = 0; + + return 0; +} + + +/* exported function */ +int +evdns_resume(void) +{ + evdns_requests_pump_waiting_queue(); + return 0; +} + +static int +_evdns_nameserver_add_impl(unsigned long int address, int port) { + /* first check to see if we already have this nameserver */ + + const struct nameserver *server = server_head, *const started_at = server_head; + struct nameserver *ns; + struct sockaddr_in sin; + int err = 0; + if (server) { + do { + if (server->address == address) return 3; + server = server->next; + } while (server != started_at); + } + + ns = (struct nameserver *) malloc(sizeof(struct nameserver)); + if (!ns) return -1; + + memset(ns, 0, sizeof(struct nameserver)); + + ns->socket = socket(PF_INET, SOCK_DGRAM, 0); + if (ns->socket < 0) { err = 1; goto out1; } + evutil_make_socket_nonblocking(ns->socket); + sin.sin_addr.s_addr = address; + sin.sin_port = htons(port); + sin.sin_family = AF_INET; + if (connect(ns->socket, (struct sockaddr *) &sin, sizeof(sin)) != 0) { + err = 2; + goto out2; + } + + ns->address = address; + ns->state = 1; + event_set(&ns->event, ns->socket, EV_READ | EV_PERSIST, nameserver_ready_callback, ns); + if (event_add(&ns->event, NULL) < 0) { + err = 2; + goto out2; + } + + log(EVDNS_LOG_DEBUG, "Added nameserver %s", debug_ntoa(address)); + + /* insert this nameserver into the list of them */ + if (!server_head) { + ns->next = ns->prev = ns; + server_head = ns; + } else { + ns->next = server_head->next; + ns->prev = server_head; + server_head->next = ns; + if (server_head->prev == server_head) { + server_head->prev = ns; + } + } + + global_good_nameservers++; + + return 0; + +out2: + CLOSE_SOCKET(ns->socket); +out1: + free(ns); + log(EVDNS_LOG_WARN, "Unable to add nameserver %s: error %d", debug_ntoa(address), err); + return err; +} + +/* exported function */ +int +evdns_nameserver_add(unsigned long int address) { + return _evdns_nameserver_add_impl(address, 53); +} + +/* exported function */ +int +evdns_nameserver_ip_add(const char *ip_as_string) { + struct in_addr ina; + int port; + char buf[20]; + const char *cp; + cp = strchr(ip_as_string, ':'); + if (! cp) { + cp = ip_as_string; + port = 53; + } else { + port = strtoint(cp+1); + if (port < 0 || port > 65535) { + return 4; + } + if ((cp-ip_as_string) >= (int)sizeof(buf)) { + return 4; + } + memcpy(buf, ip_as_string, cp-ip_as_string); + buf[cp-ip_as_string] = '\0'; + cp = buf; + } + if (!inet_aton(cp, &ina)) { + return 4; + } + return _evdns_nameserver_add_impl(ina.s_addr, port); +} + +/* insert into the tail of the queue */ +static void +evdns_request_insert(struct request *req, struct request **head) { + if (!*head) { + *head = req; + req->next = req->prev = req; + return; + } + + req->prev = (*head)->prev; + req->prev->next = req; + req->next = *head; + (*head)->prev = req; +} + +static int +string_num_dots(const char *s) { + int count = 0; + while ((s = strchr(s, '.'))) { + s++; + count++; + } + return count; +} + +static struct request * +request_new(int type, const char *name, int flags, + evdns_callback_type callback, void *user_ptr) { + const char issuing_now = + (global_requests_inflight < global_max_requests_inflight) ? 1 : 0; + + const int name_len = strlen(name); + const int request_max_len = evdns_request_len(name_len); + const u16 trans_id = issuing_now ? transaction_id_pick() : 0xffff; + /* the request data is alloced in a single block with the header */ + struct request *const req = + (struct request *) malloc(sizeof(struct request) + request_max_len); + int rlen; + (void) flags; + + if (!req) return NULL; + memset(req, 0, sizeof(struct request)); + + /* request data lives just after the header */ + req->request = ((u8 *) req) + sizeof(struct request); + /* denotes that the request data shouldn't be free()ed */ + req->request_appended = 1; + rlen = evdns_request_data_build(name, name_len, trans_id, + type, CLASS_INET, req->request, request_max_len); + if (rlen < 0) + goto err1; + req->request_len = rlen; + req->trans_id = trans_id; + req->tx_count = 0; + req->request_type = type; + req->user_pointer = user_ptr; + req->user_callback = callback; + req->ns = issuing_now ? nameserver_pick() : NULL; + req->next = req->prev = NULL; + + return req; +err1: + free(req); + return NULL; +} + +static void +request_submit(struct request *const req) { + if (req->ns) { + /* if it has a nameserver assigned then this is going */ + /* straight into the inflight queue */ + evdns_request_insert(req, &req_head); + global_requests_inflight++; + evdns_request_transmit(req); + } else { + evdns_request_insert(req, &req_waiting_head); + global_requests_waiting++; + } +} + +/* exported function */ +int evdns_resolve_ipv4(const char *name, int flags, + evdns_callback_type callback, void *ptr) { + log(EVDNS_LOG_DEBUG, "Resolve requested for %s", name); + if (flags & DNS_QUERY_NO_SEARCH) { + struct request *const req = + request_new(TYPE_A, name, flags, callback, ptr); + if (req == NULL) + return (1); + request_submit(req); + return (0); + } else { + return (search_request_new(TYPE_A, name, flags, callback, ptr)); + } +} + +/* exported function */ +int evdns_resolve_ipv6(const char *name, int flags, + evdns_callback_type callback, void *ptr) { + log(EVDNS_LOG_DEBUG, "Resolve requested for %s", name); + if (flags & DNS_QUERY_NO_SEARCH) { + struct request *const req = + request_new(TYPE_AAAA, name, flags, callback, ptr); + if (req == NULL) + return (1); + request_submit(req); + return (0); + } else { + return (search_request_new(TYPE_AAAA, name, flags, callback, ptr)); + } +} + +int evdns_resolve_reverse(struct in_addr *in, int flags, evdns_callback_type callback, void *ptr) { + char buf[32]; + struct request *req; + u32 a; + assert(in); + a = ntohl(in->s_addr); + snprintf(buf, sizeof(buf), "%d.%d.%d.%d.in-addr.arpa", + (int)(u8)((a )&0xff), + (int)(u8)((a>>8 )&0xff), + (int)(u8)((a>>16)&0xff), + (int)(u8)((a>>24)&0xff)); + log(EVDNS_LOG_DEBUG, "Resolve requested for %s (reverse)", buf); + req = request_new(TYPE_PTR, buf, flags, callback, ptr); + if (!req) return 1; + request_submit(req); + return 0; +} + +int evdns_resolve_reverse_ipv6(struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr) { + /* 32 nybbles, 32 periods, "ip6.arpa", NUL. */ + char buf[73]; + char *cp; + struct request *req; + int i; + assert(in); + cp = buf; + for (i=15; i >= 0; --i) { + u8 byte = in->s6_addr[i]; + *cp++ = "0123456789abcdef"[byte & 0x0f]; + *cp++ = '.'; + *cp++ = "0123456789abcdef"[byte >> 4]; + *cp++ = '.'; + } + assert(cp + strlen("ip6.arpa") < buf+sizeof(buf)); + memcpy(cp, "ip6.arpa", strlen("ip6.arpa")+1); + log(EVDNS_LOG_DEBUG, "Resolve requested for %s (reverse)", buf); + req = request_new(TYPE_PTR, buf, flags, callback, ptr); + if (!req) return 1; + request_submit(req); + return 0; +} + +/*/////////////////////////////////////////////////////////////////// */ +/* Search support */ +/* */ +/* the libc resolver has support for searching a number of domains */ +/* to find a name. If nothing else then it takes the single domain */ +/* from the gethostname() call. */ +/* */ +/* It can also be configured via the domain and search options in a */ +/* resolv.conf. */ +/* */ +/* The ndots option controls how many dots it takes for the resolver */ +/* to decide that a name is non-local and so try a raw lookup first. */ + +struct search_domain { + int len; + struct search_domain *next; + /* the text string is appended to this structure */ +}; + +struct search_state { + int refcount; + int ndots; + int num_domains; + struct search_domain *head; +}; + +static struct search_state *global_search_state = NULL; + +static void +search_state_decref(struct search_state *const state) { + if (!state) return; + state->refcount--; + if (!state->refcount) { + struct search_domain *next, *dom; + for (dom = state->head; dom; dom = next) { + next = dom->next; + free(dom); + } + free(state); + } +} + +static struct search_state * +search_state_new(void) { + struct search_state *state = (struct search_state *) malloc(sizeof(struct search_state)); + if (!state) return NULL; + memset(state, 0, sizeof(struct search_state)); + state->refcount = 1; + state->ndots = 1; + + return state; +} + +static void +search_postfix_clear(void) { + search_state_decref(global_search_state); + + global_search_state = search_state_new(); +} + +/* exported function */ +void +evdns_search_clear(void) { + search_postfix_clear(); +} + +static void +search_postfix_add(const char *domain) { + int domain_len; + struct search_domain *sdomain; + while (domain[0] == '.') domain++; + domain_len = strlen(domain); + + if (!global_search_state) global_search_state = search_state_new(); + if (!global_search_state) return; + global_search_state->num_domains++; + + sdomain = (struct search_domain *) malloc(sizeof(struct search_domain) + domain_len); + if (!sdomain) return; + memcpy( ((u8 *) sdomain) + sizeof(struct search_domain), domain, domain_len); + sdomain->next = global_search_state->head; + sdomain->len = domain_len; + + global_search_state->head = sdomain; +} + +/* reverse the order of members in the postfix list. This is needed because, */ +/* when parsing resolv.conf we push elements in the wrong order */ +static void +search_reverse(void) { + struct search_domain *cur, *prev = NULL, *next; + cur = global_search_state->head; + while (cur) { + next = cur->next; + cur->next = prev; + prev = cur; + cur = next; + } + + global_search_state->head = prev; +} + +/* exported function */ +void +evdns_search_add(const char *domain) { + search_postfix_add(domain); +} + +/* exported function */ +void +evdns_search_ndots_set(const int ndots) { + if (!global_search_state) global_search_state = search_state_new(); + if (!global_search_state) return; + global_search_state->ndots = ndots; +} + +static void +search_set_from_hostname(void) { + char hostname[HOST_NAME_MAX + 1], *domainname; + + search_postfix_clear(); + if (gethostname(hostname, sizeof(hostname))) return; + domainname = strchr(hostname, '.'); + if (!domainname) return; + search_postfix_add(domainname); +} + +/* warning: returns malloced string */ +static char * +search_make_new(const struct search_state *const state, int n, const char *const base_name) { + const int base_len = strlen(base_name); + const char need_to_append_dot = base_name[base_len - 1] == '.' ? 0 : 1; + struct search_domain *dom; + + for (dom = state->head; dom; dom = dom->next) { + if (!n--) { + /* this is the postfix we want */ + /* the actual postfix string is kept at the end of the structure */ + const u8 *const postfix = ((u8 *) dom) + sizeof(struct search_domain); + const int postfix_len = dom->len; + char *const newname = (char *) malloc(base_len + need_to_append_dot + postfix_len + 1); + if (!newname) return NULL; + memcpy(newname, base_name, base_len); + if (need_to_append_dot) newname[base_len] = '.'; + memcpy(newname + base_len + need_to_append_dot, postfix, postfix_len); + newname[base_len + need_to_append_dot + postfix_len] = 0; + return newname; + } + } + + /* we ran off the end of the list and still didn't find the requested string */ + abort(); + return NULL; /* unreachable; stops warnings in some compilers. */ +} + +static int +search_request_new(int type, const char *const name, int flags, evdns_callback_type user_callback, void *user_arg) { + assert(type == TYPE_A || type == TYPE_AAAA); + if ( ((flags & DNS_QUERY_NO_SEARCH) == 0) && + global_search_state && + global_search_state->num_domains) { + /* we have some domains to search */ + struct request *req; + if (string_num_dots(name) >= global_search_state->ndots) { + req = request_new(type, name, flags, user_callback, user_arg); + if (!req) return 1; + req->search_index = -1; + } else { + char *const new_name = search_make_new(global_search_state, 0, name); + if (!new_name) return 1; + req = request_new(type, new_name, flags, user_callback, user_arg); + free(new_name); + if (!req) return 1; + req->search_index = 0; + } + req->search_origname = strdup(name); + req->search_state = global_search_state; + req->search_flags = flags; + global_search_state->refcount++; + request_submit(req); + return 0; + } else { + struct request *const req = request_new(type, name, flags, user_callback, user_arg); + if (!req) return 1; + request_submit(req); + return 0; + } +} + +/* this is called when a request has failed to find a name. We need to check */ +/* if it is part of a search and, if so, try the next name in the list */ +/* returns: */ +/* 0 another request has been submitted */ +/* 1 no more requests needed */ +static int +search_try_next(struct request *const req) { + if (req->search_state) { + /* it is part of a search */ + char *new_name; + struct request *newreq; + req->search_index++; + if (req->search_index >= req->search_state->num_domains) { + /* no more postfixes to try, however we may need to try */ + /* this name without a postfix */ + if (string_num_dots(req->search_origname) < req->search_state->ndots) { + /* yep, we need to try it raw */ + newreq = request_new(req->request_type, req->search_origname, req->search_flags, req->user_callback, req->user_pointer); + log(EVDNS_LOG_DEBUG, "Search: trying raw query %s", req->search_origname); + if (newreq) { + request_submit(newreq); + return 0; + } + } + return 1; + } + + new_name = search_make_new(req->search_state, req->search_index, req->search_origname); + if (!new_name) return 1; + log(EVDNS_LOG_DEBUG, "Search: now trying %s (%d)", new_name, req->search_index); + newreq = request_new(req->request_type, new_name, req->search_flags, req->user_callback, req->user_pointer); + free(new_name); + if (!newreq) return 1; + newreq->search_origname = req->search_origname; + req->search_origname = NULL; + newreq->search_state = req->search_state; + newreq->search_flags = req->search_flags; + newreq->search_index = req->search_index; + newreq->search_state->refcount++; + request_submit(newreq); + return 0; + } + return 1; +} + +static void +search_request_finished(struct request *const req) { + if (req->search_state) { + search_state_decref(req->search_state); + req->search_state = NULL; + } + if (req->search_origname) { + free(req->search_origname); + req->search_origname = NULL; + } +} + +/*/////////////////////////////////////////////////////////////////// */ +/* Parsing resolv.conf files */ + +static void +evdns_resolv_set_defaults(int flags) { + /* if the file isn't found then we assume a local resolver */ + if (flags & DNS_OPTION_SEARCH) search_set_from_hostname(); + if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); +} + +#ifndef HAVE_STRTOK_R +static char * +strtok_r(char *s, const char *delim, char **state) { + return strtok(s, delim); +} +#endif + +/* helper version of atoi which returns -1 on error */ +static int +strtoint(const char *const str) { + char *endptr; + const int r = strtol(str, &endptr, 10); + if (*endptr) return -1; + return r; +} + +/* helper version of atoi that returns -1 on error and clips to bounds. */ +static int +strtoint_clipped(const char *const str, int min, int max) +{ + int r = strtoint(str); + if (r == -1) + return r; + else if (r<min) + return min; + else if (r>max) + return max; + else + return r; +} + +/* exported function */ +int +evdns_set_option(const char *option, const char *val, int flags) +{ + if (!strncmp(option, "ndots:", 6)) { + const int ndots = strtoint(val); + if (ndots == -1) return -1; + if (!(flags & DNS_OPTION_SEARCH)) return 0; + log(EVDNS_LOG_DEBUG, "Setting ndots to %d", ndots); + if (!global_search_state) global_search_state = search_state_new(); + if (!global_search_state) return -1; + global_search_state->ndots = ndots; + } else if (!strncmp(option, "timeout:", 8)) { + const int timeout = strtoint(val); + if (timeout == -1) return -1; + if (!(flags & DNS_OPTION_MISC)) return 0; + log(EVDNS_LOG_DEBUG, "Setting timeout to %d", timeout); + global_timeout.tv_sec = timeout; + } else if (!strncmp(option, "max-timeouts:", 12)) { + const int maxtimeout = strtoint_clipped(val, 1, 255); + if (maxtimeout == -1) return -1; + if (!(flags & DNS_OPTION_MISC)) return 0; + log(EVDNS_LOG_DEBUG, "Setting maximum allowed timeouts to %d", + maxtimeout); + global_max_nameserver_timeout = maxtimeout; + } else if (!strncmp(option, "max-inflight:", 13)) { + const int maxinflight = strtoint_clipped(val, 1, 65000); + if (maxinflight == -1) return -1; + if (!(flags & DNS_OPTION_MISC)) return 0; + log(EVDNS_LOG_DEBUG, "Setting maximum inflight requests to %d", + maxinflight); + global_max_requests_inflight = maxinflight; + } else if (!strncmp(option, "attempts:", 9)) { + int retries = strtoint(val); + if (retries == -1) return -1; + if (retries > 255) retries = 255; + if (!(flags & DNS_OPTION_MISC)) return 0; + log(EVDNS_LOG_DEBUG, "Setting retries to %d", retries); + global_max_retransmits = retries; + } + return 0; +} + +static void +resolv_conf_parse_line(char *const start, int flags) { + char *strtok_state; + static const char *const delims = " \t"; +#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) + + char *const first_token = strtok_r(start, delims, &strtok_state); + if (!first_token) return; + + if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { + const char *const nameserver = NEXT_TOKEN; + struct in_addr ina; + + if (inet_aton(nameserver, &ina)) { + /* address is valid */ + evdns_nameserver_add(ina.s_addr); + } + } else if (!strcmp(first_token, "domain") && (flags & DNS_OPTION_SEARCH)) { + const char *const domain = NEXT_TOKEN; + if (domain) { + search_postfix_clear(); + search_postfix_add(domain); + } + } else if (!strcmp(first_token, "search") && (flags & DNS_OPTION_SEARCH)) { + const char *domain; + search_postfix_clear(); + + while ((domain = NEXT_TOKEN)) { + search_postfix_add(domain); + } + search_reverse(); + } else if (!strcmp(first_token, "options")) { + const char *option; + while ((option = NEXT_TOKEN)) { + const char *val = strchr(option, ':'); + evdns_set_option(option, val ? val+1 : "", flags); + } + } +#undef NEXT_TOKEN +} + +/* exported function */ +/* returns: */ +/* 0 no errors */ +/* 1 failed to open file */ +/* 2 failed to stat file */ +/* 3 file too large */ +/* 4 out of memory */ +/* 5 short read from file */ +int +evdns_resolv_conf_parse(int flags, const char *const filename) { + struct stat st; + int fd, n, r; + u8 *resolv; + char *start; + int err = 0; + + log(EVDNS_LOG_DEBUG, "Parsing resolv.conf file %s", filename); + + fd = open(filename, O_RDONLY); + if (fd < 0) { + evdns_resolv_set_defaults(flags); + return 1; + } + + if (fstat(fd, &st)) { err = 2; goto out1; } + if (!st.st_size) { + evdns_resolv_set_defaults(flags); + err = (flags & DNS_OPTION_NAMESERVERS) ? 6 : 0; + goto out1; + } + if (st.st_size > 65535) { err = 3; goto out1; } /* no resolv.conf should be any bigger */ + + resolv = (u8 *) malloc((size_t)st.st_size + 1); + if (!resolv) { err = 4; goto out1; } + + n = 0; + while ((r = read(fd, resolv+n, (size_t)st.st_size-n)) > 0) { + n += r; + if (n == st.st_size) + break; + assert(n < st.st_size); + } + if (r < 0) { err = 5; goto out2; } + resolv[n] = 0; /* we malloced an extra byte; this should be fine. */ + + start = (char *) resolv; + for (;;) { + char *const newline = strchr(start, '\n'); + if (!newline) { + resolv_conf_parse_line(start, flags); + break; + } else { + *newline = 0; + resolv_conf_parse_line(start, flags); + start = newline + 1; + } + } + + if (!server_head && (flags & DNS_OPTION_NAMESERVERS)) { + /* no nameservers were configured. */ + evdns_nameserver_ip_add("127.0.0.1"); + err = 6; + } + if (flags & DNS_OPTION_SEARCH && (!global_search_state || global_search_state->num_domains == 0)) { + search_set_from_hostname(); + } + +out2: + free(resolv); +out1: + close(fd); + return err; +} + +#ifdef WIN32 +/* Add multiple nameservers from a space-or-comma-separated list. */ +static int +evdns_nameserver_ip_add_line(const char *ips) { + const char *addr; + char *buf; + int r; + while (*ips) { + while (ISSPACE(*ips) || *ips == ',' || *ips == '\t') + ++ips; + addr = ips; + while (ISDIGIT(*ips) || *ips == '.' || *ips == ':') + ++ips; + buf = malloc(ips-addr+1); + if (!buf) return 4; + memcpy(buf, addr, ips-addr); + buf[ips-addr] = '\0'; + r = evdns_nameserver_ip_add(buf); + free(buf); + if (r) return r; + } + return 0; +} + +typedef DWORD(WINAPI *GetNetworkParams_fn_t)(FIXED_INFO *, DWORD*); + +/* Use the windows GetNetworkParams interface in iphlpapi.dll to */ +/* figure out what our nameservers are. */ +static int +load_nameservers_with_getnetworkparams(void) +{ + /* Based on MSDN examples and inspection of c-ares code. */ + FIXED_INFO *fixed; + HMODULE handle = 0; + ULONG size = sizeof(FIXED_INFO); + void *buf = NULL; + int status = 0, r, added_any; + IP_ADDR_STRING *ns; + GetNetworkParams_fn_t fn; + + if (!(handle = LoadLibrary("iphlpapi.dll"))) { + log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); + status = -1; + goto done; + } + if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { + log(EVDNS_LOG_WARN, "Could not get address of function."); + status = -1; + goto done; + } + + buf = malloc(size); + if (!buf) { status = 4; goto done; } + fixed = buf; + r = fn(fixed, &size); + if (r != ERROR_SUCCESS && r != ERROR_BUFFER_OVERFLOW) { + status = -1; + goto done; + } + if (r != ERROR_SUCCESS) { + free(buf); + buf = malloc(size); + if (!buf) { status = 4; goto done; } + fixed = buf; + r = fn(fixed, &size); + if (r != ERROR_SUCCESS) { + log(EVDNS_LOG_DEBUG, "fn() failed."); + status = -1; + goto done; + } + } + + assert(fixed); + added_any = 0; + ns = &(fixed->DnsServerList); + while (ns) { + r = evdns_nameserver_ip_add_line(ns->IpAddress.String); + if (r) { + log(EVDNS_LOG_DEBUG,"Could not add nameserver %s to list,error: %d", + (ns->IpAddress.String),(int)GetLastError()); + status = r; + goto done; + } else { + log(EVDNS_LOG_DEBUG,"Succesfully added %s as nameserver",ns->IpAddress.String); + } + + added_any++; + ns = ns->Next; + } + + if (!added_any) { + log(EVDNS_LOG_DEBUG, "No nameservers added."); + status = -1; + } + + done: + if (buf) + free(buf); + if (handle) + FreeLibrary(handle); + return status; +} + +static int +config_nameserver_from_reg_key(HKEY key, const char *subkey) +{ + char *buf; + DWORD bufsz = 0, type = 0; + int status = 0; + + if (RegQueryValueEx(key, subkey, 0, &type, NULL, &bufsz) + != ERROR_MORE_DATA) + return -1; + if (!(buf = malloc(bufsz))) + return -1; + + if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) + == ERROR_SUCCESS && bufsz > 1) { + status = evdns_nameserver_ip_add_line(buf); + } + + free(buf); + return status; +} + +#define SERVICES_KEY "System\\CurrentControlSet\\Services\\" +#define WIN_NS_9X_KEY SERVICES_KEY "VxD\\MSTCP" +#define WIN_NS_NT_KEY SERVICES_KEY "Tcpip\\Parameters" + +static int +load_nameservers_from_registry(void) +{ + int found = 0; + int r; +#define TRY(k, name) \ + if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ + log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ + found = 1; \ + } else if (!found) { \ + log(EVDNS_LOG_DEBUG,"Didn't find nameservers in %s/%s", \ + #k,#name); \ + } + + if (((int)GetVersion()) > 0) { /* NT */ + HKEY nt_key = 0, interfaces_key = 0; + + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, + KEY_READ, &nt_key) != ERROR_SUCCESS) { + log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); + return -1; + } + r = RegOpenKeyEx(nt_key, "Interfaces", 0, + KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, + &interfaces_key); + if (r != ERROR_SUCCESS) { + log(EVDNS_LOG_DEBUG,"Couldn't open interfaces key, %d",(int)GetLastError()); + return -1; + } + TRY(nt_key, "NameServer"); + TRY(nt_key, "DhcpNameServer"); + TRY(interfaces_key, "NameServer"); + TRY(interfaces_key, "DhcpNameServer"); + RegCloseKey(interfaces_key); + RegCloseKey(nt_key); + } else { + HKEY win_key = 0; + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, + KEY_READ, &win_key) != ERROR_SUCCESS) { + log(EVDNS_LOG_DEBUG, "Couldn't open registry key, %d", (int)GetLastError()); + return -1; + } + TRY(win_key, "NameServer"); + RegCloseKey(win_key); + } + + if (found == 0) { + log(EVDNS_LOG_WARN,"Didn't find any nameservers."); + } + + return found ? 0 : -1; +#undef TRY +} + +static int +evdns_config_windows_nameservers(void) +{ + if (load_nameservers_with_getnetworkparams() == 0) + return 0; + return load_nameservers_from_registry(); +} +#endif + +int +evdns_init(void) +{ + int res = 0; +#ifdef WIN32 + res = evdns_config_windows_nameservers(); +#else + res = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, "/etc/resolv.conf"); +#endif + + return (res); +} + +const char * +evdns_err_to_string(int err) +{ + switch (err) { + case DNS_ERR_NONE: return "no error"; + case DNS_ERR_FORMAT: return "misformatted query"; + case DNS_ERR_SERVERFAILED: return "server failed"; + case DNS_ERR_NOTEXIST: return "name does not exist"; + case DNS_ERR_NOTIMPL: return "query not implemented"; + case DNS_ERR_REFUSED: return "refused"; + + case DNS_ERR_TRUNCATED: return "reply truncated or ill-formed"; + case DNS_ERR_UNKNOWN: return "unknown"; + case DNS_ERR_TIMEOUT: return "request timed out"; + case DNS_ERR_SHUTDOWN: return "dns subsystem shut down"; + default: return "[Unknown error code]"; + } +} + +void +evdns_shutdown(int fail_requests) +{ + struct nameserver *server, *server_next; + struct search_domain *dom, *dom_next; + + while (req_head) { + if (fail_requests) + reply_callback(req_head, 0, DNS_ERR_SHUTDOWN, NULL); + request_finished(req_head, &req_head); + } + while (req_waiting_head) { + if (fail_requests) + reply_callback(req_waiting_head, 0, DNS_ERR_SHUTDOWN, NULL); + request_finished(req_waiting_head, &req_waiting_head); + } + global_requests_inflight = global_requests_waiting = 0; + + for (server = server_head; server; server = server_next) { + server_next = server->next; + if (server->socket >= 0) + CLOSE_SOCKET(server->socket); + (void) event_del(&server->event); + if (server->state == 0) + (void) event_del(&server->timeout_event); + free(server); + if (server_next == server_head) + break; + } + server_head = NULL; + global_good_nameservers = 0; + + if (global_search_state) { + for (dom = global_search_state->head; dom; dom = dom_next) { + dom_next = dom->next; + free(dom); + } + free(global_search_state); + global_search_state = NULL; + } + evdns_log_fn = NULL; +} + +#ifdef EVDNS_MAIN +void +main_callback(int result, char type, int count, int ttl, + void *addrs, void *orig) { + char *n = (char*)orig; + int i; + for (i = 0; i < count; ++i) { + if (type == DNS_IPv4_A) { + printf("%s: %s\n", n, debug_ntoa(((u32*)addrs)[i])); + } else if (type == DNS_PTR) { + printf("%s: %s\n", n, ((char**)addrs)[i]); + } + } + if (!count) { + printf("%s: No answer (%d)\n", n, result); + } + fflush(stdout); +} +void +evdns_server_callback(struct evdns_server_request *req, void *data) +{ + int i, r; + (void)data; + /* dummy; give 192.168.11.11 as an answer for all A questions, + * give foo.bar.example.com as an answer for all PTR questions. */ + for (i = 0; i < req->nquestions; ++i) { + u32 ans = htonl(0xc0a80b0bUL); + if (req->questions[i]->type == EVDNS_TYPE_A && + req->questions[i]->dns_question_class == EVDNS_CLASS_INET) { + printf(" -- replying for %s (A)\n", req->questions[i]->name); + r = evdns_server_request_add_a_reply(req, req->questions[i]->name, + 1, &ans, 10); + if (r<0) + printf("eeep, didn't work.\n"); + } else if (req->questions[i]->type == EVDNS_TYPE_PTR && + req->questions[i]->dns_question_class == EVDNS_CLASS_INET) { + printf(" -- replying for %s (PTR)\n", req->questions[i]->name); + r = evdns_server_request_add_ptr_reply(req, NULL, req->questions[i]->name, + "foo.bar.example.com", 10); + } else { + printf(" -- skipping %s [%d %d]\n", req->questions[i]->name, + req->questions[i]->type, req->questions[i]->dns_question_class); + } + } + + r = evdns_request_respond(req, 0); + if (r<0) + printf("eeek, couldn't send reply.\n"); +} + +void +logfn(int is_warn, const char *msg) { + (void) is_warn; + fprintf(stderr, "%s\n", msg); +} +int +main(int c, char **v) { + int idx; + int reverse = 0, verbose = 1, servertest = 0; + if (c<2) { + fprintf(stderr, "syntax: %s [-x] [-v] hostname\n", v[0]); + fprintf(stderr, "syntax: %s [-servertest]\n", v[0]); + return 1; + } + idx = 1; + while (idx < c && v[idx][0] == '-') { + if (!strcmp(v[idx], "-x")) + reverse = 1; + else if (!strcmp(v[idx], "-v")) + verbose = 1; + else if (!strcmp(v[idx], "-servertest")) + servertest = 1; + else + fprintf(stderr, "Unknown option %s\n", v[idx]); + ++idx; + } + event_init(); + if (verbose) + evdns_set_log_fn(logfn); + evdns_resolv_conf_parse(DNS_OPTION_NAMESERVERS, "/etc/resolv.conf"); + if (servertest) { + int sock; + struct sockaddr_in my_addr; + sock = socket(PF_INET, SOCK_DGRAM, 0); + evutil_make_socket_nonblocking(sock); + my_addr.sin_family = AF_INET; + my_addr.sin_port = htons(10053); + my_addr.sin_addr.s_addr = INADDR_ANY; + if (bind(sock, (struct sockaddr*)&my_addr, sizeof(my_addr))<0) { + perror("bind"); + exit(1); + } + evdns_add_server_port(sock, 0, evdns_server_callback, NULL); + } + for (; idx < c; ++idx) { + if (reverse) { + struct in_addr addr; + if (!inet_aton(v[idx], &addr)) { + fprintf(stderr, "Skipping non-IP %s\n", v[idx]); + continue; + } + fprintf(stderr, "resolving %s...\n",v[idx]); + evdns_resolve_reverse(&addr, 0, main_callback, v[idx]); + } else { + fprintf(stderr, "resolving (fwd) %s...\n",v[idx]); + evdns_resolve_ipv4(v[idx], 0, main_callback, v[idx]); + } + } + fflush(stdout); + event_dispatch(); + return 0; +} +#endif === added file 'extra/libevent/evdns.h' --- a/extra/libevent/evdns.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evdns.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,528 @@ +/* + * Copyright (c) 2006 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * The original DNS code is due to Adam Langley with heavy + * modifications by Nick Mathewson. Adam put his DNS software in the + * public domain. You can find his original copyright below. Please, + * aware that the code as part of libevent is governed by the 3-clause + * BSD license above. + * + * This software is Public Domain. To view a copy of the public domain dedication, + * visit http://creativecommons.org/licenses/publicdomain/ or send a letter to + * Creative Commons, 559 Nathan Abbott Way, Stanford, California 94305, USA. + * + * I ask and expect, but do not require, that all derivative works contain an + * attribution similar to: + * Parts developed by Adam Langley <agl(a)imperialviolet.org> + * + * You may wish to replace the word "Parts" with something else depending on + * the amount of original code. + * + * (Derivative works does not include programs which link against, run or include + * the source verbatim in their source distributions) + */ + +/** @file evdns.h + * + * Welcome, gentle reader + * + * Async DNS lookups are really a whole lot harder than they should be, + * mostly stemming from the fact that the libc resolver has never been + * very good at them. Before you use this library you should see if libc + * can do the job for you with the modern async call getaddrinfo_a + * (see http://www.imperialviolet.org/page25.html#e498) Otherwise, + * please continue. + * + * This code is based on libevent and you must call event_init before + * any of the APIs in this file. You must also seed the OpenSSL random + * source if you are using OpenSSL for ids (see below). + * + * This library is designed to be included and shipped with your source + * code. You statically link with it. You should also test for the + * existence of strtok_r and define HAVE_STRTOK_R if you have it. + * + * The DNS protocol requires a good source of id numbers and these + * numbers should be unpredictable for spoofing reasons. There are + * three methods for generating them here and you must define exactly + * one of them. In increasing order of preference: + * + * DNS_USE_GETTIMEOFDAY_FOR_ID: + * Using the bottom 16 bits of the usec result from gettimeofday. This + * is a pretty poor solution but should work anywhere. + * DNS_USE_CPU_CLOCK_FOR_ID: + * Using the bottom 16 bits of the nsec result from the CPU's time + * counter. This is better, but may not work everywhere. Requires + * POSIX realtime support and you'll need to link against -lrt on + * glibc systems at least. + * DNS_USE_OPENSSL_FOR_ID: + * Uses the OpenSSL RAND_bytes call to generate the data. You must + * have seeded the pool before making any calls to this library. + * + * The library keeps track of the state of nameservers and will avoid + * them when they go down. Otherwise it will round robin between them. + * + * Quick start guide: + * #include "evdns.h" + * void callback(int result, char type, int count, int ttl, + * void *addresses, void *arg); + * evdns_resolv_conf_parse(DNS_OPTIONS_ALL, "/etc/resolv.conf"); + * evdns_resolve("www.hostname.com", 0, callback, NULL); + * + * When the lookup is complete the callback function is called. The + * first argument will be one of the DNS_ERR_* defines in evdns.h. + * Hopefully it will be DNS_ERR_NONE, in which case type will be + * DNS_IPv4_A, count will be the number of IP addresses, ttl is the time + * which the data can be cached for (in seconds), addresses will point + * to an array of uint32_t's and arg will be whatever you passed to + * evdns_resolve. + * + * Searching: + * + * In order for this library to be a good replacement for glibc's resolver it + * supports searching. This involves setting a list of default domains, in + * which names will be queried for. The number of dots in the query name + * determines the order in which this list is used. + * + * Searching appears to be a single lookup from the point of view of the API, + * although many DNS queries may be generated from a single call to + * evdns_resolve. Searching can also drastically slow down the resolution + * of names. + * + * To disable searching: + * 1. Never set it up. If you never call evdns_resolv_conf_parse or + * evdns_search_add then no searching will occur. + * + * 2. If you do call evdns_resolv_conf_parse then don't pass + * DNS_OPTION_SEARCH (or DNS_OPTIONS_ALL, which implies it). + * + * 3. When calling evdns_resolve, pass the DNS_QUERY_NO_SEARCH flag. + * + * The order of searches depends on the number of dots in the name. If the + * number is greater than the ndots setting then the names is first tried + * globally. Otherwise each search domain is appended in turn. + * + * The ndots setting can either be set from a resolv.conf, or by calling + * evdns_search_ndots_set. + * + * For example, with ndots set to 1 (the default) and a search domain list of + * ["myhome.net"]: + * Query: www + * Order: www.myhome.net, www. + * + * Query: www.abc + * Order: www.abc., www.abc.myhome.net + * + * Internals: + * + * Requests are kept in two queues. The first is the inflight queue. In + * this queue requests have an allocated transaction id and nameserver. + * They will soon be transmitted if they haven't already been. + * + * The second is the waiting queue. The size of the inflight ring is + * limited and all other requests wait in waiting queue for space. This + * bounds the number of concurrent requests so that we don't flood the + * nameserver. Several algorithms require a full walk of the inflight + * queue and so bounding its size keeps thing going nicely under huge + * (many thousands of requests) loads. + * + * If a nameserver loses too many requests it is considered down and we + * try not to use it. After a while we send a probe to that nameserver + * (a lookup for google.com) and, if it replies, we consider it working + * again. If the nameserver fails a probe we wait longer to try again + * with the next probe. + */ + +#ifndef EVENTDNS_H +#define EVENTDNS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* For integer types. */ +#include <evutil.h> + +/** Error codes 0-5 are as described in RFC 1035. */ +#define DNS_ERR_NONE 0 +/** The name server was unable to interpret the query */ +#define DNS_ERR_FORMAT 1 +/** The name server was unable to process this query due to a problem with the + * name server */ +#define DNS_ERR_SERVERFAILED 2 +/** The domain name does not exist */ +#define DNS_ERR_NOTEXIST 3 +/** The name server does not support the requested kind of query */ +#define DNS_ERR_NOTIMPL 4 +/** The name server refuses to reform the specified operation for policy + * reasons */ +#define DNS_ERR_REFUSED 5 +/** The reply was truncated or ill-formated */ +#define DNS_ERR_TRUNCATED 65 +/** An unknown error occurred */ +#define DNS_ERR_UNKNOWN 66 +/** Communication with the server timed out */ +#define DNS_ERR_TIMEOUT 67 +/** The request was canceled because the DNS subsystem was shut down. */ +#define DNS_ERR_SHUTDOWN 68 + +#define DNS_IPv4_A 1 +#define DNS_PTR 2 +#define DNS_IPv6_AAAA 3 + +#define DNS_QUERY_NO_SEARCH 1 + +#define DNS_OPTION_SEARCH 1 +#define DNS_OPTION_NAMESERVERS 2 +#define DNS_OPTION_MISC 4 +#define DNS_OPTIONS_ALL 7 + +/** + * The callback that contains the results from a lookup. + * - type is either DNS_IPv4_A or DNS_PTR or DNS_IPv6_AAAA + * - count contains the number of addresses of form type + * - ttl is the number of seconds the resolution may be cached for. + * - addresses needs to be cast according to type + */ +typedef void (*evdns_callback_type) (int result, char type, int count, int ttl, void *addresses, void *arg); + +/** + Initialize the asynchronous DNS library. + + This function initializes support for non-blocking name resolution by + calling evdns_resolv_conf_parse() on UNIX and + evdns_config_windows_nameservers() on Windows. + + @return 0 if successful, or -1 if an error occurred + @see evdns_shutdown() + */ +int evdns_init(void); + + +/** + Shut down the asynchronous DNS resolver and terminate all active requests. + + If the 'fail_requests' option is enabled, all active requests will return + an empty result with the error flag set to DNS_ERR_SHUTDOWN. Otherwise, + the requests will be silently discarded. + + @param fail_requests if zero, active requests will be aborted; if non-zero, + active requests will return DNS_ERR_SHUTDOWN. + @see evdns_init() + */ +void evdns_shutdown(int fail_requests); + + +/** + Convert a DNS error code to a string. + + @param err the DNS error code + @return a string containing an explanation of the error code +*/ +const char *evdns_err_to_string(int err); + + +/** + Add a nameserver. + + The address should be an IPv4 address in network byte order. + The type of address is chosen so that it matches in_addr.s_addr. + + @param address an IP address in network byte order + @return 0 if successful, or -1 if an error occurred + @see evdns_nameserver_ip_add() + */ +int evdns_nameserver_add(unsigned long int address); + + +/** + Get the number of configured nameservers. + + This returns the number of configured nameservers (not necessarily the + number of running nameservers). This is useful for double-checking + whether our calls to the various nameserver configuration functions + have been successful. + + @return the number of configured nameservers + @see evdns_nameserver_add() + */ +int evdns_count_nameservers(void); + + +/** + Remove all configured nameservers, and suspend all pending resolves. + + Resolves will not necessarily be re-attempted until evdns_resume() is called. + + @return 0 if successful, or -1 if an error occurred + @see evdns_resume() + */ +int evdns_clear_nameservers_and_suspend(void); + + +/** + Resume normal operation and continue any suspended resolve requests. + + Re-attempt resolves left in limbo after an earlier call to + evdns_clear_nameservers_and_suspend(). + + @return 0 if successful, or -1 if an error occurred + @see evdns_clear_nameservers_and_suspend() + */ +int evdns_resume(void); + + +/** + Add a nameserver. + + This wraps the evdns_nameserver_add() function by parsing a string as an IP + address and adds it as a nameserver. + + @return 0 if successful, or -1 if an error occurred + @see evdns_nameserver_add() + */ +int evdns_nameserver_ip_add(const char *ip_as_string); + + +/** + Lookup an A record for a given name. + + @param name a DNS hostname + @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. + @param callback a callback function to invoke when the request is completed + @param ptr an argument to pass to the callback function + @return 0 if successful, or -1 if an error occurred + @see evdns_resolve_ipv6(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6() + */ +int evdns_resolve_ipv4(const char *name, int flags, evdns_callback_type callback, void *ptr); + + +/** + Lookup an AAAA record for a given name. + + @param name a DNS hostname + @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. + @param callback a callback function to invoke when the request is completed + @param ptr an argument to pass to the callback function + @return 0 if successful, or -1 if an error occurred + @see evdns_resolve_ipv4(), evdns_resolve_reverse(), evdns_resolve_reverse_ipv6() + */ +int evdns_resolve_ipv6(const char *name, int flags, evdns_callback_type callback, void *ptr); + +struct in_addr; +struct in6_addr; + +/** + Lookup a PTR record for a given IP address. + + @param in an IPv4 address + @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. + @param callback a callback function to invoke when the request is completed + @param ptr an argument to pass to the callback function + @return 0 if successful, or -1 if an error occurred + @see evdns_resolve_reverse_ipv6() + */ +int evdns_resolve_reverse(struct in_addr *in, int flags, evdns_callback_type callback, void *ptr); + + +/** + Lookup a PTR record for a given IPv6 address. + + @param in an IPv6 address + @param flags either 0, or DNS_QUERY_NO_SEARCH to disable searching for this query. + @param callback a callback function to invoke when the request is completed + @param ptr an argument to pass to the callback function + @return 0 if successful, or -1 if an error occurred + @see evdns_resolve_reverse_ipv6() + */ +int evdns_resolve_reverse_ipv6(struct in6_addr *in, int flags, evdns_callback_type callback, void *ptr); + + +/** + Set the value of a configuration option. + + The currently available configuration options are: + + ndots, timeout, max-timeouts, max-inflight, and attempts + + @param option the name of the configuration option to be modified + @param val the value to be set + @param flags either 0 | DNS_OPTION_SEARCH | DNS_OPTION_MISC + @return 0 if successful, or -1 if an error occurred + */ +int evdns_set_option(const char *option, const char *val, int flags); + + +/** + Parse a resolv.conf file. + + The 'flags' parameter determines what information is parsed from the + resolv.conf file. See the man page for resolv.conf for the format of this + file. + + The following directives are not parsed from the file: sortlist, rotate, + no-check-names, inet6, debug. + + If this function encounters an error, the possible return values are: 1 = + failed to open file, 2 = failed to stat file, 3 = file too large, 4 = out of + memory, 5 = short read from file, 6 = no nameservers listed in the file + + @param flags any of DNS_OPTION_NAMESERVERS|DNS_OPTION_SEARCH|DNS_OPTION_MISC| + DNS_OPTIONS_ALL + @param filename the path to the resolv.conf file + @return 0 if successful, or various positive error codes if an error + occurred (see above) + @see resolv.conf(3), evdns_config_windows_nameservers() + */ +int evdns_resolv_conf_parse(int flags, const char *const filename); + + +/** + Obtain nameserver information using the Windows API. + + Attempt to configure a set of nameservers based on platform settings on + a win32 host. Preferentially tries to use GetNetworkParams; if that fails, + looks in the registry. + + @return 0 if successful, or -1 if an error occurred + @see evdns_resolv_conf_parse() + */ +#ifdef MS_WINDOWS +int evdns_config_windows_nameservers(void); +#endif + + +/** + Clear the list of search domains. + */ +void evdns_search_clear(void); + + +/** + Add a domain to the list of search domains + + @param domain the domain to be added to the search list + */ +void evdns_search_add(const char *domain); + + +/** + Set the 'ndots' parameter for searches. + + Sets the number of dots which, when found in a name, causes + the first query to be without any search domain. + + @param ndots the new ndots parameter + */ +void evdns_search_ndots_set(const int ndots); + +/** + A callback that is invoked when a log message is generated + + @param is_warning indicates if the log message is a 'warning' + @param msg the content of the log message + */ +typedef void (*evdns_debug_log_fn_type)(int is_warning, const char *msg); + + +/** + Set the callback function to handle log messages. + + @param fn the callback to be invoked when a log message is generated + */ +void evdns_set_log_fn(evdns_debug_log_fn_type fn); + +/** + Set a callback that will be invoked to generate transaction IDs. By + default, we pick transaction IDs based on the current clock time. + + @param fn the new callback, or NULL to use the default. + */ +void evdns_set_transaction_id_fn(ev_uint16_t (*fn)(void)); + +#define DNS_NO_SEARCH 1 + +/* + * Structures and functions used to implement a DNS server. + */ + +struct evdns_server_request { + int flags; + int nquestions; + struct evdns_server_question **questions; +}; +struct evdns_server_question { + int type; +#ifdef __cplusplus + int dns_question_class; +#else + /* You should refer to this field as "dns_question_class". The + * name "class" works in C for backward compatibility, and will be + * removed in a future version. (1.5 or later). */ + int class; +#define dns_question_class class +#endif + char name[1]; +}; +typedef void (*evdns_request_callback_fn_type)(struct evdns_server_request *, void *); +#define EVDNS_ANSWER_SECTION 0 +#define EVDNS_AUTHORITY_SECTION 1 +#define EVDNS_ADDITIONAL_SECTION 2 + +#define EVDNS_TYPE_A 1 +#define EVDNS_TYPE_NS 2 +#define EVDNS_TYPE_CNAME 5 +#define EVDNS_TYPE_SOA 6 +#define EVDNS_TYPE_PTR 12 +#define EVDNS_TYPE_MX 15 +#define EVDNS_TYPE_TXT 16 +#define EVDNS_TYPE_AAAA 28 + +#define EVDNS_QTYPE_AXFR 252 +#define EVDNS_QTYPE_ALL 255 + +#define EVDNS_CLASS_INET 1 + +struct evdns_server_port *evdns_add_server_port(int socket, int is_tcp, evdns_request_callback_fn_type callback, void *user_data); +void evdns_close_server_port(struct evdns_server_port *port); + +int evdns_server_request_add_reply(struct evdns_server_request *req, int section, const char *name, int type, int dns_class, int ttl, int datalen, int is_name, const char *data); +int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); +int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); +int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl); +int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl); + +int evdns_server_request_respond(struct evdns_server_request *req, int err); +int evdns_server_request_drop(struct evdns_server_request *req); +struct sockaddr; +int evdns_server_request_get_requesting_addr(struct evdns_server_request *_req, struct sockaddr *sa, int addr_len); + +#ifdef __cplusplus +} +#endif + +#endif /* !EVENTDNS_H */ === added file 'extra/libevent/event-config.h' --- a/extra/libevent/event-config.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/event-config.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,1409 @@ +/* event-config.h + * Generated by autoconf; post-processed by libevent. + * Do not edit this file. + * Do not rely on macros in this file existing in later versions. + */ +#ifndef _EVENT_CONFIG_H_ +#define _EVENT_CONFIG_H_ +/* include/config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.in by autoheader. */ + +/* Support big tables */ +#define _EVENT_BIG_TABLES 1 + +/* Whether features provided by the user community should be included */ +#define _EVENT_COMMUNITY_SERVER 1 + +/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP + systems. This function is required for `alloca.c' support on those systems. + */ +/* #undef _EVENT_CRAY_STACKSEG_END */ + +/* Define to 1 if using `alloca.c'. */ +/* #undef _EVENT_C_ALLOCA */ + +/* Don't use libdbug */ +/* #undef _EVENT_DBUG_OFF */ + +/* Use libdbug */ +#define _EVENT_DBUG_ON 1 + +/* all charsets are available */ +#define _EVENT_DEFINE_ALL_CHARACTER_SETS 1 + +/* Disables the use of --init-file, --skip-grant-tables and --bootstrap + options */ +/* #undef _EVENT_DISABLE_GRANT_OPTIONS */ + +/* Define if clock_gettime is available in libc */ +/* #undef _EVENT_DNS_USE_CPU_CLOCK_FOR_ID */ + +/* Define is no secure id variant is available */ +#define _EVENT_DNS_USE_GETTIMEOFDAY_FOR_ID 1 + +/* Version of .frm files */ +#define _EVENT_DOT_FRM_VERSION 6 + +/* If LOAD DATA LOCAL INFILE should be enabled by default */ +#define _EVENT_ENABLED_LOCAL_INFILE 1 + +/* If SHOW PROFILE should be enabled */ +#define _EVENT_ENABLED_PROFILING 1 + +/* Enable error injection in MySQL Server */ +/* #undef _EVENT_ERROR_INJECT_SUPPORT */ + +/* Do we have FIONREAD */ +#define _EVENT_FIONREAD_IN_SYS_IOCTL 1 + +/* READLINE: your system defines TIOCGWINSZ in sys/ioctl.h. */ +#define _EVENT_GWINSZ_IN_SYS_IOCTL 1 + +/* Define to 1 if you have the `abi::__cxa_demangle' function. */ +/* #undef _EVENT_HAVE_ABI_CXA_DEMANGLE */ + +/* Define to 1 if you have the <aio.h> header file. */ +#define _EVENT_HAVE_AIO_H 1 + +/* Define to 1 if you have the `alarm' function. */ +#define _EVENT_HAVE_ALARM 1 + +/* Define to 1 if you have `alloca', as a function or macro. */ +#define _EVENT_HAVE_ALLOCA 1 + +/* Define to 1 if you have <alloca.h> and it should be used (not on Ultrix). + */ +#define _EVENT_HAVE_ALLOCA_H 1 + +/* Define to 1 if you have the <arpa/inet.h> header file. */ +#define _EVENT_HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the <asm/termbits.h> header file. */ +#define _EVENT_HAVE_ASM_TERMBITS_H 1 + +/* Define to 1 if you have the `backtrace' function. */ +#define _EVENT_HAVE_BACKTRACE 1 + +/* Define to 1 if you have the `backtrace_symbols' function. */ +#define _EVENT_HAVE_BACKTRACE_SYMBOLS 1 + +/* Define to 1 if you have the `backtrace_symbols_fd' function. */ +#define _EVENT_HAVE_BACKTRACE_SYMBOLS_FD 1 + +/* Define to 1 if you have the `bcmp' function. */ +#define _EVENT_HAVE_BCMP 1 + +/* Define to 1 if you have the `bfill' function. */ +/* #undef _EVENT_HAVE_BFILL */ + +/* Define to 1 if you have the `bmove' function. */ +/* #undef _EVENT_HAVE_BMOVE */ + +/* bool is not defined by all C++ compilators */ +#define _EVENT_HAVE_BOOL 1 + +/* Can netinet be included */ +/* #undef _EVENT_HAVE_BROKEN_NETINET_INCLUDES */ + +/* BSD style signals */ +/* #undef _EVENT_HAVE_BSD_SIGNALS */ + +/* Define to 1 if you have the `bsearch' function. */ +#define _EVENT_HAVE_BSEARCH 1 + +/* Define to 1 if compiler defines __bss_start. */ +#define _EVENT_HAVE_BSS_START 1 + +/* Define to 1 if you have the `bzero' function. */ +#define _EVENT_HAVE_BZERO 1 + +/* Define to enable charset armscii8 */ +/* #undef _EVENT_HAVE_CHARSET_armscii8 */ + +/* Define to enable ascii character set */ +/* #undef _EVENT_HAVE_CHARSET_ascii */ + +/* Define to enable charset big5 */ +#define _EVENT_HAVE_CHARSET_big5 1 + +/* Define to enable cp1250 */ +#define _EVENT_HAVE_CHARSET_cp1250 1 + +/* Define to enable charset cp1251 */ +/* #undef _EVENT_HAVE_CHARSET_cp1251 */ + +/* Define to enable charset cp1256 */ +/* #undef _EVENT_HAVE_CHARSET_cp1256 */ + +/* Define to enable charset cp1257 */ +/* #undef _EVENT_HAVE_CHARSET_cp1257 */ + +/* Define to enable charset cp850 */ +/* #undef _EVENT_HAVE_CHARSET_cp850 */ + +/* Define to enable charset cp852 */ +/* #undef _EVENT_HAVE_CHARSET_cp852 */ + +/* Define to enable charset cp866 */ +/* #undef _EVENT_HAVE_CHARSET_cp866 */ + +/* Define to enable charset cp932 */ +#define _EVENT_HAVE_CHARSET_cp932 1 + +/* Define to enable charset dec8 */ +/* #undef _EVENT_HAVE_CHARSET_dec8 */ + +/* Define to enable charset eucjpms */ +#define _EVENT_HAVE_CHARSET_eucjpms 1 + +/* Define to enable charset euckr */ +#define _EVENT_HAVE_CHARSET_euckr 1 + +/* Define to enable charset gb2312 */ +#define _EVENT_HAVE_CHARSET_gb2312 1 + +/* Define to enable charset gbk */ +#define _EVENT_HAVE_CHARSET_gbk 1 + +/* Define to enable charset geostd8 */ +/* #undef _EVENT_HAVE_CHARSET_geostd8 */ + +/* Define to enable charset greek */ +/* #undef _EVENT_HAVE_CHARSET_greek */ + +/* Define to enable charset hebrew */ +/* #undef _EVENT_HAVE_CHARSET_hebrew */ + +/* Define to enable charset hp8 */ +/* #undef _EVENT_HAVE_CHARSET_hp8 */ + +/* Define to enable charset keybcs2 */ +/* #undef _EVENT_HAVE_CHARSET_keybcs2 */ + +/* Define to enable charset koi8r */ +/* #undef _EVENT_HAVE_CHARSET_koi8r */ + +/* Define to enable charset koi8u */ +/* #undef _EVENT_HAVE_CHARSET_koi8u */ + +/* Define to enable charset latin1 */ +#define _EVENT_HAVE_CHARSET_latin1 1 + +/* Define to enable charset latin2 */ +#define _EVENT_HAVE_CHARSET_latin2 1 + +/* Define to enable charset latin5 */ +/* #undef _EVENT_HAVE_CHARSET_latin5 */ + +/* Define to enable charset latin7 */ +/* #undef _EVENT_HAVE_CHARSET_latin7 */ + +/* Define to enable charset macce */ +/* #undef _EVENT_HAVE_CHARSET_macce */ + +/* Define to enable charset macroman */ +/* #undef _EVENT_HAVE_CHARSET_macroman */ + +/* Define to enable charset sjis */ +#define _EVENT_HAVE_CHARSET_sjis 1 + +/* Define to enable charset swe7 */ +/* #undef _EVENT_HAVE_CHARSET_swe7 */ + +/* Define to enable charset tis620 */ +#define _EVENT_HAVE_CHARSET_tis620 1 + +/* Define to enable charset ucs2 */ +#define _EVENT_HAVE_CHARSET_ucs2 1 + +/* Define to enable charset ujis */ +#define _EVENT_HAVE_CHARSET_ujis 1 + +/* Define to enable ut8 */ +#define _EVENT_HAVE_CHARSET_utf8 1 + +/* Define to 1 if you have the `chsize' function. */ +/* #undef _EVENT_HAVE_CHSIZE */ + +/* Define to 1 if you have the `clock_gettime' function. */ +/* #undef _EVENT_HAVE_CLOCK_GETTIME */ + +/* Define to enable compression support */ +#define _EVENT_HAVE_COMPRESS 1 + +/* crypt */ +#define _EVENT_HAVE_CRYPT 1 + +/* Define to 1 if you have the <crypt.h> header file. */ +#define _EVENT_HAVE_CRYPT_H 1 + +/* Define to 1 if you have the <curses.h> header file. */ +#define _EVENT_HAVE_CURSES_H 1 + +/* Define to 1 if you have the `cuserid' function. */ +#define _EVENT_HAVE_CUSERID 1 + +/* Define to 1 if you have the <cxxabi.h> header file. */ +#define _EVENT_HAVE_CXXABI_H 1 + +/* Define to 1 if you have the declaration of `madvise', and to 0 if you + don't. */ +#define _EVENT_HAVE_DECL_MADVISE 1 + +/* Define to 1 if you have the declaration of `SHM_HUGETLB', and to 0 if you + don't. */ +#define _EVENT_HAVE_DECL_SHM_HUGETLB 1 + +/* Define to 1 if you have the declaration of `tgoto', and to 0 if you don't. + */ +#define _EVENT_HAVE_DECL_TGOTO 1 + +/* Whether we are using DEC threads */ +/* #undef _EVENT_HAVE_DEC_THREADS */ + +/* Define if /dev/poll is available */ +/* #undef _EVENT_HAVE_DEVPOLL */ + +/* Define to 1 if you have the <dirent.h> header file. */ +#define _EVENT_HAVE_DIRENT_H 1 + +/* Define to 1 if you have the `dlerror' function. */ +#define _EVENT_HAVE_DLERROR 1 + +/* Define to 1 if you have the <dlfcn.h> header file. */ +#define _EVENT_HAVE_DLFCN_H 1 + +/* Define to 1 if you have the `dlopen' function. */ +#define _EVENT_HAVE_DLOPEN 1 + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef _EVENT_HAVE_DOPRNT */ + +/* Access checks in embedded library */ +/* #undef _EVENT_HAVE_EMBEDDED_PRIVILEGE_CONTROL */ + +/* Define if your system supports the epoll system calls */ +#define _EVENT_HAVE_EPOLL 1 + +/* Define to 1 if you have the `epoll_ctl' function. */ +#define _EVENT_HAVE_EPOLL_CTL 1 + +/* Define if your system supports event ports */ +/* #undef _EVENT_HAVE_EVENT_PORTS */ + +/* Define to 1 if you have the <execinfo.h> header file. */ +#define _EVENT_HAVE_EXECINFO_H 1 + +/* Defined by configure. Use explicit template instantiation. */ +#define _EVENT_HAVE_EXPLICIT_TEMPLATE_INSTANTIATION 1 + +/* Define to 1 if you have the `fchmod' function. */ +#define _EVENT_HAVE_FCHMOD 1 + +/* Define to 1 if you have the `fcntl' function. */ +#define _EVENT_HAVE_FCNTL 1 + +/* Define to 1 if you have the <fcntl.h> header file. */ +#define _EVENT_HAVE_FCNTL_H 1 + +/* Define to 1 if you have the `fconvert' function. */ +/* #undef _EVENT_HAVE_FCONVERT */ + +/* Define to 1 if you have the `fdatasync' function. */ +#define _EVENT_HAVE_FDATASYNC 1 + +/* Define to 1 if you have the `fgetln' function. */ +/* #undef _EVENT_HAVE_FGETLN */ + +/* Define to 1 if you have the `finite' function. */ +#define _EVENT_HAVE_FINITE 1 + +/* Define to 1 if you have the <floatingpoint.h> header file. */ +/* #undef _EVENT_HAVE_FLOATINGPOINT_H */ + +/* Define to 1 if you have the <float.h> header file. */ +#define _EVENT_HAVE_FLOAT_H 1 + +/* Define to 1 if you have the `flockfile' function. */ +#define _EVENT_HAVE_FLOCKFILE 1 + +/* Define to 1 if you have the <fnmatch.h> header file. */ +#define _EVENT_HAVE_FNMATCH_H 1 + +/* Define to 1 if you have the `fpresetsticky' function. */ +/* #undef _EVENT_HAVE_FPRESETSTICKY */ + +/* Define to 1 if you have the `fpsetmask' function. */ +/* #undef _EVENT_HAVE_FPSETMASK */ + +/* Define to 1 if the system has the type `fp_except'. */ +/* #undef _EVENT_HAVE_FP_EXCEPT */ + +/* Define to 1 if you have the `fsync' function. */ +#define _EVENT_HAVE_FSYNC 1 + +/* Define to 1 if you have the `ftruncate' function. */ +#define _EVENT_HAVE_FTRUNCATE 1 + +/* Define to 1 if compiler provides atomic builtins. */ +/* #undef _EVENT_HAVE_GCC_ATOMIC_BUILTINS */ + +/* Define to 1 if you have the `getaddrinfo' function. */ +#define _EVENT_HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getcwd' function. */ +#define _EVENT_HAVE_GETCWD 1 + +/* Define to 1 if you have the `gethostbyaddr_r' function. */ +#define _EVENT_HAVE_GETHOSTBYADDR_R 1 + +/* Define to 1 if you have the `gethostbyname_r' function. */ +#define _EVENT_HAVE_GETHOSTBYNAME_R 1 + +/* Solaris define gethostbyname_r with 5 arguments. glibc2 defines this with 6 + arguments */ +#define _EVENT_HAVE_GETHOSTBYNAME_R_GLIBC2_STYLE 1 + +/* In OSF 4.0f the 3'd argument to gethostbyname_r is hostent_data * */ +/* #undef _EVENT_HAVE_GETHOSTBYNAME_R_RETURN_INT */ + +/* Define to 1 if you have the `gethrtime' function. */ +/* #undef _EVENT_HAVE_GETHRTIME */ + +/* Define to 1 if you have the `getline' function. */ +#define _EVENT_HAVE_GETLINE 1 + +/* Define to 1 if you have the `getnameinfo' function. */ +#define _EVENT_HAVE_GETNAMEINFO 1 + +/* Define to 1 if you have the `getpagesize' function. */ +#define _EVENT_HAVE_GETPAGESIZE 1 + +/* Define to 1 if you have the `getpass' function. */ +#define _EVENT_HAVE_GETPASS 1 + +/* Define to 1 if you have the `getpassphrase' function. */ +/* #undef _EVENT_HAVE_GETPASSPHRASE */ + +/* Define to 1 if you have the `getpwnam' function. */ +#define _EVENT_HAVE_GETPWNAM 1 + +/* Define to 1 if you have the `getpwuid' function. */ +#define _EVENT_HAVE_GETPWUID 1 + +/* getpwent() declaration present */ +/* #undef _EVENT_HAVE_GETPW_DECLS */ + +/* Define to 1 if you have the `getrlimit' function. */ +#define _EVENT_HAVE_GETRLIMIT 1 + +/* Define to 1 if you have the `getrusage' function. */ +#define _EVENT_HAVE_GETRUSAGE 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define _EVENT_HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `getwd' function. */ +#define _EVENT_HAVE_GETWD 1 + +/* Define to 1 if you have the `gmtime_r' function. */ +#define _EVENT_HAVE_GMTIME_R 1 + +/* Define to 1 if you have the <grp.h> header file. */ +#define _EVENT_HAVE_GRP_H 1 + +/* HIST_ENTRY is defined in the outer libeditreadline */ +/* #undef _EVENT_HAVE_HIST_ENTRY */ + +/* Define to 1 if you have the <ieeefp.h> header file. */ +/* #undef _EVENT_HAVE_IEEEFP_H */ + +/* Define to 1 if you have the `index' function. */ +#define _EVENT_HAVE_INDEX 1 + +/* Define to 1 if you have the `inet_ntop' function. */ +#define _EVENT_HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `initgroups' function. */ +#define _EVENT_HAVE_INITGROUPS 1 + +/* Define to 1 if the system has the type `int16'. */ +/* #undef _EVENT_HAVE_INT16 */ + +/* Define to 1 if the system has the type `int32'. */ +/* #undef _EVENT_HAVE_INT32 */ + +/* Define to 1 if the system has the type `int64'. */ +/* #undef _EVENT_HAVE_INT64 */ + +/* Define to 1 if the system has the type `int8'. */ +/* #undef _EVENT_HAVE_INT8 */ + +/* Define to 1 if you have the <inttypes.h> header file. */ +#define _EVENT_HAVE_INTTYPES_H 1 + +/* Define to 1 if the system has the type `in_addr_t'. */ +#define _EVENT_HAVE_IN_ADDR_T 1 + +/* isinf() macro or function */ +#define _EVENT_HAVE_ISINF 1 + +/* Define to 1 if you have the `isnan' function. */ +#define _EVENT_HAVE_ISNAN 1 + +/* Define to 1 if you have the `issetugid' function. */ +/* #undef _EVENT_HAVE_ISSETUGID */ + +/* Define to 1 if you have the `kqueue' function. */ +/* #undef _EVENT_HAVE_KQUEUE */ + +/* Define if mysql_cv_langinfo_codeset=yes */ +#define _EVENT_HAVE_LANGINFO_CODESET + +/* Define to 1 if you have the <langinfo.h> header file. */ +#define _EVENT_HAVE_LANGINFO_H 1 + +/* Define if you have large pages support */ +#define _EVENT_HAVE_LARGE_PAGES 1 + +/* Define to 1 if you have the `bind' library (-lbind). */ +/* #undef _EVENT_HAVE_LIBBIND */ + +/* Define to 1 if you have the `compat' library (-lcompat). */ +/* #undef _EVENT_HAVE_LIBCOMPAT */ + +/* Define to 1 if you have the `c_r' library (-lc_r). */ +/* #undef _EVENT_HAVE_LIBC_R */ + +/* Define to 1 if you have the `dl' library (-ldl). */ +#define _EVENT_HAVE_LIBDL 1 + +/* If we want to use libevent and have connection pooling */ +#define _EVENT_HAVE_LIBEVENT 1 + +/* Define to 1 if you have the `gen' library (-lgen). */ +/* #undef _EVENT_HAVE_LIBGEN */ + +/* Define to 1 if you have the `m' library (-lm). */ +#define _EVENT_HAVE_LIBM 1 + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +#define _EVENT_HAVE_LIBNSL 1 + +/* Define to 1 if you have the `nsl_r' library (-lnsl_r). */ +/* #undef _EVENT_HAVE_LIBNSL_R */ + +/* Define to 1 if you have the `posix4' library (-lposix4). */ +/* #undef _EVENT_HAVE_LIBPOSIX4 */ + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#define _EVENT_HAVE_LIBPTHREAD 1 + +/* Define to 1 if you have the `resolv' library (-lresolv). */ +#define _EVENT_HAVE_LIBRESOLV 1 + +/* Define to 1 if you have the `rt' library (-lrt). */ +#define _EVENT_HAVE_LIBRT 1 + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef _EVENT_HAVE_LIBSOCKET */ + +/* Define if have -lwrap */ +/* #undef _EVENT_HAVE_LIBWRAP */ + +/* Define to 1 if you have the <limits.h> header file. */ +#define _EVENT_HAVE_LIMITS_H 1 + +/* Whether we are using Xavier Leroy's LinuxThreads */ +/* #undef _EVENT_HAVE_LINUXTHREADS */ + +/* Define to 1 if you have the <linux/config.h> header file. */ +/* #undef _EVENT_HAVE_LINUX_CONFIG_H */ + +/* Define to 1 if you have the <locale.h> header file. */ +#define _EVENT_HAVE_LOCALE_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define _EVENT_HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the `locking' function. */ +/* #undef _EVENT_HAVE_LOCKING */ + +/* Define to 1 if you have the `longjmp' function. */ +#define _EVENT_HAVE_LONGJMP 1 + +/* Define to 1 if you have the `lrand48' function. */ +#define _EVENT_HAVE_LRAND48 1 + +/* Define to 1 if you have the `lstat' function. */ +#define _EVENT_HAVE_LSTAT 1 + +/* Define to 1 if you have the `madvise' function. */ +#define _EVENT_HAVE_MADVISE 1 + +/* Define to 1 if you have the `mallinfo' function. */ +#define _EVENT_HAVE_MALLINFO 1 + +/* Define to 1 if you have the <malloc.h> header file. */ +#define _EVENT_HAVE_MALLOC_H 1 + +/* Define if you have mbrlen */ +#define _EVENT_HAVE_MBRLEN + +/* Define if you have mbrtowc */ +#define _EVENT_HAVE_MBRTOWC + +/* Define if you have mbsrtowcs */ +#define _EVENT_HAVE_MBSRTOWCS + +/* Define if mysql_cv_have_mbstate_t=yes */ +#define _EVENT_HAVE_MBSTATE_T + +/* Define to 1 if you have the `memcpy' function. */ +#define _EVENT_HAVE_MEMCPY 1 + +/* Define to 1 if you have the `memmove' function. */ +#define _EVENT_HAVE_MEMMOVE 1 + +/* Define to 1 if you have the <memory.h> header file. */ +#define _EVENT_HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define _EVENT_HAVE_MKSTEMP 1 + +/* Define to 1 if you have the `mlockall' function. */ +#define _EVENT_HAVE_MLOCKALL 1 + +/* Define to 1 if you have the `mmap' function. */ +#define _EVENT_HAVE_MMAP 1 + +/* Define to 1 if you have the `mmap64' function. */ +#define _EVENT_HAVE_MMAP64 1 + +/* Define to 1 if you have the <ndir.h> header file. */ +/* #undef _EVENT_HAVE_NDIR_H */ + +/* Define to 1 if you have the <netinet/in6.h> header file. */ +/* #undef _EVENT_HAVE_NETINET_IN6_H */ + +/* Define to 1 if you have the <netinet/in.h> header file. */ +#define _EVENT_HAVE_NETINET_IN_H 1 + +/* For some non posix threads */ +/* #undef _EVENT_HAVE_NONPOSIX_PTHREAD_GETSPECIFIC */ + +/* For some non posix threads */ +/* #undef _EVENT_HAVE_NONPOSIX_PTHREAD_MUTEX_INIT */ + +/* sigwait with one argument */ +/* #undef _EVENT_HAVE_NONPOSIX_SIGWAIT */ + +/* NPTL threads implementation */ +#define _EVENT_HAVE_NPTL 1 + +/* Define to 1 if the system has the type `off_t'. */ +#define _EVENT_HAVE_OFF_T 1 + +/* OpenSSL */ +#define _EVENT_HAVE_OPENSSL 1 + +/* Define to 1 if you have the <paths.h> header file. */ +#define _EVENT_HAVE_PATHS_H 1 + +/* Define to 1 if you have the `perror' function. */ +#define _EVENT_HAVE_PERROR 1 + +/* Define to 1 if you have the `poll' function. */ +#define _EVENT_HAVE_POLL 1 + +/* Define to 1 if you have the <poll.h> header file. */ +#define _EVENT_HAVE_POLL_H 1 + +/* Define to 1 if you have the `port_create' function. */ +/* #undef _EVENT_HAVE_PORT_CREATE */ + +/* Define to 1 if you have the <port.h> header file. */ +/* #undef _EVENT_HAVE_PORT_H */ + +/* Define to 1 if you have the `posix_fallocate' function. */ +#define _EVENT_HAVE_POSIX_FALLOCATE 1 + +/* Signal handling is POSIX (sigset/sighold, etc) */ +#define _EVENT_HAVE_POSIX_SIGNALS 1 + +/* Define to 1 if you have the `pread' function. */ +#define _EVENT_HAVE_PREAD 1 + +/* Define to 1 if you have the `pthread_attr_create' function. */ +/* #undef _EVENT_HAVE_PTHREAD_ATTR_CREATE */ + +/* Define to 1 if you have the `pthread_attr_getstacksize' function. */ +#define _EVENT_HAVE_PTHREAD_ATTR_GETSTACKSIZE 1 + +/* Define to 1 if you have the `pthread_attr_setprio' function. */ +/* #undef _EVENT_HAVE_PTHREAD_ATTR_SETPRIO */ + +/* Define to 1 if you have the `pthread_attr_setschedparam' function. */ +#define _EVENT_HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1 + +/* pthread_attr_setscope */ +#define _EVENT_HAVE_PTHREAD_ATTR_SETSCOPE 1 + +/* Define to 1 if you have the `pthread_attr_setstacksize' function. */ +#define _EVENT_HAVE_PTHREAD_ATTR_SETSTACKSIZE 1 + +/* Define to 1 if you have the `pthread_condattr_create' function. */ +/* #undef _EVENT_HAVE_PTHREAD_CONDATTR_CREATE */ + +/* Define to 1 if you have the `pthread_getsequence_np' function. */ +/* #undef _EVENT_HAVE_PTHREAD_GETSEQUENCE_NP */ + +/* Define to 1 if you have the `pthread_init' function. */ +/* #undef _EVENT_HAVE_PTHREAD_INIT */ + +/* Define to 1 if you have the `pthread_key_delete' function. */ +#define _EVENT_HAVE_PTHREAD_KEY_DELETE 1 + +/* Define to 1 if you have the `pthread_rwlock_rdlock' function. */ +#define _EVENT_HAVE_PTHREAD_RWLOCK_RDLOCK 1 + +/* Define to 1 if you have the `pthread_setprio' function. */ +/* #undef _EVENT_HAVE_PTHREAD_SETPRIO */ + +/* Define to 1 if you have the `pthread_setprio_np' function. */ +/* #undef _EVENT_HAVE_PTHREAD_SETPRIO_NP */ + +/* Define to 1 if you have the `pthread_setschedparam' function. */ +#define _EVENT_HAVE_PTHREAD_SETSCHEDPARAM 1 + +/* Define to 1 if you have the `pthread_setschedprio' function. */ +#define _EVENT_HAVE_PTHREAD_SETSCHEDPRIO 1 + +/* Define to 1 if you have the `pthread_sigmask' function. */ +#define _EVENT_HAVE_PTHREAD_SIGMASK 1 + +/* pthread_yield function with one argument */ +/* #undef _EVENT_HAVE_PTHREAD_YIELD_ONE_ARG */ + +/* pthread_yield that doesn't take any arguments */ +#define _EVENT_HAVE_PTHREAD_YIELD_ZERO_ARG 1 + +/* Define to 1 if you have the `putenv' function. */ +#define _EVENT_HAVE_PUTENV 1 + +/* Define to 1 if you have the <pwd.h> header file. */ +#define _EVENT_HAVE_PWD_H 1 + +/* If we want to have query cache */ +#define _EVENT_HAVE_QUERY_CACHE 1 + +/* POSIX readdir_r */ +#define _EVENT_HAVE_READDIR_R 1 + +/* Define to 1 if you have the `readlink' function. */ +#define _EVENT_HAVE_READLINK 1 + +/* Define to 1 if you have the `realpath' function. */ +#define _EVENT_HAVE_REALPATH 1 + +/* Define to 1 if you have the `regcomp' function. */ +#define _EVENT_HAVE_REGCOMP 1 + +/* Define to 1 if you have the `rename' function. */ +#define _EVENT_HAVE_RENAME 1 + +/* Define to 1 if system calls automatically restart after interruption by a + signal. */ +#define _EVENT_HAVE_RESTARTABLE_SYSCALLS 1 + +/* Define to 1 if you have the `re_comp' function. */ +#define _EVENT_HAVE_RE_COMP 1 + +/* Define to 1 if you have the `rint' function. */ +#define _EVENT_HAVE_RINT 1 + +/* RTree keys */ +#define _EVENT_HAVE_RTREE_KEYS 1 + +/* Define to 1 if you have the `rwlock_init' function. */ +/* #undef _EVENT_HAVE_RWLOCK_INIT */ + +/* Define to 1 if you have the <sched.h> header file. */ +#define _EVENT_HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_yield' function. */ +#define _EVENT_HAVE_SCHED_YIELD 1 + +/* Define to 1 if you have the `select' function. */ +#define _EVENT_HAVE_SELECT 1 + +/* Define to 1 if you have the <select.h> header file. */ +/* #undef _EVENT_HAVE_SELECT_H */ + +/* Define to 1 if you have the <semaphore.h> header file. */ +#define _EVENT_HAVE_SEMAPHORE_H 1 + +/* Define to 1 if you have the `setenv' function. */ +#define _EVENT_HAVE_SETENV 1 + +/* Define if F_SETFD is defined in <fcntl.h> */ +#define _EVENT_HAVE_SETFD 1 + +/* Define to 1 if you have the `setlocale' function. */ +#define _EVENT_HAVE_SETLOCALE 1 + +/* Define to 1 if you have the `setupterm' function. */ +/* #undef _EVENT_HAVE_SETUPTERM */ + +/* Define to 1 if you have the `shmat' function. */ +#define _EVENT_HAVE_SHMAT 1 + +/* Define to 1 if you have the `shmctl' function. */ +#define _EVENT_HAVE_SHMCTL 1 + +/* Define to 1 if you have the `shmdt' function. */ +#define _EVENT_HAVE_SHMDT 1 + +/* Define to 1 if you have the `shmget' function. */ +#define _EVENT_HAVE_SHMGET 1 + +/* Define to 1 if you have the `sigaction' function. */ +#define _EVENT_HAVE_SIGACTION 1 + +/* Define to 1 if you have the `sigaddset' function. */ +#define _EVENT_HAVE_SIGADDSET 1 + +/* Define to 1 if you have the `sigemptyset' function. */ +#define _EVENT_HAVE_SIGEMPTYSET 1 + +/* Define to 1 if you have the `sighold' function. */ +#define _EVENT_HAVE_SIGHOLD 1 + +/* Define to 1 if you have the `signal' function. */ +#define _EVENT_HAVE_SIGNAL 1 + +/* Define to 1 if you have the <signal.h> header file. */ +#define _EVENT_HAVE_SIGNAL_H 1 + +/* Define to 1 if you have the `sigset' function. */ +#define _EVENT_HAVE_SIGSET 1 + +/* Define to 1 if the system has the type `sigset_t'. */ +#define _EVENT_HAVE_SIGSET_T 1 + +/* Define to 1 if you have the `sigthreadmask' function. */ +/* #undef _EVENT_HAVE_SIGTHREADMASK */ + +/* POSIX sigwait */ +#define _EVENT_HAVE_SIGWAIT 1 + +/* Define to 1 if the system has the type `size_t'. */ +#define _EVENT_HAVE_SIZE_T 1 + +/* Define to 1 if you have the `sleep' function. */ +#define _EVENT_HAVE_SLEEP 1 + +/* Define to 1 if you have the `snprintf' function. */ +#define _EVENT_HAVE_SNPRINTF 1 + +/* Define to 1 if you have the `socket' function. */ +#define _EVENT_HAVE_SOCKET 1 + +/* Solaris define gethostbyaddr_r with 7 arguments. glibc2 defines this with 8 + arguments */ +/* #undef _EVENT_HAVE_SOLARIS_STYLE_GETHOST */ + +/* Spatial extentions */ +#define _EVENT_HAVE_SPATIAL 1 + +/* Define to 1 if you have the <stdarg.h> header file. */ +#define _EVENT_HAVE_STDARG_H 1 + +/* Define to 1 if you have the <stddef.h> header file. */ +#define _EVENT_HAVE_STDDEF_H 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +#define _EVENT_HAVE_STDINT_H 1 + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define _EVENT_HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `stpcpy' function. */ +#define _EVENT_HAVE_STPCPY 1 + +/* Define to 1 if you have the `strcasecmp' function. */ +#define _EVENT_HAVE_STRCASECMP 1 + +/* Define to 1 if you have the `strcoll' function. */ +#define _EVENT_HAVE_STRCOLL 1 + +/* Define to 1 if you have the `strdup' function. */ +#define _EVENT_HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define _EVENT_HAVE_STRERROR 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define _EVENT_HAVE_STRINGS_H 1 + +/* Define to 1 if you have the <string.h> header file. */ +#define _EVENT_HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef _EVENT_HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef _EVENT_HAVE_STRLCPY */ + +/* Define to 1 if you have the `strnlen' function. */ +#define _EVENT_HAVE_STRNLEN 1 + +/* Define to 1 if you have the `strpbrk' function. */ +#define _EVENT_HAVE_STRPBRK 1 + +/* Define to 1 if you have the `strsep' function. */ +#define _EVENT_HAVE_STRSEP 1 + +/* Define to 1 if you have the `strsignal' function. */ +#define _EVENT_HAVE_STRSIGNAL 1 + +/* Define to 1 if you have the `strstr' function. */ +#define _EVENT_HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtok_r' function. */ +#define _EVENT_HAVE_STRTOK_R 1 + +/* Define to 1 if you have the `strtol' function. */ +#define _EVENT_HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define _EVENT_HAVE_STRTOLL 1 + +/* Define to 1 if you have the `strtoul' function. */ +#define _EVENT_HAVE_STRTOUL 1 + +/* Define to 1 if you have the `strtoull' function. */ +#define _EVENT_HAVE_STRTOULL 1 + +/* Define to 1 if the system has the type `struct in6_addr'. */ +#define _EVENT_HAVE_STRUCT_IN6_ADDR 1 + +/* Define to 1 if `st_rdev' is member of `struct stat'. */ +#define _EVENT_HAVE_STRUCT_STAT_ST_RDEV 1 + +/* Define to 1 if your `struct stat' has `st_rdev'. Deprecated, use + `HAVE_STRUCT_STAT_ST_RDEV' instead. */ +#define _EVENT_HAVE_ST_RDEV 1 + +/* Define to 1 if you have the <synch.h> header file. */ +/* #undef _EVENT_HAVE_SYNCH_H */ + +/* Define to 1 if you have the <sys/cdefs.h> header file. */ +#define _EVENT_HAVE_SYS_CDEFS_H 1 + +/* Define to 1 if you have the <sys/devpoll.h> header file. */ +/* #undef _EVENT_HAVE_SYS_DEVPOLL_H */ + +/* Define to 1 if you have the <sys/dir.h> header file. */ +#define _EVENT_HAVE_SYS_DIR_H 1 + +/* Define to 1 if you have the <sys/epoll.h> header file. */ +#define _EVENT_HAVE_SYS_EPOLL_H 1 + +/* Define to 1 if you have the <sys/event.h> header file. */ +/* #undef _EVENT_HAVE_SYS_EVENT_H */ + +/* Define to 1 if you have the <sys/file.h> header file. */ +#define _EVENT_HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the <sys/ioctl.h> header file. */ +#define _EVENT_HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the <sys/ipc.h> header file. */ +#define _EVENT_HAVE_SYS_IPC_H 1 + +/* Define to 1 if you have the <sys/malloc.h> header file. */ +/* #undef _EVENT_HAVE_SYS_MALLOC_H */ + +/* Define to 1 if you have the <sys/mman.h> header file. */ +#define _EVENT_HAVE_SYS_MMAN_H 1 + +/* Define to 1 if you have the <sys/ndir.h> header file. */ +/* #undef _EVENT_HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the <sys/param.h> header file. */ +#define _EVENT_HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the <sys/prctl.h> header file. */ +#define _EVENT_HAVE_SYS_PRCTL_H 1 + +/* Define to 1 if you have the <sys/ptem.h> header file. */ +/* #undef _EVENT_HAVE_SYS_PTEM_H */ + +/* Define to 1 if you have the <sys/pte.h> header file. */ +/* #undef _EVENT_HAVE_SYS_PTE_H */ + +/* Define to 1 if you have the <sys/queue.h> header file. */ +#define _EVENT_HAVE_SYS_QUEUE_H 1 + +/* Define to 1 if you have the <sys/resource.h> header file. */ +#define _EVENT_HAVE_SYS_RESOURCE_H 1 + +/* Define to 1 if you have the <sys/select.h> header file. */ +#define _EVENT_HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the <sys/shm.h> header file. */ +#define _EVENT_HAVE_SYS_SHM_H 1 + +/* Define to 1 if you have the <sys/socket.h> header file. */ +#define _EVENT_HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#define _EVENT_HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/stream.h> header file. */ +/* #undef _EVENT_HAVE_SYS_STREAM_H */ + +/* Define to 1 if you have the <sys/timeb.h> header file. */ +#define _EVENT_HAVE_SYS_TIMEB_H 1 + +/* Define to 1 if you have the <sys/time.h> header file. */ +#define _EVENT_HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the <sys/types.h> header file. */ +#define _EVENT_HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the <sys/un.h> header file. */ +#define _EVENT_HAVE_SYS_UN_H 1 + +/* Define to 1 if you have the <sys/utime.h> header file. */ +/* #undef _EVENT_HAVE_SYS_UTIME_H */ + +/* Define to 1 if you have the <sys/vadvise.h> header file. */ +/* #undef _EVENT_HAVE_SYS_VADVISE_H */ + +/* Define to 1 if you have the <sys/wait.h> header file. */ +#define _EVENT_HAVE_SYS_WAIT_H 1 + +/* Define if TAILQ_FOREACH is defined in <sys/queue.h> */ +#define _EVENT_HAVE_TAILQFOREACH 1 + +/* Define to 1 if you have the `tcgetattr' function. */ +#define _EVENT_HAVE_TCGETATTR 1 + +/* Define to 1 if you have the `tell' function. */ +/* #undef _EVENT_HAVE_TELL */ + +/* Define to 1 if you have the `tempnam' function. */ +#define _EVENT_HAVE_TEMPNAM 1 + +/* Define to 1 if you have the <termbits.h> header file. */ +/* #undef _EVENT_HAVE_TERMBITS_H */ + +/* Define to 1 if you have the <termcap.h> header file. */ +#define _EVENT_HAVE_TERMCAP_H 1 + +/* Define to 1 if you have the <termios.h> header file. */ +#define _EVENT_HAVE_TERMIOS_H 1 + +/* Define to 1 if you have the <termio.h> header file. */ +#define _EVENT_HAVE_TERMIO_H 1 + +/* Define to 1 if you have the <term.h> header file. */ +#define _EVENT_HAVE_TERM_H 1 + +/* Define to 1 if you have the `thr_setconcurrency' function. */ +/* #undef _EVENT_HAVE_THR_SETCONCURRENCY */ + +/* Define to 1 if you have the `thr_yield' function. */ +/* #undef _EVENT_HAVE_THR_YIELD */ + +/* Define if timeradd is defined in <sys/time.h> */ +#define _EVENT_HAVE_TIMERADD 1 + +/* Define if timerclear is defined in <sys/time.h> */ +#define _EVENT_HAVE_TIMERCLEAR 1 + +/* Define if timercmp is defined in <sys/time.h> */ +#define _EVENT_HAVE_TIMERCMP 1 + +/* Define if timerisset is defined in <sys/time.h> */ +#define _EVENT_HAVE_TIMERISSET 1 + +/* Timespec has a ts_sec instead of tv_sev */ +/* #undef _EVENT_HAVE_TIMESPEC_TS_SEC */ + +/* Have the tzname variable */ +#define _EVENT_HAVE_TZNAME 1 + +/* national Unicode collations */ +#define _EVENT_HAVE_UCA_COLLATIONS 1 + +/* Define to 1 if the system has the type `uchar'. */ +/* #undef _EVENT_HAVE_UCHAR */ + +/* Define to 1 if the system has the type `uint'. */ +#define _EVENT_HAVE_UINT 1 + +/* Define to 1 if the system has the type `uint16'. */ +/* #undef _EVENT_HAVE_UINT16 */ + +/* Define to 1 if the system has the type `uint16_t'. */ +#define _EVENT_HAVE_UINT16_T 1 + +/* Define to 1 if the system has the type `uint32'. */ +/* #undef _EVENT_HAVE_UINT32 */ + +/* Define to 1 if the system has the type `uint32_t'. */ +#define _EVENT_HAVE_UINT32_T 1 + +/* Define to 1 if the system has the type `uint64'. */ +/* #undef _EVENT_HAVE_UINT64 */ + +/* Define to 1 if the system has the type `uint64_t'. */ +#define _EVENT_HAVE_UINT64_T 1 + +/* Define to 1 if the system has the type `uint8'. */ +/* #undef _EVENT_HAVE_UINT8 */ + +/* Define to 1 if the system has the type `uint8_t'. */ +#define _EVENT_HAVE_UINT8_T 1 + +/* Define to 1 if the system has the type `ulong'. */ +#define _EVENT_HAVE_ULONG 1 + +/* Define to 1 if you have the <unistd.h> header file. */ +#define _EVENT_HAVE_UNISTD_H 1 + +/* Have UnixWare 7 (or similar) almost-POSIX threading library */ +/* #undef _EVENT_HAVE_UNIXWARE7_THREADS */ + +/* sighold() is present and usable */ +/* #undef _EVENT_HAVE_USG_SIGHOLD */ + +/* certain Japanese customer */ +/* #undef _EVENT_HAVE_UTF8_GENERAL_CS */ + +/* Define to 1 if you have the <utime.h> header file. */ +#define _EVENT_HAVE_UTIME_H 1 + +/* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */ +#define _EVENT_HAVE_UTIME_NULL 1 + +/* Define to 1 if the system has the type `u_int32_t'. */ +#define _EVENT_HAVE_U_INT32_T 1 + +/* Define to 1 if you have the <varargs.h> header file. */ +/* #undef _EVENT_HAVE_VARARGS_H */ + +/* Define to 1 if you have the `vasprintf' function. */ +#define _EVENT_HAVE_VASPRINTF 1 + +/* Define to 1 if you have the `vidattr' function. */ +/* #undef _EVENT_HAVE_VIDATTR */ + +/* Define to enable buffered read. This works only if syscalls read/recv + return as soon as there is some data in the kernel buffer, no matter how + big the given buffer is. */ +#define _EVENT_HAVE_VIO_READ_BUFF 1 + +/* Found vis.h and the strvis() function */ +/* #undef _EVENT_HAVE_VIS_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define _EVENT_HAVE_VPRINTF 1 + +/* Define to 1 if you have the <wchar.h> header file. */ +#define _EVENT_HAVE_WCHAR_H 1 + +/* Define if you check wcsdup */ +#define _EVENT_HAVE_WCSDUP + +/* Define if you have wctomb */ +#define _EVENT_HAVE_WCTOMB + +/* Define to 1 if you have the <wctype.h> header file. */ +#define _EVENT_HAVE_WCTYPE_H 1 + +/* Define if you have wcwidth */ +#define _EVENT_HAVE_WCWIDTH + +/* Define to 1 if compiler supports weak symbol attribute. */ +#define _EVENT_HAVE_WEAK_SYMBOL 1 + +/* Define if kqueue works correctly with pipes */ +/* #undef _EVENT_HAVE_WORKING_KQUEUE */ + +/* Define to 1 if you have the <xfs/xfs.h> header file. */ +/* #undef _EVENT_HAVE_XFS_XFS_H */ + +/* Defined by configure. Using yaSSL for SSL. */ +#define _EVENT_HAVE_YASSL 1 + +/* Define if /proc/meminfo shows the huge page size (Linux only) */ +#define _EVENT_HUGETLB_USE_PROC_MEMINFO 1 + +/* Define if you have -lwrap */ +/* #undef _EVENT_LIBWRAP */ + +/* Machine type name, eg sparc */ +#define _EVENT_MACHINE_TYPE "x86_64" + +/* Maximum number of indexes per table */ +#define _EVENT_MAX_INDEXES 64 + +/* Define the default charset name */ +#define _EVENT_MYSQL_DEFAULT_CHARSET_NAME "latin1" + +/* Define the default charset name */ +#define _EVENT_MYSQL_DEFAULT_COLLATION_NAME "latin1_swedish_ci" + +/* Assume single-CPU mode, no concurrency */ +/* #undef _EVENT_MY_ATOMIC_MODE_DUMMY */ + +/* Use pthread rwlocks for atomic ops */ +/* #undef _EVENT_MY_ATOMIC_MODE_RWLOCKS */ + +/* Define to 1 if you want to use fast mutexes */ +/* #undef _EVENT_MY_PTHREAD_FASTMUTEX */ + +/* Including Ndb Cluster DB sci transporter */ +/* #undef _EVENT_NDB_SCI_TRANSPORTER */ + +/* Including Ndb Cluster DB shared memory transporter */ +/* #undef _EVENT_NDB_SHM_TRANSPORTER */ + +/* NDB build version */ +/* #undef _EVENT_NDB_VERSION_BUILD */ + +/* NDB major version */ +/* #undef _EVENT_NDB_VERSION_MAJOR */ + +/* NDB minor version */ +/* #undef _EVENT_NDB_VERSION_MINOR */ + +/* NDB status version */ +/* #undef _EVENT_NDB_VERSION_STATUS */ + +/* Name of package */ +#define _EVENT_PACKAGE "mysql" + +/* Define to the address where bug reports for this package should be sent. */ +#define _EVENT_PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define _EVENT_PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define _EVENT_PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define _EVENT_PACKAGE_TARNAME "" + +/* Define to the version of this package. */ +#define _EVENT_PACKAGE_VERSION "" + +/* mysql client protocol version */ +#define _EVENT_PROTOCOL_VERSION 10 + +/* qsort returns void */ +#define _EVENT_QSORT_TYPE_IS_VOID 1 + +/* The return type of qsort (int or void). */ +#define _EVENT_RETQSORTTYPE void + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define _EVENT_RETSIGTYPE void + +/* The size of `char', as computed by sizeof. */ +#define _EVENT_SIZEOF_CHAR 1 + +/* The size of `char*', as computed by sizeof. */ +#define _EVENT_SIZEOF_CHARP 8 + +/* The size of `int', as computed by sizeof. */ +#define _EVENT_SIZEOF_INT 4 + +/* The size of `long', as computed by sizeof. */ +#define _EVENT_SIZEOF_LONG 8 + +/* The size of `long long', as computed by sizeof. */ +#define _EVENT_SIZEOF_LONG_LONG 8 + +/* The size of `off_t', as computed by sizeof. */ +#define _EVENT_SIZEOF_OFF_T 8 + +/* The size of `short', as computed by sizeof. */ +#define _EVENT_SIZEOF_SHORT 2 + +/* The size of `void*', as computed by sizeof. */ +#define _EVENT_SIZEOF_VOIDP 8 + +/* The base type of the last arg to accept */ +#define _EVENT_SOCKET_SIZE_TYPE socklen_t + +/* Last argument to get/setsockopt */ +/* #undef _EVENT_SOCKOPT_OPTLEN_TYPE */ + +/* Broken sprintf */ +/* #undef _EVENT_SPRINTF_RETURNS_GARBAGE */ + +/* POSIX sprintf */ +#define _EVENT_SPRINTF_RETURNS_INT 1 + +/* Broken sprintf */ +/* #undef _EVENT_SPRINTF_RETURNS_PTR */ + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +#define _EVENT_STACK_DIRECTION -1 + +/* Define to 1 if the `S_IS*' macros in <sys/stat.h> do not work properly. */ +/* #undef _EVENT_STAT_MACROS_BROKEN */ + +/* Define to 1 if you have the ANSI C header files. */ +#define _EVENT_STDC_HEADERS 1 + +/* d_ino member present in struct dirent */ +#define _EVENT_STRUCT_DIRENT_HAS_D_INO 1 + +/* d_namlen member present in struct dirent */ +/* #undef _EVENT_STRUCT_DIRENT_HAS_D_NAMLEN */ + +/* The struct rlimit type to use with setrlimit */ +#define _EVENT_STRUCT_RLIMIT struct rlimit + +/* Name of system, eg sun-solaris */ +#define _EVENT_SYSTEM_TYPE "suse-linux-gnu" + +/* Whether we build for Linux */ +#define _EVENT_TARGET_OS_LINUX 1 + +/* Define if you want to have threaded code. This may be undef on client code + */ +#define _EVENT_THREAD 1 + +/* Should the client be thread safe */ +#define _EVENT_THREAD_SAFE_CLIENT 1 + +/* Define to 1 if time_t is unsigned */ +/* #undef _EVENT_TIME_T_UNSIGNED */ + +/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ +#define _EVENT_TIME_WITH_SYS_TIME 1 + +/* declaration of TIOCSTAT in sys/ioctl.h */ +/* #undef _EVENT_TIOCSTAT_IN_SYS_IOCTL */ + +/* Define to 1 if your <sys/time.h> declares `struct tm'. */ +/* #undef _EVENT_TM_IN_SYS_TIME */ + +/* used libedit interface (can we dereference result of + rl_completion_entry_function) */ +/* #undef _EVENT_USE_LIBEDIT_INTERFACE */ + +/* Maria is used for internal temporary tables */ +#define _EVENT_USE_MARIA_FOR_TMP_TABLES 1 + +/* Use multi-byte character routines */ +#define _EVENT_USE_MB 1 + +/* */ +#define _EVENT_USE_MB_IDENT 1 + +/* Needs to use mysys_new helpers */ +#define _EVENT_USE_MYSYS_NEW 1 + +/* used new readline interface (are rl_completion_func_t and + rl_compentry_func_t defined) */ +#define _EVENT_USE_NEW_READLINE_INTERFACE 1 + +/* the pstack backtrace library */ +/* #undef _EVENT_USE_PSTACK */ + +/* Version number of package */ +#define _EVENT_VERSION "5.1.32-maria-beta" + +/* sighandler type is void (*signal ()) (); */ +#define _EVENT_VOID_SIGHANDLER 1 + +/* Include Archive Storage Engine into mysqld */ +#define _EVENT_WITH_ARCHIVE_STORAGE_ENGINE 1 + +/* Include Basic Write-only Read-never tables into mysqld */ +#define _EVENT_WITH_BLACKHOLE_STORAGE_ENGINE 1 + +/* Include Stores tables in text CSV format into mysqld */ +#define _EVENT_WITH_CSV_STORAGE_ENGINE 1 + +/* Include Example for Storage Engines for developers into mysqld */ +/* #undef _EVENT_WITH_EXAMPLE_STORAGE_ENGINE */ + +/* Include Connects to tables on remote MySQL servers into mysqld */ +#define _EVENT_WITH_FEDERATED_STORAGE_ENGINE 1 + +/* Include Volatile memory based tables into mysqld */ +#define _EVENT_WITH_HEAP_STORAGE_ENGINE 1 + +/* Include Transactional Tables using InnoDB into mysqld */ +#define _EVENT_WITH_INNOBASE_STORAGE_ENGINE 1 + +/* Include Crash-safe tables with MyISAM heritage into mysqld */ +#define _EVENT_WITH_MARIA_STORAGE_ENGINE 1 + +/* Include Merge multiple MySQL tables into one into mysqld */ +#define _EVENT_WITH_MYISAMMRG_STORAGE_ENGINE 1 + +/* Include Traditional non-transactional MySQL tables into mysqld */ +#define _EVENT_WITH_MYISAM_STORAGE_ENGINE 1 + +/* Include High Availability Clustered tables into mysqld */ +/* #undef _EVENT_WITH_NDBCLUSTER_STORAGE_ENGINE */ + +/* Including Ndb Cluster Binlog */ +/* #undef _EVENT_WITH_NDB_BINLOG */ + +/* Include MySQL Partitioning Support into mysqld */ +#define _EVENT_WITH_PARTITION_STORAGE_ENGINE 1 + +/* Define to 1 if your processor stores words with the most significant byte + first (like Motorola and SPARC, unlike Intel and VAX). */ +/* #undef _EVENT_WORDS_BIGENDIAN */ + +/* Number of bits in a file offset, on hosts where this is settable. */ +/* #undef _EVENT__FILE_OFFSET_BITS */ + +/* makes fseeko etc. visible, on some hosts. */ +/* #undef _EVENT__LARGEFILE_SOURCE */ + +/* Large files support on AIX-style hosts. */ +/* #undef _EVENT__LARGE_FILES */ + +/* Define to appropriate substitue if compiler doesnt have __func__ */ +/* #undef _EVENT___func__ */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef _EVENT_const */ + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef _EVENT___cplusplus +/* #undef _EVENT_inline */ +#endif + +/* Define to `long int' if <sys/types.h> does not define. */ +/* #undef _EVENT_off_t */ + +/* Define to `int' if <sys/types.h> does not define. */ +/* #undef _EVENT_pid_t */ + +/* Define to `unsigned int' if <sys/types.h> does not define. */ +/* #undef _EVENT_size_t */ + +/* Define to unsigned int if you dont have it */ +/* #undef _EVENT_socklen_t */ +#endif === added file 'extra/libevent/event-internal.h' --- a/extra/libevent/event-internal.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/event-internal.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVENT_INTERNAL_H_ +#define _EVENT_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "config.h" +#include "min_heap.h" +#include "evsignal.h" + +struct eventop { + const char *name; + void *(*init)(struct event_base *); + int (*add)(void *, struct event *); + int (*del)(void *, struct event *); + int (*dispatch)(struct event_base *, void *, struct timeval *); + void (*dealloc)(struct event_base *, void *); + /* set if we need to reinitialize the event base */ + int need_reinit; +}; + +struct event_base { + const struct eventop *evsel; + void *evbase; + int event_count; /* counts number of total events */ + int event_count_active; /* counts number of active events */ + + int event_gotterm; /* Set to terminate loop */ + int event_break; /* Set to terminate loop immediately */ + + /* active event management */ + struct event_list **activequeues; + int nactivequeues; + + /* signal handling info */ + struct evsignal_info sig; + + struct event_list eventqueue; + struct timeval event_tv; + + struct min_heap timeheap; +}; + +/* Internal use only: Functions that might be missing from <sys/queue.h> */ +#ifndef HAVE_TAILQFOREACH +#define TAILQ_FIRST(head) ((head)->tqh_first) +#define TAILQ_END(head) NULL +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) +#define TAILQ_FOREACH(var, head, field) \ + for((var) = TAILQ_FIRST(head); \ + (var) != TAILQ_END(head); \ + (var) = TAILQ_NEXT(var, field)) +#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ + (elm)->field.tqe_next = (listelm); \ + *(listelm)->field.tqe_prev = (elm); \ + (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ +} while (0) +#endif /* TAILQ_FOREACH */ + +int _evsignal_set_handler(struct event_base *base, int evsignal, + void (*fn)(int)); +int _evsignal_restore_handler(struct event_base *base, int evsignal); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENT_INTERNAL_H_ */ === added file 'extra/libevent/event.c' --- a/extra/libevent/event.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/event.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,988 @@ +/* + * Copyright (c) 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#undef WIN32_LEAN_AND_MEAN +#include "misc.h" +#endif +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <sys/queue.h> +#include <stdio.h> +#include <stdlib.h> +#ifndef WIN32 +#include <unistd.h> +#endif +#include <errno.h> +#include <signal.h> +#include <string.h> +#include <assert.h> +#include <time.h> + +#include "event.h" +#include "event-internal.h" +#include "evutil.h" +#include "log.h" + +#ifdef HAVE_EVENT_PORTS +extern const struct eventop evportops; +#endif +#ifdef HAVE_SELECT +extern const struct eventop selectops; +#endif +#ifdef HAVE_POLL +extern const struct eventop pollops; +#endif +#ifdef HAVE_EPOLL +extern const struct eventop epollops; +#endif +#ifdef HAVE_WORKING_KQUEUE +extern const struct eventop kqops; +#endif +#ifdef HAVE_DEVPOLL +extern const struct eventop devpollops; +#endif +#ifdef WIN32 +extern const struct eventop win32ops; +#endif + +/* In order of preference */ +const struct eventop *eventops[] = { +#ifdef HAVE_EVENT_PORTS + &evportops, +#endif +#ifdef HAVE_WORKING_KQUEUE + &kqops, +#endif +#ifdef HAVE_EPOLL + &epollops, +#endif +#ifdef HAVE_DEVPOLL + &devpollops, +#endif +#ifdef HAVE_POLL + &pollops, +#endif +#ifdef HAVE_SELECT + &selectops, +#endif +#ifdef WIN32 + &win32ops, +#endif + NULL +}; + +/* Global state */ +struct event_base *current_base = NULL; +extern struct event_base *evsignal_base; +static int use_monotonic; + +/* Handle signals - This is a deprecated interface */ +int (*event_sigcb)(void); /* Signal callback when gotsig is set */ +volatile sig_atomic_t event_gotsig; /* Set in signal handler */ + +/* Prototypes */ +static void event_queue_insert(struct event_base *, struct event *, int); +static void event_queue_remove(struct event_base *, struct event *, int); +static int event_haveevents(struct event_base *); + +static void event_process_active(struct event_base *); + +static int timeout_next(struct event_base *, struct timeval **); +static void timeout_process(struct event_base *); +static void timeout_correct(struct event_base *, struct timeval *); + +static void +detect_monotonic(void) +{ +#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) + use_monotonic = 1; +#endif +} + +static int +gettime(struct timeval *tp) +{ +#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) + struct timespec ts; + + if (use_monotonic) { + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) + return (-1); + + tp->tv_sec = ts.tv_sec; + tp->tv_usec = ts.tv_nsec / 1000; + return (0); + } +#endif + + return (gettimeofday(tp, NULL)); +} + +struct event_base * +event_init(void) +{ + struct event_base *base = event_base_new(); + + if (base != NULL) + current_base = base; + + return (base); +} + +struct event_base * +event_base_new(void) +{ + int i; + struct event_base *base; + + if ((base = calloc(1, sizeof(struct event_base))) == NULL) + event_err(1, "%s: calloc", __func__); + + event_sigcb = NULL; + event_gotsig = 0; + + detect_monotonic(); + gettime(&base->event_tv); + + min_heap_ctor(&base->timeheap); + TAILQ_INIT(&base->eventqueue); + TAILQ_INIT(&base->sig.signalqueue); + base->sig.ev_signal_pair[0] = -1; + base->sig.ev_signal_pair[1] = -1; + + base->evbase = NULL; + for (i = 0; eventops[i] && !base->evbase; i++) { + base->evsel = eventops[i]; + + base->evbase = base->evsel->init(base); + } + + if (base->evbase == NULL) + event_errx(1, "%s: no event mechanism available", __func__); + + if (getenv("EVENT_SHOW_METHOD")) + event_msgx("libevent using: %s\n", + base->evsel->name); + + /* allocate a single active event queue */ + event_base_priority_init(base, 1); + + return (base); +} + +void +event_base_free(struct event_base *base) +{ + int i, n_deleted=0; + struct event *ev; + + if (base == NULL && current_base) + base = current_base; + if (base == current_base) + current_base = NULL; + + /* XXX(niels) - check for internal events first */ + assert(base); + /* Delete all non-internal events. */ + for (ev = TAILQ_FIRST(&base->eventqueue); ev; ) { + struct event *next = TAILQ_NEXT(ev, ev_next); + if (!(ev->ev_flags & EVLIST_INTERNAL)) { + event_del(ev); + ++n_deleted; + } + ev = next; + } + while ((ev = min_heap_top(&base->timeheap)) != NULL) { + event_del(ev); + ++n_deleted; + } + + if (n_deleted) + event_debug(("%s: %d events were still set in base", + __func__, n_deleted)); + + if (base->evsel->dealloc != NULL) + base->evsel->dealloc(base, base->evbase); + + for (i = 0; i < base->nactivequeues; ++i) + assert(TAILQ_EMPTY(base->activequeues[i])); + + assert(min_heap_empty(&base->timeheap)); + min_heap_dtor(&base->timeheap); + + for (i = 0; i < base->nactivequeues; ++i) + free(base->activequeues[i]); + free(base->activequeues); + + assert(TAILQ_EMPTY(&base->eventqueue)); + + free(base); +} + +/* reinitialized the event base after a fork */ +int +event_reinit(struct event_base *base) +{ + const struct eventop *evsel = base->evsel; + void *evbase = base->evbase; + int res = 0; + struct event *ev; + + /* check if this event mechanism requires reinit */ + if (!evsel->need_reinit) + return (0); + + if (base->evsel->dealloc != NULL) + base->evsel->dealloc(base, base->evbase); + base->evbase = evsel->init(base); + if (base->evbase == NULL) + event_errx(1, "%s: could not reinitialize event mechanism", + __func__); + + TAILQ_FOREACH(ev, &base->eventqueue, ev_next) { + if (evsel->add(evbase, ev) == -1) + res = -1; + } + + return (res); +} + +int +event_priority_init(int npriorities) +{ + return event_base_priority_init(current_base, npriorities); +} + +int +event_base_priority_init(struct event_base *base, int npriorities) +{ + int i; + + if (base->event_count_active) + return (-1); + + if (base->nactivequeues && npriorities != base->nactivequeues) { + for (i = 0; i < base->nactivequeues; ++i) { + free(base->activequeues[i]); + } + free(base->activequeues); + } + + /* Allocate our priority queues */ + base->nactivequeues = npriorities; + base->activequeues = (struct event_list **)calloc(base->nactivequeues, + npriorities * sizeof(struct event_list *)); + if (base->activequeues == NULL) + event_err(1, "%s: calloc", __func__); + + for (i = 0; i < base->nactivequeues; ++i) { + base->activequeues[i] = malloc(sizeof(struct event_list)); + if (base->activequeues[i] == NULL) + event_err(1, "%s: malloc", __func__); + TAILQ_INIT(base->activequeues[i]); + } + + return (0); +} + +int +event_haveevents(struct event_base *base) +{ + return (base->event_count > 0); +} + +/* + * Active events are stored in priority queues. Lower priorities are always + * process before higher priorities. Low priority events can starve high + * priority ones. + */ + +static void +event_process_active(struct event_base *base) +{ + struct event *ev; + struct event_list *activeq = NULL; + int i; + short ncalls; + + for (i = 0; i < base->nactivequeues; ++i) { + if (TAILQ_FIRST(base->activequeues[i]) != NULL) { + activeq = base->activequeues[i]; + break; + } + } + + assert(activeq != NULL); + + for (ev = TAILQ_FIRST(activeq); ev; ev = TAILQ_FIRST(activeq)) { + if (ev->ev_events & EV_PERSIST) + event_queue_remove(base, ev, EVLIST_ACTIVE); + else + event_del(ev); + + /* Allows deletes to work */ + ncalls = ev->ev_ncalls; + ev->ev_pncalls = &ncalls; + while (ncalls) { + ncalls--; + ev->ev_ncalls = ncalls; + (*ev->ev_callback)((int)ev->ev_fd, ev->ev_res, ev->ev_arg); + if (event_gotsig || base->event_break) + return; + } + } +} + +/* + * Wait continously for events. We exit only if no events are left. + */ + +int +event_dispatch(void) +{ + return (event_loop(0)); +} + +int +event_base_dispatch(struct event_base *event_base) +{ + return (event_base_loop(event_base, 0)); +} + +const char * +event_base_get_method(struct event_base *base) +{ + assert(base); + return (base->evsel->name); +} + +static void +event_loopexit_cb(int fd, short what, void *arg) +{ + struct event_base *base = arg; + base->event_gotterm = 1; +} + +/* not thread safe */ +int +event_loopexit(struct timeval *tv) +{ + return (event_once(-1, EV_TIMEOUT, event_loopexit_cb, + current_base, tv)); +} + +int +event_base_loopexit(struct event_base *event_base, struct timeval *tv) +{ + return (event_base_once(event_base, -1, EV_TIMEOUT, event_loopexit_cb, + event_base, tv)); +} + +/* not thread safe */ +int +event_loopbreak(void) +{ + return (event_base_loopbreak(current_base)); +} + +int +event_base_loopbreak(struct event_base *event_base) +{ + if (event_base == NULL) + return (-1); + + event_base->event_break = 1; + return (0); +} + + + +/* not thread safe */ + +int +event_loop(int flags) +{ + return event_base_loop(current_base, flags); +} + +int +event_base_loop(struct event_base *base, int flags) +{ + const struct eventop *evsel = base->evsel; + void *evbase = base->evbase; + struct timeval tv; + struct timeval *tv_p; + int res, done; + + if(!TAILQ_EMPTY(&base->sig.signalqueue)) + evsignal_base = base; + done = 0; + while (!done) { + /* Terminate the loop if we have been asked to */ + if (base->event_gotterm) { + base->event_gotterm = 0; + break; + } + + if (base->event_break) { + base->event_break = 0; + break; + } + + /* You cannot use this interface for multi-threaded apps */ + while (event_gotsig) { + event_gotsig = 0; + if (event_sigcb) { + res = (*event_sigcb)(); + if (res == -1) { + errno = EINTR; + return (-1); + } + } + } + + timeout_correct(base, &tv); + + tv_p = &tv; + if (!base->event_count_active && !(flags & EVLOOP_NONBLOCK)) { + timeout_next(base, &tv_p); + } else { + /* + * if we have active events, we just poll new events + * without waiting. + */ + evutil_timerclear(&tv); + } + + /* If we have no events, we just exit */ + if (!event_haveevents(base)) { + event_debug(("%s: no events registered.", __func__)); + return (1); + } + + res = evsel->dispatch(base, evbase, tv_p); + + if (res == -1) + return (-1); + + timeout_process(base); + + if (base->event_count_active) { + event_process_active(base); + if (!base->event_count_active && (flags & EVLOOP_ONCE)) + done = 1; + } else if (flags & EVLOOP_NONBLOCK) + done = 1; + } + + event_debug(("%s: asked to terminate loop.", __func__)); + return (0); +} + +/* Sets up an event for processing once */ + +struct event_once { + struct event ev; + + void (*cb)(int, short, void *); + void *arg; +}; + +/* One-time callback, it deletes itself */ + +static void +event_once_cb(int fd, short events, void *arg) +{ + struct event_once *eonce = arg; + + (*eonce->cb)(fd, events, eonce->arg); + free(eonce); +} + +/* not threadsafe, event scheduled once. */ +int +event_once(int fd, short events, + void (*callback)(int, short, void *), void *arg, struct timeval *tv) +{ + return event_base_once(current_base, fd, events, callback, arg, tv); +} + +/* Schedules an event once */ +int +event_base_once(struct event_base *base, int fd, short events, + void (*callback)(int, short, void *), void *arg, struct timeval *tv) +{ + struct event_once *eonce; + struct timeval etv; + int res; + + /* We cannot support signals that just fire once */ + if (events & EV_SIGNAL) + return (-1); + + if ((eonce = calloc(1, sizeof(struct event_once))) == NULL) + return (-1); + + eonce->cb = callback; + eonce->arg = arg; + + if (events == EV_TIMEOUT) { + if (tv == NULL) { + evutil_timerclear(&etv); + tv = &etv; + } + + evtimer_set(&eonce->ev, event_once_cb, eonce); + } else if (events & (EV_READ|EV_WRITE)) { + events &= EV_READ|EV_WRITE; + + event_set(&eonce->ev, fd, events, event_once_cb, eonce); + } else { + /* Bad event combination */ + free(eonce); + return (-1); + } + + res = event_base_set(base, &eonce->ev); + if (res == 0) + res = event_add(&eonce->ev, tv); + if (res != 0) { + free(eonce); + return (res); + } + + return (0); +} + +void +event_set(struct event *ev, int fd, short events, + void (*callback)(int, short, void *), void *arg) +{ + /* Take the current base - caller needs to set the real base later */ + ev->ev_base = current_base; + + ev->ev_callback = callback; + ev->ev_arg = arg; + ev->ev_fd = fd; + ev->ev_events = events; + ev->ev_res = 0; + ev->ev_flags = EVLIST_INIT; + ev->ev_ncalls = 0; + ev->ev_pncalls = NULL; + + min_heap_elem_init(ev); + + /* by default, we put new events into the middle priority */ + if(current_base) + ev->ev_pri = current_base->nactivequeues/2; +} + +int +event_base_set(struct event_base *base, struct event *ev) +{ + /* Only innocent events may be assigned to a different base */ + if (ev->ev_flags != EVLIST_INIT) + return (-1); + + ev->ev_base = base; + ev->ev_pri = base->nactivequeues/2; + + return (0); +} + +/* + * Set's the priority of an event - if an event is already scheduled + * changing the priority is going to fail. + */ + +int +event_priority_set(struct event *ev, int pri) +{ + if (ev->ev_flags & EVLIST_ACTIVE) + return (-1); + if (pri < 0 || pri >= ev->ev_base->nactivequeues) + return (-1); + + ev->ev_pri = pri; + + return (0); +} + +/* + * Checks if a specific event is pending or scheduled. + */ + +int +event_pending(struct event *ev, short event, struct timeval *tv) +{ + struct timeval now, res; + int flags = 0; + + if (ev->ev_flags & EVLIST_INSERTED) + flags |= (ev->ev_events & (EV_READ|EV_WRITE)); + if (ev->ev_flags & EVLIST_ACTIVE) + flags |= ev->ev_res; + if (ev->ev_flags & EVLIST_TIMEOUT) + flags |= EV_TIMEOUT; + if (ev->ev_flags & EVLIST_SIGNAL) + flags |= EV_SIGNAL; + + event &= (EV_TIMEOUT|EV_READ|EV_WRITE|EV_SIGNAL); + + /* See if there is a timeout that we should report */ + if (tv != NULL && (flags & event & EV_TIMEOUT)) { + gettime(&now); + evutil_timersub(&ev->ev_timeout, &now, &res); + /* correctly remap to real time */ + gettimeofday(&now, NULL); + evutil_timeradd(&now, &res, tv); + } + + return (flags & event); +} + +int +event_add(struct event *ev, struct timeval *tv) +{ + struct event_base *base = ev->ev_base; + const struct eventop *evsel = base->evsel; + void *evbase = base->evbase; + + event_debug(( + "event_add: event: %p, %s%s%scall %p", + ev, + ev->ev_events & EV_READ ? "EV_READ " : " ", + ev->ev_events & EV_WRITE ? "EV_WRITE " : " ", + tv ? "EV_TIMEOUT " : " ", + ev->ev_callback)); + + assert(!(ev->ev_flags & ~EVLIST_ALL)); + + if (tv != NULL) { + struct timeval now; + + if (ev->ev_flags & EVLIST_TIMEOUT) + event_queue_remove(base, ev, EVLIST_TIMEOUT); + else if (min_heap_reserve(&base->timeheap, + 1 + min_heap_size(&base->timeheap)) == -1) + return (-1); /* ENOMEM == errno */ + + /* Check if it is active due to a timeout. Rescheduling + * this timeout before the callback can be executed + * removes it from the active list. */ + if ((ev->ev_flags & EVLIST_ACTIVE) && + (ev->ev_res & EV_TIMEOUT)) { + /* See if we are just active executing this + * event in a loop + */ + if (ev->ev_ncalls && ev->ev_pncalls) { + /* Abort loop */ + *ev->ev_pncalls = 0; + } + + event_queue_remove(base, ev, EVLIST_ACTIVE); + } + + gettime(&now); + evutil_timeradd(&now, tv, &ev->ev_timeout); + + event_debug(( + "event_add: timeout in %d seconds, call %p", + tv->tv_sec, ev->ev_callback)); + + event_queue_insert(base, ev, EVLIST_TIMEOUT); + } + + if ((ev->ev_events & (EV_READ|EV_WRITE)) && + !(ev->ev_flags & (EVLIST_INSERTED|EVLIST_ACTIVE))) { + int res = evsel->add(evbase, ev); + if (res != -1) + event_queue_insert(base, ev, EVLIST_INSERTED); + + return (res); + } else if ((ev->ev_events & EV_SIGNAL) && + !(ev->ev_flags & EVLIST_SIGNAL)) { + int res = evsel->add(evbase, ev); + if (res != -1) + event_queue_insert(base, ev, EVLIST_SIGNAL); + + return (res); + } + + return (0); +} + +int +event_del(struct event *ev) +{ + struct event_base *base; + const struct eventop *evsel; + void *evbase; + + event_debug(("event_del: %p, callback %p", + ev, ev->ev_callback)); + + /* An event without a base has not been added */ + if (ev->ev_base == NULL) + return (-1); + + base = ev->ev_base; + evsel = base->evsel; + evbase = base->evbase; + + assert(!(ev->ev_flags & ~EVLIST_ALL)); + + /* See if we are just active executing this event in a loop */ + if (ev->ev_ncalls && ev->ev_pncalls) { + /* Abort loop */ + *ev->ev_pncalls = 0; + } + + if (ev->ev_flags & EVLIST_TIMEOUT) + event_queue_remove(base, ev, EVLIST_TIMEOUT); + + if (ev->ev_flags & EVLIST_ACTIVE) + event_queue_remove(base, ev, EVLIST_ACTIVE); + + if (ev->ev_flags & EVLIST_INSERTED) { + event_queue_remove(base, ev, EVLIST_INSERTED); + return (evsel->del(evbase, ev)); + } else if (ev->ev_flags & EVLIST_SIGNAL) { + event_queue_remove(base, ev, EVLIST_SIGNAL); + return (evsel->del(evbase, ev)); + } + + return (0); +} + +void +event_active(struct event *ev, int res, short ncalls) +{ + /* We get different kinds of events, add them together */ + if (ev->ev_flags & EVLIST_ACTIVE) { + ev->ev_res |= res; + return; + } + + ev->ev_res = res; + ev->ev_ncalls = ncalls; + ev->ev_pncalls = NULL; + event_queue_insert(ev->ev_base, ev, EVLIST_ACTIVE); +} + +static int +timeout_next(struct event_base *base, struct timeval **tv_p) +{ + struct timeval now; + struct event *ev; + struct timeval *tv = *tv_p; + + if ((ev = min_heap_top(&base->timeheap)) == NULL) { + /* if no time-based events are active wait for I/O */ + *tv_p = NULL; + return (0); + } + + if (gettime(&now) == -1) + return (-1); + + if (evutil_timercmp(&ev->ev_timeout, &now, <=)) { + evutil_timerclear(tv); + return (0); + } + + evutil_timersub(&ev->ev_timeout, &now, tv); + + assert(tv->tv_sec >= 0); + assert(tv->tv_usec >= 0); + + event_debug(("timeout_next: in %d seconds", tv->tv_sec)); + return (0); +} + +/* + * Determines if the time is running backwards by comparing the current + * time against the last time we checked. Not needed when using clock + * monotonic. + */ + +static void +timeout_correct(struct event_base *base, struct timeval *tv) +{ + struct event **pev; + unsigned int size; + struct timeval off; + + if (use_monotonic) + return; + + /* Check if time is running backwards */ + gettime(tv); + if (evutil_timercmp(tv, &base->event_tv, >=)) { + base->event_tv = *tv; + return; + } + + event_debug(("%s: time is running backwards, corrected", + __func__)); + evutil_timersub(&base->event_tv, tv, &off); + + /* + * We can modify the key element of the node without destroying + * the key, beause we apply it to all in the right order. + */ + pev = base->timeheap.p; + size = base->timeheap.n; + for (; size-- > 0; ++pev) { + struct timeval *ev_tv = &(**pev).ev_timeout; + evutil_timersub(ev_tv, &off, ev_tv); + } +} + +void +timeout_process(struct event_base *base) +{ + struct timeval now; + struct event *ev; + + if (min_heap_empty(&base->timeheap)) + return; + + gettime(&now); + + while ((ev = min_heap_top(&base->timeheap))) { + if (evutil_timercmp(&ev->ev_timeout, &now, >)) + break; + + /* delete this event from the I/O queues */ + event_del(ev); + + event_debug(("timeout_process: call %p", + ev->ev_callback)); + event_active(ev, EV_TIMEOUT, 1); + } +} + +void +event_queue_remove(struct event_base *base, struct event *ev, int queue) +{ + if (!(ev->ev_flags & queue)) + event_errx(1, "%s: %p(fd %d) not on queue %x", __func__, + ev, ev->ev_fd, queue); + + if (~ev->ev_flags & EVLIST_INTERNAL) + base->event_count--; + + ev->ev_flags &= ~queue; + switch (queue) { + case EVLIST_ACTIVE: + base->event_count_active--; + TAILQ_REMOVE(base->activequeues[ev->ev_pri], + ev, ev_active_next); + break; + case EVLIST_SIGNAL: + TAILQ_REMOVE(&base->sig.signalqueue, ev, ev_signal_next); + break; + case EVLIST_TIMEOUT: + min_heap_erase(&base->timeheap, ev); + break; + case EVLIST_INSERTED: + TAILQ_REMOVE(&base->eventqueue, ev, ev_next); + break; + default: + event_errx(1, "%s: unknown queue %x", __func__, queue); + } +} + +void +event_queue_insert(struct event_base *base, struct event *ev, int queue) +{ + if (ev->ev_flags & queue) { + /* Double insertion is possible for active events */ + if (queue & EVLIST_ACTIVE) + return; + + event_errx(1, "%s: %p(fd %d) already on queue %x", __func__, + ev, ev->ev_fd, queue); + } + + if (~ev->ev_flags & EVLIST_INTERNAL) + base->event_count++; + + ev->ev_flags |= queue; + switch (queue) { + case EVLIST_ACTIVE: + base->event_count_active++; + TAILQ_INSERT_TAIL(base->activequeues[ev->ev_pri], + ev,ev_active_next); + break; + case EVLIST_SIGNAL: + TAILQ_INSERT_TAIL(&base->sig.signalqueue, ev, ev_signal_next); + break; + case EVLIST_TIMEOUT: { + min_heap_push(&base->timeheap, ev); + break; + } + case EVLIST_INSERTED: + TAILQ_INSERT_TAIL(&base->eventqueue, ev, ev_next); + break; + default: + event_errx(1, "%s: unknown queue %x", __func__, queue); + } +} + +/* Functions for debugging */ + +const char * +event_get_version(void) +{ + return (VERSION); +} + +/* + * No thread-safe interface needed - the information should be the same + * for all threads. + */ + +const char * +event_get_method(void) +{ + return (current_base->evsel->name); +} === added file 'extra/libevent/event.h' --- a/extra/libevent/event.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/event.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,1127 @@ +/* + * Copyright (c) 2000-2007 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVENT_H_ +#define _EVENT_H_ + +/** @mainpage + + @section intro Introduction + + libevent is an event notification library for developing scalable network + servers. The libevent API provides a mechanism to execute a callback + function when a specific event occurs on a file descriptor or after a + timeout has been reached. Furthermore, libevent also support callbacks due + to signals or regular timeouts. + + libevent is meant to replace the event loop found in event driven network + servers. An application just needs to call event_dispatch() and then add or + remove events dynamically without having to change the event loop. + + Currently, libevent supports /dev/poll, kqueue(2), select(2), poll(2) and + epoll(4). It also has experimental support for real-time signals. The + internal event mechanism is completely independent of the exposed event API, + and a simple update of libevent can provide new functionality without having + to redesign the applications. As a result, Libevent allows for portable + application development and provides the most scalable event notification + mechanism available on an operating system. Libevent can also be used for + multi-threaded aplications; see Steven Grimm's explanation. Libevent should + compile on Linux, *BSD, Mac OS X, Solaris and Windows. + + @section usage Standard usage + + Every program that uses libevent must include the <event.h> header, and pass + the -levent flag to the linker. Before using any of the functions in the + library, you must call event_init() or event_base_new() to perform one-time + initialization of the libevent library. + + @section event Event notification + + For each file descriptor that you wish to monitor, you must declare an event + structure and call event_set() to initialize the members of the structure. + To enable notification, you add the structure to the list of monitored + events by calling event_add(). The event structure must remain allocated as + long as it is active, so it should be allocated on the heap. Finally, you + call event_dispatch() to loop and dispatch events. + + @section bufferevent I/O Buffers + + libevent provides an abstraction on top of the regular event callbacks. This + abstraction is called a buffered event. A buffered event provides input and + output buffers that get filled and drained automatically. The user of a + buffered event no longer deals directly with the I/O, but instead is reading + from input and writing to output buffers. + + Once initialized via bufferevent_new(), the bufferevent structure can be + used repeatedly with bufferevent_enable() and bufferevent_disable(). + Instead of reading and writing directly to a socket, you would call + bufferevent_read() and bufferevent_write(). + + When read enabled the bufferevent will try to read from the file descriptor + and call the read callback. The write callback is executed whenever the + output buffer is drained below the write low watermark, which is 0 by + default. + + @section timers Timers + + libevent can also be used to create timers that invoke a callback after a + certain amount of time has expired. The evtimer_set() function prepares an + event struct to be used as a timer. To activate the timer, call + evtimer_add(). Timers can be deactivated by calling evtimer_del(). + + @section timeouts Timeouts + + In addition to simple timers, libevent can assign timeout events to file + descriptors that are triggered whenever a certain amount of time has passed + with no activity on a file descriptor. The timeout_set() function + initializes an event struct for use as a timeout. Once initialized, the + event must be activated by using timeout_add(). To cancel the timeout, call + timeout_del(). + + @section evdns Asynchronous DNS resolution + + libevent provides an asynchronous DNS resolver that should be used instead + of the standard DNS resolver functions. These functions can be imported by + including the <evdns.h> header in your program. Before using any of the + resolver functions, you must call evdns_init() to initialize the library. To + convert a hostname to an IP address, you call the evdns_resolve_ipv4() + function. To perform a reverse lookup, you would call the + evdns_resolve_reverse() function. All of these functions use callbacks to + avoid blocking while the lookup is performed. + + @section evhttp Event-driven HTTP servers + + libevent provides a very simple event-driven HTTP server that can be + embedded in your program and used to service HTTP requests. + + To use this capability, you need to include the <evhttp.h> header in your + program. You create the server by calling evhttp_new(). Add addresses and + ports to listen on with evhttp_bind_socket(). You then register one or more + callbacks to handle incoming requests. Each URI can be assigned a callback + via the evhttp_set_cb() function. A generic callback function can also be + registered via evhttp_set_gencb(); this callback will be invoked if no other + callbacks have been registered for a given URI. + + @section evrpc A framework for RPC servers and clients + + libevents provides a framework for creating RPC servers and clients. It + takes care of marshaling and unmarshaling all data structures. + + @section api API Reference + + To browse the complete documentation of the libevent API, click on any of + the following links. + + event.h + The primary libevent header + + evdns.h + Asynchronous DNS resolution + + evhttp.h + An embedded libevent-based HTTP server + + evrpc.h + A framework for creating RPC servers and clients + + */ + +/** @file event.h + + A library for writing event-driven network servers + + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <event-config.h> +#ifdef _EVENT_HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef _EVENT_HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#ifdef _EVENT_HAVE_STDINT_H +#include <stdint.h> +#endif +#include <stdarg.h> + +/* For int types. */ +#include <evutil.h> + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#undef WIN32_LEAN_AND_MEAN +typedef unsigned char u_char; +typedef unsigned short u_short; +#endif + +#define EVLIST_TIMEOUT 0x01 +#define EVLIST_INSERTED 0x02 +#define EVLIST_SIGNAL 0x04 +#define EVLIST_ACTIVE 0x08 +#define EVLIST_INTERNAL 0x10 +#define EVLIST_INIT 0x80 + +/* EVLIST_X_ Private space: 0x1000-0xf000 */ +#define EVLIST_ALL (0xf000 | 0x9f) + +#define EV_TIMEOUT 0x01 +#define EV_READ 0x02 +#define EV_WRITE 0x04 +#define EV_SIGNAL 0x08 +#define EV_PERSIST 0x10 /* Persistant event */ + +/* Fix so that ppl dont have to run with <sys/queue.h> */ +#ifndef TAILQ_ENTRY +#define _EVENT_DEFINED_TQENTRY +#define TAILQ_ENTRY(type) \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ +} +#endif /* !TAILQ_ENTRY */ + +struct event_base; +struct event { + TAILQ_ENTRY (event) ev_next; + TAILQ_ENTRY (event) ev_active_next; + TAILQ_ENTRY (event) ev_signal_next; + unsigned int min_heap_idx; /* for managing timeouts */ + + struct event_base *ev_base; + + int ev_fd; + short ev_events; + short ev_ncalls; + short *ev_pncalls; /* Allows deletes in callback */ + + struct timeval ev_timeout; + + int ev_pri; /* smaller numbers are higher priority */ + + void (*ev_callback)(int, short, void *arg); + void *ev_arg; + + int ev_res; /* result passed to event callback */ + int ev_flags; +}; + +#define EVENT_SIGNAL(ev) (int)(ev)->ev_fd +#define EVENT_FD(ev) (int)(ev)->ev_fd + +/* + * Key-Value pairs. Can be used for HTTP headers but also for + * query argument parsing. + */ +struct evkeyval { + TAILQ_ENTRY(evkeyval) next; + + char *key; + char *value; +}; + +#ifdef _EVENT_DEFINED_TQENTRY +#undef TAILQ_ENTRY +struct event_list; +struct evkeyvalq; +#undef _EVENT_DEFINED_TQENTRY +#else +TAILQ_HEAD (event_list, event); +TAILQ_HEAD (evkeyvalq, evkeyval); +#endif /* _EVENT_DEFINED_TQENTRY */ + +/** + Initialize the event API. + + Use event_base_new() to initialize a new event base, but does not set + the current_base global. If using only event_base_new(), each event + added must have an event base set with event_base_set() + + @see event_base_set(), event_base_free(), event_init() + */ +struct event_base *event_base_new(void); + +/** + Initialize the event API. + + The event API needs to be initialized with event_init() before it can be + used. Sets the current_base global representing the default base for + events that have no base associated with them. + + @see event_base_set(), event_base_new() + */ +struct event_base *event_init(void); + +/** + Reinitialized the event base after a fork + + Some event mechanisms do not survive across fork. The event base needs + to be reinitialized with the event_reinit() function. + + @param base the event base that needs to be re-initialized + @return 0 if successful, or -1 if some events could not be re-added. + @see event_base_new(), event_init() +*/ +int event_reinit(struct event_base *base); + +/** + Loop to process events. + + In order to process events, an application needs to call + event_dispatch(). This function only returns on error, and should + replace the event core of the application program. + + @see event_base_dispatch() + */ +int event_dispatch(void); + + +/** + Threadsafe event dispatching loop. + + @param eb the event_base structure returned by event_init() + @see event_init(), event_dispatch() + */ +int event_base_dispatch(struct event_base *); + + +/** + Get the kernel event notification mechanism used by libevent. + + @param eb the event_base structure returned by event_base_new() + @return a string identifying the kernel event mechanism (kqueue, epoll, etc.) + */ +const char *event_base_get_method(struct event_base *); + + +/** + Deallocate all memory associated with an event_base, and free the base. + + Note that this function will not close any fds or free any memory passed + to event_set as the argument to callback. + + @param eb an event_base to be freed + */ +void event_base_free(struct event_base *); + + +#define _EVENT_LOG_DEBUG 0 +#define _EVENT_LOG_MSG 1 +#define _EVENT_LOG_WARN 2 +#define _EVENT_LOG_ERR 3 +typedef void (*event_log_cb)(int severity, const char *msg); +/** + Redirect libevent's log messages. + + @param cb a function taking two arguments: an integer severity between + _EVENT_LOG_DEBUG and _EVENT_LOG_ERR, and a string. If cb is NULL, + then the default log is used. + */ +void event_set_log_callback(event_log_cb cb); + +/** + Associate a different event base with an event. + + @param eb the event base + @param ev the event + */ +int event_base_set(struct event_base *, struct event *); + +/** + event_loop() flags + */ +/*@{*/ +#define EVLOOP_ONCE 0x01 /**< Block at most once. */ +#define EVLOOP_NONBLOCK 0x02 /**< Do not block. */ +/*@}*/ + +/** + Handle events. + + This is a more flexible version of event_dispatch(). + + @param flags any combination of EVLOOP_ONCE | EVLOOP_NONBLOCK + @return 0 if successful, -1 if an error occurred, or 1 if no events were + registered. + @see event_loopexit(), event_base_loop() +*/ +int event_loop(int); + +/** + Handle events (threadsafe version). + + This is a more flexible version of event_base_dispatch(). + + @param eb the event_base structure returned by event_init() + @param flags any combination of EVLOOP_ONCE | EVLOOP_NONBLOCK + @return 0 if successful, -1 if an error occurred, or 1 if no events were + registered. + @see event_loopexit(), event_base_loop() + */ +int event_base_loop(struct event_base *, int); + +/** + Exit the event loop after the specified time. + + The next event_loop() iteration after the given timer expires will + complete normally (handling all queued events) then exit without + blocking for events again. + + Subsequent invocations of event_loop() will proceed normally. + + @param tv the amount of time after which the loop should terminate. + @return 0 if successful, or -1 if an error occurred + @see event_loop(), event_base_loop(), event_base_loopexit() + */ +int event_loopexit(struct timeval *); + + +/** + Exit the event loop after the specified time (threadsafe variant). + + The next event_base_loop() iteration after the given timer expires will + complete normally (handling all queued events) then exit without + blocking for events again. + + Subsequent invocations of event_base_loop() will proceed normally. + + @param eb the event_base structure returned by event_init() + @param tv the amount of time after which the loop should terminate. + @return 0 if successful, or -1 if an error occurred + @see event_loopexit() + */ +int event_base_loopexit(struct event_base *, struct timeval *); + +/** + Abort the active event_loop() immediately. + + event_loop() will abort the loop after the next event is completed; + event_loopbreak() is typically invoked from this event's callback. + This behavior is analogous to the "break;" statement. + + Subsequent invocations of event_loop() will proceed normally. + + @return 0 if successful, or -1 if an error occurred + @see event_base_loopbreak(), event_loopexit() + */ +int event_loopbreak(void); + +/** + Abort the active event_base_loop() immediately. + + event_base_loop() will abort the loop after the next event is completed; + event_base_loopbreak() is typically invoked from this event's callback. + This behavior is analogous to the "break;" statement. + + Subsequent invocations of event_loop() will proceed normally. + + @param eb the event_base structure returned by event_init() + @return 0 if successful, or -1 if an error occurred + @see event_base_loopexit + */ +int event_base_loopbreak(struct event_base *); + + +/** + Add a timer event. + + @param ev the event struct + @param tv timeval struct + */ +#define evtimer_add(ev, tv) event_add(ev, tv) + + +/** + Define a timer event. + + @param ev event struct to be modified + @param cb callback function + @param arg argument that will be passed to the callback function + */ +#define evtimer_set(ev, cb, arg) event_set(ev, -1, 0, cb, arg) + + +/** + * Delete a timer event. + * + * @param ev the event struct to be disabled + */ +#define evtimer_del(ev) event_del(ev) +#define evtimer_pending(ev, tv) event_pending(ev, EV_TIMEOUT, tv) +#define evtimer_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +/** + * Add a timeout event. + * + * @param ev the event struct to be disabled + * @param tv the timeout value, in seconds + */ +#define timeout_add(ev, tv) event_add(ev, tv) + + +/** + * Define a timeout event. + * + * @param ev the event struct to be defined + * @param cb the callback to be invoked when the timeout expires + * @param arg the argument to be passed to the callback + */ +#define timeout_set(ev, cb, arg) event_set(ev, -1, 0, cb, arg) + + +/** + * Disable a timeout event. + * + * @param ev the timeout event to be disabled + */ +#define timeout_del(ev) event_del(ev) + +#define timeout_pending(ev, tv) event_pending(ev, EV_TIMEOUT, tv) +#define timeout_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +#define signal_add(ev, tv) event_add(ev, tv) +#define signal_set(ev, x, cb, arg) \ + event_set(ev, x, EV_SIGNAL|EV_PERSIST, cb, arg) +#define signal_del(ev) event_del(ev) +#define signal_pending(ev, tv) event_pending(ev, EV_SIGNAL, tv) +#define signal_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) + +/** + Prepare an event structure to be added. + + The function event_set() prepares the event structure ev to be used in + future calls to event_add() and event_del(). The event will be prepared to + call the function specified by the fn argument with an int argument + indicating the file descriptor, a short argument indicating the type of + event, and a void * argument given in the arg argument. The fd indicates + the file descriptor that should be monitored for events. The events can be + either EV_READ, EV_WRITE, or both. Indicating that an application can read + or write from the file descriptor respectively without blocking. + + The function fn will be called with the file descriptor that triggered the + event and the type of event which will be either EV_TIMEOUT, EV_SIGNAL, + EV_READ, or EV_WRITE. The additional flag EV_PERSIST makes an event_add() + persistent until event_del() has been called. + + @param ev an event struct to be modified + @param fd the file descriptor to be monitored + @param event desired events to monitor; can be EV_READ and/or EV_WRITE + @param fn callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + + @see event_add(), event_del(), event_once() + + */ +void event_set(struct event *, int, short, void (*)(int, short, void *), void *); + +/** + Schedule a one-time event to occur. + + The function event_once() is similar to event_set(). However, it schedules + a callback to be called exactly once and does not require the caller to + prepare an event structure. + + @param fd a file descriptor to monitor + @param events event(s) to monitor; can be any of EV_TIMEOUT | EV_READ | + EV_WRITE + @param callback callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_set() + + */ +int event_once(int, short, void (*)(int, short, void *), void *, struct timeval *); + + +/** + Schedule a one-time event (threadsafe variant) + + The function event_base_once() is similar to event_set(). However, it + schedules a callback to be called exactly once and does not require the + caller to prepare an event structure. + + @param base an event_base returned by event_init() + @param fd a file descriptor to monitor + @param events event(s) to monitor; can be any of EV_TIMEOUT | EV_READ | + EV_WRITE + @param callback callback function to be invoked when the event occurs + @param arg an argument to be passed to the callback function + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_once() + */ +int event_base_once(struct event_base *, int, short, void (*)(int, short, void *), void *, struct timeval *); + + +/** + Add an event to the set of monitored events. + + The function event_add() schedules the execution of the ev event when the + event specified in event_set() occurs or in at least the time specified in + the tv. If tv is NULL, no timeout occurs and the function will only be + called if a matching event occurs on the file descriptor. The event in the + ev argument must be already initialized by event_set() and may not be used + in calls to event_set() until it has timed out or been removed with + event_del(). If the event in the ev argument already has a scheduled + timeout, the old timeout will be replaced by the new one. + + @param ev an event struct initialized via event_set() + @param timeout the maximum amount of time to wait for the event, or NULL + to wait forever + @return 0 if successful, or -1 if an error occurred + @see event_del(), event_set() + */ +int event_add(struct event *, struct timeval *); + + +/** + Remove an event from the set of monitored events. + + The function event_del() will cancel the event in the argument ev. If the + event has already executed or has never been added the call will have no + effect. + + @param ev an event struct to be removed from the working set + @return 0 if successful, or -1 if an error occurred + @see event_add() + */ +int event_del(struct event *); + +void event_active(struct event *, int, short); + + +/** + Checks if a specific event is pending or scheduled. + + @param ev an event struct previously passed to event_add() + @param event the requested event type; any of EV_TIMEOUT|EV_READ| + EV_WRITE|EV_SIGNAL + @param tv an alternate timeout (FIXME - is this true?) + + @return 1 if the event is pending, or 0 if the event has not occurred + + */ +int event_pending(struct event *, short, struct timeval *); + + +/** + Test if an event structure has been initialized. + + The event_initialized() macro can be used to check if an event has been + initialized. + + @param ev an event structure to be tested + @return 1 if the structure has been initialized, or 0 if it has not been + initialized + */ +#ifdef WIN32 +#define event_initialized(ev) ((ev)->ev_flags & EVLIST_INIT && (ev)->ev_fd != (int)INVALID_HANDLE_VALUE) +#else +#define event_initialized(ev) ((ev)->ev_flags & EVLIST_INIT) +#endif + + +/** + Get the libevent version number. + + @return a string containing the version number of libevent + */ +const char *event_get_version(void); + + +/** + Get the kernel event notification mechanism used by libevent. + + @return a string identifying the kernel event mechanism (kqueue, epoll, etc.) + */ +const char *event_get_method(void); + + +/** + Set the number of different event priorities. + + By default libevent schedules all active events with the same priority. + However, some time it is desirable to process some events with a higher + priority than others. For that reason, libevent supports strict priority + queues. Active events with a lower priority are always processed before + events with a higher priority. + + The number of different priorities can be set initially with the + event_priority_init() function. This function should be called before the + first call to event_dispatch(). The event_priority_set() function can be + used to assign a priority to an event. By default, libevent assigns the + middle priority to all events unless their priority is explicitly set. + + @param npriorities the maximum number of priorities + @return 0 if successful, or -1 if an error occurred + @see event_base_priority_init(), event_priority_set() + + */ +int event_priority_init(int); + + +/** + Set the number of different event priorities (threadsafe variant). + + See the description of event_priority_init() for more information. + + @param eb the event_base structure returned by event_init() + @param npriorities the maximum number of priorities + @return 0 if successful, or -1 if an error occurred + @see event_priority_init(), event_priority_set() + */ +int event_base_priority_init(struct event_base *, int); + + +/** + Assign a priority to an event. + + @param ev an event struct + @param priority the new priority to be assigned + @return 0 if successful, or -1 if an error occurred + @see event_priority_init() + */ +int event_priority_set(struct event *, int); + + +/* These functions deal with buffering input and output */ + +struct evbuffer { + u_char *buffer; + u_char *orig_buffer; + + size_t misalign; + size_t totallen; + size_t off; + + void (*cb)(struct evbuffer *, size_t, size_t, void *); + void *cbarg; +}; + +/* Just for error reporting - use other constants otherwise */ +#define EVBUFFER_READ 0x01 +#define EVBUFFER_WRITE 0x02 +#define EVBUFFER_EOF 0x10 +#define EVBUFFER_ERROR 0x20 +#define EVBUFFER_TIMEOUT 0x40 + +struct bufferevent; +typedef void (*evbuffercb)(struct bufferevent *, void *); +typedef void (*everrorcb)(struct bufferevent *, short what, void *); + +struct event_watermark { + size_t low; + size_t high; +}; + +struct bufferevent { + struct event ev_read; + struct event ev_write; + + struct evbuffer *input; + struct evbuffer *output; + + struct event_watermark wm_read; + struct event_watermark wm_write; + + evbuffercb readcb; + evbuffercb writecb; + everrorcb errorcb; + void *cbarg; + + int timeout_read; /* in seconds */ + int timeout_write; /* in seconds */ + + short enabled; /* events that are currently enabled */ +}; + + +/** + Create a new bufferevent. + + libevent provides an abstraction on top of the regular event callbacks. + This abstraction is called a buffered event. A buffered event provides + input and output buffers that get filled and drained automatically. The + user of a buffered event no longer deals directly with the I/O, but + instead is reading from input and writing to output buffers. + + Once initialized, the bufferevent structure can be used repeatedly with + bufferevent_enable() and bufferevent_disable(). + + When read enabled the bufferevent will try to read from the file descriptor + and call the read callback. The write callback is executed whenever the + output buffer is drained below the write low watermark, which is 0 by + default. + + If multiple bases are in use, bufferevent_base_set() must be called before + enabling the bufferevent for the first time. + + @param fd the file descriptor from which data is read and written to. + This file descriptor is not allowed to be a pipe(2). + @param readcb callback to invoke when there is data to be read, or NULL if + no callback is desired + @param writecb callback to invoke when the file descriptor is ready for + writing, or NULL if no callback is desired + @param errorcb callback to invoke when there is an error on the file + descriptor + @param cbarg an argument that will be supplied to each of the callbacks + (readcb, writecb, and errorcb) + @return a pointer to a newly allocated bufferevent struct, or NULL if an + error occurred + @see bufferevent_base_set(), bufferevent_free() + */ +struct bufferevent *bufferevent_new(int fd, + evbuffercb readcb, evbuffercb writecb, everrorcb errorcb, void *cbarg); + + +/** + Assign a bufferevent to a specific event_base. + + @param base an event_base returned by event_init() + @param bufev a bufferevent struct returned by bufferevent_new() + @return 0 if successful, or -1 if an error occurred + @see bufferevent_new() + */ +int bufferevent_base_set(struct event_base *base, struct bufferevent *bufev); + + +/** + Assign a priority to a bufferevent. + + @param bufev a bufferevent struct + @param pri the priority to be assigned + @return 0 if successful, or -1 if an error occurred + */ +int bufferevent_priority_set(struct bufferevent *bufev, int pri); + + +/** + Deallocate the storage associated with a bufferevent structure. + + @param bufev the bufferevent structure to be freed. + */ +void bufferevent_free(struct bufferevent *bufev); + + +/** + Write data to a bufferevent buffer. + + The bufferevent_write() function can be used to write data to the file + descriptor. The data is appended to the output buffer and written to the + descriptor automatically as it becomes available for writing. + + @param bufev the bufferevent to be written to + @param data a pointer to the data to be written + @param size the length of the data, in bytes + @return 0 if successful, or -1 if an error occurred + @see bufferevent_write_buffer() + */ +int bufferevent_write(struct bufferevent *bufev, + const void *data, size_t size); + + +/** + Write data from an evbuffer to a bufferevent buffer. The evbuffer is + being drained as a result. + + @param bufev the bufferevent to be written to + @param buf the evbuffer to be written + @return 0 if successful, or -1 if an error occurred + @see bufferevent_write() + */ +int bufferevent_write_buffer(struct bufferevent *bufev, struct evbuffer *buf); + + +/** + Read data from a bufferevent buffer. + + The bufferevent_read() function is used to read data from the input buffer. + + @param bufev the bufferevent to be read from + @param data pointer to a buffer that will store the data + @param size the size of the data buffer, in bytes + @return the amount of data read, in bytes. + */ +size_t bufferevent_read(struct bufferevent *bufev, void *data, size_t size); + +/** + Enable a bufferevent. + + @param bufev the bufferevent to be enabled + @param event any combination of EV_READ | EV_WRITE. + @return 0 if successful, or -1 if an error occurred + @see bufferevent_disable() + */ +int bufferevent_enable(struct bufferevent *bufev, short event); + + +/** + Disable a bufferevent. + + @param bufev the bufferevent to be disabled + @param event any combination of EV_READ | EV_WRITE. + @return 0 if successful, or -1 if an error occurred + @see bufferevent_enable() + */ +int bufferevent_disable(struct bufferevent *bufev, short event); + + +/** + Set the read and write timeout for a buffered event. + + @param bufev the bufferevent to be modified + @param timeout_read the read timeout + @param timeout_write the write timeout + */ +void bufferevent_settimeout(struct bufferevent *bufev, + int timeout_read, int timeout_write); + + +#define EVBUFFER_LENGTH(x) (x)->off +#define EVBUFFER_DATA(x) (x)->buffer +#define EVBUFFER_INPUT(x) (x)->input +#define EVBUFFER_OUTPUT(x) (x)->output + + +/** + Allocate storage for a new evbuffer. + + @return a pointer to a newly allocated evbuffer struct, or NULL if an error + occurred + */ +struct evbuffer *evbuffer_new(void); + + +/** + Deallocate storage for an evbuffer. + + @param pointer to the evbuffer to be freed + */ +void evbuffer_free(struct evbuffer *); + + +/** + Expands the available space in an event buffer. + + Expands the available space in the event buffer to at least datlen + + @param buf the event buffer to be expanded + @param datlen the new minimum length requirement + @return 0 if successful, or -1 if an error occurred +*/ +int evbuffer_expand(struct evbuffer *, size_t); + + +/** + Append data to the end of an evbuffer. + + @param buf the event buffer to be appended to + @param data pointer to the beginning of the data buffer + @param datlen the number of bytes to be copied from the data buffer + */ +int evbuffer_add(struct evbuffer *, const void *, size_t); + + + +/** + Read data from an event buffer and drain the bytes read. + + @param buf the event buffer to be read from + @param data the destination buffer to store the result + @param datlen the maximum size of the destination buffer + @return the number of bytes read + */ +int evbuffer_remove(struct evbuffer *, void *, size_t); + + +/** + * Read a single line from an event buffer. + * + * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'. + * The returned buffer needs to be freed by the caller. + * + * @param buffer the evbuffer to read from + * @return pointer to a single line, or NULL if an error occurred + */ +char *evbuffer_readline(struct evbuffer *); + + +/** + Move data from one evbuffer into another evbuffer. + + This is a destructive add. The data from one buffer moves into + the other buffer. The destination buffer is expanded as needed. + + @param outbuf the output buffer + @param inbuf the input buffer + @return 0 if successful, or -1 if an error occurred + */ +int evbuffer_add_buffer(struct evbuffer *, struct evbuffer *); + + +/** + Append a formatted string to the end of an evbuffer. + + @param buf the evbuffer that will be appended to + @param fmt a format string + @param ... arguments that will be passed to printf(3) + @return 0 if successful, or -1 if an error occurred + */ +int evbuffer_add_printf(struct evbuffer *, const char *fmt, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 2, 3))) +#endif +; + + +/** + Append a va_list formatted string to the end of an evbuffer. + + @param buf the evbuffer that will be appended to + @param fmt a format string + @param ap a varargs va_list argument array that will be passed to vprintf(3) + @return 0 if successful, or -1 if an error occurred + */ +int evbuffer_add_vprintf(struct evbuffer *, const char *fmt, va_list ap); + + +/** + Remove a specified number of bytes data from the beginning of an evbuffer. + + @param buf the evbuffer to be drained + @param len the number of bytes to drain from the beginning of the buffer + @return 0 if successful, or -1 if an error occurred + */ +void evbuffer_drain(struct evbuffer *, size_t); + + +/** + Write the contents of an evbuffer to a file descriptor. + + The evbuffer will be drained after the bytes have been successfully written. + + @param buffer the evbuffer to be written and drained + @param fd the file descriptor to be written to + @return the number of bytes written, or -1 if an error occurred + @see evbuffer_read() + */ +int evbuffer_write(struct evbuffer *, int); + + +/** + Read from a file descriptor and store the result in an evbuffer. + + @param buf the evbuffer to store the result + @param fd the file descriptor to read from + @param howmuch the number of bytes to be read + @return the number of bytes read, or -1 if an error occurred + @see evbuffer_write() + */ +int evbuffer_read(struct evbuffer *, int, int); + + +/** + Find a string within an evbuffer. + + @param buffer the evbuffer to be searched + @param what the string to be searched for + @param len the length of the search string + @return a pointer to the beginning of the search string, or NULL if the search failed. + */ +u_char *evbuffer_find(struct evbuffer *, const u_char *, size_t); + +/** + Set a callback to invoke when the evbuffer is modified. + + @param buffer the evbuffer to be monitored + @param cb the callback function to invoke when the evbuffer is modified + @param cbarg an argument to be provided to the callback function + */ +void evbuffer_setcb(struct evbuffer *, void (*)(struct evbuffer *, size_t, size_t, void *), void *); + +/* + * Marshaling tagged data - We assume that all tags are inserted in their + * numeric order - so that unknown tags will always be higher than the + * known ones - and we can just ignore the end of an event buffer. + */ + +void evtag_init(void); + +void evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, const void *data, + ev_uint32_t len); + +/** + Encode an integer and store it in an evbuffer. + + We encode integer's by nibbles; the first nibble contains the number + of significant nibbles - 1; this allows us to encode up to 64-bit + integers. This function is byte-order independent. + + @param evbuf evbuffer to store the encoded number + @param number a 32-bit integer + */ +void encode_int(struct evbuffer *evbuf, ev_uint32_t number); + +void evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag, + ev_uint32_t integer); + +void evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag, + const char *string); + +void evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag, + struct timeval *tv); + +int evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag, + struct evbuffer *dst); +int evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag); +int evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength); +int evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength); +int evtag_consume(struct evbuffer *evbuf); + +int evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag, + ev_uint32_t *pinteger); + +int evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag, + void *data, size_t len); + +int evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag, + char **pstring); + +int evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag, + struct timeval *ptv); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVENT_H_ */ === added file 'extra/libevent/event_tagging.c' --- a/extra/libevent/event_tagging.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/event_tagging.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2003, 2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <winsock2.h> +#include <windows.h> +#undef WIN32_LEAN_AND_MEAN +#else +#include <sys/ioctl.h> +#endif + +#include <sys/queue.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef WIN32 +#include <syslog.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include "event.h" +#include "evutil.h" +#include "log.h" + +int evtag_decode_int(ev_uint32_t *pnumber, struct evbuffer *evbuf); +int evtag_encode_tag(struct evbuffer *evbuf, ev_uint32_t tag); +int evtag_decode_tag(ev_uint32_t *ptag, struct evbuffer *evbuf); + +static struct evbuffer *_buf; /* not thread safe */ + +void +evtag_init(void) +{ + if (_buf != NULL) + return; + + if ((_buf = evbuffer_new()) == NULL) + event_err(1, "%s: malloc", __func__); +} + +/* + * We encode integer's by nibbles; the first nibble contains the number + * of significant nibbles - 1; this allows us to encode up to 64-bit + * integers. This function is byte-order independent. + */ + +void +encode_int(struct evbuffer *evbuf, ev_uint32_t number) +{ + int off = 1, nibbles = 0; + ev_uint8_t data[5]; + + memset(data, 0, sizeof(data)); + while (number) { + if (off & 0x1) + data[off/2] = (data[off/2] & 0xf0) | (number & 0x0f); + else + data[off/2] = (data[off/2] & 0x0f) | + ((number & 0x0f) << 4); + number >>= 4; + off++; + } + + if (off > 2) + nibbles = off - 2; + + /* Off - 1 is the number of encoded nibbles */ + data[0] = (data[0] & 0x0f) | ((nibbles & 0x0f) << 4); + + evbuffer_add(evbuf, data, (off + 1) / 2); +} + +/* + * Support variable length encoding of tags; we use the high bit in each + * octet as a continuation signal. + */ + +int +evtag_encode_tag(struct evbuffer *evbuf, ev_uint32_t tag) +{ + int bytes = 0; + ev_uint8_t data[5]; + + memset(data, 0, sizeof(data)); + do { + ev_uint8_t lower = tag & 0x7f; + tag >>= 7; + + if (tag) + lower |= 0x80; + + data[bytes++] = lower; + } while (tag); + + if (evbuf != NULL) + evbuffer_add(evbuf, data, bytes); + + return (bytes); +} + +static int +decode_tag_internal(ev_uint32_t *ptag, struct evbuffer *evbuf, int dodrain) +{ + ev_uint32_t number = 0; + ev_uint8_t *data = EVBUFFER_DATA(evbuf); + int len = EVBUFFER_LENGTH(evbuf); + int count = 0, shift = 0, done = 0; + + while (count++ < len) { + ev_uint8_t lower = *data++; + number |= (lower & 0x7f) << shift; + shift += 7; + + if (!(lower & 0x80)) { + done = 1; + break; + } + } + + if (!done) + return (-1); + + if (dodrain) + evbuffer_drain(evbuf, count); + + if (ptag != NULL) + *ptag = number; + + return (count); +} + +int +evtag_decode_tag(ev_uint32_t *ptag, struct evbuffer *evbuf) +{ + return (decode_tag_internal(ptag, evbuf, 1 /* dodrain */)); +} + +/* + * Marshal a data type, the general format is as follows: + * + * tag number: one byte; length: var bytes; payload: var bytes + */ + +void +evtag_marshal(struct evbuffer *evbuf, ev_uint32_t tag, + const void *data, ev_uint32_t len) +{ + evtag_encode_tag(evbuf, tag); + encode_int(evbuf, len); + evbuffer_add(evbuf, (void *)data, len); +} + +/* Marshaling for integers */ +void +evtag_marshal_int(struct evbuffer *evbuf, ev_uint32_t tag, ev_uint32_t integer) +{ + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + encode_int(_buf, integer); + + evtag_encode_tag(evbuf, tag); + encode_int(evbuf, EVBUFFER_LENGTH(_buf)); + evbuffer_add_buffer(evbuf, _buf); +} + +void +evtag_marshal_string(struct evbuffer *buf, ev_uint32_t tag, const char *string) +{ + evtag_marshal(buf, tag, string, strlen(string)); +} + +void +evtag_marshal_timeval(struct evbuffer *evbuf, ev_uint32_t tag, struct timeval *tv) +{ + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + + encode_int(_buf, tv->tv_sec); + encode_int(_buf, tv->tv_usec); + + evtag_marshal(evbuf, tag, EVBUFFER_DATA(_buf), + EVBUFFER_LENGTH(_buf)); +} + +static int +decode_int_internal(ev_uint32_t *pnumber, struct evbuffer *evbuf, int dodrain) +{ + ev_uint32_t number = 0; + ev_uint8_t *data = EVBUFFER_DATA(evbuf); + int len = EVBUFFER_LENGTH(evbuf); + int nibbles = 0; + + if (!len) + return (-1); + + nibbles = ((data[0] & 0xf0) >> 4) + 1; + if (nibbles > 8 || (nibbles >> 1) + 1 > len) + return (-1); + len = (nibbles >> 1) + 1; + + while (nibbles > 0) { + number <<= 4; + if (nibbles & 0x1) + number |= data[nibbles >> 1] & 0x0f; + else + number |= (data[nibbles >> 1] & 0xf0) >> 4; + nibbles--; + } + + if (dodrain) + evbuffer_drain(evbuf, len); + + *pnumber = number; + + return (len); +} + +int +evtag_decode_int(ev_uint32_t *pnumber, struct evbuffer *evbuf) +{ + return (decode_int_internal(pnumber, evbuf, 1) == -1 ? -1 : 0); +} + +int +evtag_peek(struct evbuffer *evbuf, ev_uint32_t *ptag) +{ + return (decode_tag_internal(ptag, evbuf, 0 /* dodrain */)); +} + +int +evtag_peek_length(struct evbuffer *evbuf, ev_uint32_t *plength) +{ + struct evbuffer tmp; + int res, len; + + len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */); + if (len == -1) + return (-1); + + tmp = *evbuf; + tmp.buffer += len; + tmp.off -= len; + + res = decode_int_internal(plength, &tmp, 0); + if (res == -1) + return (-1); + + *plength += res + len; + + return (0); +} + +int +evtag_payload_length(struct evbuffer *evbuf, ev_uint32_t *plength) +{ + struct evbuffer tmp; + int res, len; + + len = decode_tag_internal(NULL, evbuf, 0 /* dodrain */); + if (len == -1) + return (-1); + + tmp = *evbuf; + tmp.buffer += len; + tmp.off -= len; + + res = decode_int_internal(plength, &tmp, 0); + if (res == -1) + return (-1); + + return (0); +} + +int +evtag_consume(struct evbuffer *evbuf) +{ + ev_uint32_t len; + if (decode_tag_internal(NULL, evbuf, 1 /* dodrain */) == -1) + return (-1); + if (evtag_decode_int(&len, evbuf) == -1) + return (-1); + evbuffer_drain(evbuf, len); + + return (0); +} + +/* Reads the data type from an event buffer */ + +int +evtag_unmarshal(struct evbuffer *src, ev_uint32_t *ptag, struct evbuffer *dst) +{ + ev_uint32_t len; + ev_uint32_t integer; + + if (decode_tag_internal(ptag, src, 1 /* dodrain */) == -1) + return (-1); + if (evtag_decode_int(&integer, src) == -1) + return (-1); + len = integer; + + if (EVBUFFER_LENGTH(src) < len) + return (-1); + + if (evbuffer_add(dst, EVBUFFER_DATA(src), len) == -1) + return (-1); + + evbuffer_drain(src, len); + + return (len); +} + +/* Marshaling for integers */ + +int +evtag_unmarshal_int(struct evbuffer *evbuf, ev_uint32_t need_tag, + ev_uint32_t *pinteger) +{ + ev_uint32_t tag; + ev_uint32_t len; + ev_uint32_t integer; + + if (decode_tag_internal(&tag, evbuf, 1 /* dodrain */) == -1) + return (-1); + if (need_tag != tag) + return (-1); + if (evtag_decode_int(&integer, evbuf) == -1) + return (-1); + len = integer; + + if (EVBUFFER_LENGTH(evbuf) < len) + return (-1); + + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + if (evbuffer_add(_buf, EVBUFFER_DATA(evbuf), len) == -1) + return (-1); + + evbuffer_drain(evbuf, len); + + return (evtag_decode_int(pinteger, _buf)); +} + +/* Unmarshal a fixed length tag */ + +int +evtag_unmarshal_fixed(struct evbuffer *src, ev_uint32_t need_tag, void *data, + size_t len) +{ + ev_uint32_t tag; + + /* Initialize this event buffer so that we can read into it */ + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + + /* Now unmarshal a tag and check that it matches the tag we want */ + if (evtag_unmarshal(src, &tag, _buf) == -1 || tag != need_tag) + return (-1); + + if (EVBUFFER_LENGTH(_buf) != len) + return (-1); + + memcpy(data, EVBUFFER_DATA(_buf), len); + return (0); +} + +int +evtag_unmarshal_string(struct evbuffer *evbuf, ev_uint32_t need_tag, + char **pstring) +{ + ev_uint32_t tag; + + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + + if (evtag_unmarshal(evbuf, &tag, _buf) == -1 || tag != need_tag) + return (-1); + + *pstring = calloc(EVBUFFER_LENGTH(_buf) + 1, 1); + if (*pstring == NULL) + event_err(1, "%s: calloc", __func__); + evbuffer_remove(_buf, *pstring, EVBUFFER_LENGTH(_buf)); + + return (0); +} + +int +evtag_unmarshal_timeval(struct evbuffer *evbuf, ev_uint32_t need_tag, + struct timeval *ptv) +{ + ev_uint32_t tag; + ev_uint32_t integer; + + evbuffer_drain(_buf, EVBUFFER_LENGTH(_buf)); + if (evtag_unmarshal(evbuf, &tag, _buf) == -1 || tag != need_tag) + return (-1); + + if (evtag_decode_int(&integer, _buf) == -1) + return (-1); + ptv->tv_sec = integer; + if (evtag_decode_int(&integer, _buf) == -1) + return (-1); + ptv->tv_usec = integer; + + return (0); +} === added file 'extra/libevent/evhttp.h' --- a/extra/libevent/evhttp.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evhttp.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVHTTP_H_ +#define _EVHTTP_H_ + +#include <event.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <winsock2.h> +#undef WIN32_LEAN_AND_MEAN +#endif + +/** @file evhttp.h + * + * Basic support for HTTP serving. + * + * As libevent is a library for dealing with event notification and most + * interesting applications are networked today, I have often found the + * need to write HTTP code. The following prototypes and definitions provide + * an application with a minimal interface for making HTTP requests and for + * creating a very simple HTTP server. + */ + +/* Response codes */ +#define HTTP_OK 200 +#define HTTP_NOCONTENT 204 +#define HTTP_MOVEPERM 301 +#define HTTP_MOVETEMP 302 +#define HTTP_NOTMODIFIED 304 +#define HTTP_BADREQUEST 400 +#define HTTP_NOTFOUND 404 +#define HTTP_SERVUNAVAIL 503 + +struct evhttp; +struct evhttp_request; +struct evkeyvalq; + +/** Create a new HTTP server + * + * @param base (optional) the event base to receive the HTTP events + * @return a pointer to a newly initialized evhttp server structure + */ +struct evhttp *evhttp_new(struct event_base *base); + +/** + * Binds an HTTP server on the specified address and port. + * + * Can be called multiple times to bind the same http server + * to multiple different ports. + * + * @param http a pointer to an evhttp object + * @param address a string containing the IP address to listen(2) on + * @param port the port number to listen on + * @return a newly allocated evhttp struct + * @see evhttp_free() + */ +int evhttp_bind_socket(struct evhttp *http, const char *address, u_short port); + +/** + * Free the previously created HTTP server. + * + * Works only if no requests are currently being served. + * + * @param http the evhttp server object to be freed + * @see evhttp_start() + */ +void evhttp_free(struct evhttp* http); + +/** Set a callback for a specified URI */ +void evhttp_set_cb(struct evhttp *, const char *, + void (*)(struct evhttp_request *, void *), void *); + +/** Removes the callback for a specified URI */ +int evhttp_del_cb(struct evhttp *, const char *); + +/** Set a callback for all requests that are not caught by specific callbacks + */ +void evhttp_set_gencb(struct evhttp *, + void (*)(struct evhttp_request *, void *), void *); + +/** + * Set the timeout for an HTTP request. + * + * @param http an evhttp object + * @param timeout_in_secs the timeout, in seconds + */ +void evhttp_set_timeout(struct evhttp *, int timeout_in_secs); + +/* Request/Response functionality */ + +/** + * Send an HTML error message to the client. + * + * @param req a request object + * @param error the HTTP error code + * @param reason a brief explanation of the error + */ +void evhttp_send_error(struct evhttp_request *req, int error, + const char *reason); + +/** + * Send an HTML reply to the client. + * + * @param req a request object + * @param code the HTTP response code to send + * @param reason a brief message to send with the response code + * @param databuf the body of the response + */ +void evhttp_send_reply(struct evhttp_request *req, int code, + const char *reason, struct evbuffer *databuf); + +/* Low-level response interface, for streaming/chunked replies */ +void evhttp_send_reply_start(struct evhttp_request *, int, const char *); +void evhttp_send_reply_chunk(struct evhttp_request *, struct evbuffer *); +void evhttp_send_reply_end(struct evhttp_request *); + +/** + * Start an HTTP server on the specified address and port + * + * DEPRECATED: it does not allow an event base to be specified + * + * @param address the address to which the HTTP server should be bound + * @param port the port number on which the HTTP server should listen + * @return an struct evhttp object + */ +struct evhttp *evhttp_start(const char *address, u_short port); + +/* + * Interfaces for making requests + */ +enum evhttp_cmd_type { EVHTTP_REQ_GET, EVHTTP_REQ_POST, EVHTTP_REQ_HEAD }; + +enum evhttp_request_kind { EVHTTP_REQUEST, EVHTTP_RESPONSE }; + +/** + * the request structure that a server receives. + * WARNING: expect this structure to change. I will try to provide + * reasonable accessors. + */ +struct evhttp_request { +#if defined(TAILQ_ENTRY) + TAILQ_ENTRY(evhttp_request) next; +#else +struct { + struct evhttp_request *tqe_next; + struct evhttp_request **tqe_prev; +} next; +#endif + + /* the connection object that this request belongs to */ + struct evhttp_connection *evcon; + int flags; +#define EVHTTP_REQ_OWN_CONNECTION 0x0001 +#define EVHTTP_PROXY_REQUEST 0x0002 + + struct evkeyvalq *input_headers; + struct evkeyvalq *output_headers; + + /* address of the remote host and the port connection came from */ + char *remote_host; + u_short remote_port; + + enum evhttp_request_kind kind; + enum evhttp_cmd_type type; + + char *uri; /* uri after HTTP request was parsed */ + + char major; /* HTTP Major number */ + char minor; /* HTTP Minor number */ + + int got_firstline; + int response_code; /* HTTP Response code */ + char *response_code_line; /* Readable response */ + + struct evbuffer *input_buffer; /* read data */ + ev_int64_t ntoread; + int chunked; + + struct evbuffer *output_buffer; /* outgoing post or data */ + + /* Callback */ + void (*cb)(struct evhttp_request *, void *); + void *cb_arg; + + /* + * Chunked data callback - call for each completed chunk if + * specified. If not specified, all the data is delivered via + * the regular callback. + */ + void (*chunk_cb)(struct evhttp_request *, void *); +}; + +/** + * Creates a new request object that needs to be filled in with the request + * parameters. The callback is executed when the request completed or an + * error occurred. + */ +struct evhttp_request *evhttp_request_new( + void (*cb)(struct evhttp_request *, void *), void *arg); + +/** enable delivery of chunks to requestor */ +void evhttp_request_set_chunked_cb(struct evhttp_request *, + void (*cb)(struct evhttp_request *, void *)); + +/** Frees the request object and removes associated events. */ +void evhttp_request_free(struct evhttp_request *req); + +/** + * A connection object that can be used to for making HTTP requests. The + * connection object tries to establish the connection when it is given an + * http request object. + */ +struct evhttp_connection *evhttp_connection_new( + const char *address, unsigned short port); + +/** Frees an http connection */ +void evhttp_connection_free(struct evhttp_connection *evcon); + +/** sets the ip address from which http connections are made */ +void evhttp_connection_set_local_address(struct evhttp_connection *evcon, + const char *address); + +/** Sets the timeout for events related to this connection */ +void evhttp_connection_set_timeout(struct evhttp_connection *evcon, + int timeout_in_secs); + +/** Sets the retry limit for this connection - -1 repeats indefnitely */ +void evhttp_connection_set_retries(struct evhttp_connection *evcon, + int retry_max); + +/** Set a callback for connection close. */ +void evhttp_connection_set_closecb(struct evhttp_connection *evcon, + void (*)(struct evhttp_connection *, void *), void *); + +/** + * Associates an event base with the connection - can only be called + * on a freshly created connection object that has not been used yet. + */ +void evhttp_connection_set_base(struct evhttp_connection *evcon, + struct event_base *base); + +/** Get the remote address and port associated with this connection. */ +void evhttp_connection_get_peer(struct evhttp_connection *evcon, + char **address, u_short *port); + +/** The connection gets ownership of the request */ +int evhttp_make_request(struct evhttp_connection *evcon, + struct evhttp_request *req, + enum evhttp_cmd_type type, const char *uri); + +const char *evhttp_request_uri(struct evhttp_request *req); + +/* Interfaces for dealing with HTTP headers */ + +const char *evhttp_find_header(const struct evkeyvalq *, const char *); +int evhttp_remove_header(struct evkeyvalq *, const char *); +int evhttp_add_header(struct evkeyvalq *, const char *, const char *); +void evhttp_clear_headers(struct evkeyvalq *); + +/* Miscellaneous utility functions */ + + +/** + Helper function to encode a URI. + + The returned string must be freed by the caller. + + @param uri an unencoded URI + @return a newly allocated URI-encoded string + */ +char *evhttp_encode_uri(const char *uri); + + +/** + Helper function to decode a URI. + + The returned string must be freed by the caller. + + @param uri an encoded URI + @return a newly allocated unencoded URI + */ +char *evhttp_decode_uri(const char *uri); + + +/** + * Helper function to parse out arguments in a query. + * The arguments are separated by key and value. + * URI should already be decoded. + */ +void evhttp_parse_query(const char *uri, struct evkeyvalq *); + + +/** + * Escape HTML character entities in a string. + * + * Replaces <, >, ", ' and & with <, >, ", + * ' and & correspondingly. + * + * The returned string needs to be freed by the caller. + * + * @param html an unescaped HTML string + * @return an escaped HTML string + */ +char *evhttp_htmlescape(const char *html); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVHTTP_H_ */ === added file 'extra/libevent/evport.c' --- a/extra/libevent/evport.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evport.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,517 @@ +/* + * Submitted by David Pacheco (dp.spambait(a)gmail.com) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY SUN MICROSYSTEMS, INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL SUN MICROSYSTEMS, INC. BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright (c) 2007 Sun Microsystems. All rights reserved. + * Use is subject to license terms. + */ + +/* + * evport.c: event backend using Solaris 10 event ports. See port_create(3C). + * This implementation is loosely modeled after the one used for select(2) (in + * select.c). + * + * The outstanding events are tracked in a data structure called evport_data. + * Each entry in the ed_fds array corresponds to a file descriptor, and contains + * pointers to the read and write events that correspond to that fd. (That is, + * when the file is readable, the "read" event should handle it, etc.) + * + * evport_add and evport_del update this data structure. evport_dispatch uses it + * to determine where to callback when an event occurs (which it gets from + * port_getn). + * + * Helper functions are used: grow() grows the file descriptor array as + * necessary when large fd's come in. reassociate() takes care of maintaining + * the proper file-descriptor/event-port associations. + * + * As in the select(2) implementation, signals are handled by evsignal. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_EVENT_PORTS + +#include <sys/time.h> +#include <assert.h> +#include <sys/queue.h> +#include <errno.h> +#include <poll.h> +#include <port.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> +#ifdef CHECK_INVARIANTS +#include <assert.h> +#endif + +#include "event.h" +#include "event-internal.h" +#include "log.h" +#include "evsignal.h" + + +/* + * Default value for ed_nevents, which is the maximum file descriptor number we + * can handle. If an event comes in for a file descriptor F > nevents, we will + * grow the array of file descriptors, doubling its size. + */ +#define DEFAULT_NFDS 16 + + +/* + * EVENTS_PER_GETN is the maximum number of events to retrieve from port_getn on + * any particular call. You can speed things up by increasing this, but it will + * (obviously) require more memory. + */ +#define EVENTS_PER_GETN 8 + +/* + * Per-file-descriptor information about what events we're subscribed to. These + * fields are NULL if no event is subscribed to either of them. + */ + +struct fd_info { + struct event* fdi_revt; /* the event responsible for the "read" */ + struct event* fdi_wevt; /* the event responsible for the "write" */ +}; + +#define FDI_HAS_READ(fdi) ((fdi)->fdi_revt != NULL) +#define FDI_HAS_WRITE(fdi) ((fdi)->fdi_wevt != NULL) +#define FDI_HAS_EVENTS(fdi) (FDI_HAS_READ(fdi) || FDI_HAS_WRITE(fdi)) +#define FDI_TO_SYSEVENTS(fdi) (FDI_HAS_READ(fdi) ? POLLIN : 0) | \ + (FDI_HAS_WRITE(fdi) ? POLLOUT : 0) + +struct evport_data { + int ed_port; /* event port for system events */ + int ed_nevents; /* number of allocated fdi's */ + struct fd_info *ed_fds; /* allocated fdi table */ + /* fdi's that we need to reassoc */ + int ed_pending[EVENTS_PER_GETN]; /* fd's with pending events */ +}; + +static void* evport_init (struct event_base *); +static int evport_add (void *, struct event *); +static int evport_del (void *, struct event *); +static int evport_dispatch (struct event_base *, void *, struct timeval *); +static void evport_dealloc (struct event_base *, void *); + +const struct eventop evportops = { + "event ports", + evport_init, + evport_add, + evport_del, + evport_dispatch, + evport_dealloc, + 1 /* need reinit */ +}; + +/* + * Initialize the event port implementation. + */ + +static void* +evport_init(struct event_base *base) +{ + struct evport_data *evpd; + int i; + /* + * Disable event ports when this environment variable is set + */ + if (getenv("EVENT_NOEVPORT")) + return (NULL); + + if (!(evpd = calloc(1, sizeof(struct evport_data)))) + return (NULL); + + if ((evpd->ed_port = port_create()) == -1) { + free(evpd); + return (NULL); + } + + /* + * Initialize file descriptor structure + */ + evpd->ed_fds = calloc(DEFAULT_NFDS, sizeof(struct fd_info)); + if (evpd->ed_fds == NULL) { + close(evpd->ed_port); + free(evpd); + return (NULL); + } + evpd->ed_nevents = DEFAULT_NFDS; + for (i = 0; i < EVENTS_PER_GETN; i++) + evpd->ed_pending[i] = -1; + + evsignal_init(base); + + return (evpd); +} + +#ifdef CHECK_INVARIANTS +/* + * Checks some basic properties about the evport_data structure. Because it + * checks all file descriptors, this function can be expensive when the maximum + * file descriptor ever used is rather large. + */ + +static void +check_evportop(struct evport_data *evpd) +{ + assert(evpd); + assert(evpd->ed_nevents > 0); + assert(evpd->ed_port > 0); + assert(evpd->ed_fds > 0); + + /* + * Verify the integrity of the fd_info struct as well as the events to + * which it points (at least, that they're valid references and correct + * for their position in the structure). + */ + int i; + for (i = 0; i < evpd->ed_nevents; ++i) { + struct event *ev; + struct fd_info *fdi; + + fdi = &evpd->ed_fds[i]; + if ((ev = fdi->fdi_revt) != NULL) { + assert(ev->ev_fd == i); + } + if ((ev = fdi->fdi_wevt) != NULL) { + assert(ev->ev_fd == i); + } + } +} + +/* + * Verifies very basic integrity of a given port_event. + */ +static void +check_event(port_event_t* pevt) +{ + /* + * We've only registered for PORT_SOURCE_FD events. The only + * other thing we can legitimately receive is PORT_SOURCE_ALERT, + * but since we're not using port_alert either, we can assume + * PORT_SOURCE_FD. + */ + assert(pevt->portev_source == PORT_SOURCE_FD); + assert(pevt->portev_user == NULL); +} + +#else +#define check_evportop(epop) +#define check_event(pevt) +#endif /* CHECK_INVARIANTS */ + +/* + * Doubles the size of the allocated file descriptor array. + */ +static int +grow(struct evport_data *epdp, int factor) +{ + struct fd_info *tmp; + int oldsize = epdp->ed_nevents; + int newsize = factor * oldsize; + assert(factor > 1); + + check_evportop(epdp); + + tmp = realloc(epdp->ed_fds, sizeof(struct fd_info) * newsize); + if (NULL == tmp) + return -1; + epdp->ed_fds = tmp; + memset((char*) (epdp->ed_fds + oldsize), 0, + (newsize - oldsize)*sizeof(struct fd_info)); + epdp->ed_nevents = newsize; + + check_evportop(epdp); + + return 0; +} + + +/* + * (Re)associates the given file descriptor with the event port. The OS events + * are specified (implicitly) from the fd_info struct. + */ +static int +reassociate(struct evport_data *epdp, struct fd_info *fdip, int fd) +{ + int sysevents = FDI_TO_SYSEVENTS(fdip); + + if (sysevents != 0) { + if (port_associate(epdp->ed_port, PORT_SOURCE_FD, + fd, sysevents, NULL) == -1) { + event_warn("port_associate"); + return (-1); + } + } + + check_evportop(epdp); + + return (0); +} + +/* + * Main event loop - polls port_getn for some number of events, and processes + * them. + */ + +static int +evport_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + int i, res; + struct evport_data *epdp = arg; + port_event_t pevtlist[EVENTS_PER_GETN]; + + /* + * port_getn will block until it has at least nevents events. It will + * also return how many it's given us (which may be more than we asked + * for, as long as it's less than our maximum (EVENTS_PER_GETN)) in + * nevents. + */ + int nevents = 1; + + /* + * We have to convert a struct timeval to a struct timespec + * (only difference is nanoseconds vs. microseconds). If no time-based + * events are active, we should wait for I/O (and tv == NULL). + */ + struct timespec ts; + struct timespec *ts_p = NULL; + if (tv != NULL) { + ts.tv_sec = tv->tv_sec; + ts.tv_nsec = tv->tv_usec * 1000; + ts_p = &ts; + } + + /* + * Before doing anything else, we need to reassociate the events we hit + * last time which need reassociation. See comment at the end of the + * loop below. + */ + for (i = 0; i < EVENTS_PER_GETN; ++i) { + struct fd_info *fdi = NULL; + if (epdp->ed_pending[i] != -1) { + fdi = &(epdp->ed_fds[epdp->ed_pending[i]]); + } + + if (fdi != NULL && FDI_HAS_EVENTS(fdi)) { + int fd = FDI_HAS_READ(fdi) ? fdi->fdi_revt->ev_fd : + fdi->fdi_wevt->ev_fd; + reassociate(epdp, fdi, fd); + epdp->ed_pending[i] = -1; + } + } + + if ((res = port_getn(epdp->ed_port, pevtlist, EVENTS_PER_GETN, + (unsigned int *) &nevents, ts_p)) == -1) { + if (errno == EINTR || errno == EAGAIN) { + evsignal_process(base); + return (0); + } else if (errno == ETIME) { + if (nevents == 0) + return (0); + } else { + event_warn("port_getn"); + return (-1); + } + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + event_debug(("%s: port_getn reports %d events", __func__, nevents)); + + for (i = 0; i < nevents; ++i) { + struct event *ev; + struct fd_info *fdi; + port_event_t *pevt = &pevtlist[i]; + int fd = (int) pevt->portev_object; + + check_evportop(epdp); + check_event(pevt); + epdp->ed_pending[i] = fd; + + /* + * Figure out what kind of event it was + * (because we have to pass this to the callback) + */ + res = 0; + if (pevt->portev_events & POLLIN) + res |= EV_READ; + if (pevt->portev_events & POLLOUT) + res |= EV_WRITE; + + assert(epdp->ed_nevents > fd); + fdi = &(epdp->ed_fds[fd]); + + /* + * We now check for each of the possible events (READ + * or WRITE). Then, we activate the event (which will + * cause its callback to be executed). + */ + + if ((res & EV_READ) && ((ev = fdi->fdi_revt) != NULL)) { + event_active(ev, res, 1); + } + + if ((res & EV_WRITE) && ((ev = fdi->fdi_wevt) != NULL)) { + event_active(ev, res, 1); + } + } /* end of all events gotten */ + + check_evportop(epdp); + + return (0); +} + + +/* + * Adds the given event (so that you will be notified when it happens via + * the callback function). + */ + +static int +evport_add(void *arg, struct event *ev) +{ + struct evport_data *evpd = arg; + struct fd_info *fdi; + int factor; + + check_evportop(evpd); + + /* + * Delegate, if it's not ours to handle. + */ + if (ev->ev_events & EV_SIGNAL) + return (evsignal_add(ev)); + + /* + * If necessary, grow the file descriptor info table + */ + + factor = 1; + while (ev->ev_fd >= factor * evpd->ed_nevents) + factor *= 2; + + if (factor > 1) { + if (-1 == grow(evpd, factor)) { + return (-1); + } + } + + fdi = &evpd->ed_fds[ev->ev_fd]; + if (ev->ev_events & EV_READ) + fdi->fdi_revt = ev; + if (ev->ev_events & EV_WRITE) + fdi->fdi_wevt = ev; + + return reassociate(evpd, fdi, ev->ev_fd); +} + +/* + * Removes the given event from the list of events to wait for. + */ + +static int +evport_del(void *arg, struct event *ev) +{ + struct evport_data *evpd = arg; + struct fd_info *fdi; + int i; + int associated = 1; + + check_evportop(evpd); + + /* + * Delegate, if it's not ours to handle + */ + if (ev->ev_events & EV_SIGNAL) { + return (evsignal_del(ev)); + } + + if (evpd->ed_nevents < ev->ev_fd) { + return (-1); + } + + for (i = 0; i < EVENTS_PER_GETN; ++i) { + if (evpd->ed_pending[i] == ev->ev_fd) { + associated = 0; + break; + } + } + + fdi = &evpd->ed_fds[ev->ev_fd]; + if (ev->ev_events & EV_READ) + fdi->fdi_revt = NULL; + if (ev->ev_events & EV_WRITE) + fdi->fdi_wevt = NULL; + + if (associated) { + if (!FDI_HAS_EVENTS(fdi) && + port_dissociate(evpd->ed_port, PORT_SOURCE_FD, + ev->ev_fd) == -1) { + /* + * Ignre EBADFD error the fd could have been closed + * before event_del() was called. + */ + if (errno != EBADFD) { + event_warn("port_dissociate"); + return (-1); + } + } else { + if (FDI_HAS_EVENTS(fdi)) { + return (reassociate(evpd, fdi, ev->ev_fd)); + } + } + } else { + if (fdi->fdi_revt == NULL && fdi->fdi_wevt == NULL) { + evpd->ed_pending[i] = -1; + } + } + return 0; +} + + +static void +evport_dealloc(struct event_base *base, void *arg) +{ + struct evport_data *evpd = arg; + + evsignal_dealloc(base); + + close(evpd->ed_port); + + if (evpd->ed_fds) + free(evpd->ed_fds); + free(evpd); +} + +#endif /* HAVE_EVENT_PORTS */ === added file 'extra/libevent/evrpc-internal.h' --- a/extra/libevent/evrpc-internal.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evrpc-internal.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2006 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVRPC_INTERNAL_H_ +#define _EVRPC_INTERNAL_H_ + +#include "http-internal.h" + +struct evrpc; + +#define EVRPC_URI_PREFIX "/.rpc." + +struct evrpc_hook { + TAILQ_ENTRY(evrpc_hook) (next); + + /* returns -1; if the rpc should be aborted, is allowed to rewrite */ + int (*process)(struct evhttp_request *, struct evbuffer *, void *); + void *process_arg; +}; + +TAILQ_HEAD(evrpc_hook_list, evrpc_hook); + +/* + * this is shared between the base and the pool, so that we can reuse + * the hook adding functions; we alias both evrpc_pool and evrpc_base + * to this common structure. + */ +struct _evrpc_hooks { + /* hooks for processing outbound and inbound rpcs */ + struct evrpc_hook_list in_hooks; + struct evrpc_hook_list out_hooks; +}; + +#define input_hooks common.in_hooks +#define output_hooks common.out_hooks + +struct evrpc_base { + struct _evrpc_hooks common; + + /* the HTTP server under which we register our RPC calls */ + struct evhttp* http_server; + + /* a list of all RPCs registered with us */ + TAILQ_HEAD(evrpc_list, evrpc) registered_rpcs; +}; + +struct evrpc_req_generic; +void evrpc_reqstate_free(struct evrpc_req_generic* rpc_state); + +/* A pool for holding evhttp_connection objects */ +struct evrpc_pool { + struct _evrpc_hooks common; + + struct event_base *base; + + struct evconq connections; + + int timeout; + + TAILQ_HEAD(evrpc_requestq, evrpc_request_wrapper) requests; +}; + + +#endif /* _EVRPC_INTERNAL_H_ */ === added file 'extra/libevent/evrpc.c' --- a/extra/libevent/evrpc.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evrpc.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,658 @@ +/* + * Copyright (c) 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <winsock2.h> +#undef WIN32_LEAN_AND_MEAN +#include "misc.h" +#endif + +#include <sys/types.h> +#ifndef WIN32 +#include <sys/socket.h> +#endif +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#include <sys/queue.h> +#include <stdio.h> +#include <stdlib.h> +#ifndef WIN32 +#include <unistd.h> +#endif +#include <errno.h> +#include <signal.h> +#include <string.h> +#include <assert.h> + +#include "event.h" +#include "evrpc.h" +#include "evrpc-internal.h" +#include "evhttp.h" +#include "evutil.h" +#include "log.h" + +struct evrpc_base * +evrpc_init(struct evhttp *http_server) +{ + struct evrpc_base* base = calloc(1, sizeof(struct evrpc_base)); + if (base == NULL) + return (NULL); + + /* we rely on the tagging sub system */ + evtag_init(); + + TAILQ_INIT(&base->registered_rpcs); + TAILQ_INIT(&base->input_hooks); + TAILQ_INIT(&base->output_hooks); + base->http_server = http_server; + + return (base); +} + +void +evrpc_free(struct evrpc_base *base) +{ + struct evrpc *rpc; + struct evrpc_hook *hook; + + while ((rpc = TAILQ_FIRST(&base->registered_rpcs)) != NULL) { + assert(evrpc_unregister_rpc(base, rpc->uri)); + } + while ((hook = TAILQ_FIRST(&base->input_hooks)) != NULL) { + assert(evrpc_remove_hook(base, INPUT, hook)); + } + while ((hook = TAILQ_FIRST(&base->output_hooks)) != NULL) { + assert(evrpc_remove_hook(base, OUTPUT, hook)); + } + free(base); +} + +void * +evrpc_add_hook(void *vbase, + enum EVRPC_HOOK_TYPE hook_type, + int (*cb)(struct evhttp_request *, struct evbuffer *, void *), + void *cb_arg) +{ + struct _evrpc_hooks *base = vbase; + struct evrpc_hook_list *head = NULL; + struct evrpc_hook *hook = NULL; + switch (hook_type) { + case INPUT: + head = &base->in_hooks; + break; + case OUTPUT: + head = &base->out_hooks; + break; + default: + assert(hook_type == INPUT || hook_type == OUTPUT); + } + + hook = calloc(1, sizeof(struct evrpc_hook)); + assert(hook != NULL); + + hook->process = cb; + hook->process_arg = cb_arg; + TAILQ_INSERT_TAIL(head, hook, next); + + return (hook); +} + +static int +evrpc_remove_hook_internal(struct evrpc_hook_list *head, void *handle) +{ + struct evrpc_hook *hook = NULL; + TAILQ_FOREACH(hook, head, next) { + if (hook == handle) { + TAILQ_REMOVE(head, hook, next); + free(hook); + return (1); + } + } + + return (0); +} + +/* + * remove the hook specified by the handle + */ + +int +evrpc_remove_hook(void *vbase, enum EVRPC_HOOK_TYPE hook_type, void *handle) +{ + struct _evrpc_hooks *base = vbase; + struct evrpc_hook_list *head = NULL; + switch (hook_type) { + case INPUT: + head = &base->in_hooks; + break; + case OUTPUT: + head = &base->out_hooks; + break; + default: + assert(hook_type == INPUT || hook_type == OUTPUT); + } + + return (evrpc_remove_hook_internal(head, handle)); +} + +static int +evrpc_process_hooks(struct evrpc_hook_list *head, + struct evhttp_request *req, struct evbuffer *evbuf) +{ + struct evrpc_hook *hook; + TAILQ_FOREACH(hook, head, next) { + if (hook->process(req, evbuf, hook->process_arg) == -1) + return (-1); + } + + return (0); +} + +static void evrpc_pool_schedule(struct evrpc_pool *pool); +static void evrpc_request_cb(struct evhttp_request *, void *); +void evrpc_request_done(struct evrpc_req_generic*); + +/* + * Registers a new RPC with the HTTP server. The evrpc object is expected + * to have been filled in via the EVRPC_REGISTER_OBJECT macro which in turn + * calls this function. + */ + +static char * +evrpc_construct_uri(const char *uri) +{ + char *constructed_uri; + int constructed_uri_len; + + constructed_uri_len = strlen(EVRPC_URI_PREFIX) + strlen(uri) + 1; + if ((constructed_uri = malloc(constructed_uri_len)) == NULL) + event_err(1, "%s: failed to register rpc at %s", + __func__, uri); + memcpy(constructed_uri, EVRPC_URI_PREFIX, strlen(EVRPC_URI_PREFIX)); + memcpy(constructed_uri + strlen(EVRPC_URI_PREFIX), uri, strlen(uri)); + constructed_uri[constructed_uri_len - 1] = '\0'; + + return (constructed_uri); +} + +int +evrpc_register_rpc(struct evrpc_base *base, struct evrpc *rpc, + void (*cb)(struct evrpc_req_generic *, void *), void *cb_arg) +{ + char *constructed_uri = evrpc_construct_uri(rpc->uri); + + rpc->base = base; + rpc->cb = cb; + rpc->cb_arg = cb_arg; + + TAILQ_INSERT_TAIL(&base->registered_rpcs, rpc, next); + + evhttp_set_cb(base->http_server, + constructed_uri, + evrpc_request_cb, + rpc); + + free(constructed_uri); + + return (0); +} + +int +evrpc_unregister_rpc(struct evrpc_base *base, const char *name) +{ + char *registered_uri = NULL; + struct evrpc *rpc; + + /* find the right rpc; linear search might be slow */ + TAILQ_FOREACH(rpc, &base->registered_rpcs, next) { + if (strcmp(rpc->uri, name) == 0) + break; + } + if (rpc == NULL) { + /* We did not find an RPC with this name */ + return (-1); + } + TAILQ_REMOVE(&base->registered_rpcs, rpc, next); + + free((char *)rpc->uri); + free(rpc); + + registered_uri = evrpc_construct_uri(name); + + /* remove the http server callback */ + assert(evhttp_del_cb(base->http_server, registered_uri) == 0); + + free(registered_uri); + return (0); +} + +static void +evrpc_request_cb(struct evhttp_request *req, void *arg) +{ + struct evrpc *rpc = arg; + struct evrpc_req_generic *rpc_state = NULL; + + /* let's verify the outside parameters */ + if (req->type != EVHTTP_REQ_POST || + EVBUFFER_LENGTH(req->input_buffer) <= 0) + goto error; + + /* + * we might want to allow hooks to suspend the processing, + * but at the moment, we assume that they just act as simple + * filters. + */ + if (evrpc_process_hooks(&rpc->base->input_hooks, + req, req->input_buffer) == -1) + goto error; + + rpc_state = calloc(1, sizeof(struct evrpc_req_generic)); + if (rpc_state == NULL) + goto error; + + /* let's check that we can parse the request */ + rpc_state->request = rpc->request_new(); + if (rpc_state->request == NULL) + goto error; + + rpc_state->rpc = rpc; + + if (rpc->request_unmarshal( + rpc_state->request, req->input_buffer) == -1) { + /* we failed to parse the request; that's a bummer */ + goto error; + } + + /* at this point, we have a well formed request, prepare the reply */ + + rpc_state->reply = rpc->reply_new(); + if (rpc_state->reply == NULL) + goto error; + + rpc_state->http_req = req; + rpc_state->done = evrpc_request_done; + + /* give the rpc to the user; they can deal with it */ + rpc->cb(rpc_state, rpc->cb_arg); + + return; + +error: + evrpc_reqstate_free(rpc_state); + evhttp_send_error(req, HTTP_SERVUNAVAIL, "Service Error"); + return; +} + +void +evrpc_reqstate_free(struct evrpc_req_generic* rpc_state) +{ + /* clean up all memory */ + if (rpc_state != NULL) { + struct evrpc *rpc = rpc_state->rpc; + + if (rpc_state->request != NULL) + rpc->request_free(rpc_state->request); + if (rpc_state->reply != NULL) + rpc->reply_free(rpc_state->reply); + free(rpc_state); + } +} + +void +evrpc_request_done(struct evrpc_req_generic* rpc_state) +{ + struct evhttp_request *req = rpc_state->http_req; + struct evrpc *rpc = rpc_state->rpc; + struct evbuffer* data = NULL; + + if (rpc->reply_complete(rpc_state->reply) == -1) { + /* the reply was not completely filled in. error out */ + goto error; + } + + if ((data = evbuffer_new()) == NULL) { + /* out of memory */ + goto error; + } + + /* serialize the reply */ + rpc->reply_marshal(data, rpc_state->reply); + + /* do hook based tweaks to the request */ + if (evrpc_process_hooks(&rpc->base->output_hooks, + req, data) == -1) + goto error; + + /* on success, we are going to transmit marshaled binary data */ + if (evhttp_find_header(req->output_headers, "Content-Type") == NULL) { + evhttp_add_header(req->output_headers, + "Content-Type", "application/octet-stream"); + } + + evhttp_send_reply(req, HTTP_OK, "OK", data); + + evbuffer_free(data); + + evrpc_reqstate_free(rpc_state); + + return; + +error: + if (data != NULL) + evbuffer_free(data); + evrpc_reqstate_free(rpc_state); + evhttp_send_error(req, HTTP_SERVUNAVAIL, "Service Error"); + return; +} + +/* Client implementation of RPC site */ + +static int evrpc_schedule_request(struct evhttp_connection *connection, + struct evrpc_request_wrapper *ctx); + +struct evrpc_pool * +evrpc_pool_new(struct event_base *base) +{ + struct evrpc_pool *pool = calloc(1, sizeof(struct evrpc_pool)); + if (pool == NULL) + return (NULL); + + TAILQ_INIT(&pool->connections); + TAILQ_INIT(&pool->requests); + + TAILQ_INIT(&pool->input_hooks); + TAILQ_INIT(&pool->output_hooks); + + pool->base = base; + pool->timeout = -1; + + return (pool); +} + +static void +evrpc_request_wrapper_free(struct evrpc_request_wrapper *request) +{ + free(request->name); + free(request); +} + +void +evrpc_pool_free(struct evrpc_pool *pool) +{ + struct evhttp_connection *connection; + struct evrpc_request_wrapper *request; + struct evrpc_hook *hook; + + while ((request = TAILQ_FIRST(&pool->requests)) != NULL) { + TAILQ_REMOVE(&pool->requests, request, next); + /* if this gets more complicated we need our own function */ + evrpc_request_wrapper_free(request); + } + + while ((connection = TAILQ_FIRST(&pool->connections)) != NULL) { + TAILQ_REMOVE(&pool->connections, connection, next); + evhttp_connection_free(connection); + } + + while ((hook = TAILQ_FIRST(&pool->input_hooks)) != NULL) { + assert(evrpc_remove_hook(pool, INPUT, hook)); + } + + while ((hook = TAILQ_FIRST(&pool->output_hooks)) != NULL) { + assert(evrpc_remove_hook(pool, OUTPUT, hook)); + } + + free(pool); +} + +/* + * Add a connection to the RPC pool. A request scheduled on the pool + * may use any available connection. + */ + +void +evrpc_pool_add_connection(struct evrpc_pool *pool, + struct evhttp_connection *connection) { + assert(connection->http_server == NULL); + TAILQ_INSERT_TAIL(&pool->connections, connection, next); + + /* + * associate an event base with this connection + */ + if (pool->base != NULL) + evhttp_connection_set_base(connection, pool->base); + + /* + * unless a timeout was specifically set for a connection, + * the connection inherits the timeout from the pool. + */ + if (connection->timeout == -1) + connection->timeout = pool->timeout; + + /* + * if we have any requests pending, schedule them with the new + * connections. + */ + + if (TAILQ_FIRST(&pool->requests) != NULL) { + struct evrpc_request_wrapper *request = + TAILQ_FIRST(&pool->requests); + TAILQ_REMOVE(&pool->requests, request, next); + evrpc_schedule_request(connection, request); + } +} + +void +evrpc_pool_set_timeout(struct evrpc_pool *pool, int timeout_in_secs) +{ + struct evhttp_connection *evcon; + TAILQ_FOREACH(evcon, &pool->connections, next) { + evcon->timeout = timeout_in_secs; + } + pool->timeout = timeout_in_secs; +} + + +static void evrpc_reply_done(struct evhttp_request *, void *); +static void evrpc_request_timeout(int, short, void *); + +/* + * Finds a connection object associated with the pool that is currently + * idle and can be used to make a request. + */ +static struct evhttp_connection * +evrpc_pool_find_connection(struct evrpc_pool *pool) +{ + struct evhttp_connection *connection; + TAILQ_FOREACH(connection, &pool->connections, next) { + if (TAILQ_FIRST(&connection->requests) == NULL) + return (connection); + } + + return (NULL); +} + +/* + * We assume that the ctx is no longer queued on the pool. + */ +static int +evrpc_schedule_request(struct evhttp_connection *connection, + struct evrpc_request_wrapper *ctx) +{ + struct evhttp_request *req = NULL; + struct evrpc_pool *pool = ctx->pool; + struct evrpc_status status; + char *uri = NULL; + int res = 0; + + if ((req = evhttp_request_new(evrpc_reply_done, ctx)) == NULL) + goto error; + + /* serialize the request data into the output buffer */ + ctx->request_marshal(req->output_buffer, ctx->request); + + uri = evrpc_construct_uri(ctx->name); + if (uri == NULL) + goto error; + + /* we need to know the connection that we might have to abort */ + ctx->evcon = connection; + + /* apply hooks to the outgoing request */ + if (evrpc_process_hooks(&pool->output_hooks, + req, req->output_buffer) == -1) + goto error; + + if (pool->timeout > 0) { + /* + * a timeout after which the whole rpc is going to be aborted. + */ + struct timeval tv; + evutil_timerclear(&tv); + tv.tv_sec = pool->timeout; + evtimer_add(&ctx->ev_timeout, &tv); + } + + /* start the request over the connection */ + res = evhttp_make_request(connection, req, EVHTTP_REQ_POST, uri); + free(uri); + + if (res == -1) + goto error; + + return (0); + +error: + memset(&status, 0, sizeof(status)); + status.error = EVRPC_STATUS_ERR_UNSTARTED; + (*ctx->cb)(&status, ctx->request, ctx->reply, ctx->cb_arg); + evrpc_request_wrapper_free(ctx); + return (-1); +} + +int +evrpc_make_request(struct evrpc_request_wrapper *ctx) +{ + struct evrpc_pool *pool = ctx->pool; + + /* initialize the event structure for this rpc */ + evtimer_set(&ctx->ev_timeout, evrpc_request_timeout, ctx); + if (pool->base != NULL) + event_base_set(pool->base, &ctx->ev_timeout); + + /* we better have some available connections on the pool */ + assert(TAILQ_FIRST(&pool->connections) != NULL); + + /* + * if no connection is available, we queue the request on the pool, + * the next time a connection is empty, the rpc will be send on that. + */ + TAILQ_INSERT_TAIL(&pool->requests, ctx, next); + + evrpc_pool_schedule(pool); + + return (0); +} + +static void +evrpc_reply_done(struct evhttp_request *req, void *arg) +{ + struct evrpc_request_wrapper *ctx = arg; + struct evrpc_pool *pool = ctx->pool; + struct evrpc_status status; + int res = -1; + + /* cancel any timeout we might have scheduled */ + event_del(&ctx->ev_timeout); + + memset(&status, 0, sizeof(status)); + status.http_req = req; + + /* we need to get the reply now */ + if (req != NULL) { + /* apply hooks to the incoming request */ + if (evrpc_process_hooks(&pool->input_hooks, + req, req->input_buffer) == -1) { + status.error = EVRPC_STATUS_ERR_HOOKABORTED; + res = -1; + } else { + res = ctx->reply_unmarshal(ctx->reply, + req->input_buffer); + if (res == -1) { + status.error = EVRPC_STATUS_ERR_BADPAYLOAD; + } + } + } else { + status.error = EVRPC_STATUS_ERR_TIMEOUT; + } + + if (res == -1) { + /* clear everything that we might have written previously */ + ctx->reply_clear(ctx->reply); + } + + (*ctx->cb)(&status, ctx->request, ctx->reply, ctx->cb_arg); + + evrpc_request_wrapper_free(ctx); + + /* the http layer owns the request structure */ + + /* see if we can schedule another request */ + evrpc_pool_schedule(pool); +} + +static void +evrpc_pool_schedule(struct evrpc_pool *pool) +{ + struct evrpc_request_wrapper *ctx = TAILQ_FIRST(&pool->requests); + struct evhttp_connection *evcon; + + /* if no requests are pending, we have no work */ + if (ctx == NULL) + return; + + if ((evcon = evrpc_pool_find_connection(pool)) != NULL) { + TAILQ_REMOVE(&pool->requests, ctx, next); + evrpc_schedule_request(evcon, ctx); + } +} + +static void +evrpc_request_timeout(int fd, short what, void *arg) +{ + struct evrpc_request_wrapper *ctx = arg; + struct evhttp_connection *evcon = ctx->evcon; + assert(evcon != NULL); + + evhttp_connection_fail(evcon, EVCON_HTTP_TIMEOUT); +} === added file 'extra/libevent/evrpc.h' --- a/extra/libevent/evrpc.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evrpc.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2006 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVRPC_H_ +#define _EVRPC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @file evrpc.h + * + * This header files provides basic support for an RPC server and client. + * + * To support RPCs in a server, every supported RPC command needs to be + * defined and registered. + * + * EVRPC_HEADER(SendCommand, Request, Reply); + * + * SendCommand is the name of the RPC command. + * Request is the name of a structure generated by event_rpcgen.py. + * It contains all parameters relating to the SendCommand RPC. The + * server needs to fill in the Reply structure. + * Reply is the name of a structure generated by event_rpcgen.py. It + * contains the answer to the RPC. + * + * To register an RPC with an HTTP server, you need to first create an RPC + * base with: + * + * struct evrpc_base *base = evrpc_init(http); + * + * A specific RPC can then be registered with + * + * EVRPC_REGISTER(base, SendCommand, Request, Reply, FunctionCB, arg); + * + * when the server receives an appropriately formatted RPC, the user callback + * is invokved. The callback needs to fill in the reply structure. + * + * void FunctionCB(EVRPC_STRUCT(SendCommand)* rpc, void *arg); + * + * To send the reply, call EVRPC_REQUEST_DONE(rpc); + * + * See the regression test for an example. + */ + +struct evbuffer; +struct event_base; +struct evrpc_req_generic; + +/* Encapsulates a request */ +struct evrpc { + TAILQ_ENTRY(evrpc) next; + + /* the URI at which the request handler lives */ + const char* uri; + + /* creates a new request structure */ + void *(*request_new)(void); + + /* frees the request structure */ + void (*request_free)(void *); + + /* unmarshals the buffer into the proper request structure */ + int (*request_unmarshal)(void *, struct evbuffer *); + + /* creates a new reply structure */ + void *(*reply_new)(void); + + /* creates a new reply structure */ + void (*reply_free)(void *); + + /* verifies that the reply is valid */ + int (*reply_complete)(void *); + + /* marshals the reply into a buffer */ + void (*reply_marshal)(struct evbuffer*, void *); + + /* the callback invoked for each received rpc */ + void (*cb)(struct evrpc_req_generic *, void *); + void *cb_arg; + + /* reference for further configuration */ + struct evrpc_base *base; +}; + +/** The type of a specific RPC Message + * + * @param rpcname the name of the RPC message + */ +#define EVRPC_STRUCT(rpcname) struct evrpc_req__##rpcname + +struct evhttp_request; +struct evrpc_status; + +/* We alias the RPC specific structs to this voided one */ +struct evrpc_req_generic { + /* the unmarshaled request object */ + void *request; + + /* the empty reply object that needs to be filled in */ + void *reply; + + /* + * the static structure for this rpc; that can be used to + * automatically unmarshal and marshal the http buffers. + */ + struct evrpc *rpc; + + /* + * the http request structure on which we need to answer. + */ + struct evhttp_request* http_req; + + /* + * callback to reply and finish answering this rpc + */ + void (*done)(struct evrpc_req_generic* rpc); +}; + +/** Creates the definitions and prototypes for an RPC + * + * You need to use EVRPC_HEADER to create structures and function prototypes + * needed by the server and client implementation. The structures have to be + * defined in an .rpc file and converted to source code via event_rpcgen.py + * + * @param rpcname the name of the RPC + * @param reqstruct the name of the RPC request structure + * @param replystruct the name of the RPC reply structure + * @see EVRPC_GENERATE() + */ +#define EVRPC_HEADER(rpcname, reqstruct, rplystruct) \ +EVRPC_STRUCT(rpcname) { \ + struct reqstruct* request; \ + struct rplystruct* reply; \ + struct evrpc* rpc; \ + struct evhttp_request* http_req; \ + void (*done)(struct evrpc_status *, \ + struct evrpc* rpc, void *request, void *reply); \ +}; \ +int evrpc_send_request_##rpcname(struct evrpc_pool *, \ + struct reqstruct *, struct rplystruct *, \ + void (*)(struct evrpc_status *, \ + struct reqstruct *, struct rplystruct *, void *cbarg), \ + void *); + +/** Generates the code for receiving and sending an RPC message + * + * EVRPC_GENERATE is used to create the code corresponding to sending + * and receiving a particular RPC message + * + * @param rpcname the name of the RPC + * @param reqstruct the name of the RPC request structure + * @param replystruct the name of the RPC reply structure + * @see EVRPC_HEADER() + */ +#define EVRPC_GENERATE(rpcname, reqstruct, rplystruct) \ +int evrpc_send_request_##rpcname(struct evrpc_pool *pool, \ + struct reqstruct *request, struct rplystruct *reply, \ + void (*cb)(struct evrpc_status *, \ + struct reqstruct *, struct rplystruct *, void *cbarg), \ + void *cbarg) { \ + struct evrpc_status status; \ + struct evrpc_request_wrapper *ctx; \ + ctx = (struct evrpc_request_wrapper *) \ + malloc(sizeof(struct evrpc_request_wrapper)); \ + if (ctx == NULL) \ + goto error; \ + ctx->pool = pool; \ + ctx->evcon = NULL; \ + ctx->name = strdup(#rpcname); \ + if (ctx->name == NULL) { \ + free(ctx); \ + goto error; \ + } \ + ctx->cb = (void (*)(struct evrpc_status *, \ + void *, void *, void *))cb; \ + ctx->cb_arg = cbarg; \ + ctx->request = (void *)request; \ + ctx->reply = (void *)reply; \ + ctx->request_marshal = (void (*)(struct evbuffer *, void *))reqstruct##_marshal; \ + ctx->reply_clear = (void (*)(void *))rplystruct##_clear; \ + ctx->reply_unmarshal = (int (*)(void *, struct evbuffer *))rplystruct##_unmarshal; \ + return (evrpc_make_request(ctx)); \ +error: \ + memset(&status, 0, sizeof(status)); \ + status.error = EVRPC_STATUS_ERR_UNSTARTED; \ + (*(cb))(&status, request, reply, cbarg); \ + return (-1); \ +} + +/** Provides access to the HTTP request object underlying an RPC + * + * Access to the underlying http object; can be used to look at headers or + * for getting the remote ip address + * + * @param rpc_req the rpc request structure provided to the server callback + * @return an struct evhttp_request object that can be inspected for + * HTTP headers or sender information. + */ +#define EVRPC_REQUEST_HTTP(rpc_req) (rpc_req)->http_req + +/** Creates the reply to an RPC request + * + * EVRPC_REQUEST_DONE is used to answer a request; the reply is expected + * to have been filled in. The request and reply pointers become invalid + * after this call has finished. + * + * @param rpc_req the rpc request structure provided to the server callback + */ +#define EVRPC_REQUEST_DONE(rpc_req) do { \ + struct evrpc_req_generic *_req = (struct evrpc_req_generic *)(rpc_req); \ + _req->done(_req); \ +} while (0) + + +/* Takes a request object and fills it in with the right magic */ +#define EVRPC_REGISTER_OBJECT(rpc, name, request, reply) \ + do { \ + (rpc)->uri = strdup(#name); \ + if ((rpc)->uri == NULL) { \ + fprintf(stderr, "failed to register object\n"); \ + exit(1); \ + } \ + (rpc)->request_new = (void *(*)(void))request##_new; \ + (rpc)->request_free = (void (*)(void *))request##_free; \ + (rpc)->request_unmarshal = (int (*)(void *, struct evbuffer *))request##_unmarshal; \ + (rpc)->reply_new = (void *(*)(void))reply##_new; \ + (rpc)->reply_free = (void (*)(void *))reply##_free; \ + (rpc)->reply_complete = (int (*)(void *))reply##_complete; \ + (rpc)->reply_marshal = (void (*)(struct evbuffer*, void *))reply##_marshal; \ + } while (0) + +struct evrpc_base; +struct evhttp; + +/* functions to start up the rpc system */ + +/** Creates a new rpc base from which RPC requests can be received + * + * @param server a pointer to an existing HTTP server + * @return a newly allocated evrpc_base struct + * @see evrpc_free() + */ +struct evrpc_base *evrpc_init(struct evhttp *server); + +/** + * Frees the evrpc base + * + * For now, you are responsible for making sure that no rpcs are ongoing. + * + * @param base the evrpc_base object to be freed + * @see evrpc_init + */ +void evrpc_free(struct evrpc_base *base); + +/** register RPCs with the HTTP Server + * + * registers a new RPC with the HTTP server, each RPC needs to have + * a unique name under which it can be identified. + * + * @param base the evrpc_base structure in which the RPC should be + * registered. + * @param name the name of the RPC + * @param request the name of the RPC request structure + * @param reply the name of the RPC reply structure + * @param callback the callback that should be invoked when the RPC + * is received. The callback has the following prototype + * void (*callback)(EVRPC_STRUCT(Message)* rpc, void *arg) + * @param cbarg an additional parameter that can be passed to the callback. + * The parameter can be used to carry around state. + */ +#define EVRPC_REGISTER(base, name, request, reply, callback, cbarg) \ + do { \ + struct evrpc* rpc = (struct evrpc *)calloc(1, sizeof(struct evrpc)); \ + EVRPC_REGISTER_OBJECT(rpc, name, request, reply); \ + evrpc_register_rpc(base, rpc, \ + (void (*)(struct evrpc_req_generic*, void *))callback, cbarg); \ + } while (0) + +int evrpc_register_rpc(struct evrpc_base *, struct evrpc *, + void (*)(struct evrpc_req_generic*, void *), void *); + +/** + * Unregisters an already registered RPC + * + * @param base the evrpc_base object from which to unregister an RPC + * @param name the name of the rpc to unregister + * @return -1 on error or 0 when successful. + * @see EVRPC_REGISTER() + */ +#define EVRPC_UNREGISTER(base, name) evrpc_unregister_rpc(base, #name) + +int evrpc_unregister_rpc(struct evrpc_base *base, const char *name); + +/* + * Client-side RPC support + */ + +struct evrpc_pool; +struct evhttp_connection; + +/** + * provides information about the completed RPC request. + */ +struct evrpc_status { +#define EVRPC_STATUS_ERR_NONE 0 +#define EVRPC_STATUS_ERR_TIMEOUT 1 +#define EVRPC_STATUS_ERR_BADPAYLOAD 2 +#define EVRPC_STATUS_ERR_UNSTARTED 3 +#define EVRPC_STATUS_ERR_HOOKABORTED 4 + int error; + + /* for looking at headers or other information */ + struct evhttp_request *http_req; +}; + +struct evrpc_request_wrapper { + TAILQ_ENTRY(evrpc_request_wrapper) next; + + /* pool on which this rpc request is being made */ + struct evrpc_pool *pool; + + /* connection on which the request is being sent */ + struct evhttp_connection *evcon; + + /* event for implementing request timeouts */ + struct event ev_timeout; + + /* the name of the rpc */ + char *name; + + /* callback */ + void (*cb)(struct evrpc_status*, void *request, void *reply, void *arg); + void *cb_arg; + + void *request; + void *reply; + + /* unmarshals the buffer into the proper request structure */ + void (*request_marshal)(struct evbuffer *, void *); + + /* removes all stored state in the reply */ + void (*reply_clear)(void *); + + /* marshals the reply into a buffer */ + int (*reply_unmarshal)(void *, struct evbuffer*); +}; + +/** launches an RPC and sends it to the server + * + * EVRPC_MAKE_REQUEST() is used by the client to send an RPC to the server. + * + * @param name the name of the RPC + * @param pool the evrpc_pool that contains the connection objects over which + * the request should be sent. + * @param request a pointer to the RPC request structure - it contains the + * data to be sent to the server. + * @param reply a pointer to the RPC reply structure. It is going to be filled + * if the request was answered successfully + * @param cb the callback to invoke when the RPC request has been answered + * @param cbarg an additional argument to be passed to the client + * @return 0 on success, -1 on failure + */ +#define EVRPC_MAKE_REQUEST(name, pool, request, reply, cb, cbarg) \ + evrpc_send_request_##name(pool, request, reply, cb, cbarg) + +int evrpc_make_request(struct evrpc_request_wrapper *); + +/** creates an rpc connection pool + * + * a pool has a number of connections associated with it. + * rpc requests are always made via a pool. + * + * @param base a pointer to an struct event_based object; can be left NULL + * in singled-threaded applications + * @return a newly allocated struct evrpc_pool object + * @see evrpc_pool_free() + */ +struct evrpc_pool *evrpc_pool_new(struct event_base *base); +/** frees an rpc connection pool + * + * @param pool a pointer to an evrpc_pool allocated via evrpc_pool_new() + * @see evrpc_pool_new() + */ +void evrpc_pool_free(struct evrpc_pool *pool); +/* + * adds a connection over which rpc can be dispatched. the connection + * object must have been newly created. + */ +void evrpc_pool_add_connection(struct evrpc_pool *, + struct evhttp_connection *); + +/** + * Sets the timeout in secs after which a request has to complete. The + * RPC is completely aborted if it does not complete by then. Setting + * the timeout to 0 means that it never timeouts and can be used to + * implement callback type RPCs. + * + * Any connection already in the pool will be updated with the new + * timeout. Connections added to the pool after set_timeout has be + * called receive the pool timeout only if no timeout has been set + * for the connection itself. + * + * @param pool a pointer to a struct evrpc_pool object + * @param timeout_in_secs the number of seconds after which a request should + * timeout and a failure be returned to the callback. + */ +void evrpc_pool_set_timeout(struct evrpc_pool *pool, int timeout_in_secs); + +/** + * Hooks for changing the input and output of RPCs; this can be used to + * implement compression, authentication, encryption, ... + */ + +enum EVRPC_HOOK_TYPE { + INPUT, /**< apply the function to an input hook */ + OUTPUT /**< apply the function to an output hook */ +}; + +/** adds a processing hook to either an rpc base or rpc pool + * + * If a hook returns -1, the processing is aborted. + * + * The add functions return handles that can be used for removing hooks. + * + * @param vbase a pointer to either struct evrpc_base or struct evrpc_pool + * @param hook_type either INPUT or OUTPUT + * @param cb the callback to call when the hook is activated + * @param cb_arg an additional argument for the callback + * @return a handle to the hook so it can be removed later + * @see evrpc_remove_hook() + */ +void *evrpc_add_hook(void *vbase, + enum EVRPC_HOOK_TYPE hook_type, + int (*cb)(struct evhttp_request *, struct evbuffer *, void *), + void *cb_arg); + +/** removes a previously added hook + * + * @param vbase a pointer to either struct evrpc_base or struct evrpc_pool + * @param hook_type either INPUT or OUTPUT + * @param handle a handle returned by evrpc_add_hook() + * @return 1 on success or 0 on failure + * @see evrpc_add_hook() + */ +int evrpc_remove_hook(void *vbase, + enum EVRPC_HOOK_TYPE hook_type, + void *handle); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVRPC_H_ */ === added file 'extra/libevent/evsignal.h' --- a/extra/libevent/evsignal.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evsignal.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,52 @@ +/* + * Copyright 2000-2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVSIGNAL_H_ +#define _EVSIGNAL_H_ + +typedef void (*ev_sighandler_t)(int); + +struct evsignal_info { + struct event_list signalqueue; + struct event ev_signal; + int ev_signal_pair[2]; + int ev_signal_added; + volatile sig_atomic_t evsignal_caught; + sig_atomic_t evsigcaught[NSIG]; +#ifdef HAVE_SIGACTION + struct sigaction **sh_old; +#else + ev_sighandler_t **sh_old; +#endif + int sh_old_max; +}; +void evsignal_init(struct event_base *); +void evsignal_process(struct event_base *); +int evsignal_add(struct event *); +int evsignal_del(struct event *); +void evsignal_dealloc(struct event_base *); + +#endif /* _EVSIGNAL_H_ */ === added file 'extra/libevent/evutil.c' --- a/extra/libevent/evutil.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evutil.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2007 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#undef WIN32_LEAN_AND_MEAN +#include <winsock2.h> +#include "misc.h" +#endif + +#include <sys/types.h> +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#include <errno.h> + +#include "evutil.h" +#include "log.h" + +int +evutil_socketpair(int family, int type, int protocol, int fd[2]) +{ +#ifndef WIN32 + return socketpair(family, type, protocol, fd); +#else + /* This code is originally from Tor. Used with permission. */ + + /* This socketpair does not work when localhost is down. So + * it's really not the same thing at all. But it's close enough + * for now, and really, when localhost is down sometimes, we + * have other problems too. + */ + int listener = -1; + int connector = -1; + int acceptor = -1; + struct sockaddr_in listen_addr; + struct sockaddr_in connect_addr; + int size; + int saved_errno = -1; + + if (protocol +#ifdef AF_UNIX + || family != AF_UNIX +#endif + ) { + EVUTIL_SET_SOCKET_ERROR(WSAEAFNOSUPPORT); + return -1; + } + if (!fd) { + EVUTIL_SET_SOCKET_ERROR(WSAEINVAL); + return -1; + } + + listener = (int)socket(AF_INET, type, 0); + if (listener < 0) + return -1; + memset(&listen_addr, 0, sizeof(listen_addr)); + listen_addr.sin_family = AF_INET; + listen_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + listen_addr.sin_port = 0; /* kernel chooses port. */ + if (bind(listener, (struct sockaddr *) &listen_addr, sizeof (listen_addr)) + == -1) + goto tidy_up_and_fail; + if (listen(listener, 1) == -1) + goto tidy_up_and_fail; + + connector = (int)socket(AF_INET, type, 0); + if (connector < 0) + goto tidy_up_and_fail; + /* We want to find out the port number to connect to. */ + size = sizeof(connect_addr); + if (getsockname(listener, (struct sockaddr *) &connect_addr, &size) == -1) + goto tidy_up_and_fail; + if (size != sizeof (connect_addr)) + goto abort_tidy_up_and_fail; + if (connect(connector, (struct sockaddr *) &connect_addr, + sizeof(connect_addr)) == -1) + goto tidy_up_and_fail; + + size = sizeof(listen_addr); + acceptor = (int)accept(listener, (struct sockaddr *) &listen_addr, &size); + if (acceptor < 0) + goto tidy_up_and_fail; + if (size != sizeof(listen_addr)) + goto abort_tidy_up_and_fail; + EVUTIL_CLOSESOCKET(listener); + /* Now check we are talking to ourself by matching port and host on the + two sockets. */ + if (getsockname(connector, (struct sockaddr *) &connect_addr, &size) == -1) + goto tidy_up_and_fail; + if (size != sizeof (connect_addr) + || listen_addr.sin_family != connect_addr.sin_family + || listen_addr.sin_addr.s_addr != connect_addr.sin_addr.s_addr + || listen_addr.sin_port != connect_addr.sin_port) + goto abort_tidy_up_and_fail; + fd[0] = connector; + fd[1] = acceptor; + + return 0; + + abort_tidy_up_and_fail: + saved_errno = WSAECONNABORTED; + tidy_up_and_fail: + if (saved_errno < 0) + saved_errno = WSAGetLastError(); + if (listener != -1) + EVUTIL_CLOSESOCKET(listener); + if (connector != -1) + EVUTIL_CLOSESOCKET(connector); + if (acceptor != -1) + EVUTIL_CLOSESOCKET(acceptor); + + EVUTIL_SET_SOCKET_ERROR(saved_errno); + return -1; +#endif +} + +int +evutil_make_socket_nonblocking(int fd) +{ +#ifdef WIN32 + { + unsigned long nonblocking = 1; + ioctlsocket(fd, FIONBIO, (unsigned long*) &nonblocking); + } +#else + if (fcntl(fd, F_SETFL, O_NONBLOCK) == -1) { + event_warn("fcntl(O_NONBLOCK)"); + return -1; +} +#endif + return 0; +} + +ev_int64_t +evutil_strtoll(const char *s, char **endptr, int base) +{ +#ifdef HAVE_STRTOLL + return (ev_int64_t)strtoll(s, endptr, base); +#elif SIZEOF_LONG == 8 + return (ev_int64_t)strtol(s, endptr, base); +#elif defined(WIN32) && defined(_MSC_VER) && _MSC_VER < 1300 + /* XXXX on old versions of MS APIs, we only support base + * 10. */ + ev_int64_t r; + if (base != 10) + return 0; + r = (ev_int64_t) _atoi64(s); + while (isspace(*s)) + ++s; + while (isdigit(*s)) + ++s; + if (endptr) + *endptr = (char*) s; + return r; +#elif defined(WIN32) + return (ev_int64_t) _strtoi64(s, endptr, base); +#else +#error "I don't know how to parse 64-bit integers." +#endif +} === added file 'extra/libevent/evutil.h' --- a/extra/libevent/evutil.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/evutil.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2007 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _EVUTIL_H_ +#define _EVUTIL_H_ + +/** @file evutil.h + + Common convenience functions for cross-platform portability and + related socket manipulations. + + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <event-config.h> +#ifdef _EVENT_HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#ifdef _EVENT_HAVE_STDINT_H +#include <stdint.h> +#elif defined(_EVENT_HAVE_INTTYPES_H) +#include <inttypes.h> +#endif +#ifdef _EVENT_HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif + +#ifdef _EVENT_HAVE_UINT64_T +#define ev_uint64_t uint64_t +#define ev_int64_t int64_t +#elif defined(WIN32) +#define ev_uint64_t unsigned __int64 +#define ev_int64_t __int64 +#elif _EVENT_SIZEOF_LONG_LONG == 8 +#define ev_uint64_t unsigned long long +#define ev_int64_t long long +#elif _EVENT_SIZEOF_LONG == 8 +#define ev_uint64_t unsigned long +#define ev_int64_t long +#else +#error "No way to define ev_uint64_t" +#endif + +#ifdef _EVENT_HAVE_UINT32_T +#define ev_uint32_t uint32_t +#elif defined(WIN32) +#define ev_uint32_t unsigned int +#elif _EVENT_SIZEOF_LONG == 4 +#define ev_uint32_t unsigned long +#elif _EVENT_SIZEOF_INT == 4 +#define ev_uint32_t unsigned int +#else +#error "No way to define ev_uint32_t" +#endif + +#ifdef _EVENT_HAVE_UINT16_T +#define ev_uint16_t uint16_t +#elif defined(WIN32) +#define ev_uint16_t unsigned short +#elif _EVENT_SIZEOF_INT == 2 +#define ev_uint16_t unsigned int +#elif _EVENT_SIZEOF_SHORT == 2 +#define ev_uint16_t unsigned short +#else +#error "No way to define ev_uint16_t" +#endif + +#ifdef _EVENT_HAVE_UINT8_T +#define ev_uint8_t uint8_t +#else +#define ev_uint8_t unsigned char +#endif + +int evutil_socketpair(int d, int type, int protocol, int sv[2]); +int evutil_make_socket_nonblocking(int sock); +#ifdef WIN32 +#define EVUTIL_CLOSESOCKET(s) closesocket(s) +#else +#define EVUTIL_CLOSESOCKET(s) close(s) +#endif + +#ifdef WIN32 +#define EVUTIL_SOCKET_ERROR() WSAGetLastError() +#define EVUTIL_SET_SOCKET_ERROR(errcode) \ + do { WSASetLastError(errcode); } while (0) +#else +#define EVUTIL_SOCKET_ERROR() (errno) +#define EVUTIL_SET_SOCKET_ERROR(errcode) \ + do { errno = (errcode); } while (0) +#endif + +/* + * Manipulation functions for struct timeval + */ +#ifdef _EVENT_HAVE_TIMERADD +#define evutil_timeradd(tvp, uvp, vvp) timeradd((tvp), (uvp), (vvp)) +#define evutil_timersub(tvp, uvp, vvp) timersub((tvp), (uvp), (vvp)) +#else +#define evutil_timeradd(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ + if ((vvp)->tv_usec >= 1000000) { \ + (vvp)->tv_sec++; \ + (vvp)->tv_usec -= 1000000; \ + } \ + } while (0) +#define evutil_timersub(tvp, uvp, vvp) \ + do { \ + (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ + if ((vvp)->tv_usec < 0) { \ + (vvp)->tv_sec--; \ + (vvp)->tv_usec += 1000000; \ + } \ + } while (0) +#endif /* !_EVENT_HAVE_HAVE_TIMERADD */ + +#ifdef _EVENT_HAVE_TIMERCLEAR +#define evutil_timerclear(tvp) timerclear(tvp) +#else +#define evutil_timerclear(tvp) (tvp)->tv_sec = (tvp)->tv_usec = 0 +#endif + +#ifdef _EVENT_HAVE_TIMERCMP +#define evutil_timercmp(tvp, uvp, cmp) timercmp((tvp), (uvp), cmp) +#else +#define evutil_timercmp(tvp, uvp, cmp) \ + (((tvp)->tv_sec == (uvp)->tv_sec) ? \ + ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ + ((tvp)->tv_sec cmp (uvp)->tv_sec)) +#endif + +#ifdef _EVENT_HAVE_TIMERISSET +#define evutil_timerisset(tvp) timerisset(tvp) +#else +#define evutil_timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_usec) +#endif + + +/* big-int related functions */ +ev_int64_t evutil_strtoll(const char *s, char **endptr, int base); + +#ifdef __cplusplus +} +#endif + +#endif /* _EVUTIL_H_ */ === added file 'extra/libevent/http-internal.h' --- a/extra/libevent/http-internal.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/http-internal.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,133 @@ +/* + * Copyright 2001 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * This header file contains definitions for dealing with HTTP requests + * that are internal to libevent. As user of the library, you should not + * need to know about these. + */ + +#ifndef _HTTP_H_ +#define _HTTP_H_ + +#define HTTP_CONNECT_TIMEOUT 45 +#define HTTP_WRITE_TIMEOUT 50 +#define HTTP_READ_TIMEOUT 50 + +#define HTTP_PREFIX "http://" +#define HTTP_DEFAULTPORT 80 + +enum evhttp_connection_error { + EVCON_HTTP_TIMEOUT, + EVCON_HTTP_EOF, + EVCON_HTTP_INVALID_HEADER +}; + +struct evbuffer; +struct addrinfo; +struct evhttp_request; + +/* A stupid connection object - maybe make this a bufferevent later */ + +enum evhttp_connection_state { + EVCON_DISCONNECTED, /* not currently connected not trying either */ + EVCON_CONNECTING, /* tries to currently connect */ + EVCON_CONNECTED /* connection is established */ +}; + +struct event_base; + +struct evhttp_connection { + /* we use tailq only if they were created for an http server */ + TAILQ_ENTRY(evhttp_connection) (next); + + int fd; + struct event ev; + struct event close_ev; + struct evbuffer *input_buffer; + struct evbuffer *output_buffer; + + char *bind_address; /* address to use for binding the src */ + + char *address; /* address to connect to */ + u_short port; + + int flags; +#define EVHTTP_CON_INCOMING 0x0001 /* only one request on it ever */ +#define EVHTTP_CON_OUTGOING 0x0002 /* multiple requests possible */ +#define EVHTTP_CON_CLOSEDETECT 0x0004 /* detecting if persistent close */ + + int timeout; /* timeout in seconds for events */ + int retry_cnt; /* retry count */ + int retry_max; /* maximum number of retries */ + + enum evhttp_connection_state state; + + /* for server connections, the http server they are connected with */ + struct evhttp *http_server; + + TAILQ_HEAD(evcon_requestq, evhttp_request) requests; + + void (*cb)(struct evhttp_connection *, void *); + void *cb_arg; + + void (*closecb)(struct evhttp_connection *, void *); + void *closecb_arg; + + struct event_base *base; +}; + +struct evhttp_cb { + TAILQ_ENTRY(evhttp_cb) next; + + char *what; + + void (*cb)(struct evhttp_request *req, void *); + void *cbarg; +}; + +/* both the http server as well as the rpc system need to queue connections */ +TAILQ_HEAD(evconq, evhttp_connection); + +struct evhttp { + struct event bind_ev; + + TAILQ_HEAD(httpcbq, evhttp_cb) callbacks; + struct evconq connections; + + int timeout; + + void (*gencb)(struct evhttp_request *req, void *); + void *gencbarg; + + struct event_base *base; +}; + +/* resets the connection; can be reused for more requests */ +void evhttp_connection_reset(struct evhttp_connection *); + +/* connects if necessary */ +int evhttp_connection_connect(struct evhttp_connection *); + +/* notifies the current request that it failed; resets connection */ +void evhttp_connection_fail(struct evhttp_connection *, + enum evhttp_connection_error error); + +void evhttp_get_request(struct evhttp *, int, struct sockaddr *, socklen_t); + +int evhttp_hostportfile(char *, char **, u_short *, char **); + +int evhttp_parse_lines(struct evhttp_request *, struct evbuffer*); + +void evhttp_start_read(struct evhttp_connection *); +void evhttp_read_header(int, short, void *); +void evhttp_make_header(struct evhttp_connection *, struct evhttp_request *); + +void evhttp_write_buffer(struct evhttp_connection *, + void (*)(struct evhttp_connection *, void *), void *); + +/* response sending HTML the data in the buffer */ +void evhttp_response_code(struct evhttp_request *, int, const char *); +void evhttp_send_page(struct evhttp_request *, struct evbuffer *); + +#endif /* _HTTP_H */ === added file 'extra/libevent/http.c' --- a/extra/libevent/http.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/http.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,2512 @@ +/* + * Copyright (c) 2002-2006 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#ifdef HAVE_SYS_IOCCOM_H +#include <sys/ioccom.h> +#endif + +#ifndef WIN32 +#include <sys/resource.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/wait.h> +#endif + +#include <sys/queue.h> + +#ifndef WIN32 +#include <netinet/in.h> +#include <netdb.h> +#endif + +#ifdef WIN32 +#include <winsock2.h> +#endif + +#include <assert.h> +#include <ctype.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef WIN32 +#include <syslog.h> +#endif +#include <signal.h> +#include <time.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#undef timeout_pending +#undef timeout_initialized + +#include "strlcpy-internal.h" +#include "event.h" +#include "evhttp.h" +#include "evutil.h" +#include "log.h" +#include "http-internal.h" + +#ifdef WIN32 +#define snprintf _snprintf +#define strcasecmp _stricmp +#define strncasecmp _strnicmp +#define strdup _strdup +#endif + +#ifndef HAVE_GETADDRINFO +struct addrinfo { + int ai_family; + int ai_socktype; + int ai_protocol; + size_t ai_addrlen; + struct sockaddr *ai_addr; + struct addrinfo *ai_next; +}; +static int +fake_getaddrinfo(const char *hostname, struct addrinfo *ai) +{ + struct hostent *he = NULL; + struct sockaddr_in *sa; + if (hostname) { + he = gethostbyname(hostname); + if (!he) + return (-1); + } + ai->ai_family = he ? he->h_addrtype : AF_INET; + ai->ai_socktype = SOCK_STREAM; + ai->ai_protocol = 0; + ai->ai_addrlen = sizeof(struct sockaddr_in); + if (NULL == (ai->ai_addr = malloc(ai->ai_addrlen))) + return (-1); + sa = (struct sockaddr_in*)ai->ai_addr; + memset(sa, 0, ai->ai_addrlen); + if (he) { + sa->sin_family = he->h_addrtype; + memcpy(&sa->sin_addr, he->h_addr_list[0], he->h_length); + } else { + sa->sin_family = AF_INET; + sa->sin_addr.s_addr = INADDR_ANY; + } + ai->ai_next = NULL; + return (0); +} +static void +fake_freeaddrinfo(struct addrinfo *ai) +{ + free(ai->ai_addr); +} +#endif + +#ifndef MIN +#define MIN(a,b) (((a)<(b))?(a):(b)) +#endif + +/* wrapper for setting the base from the http server */ +#define EVHTTP_BASE_SET(x, y) do { \ + if ((x)->base != NULL) event_base_set((x)->base, y); \ +} while (0) + +extern int debug; + +static int socket_connect(int fd, const char *address, unsigned short port); +static int bind_socket_ai(struct addrinfo *); +static int bind_socket(const char *, u_short); +static void name_from_addr(struct sockaddr *, socklen_t, char **, char **); +static int evhttp_associate_new_request_with_connection( + struct evhttp_connection *evcon); +static void evhttp_connection_start_detectclose( + struct evhttp_connection *evcon); +static void evhttp_connection_stop_detectclose( + struct evhttp_connection *evcon); +static void evhttp_request_dispatch(struct evhttp_connection* evcon); + +void evhttp_read(int, short, void *); +void evhttp_write(int, short, void *); + +#ifndef HAVE_STRSEP +/* strsep replacement for platforms that lack it. Only works if + * del is one character long. */ +static char * +strsep(char **s, const char *del) +{ + char *d, *tok; + assert(strlen(del) == 1); + if (!s || !*s) + return NULL; + tok = *s; + d = strstr(tok, del); + if (d) { + *d = '\0'; + *s = d + 1; + } else + *s = NULL; + return tok; +} +#endif + +static const char * +html_replace(char ch, char *buf) +{ + switch (ch) { + case '<': + return "<"; + case '>': + return ">"; + case '"': + return """; + case '\'': + return "'"; + case '&': + return "&"; + default: + break; + } + + /* Echo the character back */ + buf[0] = ch; + buf[1] = '\0'; + + return buf; +} + +/* + * Replaces <, >, ", ' and & with <, >, ", + * ' and & correspondingly. + * + * The returned string needs to be freed by the caller. + */ + +char * +evhttp_htmlescape(const char *html) +{ + int i, new_size = 0, old_size = strlen(html); + char *escaped_html, *p; + char scratch_space[2]; + + for (i = 0; i < old_size; ++i) + new_size += strlen(html_replace(html[i], scratch_space)); + + p = escaped_html = malloc(new_size + 1); + if (escaped_html == NULL) + event_err(1, "%s: malloc(%d)", __func__, new_size + 1); + for (i = 0; i < old_size; ++i) { + const char *replaced = html_replace(html[i], scratch_space); + /* this is length checked */ + strcpy(p, replaced); + p += strlen(replaced); + } + + *p = '\0'; + + return (escaped_html); +} + +static const char * +evhttp_method(enum evhttp_cmd_type type) +{ + const char *method; + + switch (type) { + case EVHTTP_REQ_GET: + method = "GET"; + break; + case EVHTTP_REQ_POST: + method = "POST"; + break; + case EVHTTP_REQ_HEAD: + method = "HEAD"; + break; + default: + method = NULL; + break; + } + + return (method); +} + +static void +evhttp_add_event(struct event *ev, int timeout, int default_timeout) +{ + if (timeout != 0) { + struct timeval tv; + + evutil_timerclear(&tv); + tv.tv_sec = timeout != -1 ? timeout : default_timeout; + event_add(ev, &tv); + } else { + event_add(ev, NULL); + } +} + +void +evhttp_write_buffer(struct evhttp_connection *evcon, + void (*cb)(struct evhttp_connection *, void *), void *arg) +{ + event_debug(("%s: preparing to write buffer\n", __func__)); + + /* Set call back */ + evcon->cb = cb; + evcon->cb_arg = arg; + + /* check if the event is already pending */ + if (event_pending(&evcon->ev, EV_WRITE|EV_TIMEOUT, NULL)) + event_del(&evcon->ev); + + event_set(&evcon->ev, evcon->fd, EV_WRITE, evhttp_write, evcon); + EVHTTP_BASE_SET(evcon, &evcon->ev); + evhttp_add_event(&evcon->ev, evcon->timeout, HTTP_WRITE_TIMEOUT); +} + +/* + * Create the headers need for an HTTP request + */ +static void +evhttp_make_header_request(struct evhttp_connection *evcon, + struct evhttp_request *req) +{ + char line[1024]; + const char *method; + + evhttp_remove_header(req->output_headers, "Accept-Encoding"); + evhttp_remove_header(req->output_headers, "Proxy-Connection"); + + /* Generate request line */ + method = evhttp_method(req->type); + snprintf(line, sizeof(line), "%s %s HTTP/%d.%d\r\n", + method, req->uri, req->major, req->minor); + evbuffer_add(evcon->output_buffer, line, strlen(line)); + + /* Add the content length on a post request if missing */ + if (req->type == EVHTTP_REQ_POST && + evhttp_find_header(req->output_headers, "Content-Length") == NULL){ + char size[12]; + snprintf(size, sizeof(size), "%ld", + (long)EVBUFFER_LENGTH(req->output_buffer)); + evhttp_add_header(req->output_headers, "Content-Length", size); + } +} + +static int +evhttp_is_connection_close(int flags, struct evkeyvalq* headers) +{ + if (flags & EVHTTP_PROXY_REQUEST) { + /* proxy connection */ + const char *connection = evhttp_find_header(headers, "Proxy-Connection"); + return (connection == NULL || strcasecmp(connection, "keep-alive") != 0); + } else { + const char *connection = evhttp_find_header(headers, "Connection"); + return (connection != NULL && strcasecmp(connection, "close") == 0); + } +} + +static int +evhttp_is_connection_keepalive(struct evkeyvalq* headers) +{ + const char *connection = evhttp_find_header(headers, "Connection"); + return (connection != NULL + && strncasecmp(connection, "keep-alive", 10) == 0); +} + +static void +evhttp_maybe_add_date_header(struct evkeyvalq *headers) +{ + if (evhttp_find_header(headers, "Date") == NULL) { + char date[50]; +#ifndef WIN32 + struct tm cur; +#endif + struct tm *cur_p; + time_t t = time(NULL); +#ifdef WIN32 + cur_p = gmtime(&t); +#else + gmtime_r(&t, &cur); + cur_p = &cur; +#endif + if (strftime(date, sizeof(date), + "%a, %d %b %Y %H:%M:%S GMT", cur_p) != 0) { + evhttp_add_header(headers, "Date", date); + } + } +} + +static void +evhttp_maybe_add_content_length_header(struct evkeyvalq *headers, + long content_length) +{ + if (evhttp_find_header(headers, "Transfer-Encoding") == NULL && + evhttp_find_header(headers, "Content-Length") == NULL) { + char len[12]; + snprintf(len, sizeof(len), "%ld", content_length); + evhttp_add_header(headers, "Content-Length", len); + } +} + +/* + * Create the headers needed for an HTTP reply + */ + +static void +evhttp_make_header_response(struct evhttp_connection *evcon, + struct evhttp_request *req) +{ + char line[1024]; + snprintf(line, sizeof(line), "HTTP/%d.%d %d %s\r\n", + req->major, req->minor, req->response_code, + req->response_code_line); + evbuffer_add(evcon->output_buffer, line, strlen(line)); + + if (req->major == 1 && req->minor == 1) + evhttp_maybe_add_date_header(req->output_headers); + + if (req->major == 1 && + (req->minor == 1 || + evhttp_is_connection_keepalive(req->input_headers))) { + /* + * we need to add the content length if the user did + * not give it, this is required for persistent + * connections to work. + */ + evhttp_maybe_add_content_length_header(req->output_headers, + (long)EVBUFFER_LENGTH(req->output_buffer)); + } + + /* Potentially add headers for unidentified content. */ + if (EVBUFFER_LENGTH(req->output_buffer)) { + if (evhttp_find_header(req->output_headers, + "Content-Type") == NULL) { + evhttp_add_header(req->output_headers, + "Content-Type", "text/html; charset=ISO-8859-1"); + } + } + + /* if the request asked for a close, we send a close, too */ + if (evhttp_is_connection_close(req->flags, req->input_headers)) { + evhttp_remove_header(req->output_headers, "Connection"); + if (!(req->flags & EVHTTP_PROXY_REQUEST)) + evhttp_add_header(req->output_headers, "Connection", "close"); + evhttp_remove_header(req->output_headers, "Proxy-Connection"); + } +} + +void +evhttp_make_header(struct evhttp_connection *evcon, struct evhttp_request *req) +{ + char line[1024]; + struct evkeyval *header; + + /* + * Depending if this is a HTTP request or response, we might need to + * add some new headers or remove existing headers. + */ + if (req->kind == EVHTTP_REQUEST) { + evhttp_make_header_request(evcon, req); + } else { + evhttp_make_header_response(evcon, req); + } + + TAILQ_FOREACH(header, req->output_headers, next) { + snprintf(line, sizeof(line), "%s: %s\r\n", + header->key, header->value); + evbuffer_add(evcon->output_buffer, line, strlen(line)); + } + evbuffer_add(evcon->output_buffer, "\r\n", 2); + + if (EVBUFFER_LENGTH(req->output_buffer) > 0) { + /* + * For a request, we add the POST data, for a reply, this + * is the regular data. + */ + evbuffer_add_buffer(evcon->output_buffer, req->output_buffer); + } +} + +/* Separated host, port and file from URI */ + +int +evhttp_hostportfile(char *url, char **phost, u_short *pport, char **pfile) +{ + /* XXX not threadsafe. */ + static char host[1024]; + static char file[1024]; + char *p; + const char *p2; + int len; + u_short port; + + len = strlen(HTTP_PREFIX); + if (strncasecmp(url, HTTP_PREFIX, len)) + return (-1); + + url += len; + + /* We might overrun */ + if (strlcpy(host, url, sizeof (host)) >= sizeof(host)) + return (-1); + + p = strchr(host, '/'); + if (p != NULL) { + *p = '\0'; + p2 = p + 1; + } else + p2 = NULL; + + if (pfile != NULL) { + /* Generate request file */ + if (p2 == NULL) + p2 = ""; + snprintf(file, sizeof(file), "/%s", p2); + } + + p = strchr(host, ':'); + if (p != NULL) { + *p = '\0'; + port = atoi(p + 1); + + if (port == 0) + return (-1); + } else + port = HTTP_DEFAULTPORT; + + if (phost != NULL) + *phost = host; + if (pport != NULL) + *pport = port; + if (pfile != NULL) + *pfile = file; + + return (0); +} + +static int +evhttp_connection_incoming_fail(struct evhttp_request *req, + enum evhttp_connection_error error) +{ + switch (error) { + case EVCON_HTTP_TIMEOUT: + case EVCON_HTTP_EOF: + /* + * these are cases in which we probably should just + * close the connection and not send a reply. this + * case may happen when a browser keeps a persistent + * connection open and we timeout on the read. + */ + return (-1); + case EVCON_HTTP_INVALID_HEADER: + default: /* xxx: probably should just error on default */ + /* the callback looks at the uri to determine errors */ + if (req->uri) { + free(req->uri); + req->uri = NULL; + } + + /* + * the callback needs to send a reply, once the reply has + * been send, the connection should get freed. + */ + (*req->cb)(req, req->cb_arg); + } + + return (0); +} + +void +evhttp_connection_fail(struct evhttp_connection *evcon, + enum evhttp_connection_error error) +{ + struct evhttp_request* req = TAILQ_FIRST(&evcon->requests); + void (*cb)(struct evhttp_request *, void *); + void *cb_arg; + assert(req != NULL); + + if (evcon->flags & EVHTTP_CON_INCOMING) { + /* + * for incoming requests, there are two different + * failure cases. it's either a network level error + * or an http layer error. for problems on the network + * layer like timeouts we just drop the connections. + * For HTTP problems, we might have to send back a + * reply before the connection can be freed. + */ + if (evhttp_connection_incoming_fail(req, error) == -1) + evhttp_connection_free(evcon); + return; + } + + /* save the callback for later; the cb might free our object */ + cb = req->cb; + cb_arg = req->cb_arg; + + TAILQ_REMOVE(&evcon->requests, req, next); + evhttp_request_free(req); + + /* xxx: maybe we should fail all requests??? */ + + /* reset the connection */ + evhttp_connection_reset(evcon); + + /* We are trying the next request that was queued on us */ + if (TAILQ_FIRST(&evcon->requests) != NULL) + evhttp_connection_connect(evcon); + + /* inform the user */ + if (cb != NULL) + (*cb)(NULL, cb_arg); +} + +void +evhttp_write(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + int n; + + if (what == EV_TIMEOUT) { + evhttp_connection_fail(evcon, EVCON_HTTP_TIMEOUT); + return; + } + + n = evbuffer_write(evcon->output_buffer, fd); + if (n == -1) { + event_debug(("%s: evbuffer_write", __func__)); + evhttp_connection_fail(evcon, EVCON_HTTP_EOF); + return; + } + + if (n == 0) { + event_debug(("%s: write nothing", __func__)); + evhttp_connection_fail(evcon, EVCON_HTTP_EOF); + return; + } + + if (EVBUFFER_LENGTH(evcon->output_buffer) != 0) { + evhttp_add_event(&evcon->ev, + evcon->timeout, HTTP_WRITE_TIMEOUT); + return; + } + + /* Activate our call back */ + if (evcon->cb != NULL) + (*evcon->cb)(evcon, evcon->cb_arg); +} + +static void +evhttp_connection_done(struct evhttp_connection *evcon) +{ + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + int con_outgoing = evcon->flags & EVHTTP_CON_OUTGOING; + + /* + * if this is an incoming connection, we need to leave the request + * on the connection, so that we can reply to it. + */ + if (con_outgoing) { + int need_close; + TAILQ_REMOVE(&evcon->requests, req, next); + req->evcon = NULL; + + need_close = + evhttp_is_connection_close(req->flags, req->input_headers) || + evhttp_is_connection_close(req->flags, req->output_headers); + + /* check if we got asked to close the connection */ + if (need_close) + evhttp_connection_reset(evcon); + + if (TAILQ_FIRST(&evcon->requests) != NULL) { + /* + * We have more requests; reset the connection + * and deal with the next request. xxx: no + * persistent connection right now + */ + if (evcon->state != EVCON_CONNECTED) + evhttp_connection_connect(evcon); + else + evhttp_request_dispatch(evcon); + } else if (!need_close) { + /* + * The connection is going to be persistent, but we + * need to detect if the other side closes it. + */ + evhttp_connection_start_detectclose(evcon); + } + } + + /* notify the user of the request */ + (*req->cb)(req, req->cb_arg); + + /* if this was an outgoing request, we own and it's done. so free it */ + if (con_outgoing) { + evhttp_request_free(req); + } +} + +/* + * Handles reading from a chunked request. + * return 1: all data has been read + * return 0: more data is expected + * return -1: data is corrupted + */ + +static int +evhttp_handle_chunked_read(struct evhttp_request *req, struct evbuffer *buf) +{ + int len; + + while ((len = EVBUFFER_LENGTH(buf)) > 0) { + if (req->ntoread < 0) { + /* Read chunk size */ + char *p = evbuffer_readline(buf); + char *endp; + int error; + if (p == NULL) + break; + /* the last chunk is on a new line? */ + if (strlen(p) == 0) { + free(p); + continue; + } + req->ntoread = evutil_strtoll(p, &endp, 16); + error = *p == '\0' || (*endp != '\0' && *endp != ' '); + free(p); + if (error) { + /* could not get chunk size */ + return (-1); + } + if (req->ntoread == 0) { + /* Last chunk */ + return (1); + } + continue; + } + + /* don't have enough to complete a chunk; wait for more */ + if (len < req->ntoread) + return (0); + + /* Completed chunk */ + evbuffer_add(req->input_buffer, + EVBUFFER_DATA(buf), req->ntoread); + evbuffer_drain(buf, req->ntoread); + req->ntoread = -1; + if (req->chunk_cb != NULL) { + (*req->chunk_cb)(req, req->cb_arg); + evbuffer_drain(req->input_buffer, + EVBUFFER_LENGTH(req->input_buffer)); + } + } + + return (0); +} + +static void +evhttp_read_body(struct evhttp_connection *evcon, struct evhttp_request *req) +{ + struct evbuffer *buf = evcon->input_buffer; + + if (req->chunked) { + int res = evhttp_handle_chunked_read(req, buf); + if (res == 1) { + /* finished last chunk */ + evhttp_connection_done(evcon); + return; + } else if (res == -1) { + /* corrupted data */ + evhttp_connection_fail(evcon, + EVCON_HTTP_INVALID_HEADER); + return; + } + } else if (req->ntoread < 0) { + /* Read until connection close. */ + evbuffer_add_buffer(req->input_buffer, buf); + } else if (EVBUFFER_LENGTH(buf) >= req->ntoread) { + /* Completed content length */ + evbuffer_add(req->input_buffer, EVBUFFER_DATA(buf), + req->ntoread); + evbuffer_drain(buf, req->ntoread); + req->ntoread = 0; + evhttp_connection_done(evcon); + return; + } + /* Read more! */ + event_set(&evcon->ev, evcon->fd, EV_READ, evhttp_read, evcon); + EVHTTP_BASE_SET(evcon, &evcon->ev); + evhttp_add_event(&evcon->ev, evcon->timeout, HTTP_READ_TIMEOUT); +} + +/* + * Reads data into a buffer structure until no more data + * can be read on the file descriptor or we have read all + * the data that we wanted to read. + * Execute callback when done. + */ + +void +evhttp_read(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + struct evbuffer *buf = evcon->input_buffer; + int n, len; + + if (what == EV_TIMEOUT) { + evhttp_connection_fail(evcon, EVCON_HTTP_TIMEOUT); + return; + } + n = evbuffer_read(buf, fd, -1); + len = EVBUFFER_LENGTH(buf); + event_debug(("%s: got %d on %d\n", __func__, n, fd)); + + if (n == -1) { + event_debug(("%s: evbuffer_read", __func__)); + evhttp_connection_fail(evcon, EVCON_HTTP_EOF); + return; + } else if (n == 0) { + /* Connection closed */ + evhttp_connection_done(evcon); + return; + } + evhttp_read_body(evcon, req); +} + +static void +evhttp_write_connectioncb(struct evhttp_connection *evcon, void *arg) +{ + /* This is after writing the request to the server */ + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + assert(req != NULL); + + /* We are done writing our header and are now expecting the response */ + req->kind = EVHTTP_RESPONSE; + + evhttp_start_read(evcon); +} + +/* + * Clean up a connection object + */ + +void +evhttp_connection_free(struct evhttp_connection *evcon) +{ + struct evhttp_request *req; + + /* notify interested parties that this connection is going down */ + if (evcon->fd != -1) { + if (evcon->state == EVCON_CONNECTED && evcon->closecb != NULL) + (*evcon->closecb)(evcon, evcon->closecb_arg); + } + + /* remove all requests that might be queued on this connection */ + while ((req = TAILQ_FIRST(&evcon->requests)) != NULL) { + TAILQ_REMOVE(&evcon->requests, req, next); + evhttp_request_free(req); + } + + if (evcon->http_server != NULL) { + struct evhttp *http = evcon->http_server; + TAILQ_REMOVE(&http->connections, evcon, next); + } + + if (event_initialized(&evcon->close_ev)) + event_del(&evcon->close_ev); + + if (event_initialized(&evcon->ev)) + event_del(&evcon->ev); + + if (evcon->fd != -1) + EVUTIL_CLOSESOCKET(evcon->fd); + + if (evcon->bind_address != NULL) + free(evcon->bind_address); + + if (evcon->address != NULL) + free(evcon->address); + + if (evcon->input_buffer != NULL) + evbuffer_free(evcon->input_buffer); + + if (evcon->output_buffer != NULL) + evbuffer_free(evcon->output_buffer); + + free(evcon); +} + +void +evhttp_connection_set_local_address(struct evhttp_connection *evcon, + const char *address) +{ + assert(evcon->state == EVCON_DISCONNECTED); + if (evcon->bind_address) + free(evcon->bind_address); + if ((evcon->bind_address = strdup(address)) == NULL) + event_err(1, "%s: strdup", __func__); +} + + +static void +evhttp_request_dispatch(struct evhttp_connection* evcon) +{ + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + + /* this should not usually happy but it's possible */ + if (req == NULL) + return; + + /* delete possible close detection events */ + evhttp_connection_stop_detectclose(evcon); + + /* we assume that the connection is connected already */ + assert(evcon->state == EVCON_CONNECTED); + + /* Create the header from the store arguments */ + evhttp_make_header(evcon, req); + + evhttp_write_buffer(evcon, evhttp_write_connectioncb, NULL); +} + +/* Reset our connection state */ +void +evhttp_connection_reset(struct evhttp_connection *evcon) +{ + if (event_initialized(&evcon->ev)) + event_del(&evcon->ev); + + if (evcon->fd != -1) { + /* inform interested parties about connection close */ + if (evcon->state == EVCON_CONNECTED && evcon->closecb != NULL) + (*evcon->closecb)(evcon, evcon->closecb_arg); + + EVUTIL_CLOSESOCKET(evcon->fd); + evcon->fd = -1; + } + evcon->state = EVCON_DISCONNECTED; + + /* remove unneeded flags */ + evcon->flags &= ~EVHTTP_CON_CLOSEDETECT; +} + +static void +evhttp_detect_close_cb(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + evhttp_connection_reset(evcon); +} + +static void +evhttp_connection_start_detectclose(struct evhttp_connection *evcon) +{ + evcon->flags |= EVHTTP_CON_CLOSEDETECT; + + if (event_initialized(&evcon->close_ev)) + event_del(&evcon->close_ev); + event_set(&evcon->close_ev, evcon->fd, EV_READ, + evhttp_detect_close_cb, evcon); + EVHTTP_BASE_SET(evcon, &evcon->close_ev); + event_add(&evcon->close_ev, NULL); +} + +static void +evhttp_connection_stop_detectclose(struct evhttp_connection *evcon) +{ + evcon->flags &= ~EVHTTP_CON_CLOSEDETECT; + event_del(&evcon->close_ev); +} + +static void +evhttp_connection_retry(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + + evcon->state = EVCON_DISCONNECTED; + evhttp_connection_connect(evcon); +} + +/* + * Call back for asynchronous connection attempt. + */ + +static void +evhttp_connectioncb(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + int error; + socklen_t errsz = sizeof(error); + + if (what == EV_TIMEOUT) { + event_debug(("%s: connection timeout for \"%s:%d\" on %d", + __func__, evcon->address, evcon->port, evcon->fd)); + goto cleanup; + } + + /* Check if the connection completed */ + if (getsockopt(evcon->fd, SOL_SOCKET, SO_ERROR, (void*)&error, + &errsz) == -1) { + event_debug(("%s: getsockopt for \"%s:%d\" on %d", + __func__, evcon->address, evcon->port, evcon->fd)); + goto cleanup; + } + + if (error) { + event_debug(("%s: connect failed for \"%s:%d\" on %d: %s", + __func__, evcon->address, evcon->port, evcon->fd, + strerror(error))); + goto cleanup; + } + + /* We are connected to the server now */ + event_debug(("%s: connected to \"%s:%d\" on %d\n", + __func__, evcon->address, evcon->port, evcon->fd)); + + /* Reset the retry count as we were successful in connecting */ + evcon->retry_cnt = 0; + evcon->state = EVCON_CONNECTED; + + /* try to start requests that have queued up on this connection */ + evhttp_request_dispatch(evcon); + return; + + cleanup: + if (evcon->retry_max < 0 || evcon->retry_cnt < evcon->retry_max) { + evtimer_set(&evcon->ev, evhttp_connection_retry, evcon); + EVHTTP_BASE_SET(evcon, &evcon->ev); + evhttp_add_event(&evcon->ev, MIN(3600, 2 << evcon->retry_cnt), + HTTP_CONNECT_TIMEOUT); + evcon->retry_cnt++; + return; + } + evhttp_connection_reset(evcon); + + /* for now, we just signal all requests by executing their callbacks */ + while (TAILQ_FIRST(&evcon->requests) != NULL) { + struct evhttp_request *request = TAILQ_FIRST(&evcon->requests); + TAILQ_REMOVE(&evcon->requests, request, next); + request->evcon = NULL; + + /* we might want to set an error here */ + request->cb(request, request->cb_arg); + evhttp_request_free(request); + } +} + +/* + * Check if we got a valid response code. + */ + +static int +evhttp_valid_response_code(int code) +{ + if (code == 0) + return (0); + + return (1); +} + +/* Parses the status line of a web server */ + +static int +evhttp_parse_response_line(struct evhttp_request *req, char *line) +{ + char *protocol; + char *number; + char *readable; + + protocol = strsep(&line, " "); + if (line == NULL) + return (-1); + number = strsep(&line, " "); + if (line == NULL) + return (-1); + readable = line; + + if (strcmp(protocol, "HTTP/1.0") == 0) { + req->major = 1; + req->minor = 0; + } else if (strcmp(protocol, "HTTP/1.1") == 0) { + req->major = 1; + req->minor = 1; + } else { + event_debug(("%s: bad protocol \"%s\"", + __func__, protocol)); + return (-1); + } + + req->response_code = atoi(number); + if (!evhttp_valid_response_code(req->response_code)) { + event_debug(("%s: bad response code \"%s\"", + __func__, number)); + return (-1); + } + + if ((req->response_code_line = strdup(readable)) == NULL) + event_err(1, "%s: strdup", __func__); + + return (0); +} + +/* Parse the first line of a HTTP request */ + +static int +evhttp_parse_request_line(struct evhttp_request *req, char *line) +{ + char *method; + char *uri; + char *version; + + /* Parse the request line */ + method = strsep(&line, " "); + if (line == NULL) + return (-1); + uri = strsep(&line, " "); + if (line == NULL) + return (-1); + version = strsep(&line, " "); + if (line != NULL) + return (-1); + + /* First line */ + if (strcmp(method, "GET") == 0) { + req->type = EVHTTP_REQ_GET; + } else if (strcmp(method, "POST") == 0) { + req->type = EVHTTP_REQ_POST; + } else if (strcmp(method, "HEAD") == 0) { + req->type = EVHTTP_REQ_HEAD; + } else { + event_debug(("%s: bad method %s on request %p from %s", + __func__, method, req, req->remote_host)); + return (-1); + } + + if (strcmp(version, "HTTP/1.0") == 0) { + req->major = 1; + req->minor = 0; + } else if (strcmp(version, "HTTP/1.1") == 0) { + req->major = 1; + req->minor = 1; + } else { + event_debug(("%s: bad version %s on request %p from %s", + __func__, version, req, req->remote_host)); + return (-1); + } + + if ((req->uri = strdup(uri)) == NULL) { + event_debug(("%s: evhttp_decode_uri", __func__)); + return (-1); + } + + /* determine if it's a proxy request */ + if (strlen(req->uri) > 0 && req->uri[0] != '/') + req->flags |= EVHTTP_PROXY_REQUEST; + + return (0); +} + +const char * +evhttp_find_header(const struct evkeyvalq *headers, const char *key) +{ + struct evkeyval *header; + + TAILQ_FOREACH(header, headers, next) { + if (strcasecmp(header->key, key) == 0) + return (header->value); + } + + return (NULL); +} + +void +evhttp_clear_headers(struct evkeyvalq *headers) +{ + struct evkeyval *header; + + for (header = TAILQ_FIRST(headers); + header != NULL; + header = TAILQ_FIRST(headers)) { + TAILQ_REMOVE(headers, header, next); + free(header->key); + free(header->value); + free(header); + } +} + +/* + * Returns 0, if the header was successfully removed. + * Returns -1, if the header could not be found. + */ + +int +evhttp_remove_header(struct evkeyvalq *headers, const char *key) +{ + struct evkeyval *header; + + TAILQ_FOREACH(header, headers, next) { + if (strcasecmp(header->key, key) == 0) + break; + } + + if (header == NULL) + return (-1); + + /* Free and remove the header that we found */ + TAILQ_REMOVE(headers, header, next); + free(header->key); + free(header->value); + free(header); + + return (0); +} + +int +evhttp_add_header(struct evkeyvalq *headers, + const char *key, const char *value) +{ + struct evkeyval *header = NULL; + + event_debug(("%s: key: %s val: %s\n", __func__, key, value)); + + if (strchr(value, '\r') != NULL || strchr(value, '\n') != NULL || + strchr(key, '\r') != NULL || strchr(key, '\n') != NULL) { + /* drop illegal headers */ + event_debug(("%s: dropping illegal header\n", __func__)); + return (-1); + } + + header = calloc(1, sizeof(struct evkeyval)); + if (header == NULL) { + event_warn("%s: calloc", __func__); + return (-1); + } + if ((header->key = strdup(key)) == NULL) { + free(header); + event_warn("%s: strdup", __func__); + return (-1); + } + if ((header->value = strdup(value)) == NULL) { + free(header->key); + free(header); + event_warn("%s: strdup", __func__); + return (-1); + } + + TAILQ_INSERT_TAIL(headers, header, next); + + return (0); +} + +/* + * Parses header lines from a request or a response into the specified + * request object given an event buffer. + * + * Returns + * -1 on error + * 0 when we need to read more headers + * 1 when all headers have been read. + */ + +int +evhttp_parse_lines(struct evhttp_request *req, struct evbuffer* buffer) +{ + char *line; + int done = 0; + + struct evkeyvalq* headers = req->input_headers; + while ((line = evbuffer_readline(buffer)) != NULL) { + char *skey, *svalue; + + if (*line == '\0') { /* Last header - Done */ + done = 1; + free (line); + break; + } + + /* Processing of header lines */ + if (req->got_firstline == 0) { + switch (req->kind) { + case EVHTTP_REQUEST: + if (evhttp_parse_request_line(req, line) == -1) + goto error; + break; + case EVHTTP_RESPONSE: + if (evhttp_parse_response_line(req, line) == -1) + goto error; + break; + default: + goto error; + } + req->got_firstline = 1; + } else { + /* Regular header */ + svalue = line; + skey = strsep(&svalue, ":"); + if (svalue == NULL) + goto error; + + svalue += strspn(svalue, " "); + + if (evhttp_add_header(headers, skey, svalue) == -1) + goto error; + } + + free (line); + } + + return (done); + + error: + free (line); + return (-1); +} + +static int +evhttp_get_body_length(struct evhttp_request *req) +{ + struct evkeyvalq *headers = req->input_headers; + const char *content_length; + const char *connection; + + content_length = evhttp_find_header(headers, "Content-Length"); + connection = evhttp_find_header(headers, "Connection"); + + if (content_length == NULL && connection == NULL) + req->ntoread = -1; + else if (content_length == NULL && + strcasecmp(connection, "Close") != 0) { + /* Bad combination, we don't know when it will end */ + event_warnx("%s: we got no content length, but the " + "server wants to keep the connection open: %s.", + __func__, connection); + return (-1); + } else if (content_length == NULL) { + req->ntoread = -1; + } else { + char *endp; + req->ntoread = evutil_strtoll(content_length, &endp, 10); + if (*content_length == '\0' || *endp != '\0') { + event_warnx("%s: illegal content length: %s", + __func__, content_length); + return (-1); + } + } + + event_debug(("%s: bytes to read: %d (in buffer %d)\n", + __func__, req->ntoread, + EVBUFFER_LENGTH(req->evcon->input_buffer))); + + return (0); +} + +static void +evhttp_get_body(struct evhttp_connection *evcon, struct evhttp_request *req) +{ + const char *xfer_enc; + + /* If this is a request without a body, then we are done */ + if (req->kind == EVHTTP_REQUEST && req->type != EVHTTP_REQ_POST) { + evhttp_connection_done(evcon); + return; + } + xfer_enc = evhttp_find_header(req->input_headers, "Transfer-Encoding"); + if (xfer_enc != NULL && strcasecmp(xfer_enc, "chunked") == 0) { + req->chunked = 1; + req->ntoread = -1; + } else { + if (evhttp_get_body_length(req) == -1) { + evhttp_connection_fail(evcon, + EVCON_HTTP_INVALID_HEADER); + return; + } + } + evhttp_read_body(evcon, req); +} + +void +evhttp_read_header(int fd, short what, void *arg) +{ + struct evhttp_connection *evcon = arg; + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + int n, res; + + if (what == EV_TIMEOUT) { + event_debug(("%s: timeout on %d\n", __func__, fd)); + evhttp_connection_fail(evcon, EVCON_HTTP_TIMEOUT); + return; + } + + n = evbuffer_read(evcon->input_buffer, fd, -1); + if (n == 0) { + event_debug(("%s: no more data on %d", __func__, fd)); + evhttp_connection_fail(evcon, EVCON_HTTP_EOF); + return; + } + if (n == -1) { + event_debug(("%s: bad read on %d", __func__, fd)); + evhttp_connection_fail(evcon, EVCON_HTTP_EOF); + return; + } + + res = evhttp_parse_lines(req, evcon->input_buffer); + if (res == -1) { + /* Error while reading, terminate */ + event_debug(("%s: bad header lines on %d\n", __func__, fd)); + evhttp_connection_fail(evcon, EVCON_HTTP_INVALID_HEADER); + return; + } else if (res == 0) { + /* Need more header lines */ + evhttp_add_event(&evcon->ev, + evcon->timeout, HTTP_READ_TIMEOUT); + return; + } + + /* Done reading headers, do the real work */ + switch (req->kind) { + case EVHTTP_REQUEST: + event_debug(("%s: checking for post data on %d\n", + __func__, fd)); + evhttp_get_body(evcon, req); + break; + + case EVHTTP_RESPONSE: + if (req->response_code == HTTP_NOCONTENT || + req->response_code == HTTP_NOTMODIFIED || + (req->response_code >= 100 && req->response_code < 200)) { + event_debug(("%s: skipping body for code %d\n", + __func__, req->response_code)); + evhttp_connection_done(evcon); + } else { + event_debug(("%s: start of read body for %s on %d\n", + __func__, req->remote_host, fd)); + evhttp_get_body(evcon, req); + } + break; + + default: + event_warnx("%s: bad header on %d", __func__, fd); + evhttp_connection_fail(evcon, EVCON_HTTP_INVALID_HEADER); + break; + } +} + +/* + * Creates a TCP connection to the specified port and executes a callback + * when finished. Failure or sucess is indicate by the passed connection + * object. + * + * Although this interface accepts a hostname, it is intended to take + * only numeric hostnames so that non-blocking DNS resolution can + * happen elsewhere. + */ + +struct evhttp_connection * +evhttp_connection_new(const char *address, unsigned short port) +{ + struct evhttp_connection *evcon = NULL; + + event_debug(("Attempting connection to %s:%d\n", address, port)); + + if ((evcon = calloc(1, sizeof(struct evhttp_connection))) == NULL) { + event_warn("%s: calloc failed", __func__); + goto error; + } + + evcon->fd = -1; + evcon->port = port; + + evcon->timeout = -1; + evcon->retry_cnt = evcon->retry_max = 0; + + if ((evcon->address = strdup(address)) == NULL) { + event_warn("%s: strdup failed", __func__); + goto error; + } + + if ((evcon->input_buffer = evbuffer_new()) == NULL) { + event_warn("%s: evbuffer_new failed", __func__); + goto error; + } + + if ((evcon->output_buffer = evbuffer_new()) == NULL) { + event_warn("%s: evbuffer_new failed", __func__); + goto error; + } + + evcon->state = EVCON_DISCONNECTED; + TAILQ_INIT(&evcon->requests); + + return (evcon); + + error: + if (evcon != NULL) + evhttp_connection_free(evcon); + return (NULL); +} + +void evhttp_connection_set_base(struct evhttp_connection *evcon, + struct event_base *base) +{ + assert(evcon->base == NULL); + assert(evcon->state == EVCON_DISCONNECTED); + evcon->base = base; +} + +void +evhttp_connection_set_timeout(struct evhttp_connection *evcon, + int timeout_in_secs) +{ + evcon->timeout = timeout_in_secs; +} + +void +evhttp_connection_set_retries(struct evhttp_connection *evcon, + int retry_max) +{ + evcon->retry_max = retry_max; +} + +void +evhttp_connection_set_closecb(struct evhttp_connection *evcon, + void (*cb)(struct evhttp_connection *, void *), void *cbarg) +{ + evcon->closecb = cb; + evcon->closecb_arg = cbarg; +} + +void +evhttp_connection_get_peer(struct evhttp_connection *evcon, + char **address, u_short *port) +{ + *address = evcon->address; + *port = evcon->port; +} + +int +evhttp_connection_connect(struct evhttp_connection *evcon) +{ + if (evcon->state == EVCON_CONNECTING) + return (0); + + evhttp_connection_reset(evcon); + + assert(!(evcon->flags & EVHTTP_CON_INCOMING)); + evcon->flags |= EVHTTP_CON_OUTGOING; + + evcon->fd = bind_socket(evcon->bind_address, 0); + if (evcon->fd == -1) { + event_debug(("%s: failed to bind to \"%s\"", + __func__, evcon->bind_address)); + return (-1); + } + + if (socket_connect(evcon->fd, evcon->address, evcon->port) == -1) { + EVUTIL_CLOSESOCKET(evcon->fd); evcon->fd = -1; + return (-1); + } + + /* Set up a callback for successful connection setup */ + event_set(&evcon->ev, evcon->fd, EV_WRITE, evhttp_connectioncb, evcon); + EVHTTP_BASE_SET(evcon, &evcon->ev); + evhttp_add_event(&evcon->ev, evcon->timeout, HTTP_CONNECT_TIMEOUT); + + evcon->state = EVCON_CONNECTING; + + return (0); +} + +/* + * Starts an HTTP request on the provided evhttp_connection object. + * If the connection object is not connected to the web server already, + * this will start the connection. + */ + +int +evhttp_make_request(struct evhttp_connection *evcon, + struct evhttp_request *req, + enum evhttp_cmd_type type, const char *uri) +{ + /* We are making a request */ + req->kind = EVHTTP_REQUEST; + req->type = type; + if (req->uri != NULL) + free(req->uri); + if ((req->uri = strdup(uri)) == NULL) + event_err(1, "%s: strdup", __func__); + + /* Set the protocol version if it is not supplied */ + if (!req->major && !req->minor) { + req->major = 1; + req->minor = 1; + } + + assert(req->evcon == NULL); + req->evcon = evcon; + assert(!(req->flags & EVHTTP_REQ_OWN_CONNECTION)); + + TAILQ_INSERT_TAIL(&evcon->requests, req, next); + + /* If the connection object is not connected; make it so */ + if (evcon->state != EVCON_CONNECTED) + return (evhttp_connection_connect(evcon)); + + /* + * If it's connected already and we are the first in the queue, + * then we can dispatch this request immediately. Otherwise, it + * will be dispatched once the pending requests are completed. + */ + if (TAILQ_FIRST(&evcon->requests) == req) + evhttp_request_dispatch(evcon); + + return (0); +} + +/* + * Reads data from file descriptor into request structure + * Request structure needs to be set up correctly. + */ + +void +evhttp_start_read(struct evhttp_connection *evcon) +{ + /* Set up an event to read the headers */ + if (event_initialized(&evcon->ev)) + event_del(&evcon->ev); + event_set(&evcon->ev, evcon->fd, EV_READ, evhttp_read_header, evcon); + EVHTTP_BASE_SET(evcon, &evcon->ev); + + evhttp_add_event(&evcon->ev, evcon->timeout, HTTP_READ_TIMEOUT); +} + +static void +evhttp_send_done(struct evhttp_connection *evcon, void *arg) +{ + int need_close; + struct evhttp_request *req = TAILQ_FIRST(&evcon->requests); + TAILQ_REMOVE(&evcon->requests, req, next); + + /* delete possible close detection events */ + evhttp_connection_stop_detectclose(evcon); + + need_close = + (req->minor == 0 && + !evhttp_is_connection_keepalive(req->input_headers))|| + evhttp_is_connection_close(req->flags, req->input_headers) || + evhttp_is_connection_close(req->flags, req->output_headers); + + assert(req->flags & EVHTTP_REQ_OWN_CONNECTION); + evhttp_request_free(req); + + if (need_close) { + evhttp_connection_free(evcon); + return; + } + + /* we have a persistent connection; try to accept another request. */ + if (evhttp_associate_new_request_with_connection(evcon) == -1) + evhttp_connection_free(evcon); +} + +/* + * Returns an error page. + */ + +void +evhttp_send_error(struct evhttp_request *req, int error, const char *reason) +{ +#define ERR_FORMAT "<HTML><HEAD>\n" \ + "<TITLE>%d %s</TITLE>\n" \ + "</HEAD><BODY>\n" \ + "<H1>Method Not Implemented</H1>\n" \ + "Invalid method in request<P>\n" \ + "</BODY></HTML>\n" + + struct evbuffer *buf = evbuffer_new(); + + /* close the connection on error */ + evhttp_add_header(req->output_headers, "Connection", "close"); + + evhttp_response_code(req, error, reason); + + evbuffer_add_printf(buf, ERR_FORMAT, error, reason); + + evhttp_send_page(req, buf); + + evbuffer_free(buf); +#undef ERR_FORMAT +} + +/* Requires that headers and response code are already set up */ + +static inline void +evhttp_send(struct evhttp_request *req, struct evbuffer *databuf) +{ + struct evhttp_connection *evcon = req->evcon; + + assert(TAILQ_FIRST(&evcon->requests) == req); + + /* xxx: not sure if we really should expose the data buffer this way */ + if (databuf != NULL) + evbuffer_add_buffer(req->output_buffer, databuf); + + /* Adds headers to the response */ + evhttp_make_header(evcon, req); + + evhttp_write_buffer(evcon, evhttp_send_done, NULL); +} + +void +evhttp_send_reply(struct evhttp_request *req, int code, const char *reason, + struct evbuffer *databuf) +{ + /* set up to watch for client close */ + evhttp_connection_start_detectclose(req->evcon); + evhttp_response_code(req, code, reason); + + evhttp_send(req, databuf); +} + +void +evhttp_send_reply_start(struct evhttp_request *req, int code, + const char *reason) +{ + /* set up to watch for client close */ + evhttp_connection_start_detectclose(req->evcon); + evhttp_response_code(req, code, reason); + if (req->major == 1 && req->minor == 1) { + /* use chunked encoding for HTTP/1.1 */ + evhttp_add_header(req->output_headers, "Transfer-Encoding", + "chunked"); + req->chunked = 1; + } + evhttp_make_header(req->evcon, req); + evhttp_write_buffer(req->evcon, NULL, NULL); +} + +void +evhttp_send_reply_chunk(struct evhttp_request *req, struct evbuffer *databuf) +{ + if (req->chunked) { + evbuffer_add_printf(req->evcon->output_buffer, "%x\r\n", + (unsigned)EVBUFFER_LENGTH(databuf)); + } + evbuffer_add_buffer(req->evcon->output_buffer, databuf); + if (req->chunked) { + evbuffer_add(req->evcon->output_buffer, "\r\n", 2); + } + evhttp_write_buffer(req->evcon, NULL, NULL); +} + +void +evhttp_send_reply_end(struct evhttp_request *req) +{ + struct evhttp_connection *evcon = req->evcon; + + if (req->chunked) { + evbuffer_add(req->evcon->output_buffer, "0\r\n\r\n", 5); + evhttp_write_buffer(req->evcon, evhttp_send_done, NULL); + req->chunked = 0; + } else if (!event_pending(&evcon->ev, EV_WRITE|EV_TIMEOUT, NULL)) { + /* let the connection know that we are done with the request */ + evhttp_send_done(evcon, NULL); + } else { + /* make the callback execute after all data has been written */ + evcon->cb = evhttp_send_done; + evcon->cb_arg = NULL; + } +} + +void +evhttp_response_code(struct evhttp_request *req, int code, const char *reason) +{ + req->kind = EVHTTP_RESPONSE; + req->response_code = code; + if (req->response_code_line != NULL) + free(req->response_code_line); + req->response_code_line = strdup(reason); +} + +void +evhttp_send_page(struct evhttp_request *req, struct evbuffer *databuf) +{ + if (!req->major || !req->minor) { + req->major = 1; + req->minor = 1; + } + + if (req->kind != EVHTTP_RESPONSE) + evhttp_response_code(req, 200, "OK"); + + evhttp_clear_headers(req->output_headers); + evhttp_add_header(req->output_headers, "Content-Type", "text/html"); + evhttp_add_header(req->output_headers, "Connection", "close"); + + evhttp_send(req, databuf); +} + +static const char uri_chars[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, + /* 64 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, + /* 128 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 192 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +/* + * Helper functions to encode/decode a URI. + * The returned string must be freed by the caller. + */ +char * +evhttp_encode_uri(const char *uri) +{ + struct evbuffer *buf = evbuffer_new(); + char *p; + + for (p = (char *)uri; *p != '\0'; p++) { + if (uri_chars[(u_char)(*p)]) { + evbuffer_add(buf, p, 1); + } else { + evbuffer_add_printf(buf, "%%%02X", (u_char)(*p)); + } + } + evbuffer_add(buf, "", 1); + p = strdup((char *)EVBUFFER_DATA(buf)); + evbuffer_free(buf); + + return (p); +} + +char * +evhttp_decode_uri(const char *uri) +{ + char c, *ret; + int i, j, in_query = 0; + + ret = malloc(strlen(uri) + 1); + if (ret == NULL) + event_err(1, "%s: malloc(%lu)", __func__, + (unsigned long)(strlen(uri) + 1)); + + for (i = j = 0; uri[i] != '\0'; i++) { + c = uri[i]; + if (c == '?') { + in_query = 1; + } else if (c == '+' && in_query) { + c = ' '; + } else if (c == '%' && isxdigit((unsigned char)uri[i+1]) && + isxdigit((unsigned char)uri[i+2])) { + char tmp[] = { uri[i+1], uri[i+2], '\0' }; + c = (char)strtol(tmp, NULL, 16); + i += 2; + } + ret[j++] = c; + } + ret[j] = '\0'; + + return (ret); +} + +/* + * Helper function to parse out arguments in a query. + * The arguments are separated by key and value. + * URI should already be decoded. + */ + +void +evhttp_parse_query(const char *uri, struct evkeyvalq *headers) +{ + char *line; + char *argument; + char *p; + + TAILQ_INIT(headers); + + /* No arguments - we are done */ + if (strchr(uri, '?') == NULL) + return; + + if ((line = strdup(uri)) == NULL) + event_err(1, "%s: strdup", __func__); + + + argument = line; + + /* We already know that there has to be a ? */ + strsep(&argument, "?"); + + p = argument; + while (p != NULL && *p != '\0') { + char *key, *value; + argument = strsep(&p, "&"); + + value = argument; + key = strsep(&value, "="); + if (value == NULL) + goto error; + + value = evhttp_decode_uri(value); + event_debug(("Query Param: %s -> %s\n", key, value)); + evhttp_add_header(headers, key, value); + free(value); + } + + error: + free(line); +} + +static struct evhttp_cb * +evhttp_dispatch_callback(struct httpcbq *callbacks, struct evhttp_request *req) +{ + struct evhttp_cb *cb; + size_t offset = 0; + + /* Test for different URLs */ + char *p = strchr(req->uri, '?'); + if (p != NULL) + offset = (size_t)(p - req->uri); + + TAILQ_FOREACH(cb, callbacks, next) { + int res = 0; + if (p == NULL) { + res = strcmp(cb->what, req->uri) == 0; + } else { + res = ((strncmp(cb->what, req->uri, offset) == 0) && + (cb->what[offset] == '\0')); + } + + if (res) + return (cb); + } + + return (NULL); +} + +static void +evhttp_handle_request(struct evhttp_request *req, void *arg) +{ + struct evhttp *http = arg; + struct evhttp_cb *cb = NULL; + + if (req->uri == NULL) { + evhttp_send_error(req, HTTP_BADREQUEST, "Bad Request"); + return; + } + + if ((cb = evhttp_dispatch_callback(&http->callbacks, req)) != NULL) { + (*cb->cb)(req, cb->cbarg); + return; + } + + /* Generic call back */ + if (http->gencb) { + (*http->gencb)(req, http->gencbarg); + return; + } else { + /* We need to send a 404 here */ +#define ERR_FORMAT "<html><head>" \ + "<title>404 Not Found</title>" \ + "</head><body>" \ + "<h1>Not Found</h1>" \ + "<p>The requested URL %s was not found on this server.</p>"\ + "</body></html>\n" + + char *escaped_html = evhttp_htmlescape(req->uri); + struct evbuffer *buf = evbuffer_new(); + + evhttp_response_code(req, HTTP_NOTFOUND, "Not Found"); + + evbuffer_add_printf(buf, ERR_FORMAT, escaped_html); + + free(escaped_html); + + evhttp_send_page(req, buf); + + evbuffer_free(buf); +#undef ERR_FORMAT + } +} + +static void +accept_socket(int fd, short what, void *arg) +{ + struct evhttp *http = arg; + struct sockaddr_storage ss; + socklen_t addrlen = sizeof(ss); + int nfd; + + if ((nfd = accept(fd, (struct sockaddr *)&ss, &addrlen)) == -1) { + event_warn("%s: bad accept", __func__); + return; + } + if (evutil_make_socket_nonblocking(nfd) < 0) + return; + + evhttp_get_request(http, nfd, (struct sockaddr *)&ss, addrlen); +} + +int +evhttp_bind_socket(struct evhttp *http, const char *address, u_short port) +{ + struct event *ev = &http->bind_ev; + int fd; + + if ((fd = bind_socket(address, port)) == -1) + return (-1); + + if (listen(fd, 10) == -1) { + event_warn("%s: listen", __func__); + EVUTIL_CLOSESOCKET(fd); + return (-1); + } + + /* Schedule the socket for accepting */ + event_set(ev, fd, EV_READ | EV_PERSIST, accept_socket, http); + EVHTTP_BASE_SET(http, ev); + event_add(ev, NULL); + + event_debug(("Bound to port %d - Awaiting connections ... ", port)); + + return (0); +} + +static struct evhttp* +evhttp_new_object(void) +{ + struct evhttp *http = NULL; + + if ((http = calloc(1, sizeof(struct evhttp))) == NULL) { + event_warn("%s: calloc", __func__); + return (NULL); + } + + http->timeout = -1; + + TAILQ_INIT(&http->callbacks); + TAILQ_INIT(&http->connections); + + return (http); +} + +struct evhttp * +evhttp_new(struct event_base *base) +{ + struct evhttp *http = evhttp_new_object(); + + http->base = base; + + return (http); +} + +/* + * Start a web server on the specified address and port. + */ + +struct evhttp * +evhttp_start(const char *address, u_short port) +{ + struct evhttp *http = evhttp_new_object(); + + if (evhttp_bind_socket(http, address, port) == -1) { + free(http); + return (NULL); + } + + return (http); +} + +void +evhttp_free(struct evhttp* http) +{ + struct evhttp_cb *http_cb; + struct evhttp_connection *evcon; + int fd = http->bind_ev.ev_fd; + + /* Remove the accepting part */ + event_del(&http->bind_ev); + EVUTIL_CLOSESOCKET(fd); + + while ((evcon = TAILQ_FIRST(&http->connections)) != NULL) { + /* evhttp_connection_free removes the connection */ + evhttp_connection_free(evcon); + } + + while ((http_cb = TAILQ_FIRST(&http->callbacks)) != NULL) { + TAILQ_REMOVE(&http->callbacks, http_cb, next); + free(http_cb->what); + free(http_cb); + } + + free(http); +} + +void +evhttp_set_timeout(struct evhttp* http, int timeout_in_secs) +{ + http->timeout = timeout_in_secs; +} + +void +evhttp_set_cb(struct evhttp *http, const char *uri, + void (*cb)(struct evhttp_request *, void *), void *cbarg) +{ + struct evhttp_cb *http_cb; + + if ((http_cb = calloc(1, sizeof(struct evhttp_cb))) == NULL) + event_err(1, "%s: calloc", __func__); + + http_cb->what = strdup(uri); + http_cb->cb = cb; + http_cb->cbarg = cbarg; + + TAILQ_INSERT_TAIL(&http->callbacks, http_cb, next); +} + +int +evhttp_del_cb(struct evhttp *http, const char *uri) +{ + struct evhttp_cb *http_cb; + + TAILQ_FOREACH(http_cb, &http->callbacks, next) { + if (strcmp(http_cb->what, uri) == 0) + break; + } + if (http_cb == NULL) + return (-1); + + TAILQ_REMOVE(&http->callbacks, http_cb, next); + free(http_cb->what); + free(http_cb); + + return (0); +} + +void +evhttp_set_gencb(struct evhttp *http, + void (*cb)(struct evhttp_request *, void *), void *cbarg) +{ + http->gencb = cb; + http->gencbarg = cbarg; +} + +/* + * Request related functions + */ + +struct evhttp_request * +evhttp_request_new(void (*cb)(struct evhttp_request *, void *), void *arg) +{ + struct evhttp_request *req = NULL; + + /* Allocate request structure */ + if ((req = calloc(1, sizeof(struct evhttp_request))) == NULL) { + event_warn("%s: calloc", __func__); + goto error; + } + + req->kind = EVHTTP_RESPONSE; + req->input_headers = calloc(1, sizeof(struct evkeyvalq)); + if (req->input_headers == NULL) { + event_warn("%s: calloc", __func__); + goto error; + } + TAILQ_INIT(req->input_headers); + + req->output_headers = calloc(1, sizeof(struct evkeyvalq)); + if (req->output_headers == NULL) { + event_warn("%s: calloc", __func__); + goto error; + } + TAILQ_INIT(req->output_headers); + + if ((req->input_buffer = evbuffer_new()) == NULL) { + event_warn("%s: evbuffer_new", __func__); + goto error; + } + + if ((req->output_buffer = evbuffer_new()) == NULL) { + event_warn("%s: evbuffer_new", __func__); + goto error; + } + + req->cb = cb; + req->cb_arg = arg; + + return (req); + + error: + if (req != NULL) + evhttp_request_free(req); + return (NULL); +} + +void +evhttp_request_free(struct evhttp_request *req) +{ + if (req->remote_host != NULL) + free(req->remote_host); + if (req->uri != NULL) + free(req->uri); + if (req->response_code_line != NULL) + free(req->response_code_line); + + evhttp_clear_headers(req->input_headers); + free(req->input_headers); + + evhttp_clear_headers(req->output_headers); + free(req->output_headers); + + if (req->input_buffer != NULL) + evbuffer_free(req->input_buffer); + + if (req->output_buffer != NULL) + evbuffer_free(req->output_buffer); + + free(req); +} + +void +evhttp_request_set_chunked_cb(struct evhttp_request *req, + void (*cb)(struct evhttp_request *, void *)) +{ + req->chunk_cb = cb; +} + +/* + * Allows for inspection of the request URI + */ + +const char * +evhttp_request_uri(struct evhttp_request *req) { + if (req->uri == NULL) + event_debug(("%s: request %p has no uri\n", req)); + return (req->uri); +} + +/* + * Takes a file descriptor to read a request from. + * The callback is executed once the whole request has been read. + */ + +static struct evhttp_connection* +evhttp_get_request_connection( + struct evhttp* http, + int fd, struct sockaddr *sa, socklen_t salen) +{ + struct evhttp_connection *evcon; + char *hostname, *portname; + + name_from_addr(sa, salen, &hostname, &portname); + event_debug(("%s: new request from %s:%s on %d\n", + __func__, hostname, portname, fd)); + + /* we need a connection object to put the http request on */ + if ((evcon = evhttp_connection_new(hostname, atoi(portname))) == NULL) + return (NULL); + + /* associate the base if we have one*/ + evhttp_connection_set_base(evcon, http->base); + + evcon->flags |= EVHTTP_CON_INCOMING; + evcon->state = EVCON_CONNECTED; + + evcon->fd = fd; + + return (evcon); +} + +static int +evhttp_associate_new_request_with_connection(struct evhttp_connection *evcon) +{ + struct evhttp *http = evcon->http_server; + struct evhttp_request *req; + if ((req = evhttp_request_new(evhttp_handle_request, http)) == NULL) + return (-1); + + req->evcon = evcon; /* the request ends up owning the connection */ + req->flags |= EVHTTP_REQ_OWN_CONNECTION; + + TAILQ_INSERT_TAIL(&evcon->requests, req, next); + + req->kind = EVHTTP_REQUEST; + + if ((req->remote_host = strdup(evcon->address)) == NULL) + event_err(1, "%s: strdup", __func__); + req->remote_port = evcon->port; + + evhttp_start_read(evcon); + + return (0); +} + +void +evhttp_get_request(struct evhttp *http, int fd, + struct sockaddr *sa, socklen_t salen) +{ + struct evhttp_connection *evcon; + + evcon = evhttp_get_request_connection(http, fd, sa, salen); + if (evcon == NULL) + return; + + /* the timeout can be used by the server to close idle connections */ + if (http->timeout != -1) + evhttp_connection_set_timeout(evcon, http->timeout); + + /* + * if we want to accept more than one request on a connection, + * we need to know which http server it belongs to. + */ + evcon->http_server = http; + TAILQ_INSERT_TAIL(&http->connections, evcon, next); + + if (evhttp_associate_new_request_with_connection(evcon) == -1) + evhttp_connection_free(evcon); +} + + +/* + * Network helper functions that we do not want to export to the rest of + * the world. + */ +#if 0 /* Unused */ +static struct addrinfo * +addr_from_name(char *address) +{ +#ifdef HAVE_GETADDRINFO + struct addrinfo ai, *aitop; + int ai_result; + + memset(&ai, 0, sizeof(ai)); + ai.ai_family = AF_INET; + ai.ai_socktype = SOCK_RAW; + ai.ai_flags = 0; + if ((ai_result = getaddrinfo(address, NULL, &ai, &aitop)) != 0) { + if ( ai_result == EAI_SYSTEM ) + event_warn("getaddrinfo"); + else + event_warnx("getaddrinfo: %s", gai_strerror(ai_result)); + } + + return (aitop); +#else + assert(0); + return NULL; /* XXXXX Use gethostbyname, if this function is ever used. */ +#endif +} +#endif + +static void +name_from_addr(struct sockaddr *sa, socklen_t salen, + char **phost, char **pport) +{ +#ifdef HAVE_GETNAMEINFO + /* XXXX not threadsafe. */ + static char ntop[NI_MAXHOST]; + static char strport[NI_MAXSERV]; + int ni_result; + + if ((ni_result = getnameinfo(sa, salen, + ntop, sizeof(ntop), strport, sizeof(strport), + NI_NUMERICHOST|NI_NUMERICSERV)) != 0) { + if (ni_result == EAI_SYSTEM) + event_err(1, "getnameinfo failed"); + else + event_errx(1, "getnameinfo failed: %s", gai_strerror(ni_result)); + } + + *phost = ntop; + *pport = strport; +#else + /* XXXX */ +#endif +} + +/* Either connect or bind */ + +static int +bind_socket_ai(struct addrinfo *ai) +{ + int fd, on = 1, r; + int serrno; + + /* Create listen socket */ + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == -1) { + event_warn("socket"); + return (-1); + } + + if (evutil_make_socket_nonblocking(fd) < 0) + goto out; + +#ifndef WIN32 + if (fcntl(fd, F_SETFD, 1) == -1) { + event_warn("fcntl(F_SETFD)"); + goto out; + } +#endif + + setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&on, sizeof(on)); + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void *)&on, sizeof(on)); + + r = bind(fd, ai->ai_addr, ai->ai_addrlen); + if (r == -1) + goto out; + + return (fd); + + out: + serrno = EVUTIL_SOCKET_ERROR(); + EVUTIL_CLOSESOCKET(fd); + EVUTIL_SET_SOCKET_ERROR(serrno); + return (-1); +} + +static struct addrinfo * +make_addrinfo(const char *address, u_short port) +{ + struct addrinfo *aitop = NULL; + +#ifdef HAVE_GETADDRINFO + struct addrinfo ai; + char strport[NI_MAXSERV]; + int ai_result; + + memset(&ai, 0, sizeof(ai)); + ai.ai_family = AF_INET; + ai.ai_socktype = SOCK_STREAM; + ai.ai_flags = AI_PASSIVE; /* turn NULL host name into INADDR_ANY */ + snprintf(strport, sizeof(strport), "%d", port); + if ((ai_result = getaddrinfo(address, strport, &ai, &aitop)) != 0) { + if ( ai_result == EAI_SYSTEM ) + event_warn("getaddrinfo"); + else + event_warnx("getaddrinfo: %s", gai_strerror(ai_result)); + return (NULL); + } +#else + static int cur; + static struct addrinfo ai[2]; /* We will be returning the address of some of this memory so it has to last even after this call. */ + if (++cur == 2) cur = 0; /* allow calling this function twice */ + + if (fake_getaddrinfo(address, &ai[cur]) < 0) { + event_warn("fake_getaddrinfo"); + return (NULL); + } + aitop = &ai[cur]; + ((struct sockaddr_in *) aitop->ai_addr)->sin_port = htons(port); +#endif + + return (aitop); +} + +static int +bind_socket(const char *address, u_short port) +{ + int fd; + struct addrinfo *aitop = make_addrinfo(address, port); + + if (aitop == NULL) + return (-1); + + fd = bind_socket_ai(aitop); + +#ifdef HAVE_GETADDRINFO + freeaddrinfo(aitop); +#else + fake_freeaddrinfo(aitop); +#endif + + return (fd); +} + +static int +socket_connect(int fd, const char *address, unsigned short port) +{ + struct addrinfo *ai = make_addrinfo(address, port); + int res = -1; + + if (ai == NULL) { + event_debug(("%s: make_addrinfo: \"%s:%d\"", + __func__, address, port)); + return (-1); + } + + if (connect(fd, ai->ai_addr, ai->ai_addrlen) == -1) { +#ifdef WIN32 + int tmp_error = WSAGetLastError(); + if (tmp_error != WSAEWOULDBLOCK && tmp_error != WSAEINVAL && + tmp_error != WSAEINPROGRESS) { + goto out; + } +#else + if (errno != EINPROGRESS) { + goto out; + } +#endif + } + + /* everything is fine */ + res = 0; + +out: +#ifdef HAVE_GETADDRINFO + freeaddrinfo(ai); +#else + fake_freeaddrinfo(ai); +#endif + + return (res); +} === added file 'extra/libevent/kqueue.c' --- a/extra/libevent/kqueue.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/kqueue.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,425 @@ +/* $OpenBSD: kqueue.c,v 1.5 2002/07/10 14:41:31 art Exp $ */ + +/* + * Copyright 2000-2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_WORKING_KQUEUE + +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <sys/queue.h> +#include <sys/event.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#ifdef HAVE_INTTYPES_H +#include <inttypes.h> +#endif + +/* Some platforms apparently define the udata field of struct kevent as + * intptr_t, whereas others define it as void*. There doesn't seem to be an + * easy way to tell them apart via autoconf, so we need to use OS macros. */ +#if defined(HAVE_INTTYPES_H) && !defined(__OpenBSD__) && !defined(__FreeBSD__) && !defined(__darwin__) && !defined(__APPLE__) +#define PTR_TO_UDATA(x) ((intptr_t)(x)) +#else +#define PTR_TO_UDATA(x) (x) +#endif + +#include "event.h" +#include "event-internal.h" +#include "log.h" +#include "event-internal.h" + +#define EVLIST_X_KQINKERNEL 0x1000 + +#define NEVENT 64 + +struct kqop { + struct kevent *changes; + int nchanges; + struct kevent *events; + int nevents; + int kq; + pid_t pid; +}; + +static void *kq_init (struct event_base *); +static int kq_add (void *, struct event *); +static int kq_del (void *, struct event *); +static int kq_dispatch (struct event_base *, void *, struct timeval *); +static int kq_insert (struct kqop *, struct kevent *); +static void kq_dealloc (struct event_base *, void *); + +const struct eventop kqops = { + "kqueue", + kq_init, + kq_add, + kq_del, + kq_dispatch, + kq_dealloc, + 1 /* need reinit */ +}; + +static void * +kq_init(struct event_base *base) +{ + int kq; + struct kqop *kqueueop; + + /* Disable kqueue when this environment variable is set */ + if (getenv("EVENT_NOKQUEUE")) + return (NULL); + + if (!(kqueueop = calloc(1, sizeof(struct kqop)))) + return (NULL); + + /* Initalize the kernel queue */ + + if ((kq = kqueue()) == -1) { + event_warn("kqueue"); + free (kqueueop); + return (NULL); + } + + kqueueop->kq = kq; + + kqueueop->pid = getpid(); + + /* Initalize fields */ + kqueueop->changes = malloc(NEVENT * sizeof(struct kevent)); + if (kqueueop->changes == NULL) { + free (kqueueop); + return (NULL); + } + kqueueop->events = malloc(NEVENT * sizeof(struct kevent)); + if (kqueueop->events == NULL) { + free (kqueueop->changes); + free (kqueueop); + return (NULL); + } + kqueueop->nevents = NEVENT; + + /* Check for Mac OS X kqueue bug. */ + kqueueop->changes[0].ident = -1; + kqueueop->changes[0].filter = EVFILT_READ; + kqueueop->changes[0].flags = EV_ADD; + /* + * If kqueue works, then kevent will succeed, and it will + * stick an error in events[0]. If kqueue is broken, then + * kevent will fail. + */ + if (kevent(kq, + kqueueop->changes, 1, kqueueop->events, NEVENT, NULL) != 1 || + kqueueop->events[0].ident != -1 || + kqueueop->events[0].flags != EV_ERROR) { + event_warn("%s: detected broken kqueue; not using.", __func__); + free(kqueueop->changes); + free(kqueueop->events); + free(kqueueop); + close(kq); + return (NULL); + } + + return (kqueueop); +} + +static int +kq_insert(struct kqop *kqop, struct kevent *kev) +{ + int nevents = kqop->nevents; + + if (kqop->nchanges == nevents) { + struct kevent *newchange; + struct kevent *newresult; + + nevents *= 2; + + newchange = realloc(kqop->changes, + nevents * sizeof(struct kevent)); + if (newchange == NULL) { + event_warn("%s: malloc", __func__); + return (-1); + } + kqop->changes = newchange; + + newresult = realloc(kqop->events, + nevents * sizeof(struct kevent)); + + /* + * If we fail, we don't have to worry about freeing, + * the next realloc will pick it up. + */ + if (newresult == NULL) { + event_warn("%s: malloc", __func__); + return (-1); + } + kqop->events = newresult; + + kqop->nevents = nevents; + } + + memcpy(&kqop->changes[kqop->nchanges++], kev, sizeof(struct kevent)); + + event_debug(("%s: fd %d %s%s", + __func__, kev->ident, + kev->filter == EVFILT_READ ? "EVFILT_READ" : "EVFILT_WRITE", + kev->flags == EV_DELETE ? " (del)" : "")); + + return (0); +} + +static void +kq_sighandler(int sig) +{ + /* Do nothing here */ +} + +static int +kq_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + struct kqop *kqop = arg; + struct kevent *changes = kqop->changes; + struct kevent *events = kqop->events; + struct event *ev; + struct timespec ts, *ts_p = NULL; + int i, res; + + if (tv != NULL) { + TIMEVAL_TO_TIMESPEC(tv, &ts); + ts_p = &ts; + } + + res = kevent(kqop->kq, changes, kqop->nchanges, + events, kqop->nevents, ts_p); + kqop->nchanges = 0; + if (res == -1) { + if (errno != EINTR) { + event_warn("kevent"); + return (-1); + } + + return (0); + } + + event_debug(("%s: kevent reports %d", __func__, res)); + + for (i = 0; i < res; i++) { + int which = 0; + + if (events[i].flags & EV_ERROR) { + /* + * Error messages that can happen, when a delete fails. + * EBADF happens when the file discriptor has been + * closed, + * ENOENT when the file discriptor was closed and + * then reopened. + * EINVAL for some reasons not understood; EINVAL + * should not be returned ever; but FreeBSD does :-\ + * An error is also indicated when a callback deletes + * an event we are still processing. In that case + * the data field is set to ENOENT. + */ + if (events[i].data == EBADF || + events[i].data == EINVAL || + events[i].data == ENOENT) + continue; + errno = events[i].data; + return (-1); + } + + ev = (struct event *)events[i].udata; + + if (events[i].filter == EVFILT_READ) { + which |= EV_READ; + } else if (events[i].filter == EVFILT_WRITE) { + which |= EV_WRITE; + } else if (events[i].filter == EVFILT_SIGNAL) { + which |= EV_SIGNAL; + } + + if (!which) + continue; + + if (!(ev->ev_events & EV_PERSIST)) + ev->ev_flags &= ~EVLIST_X_KQINKERNEL; + + event_active(ev, which, + ev->ev_events & EV_SIGNAL ? events[i].data : 1); + } + + return (0); +} + + +static int +kq_add(void *arg, struct event *ev) +{ + struct kqop *kqop = arg; + struct kevent kev; + + if (ev->ev_events & EV_SIGNAL) { + int nsignal = EVENT_SIGNAL(ev); + struct timespec timeout = { 0, 0 }; + + memset(&kev, 0, sizeof(kev)); + kev.ident = nsignal; + kev.filter = EVFILT_SIGNAL; + kev.flags = EV_ADD; + if (!(ev->ev_events & EV_PERSIST)) + kev.flags |= EV_ONESHOT; + kev.udata = PTR_TO_UDATA(ev); + + /* Be ready for the signal if it is sent any time between + * now and the next call to kq_dispatch. */ + if (kevent(kqop->kq, &kev, 1, NULL, 0, &timeout) == -1) + return (-1); + + if (_evsignal_set_handler(ev->ev_base, nsignal, + kq_sighandler) == -1) + return (-1); + + ev->ev_flags |= EVLIST_X_KQINKERNEL; + return (0); + } + + if (ev->ev_events & EV_READ) { + memset(&kev, 0, sizeof(kev)); + kev.ident = ev->ev_fd; + kev.filter = EVFILT_READ; +#ifdef NOTE_EOF + /* Make it behave like select() and poll() */ + kev.fflags = NOTE_EOF; +#endif + kev.flags = EV_ADD; + if (!(ev->ev_events & EV_PERSIST)) + kev.flags |= EV_ONESHOT; + kev.udata = PTR_TO_UDATA(ev); + + if (kq_insert(kqop, &kev) == -1) + return (-1); + + ev->ev_flags |= EVLIST_X_KQINKERNEL; + } + + if (ev->ev_events & EV_WRITE) { + memset(&kev, 0, sizeof(kev)); + kev.ident = ev->ev_fd; + kev.filter = EVFILT_WRITE; + kev.flags = EV_ADD; + if (!(ev->ev_events & EV_PERSIST)) + kev.flags |= EV_ONESHOT; + kev.udata = PTR_TO_UDATA(ev); + + if (kq_insert(kqop, &kev) == -1) + return (-1); + + ev->ev_flags |= EVLIST_X_KQINKERNEL; + } + + return (0); +} + +static int +kq_del(void *arg, struct event *ev) +{ + struct kqop *kqop = arg; + struct kevent kev; + + if (!(ev->ev_flags & EVLIST_X_KQINKERNEL)) + return (0); + + if (ev->ev_events & EV_SIGNAL) { + int nsignal = EVENT_SIGNAL(ev); + + memset(&kev, 0, sizeof(kev)); + kev.ident = nsignal; + kev.filter = EVFILT_SIGNAL; + kev.flags = EV_DELETE; + + if (kq_insert(kqop, &kev) == -1) + return (-1); + + if (_evsignal_restore_handler(ev->ev_base, nsignal) == -1) + return (-1); + + ev->ev_flags &= ~EVLIST_X_KQINKERNEL; + return (0); + } + + if (ev->ev_events & EV_READ) { + memset(&kev, 0, sizeof(kev)); + kev.ident = ev->ev_fd; + kev.filter = EVFILT_READ; + kev.flags = EV_DELETE; + + if (kq_insert(kqop, &kev) == -1) + return (-1); + + ev->ev_flags &= ~EVLIST_X_KQINKERNEL; + } + + if (ev->ev_events & EV_WRITE) { + memset(&kev, 0, sizeof(kev)); + kev.ident = ev->ev_fd; + kev.filter = EVFILT_WRITE; + kev.flags = EV_DELETE; + + if (kq_insert(kqop, &kev) == -1) + return (-1); + + ev->ev_flags &= ~EVLIST_X_KQINKERNEL; + } + + return (0); +} + +static void +kq_dealloc(struct event_base *base, void *arg) +{ + struct kqop *kqop = arg; + + if (kqop->changes) + free(kqop->changes); + if (kqop->events) + free(kqop->events); + if (kqop->kq >= 0 && kqop->pid == getpid()) + close(kqop->kq); + memset(kqop, 0, sizeof(struct kqop)); + free(kqop); +} + +#endif /* HAVE_WORKING_KQUEUE */ === added file 'extra/libevent/log.c' --- a/extra/libevent/log.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/log.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,218 @@ +/* $OpenBSD: err.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ + +/* + * log.c + * + * Based on err.c, which was adapted from OpenBSD libc *err* *warn* code. + * + * Copyright (c) 2005 Nick Mathewson <nickm(a)freehaven.net> + * + * Copyright (c) 2000 Dug Song <dugsong(a)monkey.org> + * + * Copyright (c) 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#undef WIN32_LEAN_AND_MEAN +#include "misc.h" +#endif +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <string.h> +#include <errno.h> +#include "event.h" + +#include "log.h" + +static void _warn_helper(int severity, int log_errno, const char *fmt, + va_list ap); +static void event_log(int severity, const char *msg); + +static int +event_vsnprintf(char *str, size_t size, const char *format, va_list args) +{ + int r; + if (size == 0) + return -1; +#ifdef WIN32 + r = _vsnprintf(str, size, format, args); +#else + r = vsnprintf(str, size, format, args); +#endif + str[size-1] = '\0'; + if (r < 0 || ((size_t)r) >= size) { + /* different platforms behave differently on overflow; + * handle both kinds. */ + return -1; + } + return r; +} + +static int +event_snprintf(char *str, size_t size, const char *format, ...) +{ + va_list ap; + int r; + va_start(ap, format); + r = event_vsnprintf(str, size, format, ap); + va_end(ap); + return r; +} + +void +event_err(int eval, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_ERR, errno, fmt, ap); + va_end(ap); + exit(eval); +} + +void +event_warn(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_WARN, errno, fmt, ap); + va_end(ap); +} + +void +event_errx(int eval, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_ERR, -1, fmt, ap); + va_end(ap); + exit(eval); +} + +void +event_warnx(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_WARN, -1, fmt, ap); + va_end(ap); +} + +void +event_msgx(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_MSG, -1, fmt, ap); + va_end(ap); +} + +void +_event_debugx(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + _warn_helper(_EVENT_LOG_DEBUG, -1, fmt, ap); + va_end(ap); +} + +static void +_warn_helper(int severity, int log_errno, const char *fmt, va_list ap) +{ + char buf[1024]; + size_t len; + + if (fmt != NULL) + event_vsnprintf(buf, sizeof(buf), fmt, ap); + else + buf[0] = '\0'; + + if (log_errno >= 0) { + len = strlen(buf); + if (len < sizeof(buf) - 3) { + event_snprintf(buf + len, sizeof(buf) - len, ": %s", + strerror(log_errno)); + } + } + + event_log(severity, buf); +} + +static event_log_cb log_fn = NULL; + +void +event_set_log_callback(event_log_cb cb) +{ + log_fn = cb; +} + +static void +event_log(int severity, const char *msg) +{ + if (log_fn) + log_fn(severity, msg); + else { + const char *severity_str; + switch (severity) { + case _EVENT_LOG_DEBUG: + severity_str = "debug"; + break; + case _EVENT_LOG_MSG: + severity_str = "msg"; + break; + case _EVENT_LOG_WARN: + severity_str = "warn"; + break; + case _EVENT_LOG_ERR: + severity_str = "err"; + break; + default: + severity_str = "???"; + break; + } + (void)fprintf(stderr, "[%s] %s\n", severity_str, msg); + } +} === added file 'extra/libevent/log.h' --- a/extra/libevent/log.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/log.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2000-2004 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _LOG_H_ +#define _LOG_H_ + +#ifdef __GNUC__ +#define EV_CHECK_FMT(a,b) __attribute__((format(printf, a, b))) +#else +#define EV_CHECK_FMT(a,b) +#endif + +void event_err(int eval, const char *fmt, ...) EV_CHECK_FMT(2,3); +void event_warn(const char *fmt, ...) EV_CHECK_FMT(1,2); +void event_errx(int eval, const char *fmt, ...) EV_CHECK_FMT(2,3); +void event_warnx(const char *fmt, ...) EV_CHECK_FMT(1,2); +void event_msgx(const char *fmt, ...) EV_CHECK_FMT(1,2); +void _event_debugx(const char *fmt, ...) EV_CHECK_FMT(1,2); + +#ifdef USE_DEBUG +#define event_debug(x) _event_debugx x +#else +#define event_debug(x) do {;} while (0) +#endif + +#undef EV_CHECK_FMT + +#endif === added file 'extra/libevent/min_heap.h' --- a/extra/libevent/min_heap.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/min_heap.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin(a)gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _MIN_HEAP_H_ +#define _MIN_HEAP_H_ + +#include "event.h" + +typedef struct min_heap +{ + struct event** p; + unsigned n, a; +} min_heap_t; + +static inline void min_heap_ctor(min_heap_t* s); +static inline void min_heap_dtor(min_heap_t* s); +static inline void min_heap_elem_init(struct event* e); +static inline int min_heap_elem_greater(struct event *a, struct event *b); +static inline int min_heap_empty(min_heap_t* s); +static inline unsigned min_heap_size(min_heap_t* s); +static inline struct event* min_heap_top(min_heap_t* s); +static inline int min_heap_reserve(min_heap_t* s, unsigned n); +static inline int min_heap_push(min_heap_t* s, struct event* e); +static inline struct event* min_heap_pop(min_heap_t* s); +static inline int min_heap_erase(min_heap_t* s, struct event* e); +static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e); +static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e); + +int min_heap_elem_greater(struct event *a, struct event *b) +{ + return timercmp(&a->ev_timeout, &b->ev_timeout, >); +} + +void min_heap_ctor(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } +void min_heap_dtor(min_heap_t* s) { free(s->p); } +void min_heap_elem_init(struct event* e) { e->min_heap_idx = -1; } +int min_heap_empty(min_heap_t* s) { return 0u == s->n; } +unsigned min_heap_size(min_heap_t* s) { return s->n; } +struct event* min_heap_top(min_heap_t* s) { return s->n ? *s->p : 0; } + +int min_heap_push(min_heap_t* s, struct event* e) +{ + if(min_heap_reserve(s, s->n + 1)) + return -1; + min_heap_shift_up_(s, s->n++, e); + return 0; +} + +struct event* min_heap_pop(min_heap_t* s) +{ + if(s->n) + { + struct event* e = *s->p; + e->min_heap_idx = -1; + min_heap_shift_down_(s, 0u, s->p[--s->n]); + return e; + } + return 0; +} + +int min_heap_erase(min_heap_t* s, struct event* e) +{ + if(((unsigned int)-1) != e->min_heap_idx) + { + min_heap_shift_down_(s, e->min_heap_idx, s->p[--s->n]); + e->min_heap_idx = -1; + return 0; + } + return -1; +} + +int min_heap_reserve(min_heap_t* s, unsigned n) +{ + if(s->a < n) + { + struct event** p; + unsigned a = s->a ? s->a * 2 : 8; + if(a < n) + a = n; + if(!(p = (struct event**)realloc(s->p, a * sizeof *p))) + return -1; + s->p = p; + s->a = a; + } + return 0; +} + +void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e) +{ + unsigned parent = (hole_index - 1) / 2; + while(hole_index && min_heap_elem_greater(s->p[parent], e)) + { + (s->p[hole_index] = s->p[parent])->min_heap_idx = hole_index; + hole_index = parent; + parent = (hole_index - 1) / 2; + } + (s->p[hole_index] = e)->min_heap_idx = hole_index; +} + +void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e) +{ + unsigned min_child = 2 * (hole_index + 1); + while(min_child <= s->n) + { + min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); + if(!(min_heap_elem_greater(e, s->p[min_child]))) + break; + (s->p[hole_index] = s->p[min_child])->min_heap_idx = hole_index; + hole_index = min_child; + min_child = 2 * (hole_index + 1); + } + min_heap_shift_up_(s, hole_index, e); +} + +#endif /* _MIN_HEAP_H_ */ === added file 'extra/libevent/poll.c' --- a/extra/libevent/poll.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/poll.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,378 @@ +/* $OpenBSD: poll.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ + +/* + * Copyright 2000-2003 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_POLL + +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#include <sys/queue.h> +#include <poll.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#ifdef CHECK_INVARIANTS +#include <assert.h> +#endif + +#include "event.h" +#include "event-internal.h" +#include "evsignal.h" +#include "log.h" + +struct pollop { + int event_count; /* Highest number alloc */ + int nfds; /* Size of event_* */ + int fd_count; /* Size of idxplus1_by_fd */ + struct pollfd *event_set; + struct event **event_r_back; + struct event **event_w_back; + int *idxplus1_by_fd; /* Index into event_set by fd; we add 1 so + * that 0 (which is easy to memset) can mean + * "no entry." */ +}; + +static void *poll_init (struct event_base *); +static int poll_add (void *, struct event *); +static int poll_del (void *, struct event *); +static int poll_dispatch (struct event_base *, void *, struct timeval *); +static void poll_dealloc (struct event_base *, void *); + +const struct eventop pollops = { + "poll", + poll_init, + poll_add, + poll_del, + poll_dispatch, + poll_dealloc, + 0 +}; + +static void * +poll_init(struct event_base *base) +{ + struct pollop *pollop; + + /* Disable poll when this environment variable is set */ + if (getenv("EVENT_NOPOLL")) + return (NULL); + + if (!(pollop = calloc(1, sizeof(struct pollop)))) + return (NULL); + + evsignal_init(base); + + return (pollop); +} + +#ifdef CHECK_INVARIANTS +static void +poll_check_ok(struct pollop *pop) +{ + int i, idx; + struct event *ev; + + for (i = 0; i < pop->fd_count; ++i) { + idx = pop->idxplus1_by_fd[i]-1; + if (idx < 0) + continue; + assert(pop->event_set[idx].fd == i); + if (pop->event_set[idx].events & POLLIN) { + ev = pop->event_r_back[idx]; + assert(ev); + assert(ev->ev_events & EV_READ); + assert(ev->ev_fd == i); + } + if (pop->event_set[idx].events & POLLOUT) { + ev = pop->event_w_back[idx]; + assert(ev); + assert(ev->ev_events & EV_WRITE); + assert(ev->ev_fd == i); + } + } + for (i = 0; i < pop->nfds; ++i) { + struct pollfd *pfd = &pop->event_set[i]; + assert(pop->idxplus1_by_fd[pfd->fd] == i+1); + } +} +#else +#define poll_check_ok(pop) +#endif + +static int +poll_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + int res, i, msec = -1, nfds; + struct pollop *pop = arg; + + poll_check_ok(pop); + + if (tv != NULL) + msec = tv->tv_sec * 1000 + (tv->tv_usec + 999) / 1000; + + nfds = pop->nfds; + res = poll(pop->event_set, nfds, msec); + + if (res == -1) { + if (errno != EINTR) { + event_warn("poll"); + return (-1); + } + + evsignal_process(base); + return (0); + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + event_debug(("%s: poll reports %d", __func__, res)); + + if (res == 0) + return (0); + + for (i = 0; i < nfds; i++) { + int what = pop->event_set[i].revents; + struct event *r_ev = NULL, *w_ev = NULL; + if (!what) + continue; + + res = 0; + + /* If the file gets closed notify */ + if (what & (POLLHUP|POLLERR)) + what |= POLLIN|POLLOUT; + if (what & POLLIN) { + res |= EV_READ; + r_ev = pop->event_r_back[i]; + } + if (what & POLLOUT) { + res |= EV_WRITE; + w_ev = pop->event_w_back[i]; + } + if (res == 0) + continue; + + if (r_ev && (res & r_ev->ev_events)) { + event_active(r_ev, res & r_ev->ev_events, 1); + } + if (w_ev && w_ev != r_ev && (res & w_ev->ev_events)) { + event_active(w_ev, res & w_ev->ev_events, 1); + } + } + + return (0); +} + +static int +poll_add(void *arg, struct event *ev) +{ + struct pollop *pop = arg; + struct pollfd *pfd = NULL; + int i; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_add(ev)); + if (!(ev->ev_events & (EV_READ|EV_WRITE))) + return (0); + + poll_check_ok(pop); + if (pop->nfds + 1 >= pop->event_count) { + struct pollfd *tmp_event_set; + struct event **tmp_event_r_back; + struct event **tmp_event_w_back; + int tmp_event_count; + + if (pop->event_count < 32) + tmp_event_count = 32; + else + tmp_event_count = pop->event_count * 2; + + /* We need more file descriptors */ + tmp_event_set = realloc(pop->event_set, + tmp_event_count * sizeof(struct pollfd)); + if (tmp_event_set == NULL) { + event_warn("realloc"); + return (-1); + } + pop->event_set = tmp_event_set; + + tmp_event_r_back = realloc(pop->event_r_back, + tmp_event_count * sizeof(struct event *)); + if (tmp_event_r_back == NULL) { + /* event_set overallocated; that's okay. */ + event_warn("realloc"); + return (-1); + } + pop->event_r_back = tmp_event_r_back; + + tmp_event_w_back = realloc(pop->event_w_back, + tmp_event_count * sizeof(struct event *)); + if (tmp_event_w_back == NULL) { + /* event_set and event_r_back overallocated; that's + * okay. */ + event_warn("realloc"); + return (-1); + } + pop->event_w_back = tmp_event_w_back; + + pop->event_count = tmp_event_count; + } + if (ev->ev_fd >= pop->fd_count) { + int *tmp_idxplus1_by_fd; + int new_count; + if (pop->fd_count < 32) + new_count = 32; + else + new_count = pop->fd_count * 2; + while (new_count <= ev->ev_fd) + new_count *= 2; + tmp_idxplus1_by_fd = + realloc(pop->idxplus1_by_fd, new_count * sizeof(int)); + if (tmp_idxplus1_by_fd == NULL) { + event_warn("realloc"); + return (-1); + } + pop->idxplus1_by_fd = tmp_idxplus1_by_fd; + memset(pop->idxplus1_by_fd + pop->fd_count, + 0, sizeof(int)*(new_count - pop->fd_count)); + pop->fd_count = new_count; + } + + i = pop->idxplus1_by_fd[ev->ev_fd] - 1; + if (i >= 0) { + pfd = &pop->event_set[i]; + } else { + i = pop->nfds++; + pfd = &pop->event_set[i]; + pfd->events = 0; + pfd->fd = ev->ev_fd; + pop->event_w_back[i] = pop->event_r_back[i] = NULL; + pop->idxplus1_by_fd[ev->ev_fd] = i + 1; + } + + pfd->revents = 0; + if (ev->ev_events & EV_WRITE) { + pfd->events |= POLLOUT; + pop->event_w_back[i] = ev; + } + if (ev->ev_events & EV_READ) { + pfd->events |= POLLIN; + pop->event_r_back[i] = ev; + } + poll_check_ok(pop); + + return (0); +} + +/* + * Nothing to be done here. + */ + +static int +poll_del(void *arg, struct event *ev) +{ + struct pollop *pop = arg; + struct pollfd *pfd = NULL; + int i; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_del(ev)); + + if (!(ev->ev_events & (EV_READ|EV_WRITE))) + return (0); + + poll_check_ok(pop); + i = pop->idxplus1_by_fd[ev->ev_fd] - 1; + if (i < 0) + return (-1); + + /* Do we still want to read or write? */ + pfd = &pop->event_set[i]; + if (ev->ev_events & EV_READ) { + pfd->events &= ~POLLIN; + pop->event_r_back[i] = NULL; + } + if (ev->ev_events & EV_WRITE) { + pfd->events &= ~POLLOUT; + pop->event_w_back[i] = NULL; + } + poll_check_ok(pop); + if (pfd->events) + /* Another event cares about that fd. */ + return (0); + + /* Okay, so we aren't interested in that fd anymore. */ + pop->idxplus1_by_fd[ev->ev_fd] = 0; + + --pop->nfds; + if (i != pop->nfds) { + /* + * Shift the last pollfd down into the now-unoccupied + * position. + */ + memcpy(&pop->event_set[i], &pop->event_set[pop->nfds], + sizeof(struct pollfd)); + pop->event_r_back[i] = pop->event_r_back[pop->nfds]; + pop->event_w_back[i] = pop->event_w_back[pop->nfds]; + pop->idxplus1_by_fd[pop->event_set[i].fd] = i + 1; + } + + poll_check_ok(pop); + return (0); +} + +static void +poll_dealloc(struct event_base *base, void *arg) +{ + struct pollop *pop = arg; + + evsignal_dealloc(base); + if (pop->event_set) + free(pop->event_set); + if (pop->event_r_back) + free(pop->event_r_back); + if (pop->event_w_back) + free(pop->event_w_back); + if (pop->idxplus1_by_fd) + free(pop->idxplus1_by_fd); + + memset(pop, 0, sizeof(struct pollop)); + free(pop); +} + +#endif /* HAVE_POLL */ === added file 'extra/libevent/select.c' --- a/extra/libevent/select.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/select.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,356 @@ +/* $OpenBSD: select.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ + +/* + * Copyright 2000-2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_SELECT + +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#else +#include <sys/_time.h> +#endif +#ifdef HAVE_SYS_SELECT_H +#include <sys/select.h> +#endif +#include <sys/queue.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> +#ifdef CHECK_INVARIANTS +#include <assert.h> +#endif + +#include "event.h" +#include "event-internal.h" +#include "evsignal.h" +#include "log.h" + +#ifndef howmany +#define howmany(x, y) (((x)+((y)-1))/(y)) +#endif + +struct selectop { + int event_fds; /* Highest fd in fd set */ + int event_fdsz; + fd_set *event_readset_in; + fd_set *event_writeset_in; + fd_set *event_readset_out; + fd_set *event_writeset_out; + struct event **event_r_by_fd; + struct event **event_w_by_fd; +}; + +static void *select_init (struct event_base *); +static int select_add (void *, struct event *); +static int select_del (void *, struct event *); +static int select_dispatch (struct event_base *, void *, struct timeval *); +static void select_dealloc (struct event_base *, void *); + +const struct eventop selectops = { + "select", + select_init, + select_add, + select_del, + select_dispatch, + select_dealloc, + 0 +}; + +static int select_resize(struct selectop *sop, int fdsz); + +static void * +select_init(struct event_base *base) +{ + struct selectop *sop; + + /* Disable select when this environment variable is set */ + if (getenv("EVENT_NOSELECT")) + return (NULL); + + if (!(sop = calloc(1, sizeof(struct selectop)))) + return (NULL); + + select_resize(sop, howmany(32 + 1, NFDBITS)*sizeof(fd_mask)); + + evsignal_init(base); + + return (sop); +} + +#ifdef CHECK_INVARIANTS +static void +check_selectop(struct selectop *sop) +{ + int i; + for (i = 0; i <= sop->event_fds; ++i) { + if (FD_ISSET(i, sop->event_readset_in)) { + assert(sop->event_r_by_fd[i]); + assert(sop->event_r_by_fd[i]->ev_events & EV_READ); + assert(sop->event_r_by_fd[i]->ev_fd == i); + } else { + assert(! sop->event_r_by_fd[i]); + } + if (FD_ISSET(i, sop->event_writeset_in)) { + assert(sop->event_w_by_fd[i]); + assert(sop->event_w_by_fd[i]->ev_events & EV_WRITE); + assert(sop->event_w_by_fd[i]->ev_fd == i); + } else { + assert(! sop->event_w_by_fd[i]); + } + } + +} +#else +#define check_selectop(sop) do { (void) sop; } while (0) +#endif + +static int +select_dispatch(struct event_base *base, void *arg, struct timeval *tv) +{ + int res, i; + struct selectop *sop = arg; + + check_selectop(sop); + + memcpy(sop->event_readset_out, sop->event_readset_in, + sop->event_fdsz); + memcpy(sop->event_writeset_out, sop->event_writeset_in, + sop->event_fdsz); + + res = select(sop->event_fds + 1, sop->event_readset_out, + sop->event_writeset_out, NULL, tv); + + check_selectop(sop); + + if (res == -1) { + if (errno != EINTR) { + event_warn("select"); + return (-1); + } + + evsignal_process(base); + return (0); + } else if (base->sig.evsignal_caught) { + evsignal_process(base); + } + + event_debug(("%s: select reports %d", __func__, res)); + + check_selectop(sop); + for (i = 0; i <= sop->event_fds; ++i) { + struct event *r_ev = NULL, *w_ev = NULL; + res = 0; + if (FD_ISSET(i, sop->event_readset_out)) { + r_ev = sop->event_r_by_fd[i]; + res |= EV_READ; + } + if (FD_ISSET(i, sop->event_writeset_out)) { + w_ev = sop->event_w_by_fd[i]; + res |= EV_WRITE; + } + if (r_ev && (res & r_ev->ev_events)) { + event_active(r_ev, res & r_ev->ev_events, 1); + } + if (w_ev && w_ev != r_ev && (res & w_ev->ev_events)) { + event_active(w_ev, res & w_ev->ev_events, 1); + } + } + check_selectop(sop); + + return (0); +} + + +static int +select_resize(struct selectop *sop, int fdsz) +{ + int n_events, n_events_old; + + fd_set *readset_in = NULL; + fd_set *writeset_in = NULL; + fd_set *readset_out = NULL; + fd_set *writeset_out = NULL; + struct event **r_by_fd = NULL; + struct event **w_by_fd = NULL; + + n_events = (fdsz/sizeof(fd_mask)) * NFDBITS; + n_events_old = (sop->event_fdsz/sizeof(fd_mask)) * NFDBITS; + + if (sop->event_readset_in) + check_selectop(sop); + + if ((readset_in = realloc(sop->event_readset_in, fdsz)) == NULL) + goto error; + sop->event_readset_in = readset_in; + if ((readset_out = realloc(sop->event_readset_out, fdsz)) == NULL) + goto error; + sop->event_readset_out = readset_out; + if ((writeset_in = realloc(sop->event_writeset_in, fdsz)) == NULL) + goto error; + sop->event_writeset_in = writeset_in; + if ((writeset_out = realloc(sop->event_writeset_out, fdsz)) == NULL) + goto error; + sop->event_writeset_out = writeset_out; + if ((r_by_fd = realloc(sop->event_r_by_fd, + n_events*sizeof(struct event*))) == NULL) + goto error; + sop->event_r_by_fd = r_by_fd; + if ((w_by_fd = realloc(sop->event_w_by_fd, + n_events * sizeof(struct event*))) == NULL) + goto error; + sop->event_w_by_fd = w_by_fd; + + memset((char *)sop->event_readset_in + sop->event_fdsz, 0, + fdsz - sop->event_fdsz); + memset((char *)sop->event_writeset_in + sop->event_fdsz, 0, + fdsz - sop->event_fdsz); + memset(sop->event_r_by_fd + n_events_old, 0, + (n_events-n_events_old) * sizeof(struct event*)); + memset(sop->event_w_by_fd + n_events_old, 0, + (n_events-n_events_old) * sizeof(struct event*)); + + sop->event_fdsz = fdsz; + check_selectop(sop); + + return (0); + + error: + event_warn("malloc"); + return (-1); +} + + +static int +select_add(void *arg, struct event *ev) +{ + struct selectop *sop = arg; + + if (ev->ev_events & EV_SIGNAL) + return (evsignal_add(ev)); + + check_selectop(sop); + /* + * Keep track of the highest fd, so that we can calculate the size + * of the fd_sets for select(2) + */ + if (sop->event_fds < ev->ev_fd) { + int fdsz = sop->event_fdsz; + + if (fdsz < sizeof(fd_mask)) + fdsz = sizeof(fd_mask); + + while (fdsz < + (howmany(ev->ev_fd + 1, NFDBITS) * sizeof(fd_mask))) + fdsz *= 2; + + if (fdsz != sop->event_fdsz) { + if (select_resize(sop, fdsz)) { + check_selectop(sop); + return (-1); + } + } + + sop->event_fds = ev->ev_fd; + } + + if (ev->ev_events & EV_READ) { + FD_SET(ev->ev_fd, sop->event_readset_in); + sop->event_r_by_fd[ev->ev_fd] = ev; + } + if (ev->ev_events & EV_WRITE) { + FD_SET(ev->ev_fd, sop->event_writeset_in); + sop->event_w_by_fd[ev->ev_fd] = ev; + } + check_selectop(sop); + + return (0); +} + +/* + * Nothing to be done here. + */ + +static int +select_del(void *arg, struct event *ev) +{ + struct selectop *sop = arg; + + check_selectop(sop); + if (ev->ev_events & EV_SIGNAL) + return (evsignal_del(ev)); + + if (sop->event_fds < ev->ev_fd) { + check_selectop(sop); + return (0); + } + + if (ev->ev_events & EV_READ) { + FD_CLR(ev->ev_fd, sop->event_readset_in); + sop->event_r_by_fd[ev->ev_fd] = NULL; + } + + if (ev->ev_events & EV_WRITE) { + FD_CLR(ev->ev_fd, sop->event_writeset_in); + sop->event_w_by_fd[ev->ev_fd] = NULL; + } + + check_selectop(sop); + return (0); +} + +static void +select_dealloc(struct event_base *base, void *arg) +{ + struct selectop *sop = arg; + + evsignal_dealloc(base); + if (sop->event_readset_in) + free(sop->event_readset_in); + if (sop->event_writeset_in) + free(sop->event_writeset_in); + if (sop->event_readset_out) + free(sop->event_readset_out); + if (sop->event_writeset_out) + free(sop->event_writeset_out); + if (sop->event_r_by_fd) + free(sop->event_r_by_fd); + if (sop->event_w_by_fd) + free(sop->event_w_by_fd); + + memset(sop, 0, sizeof(struct selectop)); + free(sop); +} + +#endif /* HAVE_SELECT */ === added file 'extra/libevent/signal.c' --- a/extra/libevent/signal.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/signal.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,304 @@ +/* $OpenBSD: select.c,v 1.2 2002/06/25 15:50:15 mickey Exp $ */ + +/* + * Copyright 2000-2002 Niels Provos <provos(a)citi.umich.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <winsock2.h> +#undef WIN32_LEAN_AND_MEAN +#endif +#include <sys/types.h> +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#include <sys/queue.h> +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#include <errno.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#include <assert.h> + +#include "event.h" +#include "event-internal.h" +#include "evsignal.h" +#include "evutil.h" +#include "log.h" + +struct event_base *evsignal_base = NULL; + +static void evsignal_handler(int sig); + +/* Callback for when the signal handler write a byte to our signaling socket */ +static void +evsignal_cb(int fd, short what, void *arg) +{ + static char signals[100]; +#ifdef WIN32 + SSIZE_T n; +#else + ssize_t n; +#endif + + n = recv(fd, signals, sizeof(signals), 0); + if (n == -1) + event_err(1, "%s: read", __func__); +} + +#ifdef HAVE_SETFD +#define FD_CLOSEONEXEC(x) do { \ + if (fcntl(x, F_SETFD, 1) == -1) \ + event_warn("fcntl(%d, F_SETFD)", x); \ +} while (0) +#else +#define FD_CLOSEONEXEC(x) +#endif + +void +evsignal_init(struct event_base *base) +{ + /* + * Our signal handler is going to write to one end of the socket + * pair to wake up our event loop. The event loop then scans for + * signals that got delivered. + */ + if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, base->sig.ev_signal_pair) == -1) + event_err(1, "%s: socketpair", __func__); + + FD_CLOSEONEXEC(base->sig.ev_signal_pair[0]); + FD_CLOSEONEXEC(base->sig.ev_signal_pair[1]); + base->sig.sh_old = NULL; + base->sig.sh_old_max = 0; + base->sig.evsignal_caught = 0; + memset(&base->sig.evsigcaught, 0, sizeof(sig_atomic_t)*NSIG); + + evutil_make_socket_nonblocking(base->sig.ev_signal_pair[0]); + + event_set(&base->sig.ev_signal, base->sig.ev_signal_pair[1], + EV_READ | EV_PERSIST, evsignal_cb, &base->sig.ev_signal); + base->sig.ev_signal.ev_base = base; + base->sig.ev_signal.ev_flags |= EVLIST_INTERNAL; +} + +/* Helper: set the signal handler for evsignal to handler in base, so that + * we can restore the original handler when we clear the current one. */ +int +_evsignal_set_handler(struct event_base *base, + int evsignal, void (*handler)(int)) +{ +#ifdef HAVE_SIGACTION + struct sigaction sa; +#else + ev_sighandler_t sh; +#endif + struct evsignal_info *sig = &base->sig; + void *p; + + /* + * resize saved signal handler array up to the highest signal number. + * a dynamic array is used to keep footprint on the low side. + */ + if (evsignal >= sig->sh_old_max) { + event_debug(("%s: evsignal (%d) >= sh_old_max (%d), resizing", + __func__, evsignal, sig->sh_old_max)); + sig->sh_old_max = evsignal + 1; + p = realloc(sig->sh_old, sig->sh_old_max * sizeof *sig->sh_old); + if (p == NULL) { + event_warn("realloc"); + return (-1); + } + sig->sh_old = p; + } + + /* allocate space for previous handler out of dynamic array */ + sig->sh_old[evsignal] = malloc(sizeof *sig->sh_old[evsignal]); + if (sig->sh_old[evsignal] == NULL) { + event_warn("malloc"); + return (-1); + } + + /* save previous handler and setup new handler */ +#ifdef HAVE_SIGACTION + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = handler; + sa.sa_flags |= SA_RESTART; + sigfillset(&sa.sa_mask); + + if (sigaction(evsignal, &sa, sig->sh_old[evsignal]) == -1) { + event_warn("sigaction"); + free(sig->sh_old[evsignal]); + return (-1); + } +#else + if ((sh = signal(evsignal, handler)) == SIG_ERR) { + event_warn("signal"); + free(sig->sh_old[evsignal]); + return (-1); + } + *sig->sh_old[evsignal] = sh; +#endif + + return (0); +} + +int +evsignal_add(struct event *ev) +{ + int evsignal; + struct event_base *base = ev->ev_base; + struct evsignal_info *sig = &ev->ev_base->sig; + + if (ev->ev_events & (EV_READ|EV_WRITE)) + event_errx(1, "%s: EV_SIGNAL incompatible use", __func__); + evsignal = EVENT_SIGNAL(ev); + + event_debug(("%s: %p: changing signal handler", __func__, ev)); + if (_evsignal_set_handler(base, evsignal, evsignal_handler) == -1) + return (-1); + + /* catch signals if they happen quickly */ + evsignal_base = base; + + if (!sig->ev_signal_added) { + sig->ev_signal_added = 1; + event_add(&sig->ev_signal, NULL); + } + + return (0); +} + +int +_evsignal_restore_handler(struct event_base *base, int evsignal) +{ + int ret = 0; + struct evsignal_info *sig = &base->sig; +#ifdef HAVE_SIGACTION + struct sigaction *sh; +#else + ev_sighandler_t *sh; +#endif + + /* restore previous handler */ + sh = sig->sh_old[evsignal]; + sig->sh_old[evsignal] = NULL; +#ifdef HAVE_SIGACTION + if (sigaction(evsignal, sh, NULL) == -1) { + event_warn("sigaction"); + ret = -1; + } +#else + if (signal(evsignal, *sh) == SIG_ERR) { + event_warn("signal"); + ret = -1; + } +#endif + free(sh); + + return ret; +} + +int +evsignal_del(struct event *ev) +{ + event_debug(("%s: %p: restoring signal handler", __func__, ev)); + return _evsignal_restore_handler(ev->ev_base, EVENT_SIGNAL(ev)); +} + +static void +evsignal_handler(int sig) +{ + int save_errno = errno; + + if(evsignal_base == NULL) { + event_warn( + "%s: received signal %d, but have no base configured", + __func__, sig); + return; + } + + evsignal_base->sig.evsigcaught[sig]++; + evsignal_base->sig.evsignal_caught = 1; + +#ifndef HAVE_SIGACTION + signal(sig, evsignal_handler); +#endif + + /* Wake up our notification mechanism */ + send(evsignal_base->sig.ev_signal_pair[0], "a", 1, 0); + errno = save_errno; +} + +void +evsignal_process(struct event_base *base) +{ + struct event *ev; + sig_atomic_t ncalls; + + base->sig.evsignal_caught = 0; + TAILQ_FOREACH(ev, &base->sig.signalqueue, ev_signal_next) { + ncalls = base->sig.evsigcaught[EVENT_SIGNAL(ev)]; + if (ncalls) { + if (!(ev->ev_events & EV_PERSIST)) + event_del(ev); + event_active(ev, EV_SIGNAL, ncalls); + base->sig.evsigcaught[EVENT_SIGNAL(ev)] = 0; + } + } +} + +void +evsignal_dealloc(struct event_base *base) +{ + if(base->sig.ev_signal_added) { + event_del(&base->sig.ev_signal); + base->sig.ev_signal_added = 0; + } + assert(TAILQ_EMPTY(&base->sig.signalqueue)); + + EVUTIL_CLOSESOCKET(base->sig.ev_signal_pair[0]); + base->sig.ev_signal_pair[0] = -1; + EVUTIL_CLOSESOCKET(base->sig.ev_signal_pair[1]); + base->sig.ev_signal_pair[1] = -1; + base->sig.sh_old_max = 0; + + /* per index frees are handled in evsignal_del() */ + free(base->sig.sh_old); +} === added file 'extra/libevent/strlcpy-internal.h' --- a/extra/libevent/strlcpy-internal.h 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/strlcpy-internal.h 2009-03-12 22:27:35 +0000 @@ -0,0 +1,23 @@ +#ifndef _STRLCPY_INTERNAL_H_ +#define _STRLCPY_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#ifndef HAVE_STRLCPY +#include <string.h> +size_t _event_strlcpy(char *dst, const char *src, size_t siz); +#define strlcpy _event_strlcpy +#endif + +#ifdef __cplusplus +} +#endif + +#endif + === added file 'extra/libevent/strlcpy.c' --- a/extra/libevent/strlcpy.c 1970-01-01 00:00:00 +0000 +++ b/extra/libevent/strlcpy.c 2009-03-12 22:27:35 +0000 @@ -0,0 +1,76 @@ +/* $OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $ */ + +/* + * Copyright (c) 1998 Todd C. Miller <Todd.Miller(a)courtesan.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if defined(LIBC_SCCS) && !defined(lint) +static char *rcsid = "$OpenBSD: strlcpy.c,v 1.5 2001/05/13 15:40:16 deraadt Exp $"; +#endif /* LIBC_SCCS and not lint */ + +#include <sys/types.h> + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#ifndef HAVE_STRLCPY +#include "strlcpy-internal.h" + +/* + * Copy src to string dst of size siz. At most siz-1 characters + * will be copied. Always NUL terminates (unless siz == 0). + * Returns strlen(src); if retval >= siz, truncation occurred. + */ +size_t +_event_strlcpy(dst, src, siz) + char *dst; + const char *src; + size_t siz; +{ + register char *d = dst; + register const char *s = src; + register size_t n = siz; + + /* Copy as many bytes as will fit */ + if (n != 0 && --n != 0) { + do { + if ((*d++ = *s++) == 0) + break; + } while (--n != 0); + } + + /* Not enough room in dst, add NUL and traverse rest of src */ + if (n == 0) { + if (siz != 0) + *d = '\0'; /* NUL-terminate dst */ + while (*s++) + ; + } + + return(s - src - 1); /* count does not include NUL */ +} +#endif === modified file 'include/config-win.h' --- a/include/config-win.h 2009-02-19 09:01:25 +0000 +++ b/include/config-win.h 2009-03-12 22:27:35 +0000 @@ -436,3 +436,7 @@ inline ulonglong double2ulonglong(double #define HAVE_UCA_COLLATIONS 1 #define HAVE_BOOL 1 +#ifndef EMBEDDED_LIBRARY +#define HAVE_LIBEVENT 1 +#define HAVE_POOL_OF_THREADS 1 +#endif === modified file 'include/my_dbug.h' --- a/include/my_dbug.h 2008-12-10 09:02:25 +0000 +++ b/include/my_dbug.h 2009-03-12 22:27:35 +0000 @@ -33,6 +33,7 @@ extern my_bool _dbug_on_; extern my_bool _db_keyword_(struct _db_code_state_ *, const char *, int); extern int _db_explain_(struct _db_code_state_ *cs, char *buf, size_t len); extern int _db_explain_init_(char *buf, size_t len); +extern int _db_is_pushed_(void); extern void _db_setjmp_(void); extern void _db_longjmp_(void); extern void _db_process_(const char *name); === modified file 'include/mysql.h.pp' --- a/include/mysql.h.pp 2008-10-10 15:28:41 +0000 +++ b/include/mysql.h.pp 2009-03-12 22:27:35 +0000 @@ -223,8 +223,10 @@ typedef struct st_typelib { unsigned int *type_lengths; } TYPELIB; extern my_ulonglong find_typeset(char *x, TYPELIB *typelib,int *error_position); -extern int find_type_or_exit(const char *x, TYPELIB *typelib, - const char *option); +extern int find_type_with_warning(const char *x, TYPELIB *typelib, + const char *option); +extern uint find_type_or_exit(const char *x, TYPELIB *typelib, + const char *option); extern int find_type(char *x, const TYPELIB *typelib, unsigned int full_name); extern void make_type(char *to,unsigned int nr,TYPELIB *typelib); extern const char *get_type(TYPELIB *typelib,unsigned int nr); === modified file 'include/typelib.h' --- a/include/typelib.h 2007-04-27 17:09:39 +0000 +++ b/include/typelib.h 2009-03-12 22:27:35 +0000 @@ -27,8 +27,10 @@ typedef struct st_typelib { /* Different } TYPELIB; extern my_ulonglong find_typeset(char *x, TYPELIB *typelib,int *error_position); -extern int find_type_or_exit(const char *x, TYPELIB *typelib, - const char *option); +extern int find_type_with_warning(const char *x, TYPELIB *typelib, + const char *option); +extern uint find_type_or_exit(const char *x, TYPELIB *typelib, + const char *option); extern int find_type(char *x, const TYPELIB *typelib, unsigned int full_name); extern void make_type(char *to,unsigned int nr,TYPELIB *typelib); extern const char *get_type(TYPELIB *typelib,unsigned int nr); === modified file 'include/violite.h' --- a/include/violite.h 2007-06-15 17:07:59 +0000 +++ b/include/violite.h 2009-03-12 22:27:35 +0000 @@ -88,6 +88,7 @@ my_bool vio_peer_addr(Vio* vio, char *bu /* Remotes in_addr */ void vio_in_addr(Vio *vio, struct in_addr *in); my_bool vio_poll_read(Vio *vio,uint timeout); +ssize_t vio_pending(Vio *vio); #ifdef HAVE_OPENSSL #include <openssl/opensslv.h> === modified file 'libmysqld/Makefile.am' --- a/libmysqld/Makefile.am 2008-10-10 15:28:41 +0000 +++ b/libmysqld/Makefile.am 2009-03-12 22:27:35 +0000 @@ -98,6 +98,7 @@ INC_LIB= $(top_builddir)/regex/libregex. $(top_builddir)/vio/libvio.a \ @NDB_SCI_LIBS@ \ @mysql_plugin_libs@ \ + $(libevent_inc_libs) \ $(yassl_inc_libs) if HAVE_YASSL === added file 'mysql-test/include/have_pool_of_threads.inc' --- a/mysql-test/include/have_pool_of_threads.inc 1970-01-01 00:00:00 +0000 +++ b/mysql-test/include/have_pool_of_threads.inc 2009-03-12 22:27:35 +0000 @@ -0,0 +1,4 @@ +-- require r/have_pool_of_threads.require +disable_query_log; +show variables like 'thread_handling'; +enable_query_log; === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2009-02-19 09:01:25 +0000 +++ b/mysql-test/mysql-test-run.pl 2009-03-12 22:27:35 +0000 @@ -1200,11 +1200,23 @@ sub command_line_setup { { # Indicate that we are using debugger $glob_debugger= 1; + $opt_testcase_timeout= 60*60*24; # Don't abort debugging with timeout + $opt_suite_timeout= $opt_testcase_timeout; + $opt_retry= 1; + $opt_retry_failure= 1; + if ( using_extern() ) { mtr_error("Can't use --extern when using debugger"); } } + if ($opt_debug) + { + $opt_testcase_timeout= 60*60*24; # Don't abort debugging with timeout + $opt_suite_timeout= $opt_testcase_timeout; + $opt_retry= 1; + $opt_retry_failure= 1; + } # -------------------------------------------------------------------------- # Check timeout arguments === modified file 'mysql-test/r/crash_commit_before.result' --- a/mysql-test/r/crash_commit_before.result 2007-04-03 09:36:33 +0000 +++ b/mysql-test/r/crash_commit_before.result 2009-03-12 22:27:35 +0000 @@ -1,7 +1,7 @@ CREATE TABLE t1(a int) engine=innodb; START TRANSACTION; insert into t1 values(9); -SET SESSION debug="d,crash_commit_before"; +SET GLOBAL debug="d,crash_commit_before"; COMMIT; ERROR HY000: Lost connection to MySQL server during query SHOW CREATE TABLE t1; === added file 'mysql-test/r/have_pool_of_threads.require' --- a/mysql-test/r/have_pool_of_threads.require 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/have_pool_of_threads.require 2009-03-12 22:27:35 +0000 @@ -0,0 +1,2 @@ +Variable_name Value +thread_handling pool-of-threads === added file 'mysql-test/r/pool_of_threads.result' --- a/mysql-test/r/pool_of_threads.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/pool_of_threads.result 2009-03-12 22:27:35 +0000 @@ -0,0 +1,2153 @@ +drop table if exists t1,t2,t3,t4; +CREATE TABLE t1 ( +Period smallint(4) unsigned zerofill DEFAULT '0000' NOT NULL, +Varor_period smallint(4) unsigned DEFAULT '0' NOT NULL +); +INSERT INTO t1 VALUES (9410,9412); +select period from t1; +period +9410 +select * from t1; +Period Varor_period +9410 9412 +select t1.* from t1; +Period Varor_period +9410 9412 +CREATE TABLE t2 ( +auto int not null auto_increment, +fld1 int(6) unsigned zerofill DEFAULT '000000' NOT NULL, +companynr tinyint(2) unsigned zerofill DEFAULT '00' NOT NULL, +fld3 char(30) DEFAULT '' NOT NULL, +fld4 char(35) DEFAULT '' NOT NULL, +fld5 char(35) DEFAULT '' NOT NULL, +fld6 char(4) DEFAULT '' NOT NULL, +UNIQUE fld1 (fld1), +KEY fld3 (fld3), +PRIMARY KEY (auto) +); +select t2.fld3 from t2 where companynr = 58 and fld3 like "%imaginable%"; +fld3 +imaginable +select fld3 from t2 where fld3 like "%cultivation" ; +fld3 +cultivation +select t2.fld3,companynr from t2 where companynr = 57+1 order by fld3; +fld3 companynr +concoct 58 +druggists 58 +engrossing 58 +Eurydice 58 +exclaimers 58 +ferociousness 58 +hopelessness 58 +Huey 58 +imaginable 58 +judges 58 +merging 58 +ostrich 58 +peering 58 +Phelps 58 +presumes 58 +Ruth 58 +sentences 58 +Shylock 58 +straggled 58 +synergy 58 +thanking 58 +tying 58 +unlocks 58 +select fld3,companynr from t2 where companynr = 58 order by fld3; +fld3 companynr +concoct 58 +druggists 58 +engrossing 58 +Eurydice 58 +exclaimers 58 +ferociousness 58 +hopelessness 58 +Huey 58 +imaginable 58 +judges 58 +merging 58 +ostrich 58 +peering 58 +Phelps 58 +presumes 58 +Ruth 58 +sentences 58 +Shylock 58 +straggled 58 +synergy 58 +thanking 58 +tying 58 +unlocks 58 +select fld3 from t2 order by fld3 desc limit 10; +fld3 +youthfulness +yelped +Wotan +workers +Witt +witchcraft +Winsett +Willy +willed +wildcats +select fld3 from t2 order by fld3 desc limit 5; +fld3 +youthfulness +yelped +Wotan +workers +Witt +select fld3 from t2 order by fld3 desc limit 5,5; +fld3 +witchcraft +Winsett +Willy +willed +wildcats +select t2.fld3 from t2 where fld3 = 'honeysuckle'; +fld3 +honeysuckle +select t2.fld3 from t2 where fld3 LIKE 'honeysuckl_'; +fld3 +honeysuckle +select t2.fld3 from t2 where fld3 LIKE 'hon_ysuckl_'; +fld3 +honeysuckle +select t2.fld3 from t2 where fld3 LIKE 'honeysuckle%'; +fld3 +honeysuckle +select t2.fld3 from t2 where fld3 LIKE 'h%le'; +fld3 +honeysuckle +select t2.fld3 from t2 where fld3 LIKE 'honeysuckle_'; +fld3 +select t2.fld3 from t2 where fld3 LIKE 'don_t_find_me_please%'; +fld3 +explain select t2.fld3 from t2 where fld3 = 'honeysuckle'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ref fld3 fld3 30 const 1 Using where; Using index +explain select fld3 from t2 ignore index (fld3) where fld3 = 'honeysuckle'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +explain select fld3 from t2 use index (fld1) where fld3 = 'honeysuckle'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +explain select fld3 from t2 use index (fld3) where fld3 = 'honeysuckle'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ref fld3 fld3 30 const 1 Using where; Using index +explain select fld3 from t2 use index (fld1,fld3) where fld3 = 'honeysuckle'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ref fld3 fld3 30 const 1 Using where; Using index +explain select fld3 from t2 ignore index (fld3,not_used); +ERROR 42000: Key 'not_used' doesn't exist in table 't2' +explain select fld3 from t2 use index (not_used); +ERROR 42000: Key 'not_used' doesn't exist in table 't2' +select t2.fld3 from t2 where fld3 >= 'honeysuckle' and fld3 <= 'honoring' order by fld3; +fld3 +honeysuckle +honoring +explain select t2.fld3 from t2 where fld3 >= 'honeysuckle' and fld3 <= 'honoring' order by fld3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range fld3 fld3 30 NULL 2 Using where; Using index +select fld1,fld3 from t2 where fld3="Colombo" or fld3 = "nondecreasing" order by fld3; +fld1 fld3 +148504 Colombo +068305 Colombo +000000 nondecreasing +select fld1,fld3 from t2 where companynr = 37 and fld3 = 'appendixes'; +fld1 fld3 +232605 appendixes +1232605 appendixes +1232606 appendixes +1232607 appendixes +1232608 appendixes +1232609 appendixes +select fld1 from t2 where fld1=250501 or fld1="250502"; +fld1 +250501 +250502 +explain select fld1 from t2 where fld1=250501 or fld1="250502"; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range fld1 fld1 4 NULL 2 Using where; Using index +select fld1 from t2 where fld1=250501 or fld1=250502 or fld1 >= 250505 and fld1 <= 250601 or fld1 between 250501 and 250502; +fld1 +250501 +250502 +250505 +250601 +explain select fld1 from t2 where fld1=250501 or fld1=250502 or fld1 >= 250505 and fld1 <= 250601 or fld1 between 250501 and 250502; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range fld1 fld1 4 NULL 4 Using where; Using index +select fld1,fld3 from t2 where companynr = 37 and fld3 like 'f%'; +fld1 fld3 +218401 faithful +018007 fanatic +228311 fated +018017 featherweight +218022 feed +088303 feminine +058004 Fenton +038017 fetched +018054 fetters +208101 fiftieth +238007 filial +013606 fingerings +218008 finishers +038205 firearm +188505 fitting +202301 Fitzpatrick +238008 fixedly +012001 flanking +018103 flint +018104 flopping +188007 flurried +013602 foldout +226205 foothill +232102 forgivably +228306 forthcoming +186002 freakish +208113 freest +231315 freezes +036002 funereal +226209 furnishings +198006 furthermore +select fld3 from t2 where fld3 like "L%" and fld3 = "ok"; +fld3 +select fld3 from t2 where (fld3 like "C%" and fld3 = "Chantilly"); +fld3 +Chantilly +select fld1,fld3 from t2 where fld1 like "25050%"; +fld1 fld3 +250501 poisoning +250502 Iraqis +250503 heaving +250504 population +250505 bomb +select fld1,fld3 from t2 where fld1 like "25050_"; +fld1 fld3 +250501 poisoning +250502 Iraqis +250503 heaving +250504 population +250505 bomb +select distinct companynr from t2; +companynr +00 +37 +36 +50 +58 +29 +40 +53 +65 +41 +34 +68 +select distinct companynr from t2 order by companynr; +companynr +00 +29 +34 +36 +37 +40 +41 +50 +53 +58 +65 +68 +select distinct companynr from t2 order by companynr desc; +companynr +68 +65 +58 +53 +50 +41 +40 +37 +36 +34 +29 +00 +select distinct t2.fld3,period from t2,t1 where companynr=37 and fld3 like "O%"; +fld3 period +obliterates 9410 +offload 9410 +opaquely 9410 +organizer 9410 +overestimating 9410 +overlay 9410 +select distinct fld3 from t2 where companynr = 34 order by fld3; +fld3 +absentee +accessed +ahead +alphabetic +Asiaticizations +attitude +aye +bankruptcies +belays +Blythe +bomb +boulevard +bulldozes +cannot +caressing +charcoal +checksumming +chess +clubroom +colorful +cosy +creator +crying +Darius +diffusing +duality +Eiffel +Epiphany +Ernestine +explorers +exterminated +famine +forked +Gershwins +heaving +Hodges +Iraqis +Italianization +Lagos +landslide +libretto +Majorca +mastering +narrowed +occurred +offerers +Palestine +Peruvianizes +pharmaceutic +poisoning +population +Pygmalion +rats +realest +recording +regimented +retransmitting +reviver +rouses +scars +sicker +sleepwalk +stopped +sugars +translatable +uncles +unexpected +uprisings +versatility +vest +select distinct fld3 from t2 limit 10; +fld3 +abates +abiding +Abraham +abrogating +absentee +abut +accessed +accruing +accumulating +accuracies +select distinct fld3 from t2 having fld3 like "A%" limit 10; +fld3 +abates +abiding +Abraham +abrogating +absentee +abut +accessed +accruing +accumulating +accuracies +select distinct substring(fld3,1,3) from t2 where fld3 like "A%"; +substring(fld3,1,3) +aba +abi +Abr +abs +abu +acc +acq +acu +Ade +adj +Adl +adm +Ado +ads +adv +aer +aff +afi +afl +afo +agi +ahe +aim +air +Ald +alg +ali +all +alp +alr +ama +ame +amm +ana +and +ane +Ang +ani +Ann +Ant +api +app +aqu +Ara +arc +Arm +arr +Art +Asi +ask +asp +ass +ast +att +aud +Aug +aut +ave +avo +awe +aye +Azt +select distinct substring(fld3,1,3) as a from t2 having a like "A%" order by a limit 10; +a +aba +abi +Abr +abs +abu +acc +acq +acu +Ade +adj +select distinct substring(fld3,1,3) from t2 where fld3 like "A%" limit 10; +substring(fld3,1,3) +aba +abi +Abr +abs +abu +acc +acq +acu +Ade +adj +select distinct substring(fld3,1,3) as a from t2 having a like "A%" limit 10; +a +aba +abi +Abr +abs +abu +acc +acq +acu +Ade +adj +create table t3 ( +period int not null, +name char(32) not null, +companynr int not null, +price double(11,0), +price2 double(11,0), +key (period), +key (name) +); +create temporary table tmp engine = myisam select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +insert into tmp select * from t3; +insert into t3 select * from tmp; +alter table t3 add t2nr int not null auto_increment primary key first; +drop table tmp; +SET SQL_BIG_TABLES=1; +select distinct concat(fld3," ",fld3) as namn from t2,t3 where t2.fld1=t3.t2nr order by namn limit 10; +namn +Abraham Abraham +abrogating abrogating +admonishing admonishing +Adolph Adolph +afield afield +aging aging +ammonium ammonium +analyzable analyzable +animals animals +animized animized +SET SQL_BIG_TABLES=0; +select distinct concat(fld3," ",fld3) from t2,t3 where t2.fld1=t3.t2nr order by fld3 limit 10; +concat(fld3," ",fld3) +Abraham Abraham +abrogating abrogating +admonishing admonishing +Adolph Adolph +afield afield +aging aging +ammonium ammonium +analyzable analyzable +animals animals +animized animized +select distinct fld5 from t2 limit 10; +fld5 +neat +Steinberg +jarring +tinily +balled +persist +attainments +fanatic +measures +rightfulness +select distinct fld3,count(*) from t2 group by companynr,fld3 limit 10; +fld3 count(*) +affixed 1 +and 1 +annoyers 1 +Anthony 1 +assayed 1 +assurers 1 +attendants 1 +bedlam 1 +bedpost 1 +boasted 1 +SET SQL_BIG_TABLES=1; +select distinct fld3,count(*) from t2 group by companynr,fld3 limit 10; +fld3 count(*) +affixed 1 +and 1 +annoyers 1 +Anthony 1 +assayed 1 +assurers 1 +attendants 1 +bedlam 1 +bedpost 1 +boasted 1 +SET SQL_BIG_TABLES=0; +select distinct fld3,repeat("a",length(fld3)),count(*) from t2 group by companynr,fld3 limit 100,10; +fld3 repeat("a",length(fld3)) count(*) +circus aaaaaa 1 +cited aaaaa 1 +Colombo aaaaaaa 1 +congresswoman aaaaaaaaaaaaa 1 +contrition aaaaaaaaaa 1 +corny aaaaa 1 +cultivation aaaaaaaaaaa 1 +definiteness aaaaaaaaaaaa 1 +demultiplex aaaaaaaaaaa 1 +disappointing aaaaaaaaaaaaa 1 +select distinct companynr,rtrim(space(512+companynr)) from t3 order by 1,2; +companynr rtrim(space(512+companynr)) +37 +78 +101 +154 +311 +447 +512 +select distinct fld3 from t2,t3 where t2.companynr = 34 and t2.fld1=t3.t2nr order by fld3; +fld3 +explain select t3.t2nr,fld3 from t2,t3 where t2.companynr = 34 and t2.fld1=t3.t2nr order by t3.t2nr,fld3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL fld1 NULL NULL NULL 1199 Using where; Using temporary; Using filesort +1 SIMPLE t3 eq_ref PRIMARY PRIMARY 4 test.t2.fld1 1 Using where; Using index +explain select * from t3 as t1,t3 where t1.period=t3.period order by t3.period; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL period NULL NULL NULL 41810 Using temporary; Using filesort +1 SIMPLE t3 ref period period 4 test.t1.period 4181 +explain select * from t3 as t1,t3 where t1.period=t3.period order by t3.period limit 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t3 index period period 4 NULL 1 +1 SIMPLE t1 ref period period 4 test.t3.period 4181 +explain select * from t3 as t1,t3 where t1.period=t3.period order by t1.period limit 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index period period 4 NULL 1 +1 SIMPLE t3 ref period period 4 test.t1.period 4181 +select period from t1; +period +9410 +select period from t1 where period=1900; +period +select fld3,period from t1,t2 where fld1 = 011401 order by period; +fld3 period +breaking 9410 +select fld3,period from t2,t3 where t2.fld1 = 011401 and t2.fld1=t3.t2nr and t3.period=1001; +fld3 period +breaking 1001 +explain select fld3,period from t2,t3 where t2.fld1 = 011401 and t3.t2nr=t2.fld1 and 1001 = t3.period; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 const fld1 fld1 4 const 1 +1 SIMPLE t3 const PRIMARY,period PRIMARY 4 const 1 +select fld3,period from t2,t1 where companynr*10 = 37*10; +fld3 period +breaking 9410 +Romans 9410 +intercepted 9410 +bewilderingly 9410 +astound 9410 +admonishing 9410 +sumac 9410 +flanking 9410 +combed 9410 +subjective 9410 +scatterbrain 9410 +Eulerian 9410 +Kane 9410 +overlay 9410 +perturb 9410 +goblins 9410 +annihilates 9410 +Wotan 9410 +snatching 9410 +concludes 9410 +laterally 9410 +yelped 9410 +grazing 9410 +Baird 9410 +celery 9410 +misunderstander 9410 +handgun 9410 +foldout 9410 +mystic 9410 +succumbed 9410 +Nabisco 9410 +fingerings 9410 +aging 9410 +afield 9410 +ammonium 9410 +boat 9410 +intelligibility 9410 +Augustine 9410 +teethe 9410 +dreaded 9410 +scholastics 9410 +audiology 9410 +wallet 9410 +parters 9410 +eschew 9410 +quitter 9410 +neat 9410 +Steinberg 9410 +jarring 9410 +tinily 9410 +balled 9410 +persist 9410 +attainments 9410 +fanatic 9410 +measures 9410 +rightfulness 9410 +capably 9410 +impulsive 9410 +starlet 9410 +terminators 9410 +untying 9410 +announces 9410 +featherweight 9410 +pessimist 9410 +daughter 9410 +decliner 9410 +lawgiver 9410 +stated 9410 +readable 9410 +attrition 9410 +cascade 9410 +motors 9410 +interrogate 9410 +pests 9410 +stairway 9410 +dopers 9410 +testicle 9410 +Parsifal 9410 +leavings 9410 +postulation 9410 +squeaking 9410 +contrasted 9410 +leftover 9410 +whiteners 9410 +erases 9410 +Punjab 9410 +Merritt 9410 +Quixotism 9410 +sweetish 9410 +dogging 9410 +scornfully 9410 +bellow 9410 +bills 9410 +cupboard 9410 +sureties 9410 +puddings 9410 +fetters 9410 +bivalves 9410 +incurring 9410 +Adolph 9410 +pithed 9410 +Miles 9410 +trimmings 9410 +tragedies 9410 +skulking 9410 +flint 9410 +flopping 9410 +relaxing 9410 +offload 9410 +suites 9410 +lists 9410 +animized 9410 +multilayer 9410 +standardizes 9410 +Judas 9410 +vacuuming 9410 +dentally 9410 +humanness 9410 +inch 9410 +Weissmuller 9410 +irresponsibly 9410 +luckily 9410 +culled 9410 +medical 9410 +bloodbath 9410 +subschema 9410 +animals 9410 +Micronesia 9410 +repetitions 9410 +Antares 9410 +ventilate 9410 +pityingly 9410 +interdependent 9410 +Graves 9410 +neonatal 9410 +chafe 9410 +honoring 9410 +realtor 9410 +elite 9410 +funereal 9410 +abrogating 9410 +sorters 9410 +Conley 9410 +lectured 9410 +Abraham 9410 +Hawaii 9410 +cage 9410 +hushes 9410 +Simla 9410 +reporters 9410 +Dutchman 9410 +descendants 9410 +groupings 9410 +dissociate 9410 +coexist 9410 +Beebe 9410 +Taoism 9410 +Connally 9410 +fetched 9410 +checkpoints 9410 +rusting 9410 +galling 9410 +obliterates 9410 +traitor 9410 +resumes 9410 +analyzable 9410 +terminator 9410 +gritty 9410 +firearm 9410 +minima 9410 +Selfridge 9410 +disable 9410 +witchcraft 9410 +betroth 9410 +Manhattanize 9410 +imprint 9410 +peeked 9410 +swelling 9410 +interrelationships 9410 +riser 9410 +Gandhian 9410 +peacock 9410 +bee 9410 +kanji 9410 +dental 9410 +scarf 9410 +chasm 9410 +insolence 9410 +syndicate 9410 +alike 9410 +imperial 9410 +convulsion 9410 +railway 9410 +validate 9410 +normalizes 9410 +comprehensive 9410 +chewing 9410 +denizen 9410 +schemer 9410 +chronicle 9410 +Kline 9410 +Anatole 9410 +partridges 9410 +brunch 9410 +recruited 9410 +dimensions 9410 +Chicana 9410 +announced 9410 +praised 9410 +employing 9410 +linear 9410 +quagmire 9410 +western 9410 +relishing 9410 +serving 9410 +scheduling 9410 +lore 9410 +eventful 9410 +arteriole 9410 +disentangle 9410 +cured 9410 +Fenton 9410 +avoidable 9410 +drains 9410 +detectably 9410 +husky 9410 +impelling 9410 +undoes 9410 +evened 9410 +squeezes 9410 +destroyer 9410 +rudeness 9410 +beaner 9410 +boorish 9410 +Everhart 9410 +encompass 9410 +mushrooms 9410 +Alison 9410 +externally 9410 +pellagra 9410 +cult 9410 +creek 9410 +Huffman 9410 +Majorca 9410 +governing 9410 +gadfly 9410 +reassigned 9410 +intentness 9410 +craziness 9410 +psychic 9410 +squabbled 9410 +burlesque 9410 +capped 9410 +extracted 9410 +DiMaggio 9410 +exclamation 9410 +subdirectory 9410 +Gothicism 9410 +feminine 9410 +metaphysically 9410 +sanding 9410 +Miltonism 9410 +freakish 9410 +index 9410 +straight 9410 +flurried 9410 +denotative 9410 +coming 9410 +commencements 9410 +gentleman 9410 +gifted 9410 +Shanghais 9410 +sportswriting 9410 +sloping 9410 +navies 9410 +leaflet 9410 +shooter 9410 +Joplin 9410 +babies 9410 +assails 9410 +admiring 9410 +swaying 9410 +Goldstine 9410 +fitting 9410 +Norwalk 9410 +analogy 9410 +deludes 9410 +cokes 9410 +Clayton 9410 +exhausts 9410 +causality 9410 +sating 9410 +icon 9410 +throttles 9410 +communicants 9410 +dehydrate 9410 +priceless 9410 +publicly 9410 +incidentals 9410 +commonplace 9410 +mumbles 9410 +furthermore 9410 +cautioned 9410 +parametrized 9410 +registration 9410 +sadly 9410 +positioning 9410 +babysitting 9410 +eternal 9410 +hoarder 9410 +congregates 9410 +rains 9410 +workers 9410 +sags 9410 +unplug 9410 +garage 9410 +boulder 9410 +specifics 9410 +Teresa 9410 +Winsett 9410 +convenient 9410 +buckboards 9410 +amenities 9410 +resplendent 9410 +sews 9410 +participated 9410 +Simon 9410 +certificates 9410 +Fitzpatrick 9410 +Evanston 9410 +misted 9410 +textures 9410 +save 9410 +count 9410 +rightful 9410 +chaperone 9410 +Lizzy 9410 +clenched 9410 +effortlessly 9410 +accessed 9410 +beaters 9410 +Hornblower 9410 +vests 9410 +indulgences 9410 +infallibly 9410 +unwilling 9410 +excrete 9410 +spools 9410 +crunches 9410 +overestimating 9410 +ineffective 9410 +humiliation 9410 +sophomore 9410 +star 9410 +rifles 9410 +dialysis 9410 +arriving 9410 +indulge 9410 +clockers 9410 +languages 9410 +Antarctica 9410 +percentage 9410 +ceiling 9410 +specification 9410 +regimented 9410 +ciphers 9410 +pictures 9410 +serpents 9410 +allot 9410 +realized 9410 +mayoral 9410 +opaquely 9410 +hostess 9410 +fiftieth 9410 +incorrectly 9410 +decomposition 9410 +stranglings 9410 +mixture 9410 +electroencephalography 9410 +similarities 9410 +charges 9410 +freest 9410 +Greenberg 9410 +tinting 9410 +expelled 9410 +warm 9410 +smoothed 9410 +deductions 9410 +Romano 9410 +bitterroot 9410 +corset 9410 +securing 9410 +environing 9410 +cute 9410 +Crays 9410 +heiress 9410 +inform 9410 +avenge 9410 +universals 9410 +Kinsey 9410 +ravines 9410 +bestseller 9410 +equilibrium 9410 +extents 9410 +relatively 9410 +pressure 9410 +critiques 9410 +befouled 9410 +rightfully 9410 +mechanizing 9410 +Latinizes 9410 +timesharing 9410 +Aden 9410 +embassies 9410 +males 9410 +shapelessly 9410 +mastering 9410 +Newtonian 9410 +finishers 9410 +abates 9410 +teem 9410 +kiting 9410 +stodgy 9410 +feed 9410 +guitars 9410 +airships 9410 +store 9410 +denounces 9410 +Pyle 9410 +Saxony 9410 +serializations 9410 +Peruvian 9410 +taxonomically 9410 +kingdom 9410 +stint 9410 +Sault 9410 +faithful 9410 +Ganymede 9410 +tidiness 9410 +gainful 9410 +contrary 9410 +Tipperary 9410 +tropics 9410 +theorizers 9410 +renew 9410 +already 9410 +terminal 9410 +Hegelian 9410 +hypothesizer 9410 +warningly 9410 +journalizing 9410 +nested 9410 +Lars 9410 +saplings 9410 +foothill 9410 +labeled 9410 +imperiously 9410 +reporters 9410 +furnishings 9410 +precipitable 9410 +discounts 9410 +excises 9410 +Stalin 9410 +despot 9410 +ripeness 9410 +Arabia 9410 +unruly 9410 +mournfulness 9410 +boom 9410 +slaughter 9410 +Sabine 9410 +handy 9410 +rural 9410 +organizer 9410 +shipyard 9410 +civics 9410 +inaccuracy 9410 +rules 9410 +juveniles 9410 +comprised 9410 +investigations 9410 +stabilizes 9410 +seminaries 9410 +Hunter 9410 +sporty 9410 +test 9410 +weasels 9410 +CERN 9410 +tempering 9410 +afore 9410 +Galatean 9410 +techniques 9410 +error 9410 +veranda 9410 +severely 9410 +Cassites 9410 +forthcoming 9410 +guides 9410 +vanish 9410 +lied 9410 +sawtooth 9410 +fated 9410 +gradually 9410 +widens 9410 +preclude 9410 +evenhandedly 9410 +percentage 9410 +disobedience 9410 +humility 9410 +gleaning 9410 +petted 9410 +bloater 9410 +minion 9410 +marginal 9410 +apiary 9410 +measures 9410 +precaution 9410 +repelled 9410 +primary 9410 +coverings 9410 +Artemia 9410 +navigate 9410 +spatial 9410 +Gurkha 9410 +meanwhile 9410 +Melinda 9410 +Butterfield 9410 +Aldrich 9410 +previewing 9410 +glut 9410 +unaffected 9410 +inmate 9410 +mineral 9410 +impending 9410 +meditation 9410 +ideas 9410 +miniaturizes 9410 +lewdly 9410 +title 9410 +youthfulness 9410 +creak 9410 +Chippewa 9410 +clamored 9410 +freezes 9410 +forgivably 9410 +reduce 9410 +McGovern 9410 +Nazis 9410 +epistle 9410 +socializes 9410 +conceptions 9410 +Kevin 9410 +uncovering 9410 +chews 9410 +appendixes 9410 +appendixes 9410 +appendixes 9410 +appendixes 9410 +appendixes 9410 +appendixes 9410 +raining 9410 +infest 9410 +compartment 9410 +minting 9410 +ducks 9410 +roped 9410 +waltz 9410 +Lillian 9410 +repressions 9410 +chillingly 9410 +noncritical 9410 +lithograph 9410 +spongers 9410 +parenthood 9410 +posed 9410 +instruments 9410 +filial 9410 +fixedly 9410 +relives 9410 +Pandora 9410 +watering 9410 +ungrateful 9410 +secures 9410 +poison 9410 +dusted 9410 +encompasses 9410 +presentation 9410 +Kantian 9410 +select fld3,period,price,price2 from t2,t3 where t2.fld1=t3.t2nr and period >= 1001 and period <= 1002 and t2.companynr = 37 order by fld3,period, price; +fld3 period price price2 +admonishing 1002 28357832 8723648 +analyzable 1002 28357832 8723648 +annihilates 1001 5987435 234724 +Antares 1002 28357832 8723648 +astound 1001 5987435 234724 +audiology 1001 5987435 234724 +Augustine 1002 28357832 8723648 +Baird 1002 28357832 8723648 +bewilderingly 1001 5987435 234724 +breaking 1001 5987435 234724 +Conley 1001 5987435 234724 +dentally 1002 28357832 8723648 +dissociate 1002 28357832 8723648 +elite 1001 5987435 234724 +eschew 1001 5987435 234724 +Eulerian 1001 5987435 234724 +flanking 1001 5987435 234724 +foldout 1002 28357832 8723648 +funereal 1002 28357832 8723648 +galling 1002 28357832 8723648 +Graves 1001 5987435 234724 +grazing 1001 5987435 234724 +groupings 1001 5987435 234724 +handgun 1001 5987435 234724 +humility 1002 28357832 8723648 +impulsive 1002 28357832 8723648 +inch 1001 5987435 234724 +intelligibility 1001 5987435 234724 +jarring 1001 5987435 234724 +lawgiver 1001 5987435 234724 +lectured 1002 28357832 8723648 +Merritt 1002 28357832 8723648 +neonatal 1001 5987435 234724 +offload 1002 28357832 8723648 +parters 1002 28357832 8723648 +pityingly 1002 28357832 8723648 +puddings 1002 28357832 8723648 +Punjab 1001 5987435 234724 +quitter 1002 28357832 8723648 +realtor 1001 5987435 234724 +relaxing 1001 5987435 234724 +repetitions 1001 5987435 234724 +resumes 1001 5987435 234724 +Romans 1002 28357832 8723648 +rusting 1001 5987435 234724 +scholastics 1001 5987435 234724 +skulking 1002 28357832 8723648 +stated 1002 28357832 8723648 +suites 1002 28357832 8723648 +sureties 1001 5987435 234724 +testicle 1002 28357832 8723648 +tinily 1002 28357832 8723648 +tragedies 1001 5987435 234724 +trimmings 1001 5987435 234724 +vacuuming 1001 5987435 234724 +ventilate 1001 5987435 234724 +wallet 1001 5987435 234724 +Weissmuller 1002 28357832 8723648 +Wotan 1002 28357832 8723648 +select t2.fld1,fld3,period,price,price2 from t2,t3 where t2.fld1>= 18201 and t2.fld1 <= 18811 and t2.fld1=t3.t2nr and period = 1001 and t2.companynr = 37; +fld1 fld3 period price price2 +018201 relaxing 1001 5987435 234724 +018601 vacuuming 1001 5987435 234724 +018801 inch 1001 5987435 234724 +018811 repetitions 1001 5987435 234724 +create table t4 ( +companynr tinyint(2) unsigned zerofill NOT NULL default '00', +companyname char(30) NOT NULL default '', +PRIMARY KEY (companynr), +UNIQUE KEY companyname(companyname) +) ENGINE=MyISAM MAX_ROWS=50 PACK_KEYS=1 COMMENT='companynames'; +select STRAIGHT_JOIN t2.companynr,companyname from t4,t2 where t2.companynr=t4.companynr group by t2.companynr; +companynr companyname +00 Unknown +29 company 1 +34 company 2 +36 company 3 +37 company 4 +40 company 5 +41 company 6 +50 company 11 +53 company 7 +58 company 8 +65 company 9 +68 company 10 +select SQL_SMALL_RESULT t2.companynr,companyname from t4,t2 where t2.companynr=t4.companynr group by t2.companynr; +companynr companyname +00 Unknown +29 company 1 +34 company 2 +36 company 3 +37 company 4 +40 company 5 +41 company 6 +50 company 11 +53 company 7 +58 company 8 +65 company 9 +68 company 10 +select * from t1,t1 t12; +Period Varor_period Period Varor_period +9410 9412 9410 9412 +select t2.fld1,t22.fld1 from t2,t2 t22 where t2.fld1 >= 250501 and t2.fld1 <= 250505 and t22.fld1 >= 250501 and t22.fld1 <= 250505; +fld1 fld1 +250501 250501 +250502 250501 +250503 250501 +250504 250501 +250505 250501 +250501 250502 +250502 250502 +250503 250502 +250504 250502 +250505 250502 +250501 250503 +250502 250503 +250503 250503 +250504 250503 +250505 250503 +250501 250504 +250502 250504 +250503 250504 +250504 250504 +250505 250504 +250501 250505 +250502 250505 +250503 250505 +250504 250505 +250505 250505 +insert into t2 (fld1, companynr) values (999999,99); +select t2.companynr,companyname from t2 left join t4 using (companynr) where t4.companynr is null; +companynr companyname +99 NULL +select count(*) from t2 left join t4 using (companynr) where t4.companynr is not null; +count(*) +1199 +explain select t2.companynr,companyname from t2 left join t4 using (companynr) where t4.companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1200 +1 SIMPLE t4 eq_ref PRIMARY PRIMARY 1 test.t2.companynr 1 Using where; Not exists +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL NULL NULL NULL NULL 12 +1 SIMPLE t2 ALL NULL NULL NULL NULL 1200 Using where; Not exists +select companynr,companyname from t2 left join t4 using (companynr) where companynr is null; +companynr companyname +select count(*) from t2 left join t4 using (companynr) where companynr is not null; +count(*) +1200 +explain select companynr,companyname from t2 left join t4 using (companynr) where companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +delete from t2 where fld1=999999; +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +1 SIMPLE t4 eq_ref PRIMARY PRIMARY 1 test.t2.companynr 1 +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr > 0 or t2.companynr < 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +1 SIMPLE t4 eq_ref PRIMARY PRIMARY 1 test.t2.companynr 1 +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr > 0 and t4.companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +1 SIMPLE t4 eq_ref PRIMARY PRIMARY 1 test.t2.companynr 1 +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr > 0 or companynr < 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr > 0 and companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr > 0 or t2.companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL NULL NULL NULL NULL 12 +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where t2.companynr > 0 or t2.companynr < 0 or t4.companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +explain select t2.companynr,companyname from t4 left join t2 using (companynr) where ifnull(t2.companynr,1)>0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL NULL NULL NULL NULL 12 +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr > 0 or companynr is null; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +explain select companynr,companyname from t4 left join t2 using (companynr) where companynr > 0 or companynr < 0 or companynr > 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL PRIMARY NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +explain select companynr,companyname from t4 left join t2 using (companynr) where ifnull(companynr,1)>0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 ALL NULL NULL NULL NULL 12 Using where +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +select distinct t2.companynr,t4.companynr from t2,t4 where t2.companynr=t4.companynr+1; +companynr companynr +37 36 +41 40 +explain select distinct t2.companynr,t4.companynr from t2,t4 where t2.companynr=t4.companynr+1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t4 index NULL PRIMARY 1 NULL 12 Using index; Using temporary +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 Using where; Using join buffer +select t2.fld1,t2.companynr,fld3,period from t3,t2 where t2.fld1 = 38208 and t2.fld1=t3.t2nr and period = 1008 or t2.fld1 = 38008 and t2.fld1 =t3.t2nr and period = 1008; +fld1 companynr fld3 period +038008 37 reporters 1008 +038208 37 Selfridge 1008 +select t2.fld1,t2.companynr,fld3,period from t3,t2 where (t2.fld1 = 38208 or t2.fld1 = 38008) and t2.fld1=t3.t2nr and period>=1008 and period<=1009; +fld1 companynr fld3 period +038008 37 reporters 1008 +038208 37 Selfridge 1008 +select t2.fld1,t2.companynr,fld3,period from t3,t2 where (t3.t2nr = 38208 or t3.t2nr = 38008) and t2.fld1=t3.t2nr and period>=1008 and period<=1009; +fld1 companynr fld3 period +038008 37 reporters 1008 +038208 37 Selfridge 1008 +select period from t1 where (((period > 0) or period < 10000 or (period = 1900)) and (period=1900 and period <= 1901) or (period=1903 and (period=1903)) and period>=1902) or ((period=1904 or period=1905) or (period=1906 or period>1907)) or (period=1908 and period = 1909); +period +9410 +select period from t1 where ((period > 0 and period < 1) or (((period > 0 and period < 100) and (period > 10)) or (period > 10)) or (period > 0 and (period > 5 or period > 6))); +period +9410 +select a.fld1 from t2 as a,t2 b where ((a.fld1 = 250501 and a.fld1=b.fld1) or a.fld1=250502 or a.fld1=250503 or (a.fld1=250505 and a.fld1<=b.fld1 and b.fld1>=a.fld1)) and a.fld1=b.fld1; +fld1 +250501 +250502 +250503 +250505 +select fld1 from t2 where fld1 in (250502,98005,98006,250503,250605,250606) and fld1 >=250502 and fld1 not in (250605,250606); +fld1 +250502 +250503 +select fld1 from t2 where fld1 between 250502 and 250504; +fld1 +250502 +250503 +250504 +select fld3 from t2 where (((fld3 like "_%L%" ) or (fld3 like "%ok%")) and ( fld3 like "L%" or fld3 like "G%")) and fld3 like "L%" ; +fld3 +label +labeled +labeled +landslide +laterally +leaflet +lewdly +Lillian +luckily +select count(*) from t1; +count(*) +1 +select companynr,count(*),sum(fld1) from t2 group by companynr; +companynr count(*) sum(fld1) +00 82 10355753 +29 95 14473298 +34 70 17788966 +36 215 22786296 +37 588 83602098 +40 37 6618386 +41 52 12816335 +50 11 1595438 +53 4 793210 +58 23 2254293 +65 10 2284055 +68 12 3097288 +select companynr,count(*) from t2 group by companynr order by companynr desc limit 5; +companynr count(*) +68 12 +65 10 +58 23 +53 4 +50 11 +select count(*),min(fld4),max(fld4),sum(fld1),avg(fld1),std(fld1),variance(fld1) from t2 where companynr = 34 and fld4<>""; +count(*) min(fld4) max(fld4) sum(fld1) avg(fld1) std(fld1) variance(fld1) +70 absentee vest 17788966 254128.0857 3272.5940 10709871.3069 +explain extended select count(*),min(fld4),max(fld4),sum(fld1),avg(fld1),std(fld1),variance(fld1) from t2 where companynr = 34 and fld4<>""; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 100.00 Using where +Warnings: +Note 1003 select count(0) AS `count(*)`,min(`test`.`t2`.`fld4`) AS `min(fld4)`,max(`test`.`t2`.`fld4`) AS `max(fld4)`,sum(`test`.`t2`.`fld1`) AS `sum(fld1)`,avg(`test`.`t2`.`fld1`) AS `avg(fld1)`,std(`test`.`t2`.`fld1`) AS `std(fld1)`,variance(`test`.`t2`.`fld1`) AS `variance(fld1)` from `test`.`t2` where ((`test`.`t2`.`companynr` = 34) and (`test`.`t2`.`fld4` <> '')) +select companynr,count(*),min(fld4),max(fld4),sum(fld1),avg(fld1),std(fld1),variance(fld1) from t2 group by companynr limit 3; +companynr count(*) min(fld4) max(fld4) sum(fld1) avg(fld1) std(fld1) variance(fld1) +00 82 Anthony windmills 10355753 126289.6707 115550.9757 13352027981.7087 +29 95 abut wetness 14473298 152350.5053 8368.5480 70032594.9026 +34 70 absentee vest 17788966 254128.0857 3272.5940 10709871.3069 +select companynr,t2nr,count(price),sum(price),min(price),max(price),avg(price) from t3 where companynr = 37 group by companynr,t2nr limit 10; +companynr t2nr count(price) sum(price) min(price) max(price) avg(price) +37 1 1 5987435 5987435 5987435 5987435.0000 +37 2 1 28357832 28357832 28357832 28357832.0000 +37 3 1 39654943 39654943 39654943 39654943.0000 +37 11 1 5987435 5987435 5987435 5987435.0000 +37 12 1 28357832 28357832 28357832 28357832.0000 +37 13 1 39654943 39654943 39654943 39654943.0000 +37 21 1 5987435 5987435 5987435 5987435.0000 +37 22 1 28357832 28357832 28357832 28357832.0000 +37 23 1 39654943 39654943 39654943 39654943.0000 +37 31 1 5987435 5987435 5987435 5987435.0000 +select /*! SQL_SMALL_RESULT */ companynr,t2nr,count(price),sum(price),min(price),max(price),avg(price) from t3 where companynr = 37 group by companynr,t2nr limit 10; +companynr t2nr count(price) sum(price) min(price) max(price) avg(price) +37 1 1 5987435 5987435 5987435 5987435.0000 +37 2 1 28357832 28357832 28357832 28357832.0000 +37 3 1 39654943 39654943 39654943 39654943.0000 +37 11 1 5987435 5987435 5987435 5987435.0000 +37 12 1 28357832 28357832 28357832 28357832.0000 +37 13 1 39654943 39654943 39654943 39654943.0000 +37 21 1 5987435 5987435 5987435 5987435.0000 +37 22 1 28357832 28357832 28357832 28357832.0000 +37 23 1 39654943 39654943 39654943 39654943.0000 +37 31 1 5987435 5987435 5987435 5987435.0000 +select companynr,count(price),sum(price),min(price),max(price),avg(price) from t3 group by companynr ; +companynr count(price) sum(price) min(price) max(price) avg(price) +37 12543 309394878010 5987435 39654943 24666736.6667 +78 8362 414611089292 726498 98439034 49582766.0000 +101 4181 3489454238 834598 834598 834598.0000 +154 4181 4112197254950 983543950 983543950 983543950.0000 +311 4181 979599938 234298 234298 234298.0000 +447 4181 9929180954 2374834 2374834 2374834.0000 +512 4181 3288532102 786542 786542 786542.0000 +select distinct mod(companynr,10) from t4 group by companynr; +mod(companynr,10) +0 +9 +4 +6 +7 +1 +3 +8 +5 +select distinct 1 from t4 group by companynr; +1 +1 +select count(distinct fld1) from t2; +count(distinct fld1) +1199 +select companynr,count(distinct fld1) from t2 group by companynr; +companynr count(distinct fld1) +00 82 +29 95 +34 70 +36 215 +37 588 +40 37 +41 52 +50 11 +53 4 +58 23 +65 10 +68 12 +select companynr,count(*) from t2 group by companynr; +companynr count(*) +00 82 +29 95 +34 70 +36 215 +37 588 +40 37 +41 52 +50 11 +53 4 +58 23 +65 10 +68 12 +select companynr,count(distinct concat(fld1,repeat(65,1000))) from t2 group by companynr; +companynr count(distinct concat(fld1,repeat(65,1000))) +00 82 +29 95 +34 70 +36 215 +37 588 +40 37 +41 52 +50 11 +53 4 +58 23 +65 10 +68 12 +select companynr,count(distinct concat(fld1,repeat(65,200))) from t2 group by companynr; +companynr count(distinct concat(fld1,repeat(65,200))) +00 82 +29 95 +34 70 +36 215 +37 588 +40 37 +41 52 +50 11 +53 4 +58 23 +65 10 +68 12 +select companynr,count(distinct floor(fld1/100)) from t2 group by companynr; +companynr count(distinct floor(fld1/100)) +00 47 +29 35 +34 14 +36 69 +37 108 +40 16 +41 11 +50 9 +53 1 +58 1 +65 1 +68 1 +select companynr,count(distinct concat(repeat(65,1000),floor(fld1/100))) from t2 group by companynr; +companynr count(distinct concat(repeat(65,1000),floor(fld1/100))) +00 47 +29 35 +34 14 +36 69 +37 108 +40 16 +41 11 +50 9 +53 1 +58 1 +65 1 +68 1 +select sum(fld1),fld3 from t2 where fld3="Romans" group by fld1 limit 10; +sum(fld1) fld3 +11402 Romans +select name,count(*) from t3 where name='cloakroom' group by name; +name count(*) +cloakroom 4181 +select name,count(*) from t3 where name='cloakroom' and price>10 group by name; +name count(*) +cloakroom 4181 +select count(*) from t3 where name='cloakroom' and price2=823742; +count(*) +4181 +select name,count(*) from t3 where name='cloakroom' and price2=823742 group by name; +name count(*) +cloakroom 4181 +select name,count(*) from t3 where name >= "extramarital" and price <= 39654943 group by name; +name count(*) +extramarital 4181 +gazer 4181 +gems 4181 +Iranizes 4181 +spates 4181 +tucked 4181 +violinist 4181 +select t2.fld3,count(*) from t2,t3 where t2.fld1=158402 and t3.name=t2.fld3 group by t3.name; +fld3 count(*) +spates 4181 +select companynr|0,companyname from t4 group by 1; +companynr|0 companyname +0 Unknown +29 company 1 +34 company 2 +36 company 3 +37 company 4 +40 company 5 +41 company 6 +50 company 11 +53 company 7 +58 company 8 +65 company 9 +68 company 10 +select t2.companynr,companyname,count(*) from t2,t4 where t2.companynr=t4.companynr group by t2.companynr order by companyname; +companynr companyname count(*) +29 company 1 95 +68 company 10 12 +50 company 11 11 +34 company 2 70 +36 company 3 215 +37 company 4 588 +40 company 5 37 +41 company 6 52 +53 company 7 4 +58 company 8 23 +65 company 9 10 +00 Unknown 82 +select t2.fld1,count(*) from t2,t3 where t2.fld1=158402 and t3.name=t2.fld3 group by t3.name; +fld1 count(*) +158402 4181 +select sum(Period)/count(*) from t1; +sum(Period)/count(*) +9410.0000 +select companynr,count(price) as "count",sum(price) as "sum" ,abs(sum(price)/count(price)-avg(price)) as "diff",(0+count(price))*companynr as func from t3 group by companynr; +companynr count sum diff func +37 12543 309394878010 0.0000 464091 +78 8362 414611089292 0.0000 652236 +101 4181 3489454238 0.0000 422281 +154 4181 4112197254950 0.0000 643874 +311 4181 979599938 0.0000 1300291 +447 4181 9929180954 0.0000 1868907 +512 4181 3288532102 0.0000 2140672 +select companynr,sum(price)/count(price) as avg from t3 group by companynr having avg > 70000000 order by avg; +companynr avg +154 983543950.0000 +select companynr,count(*) from t2 group by companynr order by 2 desc; +companynr count(*) +37 588 +36 215 +29 95 +00 82 +34 70 +41 52 +40 37 +58 23 +68 12 +50 11 +65 10 +53 4 +select companynr,count(*) from t2 where companynr > 40 group by companynr order by 2 desc; +companynr count(*) +41 52 +58 23 +68 12 +50 11 +65 10 +53 4 +select t2.fld4,t2.fld1,count(price),sum(price),min(price),max(price),avg(price) from t3,t2 where t3.companynr = 37 and t2.fld1 = t3.t2nr group by fld1,t2.fld4; +fld4 fld1 count(price) sum(price) min(price) max(price) avg(price) +teethe 000001 1 5987435 5987435 5987435 5987435.0000 +dreaded 011401 1 5987435 5987435 5987435 5987435.0000 +scholastics 011402 1 28357832 28357832 28357832 28357832.0000 +audiology 011403 1 39654943 39654943 39654943 39654943.0000 +wallet 011501 1 5987435 5987435 5987435 5987435.0000 +parters 011701 1 5987435 5987435 5987435 5987435.0000 +eschew 011702 1 28357832 28357832 28357832 28357832.0000 +quitter 011703 1 39654943 39654943 39654943 39654943.0000 +neat 012001 1 5987435 5987435 5987435 5987435.0000 +Steinberg 012003 1 39654943 39654943 39654943 39654943.0000 +balled 012301 1 5987435 5987435 5987435 5987435.0000 +persist 012302 1 28357832 28357832 28357832 28357832.0000 +attainments 012303 1 39654943 39654943 39654943 39654943.0000 +capably 012501 1 5987435 5987435 5987435 5987435.0000 +impulsive 012602 1 28357832 28357832 28357832 28357832.0000 +starlet 012603 1 39654943 39654943 39654943 39654943.0000 +featherweight 012701 1 5987435 5987435 5987435 5987435.0000 +pessimist 012702 1 28357832 28357832 28357832 28357832.0000 +daughter 012703 1 39654943 39654943 39654943 39654943.0000 +lawgiver 013601 1 5987435 5987435 5987435 5987435.0000 +stated 013602 1 28357832 28357832 28357832 28357832.0000 +readable 013603 1 39654943 39654943 39654943 39654943.0000 +testicle 013801 1 5987435 5987435 5987435 5987435.0000 +Parsifal 013802 1 28357832 28357832 28357832 28357832.0000 +leavings 013803 1 39654943 39654943 39654943 39654943.0000 +squeaking 013901 1 5987435 5987435 5987435 5987435.0000 +contrasted 016001 1 5987435 5987435 5987435 5987435.0000 +leftover 016201 1 5987435 5987435 5987435 5987435.0000 +whiteners 016202 1 28357832 28357832 28357832 28357832.0000 +erases 016301 1 5987435 5987435 5987435 5987435.0000 +Punjab 016302 1 28357832 28357832 28357832 28357832.0000 +Merritt 016303 1 39654943 39654943 39654943 39654943.0000 +sweetish 018001 1 5987435 5987435 5987435 5987435.0000 +dogging 018002 1 28357832 28357832 28357832 28357832.0000 +scornfully 018003 1 39654943 39654943 39654943 39654943.0000 +fetters 018012 1 28357832 28357832 28357832 28357832.0000 +bivalves 018013 1 39654943 39654943 39654943 39654943.0000 +skulking 018021 1 5987435 5987435 5987435 5987435.0000 +flint 018022 1 28357832 28357832 28357832 28357832.0000 +flopping 018023 1 39654943 39654943 39654943 39654943.0000 +Judas 018032 1 28357832 28357832 28357832 28357832.0000 +vacuuming 018033 1 39654943 39654943 39654943 39654943.0000 +medical 018041 1 5987435 5987435 5987435 5987435.0000 +bloodbath 018042 1 28357832 28357832 28357832 28357832.0000 +subschema 018043 1 39654943 39654943 39654943 39654943.0000 +interdependent 018051 1 5987435 5987435 5987435 5987435.0000 +Graves 018052 1 28357832 28357832 28357832 28357832.0000 +neonatal 018053 1 39654943 39654943 39654943 39654943.0000 +sorters 018061 1 5987435 5987435 5987435 5987435.0000 +epistle 018062 1 28357832 28357832 28357832 28357832.0000 +Conley 018101 1 5987435 5987435 5987435 5987435.0000 +lectured 018102 1 28357832 28357832 28357832 28357832.0000 +Abraham 018103 1 39654943 39654943 39654943 39654943.0000 +cage 018201 1 5987435 5987435 5987435 5987435.0000 +hushes 018202 1 28357832 28357832 28357832 28357832.0000 +Simla 018402 1 28357832 28357832 28357832 28357832.0000 +reporters 018403 1 39654943 39654943 39654943 39654943.0000 +coexist 018601 1 5987435 5987435 5987435 5987435.0000 +Beebe 018602 1 28357832 28357832 28357832 28357832.0000 +Taoism 018603 1 39654943 39654943 39654943 39654943.0000 +Connally 018801 1 5987435 5987435 5987435 5987435.0000 +fetched 018802 1 28357832 28357832 28357832 28357832.0000 +checkpoints 018803 1 39654943 39654943 39654943 39654943.0000 +gritty 018811 1 5987435 5987435 5987435 5987435.0000 +firearm 018812 1 28357832 28357832 28357832 28357832.0000 +minima 019101 1 5987435 5987435 5987435 5987435.0000 +Selfridge 019102 1 28357832 28357832 28357832 28357832.0000 +disable 019103 1 39654943 39654943 39654943 39654943.0000 +witchcraft 019201 1 5987435 5987435 5987435 5987435.0000 +betroth 030501 1 5987435 5987435 5987435 5987435.0000 +Manhattanize 030502 1 28357832 28357832 28357832 28357832.0000 +imprint 030503 1 39654943 39654943 39654943 39654943.0000 +swelling 031901 1 5987435 5987435 5987435 5987435.0000 +interrelationships 036001 1 5987435 5987435 5987435 5987435.0000 +riser 036002 1 28357832 28357832 28357832 28357832.0000 +bee 038001 1 5987435 5987435 5987435 5987435.0000 +kanji 038002 1 28357832 28357832 28357832 28357832.0000 +dental 038003 1 39654943 39654943 39654943 39654943.0000 +railway 038011 1 5987435 5987435 5987435 5987435.0000 +validate 038012 1 28357832 28357832 28357832 28357832.0000 +normalizes 038013 1 39654943 39654943 39654943 39654943.0000 +Kline 038101 1 5987435 5987435 5987435 5987435.0000 +Anatole 038102 1 28357832 28357832 28357832 28357832.0000 +partridges 038103 1 39654943 39654943 39654943 39654943.0000 +recruited 038201 1 5987435 5987435 5987435 5987435.0000 +dimensions 038202 1 28357832 28357832 28357832 28357832.0000 +Chicana 038203 1 39654943 39654943 39654943 39654943.0000 +select t3.companynr,fld3,sum(price) from t3,t2 where t2.fld1 = t3.t2nr and t3.companynr = 512 group by companynr,fld3; +companynr fld3 sum(price) +512 boat 786542 +512 capably 786542 +512 cupboard 786542 +512 decliner 786542 +512 descendants 786542 +512 dopers 786542 +512 erases 786542 +512 Micronesia 786542 +512 Miles 786542 +512 skies 786542 +select t2.companynr,count(*),min(fld3),max(fld3),sum(price),avg(price) from t2,t3 where t3.companynr >= 30 and t3.companynr <= 58 and t3.t2nr = t2.fld1 and 1+1=2 group by t2.companynr; +companynr count(*) min(fld3) max(fld3) sum(price) avg(price) +00 1 Omaha Omaha 5987435 5987435.0000 +36 1 dubbed dubbed 28357832 28357832.0000 +37 83 Abraham Wotan 1908978016 22999735.1325 +50 2 scribbled tapestry 68012775 34006387.5000 +select t3.companynr+0,t3.t2nr,fld3,sum(price) from t3,t2 where t2.fld1 = t3.t2nr and t3.companynr = 37 group by 1,t3.t2nr,fld3,fld3,fld3,fld3,fld3 order by fld1; +t3.companynr+0 t2nr fld3 sum(price) +37 1 Omaha 5987435 +37 11401 breaking 5987435 +37 11402 Romans 28357832 +37 11403 intercepted 39654943 +37 11501 bewilderingly 5987435 +37 11701 astound 5987435 +37 11702 admonishing 28357832 +37 11703 sumac 39654943 +37 12001 flanking 5987435 +37 12003 combed 39654943 +37 12301 Eulerian 5987435 +37 12302 dubbed 28357832 +37 12303 Kane 39654943 +37 12501 annihilates 5987435 +37 12602 Wotan 28357832 +37 12603 snatching 39654943 +37 12701 grazing 5987435 +37 12702 Baird 28357832 +37 12703 celery 39654943 +37 13601 handgun 5987435 +37 13602 foldout 28357832 +37 13603 mystic 39654943 +37 13801 intelligibility 5987435 +37 13802 Augustine 28357832 +37 13803 teethe 39654943 +37 13901 scholastics 5987435 +37 16001 audiology 5987435 +37 16201 wallet 5987435 +37 16202 parters 28357832 +37 16301 eschew 5987435 +37 16302 quitter 28357832 +37 16303 neat 39654943 +37 18001 jarring 5987435 +37 18002 tinily 28357832 +37 18003 balled 39654943 +37 18012 impulsive 28357832 +37 18013 starlet 39654943 +37 18021 lawgiver 5987435 +37 18022 stated 28357832 +37 18023 readable 39654943 +37 18032 testicle 28357832 +37 18033 Parsifal 39654943 +37 18041 Punjab 5987435 +37 18042 Merritt 28357832 +37 18043 Quixotism 39654943 +37 18051 sureties 5987435 +37 18052 puddings 28357832 +37 18053 tapestry 39654943 +37 18061 trimmings 5987435 +37 18062 humility 28357832 +37 18101 tragedies 5987435 +37 18102 skulking 28357832 +37 18103 flint 39654943 +37 18201 relaxing 5987435 +37 18202 offload 28357832 +37 18402 suites 28357832 +37 18403 lists 39654943 +37 18601 vacuuming 5987435 +37 18602 dentally 28357832 +37 18603 humanness 39654943 +37 18801 inch 5987435 +37 18802 Weissmuller 28357832 +37 18803 irresponsibly 39654943 +37 18811 repetitions 5987435 +37 18812 Antares 28357832 +37 19101 ventilate 5987435 +37 19102 pityingly 28357832 +37 19103 interdependent 39654943 +37 19201 Graves 5987435 +37 30501 neonatal 5987435 +37 30502 scribbled 28357832 +37 30503 chafe 39654943 +37 31901 realtor 5987435 +37 36001 elite 5987435 +37 36002 funereal 28357832 +37 38001 Conley 5987435 +37 38002 lectured 28357832 +37 38003 Abraham 39654943 +37 38011 groupings 5987435 +37 38012 dissociate 28357832 +37 38013 coexist 39654943 +37 38101 rusting 5987435 +37 38102 galling 28357832 +37 38103 obliterates 39654943 +37 38201 resumes 5987435 +37 38202 analyzable 28357832 +37 38203 terminator 39654943 +select sum(price) from t3,t2 where t2.fld1 = t3.t2nr and t3.companynr = 512 and t3.t2nr = 38008 and t2.fld1 = 38008 or t2.fld1= t3.t2nr and t3.t2nr = 38008 and t2.fld1 = 38008; +sum(price) +234298 +select t2.fld1,sum(price) from t3,t2 where t2.fld1 = t3.t2nr and t3.companynr = 512 and t3.t2nr = 38008 and t2.fld1 = 38008 or t2.fld1 = t3.t2nr and t3.t2nr = 38008 and t2.fld1 = 38008 or t3.t2nr = t2.fld1 and t2.fld1 = 38008 group by t2.fld1; +fld1 sum(price) +038008 234298 +explain select fld3 from t2 where 1>2 or 2>3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +explain select fld3 from t2 where fld1=fld1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 1199 +select companynr,fld1 from t2 HAVING fld1=250501 or fld1=250502; +companynr fld1 +34 250501 +34 250502 +select companynr,fld1 from t2 WHERE fld1>=250501 HAVING fld1<=250502; +companynr fld1 +34 250501 +34 250502 +select companynr,count(*) as count,sum(fld1) as sum from t2 group by companynr having count > 40 and sum/count >= 120000; +companynr count sum +00 82 10355753 +29 95 14473298 +34 70 17788966 +37 588 83602098 +41 52 12816335 +select companynr from t2 group by companynr having count(*) > 40 and sum(fld1)/count(*) >= 120000 ; +companynr +00 +29 +34 +37 +41 +select t2.companynr,companyname,count(*) from t2,t4 where t2.companynr=t4.companynr group by companyname having t2.companynr >= 40; +companynr companyname count(*) +68 company 10 12 +50 company 11 11 +40 company 5 37 +41 company 6 52 +53 company 7 4 +58 company 8 23 +65 company 9 10 +select count(*) from t2; +count(*) +1199 +select count(*) from t2 where fld1 < 098024; +count(*) +387 +select min(fld1) from t2 where fld1>= 098024; +min(fld1) +98024 +select max(fld1) from t2 where fld1>= 098024; +max(fld1) +1232609 +select count(*) from t3 where price2=76234234; +count(*) +4181 +select count(*) from t3 where companynr=512 and price2=76234234; +count(*) +4181 +explain select min(fld1),max(fld1),count(*) from t2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away +select min(fld1),max(fld1),count(*) from t2; +min(fld1) max(fld1) count(*) +0 1232609 1199 +select min(t2nr),max(t2nr) from t3 where t2nr=2115 and price2=823742; +min(t2nr) max(t2nr) +2115 2115 +select count(*),min(t2nr),max(t2nr) from t3 where name='spates' and companynr=78; +count(*) min(t2nr) max(t2nr) +4181 4 41804 +select t2nr,count(*) from t3 where name='gems' group by t2nr limit 20; +t2nr count(*) +9 1 +19 1 +29 1 +39 1 +49 1 +59 1 +69 1 +79 1 +89 1 +99 1 +109 1 +119 1 +129 1 +139 1 +149 1 +159 1 +169 1 +179 1 +189 1 +199 1 +select max(t2nr) from t3 where price=983543950; +max(t2nr) +41807 +select t1.period from t3 = t1 limit 1; +period +1001 +select t1.period from t1 as t1 limit 1; +period +9410 +select t1.period as "Nuvarande period" from t1 as t1 limit 1; +Nuvarande period +9410 +select period as ok_period from t1 limit 1; +ok_period +9410 +select period as ok_period from t1 group by ok_period limit 1; +ok_period +9410 +select 1+1 as summa from t1 group by summa limit 1; +summa +2 +select period as "Nuvarande period" from t1 group by "Nuvarande period" limit 1; +Nuvarande period +9410 +show tables; +Tables_in_test +t1 +t2 +t3 +t4 +show tables from test like "s%"; +Tables_in_test (s%) +show tables from test like "t?"; +Tables_in_test (t?) +show full columns from t2; +Field Type Collation Null Key Default Extra Privileges Comment +auto int(11) NULL NO PRI NULL auto_increment # +fld1 int(6) unsigned zerofill NULL NO UNI 000000 # +companynr tinyint(2) unsigned zerofill NULL NO 00 # +fld3 char(30) latin1_swedish_ci NO MUL # +fld4 char(35) latin1_swedish_ci NO # +fld5 char(35) latin1_swedish_ci NO # +fld6 char(4) latin1_swedish_ci NO # +show full columns from t2 from test like 'f%'; +Field Type Collation Null Key Default Extra Privileges Comment +fld1 int(6) unsigned zerofill NULL NO UNI 000000 # +fld3 char(30) latin1_swedish_ci NO MUL # +fld4 char(35) latin1_swedish_ci NO # +fld5 char(35) latin1_swedish_ci NO # +fld6 char(4) latin1_swedish_ci NO # +show full columns from t2 from test like 's%'; +Field Type Collation Null Key Default Extra Privileges Comment +show keys from t2; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment +t2 0 PRIMARY 1 auto A 1199 NULL NULL BTREE +t2 0 fld1 1 fld1 A 1199 NULL NULL BTREE +t2 1 fld3 1 fld3 A NULL NULL NULL BTREE +drop table t4, t3, t2, t1; +CREATE TABLE t1 ( +cont_nr int(11) NOT NULL auto_increment, +ver_nr int(11) NOT NULL default '0', +aufnr int(11) NOT NULL default '0', +username varchar(50) NOT NULL default '', +hdl_nr int(11) NOT NULL default '0', +eintrag date NOT NULL default '0000-00-00', +st_klasse varchar(40) NOT NULL default '', +st_wert varchar(40) NOT NULL default '', +st_zusatz varchar(40) NOT NULL default '', +st_bemerkung varchar(255) NOT NULL default '', +kunden_art varchar(40) NOT NULL default '', +mcbs_knr int(11) default NULL, +mcbs_aufnr int(11) NOT NULL default '0', +schufa_status char(1) default '?', +bemerkung text, +wirknetz text, +wf_igz int(11) NOT NULL default '0', +tarifcode varchar(80) default NULL, +recycle char(1) default NULL, +sim varchar(30) default NULL, +mcbs_tpl varchar(30) default NULL, +emp_nr int(11) NOT NULL default '0', +laufzeit int(11) default NULL, +hdl_name varchar(30) default NULL, +prov_hdl_nr int(11) NOT NULL default '0', +auto_wirknetz varchar(50) default NULL, +auto_billing varchar(50) default NULL, +touch timestamp NOT NULL, +kategorie varchar(50) default NULL, +kundentyp varchar(20) NOT NULL default '', +sammel_rech_msisdn varchar(30) NOT NULL default '', +p_nr varchar(9) NOT NULL default '', +suffix char(3) NOT NULL default '', +PRIMARY KEY (cont_nr), +KEY idx_aufnr(aufnr), +KEY idx_hdl_nr(hdl_nr), +KEY idx_st_klasse(st_klasse), +KEY ver_nr(ver_nr), +KEY eintrag_idx(eintrag), +KEY emp_nr_idx(emp_nr), +KEY wf_igz(wf_igz), +KEY touch(touch), +KEY hdl_tag(eintrag,hdl_nr), +KEY prov_hdl_nr(prov_hdl_nr), +KEY mcbs_aufnr(mcbs_aufnr), +KEY kundentyp(kundentyp), +KEY p_nr(p_nr,suffix) +) ENGINE=MyISAM; +INSERT INTO t1 VALUES (3359356,405,3359356,'Mustermann Musterfrau',52500,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1485525,2122316,'+','','N',1909160,'MobilComSuper92000D2',NULL,NULL,'MS9ND2',3,24,'MobilCom Shop Koeln',52500,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +INSERT INTO t1 VALUES (3359357,468,3359357,'Mustermann Musterfrau',7001,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1503580,2139699,'+','','P',1909171,'MobilComSuper9D1T10SFreisprech(Akquise)',NULL,NULL,'MS9NS1',327,24,'MobilCom Intern',7003,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +INSERT INTO t1 VALUES (3359358,407,3359358,'Mustermann Musterfrau',7001,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1501358,2137473,'N','','N',1909159,'MobilComSuper92000D2',NULL,NULL,'MS9ND2',325,24,'MobilCom Intern',7003,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +INSERT INTO t1 VALUES (3359359,468,3359359,'Mustermann Musterfrau',7001,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1507831,2143894,'+','','P',1909162,'MobilComSuper9D1T10SFreisprech(Akquise)',NULL,NULL,'MS9NS1',327,24,'MobilCom Intern',7003,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +INSERT INTO t1 VALUES (3359360,0,0,'Mustermann Musterfrau',29674907,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1900169997,2414578,'+',NULL,'N',1909148,'',NULL,NULL,'RV99066_2',20,NULL,'POS',29674907,NULL,NULL,20010202105916,'Mobilfunk','','','97317481','007'); +INSERT INTO t1 VALUES (3359361,406,3359361,'Mustermann Musterfrau',7001,'2000-05-20','workflow','Auftrag storniert','','(7001-84):Storno, Kd. m��chte nicht mehr','privat',NULL,0,'+','','P',1909150,'MobilComSuper92000D1(Akquise)',NULL,NULL,'MS9ND1',325,24,'MobilCom Intern',7003,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +INSERT INTO t1 VALUES (3359362,406,3359362,'Mustermann Musterfrau',7001,'2000-05-20','workflow','Auftrag erledigt','Originalvertrag eingegangen und gepr��ft','','privat',1509984,2145874,'+','','P',1909154,'MobilComSuper92000D1(Akquise)',NULL,NULL,'MS9ND1',327,24,'MobilCom Intern',7003,NULL,'auto',20010202105916,'Mobilfunk','PP','','',''); +SELECT ELT(FIELD(kundentyp,'PP','PPA','PG','PGA','FK','FKA','FP','FPA','K','KA','V','VA',''), 'Privat (Private Nutzung)','Privat (Private Nutzung) Sitz im Ausland','Privat (geschaeftliche Nutzung)','Privat (geschaeftliche Nutzung) Sitz im Ausland','Firma (Kapitalgesellschaft)','Firma (Kapitalgesellschaft) Sitz im Ausland','Firma (Personengesellschaft)','Firma (Personengesellschaft) Sitz im Ausland','oeff. rechtl. Koerperschaft','oeff. rechtl. Koerperschaft Sitz im Ausland','Eingetragener Verein','Eingetragener Verein Sitz im Ausland','Typ unbekannt') AS Kundentyp ,kategorie FROM t1 WHERE hdl_nr < 2000000 AND kategorie IN ('Prepaid','Mobilfunk') AND st_klasse = 'Workflow' GROUP BY kundentyp ORDER BY kategorie; +Kundentyp kategorie +Privat (Private Nutzung) Mobilfunk +Warnings: +Warning 1052 Column 'kundentyp' in group statement is ambiguous +drop table t1; === modified file 'mysql-test/r/subselect_debug.result' --- a/mysql-test/r/subselect_debug.result 2009-02-01 12:02:29 +0000 +++ b/mysql-test/r/subselect_debug.result 2009-03-12 22:27:35 +0000 @@ -2,12 +2,12 @@ CREATE TABLE t1(id INT); INSERT INTO t1 VALUES (1),(2),(3),(4); INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d; SET @orig_debug=@@debug; -SET SESSION debug="d,subselect_exec_fail"; +SET GLOBAL debug="d,subselect_exec_fail"; SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1; SUM(EXISTS(SELECT RAND() FROM t1)) 0 SELECT REVERSE(EXISTS(SELECT RAND() FROM t1)); REVERSE(EXISTS(SELECT RAND() FROM t1)) 0 -SET SESSION debug=@orig_debug; +SET GLOBAL debug=@orig_debug; DROP TABLE t1; === modified file 'mysql-test/t/crash_commit_before.test' --- a/mysql-test/t/crash_commit_before.test 2007-12-12 17:19:24 +0000 +++ b/mysql-test/t/crash_commit_before.test 2009-03-12 22:27:35 +0000 @@ -12,7 +12,7 @@ START TRANSACTION; insert into t1 values(9); # Setup the mysqld to crash at certain point -SET SESSION debug="d,crash_commit_before"; +SET GLOBAL debug="d,crash_commit_before"; # Write file to make mysql-test-run.pl expect crash and restart --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect === modified file 'mysql-test/t/merge-big.test' --- a/mysql-test/t/merge-big.test 2008-02-07 11:04:19 +0000 +++ b/mysql-test/t/merge-big.test 2009-03-12 22:27:35 +0000 @@ -43,7 +43,7 @@ LOCK TABLE t1 WRITE; --echo # connection con1 connect (con1,localhost,root,,); let $con1_id= `SELECT CONNECTION_ID()`; - SET SESSION debug="+d,sleep_open_and_lock_after_open"; + SET GLOBAL debug="+d,sleep_open_and_lock_after_open"; send INSERT INTO t1 VALUES (1); --echo # connection default connection default; @@ -74,7 +74,7 @@ UNLOCK TABLES; --echo # connection con1 connection con1; reap; - SET SESSION debug="-d,sleep_open_and_lock_after_open"; + SET GLOBAL debug="-d,sleep_open_and_lock_after_open"; disconnect con1; --echo # connection default connection default; === added file 'mysql-test/t/pool_of_threads-master.opt' --- a/mysql-test/t/pool_of_threads-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/pool_of_threads-master.opt 2009-03-12 22:27:35 +0000 @@ -0,0 +1 @@ +--test-ignore-wrong-options --thread-handling=pool-of-threads === added file 'mysql-test/t/pool_of_threads.test' --- a/mysql-test/t/pool_of_threads.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/pool_of_threads.test 2009-03-12 22:27:35 +0000 @@ -0,0 +1,7 @@ +# Start with thread_handling=pool-of-threads +# and run a number of tests + +-- source include/have_pool_of_threads.inc + +-- source include/common-tests.inc + === modified file 'mysql-test/t/subselect_debug.test' --- a/mysql-test/t/subselect_debug.test 2009-02-01 12:02:29 +0000 +++ b/mysql-test/t/subselect_debug.test 2009-03-12 22:27:35 +0000 @@ -10,8 +10,8 @@ INSERT INTO t1 VALUES (1),(2),(3),(4); INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d; # Setup the mysqld to crash at certain point SET @orig_debug=@@debug; -SET SESSION debug="d,subselect_exec_fail"; +SET GLOBAL debug="d,subselect_exec_fail"; SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1; SELECT REVERSE(EXISTS(SELECT RAND() FROM t1)); -SET SESSION debug=@orig_debug; +SET GLOBAL debug=@orig_debug; DROP TABLE t1; === modified file 'mysys/typelib.c' --- a/mysys/typelib.c 2007-05-24 16:47:58 +0000 +++ b/mysys/typelib.c 2009-03-12 22:27:35 +0000 @@ -22,7 +22,7 @@ static const char field_separator=','; -int find_type_or_exit(const char *x, TYPELIB *typelib, const char *option) +int find_type_with_warning(const char *x, TYPELIB *typelib, const char *option) { int res; const char **ptr; @@ -38,12 +38,20 @@ int find_type_or_exit(const char *x, TYP while (*++ptr) fprintf(stderr, ",'%s'", *ptr); fprintf(stderr, "\n"); - exit(1); } return res; } +uint find_type_or_exit(const char *x, TYPELIB *typelib, const char *option) +{ + int res; + if ((res= find_type_with_warning(x, typelib, option)) <= 0) + exit(1); + return (uint) res; +} + + /* Search after a string in a list of strings. Endspace in x is not compared. === modified file 'sql/Makefile.am' --- a/sql/Makefile.am 2008-11-21 14:21:50 +0000 +++ b/sql/Makefile.am 2009-03-12 22:27:35 +0000 @@ -22,7 +22,8 @@ MYSQLLIBdir= $(pkglibdir) pkgplugindir = $(pkglibdir)/plugin INCLUDES = @ZLIB_INCLUDES@ \ -I$(top_builddir)/include -I$(top_srcdir)/include \ - -I$(top_srcdir)/regex -I$(srcdir) $(openssl_includes) + -I$(top_srcdir)/regex -I$(srcdir) $(openssl_includes) \ + $(libevent_includes) WRAPLIBS= @WRAPLIBS@ SUBDIRS = share libexec_PROGRAMS = mysqld @@ -41,7 +42,7 @@ mysqld_DEPENDENCIES= @mysql_plugin_libs@ LDADD = $(SUPPORTING_LIBS) @ZLIB_LIBS@ @NDB_SCI_LIBS@ mysqld_LDADD = libndb.la \ @MYSQLD_EXTRA_LDFLAGS@ \ - @pstack_libs@ \ + @pstack_libs@ $(libevent_libs) \ @mysql_plugin_libs@ \ $(LDADD) $(CXXLDFLAGS) $(WRAPLIBS) @LIBDL@ \ $(yassl_libs) $(openssl_libs) @MYSQLD_EXTRA_LIBS@ === modified file 'sql/handler.cc' --- a/sql/handler.cc 2009-02-19 09:01:25 +0000 +++ b/sql/handler.cc 2009-03-12 22:27:35 +0000 @@ -377,8 +377,7 @@ int ha_finalize_handlerton(st_plugin_int if (!hton) goto end; - switch (hton->state) - { + switch (hton->state) { case SHOW_OPTION_NO: case SHOW_OPTION_DISABLED: break; @@ -435,6 +434,9 @@ int ha_initialize_handlerton(st_plugin_i { sql_print_error("Plugin '%s' init function returned error.", plugin->name.str); + /* Free data, so that we don't refer to it in ha_finalize_handlerton */ + my_free(hton, MYF(0)); + plugin->data= 0; goto err; } } @@ -463,6 +465,8 @@ int ha_initialize_handlerton(st_plugin_i if (idx == (int) DB_TYPE_DEFAULT) { sql_print_warning("Too many storage engines!"); + my_free(hton, MYF(0)); + plugin->data= 0; DBUG_RETURN(1); } if (hton->db_type != DB_TYPE_UNKNOWN) === modified file 'sql/mysql_priv.h' --- a/sql/mysql_priv.h 2009-02-19 09:01:25 +0000 +++ b/sql/mysql_priv.h 2009-03-12 22:27:35 +0000 @@ -1023,6 +1023,9 @@ void time_out_user_resource_limits(THD * void decrease_user_connections(USER_CONN *uc); void thd_init_client_charset(THD *thd, uint cs_number); bool setup_connection_thread_globals(THD *thd); +bool login_connection(THD *thd); +void end_connection(THD *thd); +void prepare_new_connection_state(THD* thd); int mysql_create_db(THD *thd, char *db, HA_CREATE_INFO *create, bool silent); bool mysql_alter_db(THD *thd, const char *db, HA_CREATE_INFO *create); @@ -1912,6 +1915,7 @@ extern ulong query_cache_size, query_cac extern ulong slow_launch_threads, slow_launch_time; extern ulong table_cache_size, table_def_size; extern ulong max_connections,max_connect_errors, connect_timeout; +extern ulong extra_max_connections; extern ulong slave_net_timeout, slave_trans_retries; extern uint max_user_connections; extern ulong what_to_log,flush_time; @@ -1933,7 +1937,7 @@ extern ulong opt_tc_log_size, tc_log_max extern ulong tc_log_page_waits; extern my_bool relay_log_purge, opt_innodb_safe_binlog, opt_innodb; extern uint test_flags,select_errors,ha_open_options; -extern uint protocol_version, mysqld_port, dropping_tables; +extern uint protocol_version, mysqld_port, mysqld_extra_port, dropping_tables; extern uint delay_key_write_options; #endif /* MYSQL_SERVER */ #if defined MYSQL_SERVER || defined INNODB_COMPATIBILITY_HOOKS @@ -1957,7 +1961,8 @@ extern bool opt_disable_networking, opt_ extern my_bool opt_character_set_client_handshake; extern bool volatile abort_loop, shutdown_in_progress; extern uint volatile thread_count, thread_running, global_read_lock; -extern uint connection_count; +extern ulong thread_created; +extern uint connection_count, extra_connection_count; extern my_bool opt_sql_bin_update, opt_safe_user_create, opt_no_mix_types; extern my_bool opt_safe_show_db, opt_local_infile, opt_myisam_use_mmap; extern my_bool opt_slave_compressed_protocol, use_temp_pool; === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-02-19 09:01:25 +0000 +++ b/sql/mysqld.cc 2009-03-12 22:27:35 +0000 @@ -357,8 +357,9 @@ static bool volatile select_thread_in_us static bool volatile ready_to_exit; static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0; static my_bool opt_short_log_format= 0; +static my_bool opt_ignore_wrong_options= 0; static uint kill_cached_threads, wake_thread; -static ulong thread_created; +ulong thread_created; static ulong max_used_connections; static ulong my_bind_addr; /**< the address we bind to */ static volatile ulong cached_thread_count= 0; @@ -469,6 +470,7 @@ const char *opt_binlog_format= binlog_fo static bool calling_initgroups= FALSE; /**< Used in SIGSEGV handler. */ #endif uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options; +uint mysqld_extra_port; uint mysqld_port_timeout; uint delay_key_write_options, protocol_version; uint lower_case_table_names; @@ -495,6 +497,7 @@ ulong delayed_insert_errors,flush_time; ulong specialflag=0; ulong binlog_cache_use= 0, binlog_cache_disk_use= 0; ulong max_connections, max_connect_errors; +ulong extra_max_connections; uint max_user_connections= 0; /** Limit of the total number of prepared statements in the server. @@ -648,7 +651,7 @@ static int defaults_argc; static char **defaults_argv; static char *opt_bin_logname; -static my_socket unix_sock,ip_sock; +static my_socket unix_sock, base_ip_sock, extra_ip_sock; struct my_rnd_struct sql_rand; ///< used by sql_class.cc:THD::THD() #ifndef EMBEDDED_LIBRARY @@ -708,7 +711,7 @@ my_bool opt_enable_shared_memory; HANDLE smem_event_connect_request= 0; #endif -scheduler_functions thread_scheduler; +scheduler_functions thread_scheduler, extra_thread_scheduler; #define SSL_VARS_NOT_STATIC #include "sslopt-vars.h" @@ -735,7 +738,7 @@ struct st_VioSSLFd *ssl_acceptor_fd; Number of currently active user connections. The variable is protected by LOCK_connection_count. */ -uint connection_count= 0; +uint connection_count= 0, extra_connection_count= 0; /* Function declarations */ @@ -831,11 +834,17 @@ static void close_connections(void) DBUG_PRINT("quit",("Closing sockets")); if (!opt_disable_networking ) { - if (ip_sock != INVALID_SOCKET) + if (base_ip_sock != INVALID_SOCKET) { - (void) shutdown(ip_sock, SHUT_RDWR); - (void) closesocket(ip_sock); - ip_sock= INVALID_SOCKET; + (void) shutdown(base_ip_sock, SHUT_RDWR); + (void) closesocket(base_ip_sock); + base_ip_sock= INVALID_SOCKET; + } + if (extra_ip_sock != INVALID_SOCKET) + { + (void) shutdown(extra_ip_sock, SHUT_RDWR); + (void) closesocket(extra_ip_sock); + extra_ip_sock= INVALID_SOCKET; } } #ifdef __NT__ @@ -969,42 +978,39 @@ static void close_connections(void) } -static void close_server_sock() +static void close_socket(my_socket sock, const char *info) { -#ifdef HAVE_CLOSE_SERVER_SOCK - DBUG_ENTER("close_server_sock"); - my_socket tmp_sock; - tmp_sock=ip_sock; - if (tmp_sock != INVALID_SOCKET) - { - ip_sock=INVALID_SOCKET; - DBUG_PRINT("info",("calling shutdown on TCP/IP socket")); - VOID(shutdown(tmp_sock, SHUT_RDWR)); + DBUG_ENTER("close_socket"); + + if (sock != INVALID_SOCKET) + { + DBUG_PRINT("info", ("calling shutdown on %s socket", info)); + (void) shutdown(sock, SHUT_RDWR); #if defined(__NETWARE__) /* The following code is disabled for normal systems as it causes MySQL to hang on AIX 4.3 during shutdown */ - DBUG_PRINT("info",("calling closesocket on TCP/IP socket")); - VOID(closesocket(tmp_sock)); + DBUG_PRINT("info", ("calling closesocket on %s socket", info)); + (void) closesocket(tmp_sock); #endif } - tmp_sock=unix_sock; - if (tmp_sock != INVALID_SOCKET) - { - unix_sock=INVALID_SOCKET; - DBUG_PRINT("info",("calling shutdown on unix socket")); - VOID(shutdown(tmp_sock, SHUT_RDWR)); -#if defined(__NETWARE__) - /* - The following code is disabled for normal systems as it may cause MySQL - to hang on AIX 4.3 during shutdown - */ - DBUG_PRINT("info",("calling closesocket on unix/IP socket")); - VOID(closesocket(tmp_sock)); -#endif + DBUG_VOID_RETURN; +} + + +static void close_server_sock() +{ +#ifdef HAVE_CLOSE_SERVER_SOCK + DBUG_ENTER("close_server_sock"); + + close_socket(base_ip_sock, "TCP/IP"); + close_socket(extra_ip_sock, "TCP/IP"); + close_socket(unix_sock, "unix/IP"); + + if (unix_sock != INVALID_SOCKET) VOID(unlink(mysqld_unix_port)); - } + base_ip_sock= extra_ip_sock= unix_sock= INVALID_SOCKET; DBUG_VOID_RETURN; #endif } @@ -1592,17 +1598,86 @@ static void set_root(const char *path) #endif } -static void network_init(void) +/** + Activate usage of a tcp port +*/ + +static my_socket activate_tcp_port(uint port) { - struct sockaddr_in IPaddr; -#ifdef HAVE_SYS_UN_H - struct sockaddr_un UNIXaddr; -#endif + struct sockaddr_in IPaddr; + my_socket ip_sock; int arg=1; int ret; uint waited; uint this_wait; uint retry; + DBUG_ENTER("activate_tcp_port"); + DBUG_PRINT("enter",("port: %u", port)); + + ip_sock = socket(AF_INET, SOCK_STREAM, 0); + if (ip_sock == INVALID_SOCKET) + { + DBUG_PRINT("error",("Got error: %d from socket()",socket_errno)); + sql_perror(ER(ER_IPSOCK_ERROR)); /* purecov: tested */ + unireg_abort(1); /* purecov: tested */ + } + bzero((char*) &IPaddr, sizeof(IPaddr)); + IPaddr.sin_family = AF_INET; + IPaddr.sin_addr.s_addr = my_bind_addr; + IPaddr.sin_port = (unsigned short) htons((unsigned short) port); + +#ifndef __WIN__ + /* + We should not use SO_REUSEADDR on windows as this would enable a + user to open two mysqld servers with the same TCP/IP port. + */ + (void) setsockopt(ip_sock,SOL_SOCKET,SO_REUSEADDR,(char*)&arg,sizeof(arg)); +#endif /* __WIN__ */ + /* + Sometimes the port is not released fast enough when stopping and + restarting the server. This happens quite often with the test suite + on busy Linux systems. Retry to bind the address at these intervals: + Sleep intervals: 1, 2, 4, 6, 9, 13, 17, 22, ... + Retry at second: 1, 3, 7, 13, 22, 35, 52, 74, ... + Limit the sequence by mysqld_port_timeout (set --port-open-timeout=#). + */ + + for (waited= 0, retry= 1; ; retry++, waited+= this_wait) + { + if (((ret= bind(ip_sock, my_reinterpret_cast(struct sockaddr *) (&IPaddr), + sizeof(IPaddr))) >= 0) || + (socket_errno != SOCKET_EADDRINUSE) || + (waited >= mysqld_port_timeout)) + break; + sql_print_information("Retrying bind on TCP/IP port %u", port); + this_wait= retry * retry / 3 + 1; + sleep(this_wait); + } + if (ret < 0) + { + DBUG_PRINT("error",("Got error: %d from bind",socket_errno)); + sql_perror("Can't start server: Bind on TCP/IP port"); + sql_print_error("Do you already have another mysqld server running on " + "port: %u ?", port); + unireg_abort(1); + } + if (listen(ip_sock,(int) back_log) < 0) + { + sql_perror("Can't start server: listen() on TCP/IP port"); + sql_print_error("listen() on TCP/IP failed with error %d", + socket_errno); + unireg_abort(1); + } + DBUG_RETURN(ip_sock); +} + + +static void network_init(void) +{ +#ifdef HAVE_SYS_UN_H + struct sockaddr_un UNIXaddr; +#endif + int arg=1; DBUG_ENTER("network_init"); LINT_INIT(ret); @@ -1611,61 +1686,12 @@ static void network_init(void) set_ports(); - if (mysqld_port != 0 && !opt_disable_networking && !opt_bootstrap) + if (!opt_disable_networking && !opt_bootstrap) { - DBUG_PRINT("general",("IP Socket is %d",mysqld_port)); - ip_sock = socket(AF_INET, SOCK_STREAM, 0); - if (ip_sock == INVALID_SOCKET) - { - DBUG_PRINT("error",("Got error: %d from socket()",socket_errno)); - sql_perror(ER(ER_IPSOCK_ERROR)); /* purecov: tested */ - unireg_abort(1); /* purecov: tested */ - } - bzero((char*) &IPaddr, sizeof(IPaddr)); - IPaddr.sin_family = AF_INET; - IPaddr.sin_addr.s_addr = my_bind_addr; - IPaddr.sin_port = (unsigned short) htons((unsigned short) mysqld_port); - -#ifndef __WIN__ - /* - We should not use SO_REUSEADDR on windows as this would enable a - user to open two mysqld servers with the same TCP/IP port. - */ - (void) setsockopt(ip_sock,SOL_SOCKET,SO_REUSEADDR,(char*)&arg,sizeof(arg)); -#endif /* __WIN__ */ - /* - Sometimes the port is not released fast enough when stopping and - restarting the server. This happens quite often with the test suite - on busy Linux systems. Retry to bind the address at these intervals: - Sleep intervals: 1, 2, 4, 6, 9, 13, 17, 22, ... - Retry at second: 1, 3, 7, 13, 22, 35, 52, 74, ... - Limit the sequence by mysqld_port_timeout (set --port-open-timeout=#). - */ - for (waited= 0, retry= 1; ; retry++, waited+= this_wait) - { - if (((ret= bind(ip_sock, my_reinterpret_cast(struct sockaddr *) (&IPaddr), - sizeof(IPaddr))) >= 0) || - (socket_errno != SOCKET_EADDRINUSE) || - (waited >= mysqld_port_timeout)) - break; - sql_print_information("Retrying bind on TCP/IP port %u", mysqld_port); - this_wait= retry * retry / 3 + 1; - sleep(this_wait); - } - if (ret < 0) - { - DBUG_PRINT("error",("Got error: %d from bind",socket_errno)); - sql_perror("Can't start server: Bind on TCP/IP port"); - sql_print_error("Do you already have another mysqld server running on port: %d ?",mysqld_port); - unireg_abort(1); - } - if (listen(ip_sock,(int) back_log) < 0) - { - sql_perror("Can't start server: listen() on TCP/IP port"); - sql_print_error("listen() on TCP/IP failed with error %d", - socket_errno); - unireg_abort(1); - } + if (mysqld_port) + base_ip_sock= activate_tcp_port(mysqld_port); + if (mysqld_extra_port) + extra_ip_sock= activate_tcp_port(mysqld_extra_port); } #ifdef __NT__ @@ -1829,7 +1855,7 @@ void unlink_thd(THD *thd) thd->cleanup(); pthread_mutex_lock(&LOCK_connection_count); - --connection_count; + (*thd->scheduler->connection_count)--; pthread_mutex_unlock(&LOCK_connection_count); (void) pthread_mutex_lock(&LOCK_thread_count); @@ -2438,15 +2464,17 @@ and this may fail.\n\n"); (ulong) dflt_key_cache->key_cache_mem_size); fprintf(stderr, "read_buffer_size=%ld\n", (long) global_system_variables.read_buff_size); fprintf(stderr, "max_used_connections=%lu\n", max_used_connections); - fprintf(stderr, "max_threads=%u\n", thread_scheduler.max_threads); + fprintf(stderr, "max_threads=%u\n", thread_scheduler.max_threads + + (uint) extra_max_connections); fprintf(stderr, "threads_connected=%u\n", thread_count); fprintf(stderr, "It is possible that mysqld could use up to \n\ key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = %lu K\n\ bytes of memory\n", ((ulong) dflt_key_cache->key_cache_mem_size + (global_system_variables.read_buff_size + global_system_variables.sortbuff_size) * - thread_scheduler.max_threads + - max_connections * sizeof(THD)) / 1024); + (thread_scheduler.max_threads + extra_max_connections) + + (max_connections + extra_max_connections)* sizeof(THD)) + / 1024); fprintf(stderr, "Hope that's ok; if not, decrease some variables in the equation.\n\n"); #if defined(HAVE_LINUXTHREADS) @@ -2692,7 +2720,7 @@ pthread_handler_t signal_hand(void *arg This should actually be '+ max_number_of_slaves' instead of +10, but the +10 should be quite safe. */ - init_thr_alarm(thread_scheduler.max_threads + + init_thr_alarm(thread_scheduler.max_threads + extra_max_connections + global_system_variables.max_insert_delayed_threads + 10); if (test_flags & TEST_SIGINT) { @@ -3005,10 +3033,8 @@ sizeof(load_default_groups)/sizeof(load_ The default value is taken from either opt_date_time_formats[] or the ISO format (ANSI SQL) - @retval - 0 ok - @retval - 1 error + @retval 0 ok + @retval 1 error */ static bool init_global_datetime_format(timestamp_type format_type, @@ -3303,7 +3329,8 @@ static int init_common_variables(const c uint files, wanted_files, max_open_files; /* MyISAM requires two file handles per table. */ - wanted_files= 10+max_connections+table_cache_size*2; + wanted_files= (10 + max_connections + extra_max_connections + + table_cache_size*2); /* We are trying to allocate no less than max_connections*5 file handles (i.e. we are trying to set the limit so that they will @@ -3314,7 +3341,8 @@ static int init_common_variables(const c can't get max_connections*5 but still got no less than was requested (value of wanted_files). */ - max_open_files= max(max(wanted_files, max_connections*5), + max_open_files= max(max(wanted_files, + (max_connections + extra_max_connections)*5), open_files_limit); files= my_set_max_open_files(max_open_files); @@ -4585,10 +4613,8 @@ static char *add_quoted_string(char *to, @param file_path Path to this program @param startup_option Startup option to mysqld - @retval - 0 option handled - @retval - 1 Could not handle option + @retval 0 option handled + @retval 1 Could not handle option */ static bool @@ -4851,7 +4877,7 @@ void create_thread_to_handle_connection( (void) pthread_mutex_unlock(&LOCK_thread_count); pthread_mutex_lock(&LOCK_connection_count); - --connection_count; + (*thd->scheduler->connection_count)--; pthread_mutex_unlock(&LOCK_connection_count); statistic_increment(aborted_connects,&LOCK_status); @@ -4900,7 +4926,8 @@ static void create_new_thread(THD *thd) pthread_mutex_lock(&LOCK_connection_count); - if (connection_count >= max_connections + 1 || abort_loop) + if (*thd->scheduler->connection_count >= + *thd->scheduler->max_connections + 1|| abort_loop) { pthread_mutex_unlock(&LOCK_connection_count); @@ -4910,10 +4937,10 @@ static void create_new_thread(THD *thd) DBUG_VOID_RETURN; } - ++connection_count; + ++*thd->scheduler->connection_count; - if (connection_count > max_used_connections) - max_used_connections= connection_count; + if (connection_count + extra_connection_count > max_used_connections) + max_used_connections= connection_count + extra_connection_count; pthread_mutex_unlock(&LOCK_connection_count); @@ -4930,7 +4957,7 @@ static void create_new_thread(THD *thd) thread_count++; - thread_scheduler.add_connection(thd); + thd->scheduler->add_connection(thd); DBUG_VOID_RETURN; } @@ -4945,7 +4972,8 @@ inline void kill_broken_server() #if !defined(__NETWARE__) unix_sock == INVALID_SOCKET || #endif - (!opt_disable_networking && ip_sock == INVALID_SOCKET)) + (!opt_disable_networking && + (base_ip_sock == INVALID_SOCKET || extra_ip_sock != INVALID_SOCKET))) { select_thread_in_use = 0; /* The following call will never return */ @@ -4964,26 +4992,38 @@ pthread_handler_t handle_connections_soc { my_socket sock,new_sock; uint error_count=0; - uint max_used_connection= (uint) (max(ip_sock,unix_sock)+1); + uint max_used_connection; fd_set readFDs,clientFDs; THD *thd; struct sockaddr_in cAddr; - int ip_flags=0,socket_flags=0,flags; + int base_ip_flags=0, extra_ip_flags= 0, socket_flags=0, flags; st_vio *vio_tmp; DBUG_ENTER("handle_connections_sockets"); + max_used_connection= (uint) (max(base_ip_sock, unix_sock)); + max_used_connection= (uint) (max(extra_ip_sock, (int) max_used_connection)); + max_used_connection++; + LINT_INIT(new_sock); (void) my_pthread_getprio(pthread_self()); // For debugging FD_ZERO(&clientFDs); - if (ip_sock != INVALID_SOCKET) + if (base_ip_sock != INVALID_SOCKET) + { + FD_SET(base_ip_sock, &clientFDs); +#ifdef HAVE_FCNTL + base_ip_flags = fcntl(base_ip_sock, F_GETFL, 0); +#endif + } + if (extra_ip_sock != INVALID_SOCKET) { - FD_SET(ip_sock,&clientFDs); + FD_SET(extra_ip_sock, &clientFDs); #ifdef HAVE_FCNTL - ip_flags = fcntl(ip_sock, F_GETFL, 0); + extra_ip_flags = fcntl(extra_ip_sock, F_GETFL, 0); #endif } + #ifdef HAVE_SYS_UN_H FD_SET(unix_sock,&clientFDs); #ifdef HAVE_FCNTL @@ -5021,14 +5061,22 @@ pthread_handler_t handle_connections_soc #ifdef HAVE_SYS_UN_H if (FD_ISSET(unix_sock,&readFDs)) { - sock = unix_sock; + sock= unix_sock; flags= socket_flags; } else #endif { - sock = ip_sock; - flags= ip_flags; + if (FD_ISSET(base_ip_sock,&readFDs)) + { + sock= base_ip_sock; + flags= base_ip_flags; + } + else + { + sock= extra_ip_sock; + flags= extra_ip_flags; + } } #if !defined(NO_FCNTL_NONBLOCK) @@ -5081,7 +5129,7 @@ pthread_handler_t handle_connections_soc #ifdef HAVE_LIBWRAP { - if (sock == ip_sock) + if (sock == base_ip_sock || sock == extra_ip_sock) { struct request_info req; signal(SIGCHLD, SIG_DFL); @@ -5161,6 +5209,11 @@ pthread_handler_t handle_connections_soc if (sock == unix_sock) thd->security_ctx->host=(char*) my_localhost; + if (sock == extra_ip_sock) + { + thd->extra_port= 1; + thd->scheduler= &extra_thread_scheduler; + } create_new_thread(thd); } @@ -5502,6 +5555,7 @@ enum options_mysqld OPT_SKIP_GRANT, OPT_SKIP_LOCK, OPT_ENABLE_LOCK, OPT_USE_LOCKING, OPT_SOCKET, OPT_UPDATE_LOG, + OPT_EXTRA_PORT, OPT_BIN_LOG, OPT_SKIP_RESOLVE, OPT_SKIP_NETWORKING, OPT_BIN_LOG_INDEX, OPT_BIND_ADDRESS, OPT_PID_FILE, @@ -5660,6 +5714,7 @@ enum options_mysqld OPT_MIN_EXAMINED_ROW_LIMIT, OPT_LOG_SLOW_SLAVE_STATEMENTS, OPT_DEBUG_CRC, OPT_DEBUG_ON, OPT_OLD_MODE, + OPT_TEST_IGNORE_WRONG_OPTIONS, OPT_SLAVE_EXEC_MODE, OPT_DEADLOCK_SEARCH_DEPTH_SHORT, OPT_DEADLOCK_SEARCH_DEPTH_LONG, @@ -5884,6 +5939,15 @@ struct my_option my_long_options[] = GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, /* We must always support the next option to make scripts like mysqltest easier to do */ + {"extra-port", OPT_EXTRA_PORT, + "Extra port number to use for tcp-connections in a one-thread-per-connection manner. 0 means don't use another port", + (uchar**) &mysqld_extra_port, + (uchar**) &mysqld_extra_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"extra-max-connections", OPT_MAX_CONNECTIONS, + "The number of connections on 'extra-port.", + (uchar**) &extra_max_connections, + (uchar**) &extra_max_connections, 0, GET_ULONG, REQUIRED_ARG, 1, 1, 100000, + 0, 1, 0}, {"gdb", OPT_DEBUGGING, "Set up signals usable for debugging", (uchar**) &opt_debugging, (uchar**) &opt_debugging, @@ -6471,6 +6535,10 @@ log and this option does nothing anymore (uchar**) &use_temp_pool, (uchar**) &use_temp_pool, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"test-ignore-wrong-options", OPT_TEST_IGNORE_WRONG_OPTIONS, + "Ignore wrong enums values in command line arguments. Usefull only for test scripts", + (uchar**) &opt_ignore_wrong_options, (uchar**) &opt_ignore_wrong_options, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"timed_mutexes", OPT_TIMED_MUTEXES, "Specify whether to time mutexes (only InnoDB mutexes are currently supported)", (uchar**) &timed_mutexes, (uchar**) &timed_mutexes, 0, GET_BOOL, NO_ARG, 0, @@ -7648,7 +7716,7 @@ static int mysql_init_variables(void) slave_exec_mode_options= (uint) find_bit_type_or_exit(slave_exec_mode_str, &slave_exec_mode_typelib, NULL); opt_specialflag= SPECIAL_ENGLISH; - unix_sock= ip_sock= INVALID_SOCKET; + unix_sock= base_ip_sock= extra_ip_sock= INVALID_SOCKET; mysql_home_ptr= mysql_home; pidfile_name_ptr= pidfile_name; log_error_file_ptr= log_error_file; @@ -7820,6 +7888,38 @@ static int mysql_init_variables(void) } +/** + Find type for option + + If opt_ignore_wrong_options is set ignore wrong values + otherwise exit + + @return + @retval 0 ok ; *result is updated + @retval 1 error ; *result is not touched +*/ + +static my_bool find_opt_type(const char *x, TYPELIB *typelib, + const char *option, int *result) +{ + int res; + + if (opt_ignore_wrong_options) + { + if ((res= find_type_with_warning(x, typelib, option)) <= 0) + return 1; + } + else + res= find_type_or_exit(x, typelib, option); + *result= res; + return 0; +} + + +/** + Get next option from the command line +*/ + my_bool mysqld_get_one_option(int optid, const struct my_option *opt __attribute__((unused)), @@ -7923,9 +8023,11 @@ mysqld_get_one_option(int optid, case (int) OPT_INIT_RPL_ROLE: { int role; - role= find_type_or_exit(argument, &rpl_role_typelib, opt->name); - rpl_status = (role == 1) ? RPL_AUTH_MASTER : RPL_IDLE_SLAVE; - break; + if (!find_opt_type(argument, &rpl_role_typelib, opt->name, &role)) + { + rpl_status = (role == 1) ? RPL_AUTH_MASTER : RPL_IDLE_SLAVE; + break; + } } case (int)OPT_REPLICATE_IGNORE_DB: { @@ -7979,8 +8081,10 @@ mysqld_get_one_option(int optid, case OPT_BINLOG_FORMAT: { int id; - id= find_type_or_exit(argument, &binlog_format_typelib, opt->name); - global_system_variables.binlog_format= opt_binlog_format_id= id - 1; + if (!find_opt_type(argument, &binlog_format_typelib, opt->name, &id)) + { + global_system_variables.binlog_format= opt_binlog_format_id= id - 1; + } break; } case (int)OPT_BINLOG_DO_DB: @@ -8093,7 +8197,7 @@ mysqld_get_one_option(int optid, exit(1); #endif opt_disable_networking=1; - mysqld_port=0; + mysqld_port= mysqld_extra_port= 0; break; case (int) OPT_SKIP_SHOW_DB: opt_skip_show_db=1; @@ -8192,8 +8296,8 @@ mysqld_get_one_option(int optid, else { int type; - type= find_type_or_exit(argument, &delay_key_write_typelib, opt->name); - delay_key_write_options= (uint) type-1; + if (!find_opt_type(argument, &delay_key_write_typelib, opt->name, &type)) + delay_key_write_options= (uint) type-1; } break; case OPT_CHARSETS_DIR: @@ -8203,8 +8307,8 @@ mysqld_get_one_option(int optid, case OPT_TX_ISOLATION: { int type; - type= find_type_or_exit(argument, &tx_isolation_typelib, opt->name); - global_system_variables.tx_isolation= (type-1); + if (!find_opt_type(argument, &tx_isolation_typelib, opt->name, &type)) + global_system_variables.tx_isolation= (type-1); break; } #ifdef WITH_NDBCLUSTER_STORAGE_ENGINE @@ -8233,8 +8337,8 @@ mysqld_get_one_option(int optid, break; case OPT_NDB_DISTRIBUTION: int id; - id= find_type_or_exit(argument, &ndb_distribution_typelib, opt->name); - opt_ndb_distribution_id= (enum ndb_distribution)(id-1); + if (!find_opt_type(argument, &ndb_distribution_typelib, opt->name, &id)) + opt_ndb_distribution_id= (enum ndb_distribution)(id-1); break; case OPT_NDB_EXTRA_LOGGING: if (!argument) @@ -8274,9 +8378,8 @@ mysqld_get_one_option(int optid, myisam_concurrent_insert= 0; /* --skip-concurrent-insert */ break; case OPT_TC_HEURISTIC_RECOVER: - tc_heuristic_recover= find_type_or_exit(argument, - &tc_heuristic_recover_typelib, - opt->name); + find_opt_type(argument, &tc_heuristic_recover_typelib, + opt->name, (int*) &tc_heuristic_recover); break; case OPT_MYISAM_STATS_METHOD: { @@ -8285,21 +8388,23 @@ mysqld_get_one_option(int optid, LINT_INIT(method_conv); myisam_stats_method_str= argument; - method= find_type_or_exit(argument, &myisam_stats_method_typelib, - opt->name); - switch (method-1) { - case 2: - method_conv= MI_STATS_METHOD_IGNORE_NULLS; - break; - case 1: - method_conv= MI_STATS_METHOD_NULLS_EQUAL; - break; - case 0: - default: - method_conv= MI_STATS_METHOD_NULLS_NOT_EQUAL; - break; + if (!find_opt_type(argument, &myisam_stats_method_typelib, + opt->name, &method)) + { + switch (method-1) { + case 2: + method_conv= MI_STATS_METHOD_IGNORE_NULLS; + break; + case 1: + method_conv= MI_STATS_METHOD_NULLS_EQUAL; + break; + case 0: + default: + method_conv= MI_STATS_METHOD_NULLS_NOT_EQUAL; + break; + } + global_system_variables.myisam_stats_method= method_conv; } - global_system_variables.myisam_stats_method= method_conv; break; } case OPT_SQL_MODE: @@ -8317,8 +8422,9 @@ mysqld_get_one_option(int optid, break; case OPT_THREAD_HANDLING: { - global_system_variables.thread_handling= - find_type_or_exit(argument, &thread_handling_typelib, opt->name)-1; + int id; + if (!find_opt_type(argument, &thread_handling_typelib, opt->name, &id)) + global_system_variables.thread_handling= id - 1; break; } case OPT_FT_BOOLEAN_SYNTAX: @@ -8338,6 +8444,10 @@ mysqld_get_one_option(int optid, lower_case_table_names= argument ? atoi(argument) : 1; lower_case_table_names_used= 1; break; + case OPT_TEST_IGNORE_WRONG_OPTIONS: + /* Used for testing options */ + opt_ignore_wrong_options= 1; + break; } return 0; } @@ -8483,14 +8593,19 @@ static void get_options(int *argc,char * #ifdef EMBEDDED_LIBRARY one_thread_scheduler(&thread_scheduler); + one_thread_scheduler(&extra_thread_scheduler); #else if (global_system_variables.thread_handling <= SCHEDULER_ONE_THREAD_PER_CONNECTION) - one_thread_per_connection_scheduler(&thread_scheduler); + one_thread_per_connection_scheduler(&thread_scheduler, &max_connections, + &connection_count); else if (global_system_variables.thread_handling == SCHEDULER_NO_THREADS) one_thread_scheduler(&thread_scheduler); else pool_of_threads_scheduler(&thread_scheduler); /* purecov: tested */ + one_thread_per_connection_scheduler(&extra_thread_scheduler, + &extra_max_connections, + &extra_connection_count); #endif } @@ -8715,12 +8830,9 @@ skip: ; @param dir_name Directory to test - @retval - -1 Don't know (Test failed) - @retval - 0 File system is case sensitive - @retval - 1 File system is case insensitive + @retval -1 Don't know (Test failed) + @retval 0 File system is case sensitive + @retval 1 File system is case insensitive */ static int test_if_case_insensitive(const char *dir_name) === modified file 'sql/scheduler.cc' --- a/sql/scheduler.cc 2007-02-23 11:13:55 +0000 +++ b/sql/scheduler.cc 2009-03-12 22:27:35 +0000 @@ -22,6 +22,9 @@ #endif #include <mysql_priv.h> +#if MYSQL_VERSION_ID >= 60000 +#include "sql_audit.h" +#endif /* 'Dummy' functions to be used when we don't need any handling for a scheduler @@ -29,7 +32,7 @@ */ static bool init_dummy(void) {return 0;} -static void post_kill_dummy(THD* thd) {} +static void post_kill_dummy(THD *thd) {} static void end_dummy(void) {} static bool end_thread_dummy(THD *thd, bool cache_thread) { return 0; } @@ -63,9 +66,12 @@ static bool no_threads_end(THD *thd, boo Initailize scheduler for --thread-handling=no-threads */ -void one_thread_scheduler(scheduler_functions* func) +void one_thread_scheduler(scheduler_functions *func) { func->max_threads= 1; + max_connections= 1; + func->max_connections= &max_connections; + func->connection_count= &connection_count; #ifndef EMBEDDED_LIBRARY func->add_connection= handle_connection_in_main_thread; #endif @@ -79,10 +85,674 @@ void one_thread_scheduler(scheduler_func */ #ifndef EMBEDDED_LIBRARY -void one_thread_per_connection_scheduler(scheduler_functions* func) +void one_thread_per_connection_scheduler(scheduler_functions *func, + ulong *arg_max_connections, + uint *arg_connection_count) { - func->max_threads= max_connections; + func->max_threads= *arg_max_connections + 1; + func->max_connections= arg_max_connections; + func->connection_count= arg_connection_count; func->add_connection= create_thread_to_handle_connection; func->end_thread= one_thread_per_connection_end; } #endif /* EMBEDDED_LIBRARY */ + + +#if defined(HAVE_LIBEVENT) && HAVE_POOL_OF_THREADS == 1 + +#include "event.h" + +static struct event_base *base; + +static uint created_threads, killed_threads; +static bool kill_pool_threads; + +static struct event thd_add_event; +static struct event thd_kill_event; + +static pthread_mutex_t LOCK_thd_add; /* protects thds_need_adding */ +static LIST *thds_need_adding; /* list of thds to add to libevent queue */ + +static int thd_add_pair[2]; /* pipe to signal add a connection to libevent*/ +static int thd_kill_pair[2]; /* pipe to signal kill a connection in libevent */ + +/* + LOCK_event_loop protects the non-thread safe libevent calls (event_add and + event_del) and thds_need_processing and thds_waiting_for_io. +*/ +static pthread_mutex_t LOCK_event_loop; +static LIST *thds_need_processing; /* list of thds that needs some processing */ +static LIST *thds_waiting_for_io; /* list of thds with added events */ + +pthread_handler_t libevent_thread_proc(void *arg); +static void libevent_end(); +static bool libevent_needs_immediate_processing(THD *thd); +static void libevent_connection_close(THD *thd); +static bool libevent_should_close_connection(THD* thd); +static void libevent_thd_add(THD* thd); +void libevent_io_callback(int Fd, short Operation, void *ctx); +void libevent_add_thd_callback(int Fd, short Operation, void *ctx); +void libevent_kill_thd_callback(int Fd, short Operation, void *ctx); + + +/* + Create a pipe and set to non-blocking. + Returns TRUE if there is an error. +*/ + +static bool init_socketpair(int sock_pair[]) +{ + sock_pair[0]= sock_pair[1]= -1; + return (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, sock_pair) < 0 || + evutil_make_socket_nonblocking(sock_pair[0]) == -1 || + evutil_make_socket_nonblocking(sock_pair[1]) == -1); +} + +static void close_socketpair(int sock_pair[]) +{ + if (sock_pair[0] != -1) + EVUTIL_CLOSESOCKET(sock_pair[0]); + if (sock_pair[1] != -1) + EVUTIL_CLOSESOCKET(sock_pair[1]); +} + +/* + thd_scheduler keeps the link between THD and events. + It's embedded in the THD class. +*/ + +thd_scheduler::thd_scheduler() + : logged_in(FALSE), io_event(NULL), thread_attached(FALSE) +{ +#ifndef DBUG_OFF + dbug_explain[0]= '\0'; + set_explain= FALSE; +#endif +} + + +thd_scheduler::~thd_scheduler() +{ + my_free(io_event, MYF(MY_ALLOW_ZERO_PTR)); +} + + +bool thd_scheduler::init(THD *parent_thd) +{ + io_event= + (struct event*)my_malloc(sizeof(*io_event),MYF(MY_ZEROFILL|MY_WME)); + + if (!io_event) + { + sql_print_error("Memory allocation error in thd_scheduler::init\n"); + return TRUE; + } + + event_set(io_event, (int)parent_thd->net.vio->sd, EV_READ, + libevent_io_callback, (void*)parent_thd); + + list.data= parent_thd; + + return FALSE; +} + + +/* + Attach/associate the connection with the OS thread, for command processing. +*/ + +bool thd_scheduler::thread_attach() +{ + DBUG_ASSERT(!thread_attached); + THD* thd = (THD*)list.data; + if (libevent_should_close_connection(thd) || + setup_connection_thread_globals(thd)) + { + return TRUE; + } + my_errno= 0; + thd->mysys_var->abort= 0; + thread_attached= TRUE; +#ifndef DBUG_OFF + /* + When we attach the thread for a connection for the first time, + we know that there is no session value set yet. Thus + the initial setting of set_explain to FALSE is OK. + */ + if (set_explain) + DBUG_SET(dbug_explain); +#endif + return FALSE; +} + + +/* + Detach/disassociate the connection with the OS thread. +*/ + +void thd_scheduler::thread_detach() +{ + if (thread_attached) + { + THD* thd = (THD*)list.data; + pthread_mutex_lock(&thd->LOCK_delete); + thd->mysys_var= NULL; + pthread_mutex_unlock(&thd->LOCK_delete); + thread_attached= FALSE; +#ifndef DBUG_OFF + /* + If during the session @@session.dbug was assigned, the + dbug options/state has been pushed. Check if this is the + case, to be able to restore the state when we attach this + logical connection to a physical thread. + */ + if (_db_is_pushed_()) + { + set_explain= TRUE; + if (DBUG_EXPLAIN(dbug_explain, sizeof(dbug_explain))) + sql_print_error("thd_scheduler: DBUG_EXPLAIN buffer is too small"); + } + /* DBUG_POP() is a no-op in case there is no session state */ + DBUG_POP(); +#endif + } +} + +/** + Create all threads for the thread pool + + NOTES + After threads are created we wait until all threads has signaled that + they have started before we return + + RETURN + 0 ok + 1 We got an error creating the thread pool + In this case we will abort all created threads +*/ + +static bool libevent_init(void) +{ + uint i; + DBUG_ENTER("libevent_init"); + + base= (struct event_base *) event_init(); + + created_threads= 0; + killed_threads= 0; + kill_pool_threads= FALSE; + + pthread_mutex_init(&LOCK_event_loop, NULL); + pthread_mutex_init(&LOCK_thd_add, NULL); + + /* set up sockets used to add new thds to the event pool */ + if (init_socketpair(thd_add_pair)) + { + sql_print_error("init_socketpair(thd_add_spair) error in libevent_init"); + DBUG_RETURN(1); + } + /* set up sockets used to kill thds in the event queue */ + if (init_socketpair(thd_kill_pair)) + { + sql_print_error("init_socketpair(thd_kill_pair) error in libevent_init"); + close_socketpair(thd_add_pair); + DBUG_RETURN(1); + } + event_set(&thd_add_event, thd_add_pair[0], EV_READ|EV_PERSIST, + libevent_add_thd_callback, NULL); + event_set(&thd_kill_event, thd_kill_pair[0], EV_READ|EV_PERSIST, + libevent_kill_thd_callback, NULL); + + if (event_add(&thd_add_event, NULL) || event_add(&thd_kill_event, NULL)) + { + sql_print_error("thd_add_event event_add error in libevent_init"); + libevent_end(); + DBUG_RETURN(1); + } + /* Set up the thread pool */ + created_threads= killed_threads= 0; + pthread_mutex_lock(&LOCK_thread_count); + + for (i= 0; i < thread_pool_size; i++) + { + pthread_t thread; + int error; + if ((error= pthread_create(&thread, &connection_attrib, + libevent_thread_proc, 0))) + { + sql_print_error("Can't create completion port thread (error %d)", + error); + pthread_mutex_unlock(&LOCK_thread_count); + libevent_end(); // Cleanup + DBUG_RETURN(TRUE); + } + } + + /* Wait until all threads are created */ + while (created_threads != thread_pool_size) + pthread_cond_wait(&COND_thread_count,&LOCK_thread_count); + pthread_mutex_unlock(&LOCK_thread_count); + + DBUG_PRINT("info", ("%u threads created", (uint) thread_pool_size)); + DBUG_RETURN(FALSE); +} + + +/* + This is called when data is ready on the socket. + + NOTES + This is only called by the thread that owns LOCK_event_loop. + + We add the thd that got the data to thds_need_processing, and + cause the libevent event_loop() to terminate. Then this same thread will + return from event_loop and pick the thd value back up for processing. +*/ + +void libevent_io_callback(int, short, void *ctx) +{ + safe_mutex_assert_owner(&LOCK_event_loop); + THD *thd= (THD*)ctx; + thds_waiting_for_io= list_delete(thds_waiting_for_io, &thd->event_scheduler.list); + thds_need_processing= list_add(thds_need_processing, &thd->event_scheduler.list); +} + +/* + This is called when we have a thread we want to be killed. + + NOTES + This is only called by the thread that owns LOCK_event_loop. +*/ + +void libevent_kill_thd_callback(int Fd, short, void*) +{ + safe_mutex_assert_owner(&LOCK_event_loop); + + /* clear the pending events */ + char c; + while (recv(Fd, &c, sizeof(c), 0) == sizeof(c)) + {} + + LIST* list= thds_waiting_for_io; + while (list) + { + THD *thd= (THD*)list->data; + list= list_rest(list); + if (thd->killed == THD::KILL_CONNECTION) + { + /* + Delete from libevent and add to the processing queue. + */ + event_del(thd->event_scheduler.io_event); + thds_waiting_for_io= list_delete(thds_waiting_for_io, + &thd->event_scheduler.list); + thds_need_processing= list_add(thds_need_processing, + &thd->event_scheduler.list); + } + } +} + + +/* + This is used to add connections to the pool. This callback is invoked from + the libevent event_loop() call whenever the thd_add_pair[1] has a byte + written to it. + + NOTES + This is only called by the thread that owns LOCK_event_loop. +*/ + +void libevent_add_thd_callback(int Fd, short, void *) +{ + safe_mutex_assert_owner(&LOCK_event_loop); + + /* clear the pending events */ + char c; + while (recv(Fd, &c, sizeof(c), 0) == sizeof(c)) + {} + + pthread_mutex_lock(&LOCK_thd_add); + while (thds_need_adding) + { + /* pop the first thd off the list */ + THD* thd= (THD*)thds_need_adding->data; + thds_need_adding= list_delete(thds_need_adding, thds_need_adding); + + pthread_mutex_unlock(&LOCK_thd_add); + + if (!thd->event_scheduler.logged_in || libevent_should_close_connection(thd)) + { + /* + Add thd to thds_need_processing list. If it needs closing we'll close + it outside of event_loop(). + */ + thds_need_processing= list_add(thds_need_processing, + &thd->event_scheduler.list); + } + else + { + /* Add to libevent */ + if (event_add(thd->event_scheduler.io_event, NULL)) + { + sql_print_error("event_add error in libevent_add_thd_callback"); + libevent_connection_close(thd); + } + else + { + thds_waiting_for_io= list_add(thds_waiting_for_io, + &thd->event_scheduler.list); + } + } + pthread_mutex_lock(&LOCK_thd_add); + } + pthread_mutex_unlock(&LOCK_thd_add); +} + + +/** + Notify the thread pool about a new connection + + NOTES + LOCK_thread_count is locked on entry. This function MUST unlock it! +*/ + +static void libevent_add_connection(THD *thd) +{ + DBUG_ENTER("libevent_add_connection"); + DBUG_PRINT("enter", ("thd: %p thread_id: %lu", + thd, thd->thread_id)); + + if (thd->event_scheduler.init(thd)) + { + sql_print_error("Scheduler init error in libevent_add_new_connection"); + pthread_mutex_unlock(&LOCK_thread_count); + libevent_connection_close(thd); + DBUG_VOID_RETURN; + } + threads.append(thd); + libevent_thd_add(thd); + + pthread_mutex_unlock(&LOCK_thread_count); + DBUG_VOID_RETURN; +} + + +/** + @brief Signal a waiting connection it's time to die. + + @details This function will signal libevent the THD should be killed. + Either the global LOCK_thd_count or the THD's LOCK_delete must be locked + upon entry. + + @param[in] thd The connection to kill +*/ + +static void libevent_post_kill_notification(THD *) +{ + /* + Note, we just wake up libevent with an event that a THD should be killed, + It will search its list of thds for thd->killed == KILL_CONNECTION to + find the THDs it should kill. + + So we don't actually tell it which one and we don't actually use the + THD being passed to us, but that's just a design detail that could change + later. + */ + char c= 0; + send(thd_kill_pair[1], &c, sizeof(c), 0); +} + + +/* + Close and delete a connection. +*/ + +static void libevent_connection_close(THD *thd) +{ + DBUG_ENTER("libevent_connection_close"); + DBUG_PRINT("enter", ("thd: %p", thd)); + + thd->killed= THD::KILL_CONNECTION; // Avoid error messages + + if (thd->net.vio->sd >= 0) // not already closed + { + end_connection(thd); + close_connection(thd, 0, 1); + } + thd->event_scheduler.thread_detach(); + unlink_thd(thd); /* locks LOCK_thread_count and deletes thd */ + pthread_mutex_unlock(&LOCK_thread_count); + + DBUG_VOID_RETURN; +} + + +/* + Returns true if we should close and delete a THD connection. +*/ + +static bool libevent_should_close_connection(THD* thd) +{ + return thd->net.error || + thd->net.vio == 0 || + thd->killed == THD::KILL_CONNECTION; +} + + +/* + libevent_thread_proc is the outer loop of each thread in the thread pool. + These procs only return/terminate on shutdown (kill_pool_threads == true). +*/ + +pthread_handler_t libevent_thread_proc(void *arg) +{ + if (init_new_connection_handler_thread()) + { + my_thread_global_end(); + sql_print_error("libevent_thread_proc: my_thread_init() failed"); + exit(1); + } + DBUG_ENTER("libevent_thread_proc"); + + /* + Signal libevent_init() when all threads has been created and are ready to + receive events. + */ + (void) pthread_mutex_lock(&LOCK_thread_count); + created_threads++; + thread_created++; + if (created_threads == thread_pool_size) + (void) pthread_cond_signal(&COND_thread_count); + (void) pthread_mutex_unlock(&LOCK_thread_count); + + for (;;) + { + THD *thd= NULL; + (void) pthread_mutex_lock(&LOCK_event_loop); + + /* get thd(s) to process */ + while (!thds_need_processing) + { + if (kill_pool_threads) + { + /* the flag that we should die has been set */ + (void) pthread_mutex_unlock(&LOCK_event_loop); + goto thread_exit; + } + event_loop(EVLOOP_ONCE); + } + + /* pop the first thd off the list */ + thd= (THD*)thds_need_processing->data; + thds_need_processing= list_delete(thds_need_processing, + thds_need_processing); + + (void) pthread_mutex_unlock(&LOCK_event_loop); + + /* now we process the connection (thd) */ + + /* set up the thd<->thread links. */ + thd->thread_stack= (char*) &thd; + + if (thd->event_scheduler.thread_attach()) + { + libevent_connection_close(thd); + continue; + } + + /* is the connection logged in yet? */ + if (!thd->event_scheduler.logged_in) + { + DBUG_PRINT("info", ("init new connection. sd: %d", + thd->net.vio->sd)); + lex_start(thd); + if (login_connection(thd)) + { + /* Failed to log in */ + libevent_connection_close(thd); + continue; + } + else + { + /* login successful */ +#if MYSQL_VERSION_ID >= 60000 + MYSQL_CONNECTION_START(thd->thread_id, thd->security_ctx->priv_user, + (char *) thd->security_ctx->host_or_ip); +#endif + thd->event_scheduler.logged_in= TRUE; + prepare_new_connection_state(thd); + if (!libevent_needs_immediate_processing(thd)) + continue; /* New connection is now waiting for data in libevent*/ + } + } + + do + { + /* Process a query */ + if (do_command(thd)) + { + libevent_connection_close(thd); + break; + } + } while (libevent_needs_immediate_processing(thd)); + } + +thread_exit: + DBUG_PRINT("exit", ("ending thread")); + (void) pthread_mutex_lock(&LOCK_thread_count); + killed_threads++; + pthread_cond_broadcast(&COND_thread_count); + (void) pthread_mutex_unlock(&LOCK_thread_count); + my_thread_end(); + pthread_exit(0); + DBUG_RETURN(0); /* purify: deadcode */ +} + + +/* + Returns TRUE if the connection needs immediate processing and FALSE if + instead it's queued for libevent processing or closed, +*/ + +static bool libevent_needs_immediate_processing(THD *thd) +{ + if (libevent_should_close_connection(thd)) + { + libevent_connection_close(thd); + return FALSE; + } + /* + If more data in the socket buffer, return TRUE to process another command. + + Note: we cannot add for event processing because the whole request might + already be buffered and we wouldn't receive an event. + */ + if (vio_pending(thd->net.vio) > 0) + return TRUE; + + thd->event_scheduler.thread_detach(); + libevent_thd_add(thd); + return FALSE; +} + + +/* + Adds a THD to queued for libevent processing. + + This call does not actually register the event with libevent. + Instead, it places the THD onto a queue and signals libevent by writing + a byte into thd_add_pair, which will cause our libevent_add_thd_callback to + be invoked which will find the THD on the queue and add it to libevent. +*/ + +static void libevent_thd_add(THD* thd) +{ + char c= 0; + /* release any audit resources, this thd is going to sleep */ +#if MYSQL_VERSION_ID >= 60000 + mysql_audit_release(thd); +#endif + pthread_mutex_lock(&LOCK_thd_add); + /* queue for libevent */ + thds_need_adding= list_add(thds_need_adding, &thd->event_scheduler.list); + /* notify libevent */ + send(thd_add_pair[1], &c, sizeof(c), 0); + pthread_mutex_unlock(&LOCK_thd_add); +} + + +/** + Wait until all pool threads have been deleted for clean shutdown +*/ + +static void libevent_end() +{ + DBUG_ENTER("libevent_end"); + DBUG_PRINT("enter", ("created_threads: %d killed_threads: %u", + created_threads, killed_threads)); + + /* + check if initialized. This may not be the case if get an error at + startup + */ + if (!base) + DBUG_VOID_RETURN; + + (void) pthread_mutex_lock(&LOCK_thread_count); + + + kill_pool_threads= TRUE; + while (killed_threads != created_threads) + { + /* wake up the event loop */ + char c= 0; + send(thd_add_pair[1], &c, sizeof(c), 0); + pthread_cond_wait(&COND_thread_count, &LOCK_thread_count); + } + (void) pthread_mutex_unlock(&LOCK_thread_count); + + event_del(&thd_add_event); + close_socketpair(thd_add_pair); + event_del(&thd_kill_event); + close_socketpair(thd_kill_pair); + event_base_free(base); + base= 0; + + (void) pthread_mutex_destroy(&LOCK_event_loop); + (void) pthread_mutex_destroy(&LOCK_thd_add); + DBUG_VOID_RETURN; +} + + +void pool_of_threads_scheduler(scheduler_functions* func) +{ + func->max_threads= thread_pool_size; + func->max_connections= &max_connections; + func->connection_count= &connection_count; + func->init= libevent_init; + func->end= libevent_end; + func->post_kill_notification= libevent_post_kill_notification; + func->add_connection= libevent_add_connection; +} + +#endif === modified file 'sql/scheduler.h' --- a/sql/scheduler.h 2007-11-14 21:12:46 +0000 +++ b/sql/scheduler.h 2009-03-12 22:27:35 +0000 @@ -28,7 +28,8 @@ class THD; class scheduler_functions { public: - uint max_threads; + uint max_threads, *connection_count; + ulong *max_connections; bool (*init)(void); bool (*init_new_connection_thread)(void); void (*add_connection)(THD *thd); @@ -45,16 +46,45 @@ enum scheduler_types SCHEDULER_POOL_OF_THREADS }; -void one_thread_per_connection_scheduler(scheduler_functions* func); +void one_thread_per_connection_scheduler(scheduler_functions *func, + ulong *arg_max_connections, + uint *arg_connection_count); void one_thread_scheduler(scheduler_functions* func); -enum pool_command_op +#if defined(HAVE_LIBEVENT) && !defined(EMBEDDED_LIBRARY) + +#define HAVE_POOL_OF_THREADS 1 + +struct event; + +class thd_scheduler { - NOT_IN_USE_OP= 0, NORMAL_OP= 1, CONNECT_OP, KILL_OP, DIE_OP +public: + bool logged_in; + struct event* io_event; + LIST list; + bool thread_attached; /* Indicates if THD is attached to the OS thread */ + +#ifndef DBUG_OFF + char dbug_explain[256]; + bool set_explain; +#endif + + thd_scheduler(); + ~thd_scheduler(); + bool init(THD* parent_thd); + bool thread_attach(); + void thread_detach(); }; +void pool_of_threads_scheduler(scheduler_functions* func); + +#else + #define HAVE_POOL_OF_THREADS 0 /* For easyer tests */ #define pool_of_threads_scheduler(A) one_thread_per_connection_scheduler(A) class thd_scheduler {}; + +#endif === modified file 'sql/set_var.cc' --- a/sql/set_var.cc 2009-02-19 09:01:25 +0000 +++ b/sql/set_var.cc 2009-03-12 22:27:35 +0000 @@ -265,6 +265,13 @@ static sys_var_long_ptr sys_delayed_queu static sys_var_event_scheduler sys_event_scheduler(&vars, "event_scheduler"); #endif +static sys_var_const sys_extra_port(&vars, "extra_port", + OPT_GLOBAL, SHOW_INT, + (uchar*) &mysqld_extra_port); +static sys_var_long_ptr sys_extra_max_connections(&vars, + "extra_max_connections", + &extra_max_connections, + fix_max_connections); static sys_var_long_ptr sys_expire_logs_days(&vars, "expire_logs_days", &expire_logs_days); static sys_var_bool_ptr sys_flush(&vars, "flush", &myisam_flush); @@ -1357,7 +1364,7 @@ static int check_max_delayed_threads(THD static void fix_max_connections(THD *thd, enum_var_type type) { #ifndef EMBEDDED_LIBRARY - resize_thr_alarm(max_connections + + resize_thr_alarm(max_connections + extra_max_connections + global_system_variables.max_insert_delayed_threads + 10); #endif } === modified file 'sql/sql_class.cc' --- a/sql/sql_class.cc 2009-02-19 09:01:25 +0000 +++ b/sql/sql_class.cc 2009-03-12 22:27:35 +0000 @@ -572,6 +572,8 @@ THD::THD() init_sql_alloc(&main_mem_root, ALLOC_ROOT_MIN_BLOCK_SIZE, 0); stmt_arena= this; thread_stack= 0; + scheduler= &thread_scheduler; // Will be fixed later + extra_port= 0; catalog= (char*)"std"; // the only catalog we have for now main_security_ctx.init(); security_ctx= &main_security_ctx; === modified file 'sql/sql_class.h' --- a/sql/sql_class.h 2009-02-19 09:01:25 +0000 +++ b/sql/sql_class.h 2009-03-12 22:27:35 +0000 @@ -1260,6 +1260,7 @@ public: struct st_mysql_stmt *current_stmt; #endif NET net; // client connection descriptor + scheduler_functions *scheduler; // Scheduler for this connection MEM_ROOT warn_root; // For warnings and errors Protocol *protocol; // Current protocol Protocol_text protocol_text; // Normal protocol @@ -1346,7 +1347,7 @@ public: uint32 server_id; uint32 file_id; // for LOAD DATA INFILE /* remote (peer) port */ - uint16 peer_port; + uint16 peer_port; time_t start_time, user_time; ulonglong connect_utime, thr_create_utime; // track down slow pthread_create ulonglong start_utime, utime_after_lock; @@ -1722,6 +1723,8 @@ public: bool locked, some_tables_deleted; bool last_cuted_field; bool no_errors, password; + bool extra_port; /* If extra connection */ + /** Set to TRUE if execution of the current compound statement can not continue. In particular, disables activation of @@ -2206,7 +2209,7 @@ public: *p_db_length= db_length; return FALSE; } - thd_scheduler scheduler; + thd_scheduler event_scheduler; public: /** === modified file 'sql/sql_connect.cc' --- a/sql/sql_connect.cc 2009-01-31 21:22:44 +0000 +++ b/sql/sql_connect.cc 2009-03-12 22:27:35 +0000 @@ -402,11 +402,15 @@ check_user(THD *thd, enum enum_server_co if (check_count) { - pthread_mutex_lock(&LOCK_connection_count); - bool count_ok= connection_count <= max_connections || - (thd->main_security_ctx.master_access & SUPER_ACL); - VOID(pthread_mutex_unlock(&LOCK_connection_count)); - + bool count_ok= 1; + + if (!(thd->main_security_ctx.master_access & SUPER_ACL)) + { + pthread_mutex_lock(&LOCK_connection_count); + count_ok= (*thd->scheduler->connection_count <= + *thd->scheduler->max_connections); + VOID(pthread_mutex_unlock(&LOCK_connection_count)); + } if (!count_ok) { // too many connections my_error(ER_CON_COUNT_ERROR, MYF(0)); @@ -917,7 +921,7 @@ bool setup_connection_thread_globals(THD { close_connection(thd, ER_OUT_OF_RESOURCES, 1); statistic_increment(aborted_connects,&LOCK_status); - thread_scheduler.end_thread(thd, 0); + thd->scheduler->end_thread(thd, 0); return 1; // Error } return 0; @@ -939,8 +943,7 @@ bool setup_connection_thread_globals(THD 1 error */ - -static bool login_connection(THD *thd) +bool login_connection(THD *thd) { NET *net= &thd->net; int error; @@ -978,7 +981,7 @@ static bool login_connection(THD *thd) This mainly updates status variables */ -static void end_connection(THD *thd) +void end_connection(THD *thd) { NET *net= &thd->net; plugin_thdvar_cleanup(thd); @@ -1011,7 +1014,7 @@ static void end_connection(THD *thd) Initialize THD to handle queries */ -static void prepare_new_connection_state(THD* thd) +void prepare_new_connection_state(THD* thd) { Security_context *sctx= thd->security_ctx; @@ -1081,7 +1084,7 @@ pthread_handler_t handle_one_connection( { close_connection(thd, ER_OUT_OF_RESOURCES, 1); statistic_increment(aborted_connects,&LOCK_status); - thread_scheduler.end_thread(thd,0); + thd->scheduler->end_thread(thd,0); return 0; } if (launch_time >= slow_launch_time*1000000L) @@ -1119,7 +1122,7 @@ pthread_handler_t handle_one_connection( end_thread: close_connection(thd, 0, 1); - if (thread_scheduler.end_thread(thd,1)) + if (thd->scheduler->end_thread(thd,1)) return 0; // Probably no-threads /* === modified file 'vio/viosocket.c' --- a/vio/viosocket.c 2007-08-13 13:11:25 +0000 +++ b/vio/viosocket.c 2009-03-12 22:27:35 +0000 @@ -637,3 +637,21 @@ int vio_close_shared_memory(Vio * vio) } #endif /* HAVE_SMEM */ #endif /* __WIN__ */ + + +/** + Number of bytes in the read buffer. + + @return number of bytes in the read buffer or < 0 if error. +*/ + +ssize_t vio_pending(Vio *vio) +{ + if (vio->read_pos < vio->read_end) + return vio->read_end - vio->read_pos; +#ifdef HAVE_OPENSSL + if (vio->ssl_arg) + return SSL_pending((SSL*) vio->ssl_arg); +#endif + return 0; +}

3 5

[Maria-developers] Proposed test patches
by Patrick Crews 09 Apr '09

09 Apr '09

Hello, I have attached a proposed patch file for the MariaDB test suite. These are related to pool of threads. pool_of_threads.test - Nothing major here. Added some extra comments, output to .result file (for easier tying .test to .result), and disconnects of the test connections (We have found in MySQL that while the connections will go away, there is a little bit of time where the leftover connections can cause subsequent tests to fail - they show up in a 'connection audit' and throw off counts, etc). connect.test - Added a test to ensure that we can only have max-extra-connections + 1 SUPER user. Previously, the test used only the 'root' account and did not ensure it was +1 SUPER. I created a limited-access user and tried to connect via extra-port after max-extra-connections were used. As expected, this connection is not successful, but the root account CAN connect. Just wanted this in place to ensure the proper behavior is tested and won't change (test will fail if it does). variables.test - Added some tests of the pool of threads related variables. Ensure that the read-only ones can't be changed via SQL and that the others can be. Two questions: 1) I performed a test where I tried to set the main port and extra-port to the same value. It seems extra-port was able to claim the port# first and the server died as the value was already in-use. Is this correct behavior? I know nobody *should* do this, but it raises the following extra questions: a. Should we let 'port' take precedence on claims over 'extra-port' (ie, the server starts as expected, but throws a warning about extra-port being an in-use value)? b. Should we throw a more informative error message? It is pretty easy to troubleshoot, but I just wanted to raise the question directly and not make assumptions 2) Do you guys want a test of the behavior described in #1? 3) Finally, I tried setting extra-max-connections to 0. It defaults to 1. Since the behavior of extra-port is <extra-max-connections> regular connections + 1 SUPER user, should we let 0 be an acceptable value? A super user would/should still be able to connect, but an admin might not want to allow other connections via extra-port. Thanks, Patrick

1 0

[Maria-developers] Working on microsecond patches
by Michael Widenius 09 Apr '09

09 Apr '09

Hi! Just wanted to inform that I just pushed and extended pool-of-threads handling to the active MariaDB tree at launchpad. http://askmonty.org/wiki/index.php/MariaDB_versus_MySQL#Pool_of_threads_.28… I plan to during the next few days also add the following patches to MariaDB: http://www.percona.com/mysql/5.0.77-b13/patches/microslow_innodb.patch http://www.percona.com/mysql/5.0.77-b13/patches/microsec_process.patch Regards, Monty

3 5

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (monty:2691)
by Michael Widenius 08 Apr '09

08 Apr '09

#At lp:maria based on revid:sanja@askmonty.org-20090403132308-1rukse34u7ipth73 2691 Michael Widenius 2009-04-08 Fixed wrong test for extra_ip_sock that caused mysqld to die on Mac if --extra-port was used modified: sql/mysqld.cc sql/sql_string.cc sql/strfunc.cc per-file messages: sql/mysqld.cc Fixed wrong test for extra_ip_sock that caused mysqld to die on Mac if --extra-port was used sql/sql_string.cc Optimized tests to do fewer reallocs sql/strfunc.cc Indentation fix === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-03-18 02:14:22 +0000 +++ b/sql/mysqld.cc 2009-04-08 20:08:38 +0000 @@ -4974,11 +4974,11 @@ inline void kill_broken_server() #if !defined(__NETWARE__) unix_sock == INVALID_SOCKET || #endif - (!opt_disable_networking && - (base_ip_sock == INVALID_SOCKET || extra_ip_sock != INVALID_SOCKET))) + (!opt_disable_networking && ip_sock == INVALID_SOCKET)) { select_thread_in_use = 0; /* The following call will never return */ + DBUG_PRINT("general", ("killing server because socket is closed")); kill_server(IF_NETWARE(MYSQL_KILL_SIGNAL, (void*) MYSQL_KILL_SIGNAL)); } } === modified file 'sql/sql_string.cc' --- a/sql/sql_string.cc 2008-02-29 13:56:50 +0000 +++ b/sql/sql_string.cc 2009-04-08 20:08:38 +0000 @@ -65,10 +65,10 @@ bool String::real_alloc(uint32 arg_lengt bool String::realloc(uint32 alloc_length) { - uint32 len=ALIGN_SIZE(alloc_length+1); - if (Alloced_length < len) + if (Alloced_length <= alloc_length) { char *new_ptr; + uint32 len= ALIGN_SIZE(alloc_length+1); if (alloced) { if ((new_ptr= (char*) my_realloc(Ptr,len,MYF(MY_WME)))) === modified file 'sql/strfunc.cc' --- a/sql/strfunc.cc 2007-08-13 13:11:25 +0000 +++ b/sql/strfunc.cc 2009-04-08 20:08:38 +0000 @@ -38,7 +38,8 @@ static const char field_separator=','; -ulonglong find_set(TYPELIB *lib, const char *str, uint length, CHARSET_INFO *cs, +ulonglong find_set(TYPELIB *lib, const char *str, uint length, + CHARSET_INFO *cs, char **err_pos, uint *err_len, bool *set_warning) { CHARSET_INFO *strip= cs ? cs : &my_charset_latin1;

1 0

[Maria-developers] Some notes on pool_of_threads
by Patrick Crews 08 Apr '09

08 Apr '09

Hi there, I've been playing around with the pool_of_threads addition and here is some of what I've found: All of this was on Mac OSX 10.5 1) Biggie here: Running the server with the --extra-port=NNNN option (with pool-of-threads set) causes the server to start then immediately stop. I'm still looking through logs and trace files to see if I can find anything meaningful. I am able to run without setting extra-port though. 2) We can set extra-max-connections when --extra-port isn't set. The limit is 100000, at least on my system. As I haven't been able to get the server running with extra-port yet, I haven't tested much beyond this. 3) We can also set thread-pool-size even if thread-handling isn't set to pool-of-threads. It's limit appears to be < 2560 (I was running into some mess on my system so I haven't pinpointed it yet). The biggest thing is that if thread-pool-size is set too large, we just get a start and stop (Once again, my logs are a bit of a mess due to 'play', so I'll have more info on the server behavior in a bit). 4) I found that if the number of connections is = thread-pool-size, new connections will terminate within 'connect-timeout' seconds (there was some question about whether we had to wait for the denied connection to receive a message until after a thread was freed). 5) Due to these problems (most especially #1), main.pool_of_threads.test fails on my machine. If I remove the --extra-port stuff, the test will run more than otherwise (the server dies quickly with extra-port stuff). For extra-max-connections and thread-pool-size, do we want to allow these to be set even if they aren't relevant to the way the server is currently running? Is there any specific information I can provide if anyone wants to look into why the server dies when I try to use --extra-port? Testing notes: 1) I have some changes to make to pool_of_threads.test (style, additional testing, etc). 2) We should also add some testing for extra-max-connections, thread-pool-size, etc. Possibly to variables.test, though somewhere else would work. I've played around a bit with dynamically changing the variables, etc. I just can't get MTR to run the test to really polish my proposed changes before submitting a patch. Thanks, Patrick.

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2694)
by knielsen＠knielsen-hq.org 08 Apr '09

08 Apr '09

#At lp:maria 2694 knielsen(a)knielsen-hq.org 2009-04-08 [merge] Merge mariadb-5.1 modified: storage/maria/ma_blockrec.c === modified file 'storage/maria/ma_blockrec.c' --- a/storage/maria/ma_blockrec.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_blockrec.c 2009-04-08 08:47:30 +0000 @@ -2874,7 +2874,7 @@ static my_bool write_block_record(MARIA_ if (cur_block->used & BLOCKUSED_TAIL) { DBUG_ASSERT(data_length < MAX_TAIL_SIZE(block_size)); - /* tail written to full tail page */ + /* tail written to tail page */ cur_block->used|= BLOCKUSED_USED; head_tail_block= cur_block; }

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2693)
by knielsen＠knielsen-hq.org 08 Apr '09

08 Apr '09

#At lp:maria 2693 knielsen(a)knielsen-hq.org 2009-04-08 Fix Valgrind errors seen in buildbot. Fix mysql-test-run.pl to not terminate early when warnings in error logs are detected during server shutdown. Instead, give a nice summary report at the end of the failures. Fix code to make 100% sure no failures will go undetected. Revert earlier wrong change. Fix race with port allocation semaphore file permissions. Adjust testsuite to copy with new PBXT engine now in the tree. The PBXT engine causes an extra table to appear in the INFORMATION_SCHEMA. This causes different output for a few test cases. added: mysql-test/include/have_pbxt.inc mysql-test/r/information_schema_all_engines.result mysql-test/t/information_schema_all_engines.test modified: dbug/dbug.c include/my_global.h mysql-test/extra/rpl_tests/rpl_auto_increment.test mysql-test/lib/mtr_report.pm mysql-test/lib/mtr_unique.pm mysql-test/mysql-test-run.pl mysql-test/r/information_schema.result mysql-test/r/information_schema_db.result mysql-test/r/innodb-autoinc.result mysql-test/r/mysqlshow.result mysql-test/suite/rpl/r/rpl_auto_increment.result mysql-test/t/information_schema.test mysql-test/t/information_schema_db.test mysql-test/t/innodb-autoinc.test mysql-test/t/mysqlshow.test mysql-test/valgrind.supp sql/mysqld.cc sql/set_var.cc sql/sql_select.cc sql/sql_string.h storage/archive/azio.c per-file messages: dbug/dbug.c If DbugParse() is called multiple times, the stack->keywords for the top stack frame could be overwritten without being freed, causing a memory leak reported by Valgrind. include/my_global.h Add useful macro for different values when Valgrind (HAVE_purify) and not. mysql-test/extra/rpl_tests/rpl_auto_increment.test Omit pbxt variables from show variables output. mysql-test/include/have_pbxt.inc Add facility to disable test if PBXT engine is not available. mysql-test/lib/mtr_report.pm Give a nice summary report at the end of tests of any warnings seen in logs during server shutdowns. mysql-test/lib/mtr_unique.pm Move chmod 777 to greatly reduce the risk of leaving the port semaphore file unaccessible bu other users. mysql-test/mysql-test-run.pl Don't abort in case of warnings detected, instead give a nice summary report. Fix code to make 100% sure no failures will go undetected. Revert earlier wrong change when master disconnects early. mysql-test/r/information_schema.result Omit PBXT INFORMATION_SCHEMA table from output. Move part of test to information_schema_all_engines. mysql-test/r/information_schema_all_engines.result New file for information_schema tests that depend on which engines are available. mysql-test/r/information_schema_db.result Move part of test to information_schema_all_engines. mysql-test/r/innodb-autoinc.result Omit pbxt variables from show variables output. mysql-test/r/mysqlshow.result Move part of test to information_schema_all_engines. mysql-test/suite/rpl/r/rpl_auto_increment.result Omit pbxt variables from show variables output. mysql-test/t/information_schema.test Omit PBXT INFORMATION_SCHEMA table from output. Move part of test to information_schema_all_engines. mysql-test/t/information_schema_all_engines.test New file for information_schema tests that depend on which engines are available. mysql-test/t/information_schema_db.test Move part of test to information_schema_all_engines. mysql-test/t/innodb-autoinc.test Omit pbxt variables from show variables output. mysql-test/t/mysqlshow.test Move part of test to information_schema_all_engines. mysql-test/valgrind.supp Add variant suppression (different system library versions). Add suppression for problem with inet_ntoa(). sql/mysqld.cc Fix missing DBUG_RETURN. Fix uninitialised thd->connect_utime, likely introduced by pool_of_threads. sql/set_var.cc Fix one-byte buffer overflow in several places. Fix unsafe use of String::c_ptr() of stack-allocated String buffer. sql/sql_select.cc Silence valgrind warning due to GCC bug. sql/sql_string.h Document potential problem with String::c_ptr() and String() constructor with caller-supplied buffer. storage/archive/azio.c Silence Valgrind false warning for libz. === modified file 'dbug/dbug.c' --- a/dbug/dbug.c 2009-03-22 12:16:09 +0000 +++ b/dbug/dbug.c 2009-04-08 16:55:26 +0000 @@ -506,6 +506,9 @@ int DbugParse(CODE_STATE *cs, const char rel= control[0] == '+' || control[0] == '-'; if ((!rel || (!stack->out_file && !stack->next))) { + /* If overwriting previous state, be sure to free old to avoid leak. */ + if (stack->out_file) + FreeState(cs, stack, 0); stack->flags= 0; stack->delay= 0; stack->maxdepth= 0; @@ -1648,10 +1651,12 @@ static void FreeState(CODE_STATE *cs, st FreeList(state->processes); if (!is_shared(state, p_functions)) FreeList(state->p_functions); - if (!is_shared(state, out_file)) + if (!is_shared(state, out_file) && + state->out_file != stderr && state->out_file != stdout) DBUGCloseFile(cs, state->out_file); (void) fflush(cs->stack->out_file); - if (state->prof_file) + if (state->prof_file && + state->prof_file != stderr && state->prof_file != stdout) DBUGCloseFile(cs, state->prof_file); if (free_state) free((void*) state); === modified file 'include/my_global.h' --- a/include/my_global.h 2009-02-19 09:01:25 +0000 +++ b/include/my_global.h 2009-04-08 16:55:26 +0000 @@ -569,6 +569,12 @@ int __void__; #define PURIFY_OR_LINT_INIT(var) #endif +#ifdef HAVE_purify +#define IF_PURIFY(A,B) (A) +#else +#define IF_PURIFY(A,B) (B) +#endif + #if !defined(HAVE_UINT) #undef HAVE_UINT #define HAVE_UINT === modified file 'mysql-test/extra/rpl_tests/rpl_auto_increment.test' --- a/mysql-test/extra/rpl_tests/rpl_auto_increment.test 2009-01-14 08:27:32 +0000 +++ b/mysql-test/extra/rpl_tests/rpl_auto_increment.test 2009-04-08 16:55:26 +0000 @@ -34,7 +34,7 @@ connection master; drop table t1; set @@session.auto_increment_increment=100, @@session.auto_increment_offset=10; -show variables like "%auto_inc%"; +show variables like "auto_inc%"; eval create table t1 (a int not null auto_increment, primary key (a)) engine=$engine_type2; # Insert with 2 insert statements to get better testing of logging === added file 'mysql-test/include/have_pbxt.inc' --- a/mysql-test/include/have_pbxt.inc 1970-01-01 00:00:00 +0000 +++ b/mysql-test/include/have_pbxt.inc 2009-04-08 16:55:26 +0000 @@ -0,0 +1,4 @@ +disable_query_log; +--require r/true.require +select (support = 'YES' or support = 'DEFAULT') as `TRUE` from information_schema.engines where engine = 'pbxt'; +enable_query_log; === modified file 'mysql-test/lib/mtr_report.pm' --- a/mysql-test/lib/mtr_report.pm 2009-02-19 09:01:25 +0000 +++ b/mysql-test/lib/mtr_report.pm 2009-04-08 16:55:26 +0000 @@ -187,8 +187,10 @@ sub mtr_report_test ($) { } -sub mtr_report_stats ($) { +sub mtr_report_stats ($$$) { + my $fail= shift; my $tests= shift; + my $extra_warnings= shift; # ---------------------------------------------------------------------- # Find out how we where doing @@ -325,10 +327,27 @@ sub mtr_report_stats ($) { print "All $tot_tests tests were successful.\n\n"; } + if (@$extra_warnings) + { + print <<MSG; +Errors/warnings were found in logfiles during server shutdown after running the +following sequence(s) of tests: +MSG + print " $_\n" for @$extra_warnings; + } + if ( $tot_failed != 0 || $found_problems) { mtr_error("there were failing test cases"); } + elsif (@$extra_warnings) + { + mtr_error("There were errors/warnings in server logs after running test cases."); + } + elsif ($fail) + { + mtr_error("Test suite failure, see messages above for possible cause(s)."); + } } === modified file 'mysql-test/lib/mtr_unique.pm' --- a/mysql-test/lib/mtr_unique.pm 2009-02-19 09:01:25 +0000 +++ b/mysql-test/lib/mtr_unique.pm 2009-04-08 16:55:26 +0000 @@ -62,13 +62,14 @@ sub mtr_get_unique_id($$) { die 'lock file is a symbolic link'; } - chmod 0777, "$file.sem"; open SEM, ">", "$file.sem" or die "can't write to $file.sem"; + chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; if(! -e $file) { open FILE, ">", $file or die "can't create $file"; close FILE; } + chmod 0777, $file; msg("HAVE THE LOCK"); @@ -76,7 +77,6 @@ sub mtr_get_unique_id($$) { die 'lock file is a symbolic link'; } - chmod 0777, $file; open FILE, "+<", $file or die "can't open $file"; #select undef,undef,undef,0.2; seek FILE, 0, 0; @@ -136,6 +136,7 @@ sub mtr_release_unique_id($) { } open SEM, ">", "$file.sem" or die "can't write to $file.sem"; + chmod 0777, "$file.sem"; flock SEM, LOCK_EX or die "can't lock $file.sem"; msg("HAVE THE LOCK"); @@ -148,6 +149,7 @@ sub mtr_release_unique_id($) { open FILE, ">", $file or die "can't create $file"; close FILE; } + chmod 0777, "$file.sem"; open FILE, "+<", $file or die "can't open $file"; #select undef,undef,undef,0.2; seek FILE, 0, 0; === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2009-04-03 08:09:55 +0000 +++ b/mysql-test/mysql-test-run.pl 2009-04-08 16:55:26 +0000 @@ -357,7 +357,8 @@ sub main { mtr_print_thick_line(); mtr_print_header(); - my ($completed, $fail)= run_test_server($server, $tests, $opt_parallel); + my ($fail, $completed, $extra_warnings)= + run_test_server($server, $tests, $opt_parallel); # Send Ctrl-C to any children still running kill("INT", keys(%children)); @@ -393,10 +394,6 @@ sub main { mtr_error("Not all tests completed"); } - if ($fail) { - mtr_error("Test suite failure."); - } - mtr_print_line(); if ( $opt_gcov ) { @@ -404,7 +401,7 @@ sub main { $opt_gcov_msg, $opt_gcov_err); } - mtr_report_stats($completed); + mtr_report_stats($fail, $completed, $extra_warnings); exit(0); } @@ -416,7 +413,8 @@ sub run_test_server ($$$) { my $num_saved_cores= 0; # Number of core files saved in vardir/log/ so far. my $num_saved_datadir= 0; # Number of datadirs saved in vardir/log/ so far. my $num_failed_test= 0; # Number of tests failed so far - my $test_failure= 0; + my $test_failure= 0; # Set true if test suite failed + my $extra_warnings= []; # Warnings found during server shutdowns # Scheduler variables my $max_ndb= $childs / 2; @@ -450,7 +448,7 @@ sub run_test_server ($$$) { $s->remove($sock); if (--$childs == 0){ $suite_timeout_proc->kill(); - return ($completed, $test_failure); + return ($test_failure, $completed, $extra_warnings); } next; } @@ -519,14 +517,14 @@ sub run_test_server ($$$) { # Test has failed, force is off $suite_timeout_proc->kill(); push(@$completed, $result); - return ($completed, 1); + return (1, $completed, $extra_warnings); } elsif ($opt_max_test_fail > 0 and $num_failed_test >= $opt_max_test_fail) { $suite_timeout_proc->kill(); mtr_report("Too many tests($num_failed_test) failed!", "Terminating..."); - return (undef, 1); + return (1, $completed, $extra_warnings); } $num_failed_test++; } @@ -580,13 +578,14 @@ sub run_test_server ($$$) { elsif ($line eq 'WARNINGS'){ my $fake_test= My::Test::read_test($sock); my $test_list= join (" ", @{$fake_test->{testnames}}); + push @$extra_warnings, $test_list; mtr_report("***Warnings generated in error logs during shutdown ". "after running tests: $test_list"); $test_failure= 1; if ( !$opt_force ) { # Test failure due to warnings, force is off $suite_timeout_proc->kill(); - return ($completed, 1); + return (1, $completed, $extra_warnings); } } else { mtr_error("Unknown response: '$line' from client"); @@ -666,7 +665,7 @@ sub run_test_server ($$$) { if ( ! $suite_timeout_proc->wait_one(0) ) { mtr_report("Test suite timeout! Terminating..."); - return (undef, 1); + return (1, $completed, $extra_warnings); } } } @@ -758,7 +757,9 @@ sub run_worker ($) { } } - die "Internal error: should not reach this place."; + stop_all_servers(); + + exit(1); } === modified file 'mysql-test/r/information_schema.result' --- a/mysql-test/r/information_schema.result 2009-02-15 10:58:34 +0000 +++ b/mysql-test/r/information_schema.result 2009-04-08 16:55:26 +0000 @@ -40,7 +40,8 @@ create view v1 (c) as SELECT table_name FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND -table_name<>'ndb_apply_status'; +table_name<>'ndb_apply_status' AND +NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%'); select * from v1; c CHARACTER_SETS @@ -850,10 +851,6 @@ VIEWS TABLE_NAME select delete from mysql.user where user='mysqltest_4'; delete from mysql.db where user='mysqltest_4'; flush privileges; -SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; -table_schema count(*) -information_schema 28 -mysql 22 create table t1 (i int, j int); create trigger trg1 before insert on t1 for each row begin @@ -1224,92 +1221,6 @@ f1() DROP FUNCTION f1; DROP PROCEDURE p1; DROP USER mysql_bug20230@localhost; -SELECT t.table_name, c1.column_name -FROM information_schema.tables t -INNER JOIN -information_schema.columns c1 -ON t.table_schema = c1.table_schema AND -t.table_name = c1.table_name -WHERE t.table_schema = 'information_schema' AND -c1.ordinal_position = -( SELECT COALESCE(MIN(c2.ordinal_position),1) -FROM information_schema.columns c2 -WHERE c2.table_schema = t.table_schema AND -c2.table_name = t.table_name AND -c2.column_name LIKE '%SCHEMA%' - ); -table_name column_name -CHARACTER_SETS CHARACTER_SET_NAME -COLLATIONS COLLATION_NAME -COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME -COLUMNS TABLE_SCHEMA -COLUMN_PRIVILEGES TABLE_SCHEMA -ENGINES ENGINE -EVENTS EVENT_SCHEMA -FILES TABLE_SCHEMA -GLOBAL_STATUS VARIABLE_NAME -GLOBAL_VARIABLES VARIABLE_NAME -KEY_COLUMN_USAGE CONSTRAINT_SCHEMA -PARTITIONS TABLE_SCHEMA -PLUGINS PLUGIN_NAME -PROCESSLIST ID -PROFILING QUERY_ID -REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA -ROUTINES ROUTINE_SCHEMA -SCHEMATA SCHEMA_NAME -SCHEMA_PRIVILEGES TABLE_SCHEMA -SESSION_STATUS VARIABLE_NAME -SESSION_VARIABLES VARIABLE_NAME -STATISTICS TABLE_SCHEMA -TABLES TABLE_SCHEMA -TABLE_CONSTRAINTS CONSTRAINT_SCHEMA -TABLE_PRIVILEGES TABLE_SCHEMA -TRIGGERS TRIGGER_SCHEMA -USER_PRIVILEGES GRANTEE -VIEWS TABLE_SCHEMA -SELECT t.table_name, c1.column_name -FROM information_schema.tables t -INNER JOIN -information_schema.columns c1 -ON t.table_schema = c1.table_schema AND -t.table_name = c1.table_name -WHERE t.table_schema = 'information_schema' AND -c1.ordinal_position = -( SELECT COALESCE(MIN(c2.ordinal_position),1) -FROM information_schema.columns c2 -WHERE c2.table_schema = 'information_schema' AND -c2.table_name = t.table_name AND -c2.column_name LIKE '%SCHEMA%' - ); -table_name column_name -CHARACTER_SETS CHARACTER_SET_NAME -COLLATIONS COLLATION_NAME -COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME -COLUMNS TABLE_SCHEMA -COLUMN_PRIVILEGES TABLE_SCHEMA -ENGINES ENGINE -EVENTS EVENT_SCHEMA -FILES TABLE_SCHEMA -GLOBAL_STATUS VARIABLE_NAME -GLOBAL_VARIABLES VARIABLE_NAME -KEY_COLUMN_USAGE CONSTRAINT_SCHEMA -PARTITIONS TABLE_SCHEMA -PLUGINS PLUGIN_NAME -PROCESSLIST ID -PROFILING QUERY_ID -REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA -ROUTINES ROUTINE_SCHEMA -SCHEMATA SCHEMA_NAME -SCHEMA_PRIVILEGES TABLE_SCHEMA -SESSION_STATUS VARIABLE_NAME -SESSION_VARIABLES VARIABLE_NAME -STATISTICS TABLE_SCHEMA -TABLES TABLE_SCHEMA -TABLE_CONSTRAINTS CONSTRAINT_SCHEMA -TABLE_PRIVILEGES TABLE_SCHEMA -TRIGGERS TRIGGER_SCHEMA -USER_PRIVILEGES GRANTEE -VIEWS TABLE_SCHEMA SELECT MAX(table_name) FROM information_schema.tables WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test'); MAX(table_name) VIEWS @@ -1355,55 +1266,6 @@ table_name t1 t2 drop table t1,t2; -select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= -(select cast(table_name as char) from information_schema.tables -order by table_name limit 1) limit 1; -f1 -1 -select t.table_name, group_concat(t.table_schema, '.', t.table_name), -count(*) as num1 -from information_schema.tables t -inner join information_schema.columns c1 -on t.table_schema = c1.table_schema AND t.table_name = c1.table_name -where t.table_schema = 'information_schema' and -c1.ordinal_position = -(select isnull(c2.column_type) - -isnull(group_concat(c2.table_schema, '.', c2.table_name)) + -count(*) as num -from information_schema.columns c2 where -c2.table_schema='information_schema' and -(c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') -group by c2.column_type order by num limit 1) -group by t.table_name order by num1, t.table_name; -table_name group_concat(t.table_schema, '.', t.table_name) num1 -CHARACTER_SETS information_schema.CHARACTER_SETS 1 -COLLATIONS information_schema.COLLATIONS 1 -COLLATION_CHARACTER_SET_APPLICABILITY information_schema.COLLATION_CHARACTER_SET_APPLICABILITY 1 -COLUMNS information_schema.COLUMNS 1 -COLUMN_PRIVILEGES information_schema.COLUMN_PRIVILEGES 1 -ENGINES information_schema.ENGINES 1 -EVENTS information_schema.EVENTS 1 -FILES information_schema.FILES 1 -GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 -GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 -KEY_COLUMN_USAGE information_schema.KEY_COLUMN_USAGE 1 -PARTITIONS information_schema.PARTITIONS 1 -PLUGINS information_schema.PLUGINS 1 -PROCESSLIST information_schema.PROCESSLIST 1 -PROFILING information_schema.PROFILING 1 -REFERENTIAL_CONSTRAINTS information_schema.REFERENTIAL_CONSTRAINTS 1 -ROUTINES information_schema.ROUTINES 1 -SCHEMATA information_schema.SCHEMATA 1 -SCHEMA_PRIVILEGES information_schema.SCHEMA_PRIVILEGES 1 -SESSION_STATUS information_schema.SESSION_STATUS 1 -SESSION_VARIABLES information_schema.SESSION_VARIABLES 1 -STATISTICS information_schema.STATISTICS 1 -TABLES information_schema.TABLES 1 -TABLE_CONSTRAINTS information_schema.TABLE_CONSTRAINTS 1 -TABLE_PRIVILEGES information_schema.TABLE_PRIVILEGES 1 -TRIGGERS information_schema.TRIGGERS 1 -USER_PRIVILEGES information_schema.USER_PRIVILEGES 1 -VIEWS information_schema.VIEWS 1 create table t1(f1 int); create view v1 as select f1+1 as a from t1; create table t2 (f1 int, f2 int); === added file 'mysql-test/r/information_schema_all_engines.result' --- a/mysql-test/r/information_schema_all_engines.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/information_schema_all_engines.result 2009-04-08 16:55:26 +0000 @@ -0,0 +1,248 @@ +use INFORMATION_SCHEMA; +show tables; +Tables_in_information_schema +CHARACTER_SETS +COLLATIONS +COLLATION_CHARACTER_SET_APPLICABILITY +COLUMNS +COLUMN_PRIVILEGES +ENGINES +EVENTS +FILES +GLOBAL_STATUS +GLOBAL_VARIABLES +KEY_COLUMN_USAGE +PARTITIONS +PLUGINS +PROCESSLIST +PROFILING +REFERENTIAL_CONSTRAINTS +ROUTINES +SCHEMATA +SCHEMA_PRIVILEGES +SESSION_STATUS +SESSION_VARIABLES +STATISTICS +TABLES +TABLE_CONSTRAINTS +TABLE_PRIVILEGES +TRIGGERS +USER_PRIVILEGES +VIEWS +PBXT_STATISTICS +SELECT t.table_name, c1.column_name +FROM information_schema.tables t +INNER JOIN +information_schema.columns c1 +ON t.table_schema = c1.table_schema AND +t.table_name = c1.table_name +WHERE t.table_schema = 'information_schema' AND +c1.ordinal_position = +( SELECT COALESCE(MIN(c2.ordinal_position),1) +FROM information_schema.columns c2 +WHERE c2.table_schema = t.table_schema AND +c2.table_name = t.table_name AND +c2.column_name LIKE '%SCHEMA%' + ); +table_name column_name +CHARACTER_SETS CHARACTER_SET_NAME +COLLATIONS COLLATION_NAME +COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME +COLUMNS TABLE_SCHEMA +COLUMN_PRIVILEGES TABLE_SCHEMA +ENGINES ENGINE +EVENTS EVENT_SCHEMA +FILES TABLE_SCHEMA +GLOBAL_STATUS VARIABLE_NAME +GLOBAL_VARIABLES VARIABLE_NAME +KEY_COLUMN_USAGE CONSTRAINT_SCHEMA +PARTITIONS TABLE_SCHEMA +PLUGINS PLUGIN_NAME +PROCESSLIST ID +PROFILING QUERY_ID +REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA +ROUTINES ROUTINE_SCHEMA +SCHEMATA SCHEMA_NAME +SCHEMA_PRIVILEGES TABLE_SCHEMA +SESSION_STATUS VARIABLE_NAME +SESSION_VARIABLES VARIABLE_NAME +STATISTICS TABLE_SCHEMA +TABLES TABLE_SCHEMA +TABLE_CONSTRAINTS CONSTRAINT_SCHEMA +TABLE_PRIVILEGES TABLE_SCHEMA +TRIGGERS TRIGGER_SCHEMA +USER_PRIVILEGES GRANTEE +VIEWS TABLE_SCHEMA +PBXT_STATISTICS ID +SELECT t.table_name, c1.column_name +FROM information_schema.tables t +INNER JOIN +information_schema.columns c1 +ON t.table_schema = c1.table_schema AND +t.table_name = c1.table_name +WHERE t.table_schema = 'information_schema' AND +c1.ordinal_position = +( SELECT COALESCE(MIN(c2.ordinal_position),1) +FROM information_schema.columns c2 +WHERE c2.table_schema = 'information_schema' AND +c2.table_name = t.table_name AND +c2.column_name LIKE '%SCHEMA%' + ); +table_name column_name +CHARACTER_SETS CHARACTER_SET_NAME +COLLATIONS COLLATION_NAME +COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME +COLUMNS TABLE_SCHEMA +COLUMN_PRIVILEGES TABLE_SCHEMA +ENGINES ENGINE +EVENTS EVENT_SCHEMA +FILES TABLE_SCHEMA +GLOBAL_STATUS VARIABLE_NAME +GLOBAL_VARIABLES VARIABLE_NAME +KEY_COLUMN_USAGE CONSTRAINT_SCHEMA +PARTITIONS TABLE_SCHEMA +PLUGINS PLUGIN_NAME +PROCESSLIST ID +PROFILING QUERY_ID +REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA +ROUTINES ROUTINE_SCHEMA +SCHEMATA SCHEMA_NAME +SCHEMA_PRIVILEGES TABLE_SCHEMA +SESSION_STATUS VARIABLE_NAME +SESSION_VARIABLES VARIABLE_NAME +STATISTICS TABLE_SCHEMA +TABLES TABLE_SCHEMA +TABLE_CONSTRAINTS CONSTRAINT_SCHEMA +TABLE_PRIVILEGES TABLE_SCHEMA +TRIGGERS TRIGGER_SCHEMA +USER_PRIVILEGES GRANTEE +VIEWS TABLE_SCHEMA +PBXT_STATISTICS ID +select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= +(select cast(table_name as char) from information_schema.tables +order by table_name limit 1) limit 1; +f1 +1 +select t.table_name, group_concat(t.table_schema, '.', t.table_name), +count(*) as num1 +from information_schema.tables t +inner join information_schema.columns c1 +on t.table_schema = c1.table_schema AND t.table_name = c1.table_name +where t.table_schema = 'information_schema' and +c1.ordinal_position = +(select isnull(c2.column_type) - +isnull(group_concat(c2.table_schema, '.', c2.table_name)) + +count(*) as num +from information_schema.columns c2 where +c2.table_schema='information_schema' and +(c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') +group by c2.column_type order by num limit 1) +group by t.table_name order by num1, t.table_name; +table_name group_concat(t.table_schema, '.', t.table_name) num1 +CHARACTER_SETS information_schema.CHARACTER_SETS 1 +COLLATIONS information_schema.COLLATIONS 1 +COLLATION_CHARACTER_SET_APPLICABILITY information_schema.COLLATION_CHARACTER_SET_APPLICABILITY 1 +COLUMNS information_schema.COLUMNS 1 +COLUMN_PRIVILEGES information_schema.COLUMN_PRIVILEGES 1 +ENGINES information_schema.ENGINES 1 +EVENTS information_schema.EVENTS 1 +FILES information_schema.FILES 1 +GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 +GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 +KEY_COLUMN_USAGE information_schema.KEY_COLUMN_USAGE 1 +PARTITIONS information_schema.PARTITIONS 1 +PBXT_STATISTICS information_schema.PBXT_STATISTICS 1 +PLUGINS information_schema.PLUGINS 1 +PROCESSLIST information_schema.PROCESSLIST 1 +PROFILING information_schema.PROFILING 1 +REFERENTIAL_CONSTRAINTS information_schema.REFERENTIAL_CONSTRAINTS 1 +ROUTINES information_schema.ROUTINES 1 +SCHEMATA information_schema.SCHEMATA 1 +SCHEMA_PRIVILEGES information_schema.SCHEMA_PRIVILEGES 1 +SESSION_STATUS information_schema.SESSION_STATUS 1 +SESSION_VARIABLES information_schema.SESSION_VARIABLES 1 +STATISTICS information_schema.STATISTICS 1 +TABLES information_schema.TABLES 1 +TABLE_CONSTRAINTS information_schema.TABLE_CONSTRAINTS 1 +TABLE_PRIVILEGES information_schema.TABLE_PRIVILEGES 1 +TRIGGERS information_schema.TRIGGERS 1 +USER_PRIVILEGES information_schema.USER_PRIVILEGES 1 +VIEWS information_schema.VIEWS 1 +Database: information_schema ++---------------------------------------+ +| Tables | ++---------------------------------------+ +| CHARACTER_SETS | +| COLLATIONS | +| COLLATION_CHARACTER_SET_APPLICABILITY | +| COLUMNS | +| COLUMN_PRIVILEGES | +| ENGINES | +| EVENTS | +| FILES | +| GLOBAL_STATUS | +| GLOBAL_VARIABLES | +| KEY_COLUMN_USAGE | +| PARTITIONS | +| PLUGINS | +| PROCESSLIST | +| PROFILING | +| REFERENTIAL_CONSTRAINTS | +| ROUTINES | +| SCHEMATA | +| SCHEMA_PRIVILEGES | +| SESSION_STATUS | +| SESSION_VARIABLES | +| STATISTICS | +| TABLES | +| TABLE_CONSTRAINTS | +| TABLE_PRIVILEGES | +| TRIGGERS | +| USER_PRIVILEGES | +| VIEWS | +| PBXT_STATISTICS | ++---------------------------------------+ +Database: INFORMATION_SCHEMA ++---------------------------------------+ +| Tables | ++---------------------------------------+ +| CHARACTER_SETS | +| COLLATIONS | +| COLLATION_CHARACTER_SET_APPLICABILITY | +| COLUMNS | +| COLUMN_PRIVILEGES | +| ENGINES | +| EVENTS | +| FILES | +| GLOBAL_STATUS | +| GLOBAL_VARIABLES | +| KEY_COLUMN_USAGE | +| PARTITIONS | +| PLUGINS | +| PROCESSLIST | +| PROFILING | +| REFERENTIAL_CONSTRAINTS | +| ROUTINES | +| SCHEMATA | +| SCHEMA_PRIVILEGES | +| SESSION_STATUS | +| SESSION_VARIABLES | +| STATISTICS | +| TABLES | +| TABLE_CONSTRAINTS | +| TABLE_PRIVILEGES | +| TRIGGERS | +| USER_PRIVILEGES | +| VIEWS | +| PBXT_STATISTICS | ++---------------------------------------+ +Wildcard: inf_rmation_schema ++--------------------+ +| Databases | ++--------------------+ +| information_schema | ++--------------------+ +SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; +table_schema count(*) +information_schema 29 +mysql 22 === modified file 'mysql-test/r/information_schema_db.result' --- a/mysql-test/r/information_schema_db.result 2008-10-02 10:08:15 +0000 +++ b/mysql-test/r/information_schema_db.result 2009-04-08 16:55:26 +0000 @@ -2,37 +2,6 @@ drop table if exists t1,t2; drop view if exists v1,v2; drop function if exists f1; drop function if exists f2; -use INFORMATION_SCHEMA; -show tables; -Tables_in_information_schema -CHARACTER_SETS -COLLATIONS -COLLATION_CHARACTER_SET_APPLICABILITY -COLUMNS -COLUMN_PRIVILEGES -ENGINES -EVENTS -FILES -GLOBAL_STATUS -GLOBAL_VARIABLES -KEY_COLUMN_USAGE -PARTITIONS -PLUGINS -PROCESSLIST -PROFILING -REFERENTIAL_CONSTRAINTS -ROUTINES -SCHEMATA -SCHEMA_PRIVILEGES -SESSION_STATUS -SESSION_VARIABLES -STATISTICS -TABLES -TABLE_CONSTRAINTS -TABLE_PRIVILEGES -TRIGGERS -USER_PRIVILEGES -VIEWS show tables from INFORMATION_SCHEMA like 'T%'; Tables_in_information_schema (T%) TABLES === modified file 'mysql-test/r/innodb-autoinc.result' --- a/mysql-test/r/innodb-autoinc.result 2008-12-19 00:28:51 +0000 +++ b/mysql-test/r/innodb-autoinc.result 2009-04-08 16:55:26 +0000 @@ -197,7 +197,7 @@ c1 c2 5 9 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -230,7 +230,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -269,7 +269,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -282,7 +282,7 @@ SELECT * FROM t1; c1 -1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -315,7 +315,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -330,7 +330,7 @@ SELECT * FROM t1; c1 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -370,7 +370,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -385,7 +385,7 @@ SELECT * FROM t1; c1 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -419,7 +419,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -434,7 +434,7 @@ c1 1 9223372036854775794 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 2 auto_increment_offset 10 @@ -452,7 +452,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -467,7 +467,7 @@ c1 1 18446744073709551603 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 2 auto_increment_offset 10 @@ -485,7 +485,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -500,7 +500,7 @@ c1 1 18446744073709551603 SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 5 auto_increment_offset 7 @@ -514,7 +514,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -533,7 +533,7 @@ c1 -9223372036854775806 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 3 auto_increment_offset 3 @@ -550,7 +550,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -568,7 +568,7 @@ SET @@SESSION.AUTO_INCREMENT_INCREMENT=1 Warnings: Warning 1292 Truncated incorrect auto_increment_increment value: '1152921504606846976' Warning 1292 Truncated incorrect auto_increment_offset value: '1152921504606846976' -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 65535 auto_increment_offset 65535 === modified file 'mysql-test/r/mysqlshow.result' --- a/mysql-test/r/mysqlshow.result 2007-04-27 20:45:01 +0000 +++ b/mysql-test/r/mysqlshow.result 2009-04-08 16:55:26 +0000 @@ -75,76 +75,4 @@ Database: test 2 rows in set. DROP TABLE t1, t2; -Database: information_schema -+---------------------------------------+ -| Tables | -+---------------------------------------+ -| CHARACTER_SETS | -| COLLATIONS | -| COLLATION_CHARACTER_SET_APPLICABILITY | -| COLUMNS | -| COLUMN_PRIVILEGES | -| ENGINES | -| EVENTS | -| FILES | -| GLOBAL_STATUS | -| GLOBAL_VARIABLES | -| KEY_COLUMN_USAGE | -| PARTITIONS | -| PLUGINS | -| PROCESSLIST | -| PROFILING | -| REFERENTIAL_CONSTRAINTS | -| ROUTINES | -| SCHEMATA | -| SCHEMA_PRIVILEGES | -| SESSION_STATUS | -| SESSION_VARIABLES | -| STATISTICS | -| TABLES | -| TABLE_CONSTRAINTS | -| TABLE_PRIVILEGES | -| TRIGGERS | -| USER_PRIVILEGES | -| VIEWS | -+---------------------------------------+ -Database: INFORMATION_SCHEMA -+---------------------------------------+ -| Tables | -+---------------------------------------+ -| CHARACTER_SETS | -| COLLATIONS | -| COLLATION_CHARACTER_SET_APPLICABILITY | -| COLUMNS | -| COLUMN_PRIVILEGES | -| ENGINES | -| EVENTS | -| FILES | -| GLOBAL_STATUS | -| GLOBAL_VARIABLES | -| KEY_COLUMN_USAGE | -| PARTITIONS | -| PLUGINS | -| PROCESSLIST | -| PROFILING | -| REFERENTIAL_CONSTRAINTS | -| ROUTINES | -| SCHEMATA | -| SCHEMA_PRIVILEGES | -| SESSION_STATUS | -| SESSION_VARIABLES | -| STATISTICS | -| TABLES | -| TABLE_CONSTRAINTS | -| TABLE_PRIVILEGES | -| TRIGGERS | -| USER_PRIVILEGES | -| VIEWS | -+---------------------------------------+ -Wildcard: inf_rmation_schema -+--------------------+ -| Databases | -+--------------------+ -| information_schema | -+--------------------+ End of 5.0 tests === modified file 'mysql-test/suite/rpl/r/rpl_auto_increment.result' --- a/mysql-test/suite/rpl/r/rpl_auto_increment.result 2009-01-14 08:27:32 +0000 +++ b/mysql-test/suite/rpl/r/rpl_auto_increment.result 2009-04-08 16:55:26 +0000 @@ -37,7 +37,7 @@ a b 32 6 drop table t1; set @@session.auto_increment_increment=100, @@session.auto_increment_offset=10; -show variables like "%auto_inc%"; +show variables like "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 === modified file 'mysql-test/t/information_schema.test' --- a/mysql-test/t/information_schema.test 2009-01-23 12:22:05 +0000 +++ b/mysql-test/t/information_schema.test 2009-04-08 16:55:26 +0000 @@ -41,7 +41,8 @@ create view v1 (c) as SELECT table_name FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND - table_name<>'ndb_apply_status'; + table_name<>'ndb_apply_status' AND + NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%'); select * from v1; select c,table_name from v1 @@ -529,13 +530,6 @@ delete from mysql.db where user='mysqlte flush privileges; # -# Bug #9404 information_schema: Weird error messages -# with SELECT SUM() ... GROUP BY queries -# -SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; - - -# # TRIGGERS table test # create table t1 (i int, j int); @@ -898,39 +892,6 @@ DROP PROCEDURE p1; DROP USER mysql_bug20230@localhost; # -# Bug#18925: subqueries with MIN/MAX functions on INFORMARTION_SCHEMA -# - -SELECT t.table_name, c1.column_name - FROM information_schema.tables t - INNER JOIN - information_schema.columns c1 - ON t.table_schema = c1.table_schema AND - t.table_name = c1.table_name - WHERE t.table_schema = 'information_schema' AND - c1.ordinal_position = - ( SELECT COALESCE(MIN(c2.ordinal_position),1) - FROM information_schema.columns c2 - WHERE c2.table_schema = t.table_schema AND - c2.table_name = t.table_name AND - c2.column_name LIKE '%SCHEMA%' - ); -SELECT t.table_name, c1.column_name - FROM information_schema.tables t - INNER JOIN - information_schema.columns c1 - ON t.table_schema = c1.table_schema AND - t.table_name = c1.table_name - WHERE t.table_schema = 'information_schema' AND - c1.ordinal_position = - ( SELECT COALESCE(MIN(c2.ordinal_position),1) - FROM information_schema.columns c2 - WHERE c2.table_schema = 'information_schema' AND - c2.table_name = t.table_name AND - c2.column_name LIKE '%SCHEMA%' - ); - -# # Bug#21231: query with a simple non-correlated subquery over # INFORMARTION_SCHEMA.TABLES # @@ -1007,29 +968,6 @@ drop table t1,t2; # -# Bug#24630 Subselect query crashes mysqld -# -select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= -(select cast(table_name as char) from information_schema.tables - order by table_name limit 1) limit 1; - -select t.table_name, group_concat(t.table_schema, '.', t.table_name), - count(*) as num1 -from information_schema.tables t -inner join information_schema.columns c1 -on t.table_schema = c1.table_schema AND t.table_name = c1.table_name -where t.table_schema = 'information_schema' and - c1.ordinal_position = - (select isnull(c2.column_type) - - isnull(group_concat(c2.table_schema, '.', c2.table_name)) + - count(*) as num - from information_schema.columns c2 where - c2.table_schema='information_schema' and - (c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') - group by c2.column_type order by num limit 1) -group by t.table_name order by num1, t.table_name; - -# # Bug#28266 IS_UPDATABLE field on VIEWS table in I_S database is wrong # create table t1(f1 int); === added file 'mysql-test/t/information_schema_all_engines.test' --- a/mysql-test/t/information_schema_all_engines.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/information_schema_all_engines.test 2009-04-08 16:55:26 +0000 @@ -0,0 +1,79 @@ +# This file contains tests moved from information_schema.test and +# information_schema_db.test whose results depends on which engines are +# available (since these engines inject tables into INFORMATION_SCHEMA). + +--source include/not_embedded.inc +--source include/have_pbxt.inc + +use INFORMATION_SCHEMA; +--replace_result Tables_in_INFORMATION_SCHEMA Tables_in_information_schema +show tables; + +# +# Bug#18925: subqueries with MIN/MAX functions on INFORMARTION_SCHEMA +# + +SELECT t.table_name, c1.column_name + FROM information_schema.tables t + INNER JOIN + information_schema.columns c1 + ON t.table_schema = c1.table_schema AND + t.table_name = c1.table_name + WHERE t.table_schema = 'information_schema' AND + c1.ordinal_position = + ( SELECT COALESCE(MIN(c2.ordinal_position),1) + FROM information_schema.columns c2 + WHERE c2.table_schema = t.table_schema AND + c2.table_name = t.table_name AND + c2.column_name LIKE '%SCHEMA%' + ); +SELECT t.table_name, c1.column_name + FROM information_schema.tables t + INNER JOIN + information_schema.columns c1 + ON t.table_schema = c1.table_schema AND + t.table_name = c1.table_name + WHERE t.table_schema = 'information_schema' AND + c1.ordinal_position = + ( SELECT COALESCE(MIN(c2.ordinal_position),1) + FROM information_schema.columns c2 + WHERE c2.table_schema = 'information_schema' AND + c2.table_name = t.table_name AND + c2.column_name LIKE '%SCHEMA%' + ); + +# +# Bug#24630 Subselect query crashes mysqld +# +select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= +(select cast(table_name as char) from information_schema.tables + order by table_name limit 1) limit 1; + +select t.table_name, group_concat(t.table_schema, '.', t.table_name), + count(*) as num1 +from information_schema.tables t +inner join information_schema.columns c1 +on t.table_schema = c1.table_schema AND t.table_name = c1.table_name +where t.table_schema = 'information_schema' and + c1.ordinal_position = + (select isnull(c2.column_type) - + isnull(group_concat(c2.table_schema, '.', c2.table_name)) + + count(*) as num + from information_schema.columns c2 where + c2.table_schema='information_schema' and + (c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') + group by c2.column_type order by num limit 1) +group by t.table_name order by num1, t.table_name; + +# +# Bug #19147: mysqlshow INFORMATION_SCHEMA does not work +# +--exec $MYSQL_SHOW information_schema +--exec $MYSQL_SHOW INFORMATION_SCHEMA +--exec $MYSQL_SHOW inf_rmation_schema + +# +# Bug #9404 information_schema: Weird error messages +# with SELECT SUM() ... GROUP BY queries +# +SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; === modified file 'mysql-test/t/information_schema_db.test' --- a/mysql-test/t/information_schema_db.test 2008-10-02 09:37:07 +0000 +++ b/mysql-test/t/information_schema_db.test 2009-04-08 16:55:26 +0000 @@ -11,9 +11,6 @@ drop function if exists f1; drop function if exists f2; --enable_warnings -use INFORMATION_SCHEMA; ---replace_result Tables_in_INFORMATION_SCHEMA Tables_in_information_schema -show tables; --replace_result 'Tables_in_INFORMATION_SCHEMA (T%)' 'Tables_in_information_schema (T%)' show tables from INFORMATION_SCHEMA like 'T%'; create database `inf%`; === modified file 'mysql-test/t/innodb-autoinc.test' --- a/mysql-test/t/innodb-autoinc.test 2009-01-23 12:22:05 +0000 +++ b/mysql-test/t/innodb-autoinc.test 2009-04-08 16:55:26 +0000 @@ -156,7 +156,7 @@ DROP TABLE t1; # # Test changes to AUTOINC next value calculation SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES (NULL),(5),(NULL); @@ -173,7 +173,7 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(0); @@ -193,13 +193,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2), (NULL),(2),(NULL); INSERT INTO t1 VALUES (250),(NULL); SELECT * FROM t1; @@ -214,13 +214,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2); INSERT INTO t1 VALUES (NULL); INSERT INTO t1 VALUES (2); @@ -240,13 +240,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2),(NULL),(2),(NULL); INSERT INTO t1 VALUES (250),(NULL); SELECT * FROM t1; @@ -262,7 +262,7 @@ DROP TABLE t1; # Check for overflow handling when increment is > 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -271,7 +271,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (9223372036854775794); #-- 2^63 - 14 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should just fit INSERT INTO t1 VALUES (NULL),(NULL),(NULL),(NULL),(NULL),(NULL); SELECT * FROM t1; @@ -281,7 +281,7 @@ DROP TABLE t1; # Check for overflow handling when increment and offser are > 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -290,7 +290,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't, it seems to be # a MySQL server bug. It wraps around to 0 for the last value. # See MySQL Bug# 39828 @@ -313,7 +313,7 @@ DROP TABLE t1; # Check for overflow handling when increment and offset are odd numbers SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -322,7 +322,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't. It fails with # a duplicate entry message because of a MySQL server bug, it wraps # around. See MySQL Bug# 39828, once MySQL fix the bug we can replace @@ -344,7 +344,7 @@ DROP TABLE t1; # and check for large -ve numbers SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -355,7 +355,7 @@ INSERT INTO t1 VALUES(-92233720368547758 INSERT INTO t1 VALUES(-9223372036854775808); #-- -2^63 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (NULL),(NULL), (NULL); SELECT * FROM t1; DROP TABLE t1; @@ -364,7 +364,7 @@ DROP TABLE t1; # large numbers 2^60 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -373,7 +373,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551610); #-- 2^64 - 2 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1152921504606846976, @@SESSION.AUTO_INCREMENT_OFFSET=1152921504606846976; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't. It wraps around # and the autoinc values look bogus too. # See MySQL Bug# 39828, once MySQL fix the bug we can enable the error === modified file 'mysql-test/t/mysqlshow.test' --- a/mysql-test/t/mysqlshow.test 2006-07-22 03:29:25 +0000 +++ b/mysql-test/t/mysqlshow.test 2009-04-08 16:55:26 +0000 @@ -26,11 +26,4 @@ select "---- -v -v -t ------" as ""; --exec $MYSQL_SHOW test -v -v -t DROP TABLE t1, t2; -# -# Bug #19147: mysqlshow INFORMATION_SCHEMA does not work -# ---exec $MYSQL_SHOW information_schema ---exec $MYSQL_SHOW INFORMATION_SCHEMA ---exec $MYSQL_SHOW inf_rmation_schema - --echo End of 5.0 tests === modified file 'mysql-test/valgrind.supp' --- a/mysql-test/valgrind.supp 2008-11-21 14:21:50 +0000 +++ b/mysql-test/valgrind.supp 2009-04-08 16:55:26 +0000 @@ -379,7 +379,7 @@ } { - dlclose memory loss from plugin + dlclose memory loss from plugin variant 1 Memcheck:Leak fun:calloc fun:_dlerror_run @@ -388,6 +388,19 @@ } { + dlclose memory loss from plugin variant 2 + Memcheck:Leak + fun:malloc + fun:_dl_close_worker + fun:_dl_close + fun:_dl_catch_error + fun:_dlerror_run + fun:dlclose + fun:_Z15free_plugin_memP12st_plugin_dl + fun:_Z13plugin_dl_delPK19st_mysql_lex_string +} + +{ dlopen / ptread_cancel_init memory loss on Suse Linux 10.3 32/64 bit Memcheck:Leak fun:*alloc @@ -588,3 +601,19 @@ fun:dlopen* } +# +# In glibc (checked version 2.7), inet_ntoa allocates an 18-byte +# per-thread static buffer for the return value. That memory is freed +# at thread exit, however if called from the main thread, Valgrind +# does not see the free (test main.no-threads). +# +# Since inet_ntoa() does not allocate memory dynamically per-call, this +# suppression is safe. +# + +{ + inet_ntoa thread local storage + Memcheck:Leak + fun:malloc + fun:inet_ntoa +} === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-03-18 02:14:22 +0000 +++ b/sql/mysqld.cc 2009-04-08 16:55:26 +0000 @@ -4815,10 +4815,10 @@ static bool read_init_file(char *file_na DBUG_ENTER("read_init_file"); DBUG_PRINT("enter",("name: %s",file_name)); if (!(file=my_fopen(file_name,O_RDONLY,MYF(MY_WME)))) - return(1); + DBUG_RETURN(1); bootstrap(file); (void) my_fclose(file,MYF(MY_WME)); - return 0; + DBUG_RETURN(0); } @@ -4839,6 +4839,7 @@ void handle_connection_in_main_thread(TH safe_mutex_assert_owner(&LOCK_thread_count); thread_cache_size=0; // Safety threads.append(thd); + thd->connect_utime= my_micro_time(); (void) pthread_mutex_unlock(&LOCK_thread_count); handle_one_connection((void*) thd); } === modified file 'sql/set_var.cc' --- a/sql/set_var.cc 2009-03-12 22:27:35 +0000 +++ b/sql/set_var.cc 2009-04-08 16:55:26 +0000 @@ -1740,7 +1740,7 @@ bool sys_var::check_enum(THD *thd, set_v { char buff[STRING_BUFFER_USUAL_SIZE]; const char *value; - String str(buff, sizeof(buff), system_charset_info), *res; + String str(buff, sizeof(buff) - 1, system_charset_info), *res; if (var->value->result_type() == STRING_RESULT) { @@ -1777,7 +1777,7 @@ bool sys_var::check_set(THD *thd, set_va bool not_used; char buff[STRING_BUFFER_USUAL_SIZE], *error= 0; uint error_len= 0; - String str(buff, sizeof(buff), system_charset_info), *res; + String str(buff, sizeof(buff) - 1, system_charset_info), *res; if (var->value->result_type() == STRING_RESULT) { @@ -1795,7 +1795,7 @@ bool sys_var::check_set(THD *thd, set_va } var->save_result.ulong_value= ((ulong) - find_set(enum_names, res->c_ptr(), + find_set(enum_names, res->ptr(), res->length(), NULL, &error, &error_len, @@ -1942,7 +1942,7 @@ bool sys_var_thd_date_time_format::updat bool sys_var_thd_date_time_format::check(THD *thd, set_var *var) { char buff[STRING_BUFFER_USUAL_SIZE]; - String str(buff,sizeof(buff), system_charset_info), *res; + String str(buff,sizeof(buff) - 1, system_charset_info), *res; DATE_TIME_FORMAT *format; if (!(res=var->value->val_str(&str))) @@ -2047,7 +2047,7 @@ bool sys_var_collation::check(THD *thd, if (var->value->result_type() == STRING_RESULT) { char buff[STRING_BUFFER_USUAL_SIZE]; - String str(buff,sizeof(buff), system_charset_info), *res; + String str(buff,sizeof(buff) - 1, system_charset_info), *res; if (!(res=var->value->val_str(&str))) { my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), name, "NULL"); @@ -2082,7 +2082,7 @@ bool sys_var_character_set::check(THD *t if (var->value->result_type() == STRING_RESULT) { char buff[STRING_BUFFER_USUAL_SIZE]; - String str(buff,sizeof(buff), system_charset_info), *res; + String str(buff,sizeof(buff) - 1, system_charset_info), *res; if (!(res=var->value->val_str(&str))) { if (!nullable) @@ -3620,7 +3620,7 @@ bool sys_var_thd_storage_engine::check(T { char buff[STRING_BUFFER_USUAL_SIZE]; const char *value; - String str(buff, sizeof(buff), &my_charset_latin1), *res; + String str(buff, sizeof(buff) - 1, &my_charset_latin1), *res; var->save_result.plugin= NULL; if (var->value->result_type() == STRING_RESULT) @@ -3737,7 +3737,7 @@ sys_var_thd_sql_mode:: symbolic_mode_representation(THD *thd, ulonglong val, LEX_STRING *rep) { char buff[STRING_BUFFER_USUAL_SIZE*8]; - String tmp(buff, sizeof(buff), &my_charset_latin1); + String tmp(buff, sizeof(buff) - 1, &my_charset_latin1); tmp.length(0); === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-02-19 09:01:25 +0000 +++ b/sql/sql_select.cc 2009-04-08 16:55:26 +0000 @@ -1994,8 +1994,17 @@ JOIN::exec() tmp_fields_list2, tmp_all_fields2, fields_list.elements, tmp_all_fields1)) DBUG_VOID_RETURN; - curr_join->tmp_fields_list2= tmp_fields_list2; - curr_join->tmp_all_fields2= tmp_all_fields2; +#ifdef HAVE_purify + /* + Some GCCs use memcpy() for struct assignment, even for x=x. + GCC bug 19410: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=19410 + */ + if (curr_join != this) +#endif + { + curr_join->tmp_fields_list2= tmp_fields_list2; + curr_join->tmp_all_fields2= tmp_all_fields2; + } } curr_fields_list= &curr_join->tmp_fields_list2; curr_all_fields= &curr_join->tmp_all_fields2; === modified file 'sql/sql_string.h' --- a/sql/sql_string.h 2009-01-15 18:11:25 +0000 +++ b/sql/sql_string.h 2009-04-08 16:55:26 +0000 @@ -63,6 +63,11 @@ public: Ptr=(char*) str; str_length=(uint) strlen(str); Alloced_length=0; alloced=0; str_charset=cs; } + /* + NOTE: If one intend to use the c_ptr() method, the following two + contructors need the size of memory for STR to be at least LEN+1 (to make + room for zero termination). + */ String(const char *str,uint32 len, CHARSET_INFO *cs) { Ptr=(char*) str; str_length=len; Alloced_length=0; alloced=0; === modified file 'storage/archive/azio.c' --- a/storage/archive/azio.c 2007-12-12 13:16:59 +0000 +++ b/storage/archive/azio.c 2009-04-08 16:55:26 +0000 @@ -37,6 +37,40 @@ void putLong(File file, uLong x); uLong getLong(azio_stream *s); void read_header(azio_stream *s, unsigned char *buffer); +/* + Valgrind normally gives false alarms for zlib operations, in the form of + "conditional jump depends on uninitialised values" etc. The reason is + explained in the zlib FAQ (http://www.zlib.net/zlib_faq.html#faq36) + + "That is intentional for performance reasons, and the output of deflate + is not affected." + + Also discussed on a blog + (http://www.sirena.org.uk/log/2006/02/19/zlib-generating-valgrind-warnings/) + + "...loop unrolling in the zlib library causes the mentioned + “Conditional jump or move depends on uninitialised value(s)” + warnings. These are safe since the results of the comparison are + subsequently ignored..." + + "the results of the calculations are discarded by bounds checking done + after the loop exits" + + Fix by initializing the memory allocated by zlib when running under Valgrind. + + This fix is safe, since such memory is only used internally by zlib, so we + will not hide any bugs in mysql this way. +*/ +static void *az_allocator(void *dummy, uInt items, uInt size) +{ + return my_malloc((size_t)items*(size_t)size, IF_PURIFY(MY_ZEROFILL, MYF(0))); +} + +static void az_free(void *dummy, void *address) +{ + my_free(address, MYF(MY_ALLOW_ZERO_PTR)); +} + /* =========================================================================== Opens a gzip (.gz) file for reading or writing. The mode parameter is as in fopen ("rb" or "wb"). The file is given either by file descriptor @@ -52,8 +86,8 @@ int az_open (azio_stream *s, const char int level = Z_DEFAULT_COMPRESSION; /* compression level */ int strategy = Z_DEFAULT_STRATEGY; /* compression strategy */ - s->stream.zalloc = (alloc_func)0; - s->stream.zfree = (free_func)0; + s->stream.zalloc = az_allocator; + s->stream.zfree = az_free; s->stream.opaque = (voidpf)0; memset(s->inbuf, 0, AZ_BUFSIZE_READ); memset(s->outbuf, 0, AZ_BUFSIZE_WRITE);

1 0

Re: [Maria-developers] Working on microsecond patches
by Michael Widenius 08 Apr '09

08 Apr '09

Hi! >>>>> "Jeremiah" == Jeremiah Gowdy <basharteg(a)basharteg.com> writes: Vadim> Michael, Vadim> Thread-pool should be used very carefully. >> Vadim> We tested it and InnoDB hangs in sysbench benchmarks when count of Vadim> client connection > size of thread-pool. >> >> Yes, this a problem when you have more locks than threads. >> The innodb deadlock detector has to timeout the hanged items. >> Vadim> The problem is transaction state. Some connections may do internals Vadim> locks and after that could not enter to pool, because all slots are busy. Vadim> I expect you will have the same problem with Maria when it can fully Vadim> support transactions. >> >> That is one of the main reasons I added --extra-port to MariaDB >> This allows you to connect and check/kill things if you get a hang. >> >> So things are not perfect now, but at least a little better. >> >> In the future we have to also look at creating more pool-threads when >> all pool-threads gets locked by a handler. >> Jeremiah> This is a good solution IMO. You could base it on how long the next Jeremiah> queue item has been in the pool. So you determine a maximum latency Jeremiah> value that makes sense, and you grow the thread pool any time tasks are Jeremiah> waiting longer than that to execute, up to a certain number of threads Jeremiah> that would be your hard limit. Once you reach the hard limit, you Jeremiah> refuse to grow the pool further, and if no queries are being dequeued Jeremiah> after a long timeout, you could determine that things are deadlocked and Jeremiah> panic / start killing things, or let the user make that decision using Jeremiah> extra-port. Agree that this is a good way to do it. I will copy the above to a worklog item; Hope you don't mind.... http://askmonty.org/worklog/Server-RawIdeaBin/index.pl?tid=13 Regards, Monty

1 0

[Maria-developers] [patch 00/11] A number of fixes for buildbot-found problems
by knielsen＠knielsen-hq.org 08 Apr '09

08 Apr '09

Hi Monty, Sanja, all, I finally managed to get a valgrind test run to pass. I needed 11 patches to get there ... It would be really good to get a review of these so I can push, they are needed to get the Valgrind run to pass in buildbot. I think all of them are ok, but for at least some of them I would really like to get another pair of eyes before pushing as I'm not very familiar with the code they touch. Sanja, could you review first 2 patches? They are of mysql-test-run.pl, one of them is related to the previous mysqld.err fixes. Monty, could you review the remaining 9 patches? They are all fixes for Valgrind warnings. Especially the last 4 I am not 100% sure of are the best possible fix, I think you can help. Serg, I checked up on your comment on my earlier dbug.c patch (thanks!), found the problem you described and I think it is fixed in this version of the patch. Monty, I will commit as one commit before pushing, but due to large number of independent changes I found it convenient to split it up using quilt. - Kristian.

3 23

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2693)
by knielsen＠knielsen-hq.org 08 Apr '09

08 Apr '09

#At lp:maria 2693 knielsen(a)knielsen-hq.org 2009-04-08 Fix test suite following merge of PBXT The PBXT engine causes an extra table to appear in the INFORMATION_SCHEMA. This causes different output for a few test cases. Fix test suite to avoid any errors from this. added: mysql-test/include/have_pbxt.inc mysql-test/r/information_schema_all_engines.result mysql-test/t/information_schema_all_engines.test modified: mysql-test/extra/rpl_tests/rpl_auto_increment.test mysql-test/r/information_schema.result mysql-test/r/information_schema_db.result mysql-test/r/innodb-autoinc.result mysql-test/r/mysqlshow.result mysql-test/suite/rpl/r/rpl_auto_increment.result mysql-test/t/information_schema.test mysql-test/t/information_schema_db.test mysql-test/t/innodb-autoinc.test mysql-test/t/mysqlshow.test per-file messages: mysql-test/extra/rpl_tests/rpl_auto_increment.test Omit pbxt variables from show variables output. mysql-test/include/have_pbxt.inc Add facility to disable test if PBXT engine is not available. mysql-test/r/information_schema.result Omit PBXT INFORMATION_SCHEMA table from output. Move part of test to information_schema_all_engines. mysql-test/r/information_schema_all_engines.result New file for information_schema tests that depend on which engines are available. mysql-test/r/information_schema_db.result Move part of test to information_schema_all_engines. mysql-test/r/innodb-autoinc.result Omit pbxt variables from show variables output. mysql-test/r/mysqlshow.result Move part of test to information_schema_all_engines. mysql-test/suite/rpl/r/rpl_auto_increment.result Omit pbxt variables from show variables output. mysql-test/t/information_schema.test Omit PBXT INFORMATION_SCHEMA table from output. Move part of test to information_schema_all_engines. mysql-test/t/information_schema_all_engines.test New file for information_schema tests that depend on which engines are available. mysql-test/t/information_schema_db.test Omit PBXT INFORMATION_SCHEMA table from output. Move part of test to information_schema_all_engines. mysql-test/t/innodb-autoinc.test Omit pbxt variables from show variables output. mysql-test/t/mysqlshow.test Move part of test to information_schema_all_engines. === modified file 'mysql-test/extra/rpl_tests/rpl_auto_increment.test' --- a/mysql-test/extra/rpl_tests/rpl_auto_increment.test 2009-01-14 08:27:32 +0000 +++ b/mysql-test/extra/rpl_tests/rpl_auto_increment.test 2009-04-08 13:40:20 +0000 @@ -34,7 +34,7 @@ connection master; drop table t1; set @@session.auto_increment_increment=100, @@session.auto_increment_offset=10; -show variables like "%auto_inc%"; +show variables like "auto_inc%"; eval create table t1 (a int not null auto_increment, primary key (a)) engine=$engine_type2; # Insert with 2 insert statements to get better testing of logging === added file 'mysql-test/include/have_pbxt.inc' --- a/mysql-test/include/have_pbxt.inc 1970-01-01 00:00:00 +0000 +++ b/mysql-test/include/have_pbxt.inc 2009-04-08 13:40:20 +0000 @@ -0,0 +1,4 @@ +disable_query_log; +--require r/true.require +select (support = 'YES' or support = 'DEFAULT') as `TRUE` from information_schema.engines where engine = 'pbxt'; +enable_query_log; === modified file 'mysql-test/r/information_schema.result' --- a/mysql-test/r/information_schema.result 2009-02-15 10:58:34 +0000 +++ b/mysql-test/r/information_schema.result 2009-04-08 13:40:20 +0000 @@ -40,7 +40,8 @@ create view v1 (c) as SELECT table_name FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND -table_name<>'ndb_apply_status'; +table_name<>'ndb_apply_status' AND +NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%'); select * from v1; c CHARACTER_SETS @@ -850,7 +851,7 @@ VIEWS TABLE_NAME select delete from mysql.user where user='mysqltest_4'; delete from mysql.db where user='mysqltest_4'; flush privileges; -SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; +SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' AND NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%') GROUP BY TABLE_SCHEMA; table_schema count(*) information_schema 28 mysql 22 @@ -1224,92 +1225,6 @@ f1() DROP FUNCTION f1; DROP PROCEDURE p1; DROP USER mysql_bug20230@localhost; -SELECT t.table_name, c1.column_name -FROM information_schema.tables t -INNER JOIN -information_schema.columns c1 -ON t.table_schema = c1.table_schema AND -t.table_name = c1.table_name -WHERE t.table_schema = 'information_schema' AND -c1.ordinal_position = -( SELECT COALESCE(MIN(c2.ordinal_position),1) -FROM information_schema.columns c2 -WHERE c2.table_schema = t.table_schema AND -c2.table_name = t.table_name AND -c2.column_name LIKE '%SCHEMA%' - ); -table_name column_name -CHARACTER_SETS CHARACTER_SET_NAME -COLLATIONS COLLATION_NAME -COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME -COLUMNS TABLE_SCHEMA -COLUMN_PRIVILEGES TABLE_SCHEMA -ENGINES ENGINE -EVENTS EVENT_SCHEMA -FILES TABLE_SCHEMA -GLOBAL_STATUS VARIABLE_NAME -GLOBAL_VARIABLES VARIABLE_NAME -KEY_COLUMN_USAGE CONSTRAINT_SCHEMA -PARTITIONS TABLE_SCHEMA -PLUGINS PLUGIN_NAME -PROCESSLIST ID -PROFILING QUERY_ID -REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA -ROUTINES ROUTINE_SCHEMA -SCHEMATA SCHEMA_NAME -SCHEMA_PRIVILEGES TABLE_SCHEMA -SESSION_STATUS VARIABLE_NAME -SESSION_VARIABLES VARIABLE_NAME -STATISTICS TABLE_SCHEMA -TABLES TABLE_SCHEMA -TABLE_CONSTRAINTS CONSTRAINT_SCHEMA -TABLE_PRIVILEGES TABLE_SCHEMA -TRIGGERS TRIGGER_SCHEMA -USER_PRIVILEGES GRANTEE -VIEWS TABLE_SCHEMA -SELECT t.table_name, c1.column_name -FROM information_schema.tables t -INNER JOIN -information_schema.columns c1 -ON t.table_schema = c1.table_schema AND -t.table_name = c1.table_name -WHERE t.table_schema = 'information_schema' AND -c1.ordinal_position = -( SELECT COALESCE(MIN(c2.ordinal_position),1) -FROM information_schema.columns c2 -WHERE c2.table_schema = 'information_schema' AND -c2.table_name = t.table_name AND -c2.column_name LIKE '%SCHEMA%' - ); -table_name column_name -CHARACTER_SETS CHARACTER_SET_NAME -COLLATIONS COLLATION_NAME -COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME -COLUMNS TABLE_SCHEMA -COLUMN_PRIVILEGES TABLE_SCHEMA -ENGINES ENGINE -EVENTS EVENT_SCHEMA -FILES TABLE_SCHEMA -GLOBAL_STATUS VARIABLE_NAME -GLOBAL_VARIABLES VARIABLE_NAME -KEY_COLUMN_USAGE CONSTRAINT_SCHEMA -PARTITIONS TABLE_SCHEMA -PLUGINS PLUGIN_NAME -PROCESSLIST ID -PROFILING QUERY_ID -REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA -ROUTINES ROUTINE_SCHEMA -SCHEMATA SCHEMA_NAME -SCHEMA_PRIVILEGES TABLE_SCHEMA -SESSION_STATUS VARIABLE_NAME -SESSION_VARIABLES VARIABLE_NAME -STATISTICS TABLE_SCHEMA -TABLES TABLE_SCHEMA -TABLE_CONSTRAINTS CONSTRAINT_SCHEMA -TABLE_PRIVILEGES TABLE_SCHEMA -TRIGGERS TRIGGER_SCHEMA -USER_PRIVILEGES GRANTEE -VIEWS TABLE_SCHEMA SELECT MAX(table_name) FROM information_schema.tables WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test'); MAX(table_name) VIEWS @@ -1355,55 +1270,6 @@ table_name t1 t2 drop table t1,t2; -select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= -(select cast(table_name as char) from information_schema.tables -order by table_name limit 1) limit 1; -f1 -1 -select t.table_name, group_concat(t.table_schema, '.', t.table_name), -count(*) as num1 -from information_schema.tables t -inner join information_schema.columns c1 -on t.table_schema = c1.table_schema AND t.table_name = c1.table_name -where t.table_schema = 'information_schema' and -c1.ordinal_position = -(select isnull(c2.column_type) - -isnull(group_concat(c2.table_schema, '.', c2.table_name)) + -count(*) as num -from information_schema.columns c2 where -c2.table_schema='information_schema' and -(c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') -group by c2.column_type order by num limit 1) -group by t.table_name order by num1, t.table_name; -table_name group_concat(t.table_schema, '.', t.table_name) num1 -CHARACTER_SETS information_schema.CHARACTER_SETS 1 -COLLATIONS information_schema.COLLATIONS 1 -COLLATION_CHARACTER_SET_APPLICABILITY information_schema.COLLATION_CHARACTER_SET_APPLICABILITY 1 -COLUMNS information_schema.COLUMNS 1 -COLUMN_PRIVILEGES information_schema.COLUMN_PRIVILEGES 1 -ENGINES information_schema.ENGINES 1 -EVENTS information_schema.EVENTS 1 -FILES information_schema.FILES 1 -GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 -GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 -KEY_COLUMN_USAGE information_schema.KEY_COLUMN_USAGE 1 -PARTITIONS information_schema.PARTITIONS 1 -PLUGINS information_schema.PLUGINS 1 -PROCESSLIST information_schema.PROCESSLIST 1 -PROFILING information_schema.PROFILING 1 -REFERENTIAL_CONSTRAINTS information_schema.REFERENTIAL_CONSTRAINTS 1 -ROUTINES information_schema.ROUTINES 1 -SCHEMATA information_schema.SCHEMATA 1 -SCHEMA_PRIVILEGES information_schema.SCHEMA_PRIVILEGES 1 -SESSION_STATUS information_schema.SESSION_STATUS 1 -SESSION_VARIABLES information_schema.SESSION_VARIABLES 1 -STATISTICS information_schema.STATISTICS 1 -TABLES information_schema.TABLES 1 -TABLE_CONSTRAINTS information_schema.TABLE_CONSTRAINTS 1 -TABLE_PRIVILEGES information_schema.TABLE_PRIVILEGES 1 -TRIGGERS information_schema.TRIGGERS 1 -USER_PRIVILEGES information_schema.USER_PRIVILEGES 1 -VIEWS information_schema.VIEWS 1 create table t1(f1 int); create view v1 as select f1+1 as a from t1; create table t2 (f1 int, f2 int); === added file 'mysql-test/r/information_schema_all_engines.result' --- a/mysql-test/r/information_schema_all_engines.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/information_schema_all_engines.result 2009-04-08 13:40:20 +0000 @@ -0,0 +1,244 @@ +use INFORMATION_SCHEMA; +show tables; +Tables_in_information_schema +CHARACTER_SETS +COLLATIONS +COLLATION_CHARACTER_SET_APPLICABILITY +COLUMNS +COLUMN_PRIVILEGES +ENGINES +EVENTS +FILES +GLOBAL_STATUS +GLOBAL_VARIABLES +KEY_COLUMN_USAGE +PARTITIONS +PLUGINS +PROCESSLIST +PROFILING +REFERENTIAL_CONSTRAINTS +ROUTINES +SCHEMATA +SCHEMA_PRIVILEGES +SESSION_STATUS +SESSION_VARIABLES +STATISTICS +TABLES +TABLE_CONSTRAINTS +TABLE_PRIVILEGES +TRIGGERS +USER_PRIVILEGES +VIEWS +PBXT_STATISTICS +SELECT t.table_name, c1.column_name +FROM information_schema.tables t +INNER JOIN +information_schema.columns c1 +ON t.table_schema = c1.table_schema AND +t.table_name = c1.table_name +WHERE t.table_schema = 'information_schema' AND +c1.ordinal_position = +( SELECT COALESCE(MIN(c2.ordinal_position),1) +FROM information_schema.columns c2 +WHERE c2.table_schema = t.table_schema AND +c2.table_name = t.table_name AND +c2.column_name LIKE '%SCHEMA%' + ); +table_name column_name +CHARACTER_SETS CHARACTER_SET_NAME +COLLATIONS COLLATION_NAME +COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME +COLUMNS TABLE_SCHEMA +COLUMN_PRIVILEGES TABLE_SCHEMA +ENGINES ENGINE +EVENTS EVENT_SCHEMA +FILES TABLE_SCHEMA +GLOBAL_STATUS VARIABLE_NAME +GLOBAL_VARIABLES VARIABLE_NAME +KEY_COLUMN_USAGE CONSTRAINT_SCHEMA +PARTITIONS TABLE_SCHEMA +PLUGINS PLUGIN_NAME +PROCESSLIST ID +PROFILING QUERY_ID +REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA +ROUTINES ROUTINE_SCHEMA +SCHEMATA SCHEMA_NAME +SCHEMA_PRIVILEGES TABLE_SCHEMA +SESSION_STATUS VARIABLE_NAME +SESSION_VARIABLES VARIABLE_NAME +STATISTICS TABLE_SCHEMA +TABLES TABLE_SCHEMA +TABLE_CONSTRAINTS CONSTRAINT_SCHEMA +TABLE_PRIVILEGES TABLE_SCHEMA +TRIGGERS TRIGGER_SCHEMA +USER_PRIVILEGES GRANTEE +VIEWS TABLE_SCHEMA +PBXT_STATISTICS ID +SELECT t.table_name, c1.column_name +FROM information_schema.tables t +INNER JOIN +information_schema.columns c1 +ON t.table_schema = c1.table_schema AND +t.table_name = c1.table_name +WHERE t.table_schema = 'information_schema' AND +c1.ordinal_position = +( SELECT COALESCE(MIN(c2.ordinal_position),1) +FROM information_schema.columns c2 +WHERE c2.table_schema = 'information_schema' AND +c2.table_name = t.table_name AND +c2.column_name LIKE '%SCHEMA%' + ); +table_name column_name +CHARACTER_SETS CHARACTER_SET_NAME +COLLATIONS COLLATION_NAME +COLLATION_CHARACTER_SET_APPLICABILITY COLLATION_NAME +COLUMNS TABLE_SCHEMA +COLUMN_PRIVILEGES TABLE_SCHEMA +ENGINES ENGINE +EVENTS EVENT_SCHEMA +FILES TABLE_SCHEMA +GLOBAL_STATUS VARIABLE_NAME +GLOBAL_VARIABLES VARIABLE_NAME +KEY_COLUMN_USAGE CONSTRAINT_SCHEMA +PARTITIONS TABLE_SCHEMA +PLUGINS PLUGIN_NAME +PROCESSLIST ID +PROFILING QUERY_ID +REFERENTIAL_CONSTRAINTS CONSTRAINT_SCHEMA +ROUTINES ROUTINE_SCHEMA +SCHEMATA SCHEMA_NAME +SCHEMA_PRIVILEGES TABLE_SCHEMA +SESSION_STATUS VARIABLE_NAME +SESSION_VARIABLES VARIABLE_NAME +STATISTICS TABLE_SCHEMA +TABLES TABLE_SCHEMA +TABLE_CONSTRAINTS CONSTRAINT_SCHEMA +TABLE_PRIVILEGES TABLE_SCHEMA +TRIGGERS TRIGGER_SCHEMA +USER_PRIVILEGES GRANTEE +VIEWS TABLE_SCHEMA +PBXT_STATISTICS ID +select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= +(select cast(table_name as char) from information_schema.tables +order by table_name limit 1) limit 1; +f1 +1 +select t.table_name, group_concat(t.table_schema, '.', t.table_name), +count(*) as num1 +from information_schema.tables t +inner join information_schema.columns c1 +on t.table_schema = c1.table_schema AND t.table_name = c1.table_name +where t.table_schema = 'information_schema' and +c1.ordinal_position = +(select isnull(c2.column_type) - +isnull(group_concat(c2.table_schema, '.', c2.table_name)) + +count(*) as num +from information_schema.columns c2 where +c2.table_schema='information_schema' and +(c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') +group by c2.column_type order by num limit 1) +group by t.table_name order by num1, t.table_name; +table_name group_concat(t.table_schema, '.', t.table_name) num1 +CHARACTER_SETS information_schema.CHARACTER_SETS 1 +COLLATIONS information_schema.COLLATIONS 1 +COLLATION_CHARACTER_SET_APPLICABILITY information_schema.COLLATION_CHARACTER_SET_APPLICABILITY 1 +COLUMNS information_schema.COLUMNS 1 +COLUMN_PRIVILEGES information_schema.COLUMN_PRIVILEGES 1 +ENGINES information_schema.ENGINES 1 +EVENTS information_schema.EVENTS 1 +FILES information_schema.FILES 1 +GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 +GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 +KEY_COLUMN_USAGE information_schema.KEY_COLUMN_USAGE 1 +PARTITIONS information_schema.PARTITIONS 1 +PBXT_STATISTICS information_schema.PBXT_STATISTICS 1 +PLUGINS information_schema.PLUGINS 1 +PROCESSLIST information_schema.PROCESSLIST 1 +PROFILING information_schema.PROFILING 1 +REFERENTIAL_CONSTRAINTS information_schema.REFERENTIAL_CONSTRAINTS 1 +ROUTINES information_schema.ROUTINES 1 +SCHEMATA information_schema.SCHEMATA 1 +SCHEMA_PRIVILEGES information_schema.SCHEMA_PRIVILEGES 1 +SESSION_STATUS information_schema.SESSION_STATUS 1 +SESSION_VARIABLES information_schema.SESSION_VARIABLES 1 +STATISTICS information_schema.STATISTICS 1 +TABLES information_schema.TABLES 1 +TABLE_CONSTRAINTS information_schema.TABLE_CONSTRAINTS 1 +TABLE_PRIVILEGES information_schema.TABLE_PRIVILEGES 1 +TRIGGERS information_schema.TRIGGERS 1 +USER_PRIVILEGES information_schema.USER_PRIVILEGES 1 +VIEWS information_schema.VIEWS 1 +Database: information_schema ++---------------------------------------+ +| Tables | ++---------------------------------------+ +| CHARACTER_SETS | +| COLLATIONS | +| COLLATION_CHARACTER_SET_APPLICABILITY | +| COLUMNS | +| COLUMN_PRIVILEGES | +| ENGINES | +| EVENTS | +| FILES | +| GLOBAL_STATUS | +| GLOBAL_VARIABLES | +| KEY_COLUMN_USAGE | +| PARTITIONS | +| PLUGINS | +| PROCESSLIST | +| PROFILING | +| REFERENTIAL_CONSTRAINTS | +| ROUTINES | +| SCHEMATA | +| SCHEMA_PRIVILEGES | +| SESSION_STATUS | +| SESSION_VARIABLES | +| STATISTICS | +| TABLES | +| TABLE_CONSTRAINTS | +| TABLE_PRIVILEGES | +| TRIGGERS | +| USER_PRIVILEGES | +| VIEWS | +| PBXT_STATISTICS | ++---------------------------------------+ +Database: INFORMATION_SCHEMA ++---------------------------------------+ +| Tables | ++---------------------------------------+ +| CHARACTER_SETS | +| COLLATIONS | +| COLLATION_CHARACTER_SET_APPLICABILITY | +| COLUMNS | +| COLUMN_PRIVILEGES | +| ENGINES | +| EVENTS | +| FILES | +| GLOBAL_STATUS | +| GLOBAL_VARIABLES | +| KEY_COLUMN_USAGE | +| PARTITIONS | +| PLUGINS | +| PROCESSLIST | +| PROFILING | +| REFERENTIAL_CONSTRAINTS | +| ROUTINES | +| SCHEMATA | +| SCHEMA_PRIVILEGES | +| SESSION_STATUS | +| SESSION_VARIABLES | +| STATISTICS | +| TABLES | +| TABLE_CONSTRAINTS | +| TABLE_PRIVILEGES | +| TRIGGERS | +| USER_PRIVILEGES | +| VIEWS | +| PBXT_STATISTICS | ++---------------------------------------+ +Wildcard: inf_rmation_schema ++--------------------+ +| Databases | ++--------------------+ +| information_schema | ++--------------------+ === modified file 'mysql-test/r/information_schema_db.result' --- a/mysql-test/r/information_schema_db.result 2008-10-02 10:08:15 +0000 +++ b/mysql-test/r/information_schema_db.result 2009-04-08 13:40:20 +0000 @@ -2,37 +2,6 @@ drop table if exists t1,t2; drop view if exists v1,v2; drop function if exists f1; drop function if exists f2; -use INFORMATION_SCHEMA; -show tables; -Tables_in_information_schema -CHARACTER_SETS -COLLATIONS -COLLATION_CHARACTER_SET_APPLICABILITY -COLUMNS -COLUMN_PRIVILEGES -ENGINES -EVENTS -FILES -GLOBAL_STATUS -GLOBAL_VARIABLES -KEY_COLUMN_USAGE -PARTITIONS -PLUGINS -PROCESSLIST -PROFILING -REFERENTIAL_CONSTRAINTS -ROUTINES -SCHEMATA -SCHEMA_PRIVILEGES -SESSION_STATUS -SESSION_VARIABLES -STATISTICS -TABLES -TABLE_CONSTRAINTS -TABLE_PRIVILEGES -TRIGGERS -USER_PRIVILEGES -VIEWS show tables from INFORMATION_SCHEMA like 'T%'; Tables_in_information_schema (T%) TABLES === modified file 'mysql-test/r/innodb-autoinc.result' --- a/mysql-test/r/innodb-autoinc.result 2008-12-19 00:28:51 +0000 +++ b/mysql-test/r/innodb-autoinc.result 2009-04-08 13:40:20 +0000 @@ -197,7 +197,7 @@ c1 c2 5 9 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -230,7 +230,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -269,7 +269,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -282,7 +282,7 @@ SELECT * FROM t1; c1 -1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -315,7 +315,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -330,7 +330,7 @@ SELECT * FROM t1; c1 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -370,7 +370,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -385,7 +385,7 @@ SELECT * FROM t1; c1 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 @@ -419,7 +419,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -434,7 +434,7 @@ c1 1 9223372036854775794 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 2 auto_increment_offset 10 @@ -452,7 +452,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -467,7 +467,7 @@ c1 1 18446744073709551603 SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 2 auto_increment_offset 10 @@ -485,7 +485,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -500,7 +500,7 @@ c1 1 18446744073709551603 SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 5 auto_increment_offset 7 @@ -514,7 +514,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -533,7 +533,7 @@ c1 -9223372036854775806 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 3 auto_increment_offset 3 @@ -550,7 +550,7 @@ c1 DROP TABLE t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 1 auto_increment_offset 1 @@ -568,7 +568,7 @@ SET @@SESSION.AUTO_INCREMENT_INCREMENT=1 Warnings: Warning 1292 Truncated incorrect auto_increment_increment value: '1152921504606846976' Warning 1292 Truncated incorrect auto_increment_offset value: '1152921504606846976' -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; Variable_name Value auto_increment_increment 65535 auto_increment_offset 65535 === modified file 'mysql-test/r/mysqlshow.result' --- a/mysql-test/r/mysqlshow.result 2007-04-27 20:45:01 +0000 +++ b/mysql-test/r/mysqlshow.result 2009-04-08 13:40:20 +0000 @@ -75,76 +75,4 @@ Database: test 2 rows in set. DROP TABLE t1, t2; -Database: information_schema -+---------------------------------------+ -| Tables | -+---------------------------------------+ -| CHARACTER_SETS | -| COLLATIONS | -| COLLATION_CHARACTER_SET_APPLICABILITY | -| COLUMNS | -| COLUMN_PRIVILEGES | -| ENGINES | -| EVENTS | -| FILES | -| GLOBAL_STATUS | -| GLOBAL_VARIABLES | -| KEY_COLUMN_USAGE | -| PARTITIONS | -| PLUGINS | -| PROCESSLIST | -| PROFILING | -| REFERENTIAL_CONSTRAINTS | -| ROUTINES | -| SCHEMATA | -| SCHEMA_PRIVILEGES | -| SESSION_STATUS | -| SESSION_VARIABLES | -| STATISTICS | -| TABLES | -| TABLE_CONSTRAINTS | -| TABLE_PRIVILEGES | -| TRIGGERS | -| USER_PRIVILEGES | -| VIEWS | -+---------------------------------------+ -Database: INFORMATION_SCHEMA -+---------------------------------------+ -| Tables | -+---------------------------------------+ -| CHARACTER_SETS | -| COLLATIONS | -| COLLATION_CHARACTER_SET_APPLICABILITY | -| COLUMNS | -| COLUMN_PRIVILEGES | -| ENGINES | -| EVENTS | -| FILES | -| GLOBAL_STATUS | -| GLOBAL_VARIABLES | -| KEY_COLUMN_USAGE | -| PARTITIONS | -| PLUGINS | -| PROCESSLIST | -| PROFILING | -| REFERENTIAL_CONSTRAINTS | -| ROUTINES | -| SCHEMATA | -| SCHEMA_PRIVILEGES | -| SESSION_STATUS | -| SESSION_VARIABLES | -| STATISTICS | -| TABLES | -| TABLE_CONSTRAINTS | -| TABLE_PRIVILEGES | -| TRIGGERS | -| USER_PRIVILEGES | -| VIEWS | -+---------------------------------------+ -Wildcard: inf_rmation_schema -+--------------------+ -| Databases | -+--------------------+ -| information_schema | -+--------------------+ End of 5.0 tests === modified file 'mysql-test/suite/rpl/r/rpl_auto_increment.result' --- a/mysql-test/suite/rpl/r/rpl_auto_increment.result 2009-01-14 08:27:32 +0000 +++ b/mysql-test/suite/rpl/r/rpl_auto_increment.result 2009-04-08 13:40:20 +0000 @@ -37,7 +37,7 @@ a b 32 6 drop table t1; set @@session.auto_increment_increment=100, @@session.auto_increment_offset=10; -show variables like "%auto_inc%"; +show variables like "auto_inc%"; Variable_name Value auto_increment_increment 100 auto_increment_offset 10 === modified file 'mysql-test/t/information_schema.test' --- a/mysql-test/t/information_schema.test 2009-01-23 12:22:05 +0000 +++ b/mysql-test/t/information_schema.test 2009-04-08 13:40:20 +0000 @@ -41,7 +41,8 @@ create view v1 (c) as SELECT table_name FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND - table_name<>'ndb_apply_status'; + table_name<>'ndb_apply_status' AND + NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%'); select * from v1; select c,table_name from v1 @@ -532,7 +533,7 @@ flush privileges; # Bug #9404 information_schema: Weird error messages # with SELECT SUM() ... GROUP BY queries # -SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; +SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' AND NOT (table_schema = 'INFORMATION_SCHEMA' AND table_name LIKE 'PBXT_%') GROUP BY TABLE_SCHEMA; # @@ -898,39 +899,6 @@ DROP PROCEDURE p1; DROP USER mysql_bug20230@localhost; # -# Bug#18925: subqueries with MIN/MAX functions on INFORMARTION_SCHEMA -# - -SELECT t.table_name, c1.column_name - FROM information_schema.tables t - INNER JOIN - information_schema.columns c1 - ON t.table_schema = c1.table_schema AND - t.table_name = c1.table_name - WHERE t.table_schema = 'information_schema' AND - c1.ordinal_position = - ( SELECT COALESCE(MIN(c2.ordinal_position),1) - FROM information_schema.columns c2 - WHERE c2.table_schema = t.table_schema AND - c2.table_name = t.table_name AND - c2.column_name LIKE '%SCHEMA%' - ); -SELECT t.table_name, c1.column_name - FROM information_schema.tables t - INNER JOIN - information_schema.columns c1 - ON t.table_schema = c1.table_schema AND - t.table_name = c1.table_name - WHERE t.table_schema = 'information_schema' AND - c1.ordinal_position = - ( SELECT COALESCE(MIN(c2.ordinal_position),1) - FROM information_schema.columns c2 - WHERE c2.table_schema = 'information_schema' AND - c2.table_name = t.table_name AND - c2.column_name LIKE '%SCHEMA%' - ); - -# # Bug#21231: query with a simple non-correlated subquery over # INFORMARTION_SCHEMA.TABLES # @@ -1007,29 +975,6 @@ drop table t1,t2; # -# Bug#24630 Subselect query crashes mysqld -# -select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= -(select cast(table_name as char) from information_schema.tables - order by table_name limit 1) limit 1; - -select t.table_name, group_concat(t.table_schema, '.', t.table_name), - count(*) as num1 -from information_schema.tables t -inner join information_schema.columns c1 -on t.table_schema = c1.table_schema AND t.table_name = c1.table_name -where t.table_schema = 'information_schema' and - c1.ordinal_position = - (select isnull(c2.column_type) - - isnull(group_concat(c2.table_schema, '.', c2.table_name)) + - count(*) as num - from information_schema.columns c2 where - c2.table_schema='information_schema' and - (c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') - group by c2.column_type order by num limit 1) -group by t.table_name order by num1, t.table_name; - -# # Bug#28266 IS_UPDATABLE field on VIEWS table in I_S database is wrong # create table t1(f1 int); === added file 'mysql-test/t/information_schema_all_engines.test' --- a/mysql-test/t/information_schema_all_engines.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/information_schema_all_engines.test 2009-04-08 13:40:20 +0000 @@ -0,0 +1,73 @@ +# This file contains tests moved from information_schema.test and +# information_schema_db.test whose results depends on which engines are +# available (since these engines inject tables into INFORMATION_SCHEMA). + +--source include/not_embedded.inc +--source include/have_pbxt.inc + +use INFORMATION_SCHEMA; +--replace_result Tables_in_INFORMATION_SCHEMA Tables_in_information_schema +show tables; + +# +# Bug#18925: subqueries with MIN/MAX functions on INFORMARTION_SCHEMA +# + +SELECT t.table_name, c1.column_name + FROM information_schema.tables t + INNER JOIN + information_schema.columns c1 + ON t.table_schema = c1.table_schema AND + t.table_name = c1.table_name + WHERE t.table_schema = 'information_schema' AND + c1.ordinal_position = + ( SELECT COALESCE(MIN(c2.ordinal_position),1) + FROM information_schema.columns c2 + WHERE c2.table_schema = t.table_schema AND + c2.table_name = t.table_name AND + c2.column_name LIKE '%SCHEMA%' + ); +SELECT t.table_name, c1.column_name + FROM information_schema.tables t + INNER JOIN + information_schema.columns c1 + ON t.table_schema = c1.table_schema AND + t.table_name = c1.table_name + WHERE t.table_schema = 'information_schema' AND + c1.ordinal_position = + ( SELECT COALESCE(MIN(c2.ordinal_position),1) + FROM information_schema.columns c2 + WHERE c2.table_schema = 'information_schema' AND + c2.table_name = t.table_name AND + c2.column_name LIKE '%SCHEMA%' + ); + +# +# Bug#24630 Subselect query crashes mysqld +# +select 1 as f1 from information_schema.tables where "CHARACTER_SETS"= +(select cast(table_name as char) from information_schema.tables + order by table_name limit 1) limit 1; + +select t.table_name, group_concat(t.table_schema, '.', t.table_name), + count(*) as num1 +from information_schema.tables t +inner join information_schema.columns c1 +on t.table_schema = c1.table_schema AND t.table_name = c1.table_name +where t.table_schema = 'information_schema' and + c1.ordinal_position = + (select isnull(c2.column_type) - + isnull(group_concat(c2.table_schema, '.', c2.table_name)) + + count(*) as num + from information_schema.columns c2 where + c2.table_schema='information_schema' and + (c2.column_type = 'varchar(7)' or c2.column_type = 'varchar(20)') + group by c2.column_type order by num limit 1) +group by t.table_name order by num1, t.table_name; + +# +# Bug #19147: mysqlshow INFORMATION_SCHEMA does not work +# +--exec $MYSQL_SHOW information_schema +--exec $MYSQL_SHOW INFORMATION_SCHEMA +--exec $MYSQL_SHOW inf_rmation_schema === modified file 'mysql-test/t/information_schema_db.test' --- a/mysql-test/t/information_schema_db.test 2008-10-02 09:37:07 +0000 +++ b/mysql-test/t/information_schema_db.test 2009-04-08 13:40:20 +0000 @@ -11,9 +11,6 @@ drop function if exists f1; drop function if exists f2; --enable_warnings -use INFORMATION_SCHEMA; ---replace_result Tables_in_INFORMATION_SCHEMA Tables_in_information_schema -show tables; --replace_result 'Tables_in_INFORMATION_SCHEMA (T%)' 'Tables_in_information_schema (T%)' show tables from INFORMATION_SCHEMA like 'T%'; create database `inf%`; === modified file 'mysql-test/t/innodb-autoinc.test' --- a/mysql-test/t/innodb-autoinc.test 2009-01-23 12:22:05 +0000 +++ b/mysql-test/t/innodb-autoinc.test 2009-04-08 13:40:20 +0000 @@ -156,7 +156,7 @@ DROP TABLE t1; # # Test changes to AUTOINC next value calculation SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES (NULL),(5),(NULL); @@ -173,7 +173,7 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(0); @@ -193,13 +193,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2), (NULL),(2),(NULL); INSERT INTO t1 VALUES (250),(NULL); SELECT * FROM t1; @@ -214,13 +214,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2); INSERT INTO t1 VALUES (NULL); INSERT INTO t1 VALUES (2); @@ -240,13 +240,13 @@ DROP TABLE t1; # Reset the AUTOINC session variables SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 INT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; INSERT INTO t1 VALUES(-1); SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=100, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (-2),(NULL),(2),(NULL); INSERT INTO t1 VALUES (250),(NULL); SELECT * FROM t1; @@ -262,7 +262,7 @@ DROP TABLE t1; # Check for overflow handling when increment is > 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -271,7 +271,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (9223372036854775794); #-- 2^63 - 14 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should just fit INSERT INTO t1 VALUES (NULL),(NULL),(NULL),(NULL),(NULL),(NULL); SELECT * FROM t1; @@ -281,7 +281,7 @@ DROP TABLE t1; # Check for overflow handling when increment and offser are > 1 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -290,7 +290,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=2, @@SESSION.AUTO_INCREMENT_OFFSET=10; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't, it seems to be # a MySQL server bug. It wraps around to 0 for the last value. # See MySQL Bug# 39828 @@ -313,7 +313,7 @@ DROP TABLE t1; # Check for overflow handling when increment and offset are odd numbers SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -322,7 +322,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551603); #-- 2^64 - 13 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=5, @@SESSION.AUTO_INCREMENT_OFFSET=7; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't. It fails with # a duplicate entry message because of a MySQL server bug, it wraps # around. See MySQL Bug# 39828, once MySQL fix the bug we can replace @@ -344,7 +344,7 @@ DROP TABLE t1; # and check for large -ve numbers SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -355,7 +355,7 @@ INSERT INTO t1 VALUES(-92233720368547758 INSERT INTO t1 VALUES(-9223372036854775808); #-- -2^63 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=3, @@SESSION.AUTO_INCREMENT_OFFSET=3; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; INSERT INTO t1 VALUES (NULL),(NULL), (NULL); SELECT * FROM t1; DROP TABLE t1; @@ -364,7 +364,7 @@ DROP TABLE t1; # large numbers 2^60 SET @@SESSION.AUTO_INCREMENT_INCREMENT=1, @@SESSION.AUTO_INCREMENT_OFFSET=1; SET @@INSERT_ID=1; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 BIGINT UNSIGNED AUTO_INCREMENT, PRIMARY KEY(c1)) ENGINE=InnoDB; # TODO: Fix the autoinc init code @@ -373,7 +373,7 @@ INSERT INTO t1 VALUES(NULL); INSERT INTO t1 VALUES (18446744073709551610); #-- 2^64 - 2 SELECT * FROM t1; SET @@SESSION.AUTO_INCREMENT_INCREMENT=1152921504606846976, @@SESSION.AUTO_INCREMENT_OFFSET=1152921504606846976; -SHOW VARIABLES LIKE "%auto_inc%"; +SHOW VARIABLES LIKE "auto_inc%"; # This should fail because of overflow but it doesn't. It wraps around # and the autoinc values look bogus too. # See MySQL Bug# 39828, once MySQL fix the bug we can enable the error === modified file 'mysql-test/t/mysqlshow.test' --- a/mysql-test/t/mysqlshow.test 2006-07-22 03:29:25 +0000 +++ b/mysql-test/t/mysqlshow.test 2009-04-08 13:40:20 +0000 @@ -26,11 +26,4 @@ select "---- -v -v -t ------" as ""; --exec $MYSQL_SHOW test -v -v -t DROP TABLE t1, t2; -# -# Bug #19147: mysqlshow INFORMATION_SCHEMA does not work -# ---exec $MYSQL_SHOW information_schema ---exec $MYSQL_SHOW INFORMATION_SCHEMA ---exec $MYSQL_SHOW inf_rmation_schema - --echo End of 5.0 tests

1 0

[Maria-developers] mysql-test-run (was Re: BuildBot)
by Kristian Nielsen 08 Apr '09

08 Apr '09

Michael Widenius <michael.widenius(a)gmail.com> writes: > To go trough the things that are wrong with mysqltest: In my opinion, the single biggest problem with mysqltest is that it is a seperate program from the test suite driver mysql-test-run.pl. It's even written in a different language (C vs. Perl), and test cases themselves are written in a third language (the mysqltest language) + a mixture of SQL and even some shell scripts. That's _5_ different languages in a single application :-( This is a disaster for the structure of mysql-test-run. Such a large part of the code (and complexity) is just trying to handle communication between the various parts, which is really hard to do well due to different languages. The first step in any major improvement (in my opinion) should be to re-implement mysqltest in Perl inside mysql-test-run.pl. But that would be a project of several months, not something I'm eager to start just now ... > I like the philosophy behind it: Write a test in pure SQL and then > ensure that the generated result doesn't change. It's easy enough to > allow any developers to generate a test case. Yes. This is a very strong point for testing that correct results are produced by a query or similar. Keeping something like this is very important if thinking about extending or replacing the test framework. The weakness with mysqltest / mysql-test-run is not that it does this, it is that it does not provide additional/separate functinality for other kinds of tests that do not fit this simple framework well. > In particular, it's good that tests and results are different files as > it makes it easier to understand what are wrong when you get a failure. I am trying to understand what precisely you have in mind here... One big problem with other test suites I have seen is that if you want to debug a failure, you first have to spend a lot of time debugging and understanding the test case code just to understand what the problem actually is. Getting a diff of queries run and results produced like mysqltest does makes this step trivial (if the problem is wrong results from a query). Also the --record functionality for generating .result files can be really good to ease the production of test case and also make it more likely the developer will not be tempted to insufficiently check the results (everything is checked by default). It does have the big risk of not properly checking that the committed .result is in fact correct, something I have seen more than once. This does not really have to mean using separate files. I have in mind something like a Perl-based suite where I could do: query <<SQL, <<RESULT SELECT a, b FROM t1 JOIN t2 ON (a = b) ORDER BY a, b; SQL RECORDME RESULT and `mysql-test-run --record` would rewrite this as query <<SQL, <<RESULT SELECT a, b FROM t1 JOIN t2 ON (a = b) ORDER BY a, b; SQL a b 1 1 2 2 4 4 RESULT And in case of different result, it could output a diff of just the RESULT section of the failing query, along with the query. It could also give the correct line number in the .test file, avoiding the need to search the .test file for the query that produced the diff. I think that might make it even easier than separate result file. Something like that would of course allow putting arbitrary Perl code in-between running queries (with lots of useful modules available) where this is more appropriate. This would eliminate the need for the mysqltest language for new test cases, which is one less language again. And it would be quite easy to have a backwards-compatible emulation of the old .test and .result files (just a small wrapper that reads the .test and .result files and passes the contents to the query() function). > One benefit is that you can usually run a test by just piping it > trough the normal MySQL client, which simplifies testing and > debugging a lot! Yes, this should be kept for simple stuff, very important. - Kristian.

3 3

Re: [Maria-developers] Maria binary builds
by Kristian Nielsen 07 Apr '09

07 Apr '09

Hi Vadim, Just an update on what I have been doing with binaries and Buildbot. I have now set up Buildbot to build and test our trees on Launchpad after every push (similar to Pushbuild at MySQL/Sun if you are familiar with that): http://askmonty.org/buildbot/index.html For now we have only few (one) build slave. The plan of course is to add more, hopefully some of them volunteered from the community (hint :-). I would be happy to make the infrastructure available to you as well, ie. to set it up to also track your Launcpad trees and build on your build slaves with your scripts and options. But you may prefer to control the buildbot master yourself; just let me know if you need a copy of my Buildbot configuration files or if I can help with any questions in setting it up. I am currently working on getting all existing test failures visible in the Buildbot runs fixed (Valgrind errors mostly). After that (next week I hope) I plan to turn to working on getting Buildbot to also produce binaries. I will need to do some research in which platforms we need to build for (Debian/Ubuntu/CentOS/Suse/Generic? x86/amd64? Any others?), and which configure / build options to use. Any help you can provide here would be much appreciated. I have lots of experience with scripting this kind of thing, but less so with the details of how people actually use the binaries and what they need (I have mostly build from source myself even for production). I also plan to look at what official Debian/Ubuntu packages provide, and at the scripts from OurDelta. My vision is to have production and test of binaries fully integrated into our Buildbot infrastructure. I want it fully automated, so that binaries are made after every push, and so that problems with the binaries are dealt with as an integrated part of development just like any other problem. Ideally, releasing binaries would then just be a matter of picking the right files from the buildbot archives. It will take some work to get there of course, but at least that is the vision. >From what I have seen of Buildbot so far, I am hopeful that it will be sufficient for our needs. It seems to have all of the right ideas, and many of the necessary features are already there. Some stuff will be missing, and my plan is to spend the time to extend it as needed. So for concrete plans next week, these currently are: 1. Setup build of generic binaries (.tar.gz) for Linux x86/amd64. For this I need to understand better how to deal with external dependencies. Should I link statically with libc? My impression is not to do this as I think this causes big problems with any type of plugin, but I also seem to recall that existing generic packages do this. Alternatively, I need to understand which version of glibc to use. 2. Work on getting build slave machines set up for this. 3. Get a first version running on Buildbot that produces the binaries. After this I think will be Debian and Ubuntu binaries with apt-able repositories, and then RPMs. I think that's it for an update of current plans. Let me know if you have any questions/comments/suggestions. Especially any help on exact CFLAGS and ./configure arguments to use would be very helpful. - Kristian.

3 3

[Maria-developers] bzr commit into Mariadb 5.2, with Maria 2.0:maria/5.2 branch (sanja:2723)
by sanja＠askmonty.org 03 Apr '09

03 Apr '09

#At lp:maria/5.2 2723 sanja(a)askmonty.org 2009-04-03 [merge] 5.1 -> 5.2 merge modified: mysys/my_malloc.c mysys/my_once.c mysys/my_realloc.c tests/fork_big2.pl === modified file 'mysys/my_malloc.c' --- a/mysys/my_malloc.c 2007-10-02 07:32:33 +0000 +++ b/mysys/my_malloc.c 2009-03-31 08:06:51 +0000 @@ -13,6 +13,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif === modified file 'mysys/my_once.c' --- a/mysys/my_once.c 2007-05-10 09:59:39 +0000 +++ b/mysys/my_once.c 2009-03-31 08:06:51 +0000 @@ -15,6 +15,9 @@ /* Not MT-SAFE */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif === modified file 'mysys/my_realloc.c' --- a/mysys/my_realloc.c 2008-04-28 16:24:05 +0000 +++ b/mysys/my_realloc.c 2009-03-31 08:06:51 +0000 @@ -13,6 +13,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +/* my_global.h may define SAFEMALLOC (through my_config.h). */ +#include <my_global.h> + #ifdef SAFEMALLOC /* We don't need SAFEMALLOC here */ #undef SAFEMALLOC #endif === modified file 'tests/fork_big2.pl' --- a/tests/fork_big2.pl 2006-02-12 21:26:30 +0000 +++ b/tests/fork_big2.pl 2009-04-03 13:23:08 +0000 @@ -16,21 +16,21 @@ package main; $opt_skip_create=$opt_skip_in=$opt_verbose=$opt_fast_insert= $opt_lock_tables=$opt_debug=$opt_skip_drop=$opt_fast=$opt_force=0; -$opt_thread_factor=1; -$opt_insert=1; -$opt_select=6;$opt_join=4; -$opt_select_count=$opt_join_count=0; -$opt_update=1;$opt_delete=0; -$opt_flush=$opt_check=$opt_repair=$opt_alter=0; -$opt_join_range=100; +$opt_thread_factor=1; +$opt_insert=1; +$opt_select=6;$opt_join=4; +$opt_select_count=$opt_join_count=0; +$opt_update=1;$opt_delete=0; +$opt_flush=$opt_check=$opt_repair=$opt_alter=0; +$opt_join_range=100; $opt_resize_interval=0; $opt_time=0; $opt_host=$opt_user=$opt_password=""; $opt_db="test"; $opt_verbose=$opt_debug=$opt_lock_tables=$opt_fast_insert=$opt_fast=$opt_skip_in=$opt_force=undef; # Ignore warnings from these -GetOptions("host=s","db=s","user=s","password=s","loop-count=i","skip-create","skip-in","skip-drop", - "verbose","fast-insert","lock-tables","debug","fast","force","thread-factor=i", - "insert=i", "select=i", "join=i", "select-count=i", "join-count=i", "update=i", "delete=i", +GetOptions("host=s","db=s","user=s","password=s","loop-count=i","skip-create","skip-in","skip-drop", + "verbose","fast-insert","lock-tables","debug","fast","force","thread-factor=i", + "insert=i", "select=i", "join=i", "select-count=i", "join-count=i", "update=i", "delete=i", "flush=i", "check=i", "repair=i", "alter=i", "resize-interval=i", "max-join_range=i", "time=i") || die "Aborted"; print "Test of multiple connections that test the following things:\n"; @@ -48,20 +48,20 @@ srand 100; # Make random numbers repea #### #### Start timeing and start test -#### - +#### + $opt_insert*=$opt_thread_factor; -$opt_select*=$opt_thread_factor; -$opt_join*=$opt_thread_factor; -$opt_select_count*=$opt_thread_factor; -$opt_join_count*=$opt_thread_factor; -$opt_update*=$opt_thread_factor; -$opt_delete*=$opt_thread_factor; - -if ($opt_time == 0 && $opt_insert == 0) -{ - $opt_insert=1; -} +$opt_select*=$opt_thread_factor; +$opt_join*=$opt_thread_factor; +$opt_select_count*=$opt_thread_factor; +$opt_join_count*=$opt_thread_factor; +$opt_update*=$opt_thread_factor; +$opt_delete*=$opt_thread_factor; + +if ($opt_time == 0 && $opt_insert == 0) +{ + $opt_insert=1; +} $start_time=new Benchmark; $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", @@ -100,71 +100,71 @@ $|= 1; # Autoflush #### #### Start the tests #### -if ($opt_time != 0) -{ - test_abort() if (($pid=fork()) == 0); $work{$pid}="abort"; +if ($opt_time != 0) +{ + test_abort() if (($pid=fork()) == 0); $work{$pid}="abort"; } for ($i=0 ; $i < $opt_insert ; $i ++) { test_insert() if (($pid=fork()) == 0); $work{$pid}="insert"; -} +} $threads=$i; -for ($i=0 ; $i < $opt_select ; $i ++) -{ - test_select() if (($pid=fork()) == 0); $work{$pid}="select"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_join ; $i ++) -{ - test_join() if (($pid=fork()) == 0); $work{$pid}="join"; -} -$threads+=$i; +for ($i=0 ; $i < $opt_select ; $i ++) +{ + test_select() if (($pid=fork()) == 0); $work{$pid}="select"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_join ; $i ++) +{ + test_join() if (($pid=fork()) == 0); $work{$pid}="join"; +} +$threads+=$i; for ($i=0 ; $i < $opt_select_count ; $i ++) { test_select_count() if (($pid=fork()) == 0); $work{$pid}="select_count"; } -$threads+=$i; -for ($i=0 ; $i < $opt_join_count ; $i ++) -{ - test_join_count() if (($pid=fork()) == 0); $work{$pid}="join_count"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_update ; $i ++) -{ - test_update() if (($pid=fork()) == 0); $work{$pid}="update"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_delete ; $i ++) -{ - test_delete() if (($pid=fork()) == 0); $work{$pid}="delete"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_flush ; $i ++) -{ - test_flush() if (($pid=fork()) == 0); $work{$pid}="flush"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_check ; $i ++) -{ - test_check() if (($pid=fork()) == 0); $work{$pid}="check"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_repair ; $i ++) -{ - test_repair() if (($pid=fork()) == 0); $work{$pid}="repair"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_alter ; $i ++) -{ - test_alter() if (($pid=fork()) == 0); $work{$pid}="alter"; -} -$threads+=$i; +$threads+=$i; +for ($i=0 ; $i < $opt_join_count ; $i ++) +{ + test_join_count() if (($pid=fork()) == 0); $work{$pid}="join_count"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_update ; $i ++) +{ + test_update() if (($pid=fork()) == 0); $work{$pid}="update"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_delete ; $i ++) +{ + test_delete() if (($pid=fork()) == 0); $work{$pid}="delete"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_flush ; $i ++) +{ + test_flush() if (($pid=fork()) == 0); $work{$pid}="flush"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_check ; $i ++) +{ + test_check() if (($pid=fork()) == 0); $work{$pid}="check"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_repair ; $i ++) +{ + test_repair() if (($pid=fork()) == 0); $work{$pid}="repair"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_alter ; $i ++) +{ + test_alter() if (($pid=fork()) == 0); $work{$pid}="alter"; +} +$threads+=$i; if ($opt_resize_interval != 0) { test_resize() if (($pid=fork()) == 0); $work{$pid}="resize"; $threads+=1; } - + print "Started $threads threads\n"; $errors=0; @@ -172,17 +172,17 @@ $running_insert_threads=$opt_insert; while (($pid=wait()) != -1) { $ret=$?/256; - print "thread '" . $work{$pid} . "' finished with exit code $ret\n"; - if ($opt_time == 0) + print "thread '" . $work{$pid} . "' finished with exit code $ret\n"; + if ($opt_time == 0) { if ($work{$pid} =~ /^insert/) { if (!--$running_insert_threads) - { - + { + # Time to stop other threads signal_abort(); - } + } } } $errors++ if ($ret != 0); @@ -214,17 +214,17 @@ print "Total time: " . exit(0); -# -# Sleep and then abort other threads -# - -sub test_abort -{ - sleep($opt_time); - signal_abort(); - exit(0); -} - +# +# Sleep and then abort other threads +# + +sub test_abort +{ + sleep($opt_time); + signal_abort(); + exit(0); +} + # # Insert records in the table @@ -363,58 +363,58 @@ sub test_join $dbh->disconnect; $dbh=0; print "Test_join: Executed $count joins\n"; exit(0); -} - -# -# select records -# Do continously joins between the first and second for range and count selected rows -# - -sub test_join_count -{ - my ($dbh, $i, $j, $count, $loop); - - $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", - $opt_user, $opt_password, - { PrintError => 0}) || die $DBI::errstr; - - $count_query=make_count_query($numtables); - $count=0; - $loop=9999; - $sum=0; - - srand(); - - $i=0; - while (($i++ % 10) || !test_if_abort($dbh)) - { - if ($loop++ >= 10) - { - $loop=0; - $row_counts=simple_query($dbh, $count_query); - } - for ($j=0 ; $j < $numtables-1 ; $j++) - { - my ($id1)= int rand $row_counts->[$j]; - my ($id2)= int rand $row_counts->[$j]; - if ($id1 > $id2) - { - my $id0=$id1; $id1=$id2; $id2=$id0; - if ($id2-$id1 > $opt_join_range) - { - $id2=$id1+$opt_join_range; - } - } - my ($t1,$t2)= ($testtables[$j]->[0],$testtables[$j+1]->[0]); - $row=simple_query($dbh, "select count(*) from $t1, $t2 where $t1.id=$t2.id and $t1.id between $id1 and $id2"); - $sum+=$row->[0]; - $count++; - } - } - $dbh->disconnect; $dbh=0; - print "Test_join_count: Executed $count joins: total $sum rows\n"; - exit(0); -} +} + +# +# select records +# Do continously joins between the first and second for range and count selected rows +# + +sub test_join_count +{ + my ($dbh, $i, $j, $count, $loop); + + $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", + $opt_user, $opt_password, + { PrintError => 0}) || die $DBI::errstr; + + $count_query=make_count_query($numtables); + $count=0; + $loop=9999; + $sum=0; + + srand(); + + $i=0; + while (($i++ % 10) || !test_if_abort($dbh)) + { + if ($loop++ >= 10) + { + $loop=0; + $row_counts=simple_query($dbh, $count_query); + } + for ($j=0 ; $j < $numtables-1 ; $j++) + { + my ($id1)= int rand $row_counts->[$j]; + my ($id2)= int rand $row_counts->[$j]; + if ($id1 > $id2) + { + my $id0=$id1; $id1=$id2; $id2=$id0; + if ($id2-$id1 > $opt_join_range) + { + $id2=$id1+$opt_join_range; + } + } + my ($t1,$t2)= ($testtables[$j]->[0],$testtables[$j+1]->[0]); + $row=simple_query($dbh, "select count(*) from $t1, $t2 where $t1.id=$t2.id and $t1.id between $id1 and $id2"); + $sum+=$row->[0]; + $count++; + } + } + $dbh->disconnect; $dbh=0; + print "Test_join_count: Executed $count joins: total $sum rows\n"; + exit(0); +} #

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (sanja:2690)
by sanja＠askmonty.org 03 Apr '09

03 Apr '09

#At lp:maria 2690 sanja(a)askmonty.org 2009-04-03 Fixed test written by windows to get it working under linux (line endings) modified: tests/fork_big2.pl per-file messages: tests/fork_big2.pl Fixed test written by windows to get it working under linux (line endings) === modified file 'tests/fork_big2.pl' --- a/tests/fork_big2.pl 2006-02-12 21:26:30 +0000 +++ b/tests/fork_big2.pl 2009-04-03 13:23:08 +0000 @@ -16,21 +16,21 @@ package main; $opt_skip_create=$opt_skip_in=$opt_verbose=$opt_fast_insert= $opt_lock_tables=$opt_debug=$opt_skip_drop=$opt_fast=$opt_force=0; -$opt_thread_factor=1; -$opt_insert=1; -$opt_select=6;$opt_join=4; -$opt_select_count=$opt_join_count=0; -$opt_update=1;$opt_delete=0; -$opt_flush=$opt_check=$opt_repair=$opt_alter=0; -$opt_join_range=100; +$opt_thread_factor=1; +$opt_insert=1; +$opt_select=6;$opt_join=4; +$opt_select_count=$opt_join_count=0; +$opt_update=1;$opt_delete=0; +$opt_flush=$opt_check=$opt_repair=$opt_alter=0; +$opt_join_range=100; $opt_resize_interval=0; $opt_time=0; $opt_host=$opt_user=$opt_password=""; $opt_db="test"; $opt_verbose=$opt_debug=$opt_lock_tables=$opt_fast_insert=$opt_fast=$opt_skip_in=$opt_force=undef; # Ignore warnings from these -GetOptions("host=s","db=s","user=s","password=s","loop-count=i","skip-create","skip-in","skip-drop", - "verbose","fast-insert","lock-tables","debug","fast","force","thread-factor=i", - "insert=i", "select=i", "join=i", "select-count=i", "join-count=i", "update=i", "delete=i", +GetOptions("host=s","db=s","user=s","password=s","loop-count=i","skip-create","skip-in","skip-drop", + "verbose","fast-insert","lock-tables","debug","fast","force","thread-factor=i", + "insert=i", "select=i", "join=i", "select-count=i", "join-count=i", "update=i", "delete=i", "flush=i", "check=i", "repair=i", "alter=i", "resize-interval=i", "max-join_range=i", "time=i") || die "Aborted"; print "Test of multiple connections that test the following things:\n"; @@ -48,20 +48,20 @@ srand 100; # Make random numbers repea #### #### Start timeing and start test -#### - +#### + $opt_insert*=$opt_thread_factor; -$opt_select*=$opt_thread_factor; -$opt_join*=$opt_thread_factor; -$opt_select_count*=$opt_thread_factor; -$opt_join_count*=$opt_thread_factor; -$opt_update*=$opt_thread_factor; -$opt_delete*=$opt_thread_factor; - -if ($opt_time == 0 && $opt_insert == 0) -{ - $opt_insert=1; -} +$opt_select*=$opt_thread_factor; +$opt_join*=$opt_thread_factor; +$opt_select_count*=$opt_thread_factor; +$opt_join_count*=$opt_thread_factor; +$opt_update*=$opt_thread_factor; +$opt_delete*=$opt_thread_factor; + +if ($opt_time == 0 && $opt_insert == 0) +{ + $opt_insert=1; +} $start_time=new Benchmark; $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", @@ -100,71 +100,71 @@ $|= 1; # Autoflush #### #### Start the tests #### -if ($opt_time != 0) -{ - test_abort() if (($pid=fork()) == 0); $work{$pid}="abort"; +if ($opt_time != 0) +{ + test_abort() if (($pid=fork()) == 0); $work{$pid}="abort"; } for ($i=0 ; $i < $opt_insert ; $i ++) { test_insert() if (($pid=fork()) == 0); $work{$pid}="insert"; -} +} $threads=$i; -for ($i=0 ; $i < $opt_select ; $i ++) -{ - test_select() if (($pid=fork()) == 0); $work{$pid}="select"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_join ; $i ++) -{ - test_join() if (($pid=fork()) == 0); $work{$pid}="join"; -} -$threads+=$i; +for ($i=0 ; $i < $opt_select ; $i ++) +{ + test_select() if (($pid=fork()) == 0); $work{$pid}="select"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_join ; $i ++) +{ + test_join() if (($pid=fork()) == 0); $work{$pid}="join"; +} +$threads+=$i; for ($i=0 ; $i < $opt_select_count ; $i ++) { test_select_count() if (($pid=fork()) == 0); $work{$pid}="select_count"; } -$threads+=$i; -for ($i=0 ; $i < $opt_join_count ; $i ++) -{ - test_join_count() if (($pid=fork()) == 0); $work{$pid}="join_count"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_update ; $i ++) -{ - test_update() if (($pid=fork()) == 0); $work{$pid}="update"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_delete ; $i ++) -{ - test_delete() if (($pid=fork()) == 0); $work{$pid}="delete"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_flush ; $i ++) -{ - test_flush() if (($pid=fork()) == 0); $work{$pid}="flush"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_check ; $i ++) -{ - test_check() if (($pid=fork()) == 0); $work{$pid}="check"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_repair ; $i ++) -{ - test_repair() if (($pid=fork()) == 0); $work{$pid}="repair"; -} -$threads+=$i; -for ($i=0 ; $i < $opt_alter ; $i ++) -{ - test_alter() if (($pid=fork()) == 0); $work{$pid}="alter"; -} -$threads+=$i; +$threads+=$i; +for ($i=0 ; $i < $opt_join_count ; $i ++) +{ + test_join_count() if (($pid=fork()) == 0); $work{$pid}="join_count"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_update ; $i ++) +{ + test_update() if (($pid=fork()) == 0); $work{$pid}="update"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_delete ; $i ++) +{ + test_delete() if (($pid=fork()) == 0); $work{$pid}="delete"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_flush ; $i ++) +{ + test_flush() if (($pid=fork()) == 0); $work{$pid}="flush"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_check ; $i ++) +{ + test_check() if (($pid=fork()) == 0); $work{$pid}="check"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_repair ; $i ++) +{ + test_repair() if (($pid=fork()) == 0); $work{$pid}="repair"; +} +$threads+=$i; +for ($i=0 ; $i < $opt_alter ; $i ++) +{ + test_alter() if (($pid=fork()) == 0); $work{$pid}="alter"; +} +$threads+=$i; if ($opt_resize_interval != 0) { test_resize() if (($pid=fork()) == 0); $work{$pid}="resize"; $threads+=1; } - + print "Started $threads threads\n"; $errors=0; @@ -172,17 +172,17 @@ $running_insert_threads=$opt_insert; while (($pid=wait()) != -1) { $ret=$?/256; - print "thread '" . $work{$pid} . "' finished with exit code $ret\n"; - if ($opt_time == 0) + print "thread '" . $work{$pid} . "' finished with exit code $ret\n"; + if ($opt_time == 0) { if ($work{$pid} =~ /^insert/) { if (!--$running_insert_threads) - { - + { + # Time to stop other threads signal_abort(); - } + } } } $errors++ if ($ret != 0); @@ -214,17 +214,17 @@ print "Total time: " . exit(0); -# -# Sleep and then abort other threads -# - -sub test_abort -{ - sleep($opt_time); - signal_abort(); - exit(0); -} - +# +# Sleep and then abort other threads +# + +sub test_abort +{ + sleep($opt_time); + signal_abort(); + exit(0); +} + # # Insert records in the table @@ -363,58 +363,58 @@ sub test_join $dbh->disconnect; $dbh=0; print "Test_join: Executed $count joins\n"; exit(0); -} - -# -# select records -# Do continously joins between the first and second for range and count selected rows -# - -sub test_join_count -{ - my ($dbh, $i, $j, $count, $loop); - - $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", - $opt_user, $opt_password, - { PrintError => 0}) || die $DBI::errstr; - - $count_query=make_count_query($numtables); - $count=0; - $loop=9999; - $sum=0; - - srand(); - - $i=0; - while (($i++ % 10) || !test_if_abort($dbh)) - { - if ($loop++ >= 10) - { - $loop=0; - $row_counts=simple_query($dbh, $count_query); - } - for ($j=0 ; $j < $numtables-1 ; $j++) - { - my ($id1)= int rand $row_counts->[$j]; - my ($id2)= int rand $row_counts->[$j]; - if ($id1 > $id2) - { - my $id0=$id1; $id1=$id2; $id2=$id0; - if ($id2-$id1 > $opt_join_range) - { - $id2=$id1+$opt_join_range; - } - } - my ($t1,$t2)= ($testtables[$j]->[0],$testtables[$j+1]->[0]); - $row=simple_query($dbh, "select count(*) from $t1, $t2 where $t1.id=$t2.id and $t1.id between $id1 and $id2"); - $sum+=$row->[0]; - $count++; - } - } - $dbh->disconnect; $dbh=0; - print "Test_join_count: Executed $count joins: total $sum rows\n"; - exit(0); -} +} + +# +# select records +# Do continously joins between the first and second for range and count selected rows +# + +sub test_join_count +{ + my ($dbh, $i, $j, $count, $loop); + + $dbh = DBI->connect("DBI:mysql:$opt_db:$opt_host", + $opt_user, $opt_password, + { PrintError => 0}) || die $DBI::errstr; + + $count_query=make_count_query($numtables); + $count=0; + $loop=9999; + $sum=0; + + srand(); + + $i=0; + while (($i++ % 10) || !test_if_abort($dbh)) + { + if ($loop++ >= 10) + { + $loop=0; + $row_counts=simple_query($dbh, $count_query); + } + for ($j=0 ; $j < $numtables-1 ; $j++) + { + my ($id1)= int rand $row_counts->[$j]; + my ($id2)= int rand $row_counts->[$j]; + if ($id1 > $id2) + { + my $id0=$id1; $id1=$id2; $id2=$id0; + if ($id2-$id1 > $opt_join_range) + { + $id2=$id1+$opt_join_range; + } + } + my ($t1,$t2)= ($testtables[$j]->[0],$testtables[$j+1]->[0]); + $row=simple_query($dbh, "select count(*) from $t1, $t2 where $t1.id=$t2.id and $t1.id between $id1 and $id2"); + $sum+=$row->[0]; + $count++; + } + } + $dbh->disconnect; $dbh=0; + print "Test_join_count: Executed $count joins: total $sum rows\n"; + exit(0); +} #

1 0