At http://bazaar.launchpad.net/~maria-captains/maria/5.1/ ------------------------------------------------------------ revno: 2898 revision-id: sergii@pisem.net-20100811081549-2jasocti5y17ktr1 parent: sergii@pisem.net-20100810143604-zt0d174fb06vrsrj committer: Sergei Golubchik <sergii@pisem.net> branch nick: 5.1 timestamp: Wed 2010-08-11 12:15:49 +0400 message: generalization of mtr to support suite.pm extensions: * no automatic --loose-skip-innodb added by mtr based on the test name. instead loose-skip-innodb is now in the default_mysqld.cnf * have_innodb_plugin.inc is changed to give a verbose "skip" message (instead of "require: true") * My::Suite class. It's support in mtr, and everywhere * support for suite.pm * when sorting tests, take combinations into account * support for SUITENAME_COMBINATIONS * no special treatment for innodb_plugin in mtr_cases.pm * two special pre-created config groups: ENV and OPT * allow option names to start from # * allow magic option to have an argument * remove dead code * fix @-substitution to works as expected * new processes take the value of $opt_verbose automatically, no need to pass it to a constructor * innodb_plugin suite uses suite.pm and combinations file to test as much as possible (innodb plugin, xtradb plugin, xtradb static - whatever available) === added file 'mysql-test/README.suites' --- a/mysql-test/README.suites 1970-01-01 00:00:00 +0000 +++ b/mysql-test/README.suites 2010-08-11 08:15:49 +0000 @@ -0,0 +1,129 @@ +These are the assorted notes that will be turned into a manual eventually. + +========================== +Tests are organized in suites. +A "suite" is a subdirectory inside, one of, + + <basedir>/mysql-test/suite + <basedir>/mysql-test + <basedir>/share/mysql-test/suite + <basedir>/share/mysql-test + <basedir>/share/mysql/mysql-test/suite + <basedir>/share/mysql/mysql-test + <basedir>/storage/*/mysql-test-suites + +This is supposed to cover running mtr from a source directory and installed. + +========================== +A suite contains .test and .result files. They can be in the t/ and r/ +directories under the suitedir or directly in the suitedir. + +========================== +A suite can contain a suite.opt file - at the same location where .test +files are. As usual, the .opt file can use $-substitutions for the +environment variables. + +========================== +A suite can have suite.pm file in the suitedir. It must declare a +package that inherits from My::Suite. + +The suite.pm needs to have @ISA=qw(My::Suite) and it must end +with bless {}; - that is it must return an object of that class. + +A suite class can define config_files() and servers() methods. + +A config_files method returns a list of additional config files (besides +my.cnf), that this suite needs to be created. For every file it specifies +a function that will create it, when given a My::Config object. For example: + + sub config_files { ( 'config.ini' => \&write_ini, + 'new.conf' => \&do_new_conf ) } + +A servers method returns a list of processes that needs to be started for +this suite. A process is specified as a pair (regex, hash). A regex must +match a section in the my.cnf template (for example, qr/mysqld\./ corresponds +to all mysqld processes), a hash contains these options: + + SORT => a number, processes are started in the order of increasing SORT + values (and stopped in the reverse order). mysqld has number 300. + START => a function to start a process. It takes two arguments, + My::Config::Group and My::Test. If START is undefined the process + will not be started. + WAIT => a function waits for the process to be started. It takes + My::Config::Group as an argument. Internallys mtr first invokes + START for all processes, then WAIT for all started processes. + +example: sub servers { ( qr/^foo$/ => { SORT => 200, + START => \&start_foo, + WAIT => \&wait_foo } ) } + +See sphinx suite for an example. + +========================== +A suite can have my.cnf template file in the suitedir. +A my.cnf template uses a normal my.cnf syntax - with groups, options, +and values, with templating extansions. They are + +* There can be groups with non-standard names, not used by mysqld. + These groups may be used by the suite.pm file somehow. + For example, they can be written in the additional config files. + See sphinx suite for an example. + +* There can be ENV group. It sets values for the environment variables. + +* Values can refer to each other - they will be expanded as needed. + A reference to a value of option looks like @groupname.optionname. + For example + + [mysqld.2] + master-port= @mysqld.1.port + + it sets the master-port in the mysqld.2 group to the value of + port in the mysqld.1 group. + +* An option name may start from '#'. In the resulting my.cnf it will look + like a comment, but it still can be referred to. For example: + + datadir = @mysqld.1.#vardir/data + +* There are two special - in this regard - groups. + Via the ENV group one can refer to any environment variable. + Via the OPT group one can refer to special values: + @OPT.vardir - a path to vardir + @OPT.port - a new port number is reserved out of the pool. It will not + match any other port number used by this test run. + See sphinx suite for an example. + +Most probably a suite my.cnf will need to start from + + !include include/default_my.cnf + +and then modify the configuration as necessary. +========================== + +A suite can have combinations file in the suitedir. It uses my.cnf syntax +but it cannot use @-substitutions. Instead, it can use $-substitutions for +the environment variables. Because the combination options will not be +merged to a my.cnf, but will be added to the command line. Example: + + [conf1] + opt1=val1 + + [conf2] + opt1=val2 + opt2=$HAVE_SOMETHING + +Such a file will cause every test from the suite to be run twice - once +with mysqld using --opt1=val1 and the other one with mysqld using +--opt1=val2 --opt2=$HAVE_SOMETHING + +One can limit mtr run to a subset of combinations by setting environment +variable SUITENAME_COMBINATIONS to the ':'-separated set of combination +names. E.g. + + RPL_COMBINATIONS=mix:row ./mtr --suite rpl + +See innodb_plugin suite for an example how suite.pm may set this variable +to exclude unsupported configurations. + +========================== === modified file 'mysql-test/include/default_mysqld.cnf' --- a/mysql-test/include/default_mysqld.cnf 2010-03-28 18:10:00 +0000 +++ b/mysql-test/include/default_mysqld.cnf 2010-08-11 08:15:49 +0000 @@ -13,6 +13,7 @@ key_buffer_size= 1M sort_buffer= 256K max_heap_table_size= 1M +loose-skip-innodb loose-innodb_data_file_path= ibdata1:10M:autoextend slave-net-timeout=120 === modified file 'mysql-test/include/have_innodb_plugin.inc' --- a/mysql-test/include/have_innodb_plugin.inc 2010-08-03 21:26:17 +0000 +++ b/mysql-test/include/have_innodb_plugin.inc 2010-08-11 08:15:49 +0000 @@ -1,4 +1,5 @@ -disable_query_log; ---require r/true.require -SELECT (plugin_library LIKE 'ha_innodb_plugin%' OR plugin_description LIKE '%xtradb%') AS `TRUE` FROM information_schema.plugins WHERE LOWER(plugin_name) = 'innodb' AND LOWER(plugin_status) = 'active'; -enable_query_log; +if (!`SELECT COUNT(*) FROM INFORMATION_SCHEMA.PLUGINS + WHERE PLUGIN_NAME = 'innodb' AND PLUGIN_STATUS = 'active' AND + (PLUGIN_LIBRARY LIKE 'ha_innodb_plugin%' OR PLUGIN_DESCRIPTION LIKE '%xtradb%')`) { + skip Need InnoDB plugin or XtraDB; +} === modified file 'mysql-test/lib/My/Config.pm' --- a/mysql-test/lib/My/Config.pm 2008-09-05 13:31:09 +0000 +++ b/mysql-test/lib/My/Config.pm 2010-08-11 08:15:49 +0000 @@ -61,7 +60,7 @@ sub insert { $option->{value}= $value; } else { - my $option= My::Config::Option->new($option_name, $value); + $option= My::Config::Option->new($option_name, $value); # Insert option in list push(@{$self->{options}}, $option); # Insert option in hash @@ -163,6 +162,62 @@ sub if_exist { return $option->value(); } +package My::Config::Group::ENV; +our @ISA=qw(My::Config::Group); + +use strict; +use warnings; +use Carp; + +sub new { + my ($class, $group_name)= @_; + bless My::Config::Group->new($group_name), $class; +} + +# +# Return value for an option in the group, fail if it does not exist +# +sub value { + my ($self, $option_name)= @_; + my $option= $self->option($option_name); + + if (! defined($option) and $ENV{$option_name}) { + my $value= $ENV{$option_name}; + $option= My::Config::Option->new($option_name, $value); + } + + croak "No option named '$option_name' in group '$self->{name}'" + if ! defined($option); + + return $option->value(); +} + +package My::Config::Group::OPT; +our @ISA=qw(My::Config::Group); + +use strict; +use warnings; +use Carp; + +sub new { + my ($class, $group_name)= @_; + bless My::Config::Group->new($group_name), $class; +} + +sub options { + my ($self)= @_; + () +} + +sub value { + my ($self, $option_name)= @_; + my $option= $self->option($option_name); + + croak "No option named '$option_name' in group '$self->{name}'" + if ! defined($option); + + return $option->value()->(); +} package My::Config; @@ -182,7 +237,10 @@ sub new { my ($class, $path)= @_; my $group_name= undef; - my $self= bless { groups => [] }, $class; + my $self= bless { groups => [ + My::Config::Group::ENV->new('ENV'), + My::Config::Group::OPT->new('OPT'), + ] }, $class; my $F= IO::File->new($path, "<") or croak "Could not open '$path': $!"; @@ -199,19 +257,13 @@ sub new { } # Magic #! comments - elsif ( $line =~ /^#\!/) { - my $magic= $line; + elsif ( $line =~ /^(#\!\S+)(?:\s*(.*?)\s*)?$/) { + my ($magic, $arg)= ($1, $2); croak "Found magic comment '$magic' outside of group" unless $group_name; #print "$magic\n"; - $self->insert($group_name, $magic, undef); - } - - # Comments - elsif ( $line =~ /^#/ || $line =~ /^;/) { - # Skip comment - next; + $self->insert($group_name, $magic, $arg); } # Empty lines @@ -236,7 +288,7 @@ sub new { } # <option> - elsif ( $line =~ /^([\@\w-]+)\s*$/ ) { + elsif ( $line =~ /^(#?[\w-]+)\s*$/ ) { my $option= $1; croak "Found option '$option' outside of group" @@ -247,7 +299,7 @@ sub new { } # <option>=<value> - elsif ( $line =~ /^([\@\w-]+)\s*=\s*(.*?)\s*$/ ) { + elsif ( $line =~ /^(#?[\w-]+)\s*=\s*(.*?)\s*$/ ) { my $option= $1; my $value= $2; @@ -256,10 +308,17 @@ sub new { #print "$option=$value\n"; $self->insert($group_name, $option, $value); - } else { - croak "Unexpected line '$line' found in '$path'"; } + # Comments + elsif ( $line =~ /^#/ || $line =~ /^;/) { + # Skip comment + next; + } + + else { + croak "Unexpected line '$line' found in '$path'"; + } } undef $F; # Close the file @@ -437,44 +496,4 @@ sub exists { return defined($option); } - -# Overload "to string"-operator with 'stringify' -use overload - '""' => \&stringify; - -# -# Return the config as a string in my.cnf file format -# -sub stringify { - my ($self)= @_; - my $res; - - foreach my $group ($self->groups()) { - $res .= "[$group->{name}]\n"; - - foreach my $option ($group->options()) { - $res .= $option->name(); - my $value= $option->value(); - if (defined $value) { - $res .= "=$value"; - } - $res .= "\n"; - } - $res .= "\n"; - } - return $res; -} - - -# -# Save the config to named file -# -sub save { - my ($self, $path)= @_; - my $F= IO::File->new($path, ">") - or croak "Could not open '$path': $!"; - print $F $self; - undef $F; # Close the file -} - 1; === modified file 'mysql-test/lib/My/ConfigFactory.pm' --- a/mysql-test/lib/My/ConfigFactory.pm 2010-04-28 12:52:24 +0000 +++ b/mysql-test/lib/My/ConfigFactory.pm 2010-08-11 08:15:49 +0000 @@ -57,16 +57,12 @@ sub fix_pidfile { sub fix_port { my ($self, $config, $group_name, $group)= @_; - my $hostname= $group->value('#host'); - return $self->{HOSTS}->{$hostname}++; + return $self->{PORT}++; } sub fix_host { my ($self)= @_; - # Get next host from HOSTS array - my @hosts= keys(%{$self->{HOSTS}});; - my $host_no= $self->{NEXT_HOST}++ % @hosts; - return $hosts[$host_no]; + 'localhost' } sub is_unique { @@ -229,7 +225,7 @@ if (IS_WINDOWS) sub fix_ndb_mgmd_port { my ($self, $config, $group_name, $group)= @_; my $hostname= $group->value('HostName'); - return $self->{HOSTS}->{$hostname}++; + return $self->{PORT}++; } @@ -428,20 +424,24 @@ sub post_check_embedded_group { sub resolve_at_variable { my ($self, $config, $group, $option)= @_; + local $_ = $option->value(); + my ($res, $after); - # Split the options value on last . - my @parts= split(/\./, $option->value()); - my $option_name= pop(@parts); - my $group_name= join('.', @parts); - - $group_name =~ s/^\@//; # Remove at - - my $from_group= $config->group($group_name) - or croak "There is no group named '$group_name' that ", - "can be used to resolve '$option_name'"; + while (m/(.*?)\@((?:\w+\.)+)(#?[-\w]+)/g) { + my ($before, $group_name, $option_name)= ($1, $2, $3); + $after = $'; + chop($group_name); - my $from= $from_group->value($option_name); - $config->insert($group->name(), $option->name(), $from) + my $from_group= $config->group($group_name) + or croak "There is no group named '$group_name' that ", + "can be used to resolve '$option_name'"; + + my $value= $from_group->value($option_name); + $res .= $before.$value; + } + $res .= $after; + + $config->insert($group->name(), $option->name(), $res) } @@ -453,7 +453,7 @@ sub post_fix_resolve_at_variables { next unless defined $option->value(); $self->resolve_at_variable($config, $group, $option) - if ($option->value() =~ /^\@/); + if ($option->value() =~ /\@/); } } } @@ -595,28 +595,18 @@ sub new_config { croak "you must pass '$required'" unless defined $args->{$required}; } - # Fill in hosts/port hash - my $hosts= {}; - my $baseport= $args->{baseport}; - $args->{hosts}= [ 'localhost' ] unless exists($args->{hosts}); - foreach my $host ( @{$args->{hosts}} ) { - $hosts->{$host}= $baseport; - } - # Open the config template my $config= My::Config->new($args->{'template_path'}); - my $extra_template_path= $args->{'extra_template_path'}; - if ($extra_template_path){ - $config->append(My::Config->new($extra_template_path)); - } my $self= bless { CONFIG => $config, ARGS => $args, - HOSTS => $hosts, - NEXT_HOST => 0, + PORT => $args->{baseport}, SERVER_ID => 1, }, $class; + # add auto-options + $config->insert('OPT', 'port' => sub { fix_port($self, $config) }); + $config->insert('OPT', 'vardir' => sub { shift->{ARGS}->{vardir} }); { # Run pre rules === modified file 'mysql-test/lib/My/Handles.pm' (properties changed: +x to -x) === modified file 'mysql-test/lib/My/SafeProcess.pm' --- a/mysql-test/lib/My/SafeProcess.pm 2010-04-28 12:52:24 +0000 +++ b/mysql-test/lib/My/SafeProcess.pm 2010-08-11 08:15:49 +0000 @@ -120,7 +120,7 @@ sub new { my $input = delete($opts{'input'}); my $output = delete($opts{'output'}); my $error = delete($opts{'error'}); - my $verbose = delete($opts{'verbose'}); + my $verbose = delete($opts{'verbose'}) || $::opt_verbose; my $nocore = delete($opts{'nocore'}); my $host = delete($opts{'host'}); my $shutdown = delete($opts{'shutdown'}); === added file 'mysql-test/lib/My/Suite.pm' --- a/mysql-test/lib/My/Suite.pm 1970-01-01 00:00:00 +0000 +++ b/mysql-test/lib/My/Suite.pm 2010-08-11 08:15:49 +0000 @@ -0,0 +1,10 @@ +# A default suite class that is used for all suites without their owns suite.pm +# see README.suites for a description + +package My::Suite; + +sub config_files { () } +sub servers { () } + +bless { }; + === modified file 'mysql-test/lib/My/Test.pm' --- a/mysql-test/lib/My/Test.pm 2009-03-20 14:18:22 +0000 +++ b/mysql-test/lib/My/Test.pm 2010-08-11 08:15:49 +0000 @@ -20,6 +20,12 @@ sub new { return $self; } +sub fullname { + my ($self)= @_; + $self->{name} . (defined $self->{combination} + ? " '$self->{combination}'" + : "") +} # # Return a unique key that can be used to === modified file 'mysql-test/lib/mtr_cases.pm' --- a/mysql-test/lib/mtr_cases.pm 2010-08-03 21:26:17 +0000 +++ b/mysql-test/lib/mtr_cases.pm 2010-08-11 08:15:49 +0000 @@ -39,7 +39,6 @@ our $enable_disabled; our $default_storage_engine; our $opt_with_ndbcluster_only; our $defaults_file; -our $defaults_extra_file; our $quick_collect; sub collect_option { @@ -68,21 +67,10 @@ require "mtr_misc.pl"; my $do_test_reg; my $skip_test_reg; -# Related to adding InnoDB plugin combinations -my $lib_innodb_plugin; - # If "Quick collect", set to 1 once a test to run has been found. my $some_test_found; -sub find_innodb_plugin { - $lib_innodb_plugin= - my_find_file($::basedir, - ["storage/innodb_plugin", "storage/innodb_plugin/.libs", - "lib/mysql/plugin", "lib/plugin"], - ["ha_innodb_plugin.dll", "ha_innodb_plugin.so", - "ha_innodb_plugin.sl"], - NOT_REQUIRED); -} +my $default_suite_object = do 'My/Suite.pm'; sub init_pattern { my ($from, $what)= @_; @@ -116,8 +104,6 @@ sub collect_test_cases ($$$) { $do_test_reg= init_pattern($do_test, "--do-test"); $skip_test_reg= init_pattern($skip_test, "--skip-test"); - &find_innodb_plugin; - # If not reordering, we also shouldn't group by suites, unless # no test cases were named. # This also effects some logic in the loop following this. @@ -188,17 +174,17 @@ sub collect_test_cases ($$$) { my $opts= $tinfo->{'master_opt'} ? $tinfo->{'master_opt'} : []; push(@criteria, join("!", sort @{$opts}) . "~"); - $sort_criteria{$tinfo->{name}} = join(" ", @criteria); + $sort_criteria{$tinfo->fullname()} = join(" ", @criteria); } @$cases = sort { - $sort_criteria{$a->{'name'}} . $a->{'name'} cmp - $sort_criteria{$b->{'name'}} . $b->{'name'}; } @$cases; + $sort_criteria{$a->fullname()} . $a->fullname() cmp + $sort_criteria{$b->fullname()} . $b->fullname() } @$cases; # For debugging the sort-order # foreach my $tinfo (@$cases) # { - # print("$sort_criteria{$tinfo->{'name'}} -> \t$tinfo->{'name'}\n"); + # print $sort_criteria{$tinfo->fullname()}," -> \t",$tinfo->fullname(),"\n"; # } } @@ -310,6 +296,17 @@ sub collect_one_suite mtr_verbose("testdir: $testdir"); mtr_verbose("resdir: $resdir"); + # + # Load the Suite object + # + unless ($::suites{$suite}) { + if (-f "$suitedir/suite.pm") { + $::suites{$suite} = do "$suitedir/suite.pm"; + } else { + $::suites{$suite} = $default_suite_object; + } + } + # ---------------------------------------------------------------------- # Build a hash of disabled testcases for this suite # ---------------------------------------------------------------------- @@ -438,14 +435,16 @@ sub collect_one_suite { # Read combinations file in my.cnf format mtr_verbose("Read combinations file"); + my %env_filter = map { $_ => 1 } split /:/, $ENV{"\U${suite}_COMBINATIONS"}; my $config= My::Config->new($combination_file); foreach my $group ($config->groups()) { my $comb= {}; $comb->{name}= $group->name(); + next if %env_filter and not $env_filter{$comb->{name}}; foreach my $option ( $group->options() ) { push(@{$comb->{comb_opt}}, $option->option()); } - push(@combinations, $comb); + push(@combinations, $comb) if $comb->{comb_opt}; } } @@ -758,7 +757,7 @@ sub collect_one_test_case { name => "$suitename.$tname", shortname => $tname, path => "$testdir/$filename", - + suite => $suitename, ); my $result_file= "$resdir/$tname.result"; @@ -956,48 +955,6 @@ sub collect_one_test_case { push(@{$tinfo->{'slave_opt'}}, "--loose-federated"); } - if ( $tinfo->{'innodb_test'} ) - { - # This is a test that needs innodb - if ( $::mysqld_variables{'innodb'} eq "OFF" || - ! exists $::mysqld_variables{'innodb'} ) - { - # innodb is not supported, skip it - $tinfo->{'skip'}= 1; - $tinfo->{'comment'}= "No innodb support"; - return $tinfo; - } - } - elsif ( $tinfo->{'innodb_plugin_test'} ) - { - # This is a test that needs the innodb plugin - if (&find_innodb_plugin) - { - my $sep= (IS_WINDOWS) ? ';' : ':'; - my $plugin_filename= basename($lib_innodb_plugin); - my $plugin_list= - "innodb=$plugin_filename$sep" . - "innodb_trx=$plugin_filename$sep" . - "innodb_locks=$plugin_filename$sep" . - "innodb_lock_waits=$plugin_filename$sep" . - "innodb_cmp=$plugin_filename$sep" . - "innodb_cmp_reset=$plugin_filename$sep" . - "innodb_cmpmem=$plugin_filename$sep" . - "innodb_cmpmem_reset=$plugin_filename"; - - foreach my $k ('master_opt', 'slave_opt') { - push(@{$tinfo->{$k}}, '--ignore-builtin-innodb'); - push(@{$tinfo->{$k}}, '--plugin-dir=' . dirname($lib_innodb_plugin)); - push(@{$tinfo->{$k}}, "--plugin-load=$plugin_list"); - } - } - } - else - { - push(@{$tinfo->{'master_opt'}}, "--loose-skip-innodb"); - push(@{$tinfo->{'slave_opt'}}, "--loose-skip-innodb"); - } - if ( $tinfo->{'need_binlog'} ) { if (grep(/^--skip-log-bin/, @::opt_extra_mysqld_opt) ) @@ -1117,12 +1074,6 @@ sub collect_one_test_case { } } - - # Set extra config file to use - if (defined $defaults_extra_file) { - $tinfo->{extra_template_path}= $defaults_extra_file; - } - # ---------------------------------------------------------------------- # Append mysqld extra options to both master and slave # ---------------------------------------------------------------------- @@ -1160,10 +1111,6 @@ my @tags= ["include/have_log_bin.inc", "need_binlog", 1], ["include/have_innodb.inc", "innodb_test", 1], - ["include/have_innodb_plugin.inc", "innodb_plugin_test", 1], - ["include/have_real.inc", "innodb_test", 1], - ["include/have_real_innodb_plugin.inc", "innodb_plugin_test", 1], - ["include/have_xtradb.inc", "innodb_test", 1], ["include/have_pbxt.inc", "pbxt_test", 1], ["include/big_test.inc", "big_test", 1], ["include/have_debug.inc", "need_debug", 1], === modified file 'mysql-test/lib/mtr_report.pm' --- a/mysql-test/lib/mtr_report.pm 2010-04-28 12:52:24 +0000 +++ b/mysql-test/lib/mtr_report.pm 2010-08-11 08:15:49 +0000 @@ -61,14 +61,10 @@ sub _name { sub _mtr_report_test_name ($) { my $tinfo= shift; - my $tname= $tinfo->{name}; + my $tname= $tinfo->fullname(); return unless defined $verbose; - # Add combination name if any - $tname.= " '$tinfo->{combination}'" - if defined $tinfo->{combination}; - print _name(). _timestamp(); printf "%-40s ", $tname; my $worker = $tinfo->{worker}; === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2010-08-10 14:36:04 +0000 +++ b/mysql-test/mysql-test-run.pl 2010-08-11 08:15:49 +0000 @@ -130,7 +130,7 @@ my $path_config_file; # The ge # executables will be used by the test suite. our $opt_vs_config = $ENV{'MTR_VS_CONFIG'}; -my $DEFAULT_SUITES= "main,binlog,federated,rpl,maria,parts,innodb,innodb_plugin,percona"; +my $DEFAULT_SUITES= "main,binlog,federated,rpl,maria,parts,innodb,innodb_plugin,percona,ndb"; my $opt_suites; our $opt_verbose= 0; # Verbose output, enable with --verbose @@ -257,6 +257,8 @@ our $debug_compiled_binaries; our %mysqld_variables; +our %suites; + my $source_dist= 0; my $opt_max_save_core= env_or_val(MTR_MAX_SAVE_CORE => 5); @@ -807,7 +809,7 @@ sub run_worker ($) { # We need to gracefully shut down the servers to see any # Valgrind memory leak errors etc. since last server restart. if ($opt_warnings) { - stop_servers(all_servers()); + stop_servers(reverse all_servers()); if(check_warnings_post_shutdown($server)) { # Warnings appeared in log file(s) during final server shutdown. exit(1); @@ -1595,7 +1597,7 @@ sub collect_mysqld_features { my $cmd= join(" ", $exe_mysqld, @$args); my $list= `$cmd`; - print "cmd: $cmd\n"; + mtr_verbose("cmd: $cmd"); foreach my $line (split('\n', $list)) { @@ -2493,7 +2495,7 @@ sub check_ndbcluster_support ($) { } -sub ndbcluster_wait_started($$){ +sub ndbcluster_wait_started { my $cluster= shift; my $ndb_waiter_extra_opt= shift; my $path_waitlog= join('/', $opt_vardir, $cluster->name(), "ndb_waiter.log"); @@ -2661,7 +2663,7 @@ sub ndbd_start { sub ndbcluster_start ($) { - my $cluster= shift; + my ($cluster) = @_; mtr_verbose("ndbcluster_start '".$cluster->name()."'"); @@ -2681,6 +2683,109 @@ sub ndbcluster_start ($) { } +sub mysql_server_start($) { + my ($mysqld, $tinfo) = @_; + + if ( $mysqld->{proc} ) + { + # Already started + + # Write start of testcase to log file + mark_log($mysqld->value('#log-error'), $tinfo); + + return; + } + + my $datadir= $mysqld->value('datadir'); + if (not $opt_start_dirty) + { + + my @options= ('log-bin', 'relay-log'); + foreach my $option_name ( @options ) { + next unless $mysqld->option($option_name); + + my $file_name= $mysqld->value($option_name); + next unless + defined $file_name and + -e $file_name; + + mtr_debug(" -removing '$file_name'"); + unlink($file_name) or die ("unable to remove file '$file_name'"); + } + + if (-d $datadir ) { + preserve_error_log($mysqld); + mtr_verbose(" - removing '$datadir'"); + rmtree($datadir); + } + } + + my $mysqld_basedir= $mysqld->value('basedir'); + if ( $basedir eq $mysqld_basedir ) + { + if (! $opt_start_dirty) # If dirty, keep possibly grown system db + { + # Copy datadir from installed system db + for my $path ( "$opt_vardir", "$opt_vardir/..") { + my $install_db= "$path/install.db"; + copytree($install_db, $datadir) + if -d $install_db; + } + mtr_error("Failed to copy system db to '$datadir'") + unless -d $datadir; + } + } + else + { + mysql_install_db($mysqld); # For versional testing + + mtr_error("Failed to install system db to '$datadir'") + unless -d $datadir; + + } + restore_error_log($mysqld); + + # Create the servers tmpdir + my $tmpdir= $mysqld->value('tmpdir'); + mkpath($tmpdir) unless -d $tmpdir; + + # Write start of testcase to log file + mark_log($mysqld->value('#log-error'), $tinfo); + + # Run <tname>-master.sh + if ($mysqld->option('#!run-master-sh') and + run_sh_script($tinfo->{master_sh}) ) + { + $tinfo->{'comment'}= "Failed to execute '$tinfo->{master_sh}'"; + return 1; + } + + # Run <tname>-slave.sh + if ($mysqld->option('#!run-slave-sh') and + run_sh_script($tinfo->{slave_sh})) + { + $tinfo->{'comment'}= "Failed to execute '$tinfo->{slave_sh}'"; + return 1; + } + + if (!$opt_embedded_server) + { + my $extra_opts= get_extra_opts($mysqld, $tinfo); + mysqld_start($mysqld,$extra_opts); + + # Save this test case information, so next can examine it + $mysqld->{'started_tinfo'}= $tinfo; + } +} + +sub mysql_server_wait { + my ($mysqld) = @_; + + return not sleep_until_file_created($mysqld->value('pid-file'), + $opt_start_timeout, + $mysqld->{'proc'}); +} + sub create_config_file_for_extern { my %opts= ( @@ -3458,6 +3563,75 @@ sub timezone { return $tinfo->{timezone} || "DEFAULT"; } +sub mycnf_create { + my ($config) = @_; + my $res; + + foreach my $group ($config->groups()) { + $res .= "[$group->{name}]\n"; + + foreach my $option ($group->options()) { + $res .= $option->name(); + my $value= $option->value(); + if (defined $value) { + $res .= "=$value"; + } + $res .= "\n"; + } + $res .= "\n"; + } + $res; +} + +sub config_files($) { + my ($tinfo) = @_; + ( + 'my.cnf' => \&mycnf_create, + $suites{$tinfo->{suite}}->config_files() + ); +} + +sub _like { return $config ? $config->like($_[0]) : (); } +sub mysqlds { return _like('mysqld\.'); } +sub ndbds { return _like('cluster_config\.ndbd\.');} +sub ndb_mgmds { return _like('cluster_config\.ndb_mgmd\.'); } + +sub fix_servers($) { + my ($tinfo) = @_; + return () unless $config; + my %servers = ( + qr/mysqld\./ => { + SORT => 300, + START => \&mysql_server_start, + WAIT => \&mysql_server_wait, + }, + qr/mysql_cluster\./ => { + SORT => 200, + START => \&ndbcluster_start, + WAIT => \&ndbcluster_wait_started, + }, + qr/cluster_config\.ndb_mgmd\./ => { + SORT => 210, + START => undef, + }, + qr/cluster_config\.ndbd\./ => { + SORT => 220, + START => undef, + }, + $suites{$tinfo->{suite}}->servers() + ); + for ($config->groups()) { + while (my ($re,$prop) = each %servers) { + @$_{keys %$prop} = values %$prop if $_->{name} =~ /^$re/; + } + } +} + +sub all_servers { + return unless $config; + ( sort { $a->{SORT} <=> $b->{SORT} } + grep { defined $_->{SORT} } $config->groups() ); +} # Storage for changed environment variables my %old_env; @@ -3500,7 +3674,7 @@ sub run_testcase ($$) { if ( @restart != 0) { # Remember that we restarted for this test case (count restarts) $tinfo->{'restarted'}= 1; - stop_servers(@restart ); + stop_servers(reverse @restart); if ($opt_warnings) { check_warnings_post_shutdown($server_socket); } @@ -3536,7 +3710,6 @@ sub run_testcase ($$) { vardir => $opt_vardir, tmpdir => $opt_tmpdir, baseport => $baseport, - #hosts => [ 'host1', 'host2' ], user => $opt_user, password => '', ssl => $opt_ssl_supported, @@ -3544,8 +3717,16 @@ sub run_testcase ($$) { } ); - # Write the new my.cnf - $config->save($path_config_file); + fix_servers($tinfo); + + # Write config files: + my %config_files = config_files($tinfo); + while (my ($file, $generate) = each %config_files) { + my ($path) = "$opt_vardir/$file"; + open (F, '>', $path) or die "Could not open '$path': $!"; + print F &$generate($config); + close F; + } # Remember current config so a restart can occur when a test need # to use a different one @@ -3688,7 +3869,7 @@ sub run_testcase ($$) { if ($opt_warnings) { # Checking error logs for warnings, so need to stop server # gracefully so that memory leaks etc. can be properly detected. - stop_servers(all_servers()); + stop_servers(reverse all_servers()); check_warnings_post_shutdown($server_socket); # Even if we got warnings here, we should not fail this # particular test, as the warnings may be caused by an earlier @@ -3846,7 +4027,8 @@ sub run_testcase ($$) { # valuable debugging information even if there is no test failure recorded. sub _preserve_error_log_names { my ($mysqld)= @_; - my $error_log_file= $mysqld->value('#log-error'); + my $error_log_file= $mysqld->if_exist('#log-error'); + return unless $error_log_file and -r $error_log_file; my $error_log_dir= dirname($error_log_file); my $save_name= $error_log_dir ."/../". $mysqld->name() .".error.log"; return ($error_log_file, $save_name); @@ -3855,14 +4037,14 @@ sub _preserve_error_log_names { sub preserve_error_log { my ($mysqld)= @_; my ($error_log_file, $save_name)= _preserve_error_log_names($mysqld); - my $res= rename($error_log_file, $save_name); + rename($error_log_file, $save_name) if $save_name; # Ignore any errors, as it's just a best-effort to keep the log if possible. } sub restore_error_log { my ($mysqld)= @_; my ($error_log_file, $save_name)= _preserve_error_log_names($mysqld); - my $res= rename($save_name, $error_log_file); + rename($save_name, $error_log_file) if $save_name; } # Keep track of last position in mysqld error log where we scanned for @@ -3906,6 +4088,8 @@ sub pre_write_errorlog { sub extract_server_log ($$) { my ($error_log, $tname) = @_; + + return unless $error_log; # Open the servers .err log file and read all lines # belonging to current test into @lines @@ -3948,9 +4132,9 @@ sub get_log_from_proc ($$) { my ($proc, $name)= @_; my $srv_log= ""; - foreach my $mysqld (mysqlds()) { + foreach my $mysqld (all_servers()) { if ($mysqld->{proc} eq $proc) { - my @srv_lines= extract_server_log($mysqld->value('#log-error'), $name); + my @srv_lines= extract_server_log($mysqld->if_exist('#log-error'), $name); $srv_log= "\nServer log from this test:\n" . join ("", @srv_lines); last; } @@ -4029,6 +4213,7 @@ sub extract_warning_lines ($) { my @antipatterns = ( qr/error .*connecting to master/, + qr/Plugin 'ndbcluster' will be forced to shutdown/, qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/, qr/InnoDB: Error: table `test`.`t[12]` does not exist in the InnoDB internal/, qr/Slave: Unknown table 't1' Error_code: 1051/, @@ -4354,22 +4538,12 @@ sub clean_datadir { mtr_error("Trying to clean datadir before all servers stopped"); } - foreach my $cluster ( clusters() ) - { - my $cluster_dir= "$opt_vardir/".$cluster->{name}; - mtr_verbose(" - removing '$cluster_dir'"); - rmtree($cluster_dir); - - } - - foreach my $mysqld ( mysqlds() ) + for (all_servers()) { - my $mysqld_dir= dirname($mysqld->value('datadir')); - preserve_error_log($mysqld); - if (-d $mysqld_dir ) { - mtr_verbose(" - removing '$mysqld_dir'"); - rmtree($mysqld_dir); - } + preserve_error_log($_); # or at least, try to + my $dir= "$opt_vardir/".$_->{name}; + mtr_verbose(" - removing '$dir'"); + rmtree($dir); } # Remove all files in tmp and var/tmp @@ -4392,17 +4566,6 @@ sub save_datadir_after_failure($$) { } -sub remove_ndbfs_from_ndbd_datadir { - my ($ndbd_datadir)= @_; - # Remove the ndb_*_fs directory from ndbd.X/ dir - foreach my $ndbfs_dir ( glob("$ndbd_datadir/ndb_*_fs") ) - { - next unless -d $ndbfs_dir; # Skip if not a directory - rmtree($ndbfs_dir); - } -} - - sub after_failure ($) { my ($tinfo)= @_; @@ -4419,31 +4582,18 @@ sub after_failure ($) { mkpath($save_dir) if ! -d $save_dir; - # Save the used my.cnf file - copy($path_config_file, $save_dir); + # Save the used config files + my %config_files = config_files($tinfo); + while (my ($file, $generate) = each %config_files) { + copy("$opt_vardir/$file", $save_dir); + } # Copy the tmp dir copytree("$opt_vardir/tmp/", "$save_dir/tmp/"); - if ( clusters() ) { - foreach my $cluster ( clusters() ) { - my $cluster_dir= "$opt_vardir/".$cluster->{name}; - - # Remove the fileystem of each ndbd - foreach my $ndbd ( in_cluster($cluster, ndbds()) ) - { - my $ndbd_datadir= $ndbd->value("DataDir"); - remove_ndbfs_from_ndbd_datadir($ndbd_datadir); - } - - save_datadir_after_failure($cluster_dir, $save_dir); - } - } - else { - foreach my $mysqld ( mysqlds() ) { - my $data_dir= $mysqld->value('datadir'); - save_datadir_after_failure(dirname($data_dir), $save_dir); - } + foreach (all_servers()) { + my $dir= "$opt_vardir/".$_->{name}; + save_datadir_after_failure($dir, $save_dir); } } @@ -4838,8 +4988,7 @@ sub server_need_restart { return 1; } - my $is_mysqld= grep ($server eq $_, mysqlds()); - if ($is_mysqld) + if ($server->name() =~ /^mysqld\./) { # Check that running process was started with same options @@ -4891,17 +5040,9 @@ sub servers_need_restart($) { -# -# Return list of specific servers -# - there is no servers in an empty config -# -sub _like { return $config ? $config->like($_[0]) : (); } -sub mysqlds { return _like('mysqld.'); } -sub ndbds { return _like('cluster_config.ndbd.');} -sub ndb_mgmds { return _like('cluster_config.ndb_mgmd.'); } -sub clusters { return _like('mysql_cluster.'); } -sub all_servers { return ( mysqlds(), ndb_mgmds(), ndbds() ); } +############################################ +############################################ # # Filter a list of servers and return only those that are part @@ -4955,28 +5096,10 @@ sub get_extra_opts { sub stop_servers($$) { my (@servers)= @_; - if ( join('|', @servers) eq join('|', all_servers()) ) - { - # All servers are going down, use some kind of order to - # avoid too many warnings in the log files + mtr_report("Restarting ", started(@servers)); - mtr_report("Restarting all servers"); - - # mysqld processes - My::SafeProcess::shutdown( $opt_shutdown_timeout, started(mysqlds()) ); - - # cluster processes - My::SafeProcess::shutdown( $opt_shutdown_timeout, - started(ndbds(), ndb_mgmds()) ); - } - else - { - mtr_report("Restarting ", started(@servers)); - - # Stop only some servers - My::SafeProcess::shutdown( $opt_shutdown_timeout, - started(@servers) ); - } + My::SafeProcess::shutdown($opt_shutdown_timeout, + started(@servers)); foreach my $server (@servers) { @@ -5003,145 +5126,14 @@ sub stop_servers($$) { sub start_servers($) { my ($tinfo)= @_; - # Start clusters - foreach my $cluster ( clusters() ) - { - ndbcluster_start($cluster); + for (all_servers()) { + $_->{START}->($_, $tinfo) if $_->{START}; } - # Start mysqlds - foreach my $mysqld ( mysqlds() ) - { - if ( $mysqld->{proc} ) - { - # Already started - - # Write start of testcase to log file - mark_log($mysqld->value('#log-error'), $tinfo); - - next; - } - - my $datadir= $mysqld->value('datadir'); - if ($opt_start_dirty) - { - # Don't delete anything if starting dirty - ; - } - else - { - - my @options= ('log-bin', 'relay-log'); - foreach my $option_name ( @options ) { - next unless $mysqld->option($option_name); - - my $file_name= $mysqld->value($option_name); - next unless - defined $file_name and - -e $file_name; - - mtr_debug(" -removing '$file_name'"); - unlink($file_name) or die ("unable to remove file '$file_name'"); - } - - if (-d $datadir ) { - preserve_error_log($mysqld); - mtr_verbose(" - removing '$datadir'"); - rmtree($datadir); - } - } - - my $mysqld_basedir= $mysqld->value('basedir'); - if ( $basedir eq $mysqld_basedir ) - { - if (! $opt_start_dirty) # If dirty, keep possibly grown system db - { - # Copy datadir from installed system db - for my $path ( "$opt_vardir", "$opt_vardir/..") { - my $install_db= "$path/install.db"; - copytree($install_db, $datadir) - if -d $install_db; - } - mtr_error("Failed to copy system db to '$datadir'") - unless -d $datadir; - } - } - else - { - mysql_install_db($mysqld); # For versional testing - - mtr_error("Failed to install system db to '$datadir'") - unless -d $datadir; - - } - restore_error_log($mysqld); - - # Create the servers tmpdir - my $tmpdir= $mysqld->value('tmpdir'); - mkpath($tmpdir) unless -d $tmpdir; - - # Write start of testcase to log file - mark_log($mysqld->value('#log-error'), $tinfo); - - # Run <tname>-master.sh - if ($mysqld->option('#!run-master-sh') and - run_sh_script($tinfo->{master_sh}) ) - { - $tinfo->{'comment'}= "Failed to execute '$tinfo->{master_sh}'"; - return 1; - } - - # Run <tname>-slave.sh - if ($mysqld->option('#!run-slave-sh') and - run_sh_script($tinfo->{slave_sh})) - { - $tinfo->{'comment'}= "Failed to execute '$tinfo->{slave_sh}'"; - return 1; - } - - if (!$opt_embedded_server) - { - my $extra_opts= get_extra_opts($mysqld, $tinfo); - mysqld_start($mysqld,$extra_opts); - - # Save this test case information, so next can examine it - $mysqld->{'started_tinfo'}= $tinfo; - } - - } - - # Wait for clusters to start - foreach my $cluster ( clusters() ) - { - if (ndbcluster_wait_started($cluster, "")) - { - # failed to start - $tinfo->{'comment'}= "Start of '".$cluster->name()."' cluster failed"; - return 1; - } - } - - # Wait for mysqlds to start - foreach my $mysqld ( mysqlds() ) - { - next if !started($mysqld); - - if (sleep_until_file_created($mysqld->value('pid-file'), - $opt_start_timeout, - $mysqld->{'proc'}) == 0) { - $tinfo->{comment}= - "Failed to start ".$mysqld->name(); - - my $logfile= $mysqld->value('#log-error'); - if ( defined $logfile and -f $logfile ) - { - my @srv_lines= extract_server_log($logfile, $tinfo->{name}); - $tinfo->{logfile}= "Server log is:\n" . join ("", @srv_lines); - } - else - { - $tinfo->{logfile}= "Could not open server logfile: '$logfile'"; - } + for (all_servers()) { + next unless $_->{WAIT} and started($_); + if ($_->{WAIT}->($_)) { + $tinfo->{comment}= "Failed to start ".$_->name(); return 1; } } === added file 'mysql-test/suite/innodb_plugin/combinations' --- a/mysql-test/suite/innodb_plugin/combinations 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/innodb_plugin/combinations 2010-08-11 08:15:49 +0000 @@ -0,0 +1,12 @@ +[innodb_plugin] +ignore-builtin-innodb +plugin-load=$HA_INNODB_PLUGIN_SO +innodb + +[xtradb_plugin] +ignore-builtin-innodb +plugin-load=$HA_XTRADB_SO +innodb + +[xtradb] +innodb === added file 'mysql-test/suite/innodb_plugin/suite.pm' --- a/mysql-test/suite/innodb_plugin/suite.pm 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/innodb_plugin/suite.pm 2010-08-11 08:15:49 +0000 @@ -0,0 +1,26 @@ +package My::Suite::InnoDB_plugin; + +@ISA = qw(My::Suite); + +############# initialization ###################### + +my @combinations=('none'); + +if ($ENV{HA_INNODB_PLUGIN_SO}) { + push @combinations, 'innodb_plugin'; +} + +if ($ENV{HA_XTRADB_SO}) { + push @combinations, 'xtradb_plugin'; +} + +if ($::mysqld_variables{'innodb'} eq "ON") { + push @combinations, 'xtradb'; +} + +$ENV{INNODB_PLUGIN_COMBINATIONS}=join ':', @combinations + unless $ENV{INNODB_PLUGIN_COMBINATIONS}; + +############# return an object ###################### +bless { }; + === modified file 'mysql-test/suite/innodb_plugin/t/innodb_bug30919-master.opt' --- a/mysql-test/suite/innodb_plugin/t/innodb_bug30919-master.opt 2010-06-03 09:48:59 +0000 +++ b/mysql-test/suite/innodb_plugin/t/innodb_bug30919-master.opt 2010-08-11 08:15:49 +0000 @@ -1 +1 @@ ---innodb --innodb_autoinc_lock_mode=0 +--innodb_autoinc_lock_mode=0