At file:///Users/hakan/work/monty_program/mariadb-tools/ ------------------------------------------------------------ revno: 8 [merge] revision-id: hakan@askmonty.org-20100204111840-0d9u3nyio2h8tavi parent: hakan@askmonty.org-20100204011956-u74a4es6oogvd50w parent: knielsen@hasky-20100202151012-pjcblb2v3bqsp5eo committer: Hakan Kuecuekyilmaz <hakan@askmonty.org> branch nick: mariadb-tools timestamp: Thu 2010-02-04 12:18:40 +0100 message: Merge. modified: buildbot/maria-master.cfg mariamaster.cfg-20091218103450-cvifjz3i70oerkej-1 buildbot/runvm runvm-20091218082659-ept0jpsqa5e8jno6-2 === modified file 'buildbot/maria-master.cfg' --- a/buildbot/maria-master.cfg 2010-01-02 21:17:58 +0000 +++ b/buildbot/maria-master.cfg 2010-02-02 15:10:12 +0000 @@ -59,7 +59,6 @@ ,mkSlave("psergey-pylon-amd64", max_builds=1) ,mkSlave("psergey-foxhole-x86", max_builds=1) ,mkSlave("psergey-pslp2-x86", max_builds=1) - ,mkSlave("hakan-mac-g5", max_builds=1) ,mkSlave("work-opensuse-amd64", max_builds=1) ,mkSlave("psergey-win32box", max_builds=1) ,mkSlave("vm-win-1", max_builds=1) @@ -114,7 +113,7 @@ "lp:~maria-captains/maria/maria-5.2-merge-5.1" : "maria-5.2-merge-5.1", "lp:~maria-captains/maria/5.2-dsmrr" : "5.2-dsmrr", "lp:~maria-captains/maria/5.3" : "5.3", - "lp:~maria-captains/maria/5.3" : "5.3-sj-subqueries" + "lp:~maria-captains/maria/5.3-sj-subqueries" : "5.3-sj-subqueries" } mailSource = mail.BzrLaunchpadEmailMaildirSource("/var/lib/buildbot/Maildir", branchMap=myBranchMap) @@ -163,7 +162,7 @@ # "lenny-amd64-dbg", "adutko-alpha", "debian5-i386-fulltest", "jaunty-x86-valgrind", "jaunty-amd64-rel", - "gentoo-x86-dbg", "ubuntu-x86-dbg", "macosx-g5-dbg", + "gentoo-x86-dbg", "ubuntu-x86-dbg", "macosx-x86-bld", "work-amd64-valgrind", "winxp-x86-nmake", "win32-rel-nmake", "hardy-amd64-fulltest", "gentoo-amd64-sanja", "opensolaris-511-x86", @@ -180,6 +179,7 @@ branches=["mariadb-5.1-knielsen", "5.1-release", "5.1", "5.1.39-oqgraph", "5.1-merge", "mariadb-5.1-monty", + "5.2", "mariadb-5.2-monty", ], treeStableTimer=1, # 1 sec for bzr properties= {"bakebranch": "lp:~maria-captains/ourdelta/ourdelta-montyprogram-fixes"}, @@ -197,7 +197,9 @@ "kvm-deb-jaunty-amd64", "kvm-deb-jaunty-x86", "kvm-deb-karmic-amd64", "kvm-deb-karmic-x86", "kvm-deb-lucid-amd64", "kvm-deb-lucid-x86", - "kvm-bintar-hardy-amd64", "kvm-bintar-hardy-x86",])) + "kvm-bintar-hardy-amd64", "kvm-bintar-hardy-x86", + "opensolaris-511-bintar", + ])) ####### BUILDERS @@ -529,7 +531,7 @@ "slavename": "adutko-centos5-amd64", "builddir": "centos5-amd64-minimal", "factory": f_minimal, - "category": "experimental", + "category": "main", } f_win32_rel_nmake = factory.BuildFactory() @@ -699,7 +701,7 @@ "slavename": "adutko-ultrasparc3", "builddir": "adutko-ultrasparc3", "factory": f_sol_sparc_32, - "category": "new", + "category": "main", } f_dbg_alpha = factory.BuildFactory() @@ -771,23 +773,6 @@ "category": "experimental", } -f_dbg_g5 = factory.BuildFactory() -f_dbg_g5.addStep(bzr_shared_repo) -f_dbg_g5.addStep(bzr_checkout) -f_dbg_g5.addStep(getCompileStep(["BUILD/compile-ppc-max"])) -f_dbg_g5.addStep(getMTR( - test_type="nm", - test_info="Normal run, no --ps-protocol", - command=["sh", "-c", "cd mysql-test && exec perl mysql-test-run.pl --force --retry=3 --skip-ndb --testcase-timeout=45 --suite-timeout=1080 --mysqld=--skip-safemalloc"], - timeout=2700)) - -bld_mac_g5 = {'name': "macosx-g5-dbg", - 'slavename': "hakan-mac-g5", - 'builddir': "hakan-mac-g5", - 'factory': f_dbg_g5, - "category": "experimental", - } - f_mac_x86 = factory.BuildFactory() f_mac_x86.addStep(bzr_shared_repo) f_mac_x86.addStep(bzr_checkout) @@ -878,6 +863,36 @@ "category": "experimental", } +# The trees for which we save binary packages. +savedPackageBranches= ["5.1-release", "5.2-release", "5.3-release"] + +# Get a build step that will archive binary packages (or source tarball). +# Only the newest 3 builds are saved for each (branch, builder) combination. +# The packages are saved under the build number of the tarbake step, so it is +# easy to locate all packages for a given release. +def getPackageArchiveStep(source, tarbuildnum): + cmdText= ("TARBUILDNUM='" + tarbuildnum + "'\n" + + "SRC='" + source + "'\n" + """ +BUILDERNAME='%(buildername)s' +BRANCH='%(branch)s' +BASE="/archive/pack/$BRANCH" +DST="$BASE/build-$TARBUILDNUM/$BUILDERNAME" +set -ex +mkdir -p "$BASE" +rm -Rf "$DST" +(ls -td $BASE/build-*/"$BUILDERNAME" || : ) | (read DUMMY || exit 0; read DUMMY || exit 0; while read VICTIM; do rm -Rf "$VICTIM"; done) +rmdir $BASE/build-* 2>/dev/null || : +mkdir -p "$DST" +cp -r "$SRC" "$DST/" +""") + return ShellCommand( + doStepIf=(lambda(step): step.getProperty("branch") in savedPackageBranches), + description=["archiving"], + descriptionDone=["archive"], + command=["sh", "-c", WithProperties(cmdText)], + ) + + f_kvm_tarbake_jaunty_x86= factory.BuildFactory() f_kvm_tarbake_jaunty_x86.addStep(Compile( description=["making", "dist"], @@ -906,35 +921,38 @@ """ set -ex cd buildbot/build/$(cat buildbot/build/bakery.txt)/ -echo mariadb-*.tar.gz > ../distname.txt -mv $(cat ../distname.txt) ../ +basename mariadb-*.tar.gz .tar.gz > ../distdirname.txt +mv "$(cat ../distdirname.txt).tar.gz" ../ """, - "= scp -P 2223 buildbot@localhost:buildbot/build/distname.txt .", + "= scp -P 2223 buildbot@localhost:buildbot/build/distdirname.txt .", "= scp -P 2223 buildbot@localhost:buildbot/build/bakery.txt .", "= scp -P 2223 'buildbot@localhost:buildbot/build/mariadb-*.tar.gz' .", "= scp -P 2223 'buildbot@localhost:buildbot/build/bakery-*.tar.gz' .", ], )) f_kvm_tarbake_jaunty_x86.addStep(SetProperty( - property="distname", - command=["cat", "distname.txt"], + property="distdirname", + command=["cat", "distdirname.txt"], )) f_kvm_tarbake_jaunty_x86.addStep(SetProperty( property="bakery", command=["cat", "bakery.txt"], )) -f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(distname)s"), - masterdest=WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distname)s"))) +f_kvm_tarbake_jaunty_x86.addStep(getPackageArchiveStep("%(distdirname)s.tar.gz", "%(buildnumber)s")) +f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(distdirname)s.tar.gz"), + masterdest=WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distdirname)s.tar.gz"))) f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(bakery)s.tar.gz"), masterdest=WithProperties("/var/lib/buildbot/OQ-bakeries/%(buildnumber)s:%(bakery)s.tar.gz"))) f_kvm_tarbake_jaunty_x86.addStep(Trigger( schedulerNames=["kvm-ourdelta-sched"], waitForFinish=False, updateSourceStamp=True, - set_properties={ "tarballpath": WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distname)s"), + set_properties={ "tarballpath": WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distdirname)s.tar.gz"), "bakerypath": WithProperties("/var/lib/buildbot/OQ-bakeries/%(buildnumber)s:%(bakery)s.tar.gz"), + "distname": WithProperties("%(distdirname)s.tar.gz"), + "tarbuildnum": WithProperties("%(buildnumber)s"), }, - copy_properties=[ "distname", "bakery" ])) + copy_properties=[ "distdirname", "bakery" ])) bld_kvm_tarbake_jaunty_x86 = {"name": "kvm-tarbake-jaunty-x86", "slavename": "knielsen-kvm-x86", @@ -977,6 +995,7 @@ "= rm -Rf rpms && mkdir rpms", "= scp -P "+port+" 'buildbot@localhost:/usr/src/redhat/RPMS/"+arch+"/*.rpm' rpms/", ])) + rpm_fact.addStep(getPackageArchiveStep("rpms", "%(tarbuildnum)s")) rpm_fact.addStep(Test( description=["testing", "install"], descriptionDone=["test", "install"], @@ -1013,11 +1032,22 @@ deb_fact.addStep(FileDownload( mastersrc=WithProperties("%(bakerypath)s"), slavedest=WithProperties("%(bakery)s.tar.gz"))) + # Extract the compiler warning suppressions file from the source tarball. + deb_fact.addStep(ShellCommand( + description=["getting", ".supp"], + descriptionDone=["get", ".supp"], + command=["sh", "-c", WithProperties(""" +rm -f compiler_warnings.supp +tar zxf "%(distname)s" --strip 2 "$(basename %(distname)s .tar.gz)/support-files/compiler_warnings.supp" +exit 0 # best-effort, not fatal if no suppression file +""")])) deb_fact.addStep(Compile( description=["making", "debs"], descriptionDone=["make", "debs"], logfiles={"kernel": "kernel_"+port+".log"}, warningPattern=gccWarningPattern, + warningExtractor=Compile.warnExtractFromRegexpGroups, + suppressionFile=WithProperties("compiler_warnings.supp"), command=["runvm", "--base-image=/kvm/vms/"+kvm_image+"-build.qcow2"] + args +["--logfile=kernel_"+port+".log", "vm-tmp-"+port+".qcow2", "rm -Rf buildbot && mkdir buildbot", WithProperties("= scp -P "+port+" %(distname)s buildbot@localhost:buildbot/"), @@ -1039,6 +1069,7 @@ "= rm -Rf debs", "= scp -r -P "+port+" buildbot@localhost:buildbot/debs .", ])) + deb_fact.addStep(getPackageArchiveStep("debs", "%(tarbuildnum)s")) deb_fact.addStep(Test( description=["testing", "install"], descriptionDone=["test", "install"], @@ -1144,6 +1175,7 @@ property="bindistname", command=["cat", "bindistname.txt"], )) + bin_fact.addStep(getPackageArchiveStep("%(bindistname)s.tar.gz", "%(tarbuildnum)s")) bin_fact.addStep(Test( description=["testing", "bintar"], descriptionDone=["test", "bintar"], @@ -1178,14 +1210,55 @@ 'slavename': 'opensolaris-x86', 'builddir': 'opensolaris-511-x86', 'factory': f_dbg_sol_64, - "category": "new", + "category": "main", } +f_sol_64_bintar = factory.BuildFactory() +f_sol_64_bintar.addStep(FileDownload(mastersrc=WithProperties("%(tarballpath)s"), + slavedest=WithProperties("%(distname)s"))) +f_sol_64_bintar.addStep(ShellCommand( + description="untarring", descriptionDone="untar", + command=["sh", "-c", WithProperties("rm -Rf %(distdirname)s && tar zxf %(distname)s")])) +f_sol_64_bintar.addStep(getCompileStep(["sh", "-c", WithProperties("cd %(distdirname)s && BUILD/compile-solaris-amd64-debug")], + subdir="%(distdirname)s")) +f_sol_64_bintar.addStep(ShellCommand( + description="packaging", descriptionDone="package", + command=["sh", "-c", WithProperties(""" +set -ex +cd %(distdirname)s +scripts/make_binary_distribution +ORIG_TAR=$(echo mariadb-*.tar.gz) +ORIG_DIR=$(basename ${ORIG_TAR} .tar.gz) +NEW_DIR="$(echo "${ORIG_DIR}" | sed -e 's/-MariaDB//')" +NEW_TAR="${NEW_DIR}.tar.gz" +rm -Rf $ORIG_DIR +tar zxf $ORIG_TAR +mv $ORIG_DIR "${NEW_DIR}" +tar zcf "${NEW_TAR}" "${NEW_DIR}/" +echo "${NEW_TAR}" > ../bindistname.txt +""")])) +f_sol_64_bintar.addStep(SetProperty( + property="bindistname", + command=["cat", "bindistname.txt"], + )) +f_sol_64_bintar.addStep(getPackageArchiveStep("%(distdirname)s/%(bindistname)s", "%(tarbuildnum)s")) +f_sol_64_bintar.addStep(getMTR( + test_type="nm", + test_info="Normal run, no --ps-protocol", + command=["sh", "-c", WithProperties("cd %(distdirname)s/mysql-test && exec perl mysql-test-run.pl --force --retry=3 --skip-ndb --parallel=2")], + mtr_subdir=WithProperties("%(distdirname)s/mysql-test"))) + +bld_opensolaris_511_bintar = {'name': 'opensolaris-511-bintar', + 'slavename': 'opensolaris-x86', + 'builddir': 'opensolaris-511-bintar', + 'factory': f_sol_64_bintar, + "category": "package", + } + c['builders'] = [bld_fulltest, bld_fulltest2, bld_work_valgrind, bld_valgrind_32, bld_rel_amd64, bld_dbg_x86, bld_makedist, bld_archivist_amd64, bld_archivist_cnc, - bld_mac_g5, bld_dbg2_x86, bld_psergey_win32box, bld_win32_tarball, bld_win32_zip, bld_win32_rel_nmake, bld_mac_x86, @@ -1205,6 +1278,7 @@ bld_kvm_deb_karmic_amd64, bld_kvm_deb_karmic_x86, bld_kvm_deb_lucid_amd64, bld_kvm_deb_lucid_x86, bld_kvm_bintar_hardy_amd64, bld_kvm_bintar_hardy_x86, + bld_opensolaris_511_bintar, bld_opensolaris_511_x86, bld_sol_sparc_32, ] === modified file 'buildbot/runvm' --- a/buildbot/runvm 2010-01-14 13:43:51 +0000 +++ b/buildbot/runvm 2010-02-01 12:44:19 +0000 @@ -52,6 +52,19 @@ my $opt_baseimage= undef; my @user_cmd_opt; +# Disable host key checking for ssh. +# This is a bit convoluted due to OpenSSH's slight security-paranoia. +# Without this, we would get a login failure if using another VM image +# (with different host key) on the same port, which is annoying. +# An alternative would be to use CheckHostIP=no and HostKeyAlias=<img.qcow2> +# to get ssh to check a different key for each image. But that would still +# cause an error if re-generating an image (with new ssh host key), and it +# doesn't really give any additional security. +my @ssh_cmd_prefix= ($ssh_exec, + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + '-o', 'LogLevel=ERROR'); + my $image; my $pidfile; @@ -177,17 +190,6 @@ return $res; } -sub exec_guest_cmd { - my ($cmd_and_args)= @_; - if (scalar(@$cmd_and_args) == 0 || $cmd_and_args->[0] eq '') - { - # Otherwise ssh will do an interactive login, which we do not want. - exit 0; - } - - exec_guest_cmd $ssh_exec, '-p', $opt_port, @user_cmd_opt, 'localhost', @$cmd_and_args; -} - sub is_port_used { socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or die "socket() failed: $!\n"; @@ -348,7 +350,7 @@ die "Fatal error: Cannot fork(): $!\n"; } elsif (!$pid) { # Child. - exec_with_print($ssh_exec, '-o', 'ConnectTimeout=4', '-p', $opt_port, + exec_with_print(@ssh_cmd_prefix, '-o', 'ConnectTimeout=4', '-p', $opt_port, @user_cmd_opt, 'localhost', 'sudo', '/sbin/shutdown', '-h', 'now'); } else { @@ -410,7 +412,7 @@ die "Fatal error: Cannot fork(): $!\n"; } elsif (!$pid) { # Child. - exec_with_print($ssh_exec, '-o', 'ConnectTimeout=4', '-p', $opt_port, + exec_with_print(@ssh_cmd_prefix, '-o', 'ConnectTimeout=4', '-p', $opt_port, @user_cmd_opt, 'localhost', '/bin/true'); } else { # Parent. @@ -506,7 +508,7 @@ $res= system(substr($arg, 1)); } else { print STDERR "+ $arg\n"; - $res= system($ssh_exec, '-p', $opt_port, @user_cmd_opt, 'localhost', $arg); + $res= system(@ssh_cmd_prefix, '-p', $opt_port, @user_cmd_opt, 'localhost', $arg); } if ($res < 0) { print STDERR "Could not spawn command: $!\n";