At file:///Users/hakan/work/monty_program/mariadb-tools/
------------------------------------------------------------
revno: 8 [merge]
revision-id: hakan@askmonty.org-20100204111840-0d9u3nyio2h8tavi
parent: hakan@askmonty.org-20100204011956-u74a4es6oogvd50w
parent: knielsen@hasky-20100202151012-pjcblb2v3bqsp5eo
committer: Hakan Kuecuekyilmaz
branch nick: mariadb-tools
timestamp: Thu 2010-02-04 12:18:40 +0100
message:
Merge.
modified:
buildbot/maria-master.cfg mariamaster.cfg-20091218103450-cvifjz3i70oerkej-1
buildbot/runvm runvm-20091218082659-ept0jpsqa5e8jno6-2
=== modified file 'buildbot/maria-master.cfg'
--- a/buildbot/maria-master.cfg 2010-01-02 21:17:58 +0000
+++ b/buildbot/maria-master.cfg 2010-02-02 15:10:12 +0000
@@ -59,7 +59,6 @@
,mkSlave("psergey-pylon-amd64", max_builds=1)
,mkSlave("psergey-foxhole-x86", max_builds=1)
,mkSlave("psergey-pslp2-x86", max_builds=1)
- ,mkSlave("hakan-mac-g5", max_builds=1)
,mkSlave("work-opensuse-amd64", max_builds=1)
,mkSlave("psergey-win32box", max_builds=1)
,mkSlave("vm-win-1", max_builds=1)
@@ -114,7 +113,7 @@
"lp:~maria-captains/maria/maria-5.2-merge-5.1" : "maria-5.2-merge-5.1",
"lp:~maria-captains/maria/5.2-dsmrr" : "5.2-dsmrr",
"lp:~maria-captains/maria/5.3" : "5.3",
- "lp:~maria-captains/maria/5.3" : "5.3-sj-subqueries"
+ "lp:~maria-captains/maria/5.3-sj-subqueries" : "5.3-sj-subqueries"
}
mailSource = mail.BzrLaunchpadEmailMaildirSource("/var/lib/buildbot/Maildir",
branchMap=myBranchMap)
@@ -163,7 +162,7 @@
# "lenny-amd64-dbg", "adutko-alpha",
"debian5-i386-fulltest",
"jaunty-x86-valgrind", "jaunty-amd64-rel",
- "gentoo-x86-dbg", "ubuntu-x86-dbg", "macosx-g5-dbg",
+ "gentoo-x86-dbg", "ubuntu-x86-dbg",
"macosx-x86-bld", "work-amd64-valgrind", "winxp-x86-nmake",
"win32-rel-nmake", "hardy-amd64-fulltest",
"gentoo-amd64-sanja", "opensolaris-511-x86",
@@ -180,6 +179,7 @@
branches=["mariadb-5.1-knielsen", "5.1-release", "5.1",
"5.1.39-oqgraph", "5.1-merge",
"mariadb-5.1-monty",
+ "5.2", "mariadb-5.2-monty",
],
treeStableTimer=1, # 1 sec for bzr
properties= {"bakebranch": "lp:~maria-captains/ourdelta/ourdelta-montyprogram-fixes"},
@@ -197,7 +197,9 @@
"kvm-deb-jaunty-amd64", "kvm-deb-jaunty-x86",
"kvm-deb-karmic-amd64", "kvm-deb-karmic-x86",
"kvm-deb-lucid-amd64", "kvm-deb-lucid-x86",
- "kvm-bintar-hardy-amd64", "kvm-bintar-hardy-x86",]))
+ "kvm-bintar-hardy-amd64", "kvm-bintar-hardy-x86",
+ "opensolaris-511-bintar",
+ ]))
####### BUILDERS
@@ -529,7 +531,7 @@
"slavename": "adutko-centos5-amd64",
"builddir": "centos5-amd64-minimal",
"factory": f_minimal,
- "category": "experimental",
+ "category": "main",
}
f_win32_rel_nmake = factory.BuildFactory()
@@ -699,7 +701,7 @@
"slavename": "adutko-ultrasparc3",
"builddir": "adutko-ultrasparc3",
"factory": f_sol_sparc_32,
- "category": "new",
+ "category": "main",
}
f_dbg_alpha = factory.BuildFactory()
@@ -771,23 +773,6 @@
"category": "experimental",
}
-f_dbg_g5 = factory.BuildFactory()
-f_dbg_g5.addStep(bzr_shared_repo)
-f_dbg_g5.addStep(bzr_checkout)
-f_dbg_g5.addStep(getCompileStep(["BUILD/compile-ppc-max"]))
-f_dbg_g5.addStep(getMTR(
- test_type="nm",
- test_info="Normal run, no --ps-protocol",
- command=["sh", "-c", "cd mysql-test && exec perl mysql-test-run.pl --force --retry=3 --skip-ndb --testcase-timeout=45 --suite-timeout=1080 --mysqld=--skip-safemalloc"],
- timeout=2700))
-
-bld_mac_g5 = {'name': "macosx-g5-dbg",
- 'slavename': "hakan-mac-g5",
- 'builddir': "hakan-mac-g5",
- 'factory': f_dbg_g5,
- "category": "experimental",
- }
-
f_mac_x86 = factory.BuildFactory()
f_mac_x86.addStep(bzr_shared_repo)
f_mac_x86.addStep(bzr_checkout)
@@ -878,6 +863,36 @@
"category": "experimental",
}
+# The trees for which we save binary packages.
+savedPackageBranches= ["5.1-release", "5.2-release", "5.3-release"]
+
+# Get a build step that will archive binary packages (or source tarball).
+# Only the newest 3 builds are saved for each (branch, builder) combination.
+# The packages are saved under the build number of the tarbake step, so it is
+# easy to locate all packages for a given release.
+def getPackageArchiveStep(source, tarbuildnum):
+ cmdText= ("TARBUILDNUM='" + tarbuildnum + "'\n" +
+ "SRC='" + source + "'\n" + """
+BUILDERNAME='%(buildername)s'
+BRANCH='%(branch)s'
+BASE="/archive/pack/$BRANCH"
+DST="$BASE/build-$TARBUILDNUM/$BUILDERNAME"
+set -ex
+mkdir -p "$BASE"
+rm -Rf "$DST"
+(ls -td $BASE/build-*/"$BUILDERNAME" || : ) | (read DUMMY || exit 0; read DUMMY || exit 0; while read VICTIM; do rm -Rf "$VICTIM"; done)
+rmdir $BASE/build-* 2>/dev/null || :
+mkdir -p "$DST"
+cp -r "$SRC" "$DST/"
+""")
+ return ShellCommand(
+ doStepIf=(lambda(step): step.getProperty("branch") in savedPackageBranches),
+ description=["archiving"],
+ descriptionDone=["archive"],
+ command=["sh", "-c", WithProperties(cmdText)],
+ )
+
+
f_kvm_tarbake_jaunty_x86= factory.BuildFactory()
f_kvm_tarbake_jaunty_x86.addStep(Compile(
description=["making", "dist"],
@@ -906,35 +921,38 @@
"""
set -ex
cd buildbot/build/$(cat buildbot/build/bakery.txt)/
-echo mariadb-*.tar.gz > ../distname.txt
-mv $(cat ../distname.txt) ../
+basename mariadb-*.tar.gz .tar.gz > ../distdirname.txt
+mv "$(cat ../distdirname.txt).tar.gz" ../
""",
- "= scp -P 2223 buildbot@localhost:buildbot/build/distname.txt .",
+ "= scp -P 2223 buildbot@localhost:buildbot/build/distdirname.txt .",
"= scp -P 2223 buildbot@localhost:buildbot/build/bakery.txt .",
"= scp -P 2223 'buildbot@localhost:buildbot/build/mariadb-*.tar.gz' .",
"= scp -P 2223 'buildbot@localhost:buildbot/build/bakery-*.tar.gz' .",
],
))
f_kvm_tarbake_jaunty_x86.addStep(SetProperty(
- property="distname",
- command=["cat", "distname.txt"],
+ property="distdirname",
+ command=["cat", "distdirname.txt"],
))
f_kvm_tarbake_jaunty_x86.addStep(SetProperty(
property="bakery",
command=["cat", "bakery.txt"],
))
-f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(distname)s"),
- masterdest=WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distname)s")))
+f_kvm_tarbake_jaunty_x86.addStep(getPackageArchiveStep("%(distdirname)s.tar.gz", "%(buildnumber)s"))
+f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(distdirname)s.tar.gz"),
+ masterdest=WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distdirname)s.tar.gz")))
f_kvm_tarbake_jaunty_x86.addStep(FileUpload(slavesrc=WithProperties("%(bakery)s.tar.gz"),
masterdest=WithProperties("/var/lib/buildbot/OQ-bakeries/%(buildnumber)s:%(bakery)s.tar.gz")))
f_kvm_tarbake_jaunty_x86.addStep(Trigger(
schedulerNames=["kvm-ourdelta-sched"],
waitForFinish=False,
updateSourceStamp=True,
- set_properties={ "tarballpath": WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distname)s"),
+ set_properties={ "tarballpath": WithProperties("/var/lib/buildbot/OQ-tarballs/%(buildnumber)s:%(distdirname)s.tar.gz"),
"bakerypath": WithProperties("/var/lib/buildbot/OQ-bakeries/%(buildnumber)s:%(bakery)s.tar.gz"),
+ "distname": WithProperties("%(distdirname)s.tar.gz"),
+ "tarbuildnum": WithProperties("%(buildnumber)s"),
},
- copy_properties=[ "distname", "bakery" ]))
+ copy_properties=[ "distdirname", "bakery" ]))
bld_kvm_tarbake_jaunty_x86 = {"name": "kvm-tarbake-jaunty-x86",
"slavename": "knielsen-kvm-x86",
@@ -977,6 +995,7 @@
"= rm -Rf rpms && mkdir rpms",
"= scp -P "+port+" 'buildbot@localhost:/usr/src/redhat/RPMS/"+arch+"/*.rpm' rpms/",
]))
+ rpm_fact.addStep(getPackageArchiveStep("rpms", "%(tarbuildnum)s"))
rpm_fact.addStep(Test(
description=["testing", "install"],
descriptionDone=["test", "install"],
@@ -1013,11 +1032,22 @@
deb_fact.addStep(FileDownload(
mastersrc=WithProperties("%(bakerypath)s"),
slavedest=WithProperties("%(bakery)s.tar.gz")))
+ # Extract the compiler warning suppressions file from the source tarball.
+ deb_fact.addStep(ShellCommand(
+ description=["getting", ".supp"],
+ descriptionDone=["get", ".supp"],
+ command=["sh", "-c", WithProperties("""
+rm -f compiler_warnings.supp
+tar zxf "%(distname)s" --strip 2 "$(basename %(distname)s .tar.gz)/support-files/compiler_warnings.supp"
+exit 0 # best-effort, not fatal if no suppression file
+""")]))
deb_fact.addStep(Compile(
description=["making", "debs"],
descriptionDone=["make", "debs"],
logfiles={"kernel": "kernel_"+port+".log"},
warningPattern=gccWarningPattern,
+ warningExtractor=Compile.warnExtractFromRegexpGroups,
+ suppressionFile=WithProperties("compiler_warnings.supp"),
command=["runvm", "--base-image=/kvm/vms/"+kvm_image+"-build.qcow2"] + args +["--logfile=kernel_"+port+".log", "vm-tmp-"+port+".qcow2",
"rm -Rf buildbot && mkdir buildbot",
WithProperties("= scp -P "+port+" %(distname)s buildbot@localhost:buildbot/"),
@@ -1039,6 +1069,7 @@
"= rm -Rf debs",
"= scp -r -P "+port+" buildbot@localhost:buildbot/debs .",
]))
+ deb_fact.addStep(getPackageArchiveStep("debs", "%(tarbuildnum)s"))
deb_fact.addStep(Test(
description=["testing", "install"],
descriptionDone=["test", "install"],
@@ -1144,6 +1175,7 @@
property="bindistname",
command=["cat", "bindistname.txt"],
))
+ bin_fact.addStep(getPackageArchiveStep("%(bindistname)s.tar.gz", "%(tarbuildnum)s"))
bin_fact.addStep(Test(
description=["testing", "bintar"],
descriptionDone=["test", "bintar"],
@@ -1178,14 +1210,55 @@
'slavename': 'opensolaris-x86',
'builddir': 'opensolaris-511-x86',
'factory': f_dbg_sol_64,
- "category": "new",
+ "category": "main",
}
+f_sol_64_bintar = factory.BuildFactory()
+f_sol_64_bintar.addStep(FileDownload(mastersrc=WithProperties("%(tarballpath)s"),
+ slavedest=WithProperties("%(distname)s")))
+f_sol_64_bintar.addStep(ShellCommand(
+ description="untarring", descriptionDone="untar",
+ command=["sh", "-c", WithProperties("rm -Rf %(distdirname)s && tar zxf %(distname)s")]))
+f_sol_64_bintar.addStep(getCompileStep(["sh", "-c", WithProperties("cd %(distdirname)s && BUILD/compile-solaris-amd64-debug")],
+ subdir="%(distdirname)s"))
+f_sol_64_bintar.addStep(ShellCommand(
+ description="packaging", descriptionDone="package",
+ command=["sh", "-c", WithProperties("""
+set -ex
+cd %(distdirname)s
+scripts/make_binary_distribution
+ORIG_TAR=$(echo mariadb-*.tar.gz)
+ORIG_DIR=$(basename ${ORIG_TAR} .tar.gz)
+NEW_DIR="$(echo "${ORIG_DIR}" | sed -e 's/-MariaDB//')"
+NEW_TAR="${NEW_DIR}.tar.gz"
+rm -Rf $ORIG_DIR
+tar zxf $ORIG_TAR
+mv $ORIG_DIR "${NEW_DIR}"
+tar zcf "${NEW_TAR}" "${NEW_DIR}/"
+echo "${NEW_TAR}" > ../bindistname.txt
+""")]))
+f_sol_64_bintar.addStep(SetProperty(
+ property="bindistname",
+ command=["cat", "bindistname.txt"],
+ ))
+f_sol_64_bintar.addStep(getPackageArchiveStep("%(distdirname)s/%(bindistname)s", "%(tarbuildnum)s"))
+f_sol_64_bintar.addStep(getMTR(
+ test_type="nm",
+ test_info="Normal run, no --ps-protocol",
+ command=["sh", "-c", WithProperties("cd %(distdirname)s/mysql-test && exec perl mysql-test-run.pl --force --retry=3 --skip-ndb --parallel=2")],
+ mtr_subdir=WithProperties("%(distdirname)s/mysql-test")))
+
+bld_opensolaris_511_bintar = {'name': 'opensolaris-511-bintar',
+ 'slavename': 'opensolaris-x86',
+ 'builddir': 'opensolaris-511-bintar',
+ 'factory': f_sol_64_bintar,
+ "category": "package",
+ }
+
c['builders'] = [bld_fulltest, bld_fulltest2, bld_work_valgrind,
bld_valgrind_32, bld_rel_amd64, bld_dbg_x86,
bld_makedist,
bld_archivist_amd64, bld_archivist_cnc,
- bld_mac_g5,
bld_dbg2_x86, bld_psergey_win32box, bld_win32_tarball,
bld_win32_zip,
bld_win32_rel_nmake, bld_mac_x86,
@@ -1205,6 +1278,7 @@
bld_kvm_deb_karmic_amd64, bld_kvm_deb_karmic_x86,
bld_kvm_deb_lucid_amd64, bld_kvm_deb_lucid_x86,
bld_kvm_bintar_hardy_amd64, bld_kvm_bintar_hardy_x86,
+ bld_opensolaris_511_bintar,
bld_opensolaris_511_x86,
bld_sol_sparc_32,
]
=== modified file 'buildbot/runvm'
--- a/buildbot/runvm 2010-01-14 13:43:51 +0000
+++ b/buildbot/runvm 2010-02-01 12:44:19 +0000
@@ -52,6 +52,19 @@
my $opt_baseimage= undef;
my @user_cmd_opt;
+# Disable host key checking for ssh.
+# This is a bit convoluted due to OpenSSH's slight security-paranoia.
+# Without this, we would get a login failure if using another VM image
+# (with different host key) on the same port, which is annoying.
+# An alternative would be to use CheckHostIP=no and HostKeyAlias=
+# to get ssh to check a different key for each image. But that would still
+# cause an error if re-generating an image (with new ssh host key), and it
+# doesn't really give any additional security.
+my @ssh_cmd_prefix= ($ssh_exec,
+ '-o', 'UserKnownHostsFile=/dev/null',
+ '-o', 'StrictHostKeyChecking=no',
+ '-o', 'LogLevel=ERROR');
+
my $image;
my $pidfile;
@@ -177,17 +190,6 @@
return $res;
}
-sub exec_guest_cmd {
- my ($cmd_and_args)= @_;
- if (scalar(@$cmd_and_args) == 0 || $cmd_and_args->[0] eq '')
- {
- # Otherwise ssh will do an interactive login, which we do not want.
- exit 0;
- }
-
- exec_guest_cmd $ssh_exec, '-p', $opt_port, @user_cmd_opt, 'localhost', @$cmd_and_args;
-}
-
sub is_port_used {
socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp'))
or die "socket() failed: $!\n";
@@ -348,7 +350,7 @@
die "Fatal error: Cannot fork(): $!\n";
} elsif (!$pid) {
# Child.
- exec_with_print($ssh_exec, '-o', 'ConnectTimeout=4', '-p', $opt_port,
+ exec_with_print(@ssh_cmd_prefix, '-o', 'ConnectTimeout=4', '-p', $opt_port,
@user_cmd_opt, 'localhost',
'sudo', '/sbin/shutdown', '-h', 'now');
} else {
@@ -410,7 +412,7 @@
die "Fatal error: Cannot fork(): $!\n";
} elsif (!$pid) {
# Child.
- exec_with_print($ssh_exec, '-o', 'ConnectTimeout=4', '-p', $opt_port,
+ exec_with_print(@ssh_cmd_prefix, '-o', 'ConnectTimeout=4', '-p', $opt_port,
@user_cmd_opt, 'localhost', '/bin/true');
} else {
# Parent.
@@ -506,7 +508,7 @@
$res= system(substr($arg, 1));
} else {
print STDERR "+ $arg\n";
- $res= system($ssh_exec, '-p', $opt_port, @user_cmd_opt, 'localhost', $arg);
+ $res= system(@ssh_cmd_prefix, '-p', $opt_port, @user_cmd_opt, 'localhost', $arg);
}
if ($res < 0) {
print STDERR "Could not spawn command: $!\n";
