At file:///Users/hakan/work/monty_program/mariadb-tools/ ------------------------------------------------------------ revno: 34 revision-id: hakan@askmonty.org-20100409191359-w2dpenia3bnmlypr parent: hakan@askmonty.org-20100408002114-36ml405ax1h61x60 committer: Hakan Kuecuekyilmaz <hakan@askmonty.org> branch nick: mariadb-tools timestamp: Fri 2010-04-09 21:13:59 +0200 message: Extracted tests to run into host specific config files. Added OProfile hook. === modified file 'sysbench/conf/au0013.inc' --- a/sysbench/conf/au0013.inc 2010-03-04 02:03:03 +0000 +++ b/sysbench/conf/au0013.inc 2010-04-09 19:13:59 +0000 @@ -27,9 +27,38 @@ # Note: sar works different on Mac OS X. SAR='/usr/bin/sar' +# Other binaries. +SUDO=/usr/bin/sudo + # Directories. TEMP_DIR='/tmp' DATA_DIR="${TEMP_DIR}/data" # Where to compile the sources. -WORK_DIR='/tmp' \ No newline at end of file +WORK_DIR='/tmp' + +# Which concurrency level(s) we want to profile with OProfile +# +# Example: profile 8 and 32 +# DO_OPROFILE='8 32' +#DO_OPROFILE='64' +# There is no OProfile on Mac OS X. +DO_OPROFILE='' + +# sysbench tests to run. +SYSBENCH_TESTS[0]="delete.lua" +SYSBENCH_TESTS[1]="insert.lua" +SYSBENCH_TESTS[2]="oltp_complex_ro.lua" +SYSBENCH_TESTS[3]="oltp_complex_rw.lua" +SYSBENCH_TESTS[4]="oltp_simple.lua" +SYSBENCH_TESTS[5]="select.lua" + +# Default option is --random-points=10. +SYSBENCH_TESTS[6]="select_random_points.lua" + +# Default options are +# --number-of-ranges=10 +# --random-ranges-delta=5. +SYSBENCH_TESTS[7]="select_random_ranges.lua" +SYSBENCH_TESTS[8]="update_index.lua" +SYSBENCH_TESTS[9]="update_non_index.lua" === modified file 'sysbench/conf/lu0012.inc' --- a/sysbench/conf/lu0012.inc 2010-03-09 23:31:27 +0000 +++ b/sysbench/conf/lu0012.inc 2010-04-09 19:13:59 +0000 @@ -25,9 +25,36 @@ # For CPU utilization statistics. MPSTAT='/usr/bin/mpstat' +# Other binaries. +SUDO=/usr/bin/sudo + # Directories. TEMP_DIR='/tmp' DATA_DIR="${TEMP_DIR}/data" # Where to compile the sources. WORK_DIR='/tmp' + +# Which concurrency level(s) we want to profile with OProfile +# +# Example: profile 8 and 32 +# DO_OPROFILE='8 32' +DO_OPROFILE='1' + +# sysbench tests to run. +SYSBENCH_TESTS[0]="delete.lua" +SYSBENCH_TESTS[1]="insert.lua" +SYSBENCH_TESTS[2]="oltp_complex_ro.lua" +SYSBENCH_TESTS[3]="oltp_complex_rw.lua" +SYSBENCH_TESTS[4]="oltp_simple.lua" +SYSBENCH_TESTS[5]="select.lua" + +# Default option is --random-points=10. +SYSBENCH_TESTS[6]="select_random_points.lua" + +# Default options are +# --number-of-ranges=10 +# --random-ranges-delta=5. +SYSBENCH_TESTS[7]="select_random_ranges.lua" +SYSBENCH_TESTS[8]="update_index.lua" +SYSBENCH_TESTS[9]="update_non_index.lua" === modified file 'sysbench/conf/perro.inc' --- a/sysbench/conf/perro.inc 2010-03-09 23:31:27 +0000 +++ b/sysbench/conf/perro.inc 2010-04-09 19:13:59 +0000 @@ -34,3 +34,27 @@ # Where to compile the sources. WORK_DIR='/tmp' + +# Which concurrency level(s) we want to profile with OProfile +# +# Example: profile 8 and 32 +# DO_OPROFILE='8 32' +DO_OPROFILE='64' + +# sysbench tests to run. +SYSBENCH_TESTS[0]="delete.lua" +SYSBENCH_TESTS[1]="insert.lua" +SYSBENCH_TESTS[2]="oltp_complex_ro.lua" +SYSBENCH_TESTS[3]="oltp_complex_rw.lua" +SYSBENCH_TESTS[4]="oltp_simple.lua" +SYSBENCH_TESTS[5]="select.lua" + +# Default option is --random-points=10. +SYSBENCH_TESTS[6]="select_random_points.lua" + +# Default options are +# --number-of-ranges=10 +# --random-ranges-delta=5. +SYSBENCH_TESTS[7]="select_random_ranges.lua" +SYSBENCH_TESTS[8]="update_index.lua" +SYSBENCH_TESTS[9]="update_non_index.lua" === modified file 'sysbench/conf/work.inc' --- a/sysbench/conf/work.inc 2010-03-09 23:31:27 +0000 +++ b/sysbench/conf/work.inc 2010-04-09 19:13:59 +0000 @@ -34,3 +34,27 @@ # Where to compile the sources. WORK_DIR='/tmp' + +# Which concurrency level(s) we want to profile with OProfile +# +# Example: profile 8 and 32 +# DO_OPROFILE='8 32' +DO_OPROFILE='64' + +# sysbench tests to run. +SYSBENCH_TESTS[0]="delete.lua" +SYSBENCH_TESTS[1]="insert.lua" +SYSBENCH_TESTS[2]="oltp_complex_ro.lua" +SYSBENCH_TESTS[3]="oltp_complex_rw.lua" +SYSBENCH_TESTS[4]="oltp_simple.lua" +SYSBENCH_TESTS[5]="select.lua" + +# Default option is --random-points=10. +SYSBENCH_TESTS[6]="select_random_points.lua" + +# Default options are +# --number-of-ranges=10 +# --random-ranges-delta=5. +SYSBENCH_TESTS[7]="select_random_ranges.lua" +SYSBENCH_TESTS[8]="update_index.lua" +SYSBENCH_TESTS[9]="update_non_index.lua" === modified file 'sysbench/run-sysbench-myisam.sh' --- a/sysbench/run-sysbench-myisam.sh 2010-04-08 00:21:14 +0000 +++ b/sysbench/run-sysbench-myisam.sh 2010-04-09 19:13:59 +0000 @@ -10,6 +10,11 @@ # reference please check # http://www.mysqlperformanceblog.com/2009/01/30/linux-schedulers-in-tpcc-like... # +# For proper work we need these two commands to be run via sudo +# with no password. Example: +# hakan ALL=NOPASSWD: /usr/bin/opcontrol +# hakan ALL=NOPASSWD: /usr/bin/tee /proc/sys/vm/drop_caches +# # Index sizes for 20 mio rows (--table-size=20000000). # * delete.lua 313M sbtest.MYI # * insert.lua 4.0K sbtest.MYI @@ -110,23 +115,6 @@ # CPU utilization, and such. MONITOR_INTERVAL=10 -SYSBENCH_TESTS[0]="delete.lua" -SYSBENCH_TESTS[1]="insert.lua" -SYSBENCH_TESTS[2]="oltp_complex_ro.lua" -SYSBENCH_TESTS[3]="oltp_complex_rw.lua" -SYSBENCH_TESTS[4]="oltp_simple.lua" -SYSBENCH_TESTS[5]="select.lua" - -# Default option is --random-points=10. -SYSBENCH_TESTS[6]="select_random_points.lua" - -# Default options are -# --number-of-ranges=10 -# --random-ranges-delta=5. -SYSBENCH_TESTS[7]="select_random_ranges.lua" -SYSBENCH_TESTS[8]="update_index.lua" -SYSBENCH_TESTS[9]="update_non_index.lua" - # # Note: myisam-max-rows has to match or exceed oltp-table-size # otherwise we get a table full error while preparing the run. @@ -216,6 +204,9 @@ exit 1 fi +# Location of our mysqld binary. +MYSQLD_BINARY="${TEMP_DIR}/build/sql/mysqld" + # # Compile sources. # TODO: Add platform detection and choose proper build script accordingly. @@ -296,11 +287,14 @@ echo "Run time is: $RUN_TIME" >> ${RESULT_DIR}/${TODAY}/${PRODUCT}/sysbench_options.txt # -# Kill possibly left over monitoring processes. +# Clean up possibly left over monitoring processes. # killall -9 iostat killall -9 mpstat killall -9 mysqladmin +$SUDO opcontrol --stop +$SUDO opcontrol --deinit +$SUDO opcontrol --reset for (( i = 0 ; i < ${#SYSBENCH_TESTS[@]} ; i++ )) do @@ -340,6 +334,16 @@ SYSBENCH_OPTIONS_WARM_UP="${SYSBENCH_OPTIONS} --num-threads=3 --max-time=$WARM_UP_TIME" SYSBENCH_OPTIONS_RUN="${SYSBENCH_OPTIONS} --num-threads=$THREADS --max-time=$RUN_TIME" + # Check whether we want a profiled run. + PROFILE_IT=-1 + for l in $DO_OPROFILE + do + if [ x"$l" = x"$THREADS" ]; then + PROFILE_IT=1 + break + fi + done + k=0 while [ $k -lt $LOOP_COUNT ] do @@ -374,8 +378,35 @@ $MYSQLADMIN $MYSQLADMIN_OPTIONS --sleep $MONITOR_INTERVAL status > ${THIS_RESULT_DIR}/server_status${k}.txt 2>&1 & SERVER_STATUS_PID=$! - + + if [ $PROFILE_IT -eq 1 ]; then + $SUDO opcontrol --setup --separate=lib,kernel,thread --no-vmlinux + $SUDO opcontrol --start-daemon + if [ $? != 0 ]; then + echo "[WARNING]: Could not start oprofile daemonl." + echo " Please check your OProfile installation." + fi + + $SUDO opcontrol --start + echo "[$(date "+%Y-%m-%d %H:%M:%S")] This is an OProfile'd sysbench run." + fi + $SYSBENCH $SYSBENCH_OPTIONS_RUN run > ${THIS_RESULT_DIR}/result${k}.txt 2>&1 + + if [ $PROFILE_IT -eq 1 ]; then + PROFILE_IT=-1 + $SUDO opcontrol --dump + $SUDO opcontrol --stop + + opreport --demangle=smart --threshold 0.5 --symbols --long-filenames --merge tgid $MYSQLD_BINARY > ${THIS_RESULT_DIR}/oprofile${k}.txt 2>&1 + + $SUDO opcontrol --deinit + $SUDO opcontrol --reset + fi + + # Copy mysqld error log for future reference. + # TODO: add chrash detection. + cp ${DATA_DIR}/${HOSTNAME}.err ${THIS_RESULT_DIR}/${HOSTNAME}${k}.err sync; sync; sync sleep 1 === modified file 'sysbench/run-sysbench.sh' --- a/sysbench/run-sysbench.sh 2010-04-08 00:21:14 +0000 +++ b/sysbench/run-sysbench.sh 2010-04-09 19:13:59 +0000 @@ -10,6 +10,11 @@ # reference please check # http://www.mysqlperformanceblog.com/2009/01/30/linux-schedulers-in-tpcc-like... # +# For proper work we need these two commands to be run via sudo +# with no password. Example: +# hakan ALL=NOPASSWD: /usr/bin/opcontrol +# hakan ALL=NOPASSWD: /usr/bin/tee /proc/sys/vm/drop_caches +# # Hakan Kuecuekyilmaz <hakan at askmonty dot org> 2010-02-19. # @@ -107,23 +112,6 @@ # CPU utilization, and such. MONITOR_INTERVAL=10 -SYSBENCH_TESTS[0]="delete.lua" -SYSBENCH_TESTS[1]="insert.lua" -SYSBENCH_TESTS[2]="oltp_complex_ro.lua" -SYSBENCH_TESTS[3]="oltp_complex_rw.lua" -SYSBENCH_TESTS[4]="oltp_simple.lua" -SYSBENCH_TESTS[5]="select.lua" - -# Default option is --random-points=10. -SYSBENCH_TESTS[6]="select_random_points.lua" - -# Default options are -# --number-of-ranges=10 -# --random-ranges-delta=5. -SYSBENCH_TESTS[7]="select_random_ranges.lua" -SYSBENCH_TESTS[8]="update_index.lua" -SYSBENCH_TESTS[9]="update_non_index.lua" - SYSBENCH_OPTIONS="--oltp-table-size=$TABLE_SIZE \ --max-requests=0 \ --mysql-table-engine=InnoDB \ @@ -208,6 +196,9 @@ exit 1 fi +# Location of our mysqld binary. +MYSQLD_BINARY="${TEMP_DIR}/build/sql/mysqld" + # # Compile sources. # TODO: Add platform detection and choose proper build script accordingly. @@ -288,11 +279,14 @@ echo "Run time is: $RUN_TIME" >> ${RESULT_DIR}/${TODAY}/${PRODUCT}/sysbench_options.txt # -# Kill possibly left over monitoring processes. +# Clean up possibly left over monitoring processes. # killall -9 iostat killall -9 mpstat killall -9 mysqladmin +$SUDO opcontrol --stop +$SUDO opcontrol --deinit +$SUDO opcontrol --reset for (( i = 0 ; i < ${#SYSBENCH_TESTS[@]} ; i++ )) do @@ -332,6 +326,16 @@ SYSBENCH_OPTIONS_WARM_UP="${SYSBENCH_OPTIONS} --num-threads=1 --max-time=$WARM_UP_TIME" SYSBENCH_OPTIONS_RUN="${SYSBENCH_OPTIONS} --num-threads=$THREADS --max-time=$RUN_TIME" + # Check whether we want a profiled run. + PROFILE_IT=-1 + for l in $DO_OPROFILE + do + if [ x"$l" = x"$THREADS" ]; then + PROFILE_IT=1 + break + fi + done + k=0 while [ $k -lt $LOOP_COUNT ] do @@ -366,8 +370,35 @@ $MYSQLADMIN $MYSQLADMIN_OPTIONS --sleep $MONITOR_INTERVAL status > ${THIS_RESULT_DIR}/server_status${k}.txt 2>&1 & SERVER_STATUS_PID=$! + if [ $PROFILE_IT -eq 1 ]; then + $SUDO opcontrol --setup --separate=lib,kernel,thread --no-vmlinux + $SUDO opcontrol --start-daemon + if [ $? != 0 ]; then + echo "[WARNING]: Could not start oprofile daemonl." + echo " Please check your OProfile installation." + fi + + $SUDO opcontrol --start + echo "[$(date "+%Y-%m-%d %H:%M:%S")] This is an OProfile'd sysbench run." + fi + $SYSBENCH $SYSBENCH_OPTIONS_RUN run > ${THIS_RESULT_DIR}/result${k}.txt 2>&1 + if [ $PROFILE_IT -eq 1 ]; then + PROFILE_IT=-1 + $SUDO opcontrol --dump + $SUDO opcontrol --stop + + opreport --demangle=smart --threshold 0.5 --symbols --long-filenames --merge tgid $MYSQLD_BINARY > ${THIS_RESULT_DIR}/oprofile${k}.txt 2>&1 + + $SUDO opcontrol --deinit + $SUDO opcontrol --reset + fi + + # Copy mysqld error log for future reference. + # TODO: add chrash detection. + cp ${DATA_DIR}/${HOSTNAME}.err ${THIS_RESULT_DIR}/${HOSTNAME}${k}.err + sync; sync; sync sleep 1