[Commits] 2e59503: Cleanup

27 Feb 2019

revision-id: 2e595032f0e4c1727f18ce89173fdc2b40416f53 ()
parent(s): b6949ccd5811d33d59fa07bd1c5d2e856c7dd3bd
author: Sergei Petrunia
committer: Sergei Petrunia
timestamp: 2019-02-27 16:01:20 -0500
message:

Cleanup

---
 mariadb-tpcds-tooling2/01-generate-dataset-pg.sh   | 41 +++++++++++++++++++++
 mariadb-tpcds-tooling2/01-generate-dataset.sh      | 39 ++++++++++++++++++++
 ...chmark-and-data-pg.sh => 02-load-dataset-pg.sh} | 22 +-----------
 mariadb-tpcds-tooling2/02-load-dataset.sh          | 25 +++++++++++++
 mariadb-tpcds-tooling2/10-prepare-queries-pg.sh    | 39 ++++++++++++++++++++
 mariadb-tpcds-tooling2/10-prepare-queries.sh       | 38 ++++++++++++++++++++
 .../{run-queries-pg.sh => 20-run-queries-pg.sh}    |  9 ++++-
 ...{100-save-result-pg.sh => 90-save-result-pg.sh} |  0
 .../prepare-benchmark-and-data.sh                  | 42 ----------------------
 mariadb-tpcds-tooling2/prepare-queries-pg.sh       | 26 --------------
 mariadb-tpcds-tooling2/prepare-queries.sh          | 25 -------------
 setup-server/setup-mariadb-current.sh              | 30 +++++++++++++---
 12 files changed, 217 insertions(+), 119 deletions(-)

diff --git a/mariadb-tpcds-tooling2/01-generate-dataset-pg.sh b/mariadb-tpcds-tooling2/01-generate-dataset-pg.sh
new file mode 100755
index 0000000..b0e0c83
--- /dev/null
+++ b/mariadb-tpcds-tooling2/01-generate-dataset-pg.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+RNGSEED_DATASET=10
+
+set -e
+
+if [ -d data-for-pg ] ; then
+  echo "Dataset is already there? Remove data-for-pg directory "
+  exit 1
+fi
+
+if [ ! -f tpc-ds-tool.zip ] ; then
+  wget http://s.petrunia.net/scratch/tpc-ds-tool.zip
+fi
+
+if [ ! -d v2.8.0rc4-pg ] ; then 
+# Compile the toolkit, unmodified
+(
+  rm -rf tmp
+  mkdir  tmp
+  (cd tmp; unzip ../tpc-ds-tool.zip)
+  mv tmp/v2.8.0rc4 v2.8.0rc4-pg
+  rm -rf tmp
+
+  cd v2.8.0rc4-pg/tools
+  make
+)
+fi
+
+# Generate the dataset
+(
+  cd v2.8.0rc4-pg/tools
+  mkdir -p ../../data-for-pg
+  ./dsdgen -terminate n -scale 1 -rngseed $RNGSEED_DATASET -dir ../../data-for-pg
+  du -sh ../../data-for-pg
+)
+
+cat > data-for-postgresql.txt <<END
+RNGSEED_DATASET=$RNGSEED_DATASET
+END
+
diff --git a/mariadb-tpcds-tooling2/01-generate-dataset.sh b/mariadb-tpcds-tooling2/01-generate-dataset.sh
new file mode 100755
index 0000000..3fb580a
--- /dev/null
+++ b/mariadb-tpcds-tooling2/01-generate-dataset.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+RNGSEED_DATASET=10
+
+set -e
+
+if [ -d data-for-mysql ] ; then
+  echo "Dataset is already there? Remove data-for-mysql directory "
+  exit 1
+fi
+
+set -e 
+if [ ! -f tpc-ds-tool.zip ] ; then
+  wget http://s.petrunia.net/scratch/tpc-ds-tool.zip
+fi
+
+if [ ! -d v2.8.0rc4 ] ; then 
+
+# Modify the toolkit to produce MySQL dialect and compile it
+  (
+    cd v2.8.0rc4/tools
+    mv makefile makefile.original
+    sed  's/^\(CFLAGS.*\)$/\1 -D_MYSQL/' < makefile.original > makefile
+    make
+  )
+fi
+
+# Generate the dataset
+(
+  cd v2.8.0rc4/tools
+  mkdir -p ../../data-for-mysql
+  ./dsdgen -scale 1 -rngseed $RNGSEED_DATASET -dir ../../data-for-mysql
+  du -sh ../../data-for-mysql
+)
+
+cat > data-for-mysql.txt <<END
+RNGSEED_DATASET=$RNGSEED_DATASET
+END
+
diff --git a/mariadb-tpcds-tooling2/prepare-benchmark-and-data-pg.sh b/mariadb-tpcds-tooling2/02-load-dataset-pg.sh
similarity index 63%
rename from mariadb-tpcds-tooling2/prepare-benchmark-and-data-pg.sh
rename to mariadb-tpcds-tooling2/02-load-dataset-pg.sh
index ff5bb63..2dac1ea 100755
--- a/mariadb-tpcds-tooling2/prepare-benchmark-and-data-pg.sh
+++ b/mariadb-tpcds-tooling2/02-load-dataset-pg.sh
@@ -1,25 +1,6 @@
 #!/bin/bash
 
-mkdir tmp
-(cd tmp; unzip ../tpc-ds-tool.zip)
-mv tmp/v2.8.0rc4 v2.8.0rc4-pg
-rm -rf tmp
-
-
-# Compile the toolkit, unmodified
-(
-  cd v2.8.0rc4-pg/tools
-  make
-)
-
-# Generate the dataset
-(
-  cd v2.8.0rc4-pg/tools
-  mkdir -p ../../data-for-pg
-  ./dsdgen -terminate n -scale 1 -rngseed 5678 -dir ../../data-for-pg
-  du -sh ../../data-for-pg
-)
-
+set -e
 
 # Load the dataset
 postgresql-11.2-inst/bin/createdb tpcds
@@ -35,4 +16,3 @@ postgresql-11.2-inst/bin/psql tpcds < load-pg.sql
 echo 'analyze;' | postgresql-11.2-inst/bin/psql tpcds
 echo 'analyze;' | postgresql-11.2-inst/bin/psql tpcds
 
-
diff --git a/mariadb-tpcds-tooling2/02-load-dataset.sh b/mariadb-tpcds-tooling2/02-load-dataset.sh
new file mode 100755
index 0000000..8065ed0
--- /dev/null
+++ b/mariadb-tpcds-tooling2/02-load-dataset.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -f mysql-config.sh ] ; then
+  echo "Cannot find mysql-config.sh - did you setup the db?"
+  exit 1;
+fi
+
+source mysql-config.sh
+echo "Loading data using $MYSQL $MYSQL_ARGS";
+
+# This should fail if the database already exists, right?
+$MYSQL $MYSQL_ARGS -e "create database tpcds"
+
+$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/ddl/tables.sql
+$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/ddl/indexes.sql
+$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/aux-tables.sql
+
+DATA_FILES_DIR="`pwd`/data-for-mysql/"
+
+bash mariadb-tpcds-tooling2/ddl/load-mysql.sql.sh $DATA_FILES_DIR > load-mysql.sql
+
+$MYSQL $MYSQL_ARGS tpcds < load-mysql.sql
+
diff --git a/mariadb-tpcds-tooling2/10-prepare-queries-pg.sh b/mariadb-tpcds-tooling2/10-prepare-queries-pg.sh
new file mode 100755
index 0000000..5f8a9bc
--- /dev/null
+++ b/mariadb-tpcds-tooling2/10-prepare-queries-pg.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+RNGSEED_QUERIES=10
+
+set -e
+
+if [ -d queries-for-pg ]; then 
+  echo "Directory queries-for-pg exists. Move it away"
+  exit 1
+fi
+
+# Generate the queries
+(
+  cd v2.8.0rc4-pg
+
+  if [ ! -f query_templates/templates-for-mariadb.lst ]; then
+    patch -p1 < ../mariadb-tpcds-tooling2/adapt-queries-for-pg-v4.diff
+  fi
+ 
+  cd tools
+  mkdir -p ../../queries-for-pg
+./dsqgen \
+  -scale 1 \
+  -dialect netezza \
+  -verbose y \
+  -qualify y  \
+  -rngseed $RNGSEED_QUERIES \
+  -streams 10 \
+  -directory ../query_templates \
+  -input ../query_templates/templates-for-mariadb.lst \
+  -output_dir ../../queries-for-pg
+
+  ls -lah ../../queries-for-pg
+)
+
+cat > queries-for-pg.txt <<END
+RNGSEED_QUERIES=$RNGSEED_QUERIES
+END
+
diff --git a/mariadb-tpcds-tooling2/10-prepare-queries.sh b/mariadb-tpcds-tooling2/10-prepare-queries.sh
new file mode 100755
index 0000000..48006d8
--- /dev/null
+++ b/mariadb-tpcds-tooling2/10-prepare-queries.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+RNGSEED_QUERIES=10
+
+set -e
+
+if [ -d queries-for-mysql ]; then 
+  echo "Directory queries-for-mysql exists. Move it away"
+  exit 1
+fi
+
+# Generate the queries
+(
+  cd v2.8.0rc4
+  if [ ! -f query_templates/templates-for-mariadb.lst ]; then
+    patch -p1 < ../mariadb-tpcds-tooling2/adapt-queries-for-mariadb-v4.diff
+  fi
+
+  cd tools
+  mkdir -p ../../queries-for-mysql
+./dsqgen \
+  -scale 1 \
+  -dialect netezza \
+  -verbose y \
+  -qualify y  \
+  -rngseed $RNGSEED_QUERIES \
+  -streams 10 \
+  -directory ../query_templates \
+  -input ../query_templates/templates-for-mariadb.lst \
+  -output_dir ../../queries-for-mysql
+
+  ls -lah ../../queries-for-mysql
+)
+
+cat > queries-for-mysql.txt <<END
+RNGSEED_QUERIES=$RNGSEED_QUERIES
+END
+
diff --git a/mariadb-tpcds-tooling2/run-queries-pg.sh b/mariadb-tpcds-tooling2/20-run-queries-pg.sh
similarity index 51%
rename from mariadb-tpcds-tooling2/run-queries-pg.sh
rename to mariadb-tpcds-tooling2/20-run-queries-pg.sh
index 16cc0d7..90c608d 100644
--- a/mariadb-tpcds-tooling2/run-queries-pg.sh
+++ b/mariadb-tpcds-tooling2/20-run-queries-pg.sh
@@ -7,5 +7,12 @@ ls queries-for-pg/*.sql | while read a ; do
   ./postgresql-11.2-inst/bin/psql tpcds < $a |  tee benchmark-output-raw.txt
 done
 
-#grep LOG_END benchmark-output-raw.txt | sort | awk '//{printf ("%s %s\n" , $2, $3) }'
+cat << END
+select 
+  query_stream, count(*), sum(query_time_ms) 
+from
+  my_tpcds_result
+group by
+  query_stream;
+END | ./postgresql-11.2-inst/bin/psql tpcds  | tee pg-result.txt
 
diff --git a/mariadb-tpcds-tooling2/100-save-result-pg.sh b/mariadb-tpcds-tooling2/90-save-result-pg.sh
similarity index 100%
rename from mariadb-tpcds-tooling2/100-save-result-pg.sh
rename to mariadb-tpcds-tooling2/90-save-result-pg.sh
diff --git a/mariadb-tpcds-tooling2/prepare-benchmark-and-data.sh b/mariadb-tpcds-tooling2/prepare-benchmark-and-data.sh
deleted file mode 100755
index 71a055f..0000000
--- a/mariadb-tpcds-tooling2/prepare-benchmark-and-data.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-set -e 
-wget http://s.petrunia.net/scratch/tpc-ds-tool.zip
-unzip tpc-ds-tool.zip
-
-# Modify the toolkit to produce MySQL dialect and compile it
-(
-  cd v2.8.0rc4/tools
-  mv makefile makefile.original
-  sed  's/^\(CFLAGS.*\)$/\1 -D_MYSQL/' < makefile.original > makefile
-  make
-)
-
-# Generate the dataset
-(
-  cd v2.8.0rc4/tools
-  mkdir -p ../../data-for-mysql
-  ./dsdgen -scale 1 -rngseed 5678 -dir ../../data-for-mysql
-  du -sh ../../data-for-mysql
-)
-
-
-# Load the dataset
-MYSQL="./mariadb-10.2/client/mysql"
-SOCKET="--socket=/tmp/mysql20.sock"
-MYSQL_USER="-uroot"
-MYSQL_ARGS="$MYSQL_USER $SOCKET"
-
-#$MYSQL $MYSQL_ARGS -e "drop database tpcds"
-$MYSQL $MYSQL_ARGS -e "create database tpcds"
-$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/ddl/tables.sql
-$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/ddl/indexes.sql
-$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/aux-tables.sql
-
-DATA_FILES_DIR="`pwd`/data-for-mysql/"
-
-bash mariadb-tpcds-tooling2/ddl/load-mysql.sql.sh $DATA_FILES_DIR > load-mysql.sql
-
-$MYSQL $MYSQL_ARGS tpcds < load-mysql.sql
-
-
diff --git a/mariadb-tpcds-tooling2/prepare-queries-pg.sh b/mariadb-tpcds-tooling2/prepare-queries-pg.sh
deleted file mode 100644
index ca914b6..0000000
--- a/mariadb-tpcds-tooling2/prepare-queries-pg.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-
-#$MYSQL $MYSQL_ARGS tpcds < mariadb-tpcds-tooling2/aux-tables.sql
-
-# Generate the queries
-(
-  cd v2.8.0rc4-pg
-  patch -p1 < ../mariadb-tpcds-tooling2/adapt-queries-for-pg-v4.diff
-  cd tools
-  mkdir -p ../../queries-for-pg
-./dsqgen \
-  -scale 1 \
-  -dialect netezza \
-  -verbose y \
-  -qualify y  \
-  -rngseed 5678 \
-  -streams 20 \
-  -directory ../query_templates \
-  -input ../query_templates/templates-for-mariadb.lst \
-  -output_dir ../../queries-for-pg
-
-  ls -lah ../../queries-for-pg
-)
-
-
diff --git a/mariadb-tpcds-tooling2/prepare-queries.sh b/mariadb-tpcds-tooling2/prepare-queries.sh
deleted file mode 100755
index a56a843..0000000
--- a/mariadb-tpcds-tooling2/prepare-queries.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Generate the queries
-(
-  cd v2.8.0rc4
-  patch -p1 < ../mariadb-tpcds-tooling2/adapt-queries-for-mariadb-v4.diff
-  cd tools
-  mkdir -p ../../queries-for-mysql
-./dsqgen \
-  -scale 1 \
-  -dialect netezza \
-  -verbose y \
-  -qualify y  \
-  -rngseed 5678 \
-  -streams 20 \
-  -directory ../query_templates \
-  -input ../query_templates/templates-for-mariadb.lst \
-  -output_dir ../../queries-for-mysql
-
-  ls -lah ../../queries-for-mysql
-)
-
-
diff --git a/setup-server/setup-mariadb-current.sh b/setup-server/setup-mariadb-current.sh
index 45c66c7..cc2635d 100755
--- a/setup-server/setup-mariadb-current.sh
+++ b/setup-server/setup-mariadb-current.sh
@@ -1,12 +1,23 @@
 #!/bin/bash
 
 set -e
+
+
 if [ "x${1}y" == "xy" ] ; then 
   echo "Usage: $0 BRANCH_NAME"
   exit 1
 fi
 
+
 BRANCH="${1}"
+HOMEDIR=`pwd`
+DATADIR="$HOMEDIR/mariadb-$BRANCH-data"
+
+if [ -d $DATADIR ] ; then
+  echo "Data directort $DATADIR exists, will not overwrite it "
+  du -hs $DATADIR
+  exit 1;
+fi
 
 git clone --branch $BRANCH --depth 1 https://github.com/MariaDB/server.git mariadb-$BRANCH
 
@@ -20,14 +31,18 @@ make -j8
 cd .. 
 )
 
-HOMEDIR=`pwd`
-DATADIR="$HOMEDIR/mariadb-$BRANCH-data"
 (
   cd mariadb-$BRANCH/mysql-test
   ./mtr alias
   cp -r var/install.db $DATADIR
 )
 
+# Guess a reasonable socket name
+source_dir=`pwd`
+socket_name=`basename $source_dir`
+SOCKETNAME="/tmp/$socket_name"
+
+
 # plugin-load=ha_rocksdb.so
 # default-storage-engine=rocksdb
 # skip-innodb
@@ -48,8 +63,8 @@ log-error
 lc_messages_dir=$HOMEDIR/mariadb-$BRANCH/sql/share
 
 tmpdir=/tmp
-port=3306
-socket=/tmp/mysql20.sock
+port=3341
+socket=$SOCKETNAME
 gdb
 server-id=12
 
@@ -57,6 +72,13 @@ innodb_buffer_pool_size=8G
 
 EOF
 
+cat > mysql-config.sh <<EOF
+MYSQL="./mariadb-$BRANCH/client/mysql"
+MYSQL_SOCKET="--socket=$SOCKETNAME"
+MYSQL_USER="-uroot"
+MYSQL_ARGS="\$MYSQL_USER \$MYSQL_SOCKET"
+EOF
+
 cd $HOMEDIR/mariadb-$BRANCH/sql
 ../sql/mysqld --defaults-file=$HOMEDIR/my-mariadb-$BRANCH.cnf &

    