[Commits] e4ec889c21e: Merge remote-tracking branch 'wsrep/10.3_wsrep_api-26' into 10.4

23 Aug 2018

revision-id: e4ec889c21ef24fbdfef3d0a36828807241877da (mariadb-10.3.6-101-ge4ec889c21e)
parent(s): d6d63f4844bf87adb6250d1b1d43e86feab9a5d0 420cfe4efe72674d4291c0d5a46205ab70b6b91c
author: Jan Lindström
committer: Jan Lindström
timestamp: 2018-08-23 17:14:13 +0300
message:

Merge remote-tracking branch 'wsrep/10.3_wsrep_api-26' into 10.4

Conflicts:
	mysql-test/include/check-testcase.test
	mysql-test/include/wait_until_connected_again.inc
	mysql-test/suite/galera/disabled.def
	mysql-test/suite/galera/r/MW-416.result
	mysql-test/suite/galera/r/galera#500.result
	mysql-test/suite/galera/r/galera_var_dirty_reads.result
	mysql-test/suite/galera/t/MW-416.test
	mysql-test/suite/galera/t/galera#500.test
	sql/handler.cc
	sql/item_strfunc.cc
	sql/item_strfunc.h
	sql/slave.cc
	sql/sql_class.cc
	sql/sql_class.h
	sql/sql_connect.cc
	sql/sql_insert.cc
	sql/sql_parse.cc
	sql/sql_plugin.cc
	sql/sql_prepare.cc
	sql/wsrep_applier.cc
	sql/wsrep_mysqld.cc
	sql/wsrep_mysqld.h
	sql/wsrep_sst.cc
	sql/wsrep_thd.cc
	storage/innobase/handler/ha_innodb.cc


 CMakeLists.txt                                     |    9 +-
 cmake/wsrep.cmake                                  |    2 +-
 include/mysql/service_wsrep.h                      |   59 +-
 include/wsrep.h                                    |    5 +
 mysql-test/include/galera_cluster.inc              |    6 +
 .../galera => }/include/galera_have_debug_sync.inc |    0
 mysql-test/include/galera_wait_sync_point.inc      |   11 +
 mysql-test/include/have_ipv6.inc                   |   20 +
 mysql-test/include/kill_galera.inc                 |   20 +
 mysql-test/include/mtr_warnings.sql                |   46 +
 mysql-test/include/restart_mysqld.inc              |    1 -
 mysql-test/include/wait_wsrep_ready.inc            |   13 +
 mysql-test/include/wsrep_wait_disconnect.inc       |   20 +
 mysql-test/lib/My/ConfigFactory.pm                 |    6 +-
 mysql-test/main/mysqld--help-notwin.result         | 1472 ++++++++++++++
 .../main/query_cache_size_functionality.result     |  226 +++
 .../main/query_cache_type_functionality.result     |  250 +++
 mysql-test/mysql-test-run.pl                       |  124 ++
 mysql-test/suite/galera/disabled.def               |    2 +-
 mysql-test/suite/galera/galera_2nodes.cnf          |   25 +-
 .../suite/galera/galera_2nodes_as_master.cnf       |   12 +
 mysql-test/suite/galera/galera_2nodes_as_slave.cnf |   47 +-
 mysql-test/suite/galera/galera_3nodes_as_slave.cnf |   59 +-
 mysql-test/suite/galera/galera_4nodes.cnf          |   15 +
 .../suite/galera/include/galera_load_provider.inc  |   68 +
 mysql-test/suite/galera/include/galera_resume.inc  |    2 +-
 .../suite/galera/include/galera_sst_restore.inc    |    2 +-
 .../suite/galera/include/galera_st_clean_slave.inc |    1 +
 .../galera/include/galera_st_disconnect_slave.inc  |    8 +
 .../galera/include/galera_unload_provider.inc      |    8 +
 mysql-test/suite/galera/r/GAL-382.result           |    2 +
 mysql-test/suite/galera/r/GAL-401.result           |    2 +
 mysql-test/suite/galera/r/GCF-1081.result          |   47 +
 mysql-test/suite/galera/r/GCF-939.result           |   12 +
 mysql-test/suite/galera/r/MDEV-15443.result        |    2 +
 mysql-test/suite/galera/r/MW-252.result            |    2 +
 mysql-test/suite/galera/r/MW-258.result            |    2 +
 mysql-test/suite/galera/r/MW-259.result            |    2 +
 mysql-test/suite/galera/r/MW-284.result            |    4 +
 mysql-test/suite/galera/r/MW-285.result            |    2 +
 mysql-test/suite/galera/r/MW-292.result            |   27 +-
 mysql-test/suite/galera/r/MW-309.result            |    2 +
 mysql-test/suite/galera/r/MW-313.result            |    2 +
 mysql-test/suite/galera/r/MW-328A.result           |   18 +-
 mysql-test/suite/galera/r/MW-328B.result           |    2 +
 mysql-test/suite/galera/r/MW-328C.result           |    2 +
 mysql-test/suite/galera/r/MW-328D.result           |    2 +
 mysql-test/suite/galera/r/MW-328E.result           |    2 +
 mysql-test/suite/galera/r/MW-329.result            |    9 +-
 mysql-test/suite/galera/r/MW-336.result            |    2 +
 mysql-test/suite/galera/r/MW-357.result            |    2 +
 mysql-test/suite/galera/r/MW-360.result            |   41 +
 mysql-test/suite/galera/r/MW-369.result            |   65 +-
 mysql-test/suite/galera/r/MW-388.result            |    4 +-
 mysql-test/suite/galera/r/MW-402.result            |   76 +-
 mysql-test/suite/galera/r/MW-416.result            |   20 +-
 mysql-test/suite/galera/r/MW-86-wait1.result       |   17 +-
 mysql-test/suite/galera/r/MW-86-wait8.result       |   16 +-
 mysql-test/suite/galera/r/MW-86.result             |   78 +
 mysql-test/suite/galera/r/basic.result             |    2 +
 mysql-test/suite/galera/r/binlog_checksum.result   |    2 +
 mysql-test/suite/galera/r/create.result            |    2 +
 .../suite/galera/r/enforce_storage_engine.result   |    2 +
 .../suite/galera/r/enforce_storage_engine2.result  |    2 +
 mysql-test/suite/galera/r/ev51914.result           |    2 +
 mysql-test/suite/galera/r/fk.result                |    2 +
 mysql-test/suite/galera/r/galera#414.result        |    2 +
 mysql-test/suite/galera/r/galera#500.result        |    6 +
 mysql-test/suite/galera/r/galera_admin.result      |    2 +
 .../galera/r/galera_alter_engine_innodb.result     |    2 +
 .../galera/r/galera_alter_engine_myisam.result     |    2 +
 .../suite/galera/r/galera_alter_table_force.result |    2 +
 .../galera/r/galera_applier_ftwrl_table.result     |    2 +
 .../r/galera_applier_ftwrl_table_alter.result      |    2 +
 mysql-test/suite/galera/r/galera_as_master.result  |    4 +
 .../suite/galera/r/galera_as_master_gtid.result    |   44 +-
 .../suite/galera/r/galera_as_master_large.result   |    2 +
 .../suite/galera/r/galera_as_slave_autoinc.result  |    4 +
 .../suite/galera/r/galera_as_slave_gtid.result     |    2 +
 .../r/galera_as_slave_gtid_replicate_do_db.result  |  160 ++
 .../galera_as_slave_gtid_replicate_do_db_cc.result |  315 +++
 .../suite/galera/r/galera_as_slave_nonprim.result  |   18 +
 .../galera/r/galera_autoinc_sst_xtrabackup.result  |    2 +
 mysql-test/suite/galera/r/galera_bf_abort.result   |    4 +-
 .../r/galera_bf_abort_flush_for_export.result      |    2 +
 .../galera/r/galera_bf_abort_for_update.result     |    6 +-
 .../suite/galera/r/galera_bf_abort_ftwrl.result    |    2 +
 .../suite/galera/r/galera_bf_abort_get_lock.result |    4 +-
 .../galera/r/galera_bf_abort_group_commit.result   |  685 +++++++
 .../galera/r/galera_bf_abort_lock_table.result     |    2 +
 .../suite/galera/r/galera_bf_abort_shutdown.result |    9 +
 .../suite/galera/r/galera_bf_abort_sleep.result    |    4 +-
 .../suite/galera/r/galera_bf_lock_wait.result      |    2 +
 .../suite/galera/r/galera_binlog_cache_size.result |    2 +
 .../suite/galera/r/galera_binlog_checksum.result   |    2 +
 .../r/galera_binlog_event_max_size_max.result      |    2 +
 .../r/galera_binlog_event_max_size_min.result      |    2 +
 .../suite/galera/r/galera_binlog_row_image.result  |    2 +
 .../suite/galera/r/galera_create_function.result   |    2 +
 .../suite/galera/r/galera_create_procedure.result  |    2 +
 .../galera/r/galera_create_table_as_select.result  |  103 +
 .../suite/galera/r/galera_create_table_like.result |    2 +
 .../suite/galera/r/galera_create_trigger.result    |    2 +
 .../suite/galera/r/galera_ddl_multiline.result     |    2 +
 mysql-test/suite/galera/r/galera_defaults.result   |   25 +-
 .../suite/galera/r/galera_delete_limit.result      |    2 +
 .../suite/galera/r/galera_desync_overlapped.result |    2 +
 mysql-test/suite/galera/r/galera_drop_multi.result |    2 +
 mysql-test/suite/galera/r/galera_enum.result       |    4 +-
 mysql-test/suite/galera/r/galera_events.result     |    2 +
 .../suite/galera/r/galera_fk_cascade_delete.result |    2 +
 .../suite/galera/r/galera_fk_cascade_update.result |    2 +
 .../suite/galera/r/galera_fk_conflict.result       |    4 +-
 .../suite/galera/r/galera_fk_mismatch.result       |    2 +
 .../suite/galera/r/galera_fk_multicolumn.result    |    2 +
 .../suite/galera/r/galera_fk_multitable.result     |    2 +
 mysql-test/suite/galera/r/galera_fk_no_pk.result   |    2 +
 .../galera/r/galera_fk_selfreferential.result      |    2 +
 mysql-test/suite/galera/r/galera_fk_setnull.result |    2 +
 .../suite/galera/r/galera_flush_local.result       |    2 +
 .../galera/r/galera_forced_binlog_format.result    |   20 +-
 mysql-test/suite/galera/r/galera_ftwrl.result      |    2 +
 .../suite/galera/r/galera_ftwrl_drain.result       |    2 +
 mysql-test/suite/galera/r/galera_fulltext.result   |    2 +
 .../r/galera_gcache_recover_full_gcache.result     |    2 +-
 .../suite/galera/r/galera_gcs_fc_limit.result      |    2 +
 .../galera/r/galera_gcs_max_packet_size.result     |    2 +
 mysql-test/suite/galera/r/galera_gra_log.result    |    2 +
 mysql-test/suite/galera/r/galera_gtid.result       |    2 +
 mysql-test/suite/galera/r/galera_gtid_slave.result |   11 +-
 .../galera/r/galera_gtid_slave_sst_rsync.result    |  106 +-
 .../suite/galera/r/galera_insert_ignore.result     |    2 +
 .../suite/galera/r/galera_insert_multi.result      |    4 +-
 .../galera/r/galera_ist_innodb_flush_logs.result   |   24 +-
 .../suite/galera/r/galera_ist_progress.result      |    5 +
 .../suite/galera/r/galera_ist_recv_bind.result     |    2 +
 .../galera/r/galera_ist_restart_joiner.result      |    3 +-
 mysql-test/suite/galera/r/galera_ist_rsync.result  |    2 +
 .../suite/galera/r/galera_ist_xtrabackup-v2.result |   40 +-
 mysql-test/suite/galera/r/galera_kill_ddl.result   |    2 +
 .../suite/galera/r/galera_kill_largechanges.result |    2 +
 .../suite/galera/r/galera_kill_smallchanges.result |    2 +
 mysql-test/suite/galera/r/galera_lock_table.result |    2 +
 mysql-test/suite/galera/r/galera_log_bin.result    |    4 +
 .../suite/galera/r/galera_log_output_csv.result    |    2 +
 .../suite/galera/r/galera_many_columns.result      |    4 +-
 .../suite/galera/r/galera_many_indexes.result      |    4 +-
 mysql-test/suite/galera/r/galera_many_rows.result  |    4 +-
 .../suite/galera/r/galera_many_tables_nopk.result  |    4 +-
 .../suite/galera/r/galera_many_tables_pk.result    |    4 +-
 mysql-test/suite/galera/r/galera_mdev_10812.result |    2 +
 mysql-test/suite/galera/r/galera_mdev_13787.result |    2 +
 mysql-test/suite/galera/r/galera_mdl_race.result   |    4 +-
 .../suite/galera/r/galera_multi_database.result    |    2 +
 .../suite/galera/r/galera_myisam_autocommit.result |    2 +
 .../galera/r/galera_myisam_transactions.result     |    2 +
 mysql-test/suite/galera/r/galera_nopk_bit.result   |    4 +-
 mysql-test/suite/galera/r/galera_nopk_blob.result  |    4 +-
 .../galera/r/galera_nopk_large_varchar.result      |    4 +-
 .../suite/galera/r/galera_nopk_unicode.result      |    4 +-
 .../r/galera_parallel_apply_lock_table.result      |    6 +-
 .../r/galera_parallel_autoinc_largetrx.result      |    4 +-
 .../r/galera_parallel_autoinc_manytrx.result       |    2 +
 .../suite/galera/r/galera_parallel_simple.result   |    2 +
 .../suite/galera/r/galera_pk_bigint_signed.result  |    4 +-
 .../galera/r/galera_pk_bigint_unsigned.result      |    4 +-
 .../galera/r/galera_prepared_statement.result      |    2 +
 .../suite/galera/r/galera_query_cache.result       |    2 +
 .../galera/r/galera_query_cache_sync_wait.result   |    2 +
 mysql-test/suite/galera/r/galera_read_only.result  |    2 +
 .../galera/r/galera_repl_key_format_flat16.result  |    2 +
 .../suite/galera/r/galera_repl_max_ws_size.result  |    4 +-
 .../suite/galera/r/galera_restart_nochanges.result |    2 +
 .../r/galera_restart_on_unknown_option.result      |    2 +
 mysql-test/suite/galera/r/galera_rsu_add_pk.result |    2 +
 .../suite/galera/r/galera_rsu_drop_pk.result       |    2 +
 mysql-test/suite/galera/r/galera_rsu_error.result  |    2 +
 mysql-test/suite/galera/r/galera_rsu_simple.result |    2 +
 .../suite/galera/r/galera_rsu_wsrep_desync.result  |    2 +
 mysql-test/suite/galera/r/galera_sbr.result        |    2 +
 mysql-test/suite/galera/r/galera_sbr_binlog.result |    2 +
 .../galera/r/galera_schema_dirty_reads.result      |    2 +
 .../suite/galera/r/galera_serializable.result      |    8 +-
 mysql-test/suite/galera/r/galera_server.result     |    2 +
 .../suite/galera/r/galera_sql_log_bin_zero.result  |    2 +
 mysql-test/suite/galera/r/galera_ssl.result        |    2 +
 .../suite/galera/r/galera_ssl_compression.result   |    2 +
 .../suite/galera/r/galera_sst_mysqldump.result     |  114 ++
 mysql-test/suite/galera/r/galera_sst_rsync.result  |    2 +-
 .../suite/galera/r/galera_sst_xtrabackup-v2.result |  110 ++
 ...alera_sst_xtrabackup-v2_encrypt_with_key.result |    2 +
 .../suite/galera/r/galera_status_cluster.result    |    2 +
 .../galera/r/galera_status_local_index.result      |    2 +
 .../galera/r/galera_status_local_state.result      |    2 +
 .../suite/galera/r/galera_suspend_slave.result     |    2 +
 .../suite/galera/r/galera_sync_wait_show.result    |    2 +
 .../r/galera_toi_alter_auto_increment.result       |    2 +
 .../suite/galera/r/galera_toi_ddl_error.result     |    5 +
 .../suite/galera/r/galera_toi_ddl_fk_update.result |    2 +
 .../suite/galera/r/galera_toi_ddl_locking.result   |   32 +-
 .../galera/r/galera_toi_ddl_nonconflicting.result  |    2 +
 .../galera/r/galera_toi_ddl_sequential.result      |    2 +
 .../suite/galera/r/galera_toi_drop_database.result |    6 +-
 mysql-test/suite/galera/r/galera_toi_ftwrl.result  |    2 +
 .../galera/r/galera_toi_lock_exclusive.result      |    4 +-
 .../suite/galera/r/galera_toi_lock_shared.result   |    2 +
 .../suite/galera/r/galera_toi_truncate.result      |    4 +-
 .../galera/r/galera_transaction_read_only.result   |    2 +
 .../galera/r/galera_transaction_replay.result      |   96 +-
 mysql-test/suite/galera/r/galera_truncate.result   |    2 +
 .../galera/r/galera_truncate_temporary.result      |    2 +
 .../galera/r/galera_unicode_identifiers.result     |    2 +
 mysql-test/suite/galera/r/galera_unicode_pk.result |    6 +-
 .../suite/galera/r/galera_update_limit.result      |    2 +
 .../suite/galera/r/galera_v1_row_events.result     |    2 +
 .../suite/galera/r/galera_var_OSU_method.result    |    2 +
 .../suite/galera/r/galera_var_OSU_method2.result   |    2 +
 .../r/galera_var_auto_inc_control_off.result       |    4 +-
 .../galera/r/galera_var_certify_nonPK_off.result   |    2 +
 .../galera/r/galera_var_cluster_address.result     |    6 +-
 .../suite/galera/r/galera_var_desync_on.result     |    2 +
 .../suite/galera/r/galera_var_dirty_reads.result   |    1 -
 .../suite/galera/r/galera_var_fkchecks.result      |    2 +
 .../galera/r/galera_var_gtid_domain_id.result      |    2 +
 .../galera/r/galera_var_ignore_apply_errors.result |  154 ++
 .../r/galera_var_innodb_disallow_writes.result     |    2 +
 .../galera/r/galera_var_load_data_splitting.result |    2 +
 .../suite/galera/r/galera_var_log_bin.result       |    2 +
 .../suite/galera/r/galera_var_max_ws_rows.result   |    2 +
 .../suite/galera/r/galera_var_max_ws_size.result   |    4 +-
 .../r/galera_var_mysql_replication_bundle.result   |    2 +
 .../suite/galera/r/galera_var_node_address.result  |    2 +
 .../galera/r/galera_var_reject_queries.result      |    2 +
 .../r/galera_var_replicate_myisam_off.result       |    2 +
 .../galera/r/galera_var_replicate_myisam_on.result |    2 +
 .../suite/galera/r/galera_var_slave_threads.result |  210 +-
 .../suite/galera/r/galera_var_sst_auth.result      |    2 +
 .../suite/galera/r/galera_var_sync_wait.result     |    2 +
 .../suite/galera/r/galera_var_wsrep_on_off.result  |    2 +
 mysql-test/suite/galera/r/galera_wan.result        |    2 +
 .../suite/galera/r/galera_wan_restart_ist.result   |    2 +
 .../suite/galera/r/galera_wan_restart_sst.result   |    2 +
 .../galera/r/galera_wsrep_desync_wsrep_on.result   |    2 +
 .../galera/r/galera_wsrep_log_conficts.result      |    4 +-
 .../suite/galera/r/galera_wsrep_new_cluster.result |    2 +
 .../r/galera_wsrep_provider_options_syntax.result  |    2 +
 .../galera/r/galera_zero_length_column.result      |    2 +
 mysql-test/suite/galera/r/grant.result             |    2 +
 mysql-test/suite/galera/r/lp1276424.result         |    2 +
 mysql-test/suite/galera/r/lp1347768.result         |    2 +
 mysql-test/suite/galera/r/lp1376747-2.result       |    2 +
 mysql-test/suite/galera/r/lp1376747-3.result       |    2 +
 mysql-test/suite/galera/r/lp1376747-4.result       |    2 +
 mysql-test/suite/galera/r/lp1376747.result         |    2 +
 mysql-test/suite/galera/r/lp1438990.result         |    2 +
 mysql-test/suite/galera/r/lp959512.result          |    2 +
 mysql-test/suite/galera/r/mdev_10518.result        |    2 +
 mysql-test/suite/galera/r/mdev_9290.result         |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#110.result   |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#198.result   |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#201.result   |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#237.result   |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#247.result   |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#31.result    |    2 +
 mysql-test/suite/galera/r/mysql-wsrep#90.result    |    2 +
 mysql-test/suite/galera/r/partition.result         |    6 +-
 mysql-test/suite/galera/r/pxc-421.result           |    2 +
 mysql-test/suite/galera/r/query_cache.result       |    2 +
 mysql-test/suite/galera/r/rename.result            |    2 +
 mysql-test/suite/galera/r/rpl_row_annotate.result  |    8 +-
 mysql-test/suite/galera/r/sql_log_bin.result       |    2 +
 mysql-test/suite/galera/r/unique_key.result        |    2 +
 mysql-test/suite/galera/r/view.result              |    2 +
 .../galera/r/wsrep_trx_fragment_size_non_sr.result |   25 +
 .../galera/r/wsrep_trx_fragment_size_sr.result     |   13 +
 mysql-test/suite/galera/t/GAL-419.test             |    4 +-
 mysql-test/suite/galera/t/GCF-1081.test            |   72 +
 mysql-test/suite/galera/t/GCF-939.test             |   25 +
 mysql-test/suite/galera/t/MW-284.test              |    2 +
 mysql-test/suite/galera/t/MW-292.test              |   50 +-
 mysql-test/suite/galera/t/MW-328A.test             |   40 +-
 mysql-test/suite/galera/t/MW-329.test              |   28 +-
 mysql-test/suite/galera/t/MW-336.test              |   10 +-
 mysql-test/suite/galera/t/MW-360-master.opt        |    2 +
 mysql-test/suite/galera/t/MW-360.test              |  100 +
 mysql-test/suite/galera/t/MW-369.inc               |    7 +-
 mysql-test/suite/galera/t/MW-369.test              |    1 -
 mysql-test/suite/galera/t/MW-388.test              |    1 -
 mysql-test/suite/galera/t/MW-402.test              |   56 +-
 mysql-test/suite/galera/t/MW-416.test              |   24 +-
 mysql-test/suite/galera/t/MW-86-master.opt         |    1 +
 mysql-test/suite/galera/t/MW-86-wait1.test         |   13 +-
 mysql-test/suite/galera/t/MW-86-wait8.test         |   11 +-
 mysql-test/suite/galera/t/MW-86.test               |  193 ++
 mysql-test/suite/galera/t/disabled.def             |   13 +
 .../t/galera_applier_ftwrl_table_alter-master.opt  |    2 +-
 mysql-test/suite/galera/t/galera_as_master.test    |    2 +
 .../suite/galera/t/galera_as_master_gtid.test      |   24 +-
 .../t/galera_as_master_gtid_change_master.test     |    2 +
 mysql-test/suite/galera/t/galera_as_slave.test     |   19 +-
 .../suite/galera/t/galera_as_slave_autoinc.test    |   20 +-
 .../suite/galera/t/galera_as_slave_gtid.test       |   21 +-
 .../t/galera_as_slave_gtid_replicate_do_db.cnf     |   17 +
 .../t/galera_as_slave_gtid_replicate_do_db.test    |  150 ++
 .../t/galera_as_slave_gtid_replicate_do_db_cc.test |  176 ++
 .../suite/galera/t/galera_as_slave_nonprim.test    |   28 +-
 .../suite/galera/t/galera_as_slave_preordered.test |   19 +-
 .../t/galera_as_slave_replication_bundle.test      |   13 +-
 .../galera/t/galera_autoinc_sst_xtrabackup.cnf     |    2 +-
 .../galera/t/galera_bf_abort_group_commit.cnf      |   15 +
 .../galera/t/galera_bf_abort_group_commit.test     |   76 +
 .../suite/galera/t/galera_bf_abort_shutdown.test   |   22 +
 .../galera/t/galera_create_table_as_select.test    |  145 ++
 mysql-test/suite/galera/t/galera_defaults.test     |    4 +-
 .../galera/t/galera_forced_binlog_format.test      |    5 +-
 mysql-test/suite/galera/t/galera_ftwrl_drain.test  |    2 +-
 .../t/galera_gcache_recover_full_gcache.test       |    4 +-
 mysql-test/suite/galera/t/galera_gtid-master.opt   |    2 +-
 mysql-test/suite/galera/t/galera_gtid_slave.test   |   14 +
 .../galera/t/galera_gtid_slave_sst_rsync.test      |  109 +-
 .../galera/t/galera_ist_innodb_flush_logs.cnf      |    2 +-
 mysql-test/suite/galera/t/galera_ist_progress.test |    4 +-
 .../suite/galera/t/galera_ist_restart_joiner.test  |    2 +-
 .../suite/galera/t/galera_ist_xtrabackup-v2.cnf    |    2 +-
 mysql-test/suite/galera/t/galera_log_bin.test      |    2 +
 mysql-test/suite/galera/t/galera_migrate.cnf       |    2 +
 .../galera/t/galera_parallel_apply_lock_table.test |    4 +-
 .../suite/galera/t/galera_parallel_simple.test     |    2 +-
 mysql-test/suite/galera/t/galera_pc_recovery.test  |   93 +
 mysql-test/suite/galera/t/galera_split_brain.test  |    6 +-
 mysql-test/suite/galera/t/galera_ssl_upgrade.test  |    4 +
 .../suite/galera/t/galera_sst_mysqldump.test       |    3 +-
 .../galera/t/galera_sst_xtrabackup-v2-options.cnf  |    2 +-
 .../suite/galera/t/galera_sst_xtrabackup-v2.cnf    |    2 +-
 .../galera_sst_xtrabackup-v2_encrypt_with_key.cnf  |    2 +-
 .../suite/galera/t/galera_toi_ddl_error.test       |    5 +
 .../suite/galera/t/galera_toi_ddl_locking.test     |   53 +-
 .../suite/galera/t/galera_transaction_replay.test  |  201 +-
 .../suite/galera/t/galera_var_cluster_address.test |    9 +-
 .../suite/galera/t/galera_var_dirty_reads.test     |    9 +-
 .../galera/t/galera_var_ignore_apply_errors.test   |  235 +++
 mysql-test/suite/galera/t/galera_var_log_bin.cnf   |    5 +
 .../suite/galera/t/galera_var_slave_threads.test   |  123 +-
 .../galera/t/galera_vote_drop_temporary-master.opt |    1 +
 .../suite/galera/t/galera_wsrep_new_cluster.test   |    1 -
 .../suite/galera/t/mysql-wsrep#198-master.opt      |    1 +
 mysql-test/suite/galera/t/partition.test           |   16 +-
 mysql-test/suite/galera/t/rpl_row_annotate.test    |    6 +-
 .../galera/t/wsrep_trx_fragment_size_non_sr.test   |   26 +
 .../suite/galera/t/wsrep_trx_fragment_size_sr.test |   22 +
 mysql-test/suite/galera_3nodes/galera_3nodes.cnf   |    1 +
 .../suite/galera_3nodes/include/galera_suspend.inc |    2 +-
 .../r/galera_certification_ccc.result              |    2 +
 .../r/galera_certification_double_failure.result   |    3 +
 .../r/galera_ipv6_xtrabackup-v2.result             |    3 -
 .../r/galera_ist_gcache_rollover.result            |    2 +
 .../galera_3nodes/r/galera_pc_bootstrap.result     |   13 +
 .../suite/galera_3nodes/r/galera_pc_weight.result  |    8 +-
 .../galera_3nodes/r/galera_wsrep_schema.result     |   77 +
 mysql-test/suite/galera_3nodes/t/GAL-501.test      |    6 +
 .../t/galera_certification_double_failure.test     |    2 +
 .../t/galera_evs_suspect_timeout.test              |    6 +-
 mysql-test/suite/galera_3nodes/t/galera_garbd.test |    5 +-
 .../galera_3nodes/t/galera_ipv6_xtrabackup-v2.cnf  |    4 +
 .../galera_3nodes/t/galera_ipv6_xtrabackup-v2.test |   30 +-
 .../t/galera_ist_gcache_rollover.test              |    2 +-
 .../suite/galera_3nodes/t/galera_pc_weight.cnf     |    8 +-
 .../suite/galera_3nodes/t/galera_pc_weight.test    |   34 +-
 .../t/galera_slave_options_ignore.test             |    1 +
 .../suite/galera_3nodes/t/galera_wsrep_schema.test |   74 +
 .../suite/galera_3nodes_ee/galera_3nodes.cnf       |    1 +
 .../include/galera_check_voting_recovery.inc       |   58 +
 mysql-test/suite/galera_3nodes_ee/my.cnf           |    1 +
 mysql-test/suite/galera_3nodes_ee/r/GCF-354.result |   23 +
 mysql-test/suite/galera_3nodes_ee/r/GCF-361.result |   16 +
 mysql-test/suite/galera_3nodes_ee/r/GCF-363.result |   31 +
 mysql-test/suite/galera_3nodes_ee/r/GCF-376.result |   48 +
 .../galera_3nodes_ee/r/galera-features#115.result  |   21 +
 .../galera_3nodes_ee/r/galera-features#119.result  |   19 +
 .../galera_3nodes_ee/r/galera-features#79.result   |   32 +
 .../r/galera_nbo_kill_master.result                |   27 +
 .../r/galera_nbo_kill_slave_ist.result             |   35 +
 .../r/galera_nbo_kill_slave_sst.result             |   33 +
 .../r/galera_nbo_master_majority_failure.result    |   46 +
 .../r/galera_nbo_master_majority_success.result    |   50 +
 .../r/galera_nbo_master_minority_failure.result    |   52 +
 .../r/galera_nbo_master_minority_success.result    |   52 +
 .../r/galera_nbo_master_non_prim_failure.result    |   42 +
 .../r/galera_nbo_master_non_prim_success.result    |   32 +
 .../r/galera_nbo_shutdown_slave_ist.result         |   28 +
 .../r/galera_nbo_slave_non_prim.result             |   35 +
 .../galera_3nodes_ee/r/galera_toi_vote.result      |   19 +
 .../r/galera_vote_rejoin_mysqldump.result          |   60 +
 .../suite/galera_3nodes_ee/r/galera_vote_sr.result |   52 +
 .../r/mysql-wsrep-features#131.result              |   12 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-354.cnf    |    4 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-354.test   |   59 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-361.cnf    |    4 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-361.test   |   47 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-363.cnf    |    8 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-363.test   |   68 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-376.cnf    |    4 +
 mysql-test/suite/galera_3nodes_ee/t/GCF-376.test   |   84 +
 .../galera_3nodes_ee/t/galera-features#115.cnf     |    4 +
 .../galera_3nodes_ee/t/galera-features#115.test    |   67 +
 .../galera_3nodes_ee/t/galera-features#119.test    |   61 +
 .../galera_3nodes_ee/t/galera-features#79.test     |   75 +
 .../galera_3nodes_ee/t/galera_nbo_kill_master.test |   73 +
 .../t/galera_nbo_kill_slave_ist.test               |   73 +
 .../t/galera_nbo_kill_slave_sst.cnf                |   11 +
 .../t/galera_nbo_kill_slave_sst.test               |   70 +
 .../t/galera_nbo_master_majority_failure.cnf       |    4 +
 .../t/galera_nbo_master_majority_failure.test      |   38 +
 .../t/galera_nbo_master_majority_success.cnf       |    4 +
 .../t/galera_nbo_master_majority_success.test      |   38 +
 .../t/galera_nbo_master_minority_failure.cnf       |    4 +
 .../t/galera_nbo_master_minority_failure.test      |   35 +
 .../t/galera_nbo_master_minority_success.cnf       |    4 +
 .../t/galera_nbo_master_minority_success.test      |   41 +
 .../t/galera_nbo_master_non_prim_failure.test      |   92 +
 .../t/galera_nbo_master_non_prim_success.test      |   80 +
 .../t/galera_nbo_shutdown_slave_ist.test           |   66 +
 .../t/galera_nbo_slave_non_prim.test               |   69 +
 .../suite/galera_3nodes_ee/t/galera_toi_vote.cnf   |    4 +
 .../suite/galera_3nodes_ee/t/galera_toi_vote.test  |   63 +
 .../t/galera_vote_rejoin_mysqldump.cnf             |    4 +
 .../t/galera_vote_rejoin_mysqldump.test            |   80 +
 .../galera_3nodes_ee/t/galera_vote_sr-master.opt   |    2 +
 .../suite/galera_3nodes_ee/t/galera_vote_sr.test   |   87 +
 .../t/mysql-wsrep-features#131.test                |   39 +
 .../suite/galera_3nodes_sr/galera_3nodes.cnf       |    1 +
 mysql-test/suite/galera_3nodes_sr/my.cnf           |    1 +
 mysql-test/suite/galera_3nodes_sr/r/GCF-336.result |   26 +
 mysql-test/suite/galera_3nodes_sr/r/GCF-582.result |   23 +
 mysql-test/suite/galera_3nodes_sr/r/GCF-606.result |   23 +
 mysql-test/suite/galera_3nodes_sr/r/GCF-609.result |   20 +
 .../suite/galera_3nodes_sr/r/GCF-810A.result       |  256 +++
 .../suite/galera_3nodes_sr/r/GCF-810B.result       |  100 +
 .../suite/galera_3nodes_sr/r/GCF-810C.result       |  177 ++
 mysql-test/suite/galera_3nodes_sr/r/GCF-817.result |   33 +
 mysql-test/suite/galera_3nodes_sr/r/GCF-832.result |   15 +
 .../r/galera_sr_isolate_master.result              |   62 +
 .../galera_3nodes_sr/r/galera_sr_join_slave.result |   25 +
 .../r/galera_sr_kill_master.result                 |   22 +
 .../r/galera_sr_kill_slave_after_apply.result      |   38 +
 ...alera_sr_kill_slave_after_apply_rollback.result |   44 +
 ...lera_sr_kill_slave_after_apply_rollback2.result |   31 +
 .../r/galera_sr_kill_slave_before_apply.result     |   27 +
 .../r/galera_sr_threeway_split.result              |   84 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-336.test   |   47 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-582.test   |   39 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-606.test   |   80 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-609.test   |   30 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-810A.test  |  137 ++
 mysql-test/suite/galera_3nodes_sr/t/GCF-810B.test  |   49 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-810C.test  |   70 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-817.test   |  101 +
 mysql-test/suite/galera_3nodes_sr/t/GCF-832.test   |   36 +
 mysql-test/suite/galera_3nodes_sr/t/disabled.def   |    0
 .../t/galera_sr_isolate_master.test                |  127 ++
 .../galera_3nodes_sr/t/galera_sr_join_slave.test   |   56 +
 .../galera_3nodes_sr/t/galera_sr_kill_master.test  |   56 +
 .../t/galera_sr_kill_slave_after_apply.test        |   75 +
 .../galera_sr_kill_slave_after_apply_rollback.test |   76 +
 ...galera_sr_kill_slave_after_apply_rollback2.test |   56 +
 .../t/galera_sr_kill_slave_before_apply.test       |   71 +
 .../t/galera_sr_threeway_split.cnf                 |    5 +
 .../t/galera_sr_threeway_split.test                |  169 ++
 mysql-test/suite/galera_ee/galera_2nodes.cnf       |    1 +
 mysql-test/suite/galera_ee/galera_4nodes.cnf       |    1 +
 mysql-test/suite/galera_ee/my.cnf                  |    1 +
 mysql-test/suite/galera_ee/r/GCF-329A.result       |   30 +
 mysql-test/suite/galera_ee/r/GCF-329B.result       |   16 +
 mysql-test/suite/galera_ee/r/GCF-360.result        |  416 ++++
 mysql-test/suite/galera_ee/r/GCF-421.result        |   14 +
 mysql-test/suite/galera_ee/r/GCF-546.result        |   28 +
 mysql-test/suite/galera_ee/r/GCF-563.result        |   52 +
 mysql-test/suite/galera_ee/r/GCF-849.result        |   11 +
 mysql-test/suite/galera_ee/r/GCF-854.result        |   16 +
 .../suite/galera_ee/r/galera-features#117.result   |   29 +
 .../r/galera_nbo_alter_conflicting.result          |   14 +
 .../suite/galera_ee/r/galera_nbo_alter_copy.result |   44 +
 .../galera_ee/r/galera_nbo_alter_engine.result     |   22 +
 .../r/galera_nbo_alter_error_duplicate.result      |   30 +
 .../galera_ee/r/galera_nbo_alter_exclusive.result  |   37 +
 .../galera_ee/r/galera_nbo_alter_inplace.result    |   39 +
 .../galera_ee/r/galera_nbo_alter_multi.result      |   11 +
 .../r/galera_nbo_alter_nonconflicting.result       |   29 +
 .../galera_ee/r/galera_nbo_alter_parallel.result   |   45 +
 .../galera_ee/r/galera_nbo_alter_partition.result  |   26 +
 .../galera_ee/r/galera_nbo_alter_rename.result     |   19 +
 .../suite/galera_ee/r/galera_nbo_alter_toi.result  |   38 +
 .../galera_ee/r/galera_nbo_create_index.result     |   36 +
 .../suite/galera_ee/r/galera_nbo_ddl_error.result  |   22 +
 .../galera_ee/r/galera_nbo_error_on_all.result     |   13 +
 .../suite/galera_ee/r/galera_nbo_local_mdl.result  |   12 +
 .../galera_ee/r/galera_nbo_processlist.result      |   35 +
 .../suite/galera_ee/r/galera_nbo_sst_slave.result  |   23 +
 .../galera_ee/r/galera_nbo_temporary_table.result  |   14 +
 .../galera_ee/r/galera_nbo_unsupported.result      |   26 +
 .../galera_ee/r/galera_vote_rejoin_ddl.result      |   44 +
 .../galera_ee/r/galera_vote_rejoin_dml.result      |   47 +
 .../galera_ee/r/mysql-wsrep-features#127.result    |    5 +
 .../galera_ee/r/mysql-wsrep-features#128.result    |    7 +
 .../galera_ee/r/mysql-wsrep-features#131.result    |   12 +
 .../galera_ee/r/mysql-wsrep-features#132.result    |   18 +
 mysql-test/suite/galera_ee/t/GCF-329A.test         |   42 +
 mysql-test/suite/galera_ee/t/GCF-329B.test         |   53 +
 mysql-test/suite/galera_ee/t/GCF-360.cnf           |   17 +
 mysql-test/suite/galera_ee/t/GCF-360.test          |   64 +
 mysql-test/suite/galera_ee/t/GCF-421.test          |   46 +
 mysql-test/suite/galera_ee/t/GCF-546.test          |   47 +
 mysql-test/suite/galera_ee/t/GCF-563.test          |   63 +
 mysql-test/suite/galera_ee/t/GCF-849.test          |   39 +
 mysql-test/suite/galera_ee/t/GCF-854.test          |   42 +
 .../suite/galera_ee/t/galera-features#117.cnf      |    4 +
 .../suite/galera_ee/t/galera-features#117.test     |   37 +
 .../galera_ee/t/galera_nbo_alter_conflicting.test  |   57 +
 .../suite/galera_ee/t/galera_nbo_alter_copy.test   |   67 +
 .../suite/galera_ee/t/galera_nbo_alter_engine.test |   31 +
 .../t/galera_nbo_alter_error_duplicate.test        |   34 +
 .../galera_ee/t/galera_nbo_alter_exclusive.test    |   64 +
 .../galera_ee/t/galera_nbo_alter_inplace.test      |   67 +
 .../suite/galera_ee/t/galera_nbo_alter_multi.test  |   15 +
 .../t/galera_nbo_alter_nonconflicting.test         |   57 +
 .../galera_ee/t/galera_nbo_alter_parallel.test     |   65 +
 .../galera_ee/t/galera_nbo_alter_partition.test    |   29 +
 .../suite/galera_ee/t/galera_nbo_alter_rename.test |   31 +
 .../suite/galera_ee/t/galera_nbo_alter_toi.test    |   53 +
 .../suite/galera_ee/t/galera_nbo_create_index.test |   34 +
 .../suite/galera_ee/t/galera_nbo_ddl_error.test    |   38 +
 .../suite/galera_ee/t/galera_nbo_error_on_all.test |   21 +
 .../suite/galera_ee/t/galera_nbo_local_mdl.test    |   55 +
 .../suite/galera_ee/t/galera_nbo_processlist.test  |   81 +
 .../suite/galera_ee/t/galera_nbo_sst_slave.test    |   89 +
 .../t/galera_nbo_temporary_table-master.opt        |    2 +
 .../galera_ee/t/galera_nbo_temporary_table.test    |   37 +
 .../suite/galera_ee/t/galera_nbo_unsupported.test  |   42 +
 .../suite/galera_ee/t/galera_vote_rejoin_ddl.cnf   |    5 +
 .../suite/galera_ee/t/galera_vote_rejoin_ddl.test  |   87 +
 .../suite/galera_ee/t/galera_vote_rejoin_dml.cnf   |    4 +
 .../suite/galera_ee/t/galera_vote_rejoin_dml.test  |   91 +
 .../galera_ee/t/mysql-wsrep-features#127.test      |   16 +
 .../t/mysql-wsrep-features#128-master.opt          |    1 +
 .../galera_ee/t/mysql-wsrep-features#128.test      |   11 +
 .../galera_ee/t/mysql-wsrep-features#131.test      |   38 +
 .../galera_ee/t/mysql-wsrep-features#132.test      |   39 +
 mysql-test/suite/galera_sr/galera_2nodes.cnf       |    1 +
 mysql-test/suite/galera_sr/my.cnf                  |    1 +
 mysql-test/suite/galera_sr/r/GCF-1008.result       |   50 +
 mysql-test/suite/galera_sr/r/GCF-1018.result       |   15 +
 mysql-test/suite/galera_sr/r/GCF-1018B.result      |    8 +
 mysql-test/suite/galera_sr/r/GCF-1043A.result      |   15 +
 mysql-test/suite/galera_sr/r/GCF-1043B.result      |   15 +
 mysql-test/suite/galera_sr/r/GCF-1051.result       |   37 +
 mysql-test/suite/galera_sr/r/GCF-1060.result       |   15 +
 mysql-test/suite/galera_sr/r/GCF-437.result        |   12 +
 mysql-test/suite/galera_sr/r/GCF-561.result        |   42 +
 mysql-test/suite/galera_sr/r/GCF-571.result        |   56 +
 mysql-test/suite/galera_sr/r/GCF-572.result        |   29 +
 mysql-test/suite/galera_sr/r/GCF-574.result        |   11 +
 mysql-test/suite/galera_sr/r/GCF-580.result        |   11 +
 mysql-test/suite/galera_sr/r/GCF-585.result        |   25 +
 mysql-test/suite/galera_sr/r/GCF-597.result        |   15 +
 mysql-test/suite/galera_sr/r/GCF-620.result        |   14 +
 mysql-test/suite/galera_sr/r/GCF-623.result        |   22 +
 mysql-test/suite/galera_sr/r/GCF-627.result        |   18 +
 mysql-test/suite/galera_sr/r/GCF-845.result        |   14 +
 mysql-test/suite/galera_sr/r/GCF-851.result        |   24 +
 mysql-test/suite/galera_sr/r/GCF-867.result        |    2 +
 mysql-test/suite/galera_sr/r/GCF-889.result        |   18 +
 mysql-test/suite/galera_sr/r/GCF-900.result        |   13 +
 .../suite/galera_sr/r/galera-features#56.result    |   21 +
 .../suite/galera_sr/r/galera_sr_bf_abort.result    |  448 +++++
 mysql-test/suite/galera_sr/r/galera_sr_blob.result |   15 +
 .../suite/galera_sr/r/galera_sr_cc_master.result   |   45 +
 .../suite/galera_sr/r/galera_sr_cc_slave.result    |   46 +
 .../suite/galera_sr/r/galera_sr_concurrent.result  |   28 +
 .../suite/galera_sr/r/galera_sr_conflict.result    |   14 +
 .../r/galera_sr_conflict_on_commit.result          |   22 +
 .../r/galera_sr_conflict_on_commit2.result         |   19 +
 .../galera_sr_conflict_with_rollback_master.result |   22 +
 .../suite/galera_sr/r/galera_sr_ddl_master.result  |   41 +
 .../suite/galera_sr/r/galera_sr_ddl_schema.result  |   18 +
 .../suite/galera_sr/r/galera_sr_ddl_slave.result   |   42 +
 .../galera_sr/r/galera_sr_ddl_unrelated.result     |   34 +
 .../galera_sr/r/galera_sr_dupkey_error.result      |   36 +
 .../suite/galera_sr/r/galera_sr_fk_conflict.result |   34 +
 mysql-test/suite/galera_sr/r/galera_sr_gtid.result |   65 +
 .../galera_sr/r/galera_sr_insert_select.result     |   11 +
 .../r/galera_sr_kill_all_nobootstrap.result        |   21 +
 .../r/galera_sr_kill_all_norecovery.result         |   22 +
 .../r/galera_sr_kill_all_pcrecovery.result         |   22 +
 .../galera_sr/r/galera_sr_kill_connection.result   |   25 +
 .../suite/galera_sr/r/galera_sr_kill_query.result  |   17 +
 .../suite/galera_sr/r/galera_sr_kill_slave.result  |   39 +
 .../galera_sr/r/galera_sr_large_fragment.result    |   28 +
 .../suite/galera_sr/r/galera_sr_load_data.result   |    8 +
 .../r/galera_sr_load_data_splitting.result         |    9 +
 .../suite/galera_sr/r/galera_sr_log_bin.result     |   87 +
 .../galera_sr/r/galera_sr_many_fragments.result    |   28 +
 .../suite/galera_sr/r/galera_sr_myisam.result      |   11 +
 .../galera_sr/r/galera_sr_mysqldump_sst.result     |   40 +
 .../galera_sr/r/galera_sr_parallel_apply.result    |   26 +
 .../suite/galera_sr/r/galera_sr_rollback.result    |   29 +
 .../galera_sr/r/galera_sr_rollback_retry.result    |   26 +
 .../r/galera_sr_rollback_savepoint.result          |   33 +
 .../r/galera_sr_rollback_statement.result          |   22 +
 mysql-test/suite/galera_sr/r/galera_sr_sbr.result  |   16 +
 .../galera_sr/r/galera_sr_shutdown_master.result   |   20 +
 .../galera_sr/r/galera_sr_shutdown_slave.result    |   29 +
 .../galera_sr/r/galera_sr_small_gcache.result      |   10 +
 .../galera_sr/r/galera_sr_table_contents.result    |  198 ++
 .../r/galera_sr_transaction_replay.result          |  101 +
 .../galera_sr/r/galera_sr_unit_statements.result   |   18 +
 .../galera_sr/r/galera_sr_v1_row_events.result     |   15 +
 .../suite/galera_sr/r/galera_sr_ws_size.result     |   29 +
 .../suite/galera_sr/r/galera_sr_ws_size2.result    |   28 +
 .../r/galera_var_ignore_apply_errors_sr.result     |   22 +
 .../suite/galera_sr/r/mysql-wsrep#215.result       |   88 +
 .../galera_sr/r/mysql-wsrep-features#136.result    |   41 +
 .../galera_sr/r/mysql-wsrep-features#138.result    |   17 +
 .../galera_sr/r/mysql-wsrep-features#14.result     |    8 +
 .../galera_sr/r/mysql-wsrep-features#148.result    |   27 +
 .../galera_sr/r/mysql-wsrep-features#15.result     |    8 +
 .../galera_sr/r/mysql-wsrep-features#165.result    |  752 ++++++++
 .../galera_sr/r/mysql-wsrep-features#213.result    |   26 +
 .../galera_sr/r/mysql-wsrep-features#214.result    |   46 +
 .../galera_sr/r/mysql-wsrep-features#22.result     |   26 +
 .../galera_sr/r/mysql-wsrep-features#27.result     |   17 +
 .../galera_sr/r/mysql-wsrep-features#29.result     |   14 +
 .../galera_sr/r/mysql-wsrep-features#32.result     |   22 +
 .../galera_sr/r/mysql-wsrep-features#35.result     |   26 +
 .../galera_sr/r/mysql-wsrep-features#8.result      |   27 +
 .../galera_sr/r/mysql-wsrep-features#9.result      |   12 +
 .../galera_sr/r/mysql-wsrep-features#93.result     |   15 +
 .../galera_sr/r/mysql-wsrep-features#96.result     |   26 +
 mysql-test/suite/galera_sr/t/GCF-1008.inc          |   36 +
 mysql-test/suite/galera_sr/t/GCF-1008.test         |   18 +
 mysql-test/suite/galera_sr/t/GCF-1018.test         |   38 +
 mysql-test/suite/galera_sr/t/GCF-1018B.test        |   40 +
 mysql-test/suite/galera_sr/t/GCF-1043A.test        |   13 +
 mysql-test/suite/galera_sr/t/GCF-1043B.test        |   13 +
 mysql-test/suite/galera_sr/t/GCF-1051.test         |   52 +
 mysql-test/suite/galera_sr/t/GCF-1060.test         |    9 +
 mysql-test/suite/galera_sr/t/GCF-437.test          |   21 +
 mysql-test/suite/galera_sr/t/GCF-561.test          |   65 +
 mysql-test/suite/galera_sr/t/GCF-571.test          |   54 +
 mysql-test/suite/galera_sr/t/GCF-572.test          |   54 +
 mysql-test/suite/galera_sr/t/GCF-574.test          |   27 +
 mysql-test/suite/galera_sr/t/GCF-580.test          |   27 +
 mysql-test/suite/galera_sr/t/GCF-585.test          |   40 +
 mysql-test/suite/galera_sr/t/GCF-597.test          |   29 +
 mysql-test/suite/galera_sr/t/GCF-620.test          |   22 +
 mysql-test/suite/galera_sr/t/GCF-623.test          |   31 +
 mysql-test/suite/galera_sr/t/GCF-627.test          |   30 +
 mysql-test/suite/galera_sr/t/GCF-845.test          |   30 +
 mysql-test/suite/galera_sr/t/GCF-851.test          |   24 +
 mysql-test/suite/galera_sr/t/GCF-867.test          |   42 +
 mysql-test/suite/galera_sr/t/GCF-889.test          |   28 +
 mysql-test/suite/galera_sr/t/GCF-900.test          |   28 +
 mysql-test/suite/galera_sr/t/disabled.def          |    4 +
 .../suite/galera_sr/t/galera-features#56.test      |   55 +
 .../suite/galera_sr/t/galera_sr_bf_abort.inc       |  145 ++
 .../suite/galera_sr/t/galera_sr_bf_abort.test      |   48 +
 mysql-test/suite/galera_sr/t/galera_sr_blob.test   |   38 +
 .../suite/galera_sr/t/galera_sr_cc_master.test     |   87 +
 .../suite/galera_sr/t/galera_sr_cc_slave.test      |   86 +
 .../suite/galera_sr/t/galera_sr_concurrent.test    |   45 +
 .../suite/galera_sr/t/galera_sr_conflict.test      |   45 +
 .../galera_sr/t/galera_sr_conflict_on_commit.test  |   45 +
 .../galera_sr/t/galera_sr_conflict_on_commit2.test |   46 +
 .../t/galera_sr_conflict_with_rollback_master.test |   44 +
 .../suite/galera_sr/t/galera_sr_ddl_master.test    |   63 +
 .../suite/galera_sr/t/galera_sr_ddl_schema.test    |   43 +
 .../suite/galera_sr/t/galera_sr_ddl_slave.test     |   65 +
 .../suite/galera_sr/t/galera_sr_ddl_unrelated.test |   53 +
 .../suite/galera_sr/t/galera_sr_dupkey_error.test  |   57 +
 .../suite/galera_sr/t/galera_sr_fk_conflict.test   |   62 +
 .../suite/galera_sr/t/galera_sr_gtid-master.opt    |    1 +
 mysql-test/suite/galera_sr/t/galera_sr_gtid.test   |   46 +
 .../suite/galera_sr/t/galera_sr_insert_select.test |   33 +
 .../t/galera_sr_kill_all_nobootstrap.test          |   52 +
 .../galera_sr/t/galera_sr_kill_all_norecovery.cnf  |    4 +
 .../galera_sr/t/galera_sr_kill_all_norecovery.test |   53 +
 .../galera_sr/t/galera_sr_kill_all_pcrecovery.test |   54 +
 .../galera_sr/t/galera_sr_kill_connection.test     |   59 +
 .../suite/galera_sr/t/galera_sr_kill_query.test    |   53 +
 .../suite/galera_sr/t/galera_sr_kill_slave.cnf     |    4 +
 .../suite/galera_sr/t/galera_sr_kill_slave.test    |   80 +
 .../t/galera_sr_large_fragment-master.opt          |    1 +
 .../galera_sr/t/galera_sr_large_fragment.test      |   58 +
 .../suite/galera_sr/t/galera_sr_load_data.test     |   39 +
 .../galera_sr/t/galera_sr_load_data_splitting.test |   50 +
 .../suite/galera_sr/t/galera_sr_log_bin-master.opt |    1 +
 .../suite/galera_sr/t/galera_sr_log_bin.test       |   66 +
 .../galera_sr/t/galera_sr_many_fragments.test      |   53 +
 mysql-test/suite/galera_sr/t/galera_sr_myisam.test |   29 +
 .../suite/galera_sr/t/galera_sr_mysqldump_sst.cnf  |   11 +
 .../suite/galera_sr/t/galera_sr_mysqldump_sst.test |   79 +
 .../galera_sr/t/galera_sr_parallel_apply.test      |   59 +
 .../suite/galera_sr/t/galera_sr_rollback.test      |   76 +
 .../galera_sr/t/galera_sr_rollback_retry.test      |   55 +
 .../galera_sr/t/galera_sr_rollback_savepoint.test  |   51 +
 .../galera_sr/t/galera_sr_rollback_statement.test  |   61 +
 mysql-test/suite/galera_sr/t/galera_sr_sbr.test    |   31 +
 .../galera_sr/t/galera_sr_shutdown_master.test     |   45 +
 .../galera_sr/t/galera_sr_shutdown_slave.test      |   63 +
 .../suite/galera_sr/t/galera_sr_small_gcache.cnf   |    6 +
 .../suite/galera_sr/t/galera_sr_small_gcache.test  |   21 +
 .../galera_sr/t/galera_sr_table_contents.test      |   49 +
 .../galera_sr/t/galera_sr_transaction_replay.test  |  260 +++
 .../galera_sr/t/galera_sr_unit_statements.test     |   47 +
 .../galera_sr/t/galera_sr_v1_row_events-master.opt |    1 +
 .../suite/galera_sr/t/galera_sr_v1_row_events.test |   27 +
 .../suite/galera_sr/t/galera_sr_ws_size.test       |   70 +
 .../suite/galera_sr/t/galera_sr_ws_size2.test      |   62 +
 .../t/galera_var_ignore_apply_errors_sr.test       |   37 +
 mysql-test/suite/galera_sr/t/mysql-wsrep#215.test  |  161 ++
 .../t/mysql-wsrep-features#136-master.opt          |    1 +
 .../galera_sr/t/mysql-wsrep-features#136.test      |   37 +
 .../galera_sr/t/mysql-wsrep-features#138.test      |   25 +
 .../suite/galera_sr/t/mysql-wsrep-features#14.test |   21 +
 .../galera_sr/t/mysql-wsrep-features#148.test      |   60 +
 .../suite/galera_sr/t/mysql-wsrep-features#15.test |   17 +
 .../suite/galera_sr/t/mysql-wsrep-features#165.inc |  104 +
 .../galera_sr/t/mysql-wsrep-features#165.test      |   41 +
 .../galera_sr/t/mysql-wsrep-features#213.test      |   63 +
 .../galera_sr/t/mysql-wsrep-features#214.test      |   86 +
 .../suite/galera_sr/t/mysql-wsrep-features#22.test |   47 +
 .../suite/galera_sr/t/mysql-wsrep-features#27.test |   29 +
 .../suite/galera_sr/t/mysql-wsrep-features#29.test |   23 +
 .../galera_sr/t/mysql-wsrep-features#32-master.opt |    1 +
 .../suite/galera_sr/t/mysql-wsrep-features#32.test |   44 +
 .../suite/galera_sr/t/mysql-wsrep-features#35.test |   46 +
 .../suite/galera_sr/t/mysql-wsrep-features#8.test  |   63 +
 .../suite/galera_sr/t/mysql-wsrep-features#9.test  |   44 +
 .../suite/galera_sr/t/mysql-wsrep-features#93.test |   29 +
 .../suite/galera_sr/t/mysql-wsrep-features#96.test |   45 +
 .../suite/innodb/r/innodb-index-online-fk.result   |   10 +-
 mysql-test/suite/innodb/t/galera.skip              |   53 +
 .../suite/innodb/t/innodb-index-online-fk.test     |   10 +-
 packaging/deb-in/CMakeLists.txt                    |  357 ++++
 plugin/wsrep_info/plugin.cc                        |    2 +
 scripts/mysqld_safe.sh                             |   11 +-
 scripts/wsrep_sst_mysqldump.sh                     |    1 +
 scripts/wsrep_sst_xtrabackup-v2.sh                 |    2 +-
 sql/CMakeLists.txt                                 |   15 +-
 sql/events.cc                                      |    1 -
 sql/ha_partition.cc                                |    7 +
 sql/ha_partition.h                                 |    3 +
 sql/handler.cc                                     |  216 ++-
 sql/item_create.cc                                 |  102 +-
 sql/item_strfunc.cc                                |  102 +
 sql/item_strfunc.h                                 |   52 +
 sql/lock.cc                                        |   25 +-
 sql/log.cc                                         |  379 +++-
 sql/log.h                                          |   19 +
 sql/log_event.cc                                   |   19 +-
 sql/mdl.cc                                         |   74 +-
 sql/mysqld.cc                                      |   51 +-
 sql/protocol.cc                                    |   21 +-
 sql/slave.cc                                       |  111 +-
 sql/sp_head.cc                                     |   29 +-
 sql/sql_alter.cc                                   |    2 +-
 sql/sql_base.cc                                    |   23 +-
 sql/sql_class.cc                                   |  145 +-
 sql/sql_class.h                                    |  373 +++-
 sql/sql_connect.cc                                 |    5 +-
 sql/sql_insert.cc                                  |   53 +-
 sql/sql_load.cc                                    |    9 +-
 sql/sql_parse.cc                                   |  604 ++++--
 sql/sql_parse.h                                    |    2 +
 sql/sql_plugin.cc                                  |    5 +
 sql/sql_plugin_services.ic                         |    9 +-
 sql/sql_prepare.cc                                 |    8 +-
 sql/sql_repl.cc                                    |   11 +
 sql/sql_table.cc                                   |   32 +-
 sql/sql_truncate.cc                                |   18 +-
 sql/sys_vars.cc                                    |   38 +-
 sql/table.cc                                       |    7 +
 sql/transaction.cc                                 |   36 +-
 sql/wsrep_applier.cc                               |  898 ++++++++-
 sql/wsrep_applier.h                                |   67 +-
 sql/wsrep_binlog.cc                                |  201 +-
 sql/wsrep_binlog.h                                 |   55 +-
 sql/wsrep_dummy.cc                                 |   19 +-
 sql/wsrep_hton.cc                                  |    1 +
 sql/wsrep_mysqld.cc                                | 2036 ++++++++++++++++----
 sql/wsrep_mysqld.h                                 |  333 +++-
 sql/wsrep_plugin.cc                                |   53 +
 sql/wsrep_priv.h                                   |    9 +-
 sql/wsrep_schema.cc                                | 1481 ++++++++++++++
 sql/wsrep_schema.h                                 |  141 ++
 sql/wsrep_sr.cc                                    |  428 ++++
 sql/wsrep_sr.h                                     |  275 +++
 sql/wsrep_sr_file.cc                               |  563 ++++++
 sql/wsrep_sr_file.h                                |  111 ++
 sql/wsrep_sr_table.cc                              |  223 +++
 sql/wsrep_sr_table.h                               |   67 +
 sql/wsrep_sst.cc                                   |  169 +-
 sql/wsrep_sst.h                                    |   10 +-
 sql/wsrep_tc.h                                     |    0
 sql/wsrep_thd.cc                                   |  751 +++++++-
 sql/wsrep_thd.h                                    |  176 +-
 sql/wsrep_thd_pool.cc                              |  122 ++
 sql/wsrep_thd_pool.h                               |   37 +
 sql/wsrep_trans_observer.cc                        | 1663 ++++++++++++++++
 sql/wsrep_trans_observer.h                         |  117 ++
 sql/wsrep_utils.cc                                 |    1 +
 sql/wsrep_utils.h                                  |   34 +
 sql/wsrep_var.cc                                   |  184 +-
 sql/wsrep_var.h                                    |    5 +-
 sql/wsrep_xid.cc                                   |   42 +-
 sql/wsrep_xid.h                                    |   12 +-
 storage/innobase/buf/buf0dump.cc                   |    6 +-
 storage/innobase/handler/ha_innodb.cc              |  228 +--
 storage/innobase/handler/ha_innodb.h               |    9 +-
 storage/innobase/include/ha_prototypes.h           |    5 +
 storage/innobase/include/trx0trx.h                 |   17 +
 storage/innobase/lock/lock0lock.cc                 |   14 +-
 storage/innobase/lock/lock0wait.cc                 |    8 +-
 storage/innobase/row/row0ins.cc                    |   22 +
 storage/innobase/row/row0sel.cc                    |   16 +
 storage/innobase/row/row0upd.cc                    |   14 +-
 storage/innobase/trx/trx0roll.cc                   |    5 +
 storage/innobase/trx/trx0rseg.cc                   |    3 +-
 wsrep/wsrep_api.h                                  |  533 +++--
 wsrep/wsrep_dummy.c                                |  116 +-
 wsrep/wsrep_gtid.c                                 |    2 +-
 wsrep/wsrep_listener.c                             |  261 +++
 wsrep/wsrep_loader.c                               |   28 +-
 wsrep/wsrep_uuid.c                                 |   11 +
 833 files changed, 36460 insertions(+), 2491 deletions(-)

diff --cc CMakeLists.txt
index 58b6cc62e9f,3ff19ae33a2..5477e68a113

--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@@ -398,13 -403,8 +403,9 @@@ IF(NOT WITHOUT_SERVER
    IF(WITH_EMBEDDED_SERVER) 
     ADD_SUBDIRECTORY(libmysqld)
     ADD_SUBDIRECTORY(libmysqld/examples)
 +   ADD_SUBDIRECTORY(unittest/embedded)
    ENDIF(WITH_EMBEDDED_SERVER)
  
-   IF(WITH_WSREP)
-     ADD_SUBDIRECTORY(wsrep)
-   ENDIF()
- 
    ADD_SUBDIRECTORY(mysql-test)
    ADD_SUBDIRECTORY(mysql-test/lib/My/SafeProcess)
    ADD_SUBDIRECTORY(sql-bench)
diff --cc include/mysql/service_wsrep.h
index 1540a5e56fe,2459d8776f0..d0d00cf2013
--- a/include/mysql/service_wsrep.h
+++ b/include/mysql/service_wsrep.h
@@@ -1,4 -1,9 +1,5 @@@
 -//#ifndef MYSQL_SERVICE_WSREP_INCLUDED
 -//#ifdef MYSQL_SERVICE_WSREP_DYNAMIC_INCLUDED && MYSQL_SERVICE_WSREP_STATIC_INCLUDED
 -#if (defined (MYSQL_DYNAMIC_PLUGIN) && defined(MYSQL_SERVICE_WSREP_DYNAMIC_INCLUDED)) || (!defined(MYSQL_DYNAMIC_PLUGIN) && defined(MYSQL_SERVICE_WSREP_STATIC_INCLUDED))
 -
 -#else
 +#ifndef MYSQL_SERVICE_WSREP_INCLUDED
+ 
  /* Copyright (c) 2015 MariaDB Corporation Ab
  
     This program is free software; you can redistribute it and/or modify
@@@ -55,8 -61,7 +57,8 @@@ enum wsrep_query_state 
      QUERY_IDLE,
      QUERY_EXEC,
      QUERY_COMMITTING,
++    QUERY_ORDERED_COMMIT,
      QUERY_EXITING,
-     QUERY_ROLLINGBACK,
  };
  
  enum wsrep_trx_status {
@@@ -65,9 -70,9 +67,9 @@@
      WSREP_TRX_SIZE_EXCEEDED,  /* trx size exceeded */
      WSREP_TRX_ERROR,          /* native mysql error */
  };
 -#endif
 +
  struct xid_t;
- struct wsrep;
+ struct wsrep_st;
  struct wsrep_ws_handle;
  struct wsrep_buf;
  
@@@ -80,17 -85,17 +82,12 @@@ extern struct wsrep_service_st 
    my_bool                     (*get_wsrep_load_data_splitting_func)();
    my_bool                     (*get_wsrep_log_conflicts_func)();
    long                        (*get_wsrep_protocol_version_func)();
-   my_bool                     (*wsrep_aborting_thd_contains_func)(THD *thd);
-   void                        (*wsrep_aborting_thd_enqueue_func)(THD *thd);
    bool                        (*wsrep_consistency_check_func)(THD *thd);
-   int                         (*wsrep_is_wsrep_xid_func)(const struct xid_t *xid);
 -  //int                         (*wsrep_is_wsrep_xid_func)(const struct xid_t *xid);
+   int                         (*wsrep_is_wsrep_xid_func)(const void *xid);
    long long                   (*wsrep_xid_seqno_func)(const struct xid_t *xid);
    const unsigned char*        (*wsrep_xid_uuid_func)(const struct xid_t *xid);
-   void                        (*wsrep_lock_rollback_func)();
-   int                         (*wsrep_on_func)(MYSQL_THD);
-   void                        (*wsrep_post_commit_func)(THD* thd, bool all);
-   bool                        (*wsrep_prepare_key_func)(const unsigned char*, size_t, const unsigned char*, size_t, struct wsrep_buf*, size_t*);
-   enum wsrep_trx_status       (*wsrep_run_wsrep_commit_func)(THD *thd, bool all);
 -  //  int                         (*wsrep_on_func)(MYSQL_THD);
+   int                         (*wsrep_on_func)(void *);
 -  //  void                        (*wsrep_post_commit_func)(THD* thd, bool all);
 -  //bool                        (*wsrep_prepare_key_func)(const unsigned char*, size_t, const unsigned char*, size_t, struct wsrep_buf*, size_t*);
+   bool                        (*wsrep_prepare_key_for_innodb_func)(THD* thd, const unsigned char*, size_t, const unsigned char*, size_t, struct wsrep_buf*, size_t*);
 -  //enum wsrep_trx_status       (*wsrep_run_wsrep_commit_func)(THD *thd, bool all);
    void                        (*wsrep_thd_LOCK_func)(THD *thd);
    void                        (*wsrep_thd_UNLOCK_func)(THD *thd);
    void                        (*wsrep_thd_awake_func)(THD *thd, my_bool signal);
@@@ -99,7 -105,8 +97,7 @@@
    enum wsrep_exec_mode        (*wsrep_thd_exec_mode_func)(THD *thd);
    const char *                (*wsrep_thd_exec_mode_str_func)(THD *thd);
    enum wsrep_conflict_state   (*wsrep_thd_get_conflict_state_func)(MYSQL_THD);
-   my_bool                     (*wsrep_thd_is_BF_func)(MYSQL_THD , my_bool);
 -  //  my_bool                     (*wsrep_thd_is_BF_func)(MYSQL_THD , my_bool);
+   my_bool                     (*wsrep_thd_is_BF_func)(void* , my_bool);
    my_bool                     (*wsrep_thd_is_wsrep_func)(MYSQL_THD thd);
    char *                      (*wsrep_thd_query_func)(THD *thd);
    enum wsrep_query_state      (*wsrep_thd_query_state_func)(THD *thd);
@@@ -111,10 -118,16 +109,11 @@@
    struct wsrep_ws_handle *    (*wsrep_thd_ws_handle_func)(THD *thd);
    int                         (*wsrep_trx_is_aborting_func)(MYSQL_THD thd);
    int                         (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD);
-   void                        (*wsrep_unlock_rollback_func)();
 -  //void                        (wsrep_thd_xid_func)(const void *thd_ptr, void *xid, size_t xid_size);
  } *wsrep_service;
  
+ 
 -#define MYSQL_SERVICE_WSREP_INCLUDED
 -#endif
 -
  #ifdef MYSQL_DYNAMIC_PLUGIN
+ 
 -#define MYSQL_SERVICE_WSREP_DYNAMIC_INCLUDED
  #define get_wsrep() wsrep_service->get_wsrep_func()
  #define get_wsrep_certify_nonPK() wsrep_service->get_wsrep_certify_nonPK_func()
  #define get_wsrep_debug() wsrep_service->get_wsrep_debug_func()
@@@ -148,7 -156,7 +142,6 @@@
  #define wsrep_thd_query_state(T) wsrep_service->wsrep_thd_query_state_func(T)
  #define wsrep_thd_query_state_str(T) wsrep_service->wsrep_thd_query_state_str_func(T)
  #define wsrep_thd_retry_counter(T) wsrep_service->wsrep_thd_retry_counter_func(T)
- #define wsrep_thd_set_conflict_state(T,S) wsrep_service->wsrep_thd_set_conflict_state_func(T,S)
 -  //#define wsrep_thd_set_conflict_state(T,S) wsrep_service->wsrep_thd_set_conflict_state_func(T,S)
  #define wsrep_thd_ignore_table(T) wsrep_service->wsrep_thd_ignore_table_func(T)
  #define wsrep_thd_trx_seqno(T) wsrep_service->wsrep_thd_trx_seqno_func(T)
  #define wsrep_thd_ws_handle(T) wsrep_service->wsrep_thd_ws_handle_func(T)
@@@ -173,10 -183,11 +167,11 @@@ extern my_bool wsrep_load_data_splittin
  extern my_bool wsrep_drupal_282555_workaround;
  extern my_bool wsrep_recovery;
  extern long wsrep_protocol_version;
+ extern my_thread_id wsrep_thd_thread_id(THD *thd);
 -  
 +
  bool wsrep_consistency_check(THD *thd);
- bool wsrep_prepare_key(const unsigned char* cache_key, size_t cache_key_len, const unsigned char* row_id, size_t row_id_len, struct wsrep_buf* key, size_t* key_len);
- char *wsrep_thd_query(THD *thd);
+ bool wsrep_prepare_key_for_innodb(THD* thd, const unsigned char* cache_key, size_t cache_key_len, const unsigned char* row_id, size_t row_id_len, struct wsrep_buf* key, size_t* key_len);
+ extern "C" char *wsrep_thd_query(THD *thd);
  const char *wsrep_thd_conflict_state_str(THD *thd);
  const char *wsrep_thd_exec_mode_str(THD *thd);
  const char *wsrep_thd_query_state_str(THD *thd);
@@@ -185,10 -196,12 +180,10 @@@ enum wsrep_conflict_state wsrep_thd_get
  enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd);
  enum wsrep_query_state wsrep_thd_query_state(THD *thd);
  enum wsrep_trx_status wsrep_run_wsrep_commit(THD *thd, bool all);
- int wsrep_is_wsrep_xid(const struct xid_t* xid);
 -//int wsrep_is_wsrep_xid(const struct xid_t* xid);
+ int wsrep_is_wsrep_xid(const void* xid);
  long long wsrep_xid_seqno(const struct xid_t* xid);
  const unsigned char* wsrep_xid_uuid(const struct xid_t* xid);
- int wsrep_on(MYSQL_THD thd);
 -//int wsrep_on(MYSQL_THD thd);
+ int wsrep_on(void *);
  int wsrep_thd_retry_counter(THD *thd);
  int wsrep_trx_is_aborting(MYSQL_THD thd);
  int wsrep_trx_order_before(MYSQL_THD thd1, MYSQL_THD thd2);
@@@ -201,12 -214,12 +196,11 @@@ my_bool get_wsrep_recovery()
  my_bool get_wsrep_load_data_splitting();
  my_bool get_wsrep_log_conflicts();
  my_bool wsrep_aborting_thd_contains(THD *thd);
- my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync);
 -//my_bool wsrep_thd_is_BF(MYSQL_THD thd, my_bool sync);
+ my_bool wsrep_thd_is_BF(void* thd, my_bool sync);
  my_bool wsrep_thd_is_wsrep(MYSQL_THD thd);
- struct wsrep *get_wsrep();
+ struct wsrep_st *get_wsrep();
  struct wsrep_ws_handle *wsrep_thd_ws_handle(THD *thd);
  void wsrep_aborting_thd_enqueue(THD *thd);
- void wsrep_lock_rollback();
  void wsrep_post_commit(THD* thd, bool all);
  void wsrep_thd_LOCK(THD *thd);
  void wsrep_thd_UNLOCK(THD *thd);
diff --cc mysql-test/main/mysqld--help-notwin.result
index 00000000000,b7db372a704..b7db372a704
mode 000000,100644..100644
--- a/mysql-test/main/mysqld--help-notwin.result
+++ b/mysql-test/main/mysqld--help-notwin.result
diff --cc mysql-test/main/query_cache_size_functionality.result
index 00000000000,7563b3e61eb..7563b3e61eb
mode 000000,100644..100644
--- a/mysql-test/main/query_cache_size_functionality.result
+++ b/mysql-test/main/query_cache_size_functionality.result
diff --cc mysql-test/main/query_cache_type_functionality.result
index 00000000000,134d8c3a066..134d8c3a066
mode 000000,100644..100644
--- a/mysql-test/main/query_cache_type_functionality.result
+++ b/mysql-test/main/query_cache_type_functionality.result
diff --cc mysql-test/suite/galera/t/galera_sst_mysqldump.test
index 835fac94a68,527072a53e9..b72fa687411
--- a/mysql-test/suite/galera/t/galera_sst_mysqldump.test
+++ b/mysql-test/suite/galera/t/galera_sst_mysqldump.test
@@@ -1,6 -1,5 +1,6 @@@
 +--source include/big_test.inc
  --source include/galera_cluster.inc
- 
+ --source include/have_innodb.inc
  --source suite/galera/include/galera_sst_set_mysqldump.inc
  
  --let $node_1=node_1
diff --cc mysql-test/suite/galera/t/galera_var_dirty_reads.test
index 1f01c4aac07,075fea83d2f..dc1698ef497
--- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test
+++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test
@@@ -4,18 -4,7 +4,13 @@@
  
  --source include/galera_cluster.inc
  --source include/have_innodb.inc
 +--source include/have_perfschema.inc
 +
 +# Save original auto_increment_offset values.
 +--let $node_1=node_1
 +--let $node_2=node_2
 +--source include/auto_increment_offset_save.inc
  
- # Save original auto_increment_offset values.
- --let $node_1=node_1
- --let $node_2=node_2
- --source include/auto_increment_offset_save.inc
- 
  --connection node_2
  --let $wsrep_cluster_address_saved = `SELECT @@global.wsrep_cluster_address`
  
diff --cc sql/CMakeLists.txt
index 0037ff23153,e7fd2680f54..8d5f0c1ce2d
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@@ -42,13 -48,25 +48,12 @@@ ${PCRE_INCLUDES
  ${ZLIB_INCLUDE_DIR}
  ${SSL_INCLUDE_DIRS}
  ${CMAKE_BINARY_DIR}/sql
- ${WSREP_INCLUDES}
  )
  
 -SET(GEN_SOURCES
 -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h 
 -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc
 -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.h
 -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc
 -${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h
 -${CMAKE_CURRENT_BINARY_DIR}/lex_token.h
 -)
  
 -SET_SOURCE_FILES_PROPERTIES(${GEN_SOURCES}
 -                            PROPERTIES GENERATED 1)
  
  
 -IF(NOT CMAKE_CROSSCOMPILING)
 -  ADD_EXECUTABLE(gen_lex_token gen_lex_token.cc
 -    ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h)
 -ENDIF()
 +
  
  ADD_CUSTOM_COMMAND(
    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h
diff --cc sql/handler.cc
index b3481c7e429,256b2a51076..cb3f156be81
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@@ -55,6 -55,10 +55,12 @@@
  #include "wsrep_mysqld.h"
  #include "wsrep.h"
  #include "wsrep_xid.h"
++#ifdef WITH_WSREP
+ #include "wsrep_sr.h"
++#endif
+ #include "wsrep_thd.h"
+ #include "wsrep_trans_observer.h" /* wsrep transaction hooks */
+ #include "log.h"
  
  /*
    While we have legacy_db_type, we have this array to
diff --cc sql/item_strfunc.cc
index 12618d68c27,c312b853764..38338174ae7
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@@ -5244,23 -5205,104 +5244,125 @@@ null
    return NULL;
  }
  
 +Item_temptable_rowid::Item_temptable_rowid(TABLE *table_arg)
 +  : Item_str_func(table_arg->in_use), table(table_arg)
 +{
 +  max_length= table->file->ref_length;
 +}
 +
 +bool Item_temptable_rowid::fix_length_and_dec()
 +{
 +  used_tables_cache= table->map;
 +  const_item_cache= false;
 +  return FALSE;
 +}
 +
 +String *Item_temptable_rowid::val_str(String *str)
 +{
 +  if (!((null_value= table->null_row)))
 +    table->file->position(table->record[0]);
 +  str_value.set((char*)(table->file->ref), max_length, &my_charset_bin);
 +  return &str_value;
 +}
++
+ #ifdef WITH_WSREP
+ 
+ #include "wsrep_mysqld.h"
+ 
+ String * Item_func_wsrep_last_written_gtid::val_str_ascii(String *str)
+ {
+   wsrep_gtid_t gtid;
+   wsrep_thd_last_written_gtid(current_thd, >id);
+   
+   if (gtid_str.alloc(WSREP_GTID_STR_LEN))
+   {
+     my_error(ER_OUTOFMEMORY, WSREP_GTID_STR_LEN);
+     null_value= true;
+     return NULL;
+   }
+   int gtid_len = wsrep_gtid_print(>id,
+                                   (char*)gtid_str.ptr(),
+                                   WSREP_GTID_STR_LEN);
+   if (gtid_len < 0)
+   {
+     my_error(ER_ERROR_WHEN_EXECUTING_COMMAND, MYF(0), func_name(),
+              "wsrep_gtid_print failed");
+     null_value= true;
+     return NULL;
+   }
+   gtid_str.length(gtid_len);
+   return >id_str;
+ }
+ 
+ String * Item_func_wsrep_last_seen_gtid::val_str_ascii(String *str)
+ {
+   wsrep_gtid_t gtid;
+   wsrep_last_committed_id(>id);
+   
+   if (gtid_str.alloc(WSREP_GTID_STR_LEN))
+   {
+     my_error(ER_OUTOFMEMORY, WSREP_GTID_STR_LEN);
+     null_value= true;
+     return NULL;
+   }
+   int gtid_len= wsrep_gtid_print(>id,
+                                  (char*)gtid_str.ptr(),
+                                  WSREP_GTID_STR_LEN);
+   if (gtid_len < 0)
+   {
+     my_error(ER_ERROR_WHEN_EXECUTING_COMMAND, MYF(0), func_name(),
+              "wsrep_gtid_print failed");
+     null_value= true;
+     return NULL;
+   }
+   gtid_str.length(gtid_len);
+   return >id_str;
+ }
+ 
+ longlong Item_func_wsrep_sync_wait_upto::val_int()
+ {
+   int timeout = -1;
+   String* gtid_str= args[0]->val_str(&value);
+   if (gtid_str == NULL)
+   {
+     my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name());
+     return 0LL;
+   }
+ 
+   if (arg_count == 2)
+   {
+     timeout = args[1]->val_int();
+   }
+ 
+   wsrep_gtid_t gtid;
+   int gtid_len= wsrep_gtid_scan(gtid_str->ptr(), gtid_str->length(), >id);
+   if (gtid_len < 0)
+   {
+     my_error(ER_WRONG_ARGUMENTS, MYF(0), func_name());
+     return 0LL;
+   }
+ 
+   if (gtid.seqno == WSREP_SEQNO_UNDEFINED &&
+       wsrep_uuid_compare(>id.uuid, &WSREP_UUID_UNDEFINED) == 0)
+   {
+     return 1LL;
+   }
+ 
+   wsrep_status_t status= wsrep_sync_wait_upto(current_thd, >id, timeout);
+ 
+   if (status != WSREP_OK)
+   {
+     int err;
+     switch (status) {
+     case WSREP_TRX_MISSING:
+       err= ER_WRONG_ARGUMENTS;
+       break;
+     default:
+       err= ER_LOCK_WAIT_TIMEOUT;
+     }
+     my_error(err, MYF(0), func_name());
+     return 0LL;
+   }
+   return 1LL;
+ }
+ #endif /* WITH_WSREP */
diff --cc sql/item_strfunc.h
index 7c4ab93dc80,20311604742..5fd13725b45
--- a/sql/item_strfunc.h
+++ b/sql/item_strfunc.h
@@@ -1784,25 -1751,54 +1784,77 @@@ public
    { return get_item_copy<Item_func_dyncol_list>(thd, this); }
  };
  
 +/*
 +  this is used by JOIN_TAB::keep_current_rowid
 +  and stores handler::position().
 +  It has nothing to do with _rowid pseudo-column, that the parser supports.
 +*/
 +class Item_temptable_rowid :public Item_str_func
 +{
 +public:
 +  TABLE *table;
 +  Item_temptable_rowid(TABLE *table_arg);
 +  const Type_handler *type_handler() const { return &type_handler_string; }
 +  Field *create_tmp_field(bool group, TABLE *table)
 +  { return create_table_field_from_handler(table); }
 +  String *val_str(String *str);
 +  enum Functype functype() const { return  TEMPTABLE_ROWID; }
 +  const char *func_name() const { return "<rowid>"; }
 +  bool fix_length_and_dec();
 +  Item *get_copy(THD *thd)
 +  { return get_item_copy<Item_temptable_rowid>(thd, this); }
 +};
 +
+ #ifdef WITH_WSREP
+ 
+ #include "../wsrep/wsrep_api.h"
+ 
+ class Item_func_wsrep_last_written_gtid: public Item_str_ascii_func
+ {
+   String gtid_str;
+ public:
+   Item_func_wsrep_last_written_gtid(THD *thd): Item_str_ascii_func(thd) {}
+   const char *func_name() const { return "wsrep_last_written_gtid"; }
+   String *val_str_ascii(String *);
 -  void fix_length_and_dec()
++  bool fix_length_and_dec()
+   {
+     max_length = WSREP_GTID_STR_LEN;
+     maybe_null = true;
++    return false;
+   }
+   Item *get_copy(THD *thd)
+   { return get_item_copy<Item_func_wsrep_last_written_gtid>(thd, this); }
+ };
+ 
+ class Item_func_wsrep_last_seen_gtid: public Item_str_ascii_func
+ {
+   String gtid_str;
+ public:
+   Item_func_wsrep_last_seen_gtid(THD *thd): Item_str_ascii_func(thd) {}
+   const char *func_name() const { return "wsrep_last_seen_gtid"; }
+   String *val_str_ascii(String *);
 -  void fix_length_and_dec()
++  bool fix_length_and_dec()
+   {
+     max_length = WSREP_GTID_STR_LEN;
+     maybe_null = true;
++    return false;
+   }
+   Item *get_copy(THD *thd)
+   { return get_item_copy<Item_func_wsrep_last_seen_gtid>(thd, this); }
+ };
+ 
+ class Item_func_wsrep_sync_wait_upto: public Item_int_func
+ {
+   String value;
+ public:
+  Item_func_wsrep_sync_wait_upto(THD *thd, Item *a): Item_int_func(thd, a) {}
+  Item_func_wsrep_sync_wait_upto(THD *thd, Item *a, Item* b): Item_int_func(thd, a, b) {}
+   const Type_handler *type_handler() const { return &type_handler_string; }
+   const char *func_name() const { return "wsrep_sync_wait_upto_gtid"; }
+   longlong val_int();
+   Item *get_copy(THD *thd)
+   { return get_item_copy<Item_func_wsrep_sync_wait_upto>(thd, this); }
+ };
+ #endif /* WITH_WSREP */
 -#endif /* ITEM_STRFUNC_INCLUDED */
+ 
 +#endif /* ITEM_STRFUNC_INCLUDED */
diff --cc sql/mysqld.cc
index ce14dc678c6,7795d063d0e..d3b9ab03640
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@@ -5330,6 -5331,30 +5347,15 @@@ static int init_server_components(
      }
    }
  
 -  if (strlen(wsrep_provider)== 0 ||
 -      !strcmp(wsrep_provider, WSREP_NONE))
++  if (WSREP_NOT_SPECIFIED)
+   {
+     // enable normal operation in case no provider is specified
++#ifndef EMBEDDED_LIBRARY
+     wsrep_ready_set(TRUE);
 -    global_system_variables.wsrep_on = 0;
 -#ifdef OUT
 -    wsrep_init_args args;
 -    args.logger_cb = wsrep_log_cb;
 -    args.options = (wsrep_provider_options) ?
 -            wsrep_provider_options : "";
 -    int rcode = wsrep->init(wsrep, &args);
 -    if (rcode)
 -    {
 -      DBUG_PRINT("wsrep",("wsrep::init() failed: %d", rcode));
 -      WSREP_ERROR("wsrep::init() failed: %d, must shutdown", rcode);
 -      wsrep->free(wsrep);
 -      free(wsrep);
 -      wsrep = NULL;
 -    }
 -    return rcode;
+ #endif
++    global_system_variables.wsrep_on = 0;
+   }
+ 
    if (opt_bin_log)
    {
      if (mysql_bin_log.open_index_file(opt_binlog_index_name, opt_bin_logname,
diff --cc sql/protocol.cc
index 29004fe24e4,41895c153f1..d370ff38e4a
--- a/sql/protocol.cc
+++ b/sql/protocol.cc
@@@ -551,8 -551,23 +551,25 @@@ static uchar *net_store_length_fast(uch
  
  void Protocol::end_statement()
  {
-   /* sanity check*/
-   DBUG_ASSERT_IF_WSREP(!(WSREP(thd) && thd->wsrep_conflict_state == REPLAYING));
+   /*
+     Commented out: This sanity check does not hold in general.
+     Thd->LOCK_wsrep_thd() must be unlocked before sending response
+     to client, so BF abort may sneak in here.
+     DBUG_ASSERT(!WSREP(thd) || thd->wsrep_conflict_state() == NO_CONFLICT);
+    */
+ 
+   /*
+     sanity check, don't send end statement while replaying
+   */
++#ifdef WITH_WSREP
+   DBUG_ASSERT(thd->wsrep_conflict_state_unsafe() != REPLAYING);
+   if (WSREP(thd) && thd->wsrep_conflict_state_unsafe()== REPLAYING)
+   {
+     WSREP_ERROR("attempting net_end_statement while replaying");
+     return;
+   }
++#endif
+ 
    DBUG_ENTER("Protocol::end_statement");
    DBUG_ASSERT(! thd->get_stmt_da()->is_sent());
    bool error= FALSE;
diff --cc sql/sql_base.cc
index e46b7675bbe,e711dd004db..0587d401461
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@@ -63,6 -63,7 +63,9 @@@
  #endif
  #include "wsrep_mysqld.h"
  #include "wsrep_thd.h"
++#ifdef WITH_WSREP
+ #include "wsrep_sr.h"
++#endif
  
  bool
  No_such_table_error_handler::handle_condition(THD *,
diff --cc sql/sql_class.h
index 25e2d736cdc,a2ae8611f41..8e08864abf7
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@@ -3212,7 -3256,6 +3232,9 @@@ public
      query_id_t first_query_id;
    } binlog_evt_union;
  
++  mysql_mutex_t             LOCK_wsrep_thd;
 +  mysql_cond_t              COND_wsrep_thd;
++
    /**
      Internal parser state.
      Note that since the parser is not re-entrant, we keep only one parser
@@@ -3298,8 -3346,9 +3325,9 @@@
      mysql_mutex_lock(&LOCK_thd_kill);
      awake_no_mutex(state_to_set);
      mysql_mutex_unlock(&LOCK_thd_kill);
+     mysql_mutex_unlock(&LOCK_wsrep_thd);
    }
-- 
++
    /** Disconnect the associated communication endpoint. */
    void disconnect();
  
@@@ -4738,13 -4666,223 +4773,221 @@@ public
    const bool                wsrep_applier; /* dedicated slave applier thread */
    bool                      wsrep_applier_closing; /* applier marked to close */
    bool                      wsrep_client_thread; /* to identify client threads*/
-   bool                      wsrep_PA_safe;
-   bool                      wsrep_converted_lock_session;
-   bool                      wsrep_apply_toi; /* applier processing in TOI */
    enum wsrep_exec_mode      wsrep_exec_mode;
    query_id_t                wsrep_last_query_id;
-   enum wsrep_query_state    wsrep_query_state;
-   enum wsrep_conflict_state wsrep_conflict_state;
+   XID                       wsrep_xid;
+ 
+   /** This flag denotes that record locking should be skipped during INSERT
+   and gap locking during SELECT. Only used by the streaming replication thread
+   that only modifies the wsrep_schema.SR table. */
+   my_bool                   wsrep_skip_locking;
+ 
+   /*
+     Delegated rollback:
+ 
+     Wsrep_query_state, wsrep_conflict_state and wsrep_exec_mode define
+     together the state for rollback process. In most of the cases the
+     client thread will handle the rollback, but if the BF abort happens when
+     the client thread is in QUERY_IDLE state, the rollback process
+     is delegated to rollbacker thread. In this case rollbacker thread
+     will do the rollback on behalf of client thread and will leave
+     the states into the following
+     * wsrep_query_state = QUERY_IDLE
+     * wsrep_conflict_state = ABORTED
+     * wsrep_exec_mode = LOCAL_ROLLBACK
+    */
+ 
+   /*
+     Set wsrep_query_state
+ 
+     Asserts LOCK_wsrep_thd ownership.
+    */
+   void set_wsrep_query_state(enum wsrep_query_state state)
+   {
+     mysql_mutex_assert_owner(&LOCK_wsrep_thd);
+     /*
+       State machine:
+ 
+       QI - QUERY_IDLE
+       QX - QUERY_EXEC
+       QC - QUERY_COMMITTING
+       QOC - QUERY_ORDERED_COMMIT
+       QE - QUERY_EXITING
+ 
+            |---> QE
+            |
+       o--> QI <-> QX -> QC -> QOC 
+                   ^     |      |
+                   |-------------
+      */
+     static const bool allowed[QUERY_EXITING + 1][QUERY_EXITING + 1] =
+     {
+       /* QI     QX     QC     QOC,   QE    To / From  */
+       {  false, true,  false, false, true   }, /* QI  */
+       {  true , false, true,  false, false  }, /* QX  */
+       {  false, true,  false, true,  false  }, /* QC  */
+       {  false, true,  false, false, false  }, /* QOC */
+       {  false, false, false, false, false  }  /* QE  */
+     };
+     if (allowed[m_wsrep_query_state][state] == false)
+     {
+       WSREP_DEBUG("Unallowed query state change %d -> %d",
+                   m_wsrep_query_state, state);
+     }
+     DBUG_ASSERT(allowed[m_wsrep_query_state][state] == true);
+     m_wsrep_query_state= state;
+   }
+ 
+   /*
+     Return current wsrep_query_state
+ 
+     Asserts LOCK_wsrep_thd ownership
+    */
+   enum wsrep_query_state wsrep_query_state() const
+   {
+     mysql_mutex_assert_owner(&LOCK_wsrep_thd);
+     return m_wsrep_query_state;
+   }
+ 
+   /*
+     Return current wsrep_query_state
+ 
+     Does not assert LOCK_wsrep_thd ownership, useful for
+     debugging and logging where strict access is not
+     absolutely required.
+    */
+   enum wsrep_query_state wsrep_query_state_unsafe() const
+   {
+     return m_wsrep_query_state;
+   }
+ 
+   void set_wsrep_conflict_state(enum wsrep_conflict_state state)
+   {
+     mysql_mutex_assert_owner(&LOCK_wsrep_thd);
+     /*
+       State machine
+ 
+       NC - NO_CONFLICT
+       MA - MUST_ABORT
+       AB - ABORTING
+       AD - ABORTED
+       MR - MUST_REPLAY
+       RI - REPLAYING
+       RA - RETRY_AUTOCOMMIT
+       CF - CERT_FAILURE
+ 
+       1) BF abort
+       2) Cert failure or rollback
+       3) Pre commit returns success, so the transaction must be replayed
+       4) Pre commit return certification failure, cleanup after certification
+          failure
+       5) Post rollback phase
+       6) Is autocommit and wsrep_retry_autocommit is set
+       7) Simultaneous certification failure and BF abort
+ 
+                          4                      5
+           -> NC <-----------------> CF -------------------
+           |  ^^                     | 7                  |
+           |  ||     1            2  |                    |
+           | 6||-----------> MA --------> AB ------> AD <-|
+           |  |              |            ^          |
+           |  |-> RA         | 3          |          |
+           |                 |            |          |
+           |                  -> MR -> RI -          |
+           |                           |             |
+           |------------------------------------------
+ 
+      */
+ 
+     static const bool allowed[CERT_FAILURE + 1][CERT_FAILURE + 1] =
+     {
+       /* NC     MA     AB     AD     MR     RI     RA     CF   To / From */
+       {  false, true,  false, false, false, false, true,  true  }, /* NC */
+       {  true,  false, true,  false, true,  false, false, false }, /* MA */
+       {  false, false, false, true,  false, false, false, false }, /* AB */
+       {  true , false, false, false, false, false, false, false }, /* AD */
+       {  false, false, false, false, false, true,  false, false }, /* MR */
+       {  true,  false, true,  false, false, false, false, false }, /* RI */
+       {  true,  false, false, false, false, false, false, false }, /* RA */
+       {  true,  false, true,  true,  false, false, false, false }  /* CF */
+     };
+     if (allowed[m_wsrep_conflict_state][state] == false)
+     {
+       WSREP_DEBUG("Unallowed conflict state change %d -> %d",
+                   m_wsrep_conflict_state, state);
+     }
+     DBUG_ASSERT(allowed[m_wsrep_conflict_state][state] == true);
+ 
+     /*
+       Sanity checks agains query state and thread context
+     */
+     switch (state)
+     {
+     case MUST_ABORT:
+       break;
+     default:
+       /*
+         Only the current thread context may change the state apart from
+         NC -> MA transition.
+       */
+       DBUG_ASSERT(this == current_thd);
+       break;
+     }
+ #ifndef DBUG_OFF
+     if (state == NO_CONFLICT)
+     {
+       m_wsrep_conflict_state_history.clear();
+     }
+     else
+     {
+       m_wsrep_conflict_state_history.push_back(m_wsrep_conflict_state);
+     }
+ #endif /* DBUG_OFF */
+     m_wsrep_conflict_state= state;
+   }
+ 
+   /*
+     Return wsrep_conflict_state
+ 
+     Asserts LOCK_wsrep_thd ownership
+    */
+   enum wsrep_conflict_state wsrep_conflict_state() const
+   {
+     mysql_mutex_assert_owner(&LOCK_wsrep_thd);
+     return m_wsrep_conflict_state;
+   }
+ 
+   /*
+     Return wsrep_conflict_state
+ 
+     Does not assert LOCK_wsrep_thd ownership, useful mostly
+     for debugging and logging where strict access is not
+     aboslutely required.
+    */
+   enum wsrep_conflict_state wsrep_conflict_state_unsafe() const
+   {
+     return m_wsrep_conflict_state;
+   }
+ 
+   /*
+     Check if wsrep rollback process has started
+    */
+   bool wsrep_is_rolling_back() const
+   {
+     return (wsrep_conflict_state() == MUST_ABORT ||
+             wsrep_conflict_state() == ABORTING);
+   }
+ 
+ private:
+   enum wsrep_query_state    m_wsrep_query_state;
+   enum wsrep_conflict_state m_wsrep_conflict_state;
+ #ifndef DBUG_OFF
+   std::vector<enum wsrep_conflict_state> m_wsrep_conflict_state_history;
+ #endif /* DBUG_OFF */
+ public:
+ 
 -  mysql_mutex_t             LOCK_wsrep_thd;
 -  mysql_cond_t              COND_wsrep_thd;
+   // changed from wsrep_seqno_t to wsrep_trx_meta_t in wsrep API rev 75
+   // wsrep_seqno_t             wsrep_trx_seqno;
    wsrep_trx_meta_t          wsrep_trx_meta;
    uint32                    wsrep_rand;
    Relay_log_info            *wsrep_rli;
diff --cc sql/sql_connect.cc
index b48070b9c8f,23ff25cdd8c..088dec72313
--- a/sql/sql_connect.cc
+++ b/sql/sql_connect.cc
@@@ -1407,9 -1406,9 +1406,9 @@@ void do_handle_one_connection(CONNECT *
  #ifdef WITH_WSREP
    if (WSREP(thd))
    {
 -    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +    mysql_mutex_lock(&thd->LOCK_thd_data);
-     thd->wsrep_query_state= QUERY_EXITING;
+     thd->set_wsrep_query_state(QUERY_EXITING);
 -    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +    mysql_mutex_unlock(&thd->LOCK_thd_data);
    }
  #endif
  end_thread:
diff --cc sql/sql_insert.cc
index 51acf10a98a,ccd566c7d43..a6aa8b999dc
--- a/sql/sql_insert.cc
+++ b/sql/sql_insert.cc
@@@ -3902,10 -3919,12 +3902,14 @@@ bool select_insert::prepare_eof(
    DBUG_PRINT("enter", ("trans_table=%d, table_type='%s'",
                         trans_table, table->file->table_type()));
  
-   error= (IF_WSREP((thd->wsrep_conflict_state == MUST_ABORT ||
-                     thd->wsrep_conflict_state == CERT_FAILURE) ? -1 :, )
-           (thd->locked_tables_mode <= LTM_LOCK_TABLES ?
-            table->file->ha_end_bulk_insert() : 0));
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
++
+   error= (IF_WSREP((thd->wsrep_conflict_state() == MUST_ABORT ||
+                     thd->wsrep_conflict_state() == CERT_FAILURE) ? -1 :, )
 -          (thd->locked_tables_mode <= LTM_LOCK_TABLES ?
 -           table->file->ha_end_bulk_insert() : 0));
++                   (thd->locked_tables_mode <= LTM_LOCK_TABLES ?
++                    table->file->ha_end_bulk_insert() : 0));
++
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
  
    if (likely(!error) && unlikely(thd->is_error()))
      error= thd->get_stmt_da()->sql_errno();
@@@ -4564,10 -4583,17 +4568,19 @@@ bool select_create::send_eof(
    if (!table->s->tmp_table)
    {
  #ifdef WITH_WSREP
 +    if (WSREP_ON)
 +    {
+       if (thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID)
+       {
+         (void) wsrep_ws_handle_for_trx(&thd->wsrep_ws_handle,
+                                        thd->wsrep_next_trx_id());
+       }
+       DBUG_ASSERT(thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID);
+       WSREP_DEBUG("CTAS key append for trx: %lu thd %llu query %lld ",
+                   thd->wsrep_trx_id(), thd->thread_id, thd->query_id);
+ 
        /*
-          append table level exclusive key for CTAS
+         append table level exclusive key for CTAS
        */
        wsrep_key_arr_t key_arr= {0, 0};
        wsrep_prepare_keys_for_isolation(thd,
@@@ -4591,8 -4617,7 +4604,8 @@@
          return true;
        }
        /* If commit fails, we should be able to reset the OK status. */
-       thd->get_stmt_da()->set_overwrite_status(TRUE);
+       thd->get_stmt_da()->set_overwrite_status(true);
 +    }
  #endif /* WITH_WSREP */
      trans_commit_stmt(thd);
      if (!(thd->variables.option_bits & OPTION_GTID_BEGIN))
@@@ -4600,14 -4625,14 +4613,14 @@@
  #ifdef WITH_WSREP
      if (WSREP_ON)
      {
-       thd->get_stmt_da()->set_overwrite_status(FALSE);
-       mysql_mutex_lock(&thd->LOCK_thd_data);
-       if (thd->wsrep_conflict_state != NO_CONFLICT)
+       thd->get_stmt_da()->set_overwrite_status(false);
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       if (thd->wsrep_conflict_state() != NO_CONFLICT)
        {
-         WSREP_DEBUG("select_create commit failed, thd: %lld  err: %d %s",
-                     (longlong) thd->thread_id, thd->wsrep_conflict_state,
-                     thd->query());
+         WSREP_DEBUG("select_create commit failed, thd: %lld err: %d %s",
+                     thd->thread_id, thd->wsrep_conflict_state(),
+                     WSREP_QUERY(thd));
 -        mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +        mysql_mutex_unlock(&thd->LOCK_thd_data);
          abort_result_set();
          DBUG_RETURN(true);
        }
diff --cc sql/sql_parse.cc
index 1baa26ab465,49629030dcf..18cded52a74
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@@ -1576,38 -1506,77 +1507,77 @@@ bool dispatch_command(enum enum_server_
    compile_time_assert(COM_END == 255);
  
  #ifdef WITH_WSREP
-   if (WSREP(thd))
+   bool wsrep_on= WSREP_ON;
+   if (wsrep_on)
    {
-     if (!thd->in_multi_stmt_transaction_mode())
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     DBUG_ASSERT(thd->wsrep_query_state() == QUERY_IDLE);
+     if (thd->wsrep_is_rolling_back())
      {
-       thd->wsrep_PA_safe= true;
+       /*
+         Rollback thread is rolling back a transaction for this thd,
+         wait until it has finished.
+       */
+       while (thd->wsrep_is_rolling_back())
+       {
+         mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+         my_sleep(1000);
+         mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       }
+       DBUG_ASSERT(thd->wsrep_conflict_state() == ABORTED);
      }
  
-     mysql_mutex_lock(&thd->LOCK_thd_data);
-     thd->wsrep_query_state= QUERY_EXEC;
-     if (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT)
-     {
-       thd->wsrep_conflict_state= NO_CONFLICT;
-     }
-     if (thd->wsrep_conflict_state== MUST_ABORT)
-     {
-       wsrep_client_rollback(thd);
-     }
-     /* We let COM_QUIT and COM_STMT_CLOSE to execute even if wsrep aborted. */
-     if (thd->wsrep_conflict_state == ABORTED &&
-         command != COM_STMT_CLOSE && command != COM_QUIT)
+     /*
+       BF aborter should atomically check thd query state and start
+       background rollback process if the thd query state is QUERY_IDLE.
+       Because the wait for the background rollback is done above we should
+       be here either in NO_CONFLICT or in ABORTED state.
+     */
+     DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT ||
+                 thd->wsrep_conflict_state() == ABORTED);
+ 
+     thd->set_wsrep_query_state(QUERY_EXEC);
+ 
+     /*
+       Aborted by background rollbacker thread. Jump straight to dispatch_end
+       label where the error handling is performed.
+ 
+       We let COM_QUIT and COM_STMT_CLOSE to execute even if wsrep aborted.
+       (see MDEV-10812).
+     */
+     if (thd->wsrep_conflict_state() == ABORTED)
      {
-       my_message(ER_LOCK_DEADLOCK, "Deadlock: wsrep aborted transaction",
-                  MYF(0));
-       WSREP_DEBUG("Deadlock error for: %s", thd->query());
-       mysql_mutex_unlock(&thd->LOCK_thd_data);
-       thd->reset_killed();
-       thd->mysys_var->abort     = 0;
-       thd->wsrep_conflict_state = NO_CONFLICT;
-       thd->wsrep_retry_counter  = 0;
-       goto dispatch_end;
+       thd->store_globals();
+       WSREP_LOG_THD(thd, "enter found BF aborted");
+       DBUG_ASSERT(!thd->get_stmt_da()->is_set());
+ 
+       if (command == COM_STMT_PREPARE          ||
+           command == COM_STMT_FETCH            ||
+           command == COM_STMT_SEND_LONG_DATA   ||
+           command == COM_STMT_CLOSE            ||
+           command == COM_QUIT
+           )
+       {
+         WSREP_DEBUG("Prepared Statement bail out or COM_QUIT");
+       }
+       else
+       {
+         thd->reset_killed();
+         thd->mysys_var->abort       = 0;
+         thd->wsrep_retry_counter    = 0;
+         mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ #ifdef WSREP_TODO
+         /*
+           Increment threads running to compensate dec_thread_running() called
+           after dispatch_end label.
+         */
+         inc_thread_running();
+ #endif /* WSREP_TODO */
+         goto dispatch_end;
+         mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       }
      }
 -    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +    mysql_mutex_unlock(&thd->LOCK_thd_data);
    }
  #endif /* WITH_WSREP */
  #if defined(ENABLED_PROFILING)
@@@ -1840,8 -1816,27 +1817,29 @@@
        break;
  
      if (WSREP_ON)
-       wsrep_mysql_parse(thd, thd->query(), thd->query_length(), &parser_state,
-                         is_com_multi, is_next_command);
+     {
+       if (wsrep_mysql_parse(thd, thd->query(), thd->query_length(),
+                             &parser_state,
+                             is_com_multi, is_next_command))
+       {
+         WSREP_DEBUG("Deadlock error for: %s", thd->query());
+         mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+         thd->killed               = NOT_KILLED;
+         thd->mysys_var->abort     = 0;
++#ifdef WITH_WSREP
+         thd->wsrep_retry_counter  = 0;
++#endif
+         mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ #ifdef WSREP_TODO
+         /*
+           Increment threads running to compensate dec_thread_running() called
+           after dispatch_end label.
+         */
+         inc_thread_running();
+ #endif /* WSREP_TODO */
+         goto dispatch_end;
+       }
+     }
      else
        mysql_parse(thd, thd->query(), thd->query_length(), &parser_state,
                    is_com_multi, is_next_command);
@@@ -1930,8 -1925,28 +1928,30 @@@
        parser_state.reset(beginning_of_next_stmt, length);
  
        if (WSREP_ON)
-         wsrep_mysql_parse(thd, beginning_of_next_stmt, length, &parser_state,
-                           is_com_multi, is_next_command);
+       {
+         if (wsrep_mysql_parse(thd, beginning_of_next_stmt,
+                               length, &parser_state,
+                               is_com_multi, is_next_command))
+         {
+           WSREP_DEBUG("Deadlock error for: %s", thd->query());
+           mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+           thd->killed               = NOT_KILLED;
+           thd->mysys_var->abort     = 0;
++#ifdef WITH_WSREP
+           thd->wsrep_retry_counter  = 0;
++#endif
+           mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ #ifdef WSREP_TODO
+           /*
+             Increment threads running to compensate dec_thread_running() called
+             after dispatch_end label.
+           */
+           inc_thread_running();
+ #endif /* WSREP_TODO */
+ 
+           goto dispatch_end;
+         }
+       }
        else
          mysql_parse(thd, beginning_of_next_stmt, length, &parser_state,
                      is_com_multi, is_next_command);
@@@ -2367,10 -2382,10 +2387,10 @@@ com_multi_end
      break;
    }
  
--#ifdef WITH_WSREP
   dispatch_end:
++#ifdef WITH_WSREP
  
-   if (WSREP(thd))
+   if (wsrep_on)
    {
      /*
        MDEV-10812
@@@ -2378,15 -2393,52 +2398,51 @@@
      */
      DBUG_ASSERT((command != COM_QUIT && command != COM_STMT_CLOSE)
                    || thd->get_stmt_da()->is_disabled());
-     /* wsrep BF abort in query exec phase */
-     mysql_mutex_lock(&thd->LOCK_thd_data);
-     do_end_of_statement= thd->wsrep_conflict_state != REPLAYING &&
-                          thd->wsrep_conflict_state != RETRY_AUTOCOMMIT;
-     mysql_mutex_unlock(&thd->LOCK_thd_data);
-   }
-   else
-     do_end_of_statement= true;
+     /*
+       BF aborted before sending response back to client
+      */
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     do_end_of_statement= thd->wsrep_conflict_state() != REPLAYING &&
+                          thd->wsrep_conflict_state() != RETRY_AUTOCOMMIT;
+     if (thd->wsrep_conflict_state() == MUST_ABORT)
+     {
+       wsrep_client_rollback(thd);
+       wsrep_post_rollback(thd);
+       DBUG_ASSERT(thd->wsrep_conflict_state() == ABORTED);
+     }
+     if (thd->wsrep_conflict_state() == ABORTED &&
+         !(command == COM_STMT_PREPARE          ||
+           command == COM_STMT_FETCH            ||
+           command == COM_STMT_SEND_LONG_DATA   ||
+           command == COM_STMT_CLOSE            ||
+           command == COM_QUIT
+           ))
+     {
+       wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+       wsrep_cleanup_transaction(thd);
+       WSREP_LOG_THD(thd, "leave");
+     }
 -    mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+     /*
+       Final check after sending response back to client. Handle possible
+       BF abort and query state change atomically. Don't call
+       wsrep_cleanup_transaction() to leave thd->wsrep_conflict_state()
+       to ABORTED so that the rollback will be detected when the client
+       returns to action.
+     */
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     if (thd->wsrep_conflict_state() == MUST_ABORT)
+     {
+       wsrep_client_rollback(thd);
+       wsrep_post_rollback(thd);
+       WSREP_LOG_THD(thd, "after return to client found BF aborted");
+     }
+     thd->set_wsrep_query_state(QUERY_IDLE);
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
  
+   } else { /* if (WSREP(thd))... */
+     do_end_of_statement= true;
+   }
  #endif /* WITH_WSREP */
  
    if (do_end_of_statement)
@@@ -4758,12 -4822,10 +4826,10 @@@ end_with_restore_list
    case SQLCOM_INSERT_SELECT:
    {
      WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE);
 -    select_result *sel_result;
 +    select_insert *sel_result;
      bool explain= MY_TEST(lex->describe);
      DBUG_ASSERT(first_table == all_tables && first_table != 0);
-     if (WSREP_CLIENT(thd) &&
-         wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE))
-       goto error;
+     WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE);
  
      if ((res= insert_precheck(thd, all_tables)))
        break;
@@@ -7854,35 -7909,35 +7961,46 @@@ static bool wsrep_mysql_parse(THD *thd
  	      com_statement_info[thd->get_command()].m_key);
        MYSQL_SET_STATEMENT_TEXT(thd->m_statement_psi, thd->query(),
  	                       thd->query_length());
 +
 +      DBUG_EXECUTE_IF("sync.wsrep_retry_autocommit",
 +                      {
 +                        const char act[]=
 +                          "now "
 +                          "SIGNAL wsrep_retry_autocommit_reached "
 +                          "WAIT_FOR wsrep_retry_autocommit_continue";
 +                        DBUG_ASSERT(!debug_sync_set_action(thd, STRING_WITH_LEN(act)));
 +                      });
      }
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
      mysql_parse(thd, rawbuf, length, parser_state, is_com_multi,
                  is_next_command);
+     (void) wsrep_after_command(thd, !thd->in_active_multi_stmt_transaction());
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
  
+     /*
+       State after after_command hook must be either NO_CONFLICT
+       or MUST_ABORT if the BF abort happened just after leaving
+       after_command hook.
+     */
+     DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT ||
+                 thd->wsrep_conflict_state() == MUST_ABORT ||
+                 thd->wsrep_conflict_state() == RETRY_AUTOCOMMIT);
      if (WSREP(thd)) {
        /* wsrep BF abort in query exec phase */
-       mysql_mutex_lock(&thd->LOCK_thd_data);
-       if (thd->wsrep_conflict_state == MUST_ABORT) {
+       if (thd->wsrep_conflict_state() == MUST_ABORT) {
          wsrep_client_rollback(thd);
  
          WSREP_DEBUG("abort in exec query state, avoiding autocommit");
        }
  
-       if (thd->wsrep_conflict_state == MUST_REPLAY)
+       /* note: this is obsolete, comin from 10.3.6 merging */
+       if (thd->wsrep_conflict_state() == MUST_REPLAY)
        {
-         mysql_mutex_unlock(&thd->LOCK_thd_data);
+         WSREP_WARN("EXPLAIN DELETED");
  	if (thd->lex->explain)
            delete_explain_query(thd->lex);
 +        mysql_mutex_lock(&thd->LOCK_thd_data);
 +
          wsrep_replay_transaction(thd);
        }
  
@@@ -7957,8 -8061,9 +8124,9 @@@
      thd->wsrep_retry_query_len  = 0;
      thd->wsrep_retry_command    = COM_CONNECT;
    }
 +#endif /* WITH_WSREP */
+   return false;
  }
 -#endif /* WITH_WSREP */
  
  /*
    When you modify mysql_parse(), you may need to modify
diff --cc sql/sql_prepare.cc
index c615356b354,6d77497e68f..c9a36b02fe4
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@@ -4197,8 -4196,8 +4197,8 @@@ reexecute
  
    if (WSREP_ON)
    {
 -    mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +    mysql_mutex_lock(&thd->LOCK_thd_data);
-     switch (thd->wsrep_conflict_state)
+     switch (thd->wsrep_conflict_state())
      {
        case CERT_FAILURE:
          WSREP_DEBUG("PS execute fail for CERT_FAILURE: thd: %lld  err: %d",
@@@ -4393,8 -4392,8 +4393,8 @@@ reexecute
  
      if (WSREP_ON)
      {
 -      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +      mysql_mutex_lock(&thd->LOCK_thd_data);
-       switch (thd->wsrep_conflict_state)
+       switch (thd->wsrep_conflict_state())
        {
        case CERT_FAILURE:
          WSREP_DEBUG("PS execute fail for CERT_FAILURE: thd: %lld  err: %d",
diff --cc sql/sql_truncate.cc
index d515aacd1d0,1fcab7e480c..2239afbb13b
--- a/sql/sql_truncate.cc
+++ b/sql/sql_truncate.cc
@@@ -416,9 -416,20 +416,23 @@@ bool Sql_cmd_truncate_table::truncate_t
      if (WSREP(thd) &&
          wsrep_to_isolation_begin(thd, table_ref->db.str, table_ref->table_name.str, 0))
          DBUG_RETURN(TRUE);
-     if (lock_table(thd, table_ref, &hton_can_recreate))
-       DBUG_RETURN(TRUE);
+ 
+     /*
+       When using non-blocking operation for TRUNCATE always fall to
+       handler_truncate() in order to avoid calling lock_table_names()
+       twice.
+      */
++#ifdef WITH_WSREP
+     if (WSREP(thd) && thd->wsrep_nbo_ctx) {
+       hton_can_recreate= false;
+     }
+     else
++#endif
+     {
+       if (lock_table(thd, table_ref, &hton_can_recreate))
+         DBUG_RETURN(TRUE);
+     }
 +
      if (hton_can_recreate)
      {
       /*
diff --cc sql/wsrep_applier.cc
index f2d90def5ef,32e153b29dd..e3de1e160a4
--- a/sql/wsrep_applier.cc
+++ b/sql/wsrep_applier.cc
@@@ -14,6 -14,6 +14,7 @@@
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
  
  #include "mariadb.h"
++#include "my_pthread.h"
  #include "wsrep_priv.h"
  #include "wsrep_binlog.h" // wsrep_dump_rbr_buf()
  #include "wsrep_xid.h"
@@@ -95,14 -146,20 +147,20 @@@ int wsrep_apply_events(THD*        thd
    {
      WSREP_INFO("applier has been aborted, skipping apply_rbr: %lld",
                 (long long) wsrep_thd_trx_seqno(thd));
-     DBUG_RETURN(WSREP_CB_FAILURE);
+     DBUG_RETURN(WSREP_ERR_ABORTED);
    }
  
 -  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_lock(&thd->LOCK_thd_data);
-   thd->wsrep_query_state= QUERY_EXEC;
-   if (thd->wsrep_conflict_state!= REPLAYING)
-     thd->wsrep_conflict_state= NO_CONFLICT;
-   mysql_mutex_unlock(&thd->LOCK_thd_data);
+   // thd->set_wsrep_query_state(QUERY_EXEC);
+   if (thd->wsrep_conflict_state() !=  REPLAYING)
+   {
+     DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+     if (thd->wsrep_conflict_state() != NO_CONFLICT)
+     {
+       thd->set_wsrep_conflict_state(NO_CONFLICT);
+     }
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
  
    if (!buf_len) WSREP_DEBUG("empty rbr buffer to apply: %lld",
                              (long long) wsrep_thd_trx_seqno(thd));
@@@ -197,9 -254,8 +255,8 @@@
    }
  
   error:
 -  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 -  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_lock(&thd->LOCK_thd_data);
-   thd->wsrep_query_state= QUERY_IDLE;
 +  mysql_mutex_unlock(&thd->LOCK_thd_data);
  
    assert(thd->wsrep_exec_mode== REPL_RECV);
  
diff --cc sql/wsrep_dummy.cc
index 0b6e7e0d5bb,4f099460919..57cc0d51997
--- a/sql/wsrep_dummy.cc
+++ b/sql/wsrep_dummy.cc
@@@ -17,7 -17,7 +17,13 @@@
  #include <sql_class.h>
  #include <mysql/service_wsrep.h>
  
- my_bool wsrep_thd_is_BF(THD *, my_bool)
++int64_t wsrep_thd_trx_seqno(const THD *)
++{ return 0;}
++
++my_thread_id wsrep_thd_thread_id(THD *thd)
++{ return (my_thread_id)0; }
++
+ my_bool wsrep_thd_is_BF(void *, my_bool)
  { return 0; }
  
  int wsrep_trx_order_before(THD *, THD *)
@@@ -38,6 -39,10 +45,9 @@@ const unsigned char* wsrep_xid_uuid(con
      return uuid;
  }
  
 -#ifdef TODO
+ bool wsrep_prepare_key_for_innodb(THD* thd, const uchar*, size_t, const uchar*, size_t, struct wsrep_buf*, size_t*)
 -{ return -1; }
 -#endif
++{ return 0; }
++
  bool wsrep_prepare_key(const uchar*, size_t, const uchar*, size_t, struct wsrep_buf*, size_t*)
  { return 0; }
  
@@@ -77,6 -82,6 +87,9 @@@ bool wsrep_consistency_check(THD *
  void wsrep_lock_rollback()
  { }
  
++int wsrep_on(void *)
++{ return 0;}
++
  int wsrep_on(THD *thd)
  { return 0; }
  
diff --cc sql/wsrep_mysqld.cc
index 0a2735fe0b7,49f60f5241b..951497e100f
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@@ -33,11 -35,19 +35,18 @@@
  #include "wsrep_var.h"
  #include "wsrep_binlog.h"
  #include "wsrep_applier.h"
+ #include "wsrep_sr.h"
+ #include "wsrep_sr_file.h"
+ #include "wsrep_sr_table.h"
+ #include "wsrep_schema.h"
+ #include "wsrep_thd_pool.h"
  #include "wsrep_xid.h"
+ #include "wsrep_trans_observer.h"
  #include <cstdio>
  #include <cstdlib>
+ #include <string>
  #include "log_event.h"
  #include <slave.h>
 -#include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
  
  wsrep_t *wsrep                  = NULL;
  /*
@@@ -145,11 -166,16 +165,15 @@@ PSI_mutex_key key_LOCK_wsrep_thd
    key_LOCK_wsrep_replaying, key_LOCK_wsrep_ready, key_LOCK_wsrep_sst,
    key_LOCK_wsrep_sst_thread, key_LOCK_wsrep_sst_init,
    key_LOCK_wsrep_slave_threads, key_LOCK_wsrep_desync,
-   key_LOCK_wsrep_config_state;
+   key_LOCK_wsrep_config_state,
+   key_LOCK_wsrep_SR_pool,
+   key_LOCK_wsrep_SR_store, key_LOCK_wsrep_thd_pool, key_LOCK_wsrep_nbo,
+   key_LOCK_wsrep_thd_queue;
  
- PSI_cond_key key_COND_wsrep_rollback,
+ PSI_cond_key key_COND_wsrep_thd,
    key_COND_wsrep_replaying, key_COND_wsrep_ready, key_COND_wsrep_sst,
-   key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread;
+   key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread,
+   key_COND_wsrep_nbo, key_COND_wsrep_thd_queue;
 -  
  
  PSI_file_key key_file_wsrep_gra_log;
  
@@@ -212,11 -242,24 +240,25 @@@ long long   wsrep_local_bf_aborts    = 
  const char* wsrep_provider_name      = provider_name;
  const char* wsrep_provider_version   = provider_version;
  const char* wsrep_provider_vendor    = provider_vendor;
+ char* wsrep_provider_capabilities    = NULL;
+ char* wsrep_cluster_capabilities     = NULL;
  /* End wsrep status variables */
  
- wsrep_uuid_t     local_uuid   = WSREP_UUID_UNDEFINED;
- wsrep_seqno_t    local_seqno  = WSREP_SEQNO_UNDEFINED;
- long             wsrep_protocol_version = 3;
+ wsrep_uuid_t               local_uuid        = WSREP_UUID_UNDEFINED;
+ wsrep_seqno_t              local_seqno       = WSREP_SEQNO_UNDEFINED;
+ wsp::node_status           local_status;
+ static wsrep_view_status_t local_view_status = WSREP_VIEW_NON_PRIMARY;
+ 
+ class SR_storage_file  *wsrep_SR_store_file = NULL;
+ class SR_storage_table *wsrep_SR_store_table = NULL;
+ class SR_storage       *wsrep_SR_store = NULL;
+ 
+ /*
+  */
+ #define WSREP_THD_POOL_SIZE 16
+ Wsrep_thd_pool* wsrep_thd_pool= 0;
+ Wsrep_schema *wsrep_schema= 0;
++static bool wsrep_schema_inited= false;
  
  wsp::Config_state *wsrep_config_state;
  
@@@ -296,19 -344,178 +343,187 @@@ void wsrep_init_sidno(const wsrep_uuid_
  }
  #endif /* GTID_SUPPORT */
  
- static wsrep_cb_status_t
- wsrep_view_handler_cb (void*                    app_ctx,
-                        void*                    recv_ctx,
-                        const wsrep_view_info_t* view,
-                        const char*              state,
-                        size_t                   state_len,
-                        void**                   sst_req,
-                        size_t*                  sst_req_len)
+ void wsrep_init_schema()
  {
-   *sst_req     = NULL;
-   *sst_req_len = 0;
 -  DBUG_ASSERT(!wsrep_schema);
 -
+   WSREP_INFO("wsrep_init_schema_and_SR %p %p", wsrep_schema, wsrep_SR_store);
+   if (!wsrep_schema)
+   {
+     if (wsrep_before_SE()) {
+       delete wsrep_thd_pool;
++      wsrep_thd_pool=0;
++    }
++
++    if (!wsrep_thd_pool) {
+       wsrep_thd_pool= new Wsrep_thd_pool(WSREP_THD_POOL_SIZE);
+     }
++
+     wsrep_schema= new Wsrep_schema(wsrep_thd_pool);
 -    if (wsrep_schema->init())
++
++    if (strcmp (wsrep_provider, WSREP_NONE))
+     {
 -      WSREP_ERROR("Failed to init wsrep schema");
 -      unireg_abort(1);
++      if (wsrep_schema->init())
++      {
++        WSREP_ERROR("Failed to init wsrep schema");
++        delete wsrep_schema;
++        delete wsrep_thd_pool;
++        unireg_abort(1);
++      }
+     }
+   }
+ }
+ 
+ void wsrep_init_SR()
+ {
+   /* initialize SR pools, now that innodb has initialized */
+   if (wsrep_SR_store && wsrep_SR_store->init(wsrep_cluster_state_uuid,
+                                              wsrep_schema)) {
+     WSREP_ERROR("wsrep SR persistency store initialization failed");
+     unireg_abort(1);
+   }
+   else {
+     if (wsrep_SR_store && wsrep_SR_store->restore(0)) {
+       WSREP_ERROR("wsrep SR persistency restore failed");
+       unireg_abort(1);
+     }
+   }
+ }
+ 
+ int wsrep_replay_from_SR_store(THD* thd, const wsrep_trx_meta_t& meta)
+ {
+   DBUG_ENTER("wsrep_replay_from_SR_store");
+   if (!wsrep_SR_store) {
+     WSREP_ERROR("no SR persistency store defined, can't replay");
+     DBUG_RETURN(1);
+   }
+ 
+   int ret= wsrep_SR_store->replay_trx(thd, meta);
+   DBUG_RETURN(ret);
+ }
+ 
+ static void wsrep_rollback_SR_connections()
+ {
+   THD *tmp;
+   mysql_mutex_lock(&LOCK_thread_count);
+ 
+   I_List_iterator<THD> it(threads);
+   while ((tmp=it++))
+   {
+     mysql_mutex_lock(&tmp->LOCK_wsrep_thd);
+     if (tmp->wsrep_client_thread && tmp->wsrep_is_streaming())
+     {
+       tmp->set_wsrep_conflict_state(MUST_ABORT);
+       if (tmp->wsrep_query_state() == QUERY_IDLE)
+       {
+         wsrep_fire_rollbacker(tmp);
+       }
+       /*
+         No need to send rollback fragment for this trx: slaves rollback
+         all SR transactions whose master goes non-Primary.
+       */
+       tmp->wsrep_SR_rollback_replicated_for_trx= tmp->wsrep_trx_id();
+     }
+     mysql_mutex_unlock(&tmp->LOCK_wsrep_thd);
+   }
+ 
+   mysql_mutex_unlock(&LOCK_thread_count);
+ }
+ 
+ /** Export the WSREP provider's capabilities as a human readable string.
+  * The result is saved in a dynamically allocated string of the form:
+  * :cap1:cap2:cap3:
+  */
+ static void wsrep_capabilities_export(wsrep_cap_t const cap, char** str)
+ {
+   static const char* names[] =
+   {
+     /* Keep in sync with wsrep/wsrep_api.h WSREP_CAP_* macros. */
+     "MULTI_MASTER",
+     "CERTIFICATION",
+     "PARALLEL_APPLYING",
+     "TRX_REPLAY",
+     "ISOLATION",
+     "PAUSE",
+     "CAUSAL_READS",
+     "CAUSAL_TRX",
+     "INCREMENTAL_WRITESET",
+     "SESSION_LOCKS",
+     "DISTRIBUTED_LOCKS",
+     "CONSISTENCY_CHECK",
+     "UNORDERED",
+     "ANNOTATION",
+     "PREORDERED",
+     "STREAMING",
+     "SNAPSHOT",
+     "NBO",
+   };
+ 
+   std::string s;
+   for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); ++i)
+   {
+     if (cap & (1ULL << i))
+     {
+       if (s.empty())
+       {
+         s = ":";
+       }
+       s += names[i];
+       s += ":";
+     }
+   }
+ 
+   /* A read from the string pointed to by *str may be started at any time,
+    * so it must never point to free(3)d memory or non '\0' terminated string. */
+ 
+   char* const previous = *str;
+ 
+   *str = strdup(s.c_str());
+ 
+   if (previous != NULL)
+   {
+     free(previous);
+   }
+ }
+ 
+ wsrep_cb_status_t
+ wsrep_connected_handler_cb(void* app_ctx,
+                            const wsrep_view_info_t* initial_view)
+ {
+   if (initial_view->my_idx < 0) {
+     WSREP_ERROR("Invalid index %d in initial view", initial_view->my_idx);
+     return WSREP_CB_FAILURE;
+   }
+ 
+   node_uuid= initial_view->members[initial_view->my_idx].id;
+   cluster_uuid= initial_view->state_id.uuid;
+   wsrep_cluster_status= cluster_status_str[initial_view->status];
+ 
+   char node_uuid_str[WSREP_UUID_STR_LEN + 1];
+   (void)wsrep_uuid_print(&node_uuid, node_uuid_str, sizeof(node_uuid_str));
+   (void)wsrep_uuid_print(&cluster_uuid, cluster_uuid_str,
+                          sizeof(cluster_uuid_str));
+ 
+   WSREP_INFO("Connected to cluster %s with id: %s",
+              cluster_uuid_str, node_uuid_str);
+   return WSREP_CB_SUCCESS;
+ }
  
-   wsrep_member_status_t memb_status= wsrep_config_state->get_status();
+ static wsrep_cb_status_t
+ wsrep_view_handler_cb(void*                    app_ctx,
+                       void*                    recv_ctx,
+                       const wsrep_view_info_t* view,
+                       /* TODO: These are unused, should be removed?*/
+                       const char*              state,
+                       size_t                   state_len)
+ {
+   /* Allow calling view handler from non-applier threads */
+   struct st_my_thread_var* tmp_thread_var= 0;
+   if (!my_thread_var) {
+     my_thread_init();
+     tmp_thread_var= my_thread_var;
+   }
+  
+   wsrep_cb_status_t ret= WSREP_CB_SUCCESS;
+   wsrep_member_status_t new_status= local_status.get();
  
    if (memcmp(&cluster_uuid, &view->state_id.uuid, sizeof(wsrep_uuid_t)))
    {
@@@ -853,6 -1233,12 +1240,14 @@@ void wsrep_init_startup (bool first
  void wsrep_deinit(bool free_options)
  {
    DBUG_ASSERT(wsrep_inited == 1);
 -  delete wsrep_schema;
++  if (wsrep_schema)
++    delete wsrep_schema;
+   wsrep_schema= 0;
+   WSREP_DEBUG("wsrep_deinit, free %d", free_options);
 -  delete wsrep_thd_pool;
++  if (wsrep_thd_pool)
++    delete wsrep_thd_pool;
+   wsrep_thd_pool= 0;
+ 
    wsrep_unload(wsrep);
    wsrep= 0;
    provider_name[0]=    '\0';
@@@ -1000,13 -1462,15 +1471,17 @@@ bool wsrep_start_replication(
  
  bool wsrep_must_sync_wait (THD* thd, uint mask)
  {
-   return (thd->variables.wsrep_sync_wait & mask) &&
+   bool ret;
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   ret= (thd->variables.wsrep_sync_wait & mask) &&
      thd->variables.wsrep_on &&
 +    !(thd->variables.wsrep_dirty_reads &&
 +      !is_update_query(thd->lex->sql_command)) &&
      !thd->in_active_multi_stmt_transaction() &&
-     thd->wsrep_conflict_state != REPLAYING &&
+     thd->wsrep_conflict_state() != REPLAYING &&
      thd->wsrep_sync_wait_gtid.seqno == WSREP_SEQNO_UNDEFINED;
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+   return ret;
  }
  
  bool wsrep_sync_wait (THD* thd, uint mask)
@@@ -1399,8 -1972,97 +1983,97 @@@ create_view_query(THD *thd, uchar** buf
  }
  
  /* Forward declarations. */
- static int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len);
- static int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
+ int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len);
+ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
+ /*
+   Rewrite DROP TABLE for TOI. Temporary tables are eliminated from
+   the query as they are visible only to client connection.
+ 
+   TODO: See comments for sql_base.cc:drop_temporary_table() and refine
+   the function to deal with transactional locked tables.
+  */
+ static int wsrep_drop_table_query(THD* thd, uchar** buf, size_t* buf_len)
+ {
+ 
+   LEX* lex= thd->lex;
 -  SELECT_LEX* select_lex= &lex->select_lex;
++  SELECT_LEX* select_lex= lex->first_select_lex();
+   TABLE_LIST* first_table= select_lex->table_list.first;
+   String buff;
+ 
+   bool found_temp_table= false;
+   for (TABLE_LIST* table= first_table; table; table= table->next_global)
+   {
+     if (thd->find_temporary_table(table->db.str, table->table_name.str))
+     {
+       found_temp_table= true;
+       break;
+     }
+   }
+ 
+   if (found_temp_table)
+   {
+     buff.append("DROP TABLE ");
+     if (lex->create_info.if_exists())
+       buff.append("IF EXISTS ");
+ 
+     for (TABLE_LIST* table= first_table; table; table= table->next_global)
+     {
+       if (!thd->find_temporary_table(table->db.str, table->table_name.str))
+       {
+         append_identifier(thd, &buff, table->db.str, table->db.length);
+         buff.append(".");
+         append_identifier(thd, &buff, table->table_name.str, table->table_name.length);
+         buff.append(",");
+       }
+     }
+ 
+     /* Chop the last comma */
+     buff.chop();
+     buff.append(" /* generated by wsrep */");
+ 
+     WSREP_DEBUG("Rewrote '%s' as '%s'", thd->query(), buff.ptr());
+ 
+     return wsrep_to_buf_helper(thd, buff.ptr(), buff.length(), buf, buf_len);
+   }
+   else
+   {
+     return wsrep_to_buf_helper(thd, thd->query(), thd->query_length(),
+                                buf, buf_len);
+   }
+ }
+ 
+ static int wsrep_TOI_event_buf(THD* thd, uchar** buf, size_t* buf_len)
+ {
+   int err;
+   switch (thd->lex->sql_command)
+   {
+   case SQLCOM_CREATE_VIEW:
+     err= create_view_query(thd, buf, buf_len);
+     break;
+   case SQLCOM_CREATE_PROCEDURE:
+   case SQLCOM_CREATE_SPFUNCTION:
+     err= wsrep_create_sp(thd, buf, buf_len);
+     break;
+   case SQLCOM_CREATE_TRIGGER:
+     err= wsrep_create_trigger_query(thd, buf, buf_len);
+     break;
+   case SQLCOM_CREATE_EVENT:
+     err= wsrep_create_event_query(thd, buf, buf_len);
+     break;
+   case SQLCOM_ALTER_EVENT:
+     err= wsrep_alter_event_query(thd, buf, buf_len);
+     break;
+   case SQLCOM_DROP_TABLE:
+     err= wsrep_drop_table_query(thd, buf, buf_len);
+     break;
+   default:
+     err= wsrep_to_buf_helper(thd, thd->query(), thd->query_length(), buf,
+                              buf_len);
+     break;
+   }
+ 
+   return err;
+ }
  
  /*
    Decide if statement should run in TOI.
@@@ -1831,10 -2773,11 +2784,11 @@@ bool wsrep_grant_mdl_exception(MDL_cont
         lock is not granted to the requester THD, thus it has to wait.
         @return false
    */
+   mysql_mutex_lock(&request_thd->LOCK_wsrep_thd);
    if (request_thd->wsrep_exec_mode == TOTAL_ORDER ||
-       request_thd->wsrep_exec_mode == REPL_RECV)
-   {
+       request_thd->wsrep_exec_mode == REPL_RECV) {
+ 
 -    mysql_mutex_unlock(&request_thd->LOCK_wsrep_thd);
 +    mysql_mutex_unlock(&request_thd->LOCK_thd_data);
      WSREP_MDL_LOG(DEBUG, "MDL conflict ", schema, schema_len,
                    request_thd, granted_thd);
      ticket->wsrep_report(wsrep_debug);
@@@ -1854,8 -2809,24 +2820,24 @@@
      {
        WSREP_DEBUG("BF thread waiting for FLUSH");
        ticket->wsrep_report(wsrep_debug);
 -      mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
 +      mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
-       ret= false;
+       ret = FALSE;
+     }
+     else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE)
+     {
+       WSREP_DEBUG("DROP caused BF abort");
+       ticket->wsrep_report(wsrep_debug);
+       mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+       wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1);
+       ret = FALSE;
+     }
+     else if (granted_thd->wsrep_query_state() == QUERY_COMMITTING)
+     {
+       WSREP_DEBUG("mdl granted, but commiting thd abort scheduled");
+       ticket->wsrep_report(wsrep_debug);
+       mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd);
+       wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1);
+       ret = FALSE;
      }
      else
      {
@@@ -1886,9 -2872,8 +2883,8 @@@
    }
    else
    {
 -    mysql_mutex_unlock(&request_thd->LOCK_wsrep_thd);
 +    mysql_mutex_unlock(&request_thd->LOCK_thd_data);
    }
- 
    return ret;
  }
  
@@@ -2043,9 -3133,9 +3144,9 @@@ static inline bool is_replaying_connect
  {
    bool ret;
  
 -  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_lock(&thd->LOCK_thd_data);
-   ret=  (thd->wsrep_conflict_state == REPLAYING) ? true : false;
+   ret=  (thd->wsrep_conflict_state() == REPLAYING) ? true : false;
 -  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_unlock(&thd->LOCK_thd_data);
  
    return ret;
  }
@@@ -2055,9 -3145,9 +3156,9 @@@ static inline bool is_committing_connec
  {
    bool ret;
  
 -  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_lock(&thd->LOCK_thd_data);
-   ret=  (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false;
+   ret=  (thd->wsrep_query_state() == QUERY_COMMITTING) ? true : false;
 -  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_unlock(&thd->LOCK_thd_data);
  
    return ret;
  }
diff --cc sql/wsrep_mysqld.h
index 699a4daf27a,82e8ceafc45..5145fe86a19
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@@ -24,36 -26,71 +24,28 @@@
  #ifdef WITH_WSREP
  
  typedef struct st_mysql_show_var SHOW_VAR;
 -#include "mysqld.h"
 -#include "log.h"
++
  #include <sql_priv.h>
--//#include "rpl_gtid.h"
  #include "../wsrep/wsrep_api.h"
  #include "mdl.h"
  #include "mysqld.h"
 -#include "log.h"
  #include "sql_table.h"
  
+ #include <vector>
+ 
  #define WSREP_UNDEFINED_TRX_ID ULONGLONG_MAX
  
  class set_var;
  class THD;
  
+ typedef std::vector<wsrep_trx_meta_t> wsrep_fragment_set;
+ 
 -enum wsrep_exec_mode {
 -    LOCAL_STATE,
 -    REPL_RECV,
 -    TOTAL_ORDER,
 -    LOCAL_COMMIT,
 -    LOCAL_ROLLBACK
 -};
 -
 -enum wsrep_query_state {
 -    QUERY_IDLE,
 -    QUERY_EXEC,
 -    QUERY_COMMITTING,
 -    QUERY_ORDERED_COMMIT,
 -    QUERY_EXITING
 -};
 -
 -enum wsrep_conflict_state {
 -    NO_CONFLICT,
 -    MUST_ABORT,
 -    ABORTING,
 -    ABORTED,
 -    MUST_REPLAY,
 -    REPLAYING,
 -    RETRY_AUTOCOMMIT,
 -    CERT_FAILURE
 -};
 -
  enum wsrep_consistency_check_mode {
      NO_CONSISTENCY_CHECK,
      CONSISTENCY_CHECK_DECLARED,
      CONSISTENCY_CHECK_RUNNING,
  };
  
 -#ifdef OLD_MARIADB
--struct wsrep_thd_shadow {
--  ulonglong            options;
--  uint                 server_status;
--  enum wsrep_exec_mode wsrep_exec_mode;
--  Vio                  *vio;
--  ulong                tx_isolation;
--  const char           *db;
--  size_t               db_length;
--  my_hrtime_t          user_time;
--  longlong             row_count_func;
--};
- 
 -#endif
  // Global wsrep parameters
  extern wsrep_t*    wsrep;
  
@@@ -77,7 -116,8 +69,6 @@@ extern const char* wsrep_start_position
  extern ulong       wsrep_max_ws_size;
  extern ulong       wsrep_max_ws_rows;
  extern const char* wsrep_notify_cmd;
- extern long        wsrep_max_protocol_version;
 -extern my_bool     wsrep_certify_nonPK;
 -//extern int        wsrep_protocol_version;
  extern ulong       wsrep_forced_binlog_format;
  extern my_bool     wsrep_desync;
  extern ulong       wsrep_reject_queries;
@@@ -114,9 -162,16 +110,17 @@@ enum enum_wsrep_sync_wait 
      WSREP_SYNC_WAIT_MAX = 0xF
  };
  
+ enum enum_wsrep_ignore_apply_error {
+     WSREP_IGNORE_ERRORS_NONE = 0x0,
+     WSREP_IGNORE_ERRORS_ON_RECONCILING_DDL = 0x1,
+     WSREP_IGNORE_ERRORS_ON_RECONCILING_DML = 0x2,
+     WSREP_IGNORE_ERRORS_ON_DDL = 0x4,
+     WSREP_IGNORE_ERRORS_MAX = 0x7
+ };
+ 
  // MySQL status variables
  extern my_bool     wsrep_connected;
 +extern my_bool     wsrep_ready;
  extern const char* wsrep_cluster_state_uuid;
  extern long long   wsrep_cluster_conf_id;
  extern const char* wsrep_cluster_status;
@@@ -126,9 -181,14 +130,12 @@@ extern long long   wsrep_local_bf_abort
  extern const char* wsrep_provider_name;
  extern const char* wsrep_provider_version;
  extern const char* wsrep_provider_vendor;
+ extern char*       wsrep_provider_capabilities;
+ extern char*       wsrep_cluster_capabilities;
  
- int  wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff,
-                        enum enum_var_type scope);
 -//int  wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff,
 -//                       enum enum_var_type scope);
+ int  wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff);
+ int  wsrep_show_ready(THD *thd, SHOW_VAR *var, char *buff);
+ void wsrep_free_status(THD *thd);
  int  wsrep_init();
  void wsrep_deinit(bool free_options);
  
@@@ -147,20 -207,43 +154,19 @@@ void wsrep_init_startup(bool before)
  // Other wsrep global variables
  extern my_bool     wsrep_inited; // whether wsrep is initialized ?
  
 -extern "C" {
 -  enum wsrep_exec_mode wsrep_thd_exec_mode(THD *thd);
 -  enum wsrep_conflict_state wsrep_thd_conflict_state(THD *thd, my_bool sync);
 -  void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
 -  void wsrep_thd_set_conflict_state(
 -        THD *thd, enum wsrep_conflict_state state);
 -//void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
 -//void wsrep_thd_set_conflict_state(
 -//         THD *thd, enum wsrep_conflict_state state);
--
- extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode);
- extern "C" void wsrep_thd_set_query_state(
-         THD *thd, enum wsrep_query_state state);
 -enum wsrep_query_state wsrep_thd_query_state(THD *thd);
 -
 -const char * wsrep_thd_exec_mode_str(THD *thd);
 -const char * wsrep_thd_conflict_state_str(THD *thd);
 -const char * wsrep_thd_query_state_str(THD *thd);
 -wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd);
 -}
--
  extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id);
--
+ extern "C" void wsrep_fire_rollbacker(THD *thd);
  extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd);
  extern "C" time_t wsrep_thd_query_start(THD *thd);
 -extern "C" my_thread_id wsrep_thd_thread_id(THD *thd);
 -extern "C" int64_t wsrep_thd_trx_seqno(const THD *thd);
  extern "C" query_id_t wsrep_thd_query_id(THD *thd);
+ extern "C" wsrep_trx_id_t wsrep_thd_next_trx_id(THD *thd);
+ extern "C" wsrep_trx_id_t wsrep_thd_trx_id(THD *thd);
 -extern "C" char * wsrep_thd_query(THD *thd);
 -//extern char * wsrep_thd_query(THD *thd);
  extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
  extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
  
- extern void wsrep_close_client_connections(my_bool wait_to_end);
+ extern "C" void wsrep_thd_last_written_gtid(THD *thd, wsrep_gtid_t *t);
+ extern "C" ulong wsrep_thd_trx_fragment_size(THD *thd);
 -extern void wsrep_thd_xid(const void *thd_ptr, void *xid, size_t xid_size);
+ 
  extern int  wsrep_wait_committing_connections_close(int wait_time);
  extern void wsrep_close_applier(THD *thd);
  extern void wsrep_wait_appliers_close(THD *thd);
@@@ -186,8 -280,11 +203,15 @@@ extern wsrep_seqno_t wsrep_locked_seqno
     wsrep_provider                     && \
     strcmp(wsrep_provider, WSREP_NONE))
  
++#define WSREP_NOT_SPECIFIED              \
++  (strlen(wsrep_provider)== 0 ||         \
++   !strcmp(wsrep_provider, WSREP_NONE))
++
  #define WSREP(thd) \
-   (WSREP_ON && thd->variables.wsrep_on)
+   (thd && WSREP_NNULL(thd))
+ 
+ #define WSREP_CLIENT_NNULL(thd) \
+   (WSREP_NNULL(thd) && thd->wsrep_client_thread)
  
  #define WSREP_CLIENT(thd) \
      (WSREP(thd) && thd->wsrep_client_thread)
@@@ -231,6 -344,30 +259,22 @@@ void wsrep_log(void (*fun)(const char *
  extern my_bool wsrep_ready_get();
  extern void wsrep_ready_wait();
  
 -#ifndef OUT
 -enum wsrep_trx_status {
 -    WSREP_TRX_OK,
 -    WSREP_TRX_CERT_FAIL,      /* certification failure, must abort */
 -    WSREP_TRX_SIZE_EXCEEDED,  /* trx size exceeded */
 -    WSREP_TRX_ERROR,          /* native mysql error */
 -};
 -#endif
+ static inline
+ wsrep_status_t wsrep_trx_status_to_wsrep_status(wsrep_trx_status status)
+ {
+   switch (status)
+   {
+   case WSREP_TRX_OK:
+     return WSREP_OK;
+   case WSREP_TRX_CERT_FAIL:
+   case WSREP_TRX_ERROR:
+     return WSREP_TRX_FAIL;
+   case WSREP_TRX_SIZE_EXCEEDED:
+     return WSREP_SIZE_EXCEEDED;
+   }
+   return WSREP_NOT_IMPLEMENTED;
+ }
+ 
  class Ha_trx_info;
  struct THD_TRANS;
  void wsrep_register_hton(THD* thd, bool all);
@@@ -237,12 -376,8 +283,6 @@@ void wsrep_post_rollback(THD* thd)
  void wsrep_brute_force_killer(THD *thd);
  int  wsrep_hire_brute_force_killer(THD *thd, uint64_t trx_id);
  
- /* this is visible for client build so that innodb plugin gets this */
- typedef struct wsrep_aborting_thd {
-   struct wsrep_aborting_thd *next;
-   THD *aborting_thd;
- } *wsrep_aborting_thd_t;
 -//extern "C" bool wsrep_consistency_check(void *thd_ptr);
--
  extern mysql_mutex_t LOCK_wsrep_ready;
  extern mysql_cond_t  COND_wsrep_ready;
  extern mysql_mutex_t LOCK_wsrep_sst;
@@@ -267,6 -402,9 +307,9 @@@ extern my_bool       wsrep_preordered_o
  extern handlerton    *wsrep_hton;
  
  #ifdef HAVE_PSI_INTERFACE
++
+ extern PSI_mutex_key key_LOCK_wsrep_thd;
+ extern PSI_cond_key  key_COND_wsrep_thd;
 -
  extern PSI_mutex_key key_LOCK_wsrep_ready;
  extern PSI_mutex_key key_COND_wsrep_ready;
  extern PSI_mutex_key key_LOCK_wsrep_sst;
@@@ -287,12 -431,142 +336,143 @@@ extern PSI_file_key key_file_wsrep_gra_
  struct TABLE_LIST;
  int wsrep_to_isolation_begin(THD *thd, const char *db_, const char *table_,
                               const TABLE_LIST* table_list);
+ 
+ void wsrep_begin_nbo_unlock(THD*);
+ void wsrep_end_nbo_lock(THD*, const TABLE_LIST *table_list);
+ 
  void wsrep_to_isolation_end(THD *thd);
+ 
  void wsrep_cleanup_transaction(THD *thd);
+ bool wsrep_append_SR_keys(THD *thd);
  int wsrep_to_buf_helper(
    THD* thd, const char *query, uint query_len, uchar** buf, size_t* buf_len);
+ int wsrep_create_sp(THD *thd, uchar** buf, size_t* buf_len);
+ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
  int wsrep_create_event_query(THD *thd, uchar** buf, size_t* buf_len);
  
+ bool wsrep_stmt_rollback_is_safe(THD* thd);
+ 
+ void wsrep_init_sidno(const wsrep_uuid_t&);
+ bool wsrep_node_is_donor();
+ bool wsrep_node_is_synced();
+ 
+ void wsrep_init_schema();
+ void wsrep_init_SR();
+ void wsrep_verify_SE_checkpoint(const wsrep_uuid_t& uuid, wsrep_seqno_t seqno);
+ int wsrep_replay_from_SR_store(THD*, const wsrep_trx_meta_t&);
+ void wsrep_node_uuid(wsrep_uuid_t&);
+ 
+ class Log_event;
+ int wsrep_ignored_error_code(Log_event* ev, int error);
+ int wsrep_must_ignore_error(THD* thd);
+ 
+ bool wsrep_replicate_GTID(THD* thd);
+ 
+ /*
+  * Helper class for non-blocking operations.
+  */
+ class Wsrep_nbo_ctx
+ {
+  public:
+  Wsrep_nbo_ctx(const void* buf, size_t buf_len,
+                uint32_t flags, const wsrep_trx_meta_t& meta) :
+     buf_    (0),
+     buf_len_(buf_len),
+     flags_  (flags),
+     meta_   (meta),
+     mutex_  (),
+     cond_   (),
+     ready_  (false),
+     executing_(false),
+     toi_released_(false)
+     {
+       mysql_mutex_init(key_LOCK_wsrep_nbo, &mutex_, MY_MUTEX_INIT_FAST);
+       mysql_cond_init(key_COND_wsrep_nbo, &cond_, NULL);
+ 
+       if ((buf_ = malloc(buf_len_)) == 0) {
+         throw std::exception();
+       }
+       memcpy(buf_, buf, buf_len);
+     }
+ 
+ 
+   ~Wsrep_nbo_ctx() {
+     free(buf_);
+   }
+ 
+   const void* buf() { return buf_; }
+   size_t buf_len() { return buf_len_; }
+   uint32_t flags() { return flags_; }
+   const wsrep_trx_meta_t& meta() { return meta_; }
+ 
+   void wait_sync()
+   {
+     mysql_mutex_lock(&mutex_);
+     while (ready_ == false)
+     {
+       mysql_cond_wait(&cond_, &mutex_);
+     }
+     mysql_mutex_unlock(&mutex_);
+   }
+ 
+   void signal()
+   {
+     mysql_mutex_lock(&mutex_);
+     ready_ = true;
+     mysql_cond_signal(&cond_);
+     mysql_mutex_unlock(&mutex_);
+   }
+ 
+   bool ready() const { return ready_; }
+ 
+   void set_executing(bool val) { executing_ = val; }
+ 
+   bool executing() const { return executing_; }
+ 
+   void set_toi_released(bool val) { toi_released_ = true; }
+ 
+   bool toi_released() const { return toi_released_; }
+ 
+  private:
+   void*            buf_;
+   size_t           buf_len_;
+   uint32_t         flags_;
+   wsrep_trx_meta_t meta_;
+   mysql_mutex_t    mutex_;
+   mysql_cond_t     cond_;
+   bool             ready_;
+   bool             executing_;
+   bool             toi_released_;
+ };
+ 
+ /*
+   Convenience macros for determining NBO start and END
+ */
+ #define WSREP_NBO_START(flags_) \
+ ((flags_ & WSREP_FLAG_ISOLATION) && (flags_ & WSREP_FLAG_TRX_START) && \
+  !(flags_ & WSREP_FLAG_TRX_END))
+ 
+ #define WSREP_NBO_END(flags_) \
+  ((flags_ & WSREP_FLAG_ISOLATION) && !(flags_ & WSREP_FLAG_TRX_START) && \
+   (flags_ & WSREP_FLAG_TRX_END))
+ 
+ #define WSREP_TOI(flags_) \
+   ((flags_ & WSREP_FLAG_ISOLATION) && (flags_ & WSREP_FLAG_TRX_START) && \
+    (flags_ & WSREP_FLAG_TRX_END))
+ 
+ typedef struct wsrep_key_arr
+ {
+     wsrep_key_t* keys;
+     size_t       keys_len;
+ } wsrep_key_arr_t;
++
+ bool wsrep_prepare_keys_for_isolation(THD*              thd,
+                                       const char*       db,
+                                       const char*       table,
+                                       const TABLE_LIST* table_list,
+                                       wsrep_key_arr_t*  ka);
+ void wsrep_keys_free(wsrep_key_arr_t* key_arr);
+ 
  extern bool
  wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
                            MDL_ticket *ticket,
@@@ -342,9 -628,9 +534,11 @@@ bool wsrep_node_is_synced()
  #define WSREP(T)  (0)
  #define WSREP_ON  (0)
  #define WSREP_EMULATE_BINLOG(thd) (0)
++#define WSREP_EMULATE_BINLOG_NNULL(thd) (0)
  #define WSREP_CLIENT(thd) (0)
  #define WSREP_FORMAT(my_format) ((ulong)my_format)
  #define WSREP_PROVIDER_EXISTS (0)
++#define WSREP_NOT_SPECIFIED (1)
  #define wsrep_emulate_bin_log (0)
  #define wsrep_to_isolation (0)
  #define wsrep_init() (1)
@@@ -366,5 -652,36 +560,38 @@@
  #define wsrep_thr_deinit() do {} while(0)
  #define wsrep_running_threads (0)
  #define WSREP_BINLOG_FORMAT(my_format) my_format
 -
+ #endif /* WITH_WSREP */
++
++#ifdef WITH_WSREP
+ /**
+  * Check if the wsrep provider (ie the Galera library) is capable of
+  * doing streaming replication.
+  * @return true if SR capable
+  */
+ bool wsrep_provider_is_SR_capable();
+ 
+ /**
+  * Mark current commit ordered if binlogging is not enabled.
+  *
+  * The purpose of this function is to leave commit order critical
+  * section if binlog is not enabled.
+  *
+  * The function can be called from inside storage engine during commit.
+  * Binlog options are checked inside the function.
+  *
+  * @return Zero in case of success, non-zero in case of failure.
+  */
+ 
+ int wsrep_ordered_commit_if_no_binlog(THD*);
+ 
+ /**
+  * Commit the current transaction with the
+  * MySQL "Transaction Coordinator Log" (see `class TC_LOG` in sql/log.h).
+  * Calling this function will generate and assign a new wsrep transaction id
+  * for `thd`.
+  * @return WSREP_OK on success or other WSREP_* error code on failure
+  */
+ wsrep_status_t wsrep_tc_log_commit(THD* thd);
+ 
 +#endif /* WITH_WSREP */
  #endif /* WSREP_MYSQLD_H */
diff --cc sql/wsrep_sr_file.cc
index 00000000000,7ea6d480dad..7745b759349
mode 000000,100644..100644
--- a/sql/wsrep_sr_file.cc
+++ b/sql/wsrep_sr_file.cc
@@@ -1,0 -1,562 +1,563 @@@
+ /* Copyright (C) 2013 Codership Oy <info@codership.com>
+ 
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+ 
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+ 
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+ 
+ #include <my_global.h>
++#include "my_pthread.h"
+ #include "wsrep_priv.h"
+ #include "wsrep_api.h"
+ #include "wsrep_sr_file.h"
+ #include "wsrep_applier.h" // wsrep_apply()
+ #include "wsrep_thd.h"
+ #include "wsrep_utils.h"
+ #include <map>
+ #include <string>
+ #include <ostream>
+ #include <iostream>
+ #include <fstream>
+ #include <list>
+ #include <dirent.h>
+ #include <sstream>
+ #include <iomanip>
+ 
+ /* to prepare wsrep_uuid_t usable in std::map */
+ inline bool operator==(const node_trx_t& lhs, const node_trx_t& rhs) {
+   return(memcmp(lhs.data, rhs.data, 24) == 0);
+ }
+ inline bool operator!=(const node_trx_t& lhs, const node_trx_t& rhs) {
+   return !operator==(lhs,rhs);
+ }
+ inline bool operator< (const node_trx_t& lhs, const node_trx_t& rhs) {
+   return(memcmp(lhs.data, rhs.data, 24) < 0);
+ }
+ inline bool operator> (const node_trx_t& lhs, const node_trx_t& rhs) {
+   return  operator< (rhs,lhs);
+ }
+ inline bool operator<=(const node_trx_t& lhs, const node_trx_t& rhs) {
+   return !operator> (lhs,rhs);
+ }
+ inline bool operator>=(const node_trx_t& lhs, const node_trx_t& rhs) {
+   return !operator< (lhs,rhs);
+ }
+ 
+ typedef std::map<uint64_t, std::vector<unsigned char> > frag_list_t;
+ typedef std::map<node_trx_t, frag_list_t>  db_t;
+ 
+ class SR_file {
+   std::string   const name_;
+   std::ofstream outfile_;
+ 
+   size_t    size_;
+   long      frags_;
+   int const order_;
+ 
+   typedef std::map<node_trx_t, bool> trxs_t;
+   trxs_t trxs_;
+ 
+   /* will go in file header */
+   signed long long min_seqno_;
+   signed long long max_seqno_;
+ 
+ public:
+   SR_file(std::string name, int order) : 
+     name_(name),
+     order_(order)
+   {
+     outfile_.open(name.c_str(), std::ios::out | std::ios::app | std::ios::binary );
+     size_      = 0;
+     min_seqno_ = 0;
+     max_seqno_ = 0;
+   }
+   SR_file() : order_(0)
+   {
+   }
+   int get_order() { return order_; }
+   std::string get_name() { return name_; }
+ 
+   void write_file_header();
+   void read_file_header();
+   void write_frag_header(
+ 			 wsrep_uuid_t *node_uuid,
+ 			 wsrep_trx_id_t trx, 
+ 			 wsrep_seqno_t seqno,
+ 			 uint32_t flags)
+   {
+     char uuid_str[37] = {'\0',};
+     wsrep_uuid_print(node_uuid, uuid_str, 37);
+ 
+     outfile_ << uuid_str;
+     outfile_ << ' ' << trx << ' ' << seqno;
+ 
+     if (flags & WSREP_FLAG_TRX_START) {
+       outfile_ << 'B';
+     } else {
+       outfile_ << ' ';
+     }
+     if (flags & WSREP_FLAG_TRX_END) {
+       outfile_ << 'C';
+     } else {
+       outfile_ << ' ';
+     }
+     if (flags & WSREP_FLAG_ROLLBACK) {
+       outfile_ << 'R';
+     } else {
+       outfile_ << ' ';
+     }
+   }
+   void append(wsrep_uuid_t *node_uuid,
+ 	      wsrep_trx_id_t trx,
+ 	      wsrep_seqno_t seqno,
+ 	      uint32_t flags,
+ 	      const uchar *buf,
+ 	      size_t buf_len)
+   {
+     if (seqno < min_seqno_ || min_seqno_ == 0) min_seqno_ = seqno;
+     if (seqno > max_seqno_ || max_seqno_ == 0) max_seqno_ = seqno;
+     
+     write_frag_header(node_uuid, trx, seqno, flags);
+     outfile_ << buf_len << '#';
+     outfile_.write((char*)(buf), buf_len);
+     size_ += buf_len;
+     size_ += 35; /* for header */
+ 
+     node_trx_t trxid = { {'\0',} };
+     memcpy(trxid.data, node_uuid->data, 16);
+     sprintf((char*)&(trxid.data[16]), "%ld", trx);
+     trxs_[trxid] = true;
+   };
+ 
+   bool remove(const wsrep_uuid_t *node_uuid,
+ 	      const wsrep_seqno_t seqno)
+   {
+     node_trx_t trxid = { {'\0',} };
+     memcpy(trxid.data, node_uuid->data, 16);
+     sprintf((char*)&(trxid.data[16]), "%ld", seqno);
+     
+     //std::string trxid = node_uuid + std::to_string(seqno);
+     trxs_[trxid] = false;
+     
+     /* check if all transactions are over in this file */
+     bool all_gone(true);
+     
+     trxs_t::iterator iterator;
+     for(iterator = trxs_.begin(); iterator != trxs_.end(); iterator++) {
+       if (iterator->second == true)
+ 	{
+ 	  all_gone = false;
+ 	  continue;
+ 	}
+     }
+     
+     return all_gone;
+   }
+   
+   size_t size()
+   {
+     return size_;
+   }
+   
+   class File_hdr {
+   };
+   
+   class Frag_hdr {
+   };
+   
+   void close()
+   {
+     outfile_.close();
+   }
+ };
+ 
+ int SR_storage_file::max_file_order()
+ {
+   int max = 0;
+   std::list<SR_file*>::iterator iterator;
+   for (iterator = files_.begin(); iterator != files_.end(); ++iterator) {
+     if ((*iterator)->get_order() > max) max = (*iterator)->get_order();
+   }
+   return max;
+ }
+ 
+ std::string *SR_storage_file::new_name(int order)
+ {
+   std::string *name = new std::string(dir_); 
+   *name += "/wsrep_SR_";
+   *name += order;
+   *name += ".dat";
+   return name;
+ }
+ 
+ SR_file *SR_storage_file::append_file() {
+ 
+   int order = max_file_order() + 1;
+   std::stringstream ss;
+   ss << dir_ <<  "/wsrep_SR_store." << order;
+   SR_file *file = new SR_file(ss.str(), order);
+   files_.push_back(file);
+   return file;
+ }
+ 
+ void SR_storage_file::remove_file(SR_file *file) {
+   remove(file->get_name().c_str());
+   delete file;
+ }
+ 
+ SR_storage_file::SR_storage_file(
+ 			   const char *dir, 
+ 			   size_t limit, 
+ 			   const char * cluster_uuid_str) 
+ {
+   dir_ = std::string(dir);
+   size_limit_ = limit;
+   wsrep_uuid_scan(cluster_uuid_str, 37, &cluster_uuid_);
+   restored_ = false;
+ 
+   curr_file_ = NULL;
+ 
+   WSREP_DEBUG("SR pool initialized, group: %s", cluster_uuid_str);
+ }
+ 
+ int SR_storage_file::init(const char* cluster_uuid_str, Wsrep_schema* unused) 
+ {
+   int rcode = 0;
+ 
+   WSREP_DEBUG("SR pool initialized,");
+   WSREP_DEBUG("cluster_uuid:str: %s", cluster_uuid_str);
+   wsrep_uuid_scan(cluster_uuid_str, 37, &cluster_uuid_);
+ 
+   return rcode;
+ }
+ 
+ THD* SR_storage_file::append_frag(THD*         thd,
+ 				  uint32_t     flags,
+ 				  const uchar* buf,
+ 				  size_t       buf_len)
+ {
+   wsrep_uuid_t *node_uuid = &(thd->wsrep_trx_meta.stid.node);
+   wsrep_trx_id_t trx      = thd->wsrep_trx_meta.stid.trx;
+   wsrep_seqno_t seqno     = thd->wsrep_trx_meta.gtid.seqno;
+ 
+   if (!restored_) return NULL;
+ 
+   wsp::auto_lock lock(&LOCK_wsrep_SR_store);
+ 
+   if (!curr_file_) curr_file_ = append_file();
+   
+   assert(curr_file_);
+     
+   curr_file_->append(node_uuid, trx, seqno, flags, buf, buf_len);
+ 
+   if (curr_file_->size() > size_limit_) 
+   {
+     curr_file_->close();
+     curr_file_ = NULL;
+   }
+ 
+   return NULL;
+ }
+ 
+ void SR_storage_file::remove_trx( THD *thd )
+ {
+   assert (thd);
+   const wsrep_uuid_t *node_uuid = &(thd->wsrep_trx_meta.stid.node);
+   const wsrep_trx_id_t trxid    = thd->wsrep_trx_meta.stid.trx;
+ 
+   /* remove from all SR files */
+   wsp::auto_lock lock(&LOCK_wsrep_SR_store);
+ 
+   std::list<SR_file*>::iterator iterator;
+   for (iterator = files_.begin(); iterator != files_.end();) {
+     if ((*iterator)->remove(node_uuid, trxid))
+     {
+       if (curr_file_ == *iterator) curr_file_ = NULL;
+ 
+       remove_file(*iterator);
+ 
+       std::list<SR_file*>::iterator prev = iterator++;
+       files_.erase(prev);
+     }
+     else
+     {
+       ++iterator;
+     }
+   }
+ }
+ 
+ void SR_storage_file::rollback_trx( THD* thd )
+ 
+ {
+   WSREP_DEBUG("SR_storage_file::commit_trx");
+   remove_trx(thd);
+ }
+ 
+ int SR_storage_file::replay_trx(THD* thd, const wsrep_trx_meta_t& meta)
+ {
+   WSREP_ERROR("SR_storage_file::replay_trx not implemented");
+   return 1;
+ }
+ 
+ 
+ void SR_storage_file::read_trxs_from_file(
+         std::string file, trxs_t *trxs, THD *thd, enum read_mode mode)
+ {
+   std::ifstream infile;
+ 
+   WSREP_DEBUG("read_trxs_from_file");
+ 
+   if (!strcmp(file.c_str(), "---"))
+   {
+     WSREP_DEBUG("SR file comment line skipped");
+     return;
+   }
+   infile.open(file.c_str(), std::ios::in | std::ios::binary);
+   infile.exceptions ( 
+     std::ifstream::failbit | std::ifstream::badbit | std::ifstream::eofbit);
+ 
+   try {
+ 
+     while (infile.good())
+     {
+       char uuid[37] = {'\0',};
+       wsrep_uuid_t node_uuid;
+       wsrep_trx_id_t trxid;
+       wsrep_seqno_t seqno;
+       char begin    = '#';
+       char commit   = '#';
+       char rollback = '#';
+       int  len;
+       char *buf;
+ 
+      /* 1 source node UUID */
+       if (infile.get(uuid, 37))
+       {
+         wsrep_uuid_scan(uuid, 37, &node_uuid);
+       }
+       else
+         break;
+ 
+       /* 2 source node trx ID */
+       infile >> trxid;
+ 
+       /* 3 trx seqno */
+       infile >> seqno;
+ 
+       /* 4 flags: Begin-Commit-Rollback */
+       begin    = (char)infile.get();
+       commit   = (char)infile.get();
+       rollback = (char)infile.get();
+ 
+       /* 5 rbr buffer length */
+       infile >> len;
+ 
+       /* # after header part */
+       if ( (char)infile.get() != '#')
+       {
+         WSREP_WARN("SR frament file bad line: %s %ld %ld %c %c %c",
+                    uuid, trxid, seqno, begin, commit, rollback);
+         return;
+       }
+ 
+       /* 6 the buffer */
+       buf = new char [len];
+       infile.read(buf, len);
+ 
+       /* */
+       node_trx_t nodetrx;
+       memcpy(nodetrx.data, node_uuid.data, 16);
+       sprintf((char*)&(nodetrx.data[16]), "%ld", trxid);
+ 
+       int flags = 0;
+       wsrep_trx_meta_t meta;
+       meta.gtid.uuid  = cluster_uuid_;
+       meta.gtid.seqno = seqno;
+       meta.stid.node  = node_uuid;
+       meta.stid.trx   = trxid;
+ 
+       if (begin == 'B')
+       {
+         if (mode == FILTER)
+         {
+           WSREP_DEBUG("new trx in SR file: trx %ld seqno %ld", trxid, seqno);
+           (*trxs)[nodetrx] = true;
+         }
+         else
+         {
+           flags |= WSREP_FLAG_TRX_START;
+         }
+       }
+       else if (!(*trxs)[nodetrx])
+       {
+         WSREP_WARN("unfinished trx in SR file: trx %ld seqno %ld",
+                    trxid, seqno);
+       }
+ 
+       if (mode == FILTER && (commit == 'C' || rollback == 'R'))
+       {
+         WSREP_DEBUG("trx commit in SR file: trx %ld seqno %ld", trxid, seqno);
+         (*trxs)[nodetrx] = false;
+       }
+ 
+       if (mode == POPULATE && (*trxs)[nodetrx])
+       {
+         /* pending transaction to launch in sr_pool */
+         WSREP_DEBUG("launching SR trx: %ld", trxid);
+ 
+         wsrep_buf_t const ws= { buf, size_t(len) };
+         wsrep_apply_error err;
+         if (wsrep_apply(thd, flags, &ws, &meta, err) !=
+             WSREP_CB_SUCCESS)
+         {
+           WSREP_WARN("Streaming Replication fragment restore failed: %s",
+                      err.c_str() ? err.c_str() : "(null)");
+           return;
+         }
+         DBUG_ASSERT(err.is_null());
+       }
+       else if (mode == POPULATE)
+         WSREP_DEBUG("not populating trx %ld seqno %ld", trxid, seqno);
+ 
+       delete[] buf;
+     }
+   }
+   catch(std::exception e)
+     {
+       if(infile.eof())
+       {
+         WSREP_DEBUG("infile EOF");
+       }
+       else
+         WSREP_DEBUG("infile exception");
+     }
+ }
+ 
+ int SR_storage_file::restore( THD *thd )
+ {
+   std::ifstream infile;
+   std::string file(dir_ + '/' + "wsrep_SR_info");
+   THD * SRthd = NULL;
+ 
+   char cluster_uuid_str[37] = {'\0',};
+ 
+   wsp::auto_lock lock(&LOCK_wsrep_SR_store);
+ 
+   if (restored_)
+   {
+     return 0;
+   }
+   wsrep_uuid_print(&cluster_uuid_, cluster_uuid_str, 37);
+ 
+   WSREP_DEBUG("SR pool restore, group %s", cluster_uuid_str);
+ 
+   /* read SR store info */
+   infile.open(file.c_str(), std::ios::in);
+   if (infile.is_open())
+   {
+     std::string line;
+     getline (infile, line);
+       
+     /* this should be cluster uuid */
+     if (line.length() != 36) 
+     {
+         WSREP_WARN("Streaming Replication info file is corrupted");
+         restored_ = true;
+         return -1;
+     }
+ 
+     if (strncmp(cluster_uuid_str, line.c_str(), 36)) {
+       WSREP_WARN("Streaming Replication cluster uuid has changed, \n"
+                  "cluster in SR file: %s\n"
+                  "current cluster:    %s",
+                  line.c_str(), cluster_uuid_str);
+       restored_ = true;
+       return -2;
+     }
+ 
+     trxs_t trxs;
+ 
+     bool thd_started = false;
+     if (!thd)
+     {
+       SRthd = wsrep_start_SR_THD((char*)&infile);
+       SRthd->wsrep_SR_thd = false;
+       thd_started = true;
+       SRthd->store_globals();
+     }
+     else
+     {
+       SRthd = thd;
+     }
+ 
+     /* read all fragment files, and filter out committed trxs */
+     while ( getline (infile, line) && strcmp(line.c_str(), "---"))
+     {
+       WSREP_DEBUG("SR file filtering line: %s", line.c_str());
+       read_trxs_from_file(line, &trxs, SRthd, FILTER);
+     }
+     /* read again, and populate pending trxs */
+     infile.seekg(infile.beg);
+     getline (infile, line);
+ 
+     while ( getline (infile, line) )
+     {
+       WSREP_DEBUG("SR file populating line: %s", line.c_str());
+       read_trxs_from_file(line, &trxs, SRthd, POPULATE);
+     }
+     infile.close();
+     if (thd_started)
+     {
+       wsrep_end_SR_THD(SRthd);
+     }
+     else
+     {
+       thd->store_globals();
+     }
+   }
+   
+   restored_ = true;
+   remove(file.c_str());
+   return 0;
+ }
+ 
+ void SR_storage_file::close() 
+ {
+   WSREP_DEBUG("SR_storage_file::close()");
+   wsp::auto_lock lock(&LOCK_wsrep_SR_store);
+ 
+   /* write store info */
+   std::string file(dir_ + '/' + "wsrep_SR_info");
+   std::ofstream srinfo;
+   char cluster_uuid_str[37] = {'\0',};
+ 
+   wsrep_uuid_print(&cluster_uuid_, cluster_uuid_str, 37);
+ 
+   srinfo.open(file.c_str());
+   srinfo << cluster_uuid_str << '\n';
+ 
+   /* close transactions */
+   std::list<SR_file*>::iterator iterator;
+   for (iterator = files_.begin(); iterator != files_.end();)
+   {
+     std::list<SR_file*>::iterator prev = iterator++;
+     WSREP_DEBUG("Closing streaming replication file: %s", 
+ 		(*prev)->get_name().c_str());
+     
+     srinfo << (*prev)->get_name() << '\n';
+     (*prev)->close();
+   }
+   srinfo << "---" << '\n';
+ 
+   srinfo.close();
+ 
+   restored_ = false;
+ }
diff --cc sql/wsrep_sr_table.cc
index 00000000000,ebbc7784510..bbde72fc30c
mode 000000,100644..100644
--- a/sql/wsrep_sr_table.cc
+++ b/sql/wsrep_sr_table.cc
@@@ -1,0 -1,222 +1,223 @@@
+ /* Copyright (C) 2013 Codership Oy <info@codership.com>
+ 
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+ 
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+ 
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+ 
 -
++#include <my_global.h>
++#include "my_pthread.h"
+ #include "wsrep_priv.h"
+ #include "wsrep_sr_table.h"
+ #include "wsrep_schema.h"
+ 
+ #include "sql_class.h"
+ 
+ 
+ SR_storage_table::SR_storage_table()
+   :
+   wsrep_schema_(0)
+ {
+ }
+ 
+ SR_storage_table::~SR_storage_table()
+ {
+ }
+ 
+ int SR_storage_table::init(const char* cluster_uuid_str,
+                            Wsrep_schema* wsrep_schema)
+ {
+   /* Storage tables have been created by Wsrep_schema */
+   wsrep_schema_= wsrep_schema;
+   return 0;
+ }
+ 
+ 
+ THD* SR_storage_table::append_frag(THD*         thd,
+                                    uint32_t     flags,
+                                    const uchar* buf,
+                                    size_t       buf_len)
+ {
+   /*
+     During restore phase fragments are read from storage, don't
+     append back.
+    */
+   if (restored_ == false) {
+     return NULL;
+   }
+ 
+   thd->wsrep_trx_meta.stid.trx = thd->wsrep_ws_handle.trx_id;
+ 
+   THD* ret = wsrep_schema_->append_frag(thd->wsrep_trx_meta, flags,
+                                         buf, buf_len);
+   if (!ret) {
+     WSREP_ERROR("Failed to append frag to persistent storage");
+   }
+   else {
+     WSREP_DEBUG("SR_storage_table::append_frag(): thd %lld, seqno %lld, "
+                 "trx_id: %ld, thd_ret: %p",
+                 thd->thread_id, (long long)thd->wsrep_trx_meta.gtid.seqno,
+                 thd->wsrep_trx_meta.stid.trx, ret);
+   }
+ 
+   thd->store_globals(); /* Restore original thread context */
+   return ret;
+ }
+ 
+ void SR_storage_table::update_frag_seqno(THD* thd, THD* orig_THD)
+ {
+   thd->store_globals();
+ 
+   if (wsrep_schema_->update_frag_seqno(thd, orig_THD->wsrep_trx_meta)) {
+     WSREP_ERROR("Failed to update seqno, must abort");
+     unireg_abort(1);
+   }
+ 
+   orig_THD->wsrep_SR_fragments.push_back(orig_THD->wsrep_trx_meta);
+   orig_THD->store_globals(); /* Restore original thread context */
+ }
+ 
+ void SR_storage_table::release_SR_thd(THD* thd)
+ {
+   thd->store_globals();
+   wsrep_schema_->release_SR_thd(thd);
+ }
+ 
+ void SR_storage_table::append_frag_apply(THD*         thd,
+                                          uint32_t     flags,
+                                          const uchar* buf,
+                                          size_t       buf_len)
+ {
+   /*
+     During restore phase fragments are read from storage, don't
+     append back.
+    */
+   if (restored_ == false) {
+     return;
+   }
+ 
+   if (wsrep_schema_->append_frag_apply(thd, thd->wsrep_trx_meta, flags,
+                                        buf, buf_len)) {
+     WSREP_ERROR("Failed to append frag to persistent storage, must abort");
+     unireg_abort(1);
+   }
+ 
+   thd->store_globals(); /* Restore original thread context */
+ }
+ 
+ void SR_storage_table::append_frag_commit(THD*         thd,
+                                           uint32_t     flags,
+                                           const uchar* buf,
+                                           size_t       buf_len)
+ {
+   /*
+     During restore phase fragments are read from storage, don't
+     append back.
+    */
+   if (restored_ == false) {
+     return;
+   }
+ 
+   if (wsrep_schema_->append_frag_commit(thd->wsrep_trx_meta, flags,
+                                         buf, buf_len)) {
+     WSREP_ERROR("Failed to append frag to persistent storage, must abort");
+     unireg_abort(1);
+   }
+ 
+   thd->wsrep_SR_fragments.push_back(thd->wsrep_trx_meta);
+   thd->store_globals(); /* Restore original thread context */
+ }
+ 
+ void SR_storage_table::remove_trx(THD* thd)
+ {
+   WSREP_DEBUG("SR_storage_table::remove_trx(%lld) seqno %lld, trx %ld",
+               thd->thread_id, (long long)thd->wsrep_trx_meta.gtid.seqno,
+               thd->wsrep_trx_meta.stid.trx);
+   int err= wsrep_schema_->remove_trx(thd, &thd->wsrep_SR_fragments);
+   if (err == -1) {
+     WSREP_DEBUG("SR_storage_table::remove_trx() interrupted");
+   } else if (err) {
+     WSREP_WARN("Failed to delete fragments from persistent storage");
+   }
+ }
+ 
+ void SR_storage_table::remove_trx(wsrep_SR_trx_info* trx)
+ {
+   remove_trx(trx->get_THD());
+ }
+ 
+ void SR_storage_table::rollback_trx(THD* thd)
+ {
+   WSREP_DEBUG("SR_storage_table::rollback_trx(%lld) seqno %lld, trx %ld",
+               thd->thread_id, (long long)thd->wsrep_trx_meta.gtid.seqno,
+               thd->wsrep_trx_meta.stid.trx);
+   int err= wsrep_schema_->rollback_trx(thd);
+   if (err == -1) {
+     WSREP_DEBUG("SR_storage_table::rollback_trx() interrupted");
+   } else if (err) {
+     WSREP_WARN("Failed to delete fragments from persistent storage");
+   }
+   thd->store_globals(); /* Restore original thread context */
+ }
+ 
+ void SR_storage_table::rollback_trx(wsrep_SR_trx_info* trx)
+ {
+   rollback_trx(trx->get_THD());
+ }
+ 
+ void SR_storage_table::trx_done(THD* thd)
+ {
+   if (thd->wsrep_conflict_state() != MUST_REPLAY)
+     thd->wsrep_SR_fragments.clear();
+ }
+ 
+ int SR_storage_table::replay_trx(THD* thd, const wsrep_trx_meta_t& meta)
+ {
+   return wsrep_schema_->replay_trx(thd, meta);
+ }
+ 
+ int SR_storage_table::restore(THD* thd)
+ {
+   if (restored_ == true) {
+     WSREP_DEBUG("SR_storage_table::restore: Already restored");
+     return 0;
+   }
+ 
+   WSREP_INFO("SR_storage_table::restore");
+   int ret= wsrep_schema_->restore_frags();
+   if (thd) {
+     thd->store_globals();
+   }
+   else {
+     my_pthread_setspecific_ptr(THR_THD, NULL);
+   }
+   restored_= true;
+   return ret;
+ }
+ 
+ void SR_storage_table::prepare_for_open_tables(THD *thd, TABLE_LIST **table_list)
+ {
+   wsrep_schema_->init_SR_table(&thd->wsrep_SR_table);
+   TABLE_LIST *ptr= *table_list;
+   if (!ptr)
+   {
+     *table_list= &thd->wsrep_SR_table;
+   }
+   else
+   {
+     while (ptr->next_global) ptr = ptr->next_global;
+     ptr->next_global = &thd->wsrep_SR_table;
+   }
+ }
+ 
+ void SR_storage_table::close()
+ {
+ }
diff --cc sql/wsrep_sst.cc
index d5b0344c456,7c5963c031f..fff7e747f9b
--- a/sql/wsrep_sst.cc
+++ b/sql/wsrep_sst.cc
@@@ -28,9 -28,8 +28,10 @@@
  #include "wsrep_xid.h"
  #include <cstdio>
  #include <cstdlib>
- 
 +#include <my_service_manager.h>
  
+ const char wsrep_defaults_group_suffix[256] = {0};
++
  static char wsrep_defaults_file[FN_REFLEN * 2 + 10 + 30 +
                                  sizeof(WSREP_SST_OPT_CONF) +
                                  sizeof(WSREP_SST_OPT_CONF_SUFFIX) +
@@@ -1363,25 -1350,14 +1369,26 @@@ void wsrep_SE_init_grab(
    if (mysql_mutex_lock (&LOCK_wsrep_sst_init)) abort();
  }
  
- void wsrep_SE_init_wait()
+ int wsrep_SE_init_wait()
  {
 +  struct timespec wtime = {WSREP_TIMEDWAIT_SECONDS, 0};
 +  uint32 total_wtime=0;
 +
-   while (SE_initialized == false)
+   while (SE_init_status == WSREP_SE_UNINITIALIZED)
    {
 -    mysql_cond_wait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init);
 +    mysql_cond_timedwait (&COND_wsrep_sst_init, &LOCK_wsrep_sst_init, &wtime);
 +
-     if (!SE_initialized)
++    if (SE_init_status == WSREP_SE_UNINITIALIZED)
 +    {
 +      total_wtime += wtime.tv_sec;
 +      WSREP_DEBUG("Waiting for SST to complete. waited %u secs.", total_wtime);
 +      service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL,
 +        "WSREP SE initialization ongoing.");
 +    }
    }
 +
    mysql_mutex_unlock (&LOCK_wsrep_sst_init);
+   return !(SE_init_status == WSREP_SE_INITIALIZED);
  }
  
  void wsrep_SE_init_done()
diff --cc sql/wsrep_thd.cc
index ce6d9688cb3,ddfc3ce70d1..489613d0c6c
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@@ -46,18 -53,153 +53,153 @@@ int wsrep_show_bf_aborts (THD *thd, SHO
    var->value = (char*)&wsrep_local_bf_aborts;
    return 0;
  }
+ void wsrep_cleanup_transaction(THD *thd)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   DBUG_ENTER("wsrep_cleanup_transaction");
+   if (thd->wsrep_exec_mode == REPL_RECV)  return;
+ 
+   DBUG_ASSERT(thd->wsrep_conflict_state() != MUST_REPLAY);
+   DBUG_ASSERT(thd->wsrep_SR_fragments.empty());
+ 
+   if (wsrep_SR_store) wsrep_SR_store->trx_done(thd);
+   if (wsrep_emulate_bin_log) wsrep_thd_binlog_trx_reset(thd);
+ 
+   wsrep_reset_SR_trans(thd);
+   thd->wsrep_exec_mode= LOCAL_STATE;
+   if (thd->wsrep_conflict_state() != NO_CONFLICT)
+   {
+     /*
+       Catch half finished rollbacks.
+      */
+     DBUG_ASSERT(thd->wsrep_conflict_state() == ABORTED ||
+                 thd->wsrep_conflict_state() == CERT_FAILURE);
+ 
+     thd->killed= NOT_KILLED;
+     thd->set_wsrep_conflict_state(NO_CONFLICT);
+   }
+ 
+   if (MUST_REPLAY != thd->wsrep_conflict_state())
+   {
+     thd->wsrep_PA_safe= true;
+     thd->wsrep_ws_handle.trx_id= WSREP_UNDEFINED_TRX_ID;
+     thd->set_wsrep_next_trx_id(WSREP_UNDEFINED_TRX_ID);
+ 
+     if (thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED)
+     {
+       thd->wsrep_last_written_gtid= thd->wsrep_trx_meta.gtid;
+     }
+     thd->wsrep_trx_meta.gtid= WSREP_GTID_UNDEFINED;
+     thd->wsrep_trx_meta.depends_on= WSREP_SEQNO_UNDEFINED;
+     thd->wsrep_affected_rows= 0;
+     thd->wsrep_skip_wsrep_GTID= false;
+     thd->wsrep_xid.null();
+   }
+ 
+   DBUG_VOID_RETURN;
+ }
+ 
+ /*
+   Run post rollback actions.
  
- /* must have (&thd->LOCK_thd_data) */
- void wsrep_client_rollback(THD *thd)
+   Assert thd->LOCK_wsrep_thd ownership
+  */
+ void wsrep_post_rollback(THD *thd)
  {
-   WSREP_DEBUG("client rollback due to BF abort for (%lld), query: %s",
-               (longlong) thd->thread_id, thd->query());
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
  
-   WSREP_ATOMIC_ADD_LONG(&wsrep_bf_aborts_counter, 1);
+   WSREP_LOG_THD(thd, NULL);
+ 
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT || /* voluntary */
+               thd->wsrep_conflict_state() == ABORTING); /* BF abort or cert failure  */
  
-   thd->wsrep_conflict_state= ABORTING;
 -  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +  mysql_mutex_unlock(&thd->LOCK_thd_data);
-   trans_rollback(thd);
+ 
+   if (thd->wsrep_trx_has_seqno())
+   {
+     if (!wsrep_gtid_mode)
+     {
+       void* ptr= NULL;
+       size_t len= 0;
+       wsrep_buf_t err= {ptr, len};
+       if (wsrep->commit_order_enter(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta))
+       {
+         WSREP_WARN("wsrep_post_rollback: failed to enter commit order");
+       }
+       if (wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta, &err))
+       {
+         WSREP_WARN("wsrep_post_rollback: failed to leave commit order");
+       }
+     }
+     /*
+       If binlogging is on commit ordering is done when dummy
+       event is written into binlog
+      */
+     else if (wsrep_write_dummy_event(thd, "rollback"))
+     {
+       WSREP_WARN("wsrep_post_rollback: failed to write dummy event");
+     }
+   }
+ 
+   if (wsrep->release(wsrep, &thd->wsrep_ws_handle))
+   {
+     WSREP_WARN("wsrep::release fail: %llu %d",
+                (long long)thd->thread_id, thd->get_stmt_da()->status());
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT ||
+               thd->wsrep_conflict_state() == ABORTING);
+ 
+   if (thd->wsrep_conflict_state() == NO_CONFLICT)
+   {
+     thd->set_wsrep_conflict_state(MUST_ABORT);
+     thd->set_wsrep_conflict_state(ABORTING);
+     thd->set_wsrep_conflict_state(ABORTED);
+   }
+   else
+   {
+     thd->set_wsrep_conflict_state(ABORTED);
+   }
+ }
+ 
+ /*
+   must have (&thd->LOCK_wsrep_thd)
+   thd->wsrep_conflict_state must be MUST_ABORT
+ */
+ void wsrep_client_rollback(THD *thd, bool rollbacker)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_ABORT ||
+               thd->wsrep_conflict_state() == CERT_FAILURE);
+   WSREP_DEBUG("client rollback due to BF abort for (%lld %lld), query: %s",
+               thd->thread_id, thd->query_id, WSREP_QUERY(thd));
+ 
+   my_atomic_add64(&wsrep_bf_aborts_counter, 1);
+ 
+   /*
+     Rollback proccess should be fired only for threads which are not
+     in the process of committing.
+   */
+   DBUG_ASSERT(thd->wsrep_query_state() != QUERY_COMMITTING);
+   if (rollbacker)
+   {
+     DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno == WSREP_SEQNO_UNDEFINED);
+   }
+ 
+   thd->set_wsrep_conflict_state(ABORTING);
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   if (thd->wsrep_is_streaming())
+   {
+     WSREP_DEBUG("wsrep_client_rollback: thd: %lld fragments %zu",
+                 thd->thread_id, thd->wsrep_SR_fragments.size());
+     wsrep_SR_store->rollback_trx(thd);
+     thd->wsrep_SR_fragments.clear();
+   }
  
    if (thd->locked_tables_mode && thd->lock)
    {
@@@ -82,12 -222,46 +222,45 @@@
  
    if (thd->get_binlog_table_maps())
    {
-     WSREP_DEBUG("clearing binlog table map for BF abort (%lld)",
-                 (longlong) thd->thread_id);
+     WSREP_DEBUG("clearing binlog table map for BF abort (%lld)", thd->thread_id);
      thd->clear_binlog_table_maps();
    }
-   mysql_mutex_lock(&thd->LOCK_thd_data);
-   thd->wsrep_conflict_state= ABORTED;
+ 
+   /*
+     trans_rolback() must be called after all locks are released since it
+     calls ha_rollback_trans() which acquires TO
+   */
+   if (trans_rollback(thd))
+   {
+     WSREP_WARN("client rollback failed for: %lld %lld, conf: %d",
+                thd->thread_id, thd->query_id,
+                thd->wsrep_conflict_state_unsafe());
+   }
+ 
+   if (rollbacker && thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED)
+   {
+     /*
+       Thd has been assigned seqno and it needs to release provider
+       resoureces. Do it in separate thread to avoid deadlocks.
+     */
+     DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_ROLLBACK);
+     if (wsrep_post_rollback_queue->push_back(thd))
+     {
+       WSREP_WARN("duplicate thd %llu for post-rollbacker",
+                  wsrep_thd_thread_id(thd));
+     }
+   }
+ 
 -  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   /*
+     If the seqno is not set there is no need for post rollback
+     actions.
+    */
+   if (rollbacker /* && wsrep_thd_trx_seqno(thd) == WSREP_SEQNO_UNDEFINED */)
+   {
+     wsrep_post_rollback(thd);
+   }
+ 
+   return;
  }
  
  #define NUMBER_OF_FIELDS_TO_IDENTIFY_COORDINATOR 1
@@@ -231,8 -410,8 +409,8 @@@ void wsrep_replay_transaction(THD *thd
  
        thd->get_stmt_da()->reset_diagnostics_area();
  
-       thd->wsrep_conflict_state= REPLAYING;
+       thd->set_wsrep_conflict_state(REPLAYING);
 -      mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +      mysql_mutex_unlock(&thd->LOCK_thd_data);
  
        thd->reset_for_next_command();
        thd->reset_killed();
@@@ -269,11 -448,16 +447,16 @@@
                                      (void *)thd);
  
        wsrep_return_from_bf_mode(thd, &shadow);
-       if (thd->wsrep_conflict_state!= REPLAYING)
-         WSREP_WARN("lost replaying mode: %d", thd->wsrep_conflict_state );
+ 
+       WSREP_DEBUG("replayed %lld, seqno %lld, rcode %d",
+                    thd->thread_id, (long long)wsrep_thd_trx_seqno(thd), rcode);
+       DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
  
 -      mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +      mysql_mutex_lock(&thd->LOCK_thd_data);
  
+       if (thd->wsrep_conflict_state() != REPLAYING)
+         WSREP_WARN("lost replaying mode: %d", thd->wsrep_conflict_state());
+ 
        switch (rcode)
        {
        case WSREP_OK:
@@@ -328,8 -527,7 +526,7 @@@
          /* we're now in inconsistent state, must abort */
  
          /* http://bazaar.launchpad.net/~codership/codership-mysql/5.6/revision/3962#sql... */
 -        mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +        mysql_mutex_unlock(&thd->LOCK_thd_data);
- 
          unireg_abort(1);
          break;
        }
@@@ -495,20 -705,19 +704,19 @@@ static void wsrep_rollback_process(THD 
         */
        mysql_mutex_unlock(&LOCK_wsrep_rollback);
  
 -      mysql_mutex_lock(&aborting->LOCK_wsrep_thd);
 +      mysql_mutex_lock(&aborting->LOCK_thd_data);
-       if (aborting->wsrep_conflict_state== ABORTED)
+       if (aborting->wsrep_conflict_state()== ABORTED)
        {
          WSREP_DEBUG("WSREP, thd already aborted: %llu state: %d",
                      (long long)aborting->real_id,
-                     aborting->wsrep_conflict_state);
+                     aborting->wsrep_conflict_state());
  
 -        mysql_mutex_unlock(&aborting->LOCK_wsrep_thd);
 +        mysql_mutex_unlock(&aborting->LOCK_thd_data);
          mysql_mutex_lock(&LOCK_wsrep_rollback);
          continue;
        }
-       aborting->wsrep_conflict_state= ABORTING;
  
 -      mysql_mutex_unlock(&aborting->LOCK_wsrep_thd);
 +      mysql_mutex_unlock(&aborting->LOCK_thd_data);
  
        set_current_thd(aborting); 
        aborting->store_globals();
@@@ -558,10 -856,10 +855,10 @@@ enum wsrep_conflict_state wsrep_thd_con
    enum wsrep_conflict_state state = NO_CONFLICT;
    if (thd)
    {
 -    if (sync) mysql_mutex_lock(&thd->LOCK_wsrep_thd);
 +    if (sync) mysql_mutex_lock(&thd->LOCK_thd_data);
      
-     state = thd->wsrep_conflict_state;
+     state = thd->wsrep_conflict_state();
 -    if (sync) mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
 +    if (sync) mysql_mutex_unlock(&thd->LOCK_thd_data);
    }
    return state;
  }
diff --cc sql/wsrep_thd_pool.cc
index 00000000000,842542cab64..d65b5bde185
mode 000000,100644..100644
--- a/sql/wsrep_thd_pool.cc
+++ b/sql/wsrep_thd_pool.cc
@@@ -1,0 -1,120 +1,122 @@@
+ /* Copyright (C) 2015 Codership Oy <info@codership.com>
+ 
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+ 
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+ 
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */
+ 
+ 
++#include <my_global.h>
++#include "my_pthread.h"
+ #include "wsrep_mysqld.h"
+ #include "wsrep_thd_pool.h"
+ #include "wsrep_utils.h"
+ #include "sql_class.h"
+ //#include "global_threads.h"
+ 
+ #include <list>
+ 
+ static THD* wsrep_thd_pool_new_thd()
+ {
+   THD *thd = new THD(next_thread_id());
+   thd->thread_stack= (char*) &thd;
+   thd->security_ctx->skip_grants();
+   thd->system_thread= SYSTEM_THREAD_GENERIC;
+ 
+   mysql_mutex_lock(&LOCK_thread_count);
+ 
+   thd->real_id=pthread_self(); // Keep purify happy
+ 
+   WSREP_DEBUG("Wsrep_thd_pool: creating system thread: %lld",
+               (long long)thd->thread_id);
+   thd->prior_thr_create_utime= thd->start_utime= thd->thr_create_utime;
+   (void) mysql_mutex_unlock(&LOCK_thread_count);
+ 
+   thd->variables.wsrep_on     = 0;
+   thd->variables.sql_log_bin  = 0;
+   thd->variables.option_bits &= ~OPTION_BIN_LOG;
+   thd->variables.tx_isolation = ISO_READ_COMMITTED;
+ 
+   thd->wsrep_exec_mode = REPL_RECV;
+   //  thd->wsrep_exec_mode = LOCAL_STATE;
+ 
+   return thd;
+ }
+ 
+ Wsrep_thd_pool::Wsrep_thd_pool(size_t threads)
+   :
+   threads_(threads),
+   pool_()
+ {
+   WSREP_DEBUG("Wsrep_thd_pool constructor");
+   wsp::auto_lock lock(&LOCK_wsrep_thd_pool);
+   pool_.reserve(threads);
+   for (size_t i= 0; i < threads; ++i)
+   {
+     pool_.push_back(wsrep_thd_pool_new_thd());
+   }
+ }
+ 
+ Wsrep_thd_pool::~Wsrep_thd_pool()
+ {
+   wsp::auto_lock lock(&LOCK_wsrep_thd_pool);
+   while (!pool_.empty())
+   {
+     THD *thd = pool_.back();
+     WSREP_DEBUG("Wsrep_thd_pool: closing thread %lld",
+                 (long long)thd->thread_id);
+ 
+      delete thd;
+ 
+     pool_.pop_back();
+   }
+ }
+ 
+ THD* Wsrep_thd_pool::get_thd(THD* thd)
+ {
+   wsp::auto_lock lock(&LOCK_wsrep_thd_pool);
+   THD *ret= NULL;
+   if (pool_.empty())
+   {
+     ret= wsrep_thd_pool_new_thd();
+   }
+   else
+   {
+     ret= pool_.back();
+     pool_.pop_back();
+   }
+   if (thd)
+   {
+     ret->thread_stack = thd->thread_stack;
+   }
+   else
+   {
+     ret->thread_stack= (char*) &ret;
+   }
+   ret->store_globals();
+   return ret;
+ }
+ 
+ void Wsrep_thd_pool::release_thd(THD* thd)
+ {
+   DBUG_ASSERT(!thd->mdl_context.has_locks());
+   DBUG_ASSERT(!thd->open_tables);
+   DBUG_ASSERT(thd->transaction.stmt.is_empty());
+   wsp::auto_lock lock(&LOCK_wsrep_thd_pool);
+   if (pool_.size() < threads_)
+   {
+     pool_.push_back(thd);
+   }
+   else
+   {
+     delete thd;
+   }
+ }
diff --cc sql/wsrep_trans_observer.cc
index 00000000000,c4bc0218309..9a914ea37b7
mode 000000,100644..100644
--- a/sql/wsrep_trans_observer.cc
+++ b/sql/wsrep_trans_observer.cc
@@@ -1,0 -1,1663 +1,1663 @@@
+ /* Copyright 2016 Codership Oy <http://www.codership.com>
+ 
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+ 
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+ 
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+ 
+ 
+ /*
+   Implementation of wsrep transaction observer hooks.
+ 
+   * wsrep_after_row(): called after each row write/update/delete, will run
+     SR step
+   * wsrep_before_prepare(): SR table cleanup
+   * wsrep_after_prepare(): will run wsrep_certify() which replicates
+     and certifies transaction for transactions which have registered
+     binlog hton
+   * wsrep_before_commit(): run wsrep_certify() for autocommit
+     DML when binlog_format = STATEMENT and grab commit time
+     critical via section wsrep->commit_order_enter()
+   * wsrep_ordered_commit(): release commit time critical section
+     via wsrep->commit_order_leave()
+   * wsrep_after_commit(): release rest of the trx resources from
+     provider
+   * wsrep_before_rollback(): on SR rollback construct SR_trx_info
+     and send rollback event before actual rollback happens. Set
+     wsrep_exec_mode to LOCAL_ROLLBACK
+   * wsrep_after_rollback(): in case of statement rollback check if
+     it is safe for SR and if not, trigger full transaction rollback
+   * wsrep_after_command():
+     * run wsrep_SR_step()
+     * perform post rollback operations for thds which have
+       wsrep_exec_mode LOCAL_ROLLBACK
+       * perform wsrep_client_rollback() for thds with wsrep_conflict_state
+         MUST_ABORT or CERT_FAILURE
+       * run wsrep_client_rollback() for thd which has seqno assigned
+       * cleanup transaction after rollback
+     * do rollback process for threads which have BF aborted or have
+       failed certification but have not rolled back yet.
+     * replay transactions which need to be replayed
+ 
+ 
+   Note: The rollback processing has been postponed to after_command hook
+   because sometimes the rollback process needs to be done for threads
+   which have valid wsrep seqno and rollback for such a threads should
+   be done after all tables are closed in order to avoid deadlocks.
+   Although wsrep rollback steps would be possible to do earlier in
+   after_rollback hook, this approach was chosen for simplicity.
+ */
+ 
+ 
+ #include "wsrep_trans_observer.h"
+ #include "wsrep_mysqld.h"
+ #include "wsrep_xid.h"
+ #include "wsrep_sr.h"
+ #include "wsrep_thd.h"
+ 
+ #include "replication.h"
+ #include "transaction.h"
+ #include "sql_base.h"   /* close_thread_tables() */
+ #include "sql_class.h"  /* THD */
+ #include "sql_parse.h"  /* stmt_causes_implicit_commit() */
+ #include "rpl_filter.h" /* Rpl_filter */
+ #include "log.h"
+ #include "debug_sync.h" /* DEBUG_SYNC */
+ 
+ #include <map>
+ 
+ extern class SR_storage *wsrep_SR_store;
+ extern Rpl_filter* binlog_filter;
+ 
+ 
+ static void wsrep_wait_for_replayers(THD *thd)
+ {
+   mysql_mutex_lock(&LOCK_wsrep_replaying);
+   int replay_round= 0;
+   while (wsrep_replaying > 0                        &&
+          thd->wsrep_conflict_state() == NO_CONFLICT &&
+          thd->killed == NOT_KILLED             &&
+          !shutdown_in_progress)
+   {
+ 
+     mysql_mutex_unlock(&LOCK_wsrep_replaying);
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+     mysql_mutex_lock(&thd->mysys_var->mutex);
+     thd_proc_info(thd, "wsrep waiting on replaying");
+     thd->mysys_var->current_mutex= &LOCK_wsrep_replaying;
+     thd->mysys_var->current_cond=  &COND_wsrep_replaying;
+     mysql_mutex_unlock(&thd->mysys_var->mutex);
+ 
+     mysql_mutex_lock(&LOCK_wsrep_replaying);
+     // Using timedwait is a hack to avoid deadlock in case if BF victim
+     // misses the signal.
+     struct timespec wtime;
+     clock_gettime(CLOCK_REALTIME, &wtime);
+     long prev_nsec = wtime.tv_nsec;
+     wtime.tv_nsec = (wtime.tv_nsec + 1000000) % 1000000000;
+     // If nsecs rolled over, increment seconds.
+     wtime.tv_sec += (wtime.tv_nsec < prev_nsec ? 1 : 0);
+     mysql_cond_timedwait(&COND_wsrep_replaying, &LOCK_wsrep_replaying,
+                          &wtime);
+ 
+     if (replay_round++ % 100000 == 0)
+       WSREP_DEBUG("commit waiting for replaying: replayers %d, thd: (%lld) "
+                   "conflict: %d (round: %d)",
+                   wsrep_replaying, thd->thread_id,
+                   thd->wsrep_conflict_state_unsafe(), replay_round);
+ 
+     mysql_mutex_unlock(&LOCK_wsrep_replaying);
+ 
+     mysql_mutex_lock(&thd->mysys_var->mutex);
+     thd->mysys_var->current_mutex= 0;
+     thd->mysys_var->current_cond=  0;
+     mysql_mutex_unlock(&thd->mysys_var->mutex);
+ 
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     mysql_mutex_lock(&LOCK_wsrep_replaying);
+   }
+   mysql_mutex_unlock(&LOCK_wsrep_replaying);
+ }
+ 
+ static int wsrep_prepare_data_for_replication(THD *thd)
+ {
+   DBUG_ENTER("wsrep_prepare_data_for_replication");
+   size_t data_len= 0;
+   IO_CACHE* cache= wsrep_get_trans_cache(thd);
+ 
+   if (cache)
+   {
+     thd->binlog_flush_pending_rows_event(true);
+     enum wsrep_trx_status rcode= wsrep_write_cache(wsrep, thd, cache, &data_len);
+     if (rcode != WSREP_TRX_OK)
+     {
+       WSREP_ERROR("rbr write fail, data_len:: %zu",
+                   data_len);
+       wsrep_override_error(thd, ER_ERROR_DURING_COMMIT,
+                            wsrep_trx_status_to_wsrep_status(rcode));
+       DBUG_RETURN(1);
+     }
+   }
+ 
+   if (data_len == 0)
+   {
+     if (thd->get_stmt_da()->is_ok()              &&
+         thd->get_stmt_da()->affected_rows() > 0  &&
+         !binlog_filter->is_on())
+     {
+       WSREP_DEBUG("empty rbr buffer, query: %s, "
+                   "affected rows: %llu, "
+                   "changed tables: %d, "
+                   "sql_log_bin: %d, "
+                   "wsrep status (%d %d %d)",
+                   WSREP_QUERY(thd),
+                   thd->get_stmt_da()->affected_rows(),
+                   stmt_has_updated_trans_table(thd),
+                   thd->variables.sql_log_bin,
+                   thd->wsrep_exec_mode, thd->wsrep_query_state_unsafe(),
+                   thd->wsrep_conflict_state_unsafe());
+     }
+     else
+     {
+       WSREP_DEBUG("empty rbr buffer, query: %s", WSREP_QUERY(thd));
+     }
+ 
+     if (!thd->wsrep_is_streaming())
+     {
+       WSREP_ERROR("I/O error reading from thd's binlog iocache: "
+                   "errno=%d, io cache code=%d", my_errno, cache->error);
+       wsrep_override_error(thd, ER_ERROR_DURING_COMMIT);
+       DBUG_RETURN(1);
+     }
+   }
+   DBUG_RETURN(0);
+ }
+ 
+ /*
+   Run wsrep pre_commit phase
+ 
+   Asserts thd->LOCK_wsrep_thd ownership
+  */
+ static wsrep_trx_status wsrep_certify(THD *thd)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   DBUG_ENTER("wsrep_certify");
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+   DBUG_ASSERT(thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID);
+ 
+   /*
+     We must not proceed for certify() if there are threads
+     replaying transactions. This is because replaying thread
+     may have released some locks which this thread then acquired.
+ 
+     Now if the replaying ends before write set gets replicated,
+     the replayed write set may fall out of this write sets
+     certification range, so the conflict won't be detected.
+     This will lead to applying error later on.
+ 
+     Conflict state must be checked out once more after waiting
+     for replayers to detect if replaying transaction (or other)
+     has BF aborted this.
+   */
+   wsrep_wait_for_replayers(thd);
+   if (thd->wsrep_conflict_state() == MUST_ABORT)
+   {
+     // Transaction was BF aborted
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   thd->set_wsrep_query_state(QUERY_COMMITTING);
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DEBUG_SYNC(thd, "wsrep_before_replication");
+ 
+   if (wsrep_prepare_data_for_replication(thd))
+   {
+     /* Error will be set in function to prepare data */
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   if (thd->killed != NOT_KILLED)
+   {
+     WSREP_INFO("thd %lld killed with signal %d, skipping replication",
+                thd->thread_id, thd->killed);
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID);
+   if (WSREP_UNDEFINED_TRX_ID == thd->wsrep_trx_id())
+   {
+     WSREP_WARN("SQL statement was ineffective, THD: %lld\n"
+                "schema: %s \n"
+                "QUERY: %s\n"
+                " => Skipping replication",
+                thd->thread_id,
+                (thd->db.str ? thd->db.str : "(null)"),
+                WSREP_QUERY(thd));
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     wsrep_override_error(thd, ER_ERROR_DURING_COMMIT);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   uint32_t flags= WSREP_FLAG_TRX_END;
+   if (!thd->wsrep_PA_safe || thd->wsrep_is_streaming())
+   {
+     flags |= WSREP_FLAG_PA_UNSAFE;
+   }
+ 
+   if (thd->wsrep_is_streaming())
+   {
+     if (!wsrep_append_SR_keys(thd))
+     {
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       wsrep_override_error(thd, ER_ERROR_DURING_COMMIT);
+       thd->set_wsrep_query_state(QUERY_EXEC);
+       DBUG_RETURN(WSREP_TRX_ERROR);
+     }
+   }
+ 
+   wsrep_status_t rcode= wsrep->certify(wsrep,
+                                        (wsrep_conn_id_t) thd->thread_id,
+                                        &thd->wsrep_ws_handle,
+                                        flags,
+                                        &thd->wsrep_trx_meta);
+ 
+   WSREP_DEBUG("Trx certify(%lu): rcode %d, seqno %lld, trx %ld, "
+               "flags %u, conf %d, SQL: %s",
+               thd->thread_id, rcode,(long long)thd->wsrep_trx_meta.gtid.seqno,
+               thd->wsrep_trx_meta.stid.trx, flags, thd->wsrep_conflict_state_unsafe(),
+               thd->query());
+ 
+   DBUG_ASSERT((thd->wsrep_trx_meta.depends_on >= 0 &&
+                thd->wsrep_trx_meta.depends_on <
+                thd->wsrep_trx_meta.gtid.seqno) ||
+               WSREP_OK != rcode);
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+ 
+   DEBUG_SYNC(thd, "wsrep_after_replication");
+ 
+   if (WSREP_OK == rcode)
+   {
+     DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
+ 
+     if (thd->wsrep_conflict_state() == MUST_ABORT)
+     {
+       rcode= WSREP_BF_ABORT;
+       WSREP_DEBUG("Not calling commit_order_enter() due to conflict state"
+                   " == MUST_ABORT thd: %lu, seqno: %lld",
+                   thd->thread_id, (long long)wsrep_thd_trx_seqno(thd));
+     }
+   }
+ 
+   wsrep_trx_status ret= WSREP_TRX_ERROR;
+ 
+   switch (rcode)
+   {
+   case WSREP_TRX_MISSING:
+     WSREP_WARN("Transaction missing in provider thd: %lld schema: %s SQL: %s",
+                thd->thread_id, (thd->db.str ? thd->db.str : "(null)"),
+                WSREP_QUERY(thd));
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_TRX_MISSING);
+     break;
+   case WSREP_BF_ABORT:
+     DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
+     thd->set_wsrep_conflict_state(MUST_REPLAY);
+     mysql_mutex_lock(&LOCK_wsrep_replaying);
+     ++wsrep_replaying;
+     mysql_mutex_unlock(&LOCK_wsrep_replaying);
+     break;
+   case WSREP_OK:
+     /*
+       Ignore BF abort from storage engine in commit phase.
+       This however requires that the storage engine BF abort
+       respects QUERY_COMMITTING query state.
+ 
+       TODO: Consider removing this and respecting BF abort.
+       Also adjust allowed state transitions in
+       THD::set_wsrep_conflict_state().
+ 
+       NOTE: If we have rcode WSREP_OK here it means that transaction
+       entered commit critical section in wsrep->commit_order_enter()
+       call. It is a bug in a provider/BF abort code if it allowed
+       BF abort after that.
+     */
+     DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
+     DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+ 
+     if (thd->wsrep_conflict_state() == MUST_ABORT)
+     {
+       thd->killed= NOT_KILLED;
+       WSREP_WARN("Ignoring MUST_ABORT state");
+       thd->set_wsrep_conflict_state(NO_CONFLICT);
+     }
+ 
+     thd->wsrep_exec_mode= LOCAL_COMMIT;
+     DBUG_PRINT("wsrep", ("replicating commit success"));
+     DBUG_EXECUTE_IF("crash_last_fragment_commit_success",
+                     DBUG_SUICIDE(););
+     ret= WSREP_TRX_OK;
+     break;
+   case WSREP_TRX_FAIL:
+     if (thd->wsrep_conflict_state() == NO_CONFLICT)
+     {
+       thd->set_wsrep_conflict_state(CERT_FAILURE);
+       WSREP_LOG_CONFLICT(NULL, thd, FALSE);
+     }
+     else
+     {
+       DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_ABORT);
+     }
+     my_error(ER_LOCK_DEADLOCK, MYF(0), WSREP_TRX_FAIL);
+     ret= WSREP_TRX_CERT_FAIL;
+     break;
+   case WSREP_SIZE_EXCEEDED:
+     WSREP_ERROR("wsrep_certify(%lu): transaction size exceeded",
+                 thd->thread_id);
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_SIZE_EXCEEDED);
+     ret= WSREP_TRX_SIZE_EXCEEDED;
+     break;
+   case WSREP_CONN_FAIL:
+     WSREP_DEBUG("wsrep_certify(%lu): replication aborted",
+                 thd->thread_id);
+     my_error(ER_LOCK_DEADLOCK, MYF(0), WSREP_CONN_FAIL);
+     break;
+   case WSREP_WARNING:
+     WSREP_WARN("provider returned warning");
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_WARNING);
+     break;
+   case WSREP_NODE_FAIL:
+     WSREP_ERROR("replication aborted");
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_NODE_FAIL);
+     break;
+   case WSREP_NOT_IMPLEMENTED:
+     WSREP_ERROR("certify() or commit_order_enter() not implemented");
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), WSREP_NOT_IMPLEMENTED);
+     break;
+   default:
+     WSREP_ERROR("wsrep_certify(%lu): unknown provider failure",
+                 thd->thread_id);
+     my_error(ER_ERROR_DURING_COMMIT, MYF(0), rcode);
+     break;
+   }
+ 
+   /*
+     In case of success we keep the QUERY_COMMITTING
+    */
+   if (rcode != WSREP_OK)
+   {
+     thd->set_wsrep_query_state(QUERY_EXEC);
+   }
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ static wsrep_trx_status wsrep_replicate_fragment(THD *thd)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   DBUG_ENTER("wsrep_replicate_fragment");
+   DBUG_ASSERT(thd->wsrep_exec_mode != REPL_RECV &&
+               thd->wsrep_exec_mode != TOTAL_ORDER);
+   DBUG_ASSERT(thd->wsrep_SR_rollback_replicated_for_trx !=
+               thd->wsrep_trx_id());
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+ 
+   wsrep_wait_for_replayers(thd);
+   if (thd->wsrep_conflict_state() == MUST_ABORT)
+   {
+     // Transaction was BF aborted
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   thd->set_wsrep_query_state(QUERY_COMMITTING);
+ 
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   bool reset_trx_meta= false;
+   IO_CACHE *cache= wsrep_get_trans_cache(thd);
+   size_t data_len= 0;
+   uint32_t flags= (thd->wsrep_PA_safe ? 0 : WSREP_FLAG_PA_UNSAFE);
+   if (thd->wsrep_fragments_sent == 0)
+   {
+     flags |= WSREP_FLAG_TRX_START;
+   }
+ 
+   /*
+     thd->wsrep_fragments_sent must be incremented before wsrep_write_cache
+     to get thd->wsrep_rbr_buf populated. This also needs to be decremented
+     whenever error is returned from this function.
+    */
+   ++thd->wsrep_fragments_sent;
+   if (cache)
+   {
+     enum wsrep_trx_status rcode;
+     if ((rcode = wsrep_write_cache(wsrep, thd, cache, &data_len)) != WSREP_TRX_OK)
+     {
+       WSREP_ERROR("SR rbr write fail, data_len: %zu ret: %d",
+                   data_len, rcode);
+       --thd->wsrep_fragments_sent;
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       thd->set_wsrep_query_state(QUERY_EXEC);
+       DBUG_RETURN(rcode);
+     }
+   }
+ 
+ #if 0
+   WSREP_INFO("wsrep_replicate_fragment: base: %lu bytes: %u data_len %zu fragments_sent: %lu",
+              wsrep_get_fragment_base(thd),
+              wsrep_get_trans_cache_position(thd),
+              data_len,
+              thd->wsrep_fragments_sent);
+ #endif
+ 
+   if (data_len == 0)
+   {
+     if (thd->get_stmt_da()->is_ok()              &&
+         thd->get_stmt_da()->affected_rows() > 0  &&
+         !binlog_filter->is_on())
+     {
+       WSREP_WARN("empty rbr buffer, query: %s, "
+                  "affected rows: %llu, "
+                  "changed tables: %d, "
+                  "sql_log_bin: %d, "
+                  "wsrep status (%d %d %d)",
+                  thd->query(), thd->get_stmt_da()->affected_rows(),
+                  stmt_has_updated_trans_table(thd),
+                  thd->variables.sql_log_bin,
+                  thd->wsrep_exec_mode, thd->wsrep_query_state(),
+                  thd->wsrep_conflict_state_unsafe());
+     }
+     else
+     {
+       WSREP_WARN("empty rbr buffer, query: %s", thd->query());
+     }
+     --thd->wsrep_fragments_sent;
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID);
+   if (WSREP_UNDEFINED_TRX_ID == thd->wsrep_trx_id())
+   {
+     WSREP_WARN("SQL statement was ineffective, THD: %lld, buf: %zu\n"
+                "QUERY: %s\n"
+                " => Skipping fragment replication",
+                thd->thread_id, data_len, thd->query());
+     --thd->wsrep_fragments_sent;
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     DBUG_RETURN(WSREP_TRX_ERROR);
+   }
+ 
+   THD *SR_thd= NULL;
+   if (wsrep_SR_store)
+   {
+     SR_thd= wsrep_SR_store->append_frag(thd, flags, thd->wsrep_rbr_buf,
+                                         data_len);
+     if (!SR_thd)
+     {
+       my_error(ER_BINLOG_ROW_LOGGING_FAILED, MYF(0));
+       --thd->wsrep_fragments_sent;
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       thd->set_wsrep_query_state(QUERY_EXEC);
+       DBUG_RETURN(WSREP_TRX_ERROR);
+     }
+   }
+ 
+   my_free(thd->wsrep_rbr_buf);
+   thd->wsrep_rbr_buf= NULL;
+ 
+   DBUG_EXECUTE_IF("crash_replicate_fragment_before_certify",
+                   DBUG_SUICIDE(););
+ 
+   wsrep_status_t rcode= wsrep->certify(wsrep,
+                                        (wsrep_conn_id_t)thd->thread_id,
+                                        &thd->wsrep_ws_handle,
+                                        flags,
+                                        &thd->wsrep_trx_meta);
+ 
+   WSREP_DEBUG("Fragment certify(%lu): rcode %d, seqno %lld, trx %ld, "
+               "flags %u, conf %d, SQL: %s",
+               thd->thread_id, rcode,(long long)thd->wsrep_trx_meta.gtid.seqno,
+               thd->wsrep_trx_meta.stid.trx, flags,
+               thd->wsrep_conflict_state_unsafe(),
+               thd->query());
+   DBUG_EXECUTE_IF("crash_replicate_fragment_after_certify",
+                   DBUG_SUICIDE(););
+ 
+   bool frag_updated= false;
+   if (WSREP_OK == rcode)
+   {
+     DBUG_ASSERT(thd->wsrep_trx_has_seqno());
+ 
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     my_bool const must_abort= (thd->wsrep_conflict_state() == MUST_ABORT);
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+     if (must_abort)
+     {
+       rcode= WSREP_BF_ABORT;
+     }
+     else
+     {
+       rcode= wsrep->commit_order_enter(wsrep, &thd->wsrep_ws_handle,
+                                        &thd->wsrep_trx_meta);
+       if (rcode == WSREP_OK)
+       {
+         wsrep_xid_init(&SR_thd->wsrep_xid, thd->wsrep_trx_meta.gtid.uuid,
+                        thd->wsrep_trx_meta.gtid.seqno);
+         wsrep_SR_store->update_frag_seqno(SR_thd, thd);
+         frag_updated= true;
+         rcode= wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                          &thd->wsrep_trx_meta, NULL);
+         if (rcode != WSREP_OK && rcode != WSREP_BF_ABORT)
+         {
+           WSREP_ERROR("wsrep_replicate_fragment(%lu): seqno %lld, trx %ld "
+                       "Failed to leave commit order %d",
+                       thd->thread_id,
+                       (long long)thd->wsrep_trx_meta.gtid.seqno,
+                       thd->wsrep_trx_meta.stid.trx,
+                       rcode);
+         }
+         if (rcode == WSREP_OK)
+         {
+           rcode= wsrep->release(wsrep, &thd->wsrep_ws_handle);
+           if (rcode != WSREP_OK)
+           {
+             WSREP_ERROR("wsrep_replicate_fragment(%lu): seqno %lld, trx %ld "
+                         "Failed to release ws handle %d",
+                         thd->thread_id,
+                         (long long)thd->wsrep_trx_meta.gtid.seqno,
+                         thd->wsrep_trx_meta.stid.trx,
+                         rcode);
+           }
+         }
+         reset_trx_meta= true;
+       }
+       else
+       {
+         DBUG_ASSERT(rcode == WSREP_BF_ABORT || rcode == WSREP_TRX_FAIL);
+       }
+     }
+   }
+ 
+   if (!frag_updated)
+   {
+     wsrep_SR_store->release_SR_thd(SR_thd);
+     SR_thd= NULL;
+     thd->store_globals();
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   /*
+     If the SR transaction was BF aborted at this stage
+     we abort whole transaction.
+   */
+   if (thd->wsrep_conflict_state() == MUST_ABORT)
+   {
+     rcode= WSREP_BF_ABORT;
+   }
+ 
+   wsrep_trx_status ret;
+ 
+   switch (rcode)
+   {
+   case WSREP_OK:
+     DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+     DBUG_PRINT("wsrep", ("replicating commit success"));
+     if (thd->killed != NOT_KILLED)
+     {
+       WSREP_DEBUG("thd %lld killed with signal %d, during fragment replication",
+                 thd->thread_id, thd->killed);
+     }
+     DBUG_EXECUTE_IF("crash_replicate_fragment_success",
+                     DBUG_SUICIDE(););
+     ret= WSREP_TRX_OK;
+     break;
+   case WSREP_BF_ABORT:
+     if (thd->wsrep_conflict_state() != MUST_ABORT)
+     {
+       thd->set_wsrep_conflict_state(MUST_ABORT);
+     }
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK, WSREP_TRX_FAIL);
+     ret= WSREP_TRX_ERROR;
+     break;
+   case WSREP_TRX_FAIL:
+     thd->set_wsrep_conflict_state(CERT_FAILURE);
+     WSREP_LOG_CONFLICT(NULL, thd, FALSE);
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK, WSREP_TRX_FAIL);
+     ret= WSREP_TRX_CERT_FAIL;
+     break;
+   case WSREP_SIZE_EXCEEDED:
+     thd->set_wsrep_conflict_state(MUST_ABORT);
+     ret= WSREP_TRX_SIZE_EXCEEDED;
+     break;
+   case WSREP_CONN_FAIL:
+     thd->set_wsrep_conflict_state(MUST_ABORT);
+     my_error(ER_LOCK_DEADLOCK, MYF(0));
+     ret= WSREP_TRX_ERROR;
+     break;
+   default:
+     thd->set_wsrep_conflict_state(MUST_ABORT);
+     ret= WSREP_TRX_ERROR;
+     break;
+   }
+ 
+   thd->set_wsrep_query_state(QUERY_EXEC);
+ 
+   if (SR_thd && wsrep_thd_trx_seqno(thd) == WSREP_SEQNO_UNDEFINED)
+   {
+     --thd->wsrep_fragments_sent;
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+     wsrep_SR_store->release_SR_thd(SR_thd);
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->store_globals();
+   }
+ 
+   /*
+     Reset trx meta if pre_commit certify() (regardless of conflict state),
+     new seqno will be used for next fragment.
+     In case of failure GTID may be required in rollback process.
+   */
+   if (reset_trx_meta)
+   {
+     WSREP_DEBUG("Reset trx meta for %lu", thd->thread_id);
+     thd->wsrep_trx_meta.gtid= WSREP_GTID_UNDEFINED;
+     thd->wsrep_trx_meta.depends_on= WSREP_SEQNO_UNDEFINED;
+   }
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ static int wsrep_SR_step(THD *thd, uint unit)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   DBUG_ENTER("wsrep_SR_step");
+   if (!wsrep_may_produce_SR_step(thd) ||
+       unit != thd->variables.wsrep_trx_fragment_unit ||
+       thd->variables.wsrep_trx_fragment_size == 0 ||
+       thd->get_stmt_da()->is_error())
+   {
+ #if 0
+     WSREP_DEBUG("wsrep_SR_step: skip, frag_size: %lu is_error: %d",
+                 thd->variables.wsrep_trx_fragment_size,
+                 thd->get_stmt_da()->is_error());
+ #endif /* 0 */
+     DBUG_RETURN(0);
+   }
+ 
+   /*
+     Flush pending rows event into IO cache buffer
+    */
+   thd->binlog_flush_pending_rows_event(true);
+ 
+   wsrep_trx_status ret= WSREP_TRX_OK;
+   uint written= (wsrep_get_trans_cache_position(thd)
+                  - wsrep_get_fragment_base(thd));
+   bool replicate= false;
+ 
+   switch (thd->variables.wsrep_trx_fragment_unit)
+   {
+   case WSREP_FRAG_BYTES:
+     if (written >= thd->variables.wsrep_trx_fragment_size)
+       replicate= true;
+     break;
+   case WSREP_FRAG_ROWS:
+   case WSREP_FRAG_STATEMENTS:
+     wsrep_append_fill_rate(thd, 1);
+     if (wsrep_get_fragment_fill(thd) >= thd->variables.wsrep_trx_fragment_size)
+       replicate= true;
+     break;
+   default:
+     wsrep_append_fill_rate(thd, 1);
+     replicate= true;
+     DBUG_ASSERT(0);
+   }
+ 
+   if (replicate)
+   {
+     WSREP_DEBUG("fragment fill: %lu fragment unit: %lu fragment size: %lu written: %u",
+                 wsrep_get_fragment_fill(thd),
+                 thd->variables.wsrep_trx_fragment_unit,
+                 thd->variables.wsrep_trx_fragment_size,
+                 written);
+ 
+     ret= wsrep_replicate_fragment(thd);
+     if (ret == WSREP_TRX_OK)
+     {
+       wsrep_reset_fragment_fill(thd, 0);
+       wsrep_step_fragment_base(thd, written);
+     }
+   }
+ 
+   if (ret)
+   {
+     if (!thd->get_stmt_da()->is_error())
+     {
+       wsrep_override_error(thd, ER_ERROR_DURING_COMMIT,
+                            wsrep_trx_status_to_wsrep_status(ret));
+     }
+   }
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ bool wsrep_replicate_GTID(THD *thd)
+ {
+   if (thd->slave_thread)
+   {
+     WSREP_DEBUG("GTID replication");
+     DBUG_ASSERT (WSREP_UNDEFINED_TRX_ID == thd->wsrep_ws_handle.trx_id);
+     thd->set_wsrep_next_trx_id(thd->query_id);
+     (void)wsrep_ws_handle_for_trx(&thd->wsrep_ws_handle,
+                                   thd->wsrep_next_trx_id());
+     DBUG_ASSERT (WSREP_UNDEFINED_TRX_ID != thd->wsrep_ws_handle.trx_id);
+ 
+     int rcode= wsrep_certify(thd);
+     if (rcode)
+     {
+       WSREP_INFO("GTID replication failed: %d", rcode);
+       if (wsrep->commit_order_enter(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta))
+       {
+           WSREP_ERROR("wsrep::commit_order_enter fail: %llu %d",
+                       (long long)thd->thread_id, thd->get_stmt_da()->status());
+       }
+ 
+       if (wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta, NULL))
+       {
+           WSREP_ERROR("wsrep::commit_order_leave fail: %llu %d",
+                       (long long)thd->thread_id, thd->get_stmt_da()->status());
+       }
+ 
+       thd->wsrep_replicate_GTID= false;
+       my_message(ER_ERROR_DURING_COMMIT,
+                  "WSREP GTID replication was interrupted", MYF(0));
+ 
+       return true;
+     }
+   }
+   thd->wsrep_replicate_GTID= false;
+   return false;
+ }
+ 
+ 
+ /*
+   Utility methods to be called from hooks
+  */
+ 
+ /*
+   Log some THD info and called context
+  */
+ static void wsrep_log_thd(THD* thd, bool is_real_trans, const char *function)
+ {
+   char message[10];
+   snprintf(message, sizeof(message), "real: %d", is_real_trans);
+   message[sizeof(message) - 1] = '\0';
+   wsrep_log_thd(thd, message, function);
+ }
+ 
+ 
+ /*
+   Determine if the hook should be run
+ 
+   @return 0 Failure
+   @return 1 Success
+ */
+ static int wsrep_run_hook(const THD* thd, bool is_real_trans,
+                           bool for_real_trans)
+ {
+   return (WSREP(thd) &&  /* THD is non NULL, wsrep is enabled for thd 
+                             and is client thread */
+           thd->wsrep_exec_mode != TOTAL_ORDER && /* not TOI execution */
+           thd->wsrep_exec_mode != REPL_RECV &&   /* not applier or replayer */
+           (!for_real_trans || is_real_trans) &&
+           !(for_real_trans && /* CTAS SELECT phase */
+             WSREP_BINLOG_FORMAT(thd->variables.binlog_format) == BINLOG_FORMAT_STMT &&
+             thd->lex->sql_command == SQLCOM_CREATE_TABLE &&
 -            thd->lex->select_lex.item_list.elements)
++	    thd->lex->current_select->item_list.elements)
+     );
+ }
+ 
+ static inline
+ int wsrep_is_effective_not_to_replay(THD *thd)
+ {
+   mysql_mutex_assert_owner(&thd->LOCK_wsrep_thd);
+   int ret= (
+             /* effective */
+             thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID &&
+             /* not to replay */
+             thd->wsrep_conflict_state()   != MUST_REPLAY
+             );
+   return ret;
+ }
+ 
+ 
+ int wsrep_after_row(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_after_row");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   /*
+     We want to run this hook for each row, not just ones which
+     end autocommits or transactions.
+    */
+   if (!wsrep_run_hook(thd, is_real_trans, false))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+ 
+ #if 0
+   /*
+     Logging this hook is disabled, it gets very verbose.
+   */
+   wsrep_log_thd(thd, is_real_trans, "wsrep_after_row enter");
+ #endif /* 0 */
+   int ret= 0;
+ 
+   /*
+     TODO: Move this before row operation, maybe before_row() hook.
+    */
+   if (!wsrep_certify_nonPK)
+   {
+     for (TABLE* table= thd->open_tables; table != NULL; table= table->next)
+     {
+       if (table->key_info == NULL || table->s->primary_key == MAX_KEY)
+       {
+         WSREP_DEBUG("No primary key found for table %s.%s",
+                     table->s->db.str, table->s->table_name.str);
+         ret= 1;
+         break;
+       }
+     }
+   }
+ 
+   if (!ret)
+   {
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     if (thd->wsrep_conflict_state() == NO_CONFLICT)
+     {
+       ret|= wsrep_SR_step(thd, WSREP_FRAG_BYTES);
+       ret|= wsrep_SR_step(thd, WSREP_FRAG_ROWS);
+     }
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+   }
+ #if 0
+   /*
+     Logging this hook is disabled, it gets very verbose.
+    */
+   wsrep_log_thd(thd, is_real_trans, "wsrep_after_row leave");
+ #endif /* 0 */
+ 
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_before_prepare(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_before_prepare");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_prepare enter");
+   }
+ 
+   int ret= 0;
+   DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE);
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (wsrep_is_effective_not_to_replay(thd))
+   {
+     if (thd->wsrep_is_streaming())
+     {
+       DBUG_EXECUTE_IF("crash_last_fragment_commit_before_fragment_removal",
+                       DBUG_SUICIDE(););
+ 
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       /*
+         We don't support implicit commit for SR transactions.
+        */
+       if (stmt_causes_implicit_commit(thd, CF_IMPLICIT_COMMIT_BEGIN))
+       {
+         wsrep_override_error(thd, ER_ERROR_DURING_COMMIT);
+         ret= 1;
+       }
+       else
+       {
+         /*
+           Disable SR temporarily in order to avoid SR step from
+           after_row() hook when deleting fragments.
+          */
+         ulong frag_size_orig= thd->variables.wsrep_trx_fragment_size;
+         thd->variables.wsrep_trx_fragment_size= 0;
+         wsrep_remove_SR_fragments(thd);
+         thd->variables.wsrep_trx_fragment_size= frag_size_orig;
+       }
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       DBUG_EXECUTE_IF("crash_last_fragment_commit_after_fragment_removal",
+                       DBUG_SUICIDE(););
+     }
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_prepare leave");
+   }
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_after_prepare(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_after_prepare");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_prepare enter");
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE);
+   /*
+     thd->wsrep_exec_mode will be set in wsrep_certify() according
+     to outcome
+   */
+   int ret= 1;
+ 
+   if (wsrep_is_effective_not_to_replay(thd))
+   {
+     if (thd->wsrep_conflict_state() == NO_CONFLICT)
+     {
+       ret= wsrep_certify(thd);
+       if (ret)
+       {
+         DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_REPLAY ||
+                     thd->get_stmt_da()->is_error());
+       }
+     }
+     else
+     {
+       /*
+         BF aborted before pre commit, set state to aborting
+         and return error to trigger rollback.
+       */
+       DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_ABORT);
+       wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+     }
+   }
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_prepare leave");
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_before_commit(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_before_commit");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   /*
+     Applier/replayer codepath
+    */
+   if (thd->wsrep_exec_mode == REPL_RECV)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_commit enter");
+     if (is_real_trans)
+     {
+       DBUG_ASSERT(thd->wsrep_trx_must_order_commit());
+       if (wsrep->commit_order_enter(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta) != WSREP_OK)
+       {
+         WSREP_ERROR("Failed to enter applier commit order critical section");
+         DBUG_RETURN(1);
+       }
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       thd->set_wsrep_query_state(QUERY_COMMITTING);
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+     }
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_commit leave");
+     DBUG_RETURN(0);
+   }
+ 
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_commit enter");
+   }
+ 
+   int ret= 0;
+   if (thd->wsrep_exec_mode == LOCAL_STATE)
+   {
+     /*
+       We got here without having prepare phase first. This may
+       happen for example via trans_commit_stmt() -> tc_log->commit(thd, false)
+       in case of autocommit DML and binlog_format = STATEMENT.
+      */
+     if (wsrep_is_effective_not_to_replay(thd))
+     {
+       if (thd->wsrep_conflict_state() == NO_CONFLICT)
+       {
+         ret= wsrep_certify(thd);
+       }
+       else
+       {
+         /*
+           BF aborted before pre commit, set state to aborting
+           and return error to trigger rollback.
+         */
+         DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_ABORT);
+         wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+         ret= 1;
+       }
+     }
+   }
+ 
+ 
+   if (!ret && wsrep_is_effective_not_to_replay(thd))
+   {
+     if (thd->wsrep_conflict_state() == MUST_ABORT)
+     {
+       ret= 1;
+     }
+     else
+     {
+       DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       wsrep_status_t rcode= wsrep->commit_order_enter(wsrep,
+                                                       &thd->wsrep_ws_handle,
+                                                       &thd->wsrep_trx_meta);
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       switch (rcode)
+       {
+       case WSREP_OK:
+         wsrep_xid_init(&thd->wsrep_xid, thd->wsrep_trx_meta.gtid.uuid,
+                        thd->wsrep_trx_meta.gtid.seqno);
+         break;
+       case WSREP_BF_ABORT:
+         DBUG_ASSERT(wsrep_thd_trx_seqno(thd) > 0);
+         if (thd->wsrep_conflict_state() != MUST_ABORT)
+           thd->set_wsrep_conflict_state(MUST_ABORT);
+         mysql_mutex_lock(&LOCK_wsrep_replaying);
+         ++wsrep_replaying;
+         mysql_mutex_unlock(&LOCK_wsrep_replaying);
+         ret= 1;
+         break;
+       default:
+         WSREP_ERROR("Could not enter commit order critical section");
+         abort();
+       }
+     }
+   }
+ 
+   DBUG_ASSERT(ret ||
+               thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID ||
+               thd->wsrep_exec_mode == LOCAL_COMMIT);
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_commit leave");
+   }
+ 
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_ordered_commit(THD* thd, bool all, const wsrep_apply_error& err)
+ {
+   DBUG_ENTER("wsrep_ordered_commit");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   /*
+     Applier/replayer codepath
+   */
+   if (thd->wsrep_exec_mode == REPL_RECV)
+   {
+     int ret= 0;
+     wsrep_log_thd(thd, is_real_trans, "wsrep_ordered_commit enter");
+     if (is_real_trans)
+     {
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       bool run_commit_order_leave=
+         (thd->wsrep_query_state() != QUERY_ORDERED_COMMIT);
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       if (run_commit_order_leave)
+       {
+         wsrep_buf_t const err_buf= err.get_buf();
+         wsrep_status_t const rcode=
+           wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta, &err_buf);
+ 
+         if (rcode != WSREP_OK)
+         {
+           DBUG_ASSERT(rcode == WSREP_NODE_FAIL);
+           if (err.is_null())
+           {
+             WSREP_ERROR("Failed to leave commit order critical section, "
+                         "rcode: %d", rcode);
+           }
+           else
+           {
+             WSREP_WARN("Replication can't continue due to the error in a "
+                        "writeset apply operation: %s", err.c_str());
+           }
+           ret= 1;
+         }
+ 
+         mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+         thd->set_wsrep_query_state(QUERY_ORDERED_COMMIT);
+         mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       }
+     }
+     DBUG_RETURN(ret);
+   }
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_ordered_commit enter");
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID ||
+               (thd->wsrep_exec_mode == LOCAL_COMMIT &&
+                thd->wsrep_query_state() == QUERY_COMMITTING));
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+   if (wsrep_is_effective_not_to_replay(thd))
+   {
+     thd->wsrep_SR_fragments.clear();
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+     if (wsrep_thd_trx_seqno(thd) != WSREP_SEQNO_UNDEFINED &&
+         wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                   &thd->wsrep_trx_meta, NULL))
+     {
+       WSREP_ERROR("wsrep::commit_order_leave fail: %llu %d",
+                   (long long)thd->thread_id, thd->get_stmt_da()->status());
+     }
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_ORDERED_COMMIT);
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_ordered_commit leave");
+   }
+ 
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(0);
+ }
+ 
+ int wsrep_after_commit(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_after_commit");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_commit enter");
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID ||
+               (thd->wsrep_exec_mode == LOCAL_COMMIT &&
+                (thd->wsrep_query_state() == QUERY_COMMITTING ||
+                 thd->wsrep_query_state() == QUERY_ORDERED_COMMIT)));
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT ||
+               thd->wsrep_conflict_state() == MUST_ABORT);
+ 
+   if (wsrep_is_effective_not_to_replay(thd))
+   {
+     if (thd->wsrep_query_state() == QUERY_COMMITTING)
+     {
+       thd->wsrep_SR_fragments.clear();
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       if (wsrep_thd_trx_seqno(thd) != WSREP_SEQNO_UNDEFINED &&
+           wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                     &thd->wsrep_trx_meta, NULL))
+       {
+         WSREP_ERROR("wsrep::commit_order_leave fail: %llu %d",
+                     (long long)thd->thread_id, thd->get_stmt_da()->status());
+       }
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       thd->set_wsrep_query_state(QUERY_ORDERED_COMMIT);
+     }
+ 
+     DBUG_ASSERT(thd->wsrep_query_state() == QUERY_ORDERED_COMMIT);
+ 
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+     if (wsrep->release(wsrep, &thd->wsrep_ws_handle))
+     {
+       WSREP_WARN("wsrep::release fail: %lld %d",
+                  (long long)thd->thread_id, thd->get_stmt_da()->status());
+     }
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+   }
+ 
+   if (thd->wsrep_conflict_state() == MUST_ABORT)
+   {
+     assert(0);
+     WSREP_LOG_THD(thd, "BF aborted at commit phase");
+     thd->killed= NOT_KILLED;
+     thd->set_wsrep_conflict_state(NO_CONFLICT);
+   }
+ 
+   wsrep_cleanup_transaction(thd);
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_commit leave");
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(0);
+ }
+ 
+ int wsrep_before_rollback(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_before_rollback");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, false))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID)
+   {
+       (void)wsrep_ws_handle_for_trx(&thd->wsrep_ws_handle,
+                                     thd->wsrep_next_trx_id());
+ 
+       if (thd->wsrep_trx_id() == WSREP_UNDEFINED_TRX_ID)
+       {
+           WSREP_DEBUG("wsrep_before_rollback: setting trx_id to undefined, thd %llu %s",
+                       thd->thread_id, thd->query());
+       }
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_rollback enter");
+   }
+ 
+   if (thd->wsrep_query_state() == QUERY_COMMITTING)
+   {
+     DBUG_ASSERT(thd->wsrep_conflict_state() == MUST_ABORT);
+     WSREP_DEBUG("Query aborted while committing");
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     thd->set_wsrep_conflict_state(MUST_REPLAY);
+     thd->wsrep_exec_mode= LOCAL_STATE;
+   }
+ 
+   if (thd->wsrep_exec_mode != LOCAL_ROLLBACK &&
+       wsrep_is_effective_not_to_replay(thd) &&
+       (is_real_trans ||
+        (thd->wsrep_is_streaming() &&
+         (!wsrep_stmt_rollback_is_safe(thd) ||
+          thd->wsrep_conflict_state() != NO_CONFLICT))))
+   {
+     if (thd->wsrep_is_streaming() &&
+         /*
+           Cert failure will generate implicit rollback event on slaves
+ 
+           TODO: Need to do SR table cleanup on certification failure here.
+         */
+         thd->wsrep_conflict_state() != CERT_FAILURE &&
+         thd->wsrep_SR_rollback_replicated_for_trx != thd->wsrep_trx_id())
+     {
+       wsrep_prepare_SR_trx_info_for_rollback(thd);
+       thd->wsrep_SR_rollback_replicated_for_trx= thd->wsrep_trx_id();
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       DEBUG_SYNC(thd, "wsrep_before_SR_rollback");
+       WSREP_DEBUG("Replicating rollback for %ld %ld",
+                   thd->thread_id, thd->wsrep_trx_id());
+       wsrep_status_t rcode= wsrep->rollback(wsrep,
+                                             thd->wsrep_trx_id(), NULL);
+       if (rcode != WSREP_OK)
+       {
+         WSREP_WARN("failed to send SR rollback for %lld", thd->thread_id);
+       }
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     }
+     thd->wsrep_exec_mode= LOCAL_ROLLBACK;
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_before_rollback leave");
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(0);
+ }
+ 
+ int wsrep_after_rollback(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_after_rollback");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, false))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_rollback enter");
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT || /* voluntary or stmt rollback */
+               thd->wsrep_conflict_state() == MUST_ABORT ||  /* BF abort */
+               thd->wsrep_conflict_state() == ABORTING || /* called from wsrep_client_rollback() */
+               thd->wsrep_conflict_state() == CERT_FAILURE || /* cert failure */
+               thd->wsrep_conflict_state() == MUST_REPLAY || /* BF abort with successful repl */
+               (thd->wsrep_conflict_state() == ABORTED) /* trans_rollback_stmt() from mysql_exec_command() */
+               );
+ 
+   if (!is_real_trans)
+   {
+     /*
+       Statement rollback
+     */
+     if ((thd->wsrep_is_streaming() && !wsrep_stmt_rollback_is_safe(thd)))
+     {
+       /*
+         Statement rollback is not safe, do full rollback and report to client.
+       */
+       if (thd->wsrep_conflict_state() == NO_CONFLICT)
+       {
+         /*
+           If the statement rollback is due to an error, such as ER_DUP_ENTRY,
+           the client may not expect a full transaction rollback.
+           Set the conflict state to must abort here so that after_command()
+           hook will override the error to ER_LOCK_DEADLOCK.
+         */
+         thd->set_wsrep_conflict_state(MUST_ABORT);
+       }
+ 
+       /*
+         From trans_rollback()
+       */
+       thd->server_status&=
+         ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY);
+       mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+       /*
+         Calling ha_rollback_trans() here will call rollback hooks recursively.
+       */
+       ha_rollback_trans(thd, TRUE);
+       mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+       thd->variables.option_bits&= ~OPTION_BEGIN;
+       //thd->transaction.all.reset_unsafe_rollback_flags();
+       thd->transaction.all.m_unsafe_rollback_flags= 0;
+       thd->lex->start_transaction_opt= 0;
+     }
+   }
+   else
+   {
+     if (thd->wsrep_conflict_state() == ABORTED)
+     {
+       thd->wsrep_exec_mode= LOCAL_ROLLBACK;
+     }
+     if (wsrep_is_effective_not_to_replay(thd))
+     {
+       /*
+         Must have gone through before_rollback() hook at least once.
+       */
+       DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_ROLLBACK);
+     }
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_rollback leave");
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(0);
+ }
+ 
+ int wsrep_after_command(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_after_command");
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   /*
+     We want to run this hook for each command, not just ones which
+     end autocommits or transactions.
+   */
+   if (!wsrep_run_hook(thd, is_real_trans, false))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_command enter");
+   }
+ 
+   DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE ||
+               thd->wsrep_exec_mode == LOCAL_ROLLBACK);
+ 
+   int ret= 0;
+ 
+   switch (thd->wsrep_exec_mode)
+   {
+   case LOCAL_STATE:
+     /*
+       Run SR step if
+       - No conflict detected
+       - Trnsaction is active, it has acquired trx_id
+       - Not read-only command
+     */
+     if (thd->wsrep_conflict_state() == NO_CONFLICT  &&
+         thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID &&
+         thd->lex->sql_command != SQLCOM_SELECT)
+     {
+       ret|= wsrep_SR_step(thd, WSREP_FRAG_BYTES);
+       ret|= wsrep_SR_step(thd, WSREP_FRAG_STATEMENTS);
+     }
+     break;
+   case LOCAL_ROLLBACK:
+     {
+       bool should_retry= false;
+       const bool forced_rollback(thd->wsrep_conflict_state() == MUST_ABORT ||
+                                  thd->wsrep_conflict_state() == CERT_FAILURE);
+ 
+       DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT ||
+                   thd->wsrep_conflict_state() == MUST_ABORT ||
+                   thd->wsrep_conflict_state() == CERT_FAILURE);
+       /*
+         If conflict state is NO_CONFLICT the transaction was either
+         voluntary or done due to deadlock.
+       */
+       if (forced_rollback)
+       {
+         should_retry= !(
+           thd->wsrep_is_streaming() || // SR transactions do not retry
+           thd->spcont                  // SP code not patched to handle retry
+           );
+         wsrep_client_rollback(thd, false);
+       }
+       wsrep_post_rollback(thd);
+       if (forced_rollback)
+       {
+         wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+       }
+       wsrep_cleanup_transaction(thd);
+       DBUG_ASSERT(thd->wsrep_exec_mode == LOCAL_STATE);
+       DBUG_ASSERT(thd->wsrep_conflict_state() == NO_CONFLICT);
+       /*
+         Retry autocommit in case of deadlock error, usually seen
+         as ER_LOCK_DEADLOCK, sometimes as ER_QUERY_INTERRUPTED.
+        */
+       if (should_retry && thd->get_stmt_da()->is_error() &&
+           (thd->get_stmt_da()->sql_errno() == ER_LOCK_DEADLOCK ||
+            thd->get_stmt_da()->sql_errno() == ER_QUERY_INTERRUPTED))
+       {
+         thd->set_wsrep_conflict_state(RETRY_AUTOCOMMIT);
+       }
+     }
+     break;
+   default:
+     break;
+   }
+ 
+   if (thd->wsrep_conflict_state() == MUST_ABORT ||
+       thd->wsrep_conflict_state() == CERT_FAILURE)
+   {
+     wsrep_client_rollback(thd);
+     wsrep_post_rollback(thd);
+     wsrep_override_error(thd, ER_LOCK_DEADLOCK);
+     wsrep_cleanup_transaction(thd);
+   }
+ 
+   if (thd->wsrep_conflict_state() == MUST_REPLAY &&
+       !thd->spcont)
+   {
+     /*
+       BF aborted during commit, must replay
+     */
+     wsrep_replay_transaction(thd);
+   }
+ 
+   if (thd->wsrep_trx_id() != WSREP_UNDEFINED_TRX_ID)
+   {
+     wsrep_log_thd(thd, is_real_trans, "wsrep_after_command leave");
+   }
+ 
+ 
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_before_GTID_binlog(THD* thd, bool all)
+ {
+   DBUG_ENTER("wsrep_before_GTID_binlogt");
+   int ret= 0;
+ 
+   bool is_real_trans= (all || thd->transaction.all.ha_list == 0);
+ 
+   if (!wsrep_run_hook(thd, is_real_trans, true))
+   {
+     DBUG_RETURN(0);
+   }
+ 
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   if (wsrep_replicate_GTID(thd))
+   {
+     ret = 1;
+   }
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   wsrep_status_t rcode;
+   if (!ret &&
+       (rcode = wsrep->commit_order_enter(wsrep, &thd->wsrep_ws_handle,
+                                          &thd->wsrep_trx_meta)))
+   {
+     WSREP_ERROR("wsrep::commit_order_enter fail: %llu %d",
+                 (long long)thd->thread_id, rcode);
+     ret= 1;
+   }
+ 
+   if (!ret &&
+       (rcode = wsrep->commit_order_leave(wsrep, &thd->wsrep_ws_handle,
+                                          &thd->wsrep_trx_meta, NULL)))
+   {
+     WSREP_ERROR("wsrep::commit_order_leave fail: %llu %d",
+                 (long long)thd->thread_id, rcode);
+     ret= 1;
+   }
+ 
+   if (!ret)
+   {
+     mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+     thd->set_wsrep_query_state(QUERY_ORDERED_COMMIT);
+     thd->set_wsrep_query_state(QUERY_EXEC);
+     mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+   }
+ 
+   if (wsrep->release(wsrep, &thd->wsrep_ws_handle))
+   {
+     WSREP_WARN("wsrep::release fail: %lld %d",
+                (long long)thd->thread_id, thd->get_stmt_da()->status());
+   }
+   mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+   wsrep_cleanup_transaction(thd);
+   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+ 
+   DBUG_RETURN(ret);
+ }
+ 
+ int wsrep_register_trans_observer(void *p)
+ {
+   return 0;
+ }
+ 
+ int wsrep_unregister_trans_observer(void *p)
+ {
+   return 0;
+ }
diff --cc sql/wsrep_utils.cc
index 3c341e222b3,3c341e222b3..e9dad5f1912
--- a/sql/wsrep_utils.cc
+++ b/sql/wsrep_utils.cc
@@@ -21,6 -21,6 +21,7 @@@
  #endif
  
  #include "mariadb.h"
++#include "my_pthread.h"
  #include "wsrep_utils.h"
  #include "wsrep_mysqld.h"
  
diff --cc sql/wsrep_utils.h
index 277cea9dc31,f6520dda86e..bb3203c87e9
--- a/sql/wsrep_utils.h
+++ b/sql/wsrep_utils.h
@@@ -219,6 -240,6 +240,7 @@@ public
      return &member_info_;
    }
  
++#ifdef WITH_WSREP
    int lock()
    {
      return mysql_mutex_lock(&LOCK_wsrep_config_state);
@@@ -228,6 -249,6 +250,7 @@@
    {
      return mysql_mutex_unlock(&LOCK_wsrep_config_state);
    }
++#endif
  
  private:
    wsrep_view_info_t                  view_;
diff --cc sql/wsrep_xid.h
index 5b33a904de1,21d0a3f8f41..04dedb1f583
--- a/sql/wsrep_xid.h
+++ b/sql/wsrep_xid.h
@@@ -24,6 -23,7 +23,7 @@@
  #include "handler.h" // XID typedef
  
  void wsrep_xid_init(xid_t*, const wsrep_uuid_t&, wsrep_seqno_t);
 -int wsrep_is_wsrep_xid(const void* xid);
++extern "C" int wsrep_is_wsrep_xid(const void* xid);
  const wsrep_uuid_t* wsrep_xid_uuid(const XID&);
  wsrep_seqno_t wsrep_xid_seqno(const XID&);
  
diff --cc storage/innobase/buf/buf0dump.cc
index a0c4baad64d,95102dcbd87..bc96e91c486
--- a/storage/innobase/buf/buf0dump.cc
+++ b/storage/innobase/buf/buf0dump.cc
@@@ -43,9 -43,12 +43,9 @@@ Created April 08, 2011 Vasil Dimo
  #include "ut0byte.h"
  
  #include <algorithm>
 -#ifdef WITH_WSREP_OUT
 -extern my_bool wsrep_recovery;
 -#endif /* WITH_WSREP */
  
--#include "mysql/service_wsrep.h" /* wsrep_recovery */
  #include <my_service_manager.h>
++#include "mysql/service_wsrep.h" /* wsrep_recovery */
  
  enum status_severity {
  	STATUS_INFO,
@@@ -858,7 -861,7 +858,7 @@@ DECLARE_THREAD(buf_dump_thread)(void*
  				"Dumping of buffer pool not started"
  				" as load was incomplete");
  #ifdef WITH_WSREP
--		} else if (wsrep_recovery) {
++		} else if (get_wsrep_recovery()) {
  #endif /* WITH_WSREP */
  		} else {
  			buf_dump(FALSE/* do complete dump at shutdown */);
diff --cc storage/innobase/handler/ha_innodb.cc
index b0af34814af,62aec753777..56d41bf00b6
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@@ -5137,7 -5087,7 +5152,8 @@@ static void innobase_kill_query(handler
  {
  	DBUG_ENTER("innobase_kill_query");
  #ifdef WITH_WSREP
--	if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT) {
++	if (wsrep_on(thd) &&
++	    wsrep_thd_get_conflict_state(thd) != NO_CONFLICT) {
  		/* if victim has been signaled by BF thread and/or aborting
  		   is already progressing, following query aborting is not necessary
  		   any more.
@@@ -16754,7 -16705,7 +16768,10 @@@ ha_innobase::get_auto_increment
  				    thd_get_thread_id(ha_thd()),
  				    current, autoinc);
  
--			if (!wsrep_on(ha_thd())) {
++#ifdef WITH_WSREP
++			if (!wsrep_on(ha_thd()))
++#endif
++			{
  				current = autoinc - m_prebuilt->autoinc_increment;
  			}
  
@@@ -18992,14 -18814,27 +18880,28 @@@ wsrep_abort_transaction
  	my_bool signal)
  {
  	DBUG_ENTER("wsrep_innobase_abort_thd");
 -	trx_t* victim_trx = thd_to_trx(victim_thd);
 -	trx_t* bf_trx     = (bf_thd) ? thd_to_trx(bf_thd) : NULL;
 +
 +	trx_t* victim_trx	= thd_to_trx(victim_thd);
 +	trx_t* bf_trx		= (bf_thd) ? thd_to_trx(bf_thd) : NULL;
+ 	if (wsrep_debug)
+ 	{
+ 		wsrep_thd_LOCK(victim_thd);
+ 		WSREP_DEBUG("abort transaction: BF: %s victim: %s conf: %d",
+ 			wsrep_thd_query(bf_thd),
+ 			wsrep_thd_query(victim_thd),
+                             wsrep_thd_conflict_state(victim_thd, FALSE));
+ 		wsrep_thd_UNLOCK(victim_thd);
+ 	}
  
- 	WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d",
+ 	if (wsrep_debug)
+ 	{
+ 		wsrep_thd_LOCK(victim_thd);
+ 		WSREP_DEBUG("abort transaction: BF: %s victim: %s conf: %d",
  			wsrep_thd_query(bf_thd),
  			wsrep_thd_query(victim_thd),
- 			wsrep_thd_conflict_state(victim_thd, FALSE));
+ 			wsrep_thd_get_conflict_state(victim_thd));
+ 		wsrep_thd_UNLOCK(victim_thd);
+ 	}
  
  	if (victim_trx) {
  		lock_mutex_enter();
diff --cc storage/innobase/include/ha_prototypes.h
index 1313705f119,5ecbd4ac600..28bc8f9e565
--- a/storage/innobase/include/ha_prototypes.h
+++ b/storage/innobase/include/ha_prototypes.h
@@@ -240,6 -240,11 +240,11 @@@ wsrep_innobase_kill_one_trx(void * cons
  ulint wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
                               unsigned char* str, unsigned int str_length,
                               unsigned int buf_length);
+ my_bool wsrep_thd_is_SR(void *thd_ptr);
+ my_bool wsrep_thd_skip_locking(void *thd_ptr);
+ void wsrep_handle_SR_rollback(void *BF_thd_ptr, void *victim_thd_ptr);
 -void wsrep_thd_xid(const void *thd_ptr, void *xid, size_t xid_size);
++extern "C" void wsrep_thd_xid(const void *thd_ptr, void *xid, size_t xid_size);
+ bool wsrep_bf_abort(void *bf_thd_ptr, void *victim_thd_ptr, bool signal);
  #endif /* WITH_WSREP */
  
  /**********************************************************************//**
diff --cc storage/innobase/row/row0ins.cc
index 37ae828d09d,fa2274ad1f4..96db940e769
--- a/storage/innobase/row/row0ins.cc
+++ b/storage/innobase/row/row0ins.cc
@@@ -50,6 -50,8 +50,11 @@@ Created 4/20/1996 Heikki Tuur
  #include "fts0types.h"
  #include "m_string.h"
  #include "gis0geo.h"
++#include "log.h"
+ #include "wsrep_mysqld.h"
++#ifdef WITH_WSREP
+ #include "wsrep_schema.h"
++#endif
  
  /*************************************************************************
  IMPORTANT NOTE: Any operation that generates redo MUST check that there
@@@ -3240,9 -3229,26 +3245,26 @@@ row_ins_clust_index_entry
  
  	n_uniq = dict_index_is_unique(index) ? index->n_uniq : 0;
  
+ #ifdef WITH_WSREP
+ 	const bool skip_locking
+ 		= wsrep_thd_skip_locking(thr_get_trx(thr)->mysql_thd);
 -	ulint	flags = skip_locking | dict_table_is_temporary(index->table)
++	ulint	flags = skip_locking | index->table->is_temporary()
+ 		? BTR_NO_LOCKING_FLAG
+ 		: index->table->no_rollback() ? BTR_NO_ROLLBACK : 0;
+ #ifdef UNIV_DEBUG
+ 	if (skip_locking && sr_table_name_full_str != index->table->name.m_name) {
+ 		WSREP_ERROR("Record locking is disabled in this thread, "
+ 			    "but the table being modified is not "
+ 			    "`%s`: `%s`.", sr_table_name_full_str.c_str(),
+ 			    index->table->name.m_name);
+ 		ut_error;
+ 	}
+ #endif /* UNIV_DEBUG */
+ #else
  	ulint	flags = index->table->no_rollback() ? BTR_NO_ROLLBACK
 -		: dict_table_is_temporary(index->table)
 +		: index->table->is_temporary()
  		? BTR_NO_LOCKING_FLAG : 0;
+ #endif /* WITH_WSREP */
  	const ulint	orig_n_fields = entry->n_fields;
  
  	/* Try first optimistic descent to the B-tree */
diff --cc storage/innobase/row/row0sel.cc
index b68efbfa7be,680fab0c128..fe49d298bbc
--- a/storage/innobase/row/row0sel.cc
+++ b/storage/innobase/row/row0sel.cc
@@@ -56,6 -56,14 +56,15 @@@ Created 12/19/1997 Heikki Tuur
  #include "ha_prototypes.h"
  #include "srv0mon.h"
  #include "ut0new.h"
+ #include "m_string.h" /* for my_sys.h */
+ #include "my_sys.h" /* DEBUG_SYNC_C */
++#ifdef WITH_WSREP
+ #include "wsrep_schema.h"
 -
++#endif
+ #include "my_compare.h" /* enum icp_result */
+ #include "thr_lock.h"
+ #include "handler.h"
+ #include "ha_innodb.h"
  
  /* Maximum number of rows to prefetch; MySQL interface has another parameter */
  #define SEL_MAX_N_PREFETCH	16

    

jan

tags

participants (1)