September 2009 - developers - lists.mariadb.org

[Maria-developers] bzr commit into Mariadb 5.2, with Maria 2.0:maria/5.2 branch (sanja:2725)
by sanja＠askmonty.org 27 Nov '09

27 Nov '09

#At lp:maria/5.2 2725 sanja(a)askmonty.org 2009-07-07 Group commit and small optimisation of flush log (in case of page without CRC and sector protection) added. (for review) modified: mysql-test/suite/maria/r/maria3.result storage/maria/ha_maria.cc storage/maria/ma_init.c storage/maria/ma_loghandler.c storage/maria/ma_loghandler.h per-file messages: mysql-test/suite/maria/r/maria3.result New variables added storage/maria/ha_maria.cc New variables for group commit and routines to control them added. storage/maria/ma_init.c Service thread stop (if it is present) storage/maria/ma_loghandler.c 2 types of group commit added. If page is not protetcted by CRC or sector protection (i.e. we do not change header when add data to the page) we do not copy beginning of the page during forcing the buffer where the page is last to close. storage/maria/ma_loghandler.h Routines to control group commit added. === modified file 'mysql-test/suite/maria/r/maria3.result' --- a/mysql-test/suite/maria/r/maria3.result 2009-02-19 09:01:25 +0000 +++ b/mysql-test/suite/maria/r/maria3.result 2009-07-07 00:37:23 +0000 @@ -264,6 +264,8 @@ Variable_name Value maria_block_size 8192 maria_checkpoint_interval 30 maria_force_start_after_recovery_failures 0 +maria_group_commit none +maria_group_commit_rate 800 maria_log_file_size 4294959104 maria_log_purge_type immediate maria_max_sort_file_size 9223372036853727232 @@ -285,6 +287,7 @@ Maria_pagecache_read_requests # Maria_pagecache_reads # Maria_pagecache_write_requests # Maria_pagecache_writes # +Maria_transaction_log_syncs # create table t1 (b char(0)); insert into t1 values(NULL),(""); select length(b) from t1; === modified file 'storage/maria/ha_maria.cc' --- a/storage/maria/ha_maria.cc 2009-05-19 09:28:05 +0000 +++ b/storage/maria/ha_maria.cc 2009-07-07 00:37:23 +0000 @@ -101,22 +101,40 @@ TYPELIB maria_translog_purge_type_typeli array_elements(maria_translog_purge_type_names) - 1, "", maria_translog_purge_type_names, NULL }; + +/* transactional log directory sync */ const char *maria_sync_log_dir_names[]= { "NEVER", "NEWFILE", "ALWAYS", NullS }; - TYPELIB maria_sync_log_dir_typelib= { array_elements(maria_sync_log_dir_names) - 1, "", maria_sync_log_dir_names, NULL }; +/* transactional log group commit */ +const char *maria_group_commit_names[]= +{ + "none", "hard", "soft", NullS +}; +TYPELIB maria_group_commit_typelib= +{ + array_elements(maria_group_commit_names) - 1, "", + maria_group_commit_names, NULL +}; + /** Interval between background checkpoints in seconds */ static ulong checkpoint_interval; static void update_checkpoint_interval(MYSQL_THD thd, struct st_mysql_sys_var *var, void *var_ptr, const void *save); +static void update_maria_group_commit(MYSQL_THD thd, + struct st_mysql_sys_var *var, + void *var_ptr, const void *save); +static void update_maria_group_commit_rate(MYSQL_THD thd, + struct st_mysql_sys_var *var, + void *var_ptr, const void *save); /** After that many consecutive recovery failures, remove logs */ static ulong force_start_after_recovery_failures; static void update_log_file_size(MYSQL_THD thd, @@ -163,6 +181,24 @@ static MYSQL_SYSVAR_ULONG(log_file_size, NULL, update_log_file_size, TRANSLOG_FILE_SIZE, TRANSLOG_MIN_FILE_SIZE, 0xffffffffL, TRANSLOG_PAGE_SIZE); +static MYSQL_SYSVAR_ENUM(group_commit, maria_group_commit, + PLUGIN_VAR_RQCMDARG, + "Specifies maria group commit mode. " + "Possible values are \"none\" (no group commit), " + "\"hard\" (with waiting to actual commit), " + "\"soft\" (no wait for commit (DANGEROUS!!!))", + NULL, update_maria_group_commit, + TRANSLOG_GCOMMIT_NONE, &maria_group_commit_typelib); + +static MYSQL_SYSVAR_ULONG(group_commit_rate, maria_group_commit_rate, + PLUGIN_VAR_RQCMDARG, + "Number of commits per 100 seconds. (in other words one commit for" + "every 100/maria_group_commit_rate second). 0 stands for no waiting" + "for other threads to come and do a commit in \"hard\" mode and no" + " sync()/commit at all in \"soft\" mode. Option has only an effect" + "if maria_group_commit is used", + NULL, update_maria_group_commit_rate, 800, 0, UINT_MAX, 1); + static MYSQL_SYSVAR_ENUM(log_purge_type, log_purge_type, PLUGIN_VAR_RQCMDARG, "Specifies how maria transactional log will be purged. " @@ -3254,6 +3290,8 @@ static struct st_mysql_sys_var* system_v MYSQL_SYSVAR(block_size), MYSQL_SYSVAR(checkpoint_interval), MYSQL_SYSVAR(force_start_after_recovery_failures), + MYSQL_SYSVAR(group_commit), + MYSQL_SYSVAR(group_commit_rate), MYSQL_SYSVAR(page_checksum), MYSQL_SYSVAR(log_dir_path), MYSQL_SYSVAR(log_file_size), @@ -3284,6 +3322,97 @@ static void update_checkpoint_interval(M } /** + @brief Updates group commit mode +*/ + +static void update_maria_group_commit(MYSQL_THD thd, + struct st_mysql_sys_var *var, + void *var_ptr, const void *save) +{ + ulong value= (ulong)*((long *)var_ptr); + DBUG_ENTER("update_maria_group_commit"); + DBUG_PRINT("enter", ("old value: %lu new value %lu rate %lu", + value, (ulong)(*(long *)save), + maria_group_commit_rate)); + /* old value */ + switch (value) { + case TRANSLOG_GCOMMIT_NONE: + break; + case TRANSLOG_GCOMMIT_HARD: + translog_hard_group_commit(FALSE); + break; + case TRANSLOG_GCOMMIT_SOFT: + translog_soft_sync(FALSE); + if (maria_group_commit_rate) + translog_soft_sync_end(); + break; + default: + DBUG_ASSERT(0); /* impossible */ + } + value= *(ulong *)var_ptr= (ulong)(*(long *)save); + translog_sync(); + /* new value */ + switch (value) { + case TRANSLOG_GCOMMIT_NONE: + break; + case TRANSLOG_GCOMMIT_HARD: + translog_hard_group_commit(TRUE); + break; + case TRANSLOG_GCOMMIT_SOFT: + translog_soft_sync(TRUE); + /* variable change made under global lock so we can just read it */ + if (maria_group_commit_rate) + translog_soft_sync_start(); + break; + default: + DBUG_ASSERT(0); /* impossible */ + } + DBUG_VOID_RETURN; +} + +/** + @brief Updates group commit rate +*/ + +static void update_maria_group_commit_rate(MYSQL_THD thd, + struct st_mysql_sys_var *var, + void *var_ptr, const void *save) +{ + ulong new_value= (ulong)*((long *)save); + ulong *value_ptr= (ulong*) var_ptr; + DBUG_ENTER("update_maria_group_commit_rate"); + DBUG_PRINT("enter", ("old value: %lu new value %lu group commit %lu", + *value_ptr, new_value, maria_group_commit)); + if (new_value && + ((TRANSLOG_RATE_BASE * 1000000000ULL / new_value + + TRANSLOG_RATE_BASE / 2) / + TRANSLOG_RATE_BASE) == 0) + new_value= 0; /* protection against too small value */ + + /* variable change made under global lock so we can just read it */ + switch (maria_group_commit) { + case TRANSLOG_GCOMMIT_NONE: + *value_ptr= new_value; + translog_set_group_commit_rate(new_value); + break; + case TRANSLOG_GCOMMIT_HARD: + *value_ptr= new_value; + translog_set_group_commit_rate(new_value); + break; + case TRANSLOG_GCOMMIT_SOFT: + if (*value_ptr) + translog_soft_sync_end(); + translog_set_group_commit_rate(new_value); + if ((*value_ptr= new_value)) + translog_soft_sync_start(); + break; + default: + DBUG_ASSERT(0); /* impossible */ + } + DBUG_VOID_RETURN; +} + +/** @brief Updates the transaction log file limit. */ @@ -3305,6 +3434,7 @@ static SHOW_VAR status_variables[]= { {"Maria_pagecache_reads", (char*) &maria_pagecache_var.global_cache_read, SHOW_LONGLONG}, {"Maria_pagecache_write_requests", (char*) &maria_pagecache_var.global_cache_w_requests, SHOW_LONGLONG}, {"Maria_pagecache_writes", (char*) &maria_pagecache_var.global_cache_write, SHOW_LONGLONG}, + {"Maria_transaction_log_syncs", (char*) &translog_syncs, SHOW_LONGLONG}, {NullS, NullS, SHOW_LONG} }; === modified file 'storage/maria/ma_init.c' --- a/storage/maria/ma_init.c 2008-10-09 20:03:54 +0000 +++ b/storage/maria/ma_init.c 2009-07-07 00:37:23 +0000 @@ -82,6 +82,11 @@ void maria_end(void) maria_inited= maria_multi_threaded= FALSE; ft_free_stopwords(); ma_checkpoint_end(); + if (translog_status == TRANSLOG_OK) + { + translog_soft_sync_end(); + translog_sync(); + } if ((trid= trnman_get_max_trid()) > max_trid_in_control_file) { /* === modified file 'storage/maria/ma_loghandler.c' --- a/storage/maria/ma_loghandler.c 2009-05-19 09:28:05 +0000 +++ b/storage/maria/ma_loghandler.c 2009-07-07 00:37:23 +0000 @@ -18,6 +18,7 @@ #include "ma_blockrec.h" /* for some constants and in-write hooks */ #include "ma_key_recover.h" /* For some in-write hooks */ #include "ma_checkpoint.h" +#include "ma_servicethread.h" /* On Windows, neither my_open() nor my_sync() work for directories. @@ -47,6 +48,15 @@ #include <m_ctype.h> #endif +/** @brief protects checkpoint_in_progress */ +static pthread_mutex_t LOCK_soft_sync; +/** @brief for killing the background checkpoint thread */ +static pthread_cond_t COND_soft_sync; +/** @brief control structure for checkpoint background thread */ +static MA_SERVICE_THREAD_CONTROL soft_sync_control= + {THREAD_DEAD, FALSE, &LOCK_soft_sync, &COND_soft_sync}; + + /* transaction log file descriptor */ typedef struct st_translog_file { @@ -124,10 +134,20 @@ struct st_translog_buffer /* Previous buffer offset to detect it flush finish */ TRANSLOG_ADDRESS prev_buffer_offset; /* + If the buffer was forced to close it save value of its horizon + otherwise LSN_IMPOSSIBLE + */ + TRANSLOG_ADDRESS pre_force_close_horizon; + /* How much is written (or will be written when copy_to_buffer_in_progress become 0) to this buffer */ translog_size_t size; + /* + How much data was skipped during moving page from previous buffer + to this one (it is optimisation of forcing buffer to finish + */ + uint skipped_data; /* File handler for this buffer */ TRANSLOG_FILE *file; /* Threads which are waiting for buffer filling/freeing */ @@ -304,6 +324,7 @@ struct st_translog_descriptor */ pthread_mutex_t log_flush_lock; pthread_cond_t log_flush_cond; + pthread_cond_t new_goal_cond; /* Protects changing of headers of finished files (max_lsn) */ pthread_mutex_t file_header_lock; @@ -344,13 +365,39 @@ static struct st_translog_descriptor log ulong log_purge_type= TRANSLOG_PURGE_IMMIDIATE; ulong log_file_size= TRANSLOG_FILE_SIZE; +/* sync() of log files directory mode */ ulong sync_log_dir= TRANSLOG_SYNC_DIR_NEWFILE; +ulong maria_group_commit= TRANSLOG_GCOMMIT_NONE; +ulong maria_group_commit_rate= 0; /* Marker for end of log */ static uchar end_of_log= 0; #define END_OF_LOG &end_of_log +/** + Switch for "soft" sync (no real sync() but periodical sync by service + thread) +*/ +static volatile my_bool soft_sync= FALSE; +/** + Switch for "hard" group commit mode +*/ +static volatile my_bool hard_group_commit= FALSE; +/** + File numbers interval which have to be sync() +*/ +static uint32 soft_sync_min= 0; +static uint32 soft_sync_max= 0; +/** + stores interval in nanoseconds/TRANSLOG_RATE_BASE (to + fit into uint32) +*/ +static uint32 group_commit_wait= 0; enum enum_translog_status translog_status= TRANSLOG_UNINITED; +ulonglong translog_syncs= 0; /* Number of sync()s */ + +/* time of last flush */ +static ulonglong flush_start= 0; /* chunk types */ #define TRANSLOG_CHUNK_LSN 0x00 /* 0 chunk refer as LSN (head or tail */ @@ -980,12 +1027,17 @@ static TRANSLOG_FILE *get_logfile_by_num static TRANSLOG_FILE *get_current_logfile() { TRANSLOG_FILE *file; + DBUG_ENTER("get_current_logfile"); rw_rdlock(&log_descriptor.open_files_lock); + DBUG_PRINT("info", ("max_file: %lu min_file: %lu open_files: %lu", + (ulong) log_descriptor.max_file, + (ulong) log_descriptor.min_file, + (ulong) log_descriptor.open_files.elements)); DBUG_ASSERT(log_descriptor.max_file - log_descriptor.min_file + 1 == log_descriptor.open_files.elements); file= *dynamic_element(&log_descriptor.open_files, 0, TRANSLOG_FILE **); rw_unlock(&log_descriptor.open_files_lock); - return (file); + DBUG_RETURN(file); } uchar NEAR maria_trans_file_magic[]= @@ -1069,6 +1121,7 @@ static my_bool translog_write_file_heade static my_bool translog_max_lsn_to_header(File file, LSN lsn) { uchar lsn_buff[LSN_STORE_SIZE]; + my_bool rc; DBUG_ENTER("translog_max_lsn_to_header"); DBUG_PRINT("enter", ("File descriptor: %ld " "lsn: (%lu,0x%lx)", @@ -1077,11 +1130,13 @@ static my_bool translog_max_lsn_to_heade lsn_store(lsn_buff, lsn); - DBUG_RETURN(my_pwrite(file, lsn_buff, - LSN_STORE_SIZE, - (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE), - log_write_flags) != 0 || - my_sync(file, MYF(MY_WME)) != 0); + if (!(rc= (my_pwrite(file, lsn_buff, + LSN_STORE_SIZE, + (LOG_HEADER_DATA_SIZE - LSN_STORE_SIZE), + log_write_flags) != 0 || + my_sync(file, MYF(MY_WME)) != 0))) + translog_syncs++; + DBUG_RETURN(rc); } @@ -1423,7 +1478,9 @@ LSN translog_get_file_max_lsn_stored(uin static my_bool translog_buffer_init(struct st_translog_buffer *buffer, int num) { DBUG_ENTER("translog_buffer_init"); - buffer->prev_last_lsn= buffer->last_lsn= LSN_IMPOSSIBLE; + buffer->pre_force_close_horizon= + buffer->prev_last_lsn= buffer->last_lsn= + LSN_IMPOSSIBLE; DBUG_PRINT("info", ("last_lsn and prev_last_lsn set to 0 buffer: 0x%lx", (ulong) buffer)); @@ -1435,6 +1492,7 @@ static my_bool translog_buffer_init(stru memset(buffer->buffer, TRANSLOG_FILLER, TRANSLOG_WRITE_BUFFER); /* Buffer size */ buffer->size= 0; + buffer->skipped_data= 0; /* cond of thread which is waiting for buffer filling */ if (pthread_cond_init(&buffer->waiting_filling_buffer, 0)) DBUG_RETURN(1); @@ -1489,7 +1547,10 @@ static my_bool translog_close_log_file(T TODO: sync only we have changed the log */ if (!file->is_sync) + { rc= my_sync(file->handler.file, MYF(MY_WME)); + translog_syncs++; + } rc|= my_close(file->handler.file, MYF(MY_WME)); my_free(file, MYF(0)); return test(rc); @@ -2044,7 +2105,8 @@ static void translog_start_buffer(struct (ulong) LSN_OFFSET(log_descriptor.horizon), (ulong) LSN_OFFSET(log_descriptor.horizon))); DBUG_ASSERT(buffer_no == buffer->buffer_no); - buffer->prev_last_lsn= buffer->last_lsn= LSN_IMPOSSIBLE; + buffer->pre_force_close_horizon= + buffer->prev_last_lsn= buffer->last_lsn= LSN_IMPOSSIBLE; DBUG_PRINT("info", ("last_lsn and prev_last_lsn set to 0 buffer: 0x%lx", (ulong) buffer)); buffer->offset= log_descriptor.horizon; @@ -2052,6 +2114,7 @@ static void translog_start_buffer(struct buffer->file= get_current_logfile(); buffer->overlay= 0; buffer->size= 0; + buffer->skipped_data= 0; translog_cursor_init(cursor, buffer, buffer_no); DBUG_PRINT("info", ("file: #%ld (%d) init cursor #%u: 0x%lx " "chaser: %d Size: %lu (%lu)", @@ -2523,6 +2586,7 @@ static my_bool translog_buffer_flush(str TRANSLOG_ADDRESS offset= buffer->offset; TRANSLOG_FILE *file= buffer->file; uint8 ver= buffer->ver; + uint skipped_data; DBUG_ENTER("translog_buffer_flush"); DBUG_PRINT("enter", ("Buffer: #%u 0x%lx file: %d offset: (%lu,0x%lx) size: %lu", @@ -2557,6 +2621,8 @@ static my_bool translog_buffer_flush(str disk */ file= buffer->file; + skipped_data= buffer->skipped_data; + DBUG_ASSERT(skipped_data < TRANSLOG_PAGE_SIZE); for (i= 0, pg= LSN_OFFSET(buffer->offset) / TRANSLOG_PAGE_SIZE; i < buffer->size; i+= TRANSLOG_PAGE_SIZE, pg++) @@ -2573,13 +2639,16 @@ static my_bool translog_buffer_flush(str DBUG_ASSERT(i + TRANSLOG_PAGE_SIZE <= buffer->size); if (translog_status != TRANSLOG_OK && translog_status != TRANSLOG_SHUTDOWN) DBUG_RETURN(1); - if (pagecache_inject(log_descriptor.pagecache, + if (pagecache_write_part(log_descriptor.pagecache, &file->handler, pg, 3, buffer->buffer + i, PAGECACHE_PLAIN_PAGE, PAGECACHE_LOCK_LEFT_UNLOCKED, - PAGECACHE_PIN_LEFT_UNPINNED, 0, - LSN_IMPOSSIBLE)) + PAGECACHE_PIN_LEFT_UNPINNED, + PAGECACHE_WRITE_DONE, 0, + LSN_IMPOSSIBLE, + skipped_data, + TRANSLOG_PAGE_SIZE - skipped_data)) { DBUG_PRINT("error", ("Can't write page (%lu,0x%lx) to pagecache, error: %d", @@ -2589,10 +2658,12 @@ static my_bool translog_buffer_flush(str translog_stop_writing(); DBUG_RETURN(1); } + skipped_data= 0; } file->is_sync= 0; - if (my_pwrite(file->handler.file, buffer->buffer, - buffer->size, LSN_OFFSET(buffer->offset), + if (my_pwrite(file->handler.file, buffer->buffer + buffer->skipped_data, + buffer->size - buffer->skipped_data, + LSN_OFFSET(buffer->offset) + buffer->skipped_data, log_write_flags)) { DBUG_PRINT("error", ("Can't write buffer (%lu,0x%lx) size %lu " @@ -2985,6 +3056,7 @@ restart: uchar *from, *table= NULL; int is_last_unfinished_page; uint last_protected_sector= 0; + uint skipped_data= curr_buffer->skipped_data; TRANSLOG_FILE file_copy; uint8 ver= curr_buffer->ver; translog_wait_for_writers(curr_buffer); @@ -2997,7 +3069,25 @@ restart: } DBUG_ASSERT(LSN_FILE_NO(addr) == LSN_FILE_NO(curr_buffer->offset)); from= curr_buffer->buffer + (addr - curr_buffer->offset); - memcpy(buffer, from, TRANSLOG_PAGE_SIZE); + if (skipped_data > (addr - curr_buffer->offset)) + { + /* + We read page part of which is not present in buffer, + so we should read absent part from file (page cache actually) + */ + file= get_logfile_by_number(file_no); + DBUG_ASSERT(file != NULL); + buffer= pagecache_read(log_descriptor.pagecache, &file->handler, + LSN_OFFSET(addr) / TRANSLOG_PAGE_SIZE, + 3, buffer, + PAGECACHE_PLAIN_PAGE, + PAGECACHE_LOCK_LEFT_UNLOCKED, + NULL); + } + else + skipped_data= 0; /* Read after skipped in buffer data */ + memcpy(buffer + skipped_data, from + skipped_data, + TRANSLOG_PAGE_SIZE - skipped_data); /* We can use copy then in translog_page_validator() because it do not put it permanently somewhere. @@ -3291,6 +3381,7 @@ static my_bool translog_truncate_log(TRA uint32 next_page_offset, page_rest; uint32 i; File fd; + int rc; TRANSLOG_VALIDATOR_DATA data; char path[FN_REFLEN]; uchar page_buff[TRANSLOG_PAGE_SIZE]; @@ -3316,14 +3407,19 @@ static my_bool translog_truncate_log(TRA TRANSLOG_PAGE_SIZE); page_rest= next_page_offset - LSN_OFFSET(addr); memset(page_buff, TRANSLOG_FILLER, page_rest); - if ((fd= open_logfile_by_number_no_cache(LSN_FILE_NO(addr))) < 0 || - ((my_chsize(fd, next_page_offset, TRANSLOG_FILLER, MYF(MY_WME)) || - (page_rest && my_pwrite(fd, page_buff, page_rest, LSN_OFFSET(addr), - log_write_flags)) || - my_sync(fd, MYF(MY_WME))) | - my_close(fd, MYF(MY_WME))) || - (sync_log_dir >= TRANSLOG_SYNC_DIR_ALWAYS && - sync_dir(log_descriptor.directory_fd, MYF(MY_WME | MY_IGNORE_BADFD)))) + rc= ((fd= open_logfile_by_number_no_cache(LSN_FILE_NO(addr))) < 0 || + ((my_chsize(fd, next_page_offset, TRANSLOG_FILLER, MYF(MY_WME)) || + (page_rest && my_pwrite(fd, page_buff, page_rest, LSN_OFFSET(addr), + log_write_flags)) || + my_sync(fd, MYF(MY_WME))))); + translog_syncs++; + rc|= (fd > 0 && my_close(fd, MYF(MY_WME))); + if (sync_log_dir >= TRANSLOG_SYNC_DIR_ALWAYS) + { + rc|= sync_dir(log_descriptor.directory_fd, MYF(MY_WME | MY_IGNORE_BADFD)); + translog_syncs++; + } + if (rc) DBUG_RETURN(1); /* fix the horizon */ @@ -3511,6 +3607,7 @@ my_bool translog_init_with_table(const c pthread_mutex_init(&log_descriptor.dirty_buffer_mask_lock, MY_MUTEX_INIT_FAST) || pthread_cond_init(&log_descriptor.log_flush_cond, 0) || + pthread_cond_init(&log_descriptor.new_goal_cond, 0) || my_rwlock_init(&log_descriptor.open_files_lock, NULL) || my_init_dynamic_array(&log_descriptor.open_files, @@ -3912,7 +4009,6 @@ my_bool translog_init_with_table(const c log_descriptor.flushed= log_descriptor.horizon; log_descriptor.in_buffers_only= log_descriptor.bc.buffer->offset; log_descriptor.max_lsn= LSN_IMPOSSIBLE; /* set to 0 */ - log_descriptor.previous_flush_horizon= log_descriptor.horizon; /* Now 'flushed' is set to 'horizon' value, but 'horizon' is (potentially) address of the next LSN and we want indicate that all LSNs that are @@ -3995,6 +4091,10 @@ my_bool translog_init_with_table(const c It is beginning of the log => there is no LSNs in the log => There is no harm in leaving it "as-is". */ + log_descriptor.previous_flush_horizon= log_descriptor.horizon; + DBUG_PRINT("info", ("previous_flush_horizon: (%lu,0x%lx)", + LSN_IN_PARTS(log_descriptor. + previous_flush_horizon))); DBUG_RETURN(0); } file_no--; @@ -4070,6 +4170,9 @@ my_bool translog_init_with_table(const c translog_free_record_header(&rec); } } + log_descriptor.previous_flush_horizon= log_descriptor.horizon; + DBUG_PRINT("info", ("previous_flush_horizon: (%lu,0x%lx)", + LSN_IN_PARTS(log_descriptor.previous_flush_horizon))); DBUG_RETURN(0); err: ma_message_no_user(0, "log initialization failed"); @@ -4157,6 +4260,7 @@ void translog_destroy() pthread_mutex_destroy(&log_descriptor.log_flush_lock); pthread_mutex_destroy(&log_descriptor.dirty_buffer_mask_lock); pthread_cond_destroy(&log_descriptor.log_flush_cond); + pthread_cond_destroy(&log_descriptor.new_goal_cond); rwlock_destroy(&log_descriptor.open_files_lock); delete_dynamic(&log_descriptor.open_files); delete_dynamic(&log_descriptor.unfinished_files); @@ -6885,11 +6989,11 @@ int translog_read_record_header_from_buf { translog_size_t res; DBUG_ENTER("translog_read_record_header_from_buffer"); + DBUG_PRINT("info", ("page byte: 0x%x offset: %u", + (uint) page[page_offset], (uint) page_offset)); DBUG_ASSERT(translog_is_LSN_chunk(page[page_offset])); DBUG_ASSERT(translog_status == TRANSLOG_OK || translog_status == TRANSLOG_READONLY); - DBUG_PRINT("info", ("page byte: 0x%x offset: %u", - (uint) page[page_offset], (uint) page_offset)); buff->type= (page[page_offset] & TRANSLOG_REC_TYPE); buff->short_trid= uint2korr(page + page_offset + 1); DBUG_PRINT("info", ("Type %u, Short TrID %u, LSN (%lu,0x%lx)", @@ -7356,27 +7460,27 @@ static void translog_force_current_buffe "Buffer addr: (%lu,0x%lx) " "Page addr: (%lu,0x%lx) " "size: %lu (%lu) Pg: %u left: %u in progress %u", - (uint) log_descriptor.bc.buffer_no, - (ulong) log_descriptor.bc.buffer, - LSN_IN_PARTS(log_descriptor.bc.buffer->offset), + (uint) old_buffer_no, + (ulong) old_buffer, + LSN_IN_PARTS(old_buffer->offset), (ulong) LSN_FILE_NO(log_descriptor.horizon), (ulong) (LSN_OFFSET(log_descriptor.horizon) - log_descriptor.bc.current_page_fill), - (ulong) log_descriptor.bc.buffer->size, + (ulong) old_buffer->size, (ulong) (log_descriptor.bc.ptr -log_descriptor.bc. buffer->buffer), (uint) log_descriptor.bc.current_page_fill, (uint) left, - (uint) log_descriptor.bc.buffer-> + (uint) old_buffer-> copy_to_buffer_in_progress)); translog_lock_assert_owner(); LINT_INIT(current_page_fill); - new_buff_beginning= log_descriptor.bc.buffer->offset; - new_buff_beginning+= log_descriptor.bc.buffer->size; /* increase offset */ + new_buff_beginning= old_buffer->offset; + new_buff_beginning+= old_buffer->size; /* increase offset */ DBUG_ASSERT(log_descriptor.bc.ptr !=NULL); DBUG_ASSERT(LSN_FILE_NO(log_descriptor.horizon) == - LSN_FILE_NO(log_descriptor.bc.buffer->offset)); + LSN_FILE_NO(old_buffer->offset)); translog_check_cursor(&log_descriptor.bc); DBUG_ASSERT(left < TRANSLOG_PAGE_SIZE); if (left) @@ -7387,18 +7491,20 @@ static void translog_force_current_buffe */ DBUG_PRINT("info", ("left: %u", (uint) left)); + old_buffer->pre_force_close_horizon= + old_buffer->offset + old_buffer->size; /* decrease offset */ new_buff_beginning-= log_descriptor.bc.current_page_fill; current_page_fill= log_descriptor.bc.current_page_fill; memset(log_descriptor.bc.ptr, TRANSLOG_FILLER, left); - log_descriptor.bc.buffer->size+= left; + old_buffer->size+= left; DBUG_PRINT("info", ("Finish Page buffer #%u: 0x%lx " "Size: %lu", - (uint) log_descriptor.bc.buffer->buffer_no, - (ulong) log_descriptor.bc.buffer, - (ulong) log_descriptor.bc.buffer->size)); - DBUG_ASSERT(log_descriptor.bc.buffer->buffer_no == + (uint) old_buffer->buffer_no, + (ulong) old_buffer, + (ulong) old_buffer->size)); + DBUG_ASSERT(old_buffer->buffer_no == log_descriptor.bc.buffer_no); } else @@ -7509,11 +7615,21 @@ static void translog_force_current_buffe if (left) { - /* - TODO: do not copy beginning of the page if we have no CRC or sector - checks on - */ - memcpy(new_buffer->buffer, data, current_page_fill); + if (log_descriptor.flags & + (TRANSLOG_PAGE_CRC | TRANSLOG_SECTOR_PROTECTION)) + memcpy(new_buffer->buffer, data, current_page_fill); + else + { + /* + This page header does not change if we add more data to the page so + we can not copy it and will not overwrite later + */ + new_buffer->skipped_data= current_page_fill; +#ifndef DBUG_OFF + memset(new_buffer->buffer, 0xa5, current_page_fill); +#endif + DBUG_ASSERT(new_buffer->skipped_data < TRANSLOG_PAGE_SIZE); + } } old_buffer->next_buffer_offset= new_buffer->offset; translog_buffer_lock(new_buffer); @@ -7561,6 +7677,7 @@ void translog_flush_set_new_goal_and_wai { log_descriptor.next_pass_max_lsn= lsn; log_descriptor.max_lsn_requester= pthread_self(); + pthread_cond_broadcast(&log_descriptor.new_goal_cond); } while (flush_no == log_descriptor.flush_no) { @@ -7572,67 +7689,79 @@ void translog_flush_set_new_goal_and_wai /** - @brief Flush the log up to given LSN (included) + @brief sync() range of files (inclusive) and directory (by request) - @param lsn log record serial number up to which (inclusive) - the log has to be flushed + @param min min internal file number to flush + @param max max internal file number to flush + @param sync_dir need sync directory - @return Operation status + return Operation status @retval 0 OK @retval 1 Error - */ -my_bool translog_flush(TRANSLOG_ADDRESS lsn) +static my_bool translog_sync_files(uint32 min, uint32 max, + my_bool sync_dir) { - LSN sent_to_disk= LSN_IMPOSSIBLE; - TRANSLOG_ADDRESS flush_horizon; - uint fn, i; - dirty_buffer_mask_t dirty_buffer_mask; - uint8 last_buffer_no, start_buffer_no; + uint fn; my_bool rc= 0; - DBUG_ENTER("translog_flush"); - DBUG_PRINT("enter", ("Flush up to LSN: (%lu,0x%lx)", LSN_IN_PARTS(lsn))); - DBUG_ASSERT(translog_status == TRANSLOG_OK || - translog_status == TRANSLOG_READONLY); - LINT_INIT(sent_to_disk); - - pthread_mutex_lock(&log_descriptor.log_flush_lock); - DBUG_PRINT("info", ("Everything is flushed up to (%lu,0x%lx)", - LSN_IN_PARTS(log_descriptor.flushed))); - if (cmp_translog_addr(log_descriptor.flushed, lsn) >= 0) + ulonglong flush_interval; + DBUG_ENTER("translog_sync_files"); + DBUG_PRINT("info", ("min: %lu max: %lu sync dir: %d", + (ulong) min, (ulong) max, (int) sync_dir)); + DBUG_ASSERT(min <= max); + + flush_interval= group_commit_wait; + if (flush_interval) + flush_start= my_micro_time(); + for (fn= min; fn <= max; fn++) { - pthread_mutex_unlock(&log_descriptor.log_flush_lock); - DBUG_RETURN(0); - } - if (log_descriptor.flush_in_progress) - { - translog_flush_set_new_goal_and_wait(lsn); - if (!pthread_equal(log_descriptor.max_lsn_requester, pthread_self())) + TRANSLOG_FILE *file= get_logfile_by_number(fn); + DBUG_ASSERT(file != NULL); + if (!file->is_sync) { - /* fix lsn if it was horizon */ - if (cmp_translog_addr(lsn, log_descriptor.bc.buffer->last_lsn) > 0) - lsn= BUFFER_MAX_LSN(log_descriptor.bc.buffer); - translog_flush_wait_for_end(lsn); - pthread_mutex_unlock(&log_descriptor.log_flush_lock); - DBUG_RETURN(0); + if (my_sync(file->handler.file, MYF(MY_WME))) + { + rc= 1; + translog_stop_writing(); + DBUG_RETURN(rc); + } + translog_syncs++; + file->is_sync= 1; } - log_descriptor.next_pass_max_lsn= LSN_IMPOSSIBLE; } - log_descriptor.flush_in_progress= 1; - flush_horizon= log_descriptor.previous_flush_horizon; - DBUG_PRINT("info", ("flush_in_progress is set")); - pthread_mutex_unlock(&log_descriptor.log_flush_lock); - translog_lock(); - if (log_descriptor.is_everything_flushed) + if (sync_dir) { - DBUG_PRINT("info", ("everything is flushed")); - rc= (translog_status == TRANSLOG_READONLY); - translog_unlock(); - goto out; + if (!(rc= sync_dir(log_descriptor.directory_fd, + MYF(MY_WME | MY_IGNORE_BADFD)))) + translog_syncs++; } + DBUG_RETURN(rc); +} + + +/* + @brief Flushes buffers with LSNs in them less or equal address <lsn> + + @param lsn address up to which all LSNs should be flushed, + can be reset to real last LSN address + @parem sent_to_disk returns 'sent to disk' position + @param flush_horizon returns horizon of the flush + + @note About terminology see comment to translog_flush(). +*/ + +void translog_flush_buffers(TRANSLOG_ADDRESS *lsn, + TRANSLOG_ADDRESS *sent_to_disk, + TRANSLOG_ADDRESS *flush_horizon) +{ + dirty_buffer_mask_t dirty_buffer_mask; + uint i; + uint8 last_buffer_no, start_buffer_no; + DBUG_ENTER("translog_flush_buffers"); + /* We will recheck information when will lock buffers one by one so we can use unprotected read here (this is just for @@ -7656,15 +7785,15 @@ my_bool translog_flush(TRANSLOG_ADDRESS /* if LSN up to which we have to flush bigger then maximum LSN of previous buffer and at least one LSN was saved in the current buffer (last_lsn != - LSN_IMPOSSIBLE) then we better finish the current buffer. + LSN_IMPOSSIBLE) then we have to close the current buffer. */ - if (cmp_translog_addr(lsn, log_descriptor.bc.buffer->prev_last_lsn) > 0 && + if (cmp_translog_addr(*lsn, log_descriptor.bc.buffer->prev_last_lsn) > 0 && log_descriptor.bc.buffer->last_lsn != LSN_IMPOSSIBLE) { struct st_translog_buffer *buffer= log_descriptor.bc.buffer; - lsn= log_descriptor.bc.buffer->last_lsn; /* fix lsn if it was horizon */ + *lsn= log_descriptor.bc.buffer->last_lsn; /* fix lsn if it was horizon */ DBUG_PRINT("info", ("LSN to flush fixed to last lsn: (%lu,0x%lx)", - LSN_IN_PARTS(log_descriptor.bc.buffer->last_lsn))); + LSN_IN_PARTS(log_descriptor.bc.buffer->last_lsn))); last_buffer_no= log_descriptor.bc.buffer_no; log_descriptor.is_everything_flushed= 1; translog_force_current_buffer_to_finish(); @@ -7676,8 +7805,10 @@ my_bool translog_flush(TRANSLOG_ADDRESS TRANSLOG_BUFFERS_NO); translog_unlock(); } - sent_to_disk= translog_get_sent_to_disk(); - if (cmp_translog_addr(lsn, sent_to_disk) > 0) + + /* flush buffers */ + *sent_to_disk= translog_get_sent_to_disk(); + if (cmp_translog_addr(*lsn, *sent_to_disk) > 0) { DBUG_PRINT("info", ("Start buffer #: %u last buffer #: %u", @@ -7697,53 +7828,237 @@ my_bool translog_flush(TRANSLOG_ADDRESS LSN_IN_PARTS(buffer->last_lsn), (buffer->file ? "dirty" : "closed"))); - if (buffer->prev_last_lsn <= lsn && + if (buffer->prev_last_lsn <= *lsn && buffer->file != NULL) { - DBUG_ASSERT(flush_horizon <= buffer->offset + buffer->size); - flush_horizon= buffer->offset + buffer->size; + DBUG_ASSERT(*flush_horizon <= buffer->offset + buffer->size); + *flush_horizon= (buffer->pre_force_close_horizon != LSN_IMPOSSIBLE ? + buffer->pre_force_close_horizon : + buffer->offset + buffer->size); + /* pre_force_close_horizon is reset during new buffer start */ + DBUG_PRINT("info", ("flush_horizon: (%lu,0x%lx)", + LSN_IN_PARTS(*flush_horizon))); + DBUG_ASSERT(*flush_horizon <= log_descriptor.horizon); + translog_buffer_flush(buffer); } translog_buffer_unlock(buffer); i= (i + 1) % TRANSLOG_BUFFERS_NO; } while (i != last_buffer_no); - sent_to_disk= translog_get_sent_to_disk(); + *sent_to_disk= translog_get_sent_to_disk(); + } + + DBUG_VOID_RETURN; +} + +/** + @brief Flush the log up to given LSN (included) + + @param lsn log record serial number up to which (inclusive) + the log has to be flushed + + @return Operation status + @retval 0 OK + @retval 1 Error + + @note + + - Non group commit logic: Commits made in passes. Thread which started + flush first is performing actual flush, other threads sets new goal (LSN) + of the next pass (if it is maximum) and waits for the pass end or just + wait for the pass end. + + - If hard group commit enabled and rate set to zero: + The first thread sends all changed buffers to disk. This is repeated + as long as there are new LSNs added. The process can not loop + forever because we have limited number of threads and they will wait + for the data to be synced. + Pseudo code: + + do + send changed buffers to disk + while new_goal + sync + + - If hard group commit switched ON and less than rate microseconds has + passed from last sync, then after buffers have been sent to disk + wait until rate microseconds has passed since last sync, do sync and return. + This ensures that if we call sync infrequently we don't do any waits. + + - If soft group commit enabled everything works as with 'non group commit' + but the thread doesn't do any real sync(). If rate is not zero the + sync() will be performed by a service thread with the given rate + when needed (new LSN appears). + + @note Terminology: + 'sent to disk' means written to disk but not sync()ed, + 'flushed' mean sent to disk and synced(). +*/ + +my_bool translog_flush(TRANSLOG_ADDRESS lsn) +{ + struct timespec abstime; + ulonglong flush_interval; + ulonglong time_spent; + LSN sent_to_disk= LSN_IMPOSSIBLE; + TRANSLOG_ADDRESS flush_horizon; + my_bool rc= 0; + my_bool hgroup_commit_at_start; + DBUG_ENTER("translog_flush"); + DBUG_PRINT("enter", ("Flush up to LSN: (%lu,0x%lx)", LSN_IN_PARTS(lsn))); + DBUG_ASSERT(translog_status == TRANSLOG_OK || + translog_status == TRANSLOG_READONLY); + LINT_INIT(sent_to_disk); + + pthread_mutex_lock(&log_descriptor.log_flush_lock); + DBUG_PRINT("info", ("Everything is flushed up to (%lu,0x%lx)", + LSN_IN_PARTS(log_descriptor.flushed))); + if (cmp_translog_addr(log_descriptor.flushed, lsn) >= 0) + + + { + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + DBUG_RETURN(0); } + if (log_descriptor.flush_in_progress) + { + translog_lock(); + /* fix lsn if it was horizon */ + if (cmp_translog_addr(lsn, log_descriptor.bc.buffer->last_lsn) > 0) + lsn= BUFFER_MAX_LSN(log_descriptor.bc.buffer); + translog_unlock(); + translog_flush_set_new_goal_and_wait(lsn); + if (!pthread_equal(log_descriptor.max_lsn_requester, pthread_self())) + { + /* + translog_flush_wait_for_end() release log_flush_lock while is + waiting then acquire it again + */ + translog_flush_wait_for_end(lsn); + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + DBUG_RETURN(0); + } + log_descriptor.next_pass_max_lsn= LSN_IMPOSSIBLE; + } + log_descriptor.flush_in_progress= 1; + flush_horizon= log_descriptor.previous_flush_horizon; + DBUG_PRINT("info", ("flush_in_progress is set, flush_horizon: (%lu,0x%lx)", + LSN_IN_PARTS(flush_horizon))); + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + + hgroup_commit_at_start= hard_group_commit; + if (hgroup_commit_at_start) + flush_interval= group_commit_wait * TRANSLOG_RATE_BASE; - /* sync files from previous flush till current one */ - for (fn= LSN_FILE_NO(log_descriptor.flushed); fn <= LSN_FILE_NO(lsn); fn++) + translog_lock(); + if (log_descriptor.is_everything_flushed) { - TRANSLOG_FILE *file= get_logfile_by_number(fn); - DBUG_ASSERT(file != NULL); - if (!file->is_sync) + DBUG_PRINT("info", ("everything is flushed")); + translog_unlock(); + pthread_mutex_lock(&log_descriptor.log_flush_lock); + goto out; + } + + for (;;) + { + /* Following function flushes buffers and makes translog_unlock() */ + translog_flush_buffers(&lsn, &sent_to_disk, &flush_horizon); + + if (!hgroup_commit_at_start) + break; /* flush pass is ended */ + +retest: + if (flush_interval != 0 && + (my_micro_time() - flush_start) >= flush_interval) + break; /* flush pass is ended */ + + pthread_mutex_lock(&log_descriptor.log_flush_lock); + if (log_descriptor.next_pass_max_lsn != LSN_IMPOSSIBLE) + { + /* take next goal */ + lsn= log_descriptor.next_pass_max_lsn; + log_descriptor.next_pass_max_lsn= LSN_IMPOSSIBLE; + /* prevent other thread from continue */ + log_descriptor.max_lsn_requester= pthread_self(); + DBUG_PRINT("info", ("flush took next goal: (%lu,0x%lx)", + LSN_IN_PARTS(lsn))); + } + else { - if (my_sync(file->handler.file, MYF(MY_WME))) + if (flush_interval == 0 || + (time_spent= (my_micro_time() - flush_start)) >= flush_interval) { - rc= 1; - translog_stop_writing(); - sent_to_disk= LSN_IMPOSSIBLE; - goto out; + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + break; } - file->is_sync= 1; + DBUG_PRINT("info", ("flush waits: %llu interval: %llu spent: %llu", + flush_interval - time_spent, + flush_interval, time_spent)); + /* wait time or next goal */ + set_timespec_nsec(abstime, flush_interval - time_spent); + pthread_cond_timedwait(&log_descriptor.new_goal_cond, + &log_descriptor.log_flush_lock, + &abstime); + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + DBUG_PRINT("info", ("retest conditions")); + goto retest; } + pthread_mutex_unlock(&log_descriptor.log_flush_lock); + + /* next flush pass */ + DBUG_PRINT("info", ("next flush pass")); + translog_lock(); } - if (sync_log_dir >= TRANSLOG_SYNC_DIR_ALWAYS && - (LSN_FILE_NO(log_descriptor.previous_flush_horizon) != - LSN_FILE_NO(flush_horizon) || - ((LSN_OFFSET(log_descriptor.previous_flush_horizon) - 1) / - TRANSLOG_PAGE_SIZE) != - ((LSN_OFFSET(flush_horizon) - 1) / TRANSLOG_PAGE_SIZE))) - rc|= sync_dir(log_descriptor.directory_fd, MYF(MY_WME | MY_IGNORE_BADFD)); + /* + sync() files from previous flush till current one + */ + if (!soft_sync || hgroup_commit_at_start) + { + if ((rc= + translog_sync_files(LSN_FILE_NO(log_descriptor.flushed), + LSN_FILE_NO(lsn), + sync_log_dir >= TRANSLOG_SYNC_DIR_ALWAYS && + (LSN_FILE_NO(log_descriptor. + previous_flush_horizon) != + LSN_FILE_NO(flush_horizon) || + (LSN_OFFSET(log_descriptor. + previous_flush_horizon) / + TRANSLOG_PAGE_SIZE) != + (LSN_OFFSET(flush_horizon) / + TRANSLOG_PAGE_SIZE))))) + { + sent_to_disk= LSN_IMPOSSIBLE; + pthread_mutex_lock(&log_descriptor.log_flush_lock); + goto out; + } + /* keep values for soft sync() and forced sync() actual */ + { + uint32 fileno= LSN_FILE_NO(lsn); + my_atomic_rwlock_wrlock(&soft_sync_rwl); + my_atomic_store32(&soft_sync_min, fileno); + my_atomic_store32(&soft_sync_max, fileno); + my_atomic_rwlock_wrunlock(&soft_sync_rwl); + } + } + else + { + my_atomic_rwlock_wrlock(&soft_sync_rwl); + my_atomic_store32(&soft_sync_max, LSN_FILE_NO(lsn)); + my_atomic_rwlock_wrunlock(&soft_sync_rwl); + } + + DBUG_ASSERT(flush_horizon <= log_descriptor.horizon); + + pthread_mutex_lock(&log_descriptor.log_flush_lock); log_descriptor.previous_flush_horizon= flush_horizon; out: - pthread_mutex_lock(&log_descriptor.log_flush_lock); if (sent_to_disk != LSN_IMPOSSIBLE) log_descriptor.flushed= sent_to_disk; log_descriptor.flush_in_progress= 0; log_descriptor.flush_no++; DBUG_PRINT("info", ("flush_in_progress is dropped")); - pthread_mutex_unlock(&log_descriptor.log_flush_lock);\ + pthread_mutex_unlock(&log_descriptor.log_flush_lock); pthread_cond_broadcast(&log_descriptor.log_flush_cond); DBUG_RETURN(rc); } @@ -8113,6 +8428,8 @@ LSN translog_first_theoretical_lsn() my_bool translog_purge(TRANSLOG_ADDRESS low) { uint32 last_need_file= LSN_FILE_NO(low); + uint32 min_unsync; + int soft; TRANSLOG_ADDRESS horizon= translog_get_horizon(); int rc= 0; DBUG_ENTER("translog_purge"); @@ -8120,12 +8437,23 @@ my_bool translog_purge(TRANSLOG_ADDRESS DBUG_ASSERT(translog_status == TRANSLOG_OK || translog_status == TRANSLOG_READONLY); + soft= soft_sync; + DBUG_PRINT("info", ("min_unsync: %lu", (ulong) min_unsync)); + if (soft && min_unsync < last_need_file) + { + last_need_file= min_unsync; + DBUG_PRINT("info", ("last_need_file set to %lu", (ulong)last_need_file)); + } + pthread_mutex_lock(&log_descriptor.purger_lock); + DBUG_PRINT("info", ("last_lsn_checked file: %lu:", + (ulong) log_descriptor.last_lsn_checked)); if (LSN_FILE_NO(log_descriptor.last_lsn_checked) < last_need_file) { uint32 i; uint32 min_file= translog_first_file(horizon, 1); DBUG_ASSERT(min_file != 0); /* log is already started */ + DBUG_PRINT("info", ("min_file: %lu:",(ulong) min_file)); for(i= min_file; i < last_need_file && rc == 0; i++) { LSN lsn= translog_get_file_max_lsn_stored(i); @@ -8356,6 +8684,153 @@ my_bool translog_log_debug_info(TRN *trn } + +/** + Sets soft sync mode + + @param mode TRUE if we need switch soft sync on else off +*/ + +void translog_soft_sync(my_bool mode) +{ + soft_sync= mode; +} + + +/** + Sets hard group commit + + @param mode TRUE if we need switch hard group commit on else off +*/ + +void translog_hard_group_commit(my_bool mode) +{ + hard_group_commit= mode; +} + + +/** + @brief forced log sync (used when we are switching modes) +*/ + +void translog_sync() +{ + uint32 max= get_current_logfile()->number; + uint32 min; + DBUG_ENTER("ma_translog_sync"); + + my_atomic_rwlock_rdlock(&soft_sync_rwl); + min= my_atomic_load32(&soft_sync_min); + my_atomic_rwlock_rdunlock(&soft_sync_rwl); + if (!min) + min= max; + + translog_sync_files(min, max, sync_log_dir >= TRANSLOG_SYNC_DIR_ALWAYS); + + DBUG_VOID_RETURN; +} + + +/** + @brief set rate for group commit + + @param rate rate to set. + + @note internally it stores interval in nanoseconds/TRANSLOG_RATE_BASE (to + fit into uint32) +*/ + +void translog_set_group_commit_rate(uint32 rate) +{ + DBUG_ENTER("translog_set_group_commit_rate"); + ulonglong wait_time; + if (rate) + { + wait_time= ((TRANSLOG_RATE_BASE * 1000000000ULL / rate + + TRANSLOG_RATE_BASE / 2) / + TRANSLOG_RATE_BASE); + if (wait_time == 0) + wait_time= 1; /* protection from getting special value */ + } + else + wait_time= 0; + group_commit_wait= wait_time; + DBUG_PRINT("info", ("rate: %lu wait: %llu", + (ulong)rate, (ulonglong)wait_time)); + DBUG_VOID_RETURN; +} + + +/** + @brief syncing service thread +*/ + +static pthread_handler_t +ma_soft_sync_background( void *arg __attribute__((unused))) +{ + + my_thread_init(); + { + DBUG_ENTER("ma_soft_sync_background"); + for(;;) + { + ulonglong prev_loop= my_micro_time(); + ulonglong time, sleep; + uint32 min, max; + my_atomic_rwlock_rdlock(&soft_sync_rwl); + min= my_atomic_load32(&soft_sync_min); + max= my_atomic_load32(&soft_sync_max); + my_atomic_store32(&soft_sync_min, max); + my_atomic_rwlock_rdunlock(&soft_sync_rwl); + + sleep= group_commit_wait * TRANSLOG_RATE_BASE; + translog_sync_files(min, max, FALSE); + time= my_micro_time() - prev_loop; + if (time > sleep) + sleep= 0; + else + sleep-= time; + if (my_service_thread_sleep(&soft_sync_control, sleep)) + break; + } + my_service_thread_signal_end(&soft_sync_control); + my_thread_end(); + DBUG_RETURN(0); + } +} + + +/** + @brief Starts syncing thread +*/ + +int translog_soft_sync_start(void) +{ + pthread_t th; + int res= 0; + DBUG_ENTER("translog_soft_sync_start"); + if (!(res= ma_service_thread_control_init(&soft_sync_control))) + if (!(res= pthread_create(&th, NULL, ma_soft_sync_background, NULL))) + soft_sync_control.status= THREAD_RUNNING; + DBUG_RETURN(res); +} + + +/** + @brief Stops syncing thread +*/ + +void translog_soft_sync_end(void) +{ + DBUG_ENTER("translog_soft_sync_end"); + if (soft_sync_control.inited) + { + ma_service_thread_control_end(&soft_sync_control); + } + DBUG_VOID_RETURN; +} + + #ifdef MARIA_DUMP_LOG #include <my_getopt.h> extern void translog_example_table_init(); === modified file 'storage/maria/ma_loghandler.h' --- a/storage/maria/ma_loghandler.h 2009-01-15 22:25:53 +0000 +++ b/storage/maria/ma_loghandler.h 2009-07-07 00:37:23 +0000 @@ -342,6 +342,14 @@ enum enum_translog_status TRANSLOG_SHUTDOWN /* going to shutdown the loghandler */ }; extern enum enum_translog_status translog_status; +extern ulonglong translog_syncs; /* Number of sync()s */ + +void translog_soft_sync(my_bool mode); +void translog_hard_group_commit(my_bool mode); +int translog_soft_sync_start(void); +void translog_soft_sync_end(void); +void translog_sync(); +void translog_set_group_commit_rate(uint32 rate); /* all the rest added because of recovery; should we make @@ -441,6 +449,18 @@ extern LOG_DESC log_record_type_descript typedef enum { + TRANSLOG_GCOMMIT_NONE, + TRANSLOG_GCOMMIT_HARD, + TRANSLOG_GCOMMIT_SOFT +} enum_maria_group_commit; +extern ulong maria_group_commit; +extern ulong maria_group_commit_rate; +/** + group commit interval is TRANSLOG_RATE_BASE/<rate> seconds +*/ +#define TRANSLOG_RATE_BASE 100 +typedef enum +{ TRANSLOG_PURGE_IMMIDIATE, TRANSLOG_PURGE_EXTERNAL, TRANSLOG_PURGE_ONDEMAND

2 1

[Maria-developers] Rewrite of the free documentation to make it about MariaDB, not MySQL
by Henrik Ingo 03 Nov '09

03 Nov '09

Daniel I glanced through some of the commits today and noticed Kristian committing the new version of MySQL 5.1.38 documentation (man pages and other free docs). The docs seem to include lots of references to where you can download MySQL rpm's, where you can buy MySQL support, Suns support lifecycle policies, etc. As part of your documentation work, could you be in touch with Kristian and find out how you could help in cleaning that up. Meaning: Either remove such sections, or replace with relevant MariaDB information. While at it, I'd like to review the Readme file of MariaDB. Could you help me with that (ie extract it from bzr and work with me to rewrite it - so that I can just participate by email). The Readme file we should fix before GA release. The other docs are only semi-urgent. henrik -- email: henrik.ingo(a)avoinelama.fi tel: +358-40-5697354 www: www.avoinelama.fi/~hingo book: www.openlife.cc

3 5

[Maria-developers] WL40: Notes/questions
by Alexi1952 12 Oct '09

12 Oct '09

With WL36 and WL40 we have: mysqlbinlog options: --database=db --rewrite-db=db_from->db_to --do-table=db.tbl --ignore-table=db.tbl --wild-do-table=pattern.pattern --wild-ignore-table=pattern.pattern replication options: --replicate-rewrite-db=db_from->db_to --replicate-do-db=db --replicate-ignore-db=db --replicate-do-table=tbl --replicate-ignore-table=tbl --replicate-wild-do-table=db.tbl --replicate-wild-ignore-table=db.tbl 1. In mysqlbinlog we do not have --do_db and --ignore_db options. Does it mean that instead it is supposed to use: --replicate-wild-do-table=db.% --replicate-wild-ignore-table=db.% respectively? Compared with other options, --database option looks like a "foreign body" : - contrary to other options, it allows to specify only one database (with multiple --database's only the last one is used); - having with --database an analog of do_db, we have no similar analog of ignore_db. 2. In replication two functions are used for filtering databases: - db_ok(const char* db) which matches db only with do-db and ignore-db rules; - db_ok_with_wild_table(const char* db) which matches db only with wild-do-table=db.% and wild-ignore-table=db.% rules. This function is applied only to CREATE DB, DROP DB, and ALTER DATABASE statements. In mysqlbinlog, should we follow the same scheme, namely: - db_ok() for matching db with --database option only; - db_ok_with_wild_table() for statements listed above? 3. According to replication filtering rules, --replicate-rewrite_db is always done _before_ other --replicate-* rules are tested; see explanation for --replicate-rewrite-db in RefMan (16.1.3.3. Replication Slave Options and Variables), or the following piece of code in og_event.cc: int Table_map_log_event::do_apply_event(Relay_log_info const* rli) { RPL_TABLE_LIST* table_list; ... strmov(table_list->db, rpl_filter->get_rewrite_db(m_dbnam, &dummy_len)); ... if (...!rpl_filter->db_ok(table_list->db) ...) ... } And what about --database + rewrite-db for mysqlbinlog? If we mean to output only database xxx with renaming it to yyy, should we use (1) mysqlbinlog --database=xxx --rewrite-db=xxx->yyy or (2) mysqlbinlog --database=yyy --rewrite-db=xxx->yyy In current WL36 design it is supposed that (1) should be used (surely, this can easily be redesigned). But this becomes confused with using of --wild-do-table + replication filtering rules, for which we should use: (3) --wild-do-table=yyy.% --rewrite-db=xxx->yyy

4 5

[Maria-developers] Rev 2731: BUG#39249 Maria:query cache returns out of date results in file:///Users/bell/maria/bzr/work-maria-5.1-querycache/
by sanja＠askmonty.org 05 Oct '09

05 Oct '09

At file:///Users/bell/maria/bzr/work-maria-5.1-querycache/ ------------------------------------------------------------ revno: 2731 revision-id: sanja(a)askmonty.org-20090923065953-l23fsjfy8lx3hkm4 parent: knielsen(a)knielsen-hq.org-20090907131358-6bq1e3qfcgi30hu8 committer: sanja(a)askmonty.org branch nick: work-maria-5.1-querycache timestamp: Wed 2009-09-23 09:59:53 +0300 message: BUG#39249 Maria:query cache returns out of date results BUG#41098 Query Cache returns wrong result with concurent insert === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-05-11 15:31:30 +0000 +++ b/BUILD/SETUP.sh 2009-09-23 06:59:53 +0000 @@ -172,6 +172,7 @@ max_no_embedded_configs="$SSL_LIBRARY --with-plugins=max" +max_no_qc_configs="$SSL_LIBRARY --with-plugins=max --without-query-cache" max_no_ndb_configs="$SSL_LIBRARY --with-plugins=max-no-ndb --with-embedded-server --with-libevent" max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server -with-libevent" # Disable NDB in maria max builds === added file 'BUILD/compile-pentium-debug-max-no-qc' --- a/BUILD/compile-pentium-debug-max-no-qc 1970-01-01 00:00:00 +0000 +++ b/BUILD/compile-pentium-debug-max-no-qc 2009-09-23 06:59:53 +0000 @@ -0,0 +1,9 @@ +#! /bin/sh + +path=`dirname $0` +. "$path/SETUP.sh" + +extra_flags="$pentium_cflags $debug_cflags" +extra_configs="$pentium_configs $debug_configs $max_no_qc_configs" + +. "$path/FINISH.sh" === modified file 'sql/mysql_priv.h' --- a/sql/mysql_priv.h 2009-09-03 14:56:46 +0000 +++ b/sql/mysql_priv.h 2009-09-23 06:59:53 +0000 @@ -933,7 +933,6 @@ #define query_cache_abort(A) #define query_cache_end_of_result(A) -#define query_cache_invalidate_by_MyISAM_filename_ref NULL #define query_cache_maybe_disabled(T) 1 #define query_cache_is_cacheable_query(L) 0 #endif /*HAVE_QUERY_CACHE*/ === modified file 'storage/maria/ha_maria.cc' --- a/storage/maria/ha_maria.cc 2009-06-29 21:03:30 +0000 +++ b/storage/maria/ha_maria.cc 2009-09-23 06:59:53 +0000 @@ -28,6 +28,7 @@ #include <my_bit.h> #include "ha_maria.h" #include "trnman_public.h" +#include "trnman.h" C_MODE_START #include "maria_def.h" @@ -907,6 +908,8 @@ if (!(file= maria_open(name, mode, test_if_locked | HA_OPEN_FROM_SQL_LAYER))) return (my_errno ? my_errno : -1); + file->s->chst_invalidator= query_cache_invalidate_by_MyISAM_filename_ref; + if (test_if_locked & (HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_TMP_TABLE)) VOID(maria_extra(file, HA_EXTRA_NO_WAIT_LOCK, 0)); @@ -3227,6 +3230,9 @@ */ *engine_data= 0; + if (file->s->now_transactional && file->s->have_versioning) + return (file->trn->trid != file->s->state.last_change_trn); + /* If a concurrent INSERT has happened just before the currently processed SELECT statement, the total size of the table is unknown. === modified file 'storage/maria/ma_delete.c' --- a/storage/maria/ma_delete.c 2009-01-12 11:12:00 +0000 +++ b/storage/maria/ma_delete.c 2009-09-23 06:59:53 +0000 @@ -118,13 +118,6 @@ mi_sizestore(lastpos, info->cur_row.lastpos); VOID(_ma_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); allow_break(); /* Allow SIGHUP & SIGINT */ - if (info->invalidator != 0) - { - DBUG_PRINT("info", ("invalidator... '%s' (delete)", - share->open_file_name.str)); - (*info->invalidator)(share->open_file_name.str); - info->invalidator=0; - } DBUG_RETURN(0); err: === modified file 'storage/maria/ma_locking.c' --- a/storage/maria/ma_locking.c 2009-02-09 21:52:42 +0000 +++ b/storage/maria/ma_locking.c 2009-09-23 06:59:53 +0000 @@ -214,7 +214,6 @@ VOID(_ma_test_if_changed(info)); info->lock_type=lock_type; - info->invalidator=share->invalidator; share->w_locks++; share->tot_locks++; break; @@ -269,7 +268,6 @@ } if (check_keybuffer) VOID(_ma_test_if_changed(info)); - info->invalidator=share->invalidator; } else if (lock_type == F_WRLCK && info->lock_type == F_RDLCK) { === modified file 'storage/maria/ma_state.c' --- a/storage/maria/ma_state.c 2008-12-27 02:05:16 +0000 +++ b/storage/maria/ma_state.c 2009-09-23 06:59:53 +0000 @@ -318,6 +318,13 @@ DBUG_ASSERT(!info->s->base.born_transactional); share->state.state= *info->state; info->state= &share->state.state; +#ifdef HAVE_QUERY_CACHE + DBUG_PRINT("info", ("invalidator... '%s' (status update)", + info->s->data_file_name.str)); + DBUG_ASSERT(info->s->chst_invalidator != NULL); + (*info->s->chst_invalidator)((const char *)info->s->data_file_name.str); +#endif + } info->append_insert_at_end= 0; } @@ -469,6 +476,8 @@ tables->state_start.checksum); history->trid= trn->commit_trid; + share->state.last_change_trn= trn->commit_trid; + if (history->next) { /* Remove not visible states */ === modified file 'storage/maria/ma_update.c' --- a/storage/maria/ma_update.c 2008-10-31 23:14:58 +0000 +++ b/storage/maria/ma_update.c 2009-09-23 06:59:53 +0000 @@ -187,13 +187,6 @@ */ VOID(_ma_writeinfo(info, WRITEINFO_UPDATE_KEYFILE)); allow_break(); /* Allow SIGHUP & SIGINT */ - if (info->invalidator != 0) - { - DBUG_PRINT("info", ("invalidator... '%s' (update)", - share->open_file_name.str)); - (*info->invalidator)(share->open_file_name.str); - info->invalidator=0; - } DBUG_RETURN(0); err: === modified file 'storage/maria/ma_write.c' --- a/storage/maria/ma_write.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_write.c 2009-09-23 06:59:53 +0000 @@ -295,13 +295,6 @@ info->cur_row.lastpos= filepos; VOID(_ma_writeinfo(info, WRITEINFO_UPDATE_KEYFILE)); - if (info->invalidator != 0) - { - DBUG_PRINT("info", ("invalidator... '%s' (update)", - share->open_file_name.str)); - (*info->invalidator)(share->open_file_name.str); - info->invalidator=0; - } /* Update status of the table. We need to do so after each row write === modified file 'storage/maria/maria_def.h' --- a/storage/maria/maria_def.h 2009-02-19 09:01:25 +0000 +++ b/storage/maria/maria_def.h 2009-09-23 06:59:53 +0000 @@ -83,6 +83,7 @@ pgcache_page_no_t first_bitmap_with_space; ulonglong auto_increment; TrID create_trid; /* Minum trid for file */ + TrID last_change_trn; /* selfdescriptive */ ulong update_count; /* Updated for each write lock */ ulong status; double *rec_per_key_part; @@ -337,7 +338,7 @@ /* Mapings to read/write the data file */ size_t (*file_read)(MARIA_HA *, uchar *, size_t, my_off_t, myf); size_t (*file_write)(MARIA_HA *, const uchar *, size_t, my_off_t, myf); - invalidator_by_filename invalidator; /* query cache invalidator */ + invalidator_by_filename chst_invalidator; /* query cache invalidator */ my_off_t key_del_current; /* delete links for index pages */ ulong this_process; /* processid */ ulong last_process; /* For table-change-check */ @@ -507,7 +508,6 @@ uint int_nod_flag; /* -""- */ uint32 int_keytree_version; /* -""- */ int (*read_record)(MARIA_HA *, uchar*, MARIA_RECORD_POS); - invalidator_by_filename invalidator; /* query cache invalidator */ ulonglong last_auto_increment; /* auto value at start of statement */ ulong this_unique; /* uniq filenumber or thread */ ulong last_unique; /* last unique number */ === modified file 'storage/myisam/ha_myisam.cc' --- a/storage/myisam/ha_myisam.cc 2009-06-29 21:03:30 +0000 +++ b/storage/myisam/ha_myisam.cc 2009-09-23 06:59:53 +0000 @@ -660,6 +660,9 @@ if (!(file=mi_open(name, mode, test_if_locked | HA_OPEN_FROM_SQL_LAYER))) return (my_errno ? my_errno : -1); + + file->s->chst_invalidator= query_cache_invalidate_by_MyISAM_filename_ref; + if (!table->s->tmp_table) /* No need to perform a check for tmp table */ { if ((my_errno= table2myisam(table, &keyinfo, &recinfo, &recs))) === modified file 'storage/myisam/mi_locking.c' --- a/storage/myisam/mi_locking.c 2009-09-03 14:05:38 +0000 +++ b/storage/myisam/mi_locking.c 2009-09-23 06:59:53 +0000 @@ -329,6 +329,12 @@ #endif info->s->state.state= *info->state; info->state= &info->s->state.state; +#ifdef HAVE_QUERY_CACHE + DBUG_PRINT("info", ("invalidator... '%s' (status update)", + info->filename)); + DBUG_ASSERT(info->s->chst_invalidator != NULL); + (*info->s->chst_invalidator)((const char *)info->filename); +#endif } info->append_insert_at_end= 0; === modified file 'storage/myisam/myisamdef.h' --- a/storage/myisam/myisamdef.h 2009-04-25 09:04:38 +0000 +++ b/storage/myisam/myisamdef.h 2009-09-23 06:59:53 +0000 @@ -190,7 +190,10 @@ const uchar *record, my_off_t pos); size_t (*file_read) (MI_INFO *, uchar *, size_t, my_off_t, myf); size_t (*file_write) (MI_INFO *, const uchar *, size_t, my_off_t, myf); - invalidator_by_filename invalidator; /* query cache invalidator */ + /* query cache invalidator for merged tables */ + invalidator_by_filename invalidator; + /* query cache invalidator for changing state */ + invalidator_by_filename chst_invalidator; ulong this_process; /* processid */ ulong last_process; /* For table-change-check */ ulong last_version; /* Version on start */

2 1

Re: [Maria-developers] automake fixes for maria 5.1
by Kristian Nielsen 30 Sep '09

30 Sep '09

Peter Lieverdink <peter(a)openquery.com> writes: > I've been working on creating packaging scripts for Maria on Debian/Ubuntu and came across a few issues with the automake scripts. They are basically all related to the use case where srcdir != builddir - as is the case in the debian foo. > > My changes are on launchpad; in https://code.launchpad.net/~ourdelta-core/maria/maria-5.1-automakefixes > > I've submitted this branch for merging back into maria/maria-5.1-merge. Thanks a lot! I pushed the merge into maria-5.1-merge. - Kristian.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2743)
by knielsen＠knielsen-hq.org 30 Sep '09

30 Sep '09

#At lp:maria 2743 knielsen(a)knielsen-hq.org 2009-09-30 Add environment variable MTR_EXTRA_OPTIONS to influence `make test*` targets. This allows to eg use --mem or --parallel for better performance on big boxes. modified: Makefile.am === modified file 'Makefile.am' --- a/Makefile.am 2009-09-07 20:50:10 +0000 +++ b/Makefile.am 2009-09-30 12:45:03 +0000 @@ -81,6 +81,9 @@ tags: # environment variable MTR_BUILD_THREAD. The script "mysql-test-run" # will then calculate the various port numbers it needs from this, # making sure each user use different ports. +# +# Set MTR_EXTRA_OPTIONS to add custom options for mysql-test-run, +# like MTR_EXTRA_OPTIONS"--parallel=3 --mem". test-unit: cd unittest; $(MAKE) test @@ -90,29 +93,29 @@ test-unit-big: test-ps: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) --ps-protocol --mysqld=--binlog-format=mixed + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) --ps-protocol --mysqld=--binlog-format=mixed test-nr: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) --mysqld=--binlog-format=row + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) --mysqld=--binlog-format=row test-pr: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) $(mem) --ps-protocol --mysqld=--binlog-format=row #@libevent_test_option@ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) $(mem) --ps-protocol --mysqld=--binlog-format=row #@libevent_test_option@ test-ns: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) $(mem) --mysqld=--binlog-format=mixed + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) $(mem) --mysqld=--binlog-format=mixed test-binlog-statement: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) --mysqld=--binlog-format=statement + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) --mysqld=--binlog-format=statement # This code is duplicated in "test-bt", see the Changeset comment of 2007-Dec-07 test-embedded: if [ -e bin/mysqltest_embedded -o -e libmysqld/examples/mysqltest_embedded ] ; then \ cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=embedded --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=embedded --force --timer \ --embedded-server --skip-rpl --skip-ndbcluster ; \ else \ echo "no program found for 'embedded' tests - skipped testing" ; \ @@ -120,14 +123,14 @@ test-embedded: test-reprepare: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(force) $(mem) --ps-protocol \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(force) $(mem) --ps-protocol \ --mysqld=--debug=+d,reprepare_each_statement test: test-ns test-pr smoke: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --do-test=s + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --do-test=s test-full: test test-nr test-ps @@ -143,43 +146,43 @@ test-force-mem: test-bt: -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=normal --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=normal --force --timer \ --skip-ndbcluster --report-features -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=ps --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=ps --force --timer \ --skip-ndbcluster --ps-protocol -if [ -e bin/ndbd -o -e storage/ndb/src/kernel/ndbd ] ; then \ cd mysql-test ; \ MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=ndb+rpl_ndb+ps --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=ndb+rpl_ndb+ps --force --timer \ --ps-protocol --mysqld=--binlog-format=row --suite=ndb,rpl_ndb ; \ MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=ndb --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=ndb --force --timer \ --with-ndbcluster-only ; \ else \ echo "no program found for 'ndbcluster' tests - skipped testing" ; \ fi -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=funcs1+ps --ps-protocol --reorder --suite=funcs_1 + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=funcs1+ps --ps-protocol --reorder --suite=funcs_1 -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=funcs2 --suite=funcs_2 + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=funcs2 --suite=funcs_2 -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=partitions --suite=parts + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=partitions --suite=parts -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=stress --suite=stress + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=stress --suite=stress -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=jp --suite=jp + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=jp --suite=jp -if [ -d mysql-test/suite/nist ] ; then \ cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=nist --force --suite=nist ; \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=nist --force --suite=nist ; \ fi -if [ -d mysql-test/suite/nist ] ; then \ cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=nist+ps --force --suite=nist --ps-protocol ; \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=nist+ps --force --suite=nist --ps-protocol ; \ fi -if [ -e bin/mysqltest_embedded -o -e libmysqld/examples/mysqltest_embedded ] ; then \ cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=embedded --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=embedded --force --timer \ --embedded-server --skip-rpl --skip-ndbcluster ; \ else \ echo "no program found for 'embedded' tests - skipped testing" ; \ @@ -187,22 +190,22 @@ test-bt: test-bt-fast: -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=ps --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=ps --force --timer \ --skip-ndbcluster --ps-protocol --report-features -if [ -e bin/ndbd -o -e storage/ndb/src/kernel/ndbd ] ; then \ cd mysql-test ; \ MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=ndb --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=ndb --force --timer \ --with-ndbcluster-only ; \ else \ echo "no program found for 'ndbcluster' tests - skipped testing" ; \ fi -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=stress --suite=stress + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --comment=stress --suite=stress test-bt-debug: -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --comment=debug --force --timer \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --comment=debug --force --timer \ --skip-ndbcluster --skip-rpl --report-features test-bt-debug-fast: @@ -216,32 +219,32 @@ test-force-full-pl: test-force-full test-ext-funcs: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --force --reorder --suite=funcs_1 ; \ - @PERL@ ./mysql-test-run.pl --force --suite=funcs_2 + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --reorder --suite=funcs_1 ; \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --suite=funcs_2 test-ext-rpl: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --force --suite=rpl + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --suite=rpl test-ext-partitions: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --force --suite=parts + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --suite=parts test-ext-jp: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --force --suite=jp + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --suite=jp test-ext-stress: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl --force --big-test --suite=stress + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) --force --big-test --suite=stress test-ext: test-ext-funcs test-ext-rpl test-ext-partitions test-ext-jp test-ext-stress test-fast: cd mysql-test ; \ - @PERL@ ./mysql-test-run.pl $(subset) --force --skip-ndb --skip-innodb --skip-im --skip-rpl ; \ - @PERL@ ./mysql-test-run.pl $(subset) --force --suite=funcs_1 --do-test=myisam ; \ - @PERL@ ./mysql-test-run.pl $(subset) --force --suite=stress --do-test=ddl_myisam + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(subset) --force --skip-ndb --skip-innodb --skip-im --skip-rpl ; \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(subset) --force --suite=funcs_1 --do-test=myisam ; \ + @PERL@ ./mysql-test-run.pl $(MTR_EXTRA_OPTIONS) $(subset) --force --suite=stress --do-test=ddl_myisam test-fast-view: $(MAKE) subset=--view-protocol test-fast

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2742)
by knielsen＠knielsen-hq.org 30 Sep '09

30 Sep '09

#At lp:maria 2742 knielsen(a)knielsen-hq.org 2009-09-30 [merge] Merge autotools fixes to be able to build with builddir != srcdir. Author: Peter Lieverdink modified: config/ac-macros/libevent.m4 configure.in extra/libevent/Makefile.am === modified file 'config/ac-macros/libevent.m4' --- a/config/ac-macros/libevent.m4 2009-03-12 22:27:35 +0000 +++ b/config/ac-macros/libevent.m4 2009-09-29 23:36:15 +0000 @@ -13,7 +13,7 @@ AC_DEFUN([MYSQL_USE_BUNDLED_LIBEVENT], [ AC_SUBST([libevent_dir]) libevent_libs="\$(top_builddir)/extra/libevent/libevent.a" - libevent_includes="-I\$(top_builddir)/extra/libevent" + libevent_includes="-I\$(top_srcdir)/extra/libevent" libevent_test_option="--mysqld=--thread-handling=pool-of-threads" AC_SUBST(libevent_libs) AC_SUBST(libevent_includes) === modified file 'configure.in' --- a/configure.in 2009-09-15 11:55:37 +0000 +++ b/configure.in 2009-09-29 22:50:10 +0000 @@ -2697,7 +2697,7 @@ case $SYSTEM_TYPE in fi # if there is no readline, but we want to build with readline, we fail - if [test "$want_to_use_readline" = "yes"] && [test ! -d "./cmd-line-utils/readline"] + if [test "$want_to_use_readline" = "yes"] && [test ! -d "$srcdir/cmd-line-utils/readline"] then AC_MSG_ERROR([This commercially licensed MySQL source package can't be built with libreadline. Please use --with-libedit to use === modified file 'extra/libevent/Makefile.am' --- a/extra/libevent/Makefile.am 2009-03-12 22:27:35 +0000 +++ b/extra/libevent/Makefile.am 2009-09-29 23:00:57 +0000 @@ -22,7 +22,7 @@ include_HEADERS = event.h evutil.h event BUILT_SOURCES = event-config.h -event-config.h: $(top_srcdir)/include/config.h +event-config.h: $(top_builddir)/include/config.h echo '/* event-config.h' > $@ echo ' * Generated by autoconf; post-processed by libevent.' >> $@ echo ' * Do not edit this file.' >> $@ @@ -33,7 +33,7 @@ event-config.h: $(top_srcdir)/include/co sed -e 's/#define /#define _EVENT_/' \ -e 's/#undef /#undef _EVENT_/' \ - -e 's/#ifndef /#ifndef _EVENT_/' < $(top_srcdir)/include/config.h >> $@ + -e 's/#ifndef /#ifndef _EVENT_/' < $(top_builddir)/include/config.h >> $@ echo "#endif" >> $@ AM_CPPFLAGS = -Icompat -I$(top_srcdir)/include

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2741)
by knielsen＠knielsen-hq.org 30 Sep '09

30 Sep '09

#At lp:maria 2741 knielsen(a)knielsen-hq.org 2009-09-30 Completely disable the innodb plugin. Don't ever build it, and don't include it in source tarballs. (we keep the sources in bzr to not get conflicts on every upstream change). renamed: storage/innodb_plugin/plug.in => storage/innodb_plugin/plug.in.disabled === renamed file 'storage/innodb_plugin/plug.in' => 'storage/innodb_plugin/plug.in.disabled'

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2740) Bug#46010
by knielsen＠knielsen-hq.org 30 Sep '09

30 Sep '09

#At lp:maria 2740 knielsen(a)knielsen-hq.org 2009-09-30 Apply patch from Bug#46010. This is needed to fix failures seen in Buildbot (main.ctype_gbk_binlog). Problem is that some tests did not RESET MASTER, causing them to pick up random binlog files depending on prior tests. modified: mysql-test/r/ctype_gbk_binlog.result mysql-test/r/mysqlbinlog-cp932.result mysql-test/suite/binlog/r/binlog_incident.result mysql-test/suite/binlog/r/binlog_tmp_table.result mysql-test/suite/binlog/t/binlog_incident.test mysql-test/suite/binlog/t/binlog_tmp_table.test mysql-test/t/ctype_gbk_binlog.test mysql-test/t/mysqlbinlog-cp932.test === modified file 'mysql-test/r/ctype_gbk_binlog.result' --- a/mysql-test/r/ctype_gbk_binlog.result 2009-07-07 14:18:44 +0000 +++ b/mysql-test/r/ctype_gbk_binlog.result 2009-09-30 06:35:31 +0000 @@ -1,3 +1,4 @@ +RESET MASTER; SET NAMES gbk; CREATE TABLE t1 ( f1 BLOB === modified file 'mysql-test/r/mysqlbinlog-cp932.result' --- a/mysql-test/r/mysqlbinlog-cp932.result 2007-02-19 10:57:06 +0000 +++ b/mysql-test/r/mysqlbinlog-cp932.result 2009-09-30 06:35:31 +0000 @@ -1,4 +1,4 @@ -flush logs; +RESET MASTER; create table t3 (f text character set utf8); create table t4 (f text character set cp932); flush logs; === modified file 'mysql-test/suite/binlog/r/binlog_incident.result' --- a/mysql-test/suite/binlog/r/binlog_incident.result 2009-05-11 11:32:38 +0000 +++ b/mysql-test/suite/binlog/r/binlog_incident.result 2009-09-30 06:35:31 +0000 @@ -1,3 +1,4 @@ +RESET MASTER; CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (1),(2),(3); SELECT * FROM t1; === modified file 'mysql-test/suite/binlog/r/binlog_tmp_table.result' --- a/mysql-test/suite/binlog/r/binlog_tmp_table.result 2009-01-28 14:35:12 +0000 +++ b/mysql-test/suite/binlog/r/binlog_tmp_table.result 2009-09-30 06:35:31 +0000 @@ -1,3 +1,4 @@ +RESET MASTER; create table foo (a int); flush logs; create temporary table tmp1_foo like foo; === modified file 'mysql-test/suite/binlog/t/binlog_incident.test' --- a/mysql-test/suite/binlog/t/binlog_incident.test 2009-05-11 11:32:38 +0000 +++ b/mysql-test/suite/binlog/t/binlog_incident.test 2009-09-30 06:35:31 +0000 @@ -6,6 +6,7 @@ source include/have_log_bin.inc; source include/have_debug.inc; let $MYSQLD_DATADIR= `select @@datadir`; +RESET MASTER; CREATE TABLE t1 (a INT); @@ -24,4 +25,4 @@ exec $MYSQL_BINLOG --start-position=106 eval SELECT cont LIKE '%RELOAD DATABASE; # Shall generate syntax error%' AS `Contain RELOAD DATABASE` FROM (SELECT load_file('$MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql') AS cont) AS tbl; --enable_query_log -remove_file $MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; \ No newline at end of file +remove_file $MYSQLTEST_VARDIR/tmp/binlog_incident-bug44442.sql; === modified file 'mysql-test/suite/binlog/t/binlog_tmp_table.test' --- a/mysql-test/suite/binlog/t/binlog_tmp_table.test 2009-01-31 21:47:50 +0000 +++ b/mysql-test/suite/binlog/t/binlog_tmp_table.test 2009-09-30 06:35:31 +0000 @@ -30,6 +30,7 @@ source include/have_binlog_format_mixed_ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (master1,127.0.0.1,root,,test,$MASTER_MYPORT,); +RESET MASTER; create table foo (a int); === modified file 'mysql-test/t/ctype_gbk_binlog.test' --- a/mysql-test/t/ctype_gbk_binlog.test 2009-07-07 14:18:44 +0000 +++ b/mysql-test/t/ctype_gbk_binlog.test 2009-09-30 06:35:31 +0000 @@ -1,6 +1,7 @@ -- source include/have_binlog_format_mixed_or_statement.inc -- source include/have_gbk.inc +RESET MASTER; SET NAMES gbk; --character_set gbk === modified file 'mysql-test/t/mysqlbinlog-cp932.test' --- a/mysql-test/t/mysqlbinlog-cp932.test 2007-12-12 17:19:24 +0000 +++ b/mysql-test/t/mysqlbinlog-cp932.test 2009-09-30 06:35:31 +0000 @@ -5,8 +5,9 @@ -- source include/have_cp932.inc -- source include/have_log_bin.inc +RESET MASTER; + # Bug#16217 (mysql client did not know how not switch its internal charset) -flush logs; create table t3 (f text character set utf8); create table t4 (f text character set cp932); --exec $MYSQL --default-character-set=utf8 test -e "insert into t3 values(_utf8'ソ')" @@ -14,7 +15,7 @@ create table t4 (f text character set cp flush logs; rename table t3 to t03, t4 to t04; let $MYSQLD_DATADIR= `select @@datadir`; ---exec $MYSQL_BINLOG --short-form $MYSQLD_DATADIR/master-bin.000002 | $MYSQL --default-character-set=utf8 +--exec $MYSQL_BINLOG --short-form $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 # original and recovered data must be equal select HEX(f) from t03; select HEX(f) from t3;

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2739)
by knielsen＠knielsen-hq.org 29 Sep '09

29 Sep '09

#At lp:maria 2739 knielsen(a)knielsen-hq.org 2009-09-29 Fix race in test of times shown in SHOW FULL PROCESSLIST. (These are by their nature difficult to test reliably due to differences in timing). modified: mysql-test/r/information_schema.result mysql-test/t/information_schema.test === modified file 'mysql-test/r/information_schema.result' --- a/mysql-test/r/information_schema.result 2009-09-23 11:03:47 +0000 +++ b/mysql-test/r/information_schema.result 2009-09-29 20:19:43 +0000 @@ -1385,7 +1385,7 @@ who other connection here SELECT IF(`time` > 0, 'OK', `time`) AS time_low, IF(`time` < 1000, 'OK', `time`) AS time_high, -IF(time_ms > 900, 'OK', time_ms) AS time_ms_low, +IF(time_ms > 1500, 'OK', time_ms) AS time_ms_low, IF(time_ms < 1000000, 'OK', time_ms) AS time_ms_high FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID=@tid; === modified file 'mysql-test/t/information_schema.test' --- a/mysql-test/t/information_schema.test 2009-09-23 11:03:47 +0000 +++ b/mysql-test/t/information_schema.test 2009-09-29 20:19:43 +0000 @@ -1105,13 +1105,13 @@ connect (pslistcon,localhost,root,,test) let $ID= `select connection_id()`; SELECT 'other connection here' AS who; connection default; -sleep 1; +sleep 2; --disable_query_log eval SET @tid=$ID; --enable_query_log SELECT IF(`time` > 0, 'OK', `time`) AS time_low, IF(`time` < 1000, 'OK', `time`) AS time_high, - IF(time_ms > 900, 'OK', time_ms) AS time_ms_low, + IF(time_ms > 1500, 'OK', time_ms) AS time_ms_low, IF(time_ms < 1000000, 'OK', time_ms) AS time_ms_high FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID=@tid;

1 0