- developers - lists.mariadb.org

[Maria-developers] MDEV-8360 Clean-up CHARSET_INFO: strnncollsp: diff_if_only_endspace_difference
by Alexander Barkov 30 Mar '16

30 Mar '16

Hi Sergei, please review a patch for MDEV-8360. Btw, shouldn't we remove: #define HA_END_SPACE_ARE_EQUAL 512 in ./include/my_base.h ? It's not really needed. Thanks.

2 1

Re: [Maria-developers] [Commits] d40d68f: Convert percent_rank to work with cursors
by Sergey Petrunia 30 Mar '16

30 Mar '16

On Mon, Mar 28, 2016 at 10:58:55PM +0300, Vicentiu Ciorbaru wrote: > revision-id: d40d68f23602be9886c1b502fdad9d23bdc9a0fb (mariadb-10.1.8-187-gd40d68f) > parent(s): bf18dac08fe0ae18975158e786fee097883949d4 > author: Vicențiu Ciorbaru > committer: Vicențiu Ciorbaru > timestamp: 2016-03-28 22:51:42 +0300 > message: > > Convert percent_rank to work with cursors > > The percent_rank function now is compatible with the cursor algorithm. > We no longer need a special implementation for it to work. > > --- > sql/item_windowfunc.h | 77 +++++++++++------------ > sql/sql_window.cc | 165 +++++++++++++++++--------------------------------- > 2 files changed, 91 insertions(+), 151 deletions(-) > > diff --git a/sql/item_windowfunc.h b/sql/item_windowfunc.h > index 6f91bc8..1dd483c 100644 > --- a/sql/item_windowfunc.h > +++ b/sql/item_windowfunc.h > @@ -317,12 +317,18 @@ class Item_context > NOTE: All two pass window functions need to implement > this interface. > */ > -class Item_sum_window_with_context : public Item_sum_num, > - public Item_context > +class Item_sum_window_with_row_count : public Item_sum_num > { > public: > - Item_sum_window_with_context(THD *thd) > - : Item_sum_num(thd), Item_context() {} > + Item_sum_window_with_row_count(THD *thd) : Item_sum_num(thd), > + partition_row_count_(0){} > + > + void set_row_count(ulonglong count) { partition_row_count_ = count; } > + > + protected: > + longlong get_row_count() { return partition_row_count_; } > + private: > + ulonglong partition_row_count_; > }; > > /* > @@ -336,12 +342,11 @@ class Item_sum_window_with_context : public Item_sum_num, > This is held within the row_count context. > - Second pass to compute rank of current row and the value of the function > */ > -class Item_sum_percent_rank: public Item_sum_window_with_context, > - public Window_context_row_count > +class Item_sum_percent_rank: public Item_sum_window_with_row_count > { > public: > Item_sum_percent_rank(THD *thd) > - : Item_sum_window_with_context(thd), cur_rank(1) {} > + : Item_sum_window_with_row_count(thd), cur_rank(1) {} > > longlong val_int() > { > @@ -359,14 +364,9 @@ class Item_sum_percent_rank: public Item_sum_window_with_context, > We can not get the real value without knowing the number of rows > in the partition. Don't divide by 0. > */ > - if (!get_context_()) > - { > - // Calling this kind of function with a context makes no sense. > - DBUG_ASSERT(0); > - return 0; > - } > - > - longlong partition_rows = get_context_()->get_field_context(result_field); > + ulonglong partition_rows = get_row_count(); > + null_value= partition_rows > 0 ? false : true; > + > return partition_rows > 1 ? > static_cast<double>(cur_rank - 1) / (partition_rows - 1) : 0; > } > @@ -381,25 +381,6 @@ class Item_sum_percent_rank: public Item_sum_window_with_context, > return "percent_rank"; > } > > - bool create_window_context() > - { > - // TODO-cvicentiu: Currently this means we must make sure to delete > - // the window context. We can potentially allocate this on the THD memroot. > - // At the same time, this is only necessary for a small portion of the > - // query execution and it does not make sense to keep it for all of it. > - context_ = new Window_context_row_count(); > - if (context_ == NULL) > - return true; > - return false; > - } > - > - void delete_window_context() > - { > - if (context_) > - delete get_context_(); > - context_ = NULL; > - } > - > void update_field() {} > > void clear() > @@ -428,13 +409,6 @@ class Item_sum_percent_rank: public Item_sum_window_with_context, > void cleanup() > { > peer_tracker.cleanup(); > - Item_sum_window_with_context::cleanup(); > - } > - > - /* Helper function so that we don't cast the context every time. */ > - Window_context_row_count* get_context_() > - { > - return static_cast<Window_context_row_count *>(context_); > } > }; > > @@ -517,6 +491,27 @@ class Item_window_func : public Item_func_or_sum > } > } > > + bool requires_partition_size() const > + { > + switch (window_func()->sum_func()) { > + case Item_sum::PERCENT_RANK_FUNC: > + case Item_sum::CUME_DIST_FUNC: > + return true; > + default: > + return false; > + } > + } > + > + bool requires_peer_size() const > + { > + switch (window_func()->sum_func()) { > + case Item_sum::CUME_DIST_FUNC: > + return true; > + default: > + return false; > + } > + } This function seem not to be used anywhere? > + > bool is_order_list_mandatory() const > { > switch (window_func()->sum_func()) { > diff --git a/sql/sql_window.cc b/sql/sql_window.cc > index e8e226b..1bfac7a 100644 > --- a/sql/sql_window.cc > +++ b/sql/sql_window.cc > @@ -865,15 +865,18 @@ class Frame_unbounded_preceding : public Frame_cursor > } > }; > > + > /* > UNBOUNDED FOLLOWING frame bound > */ > > class Frame_unbounded_following : public Frame_cursor > { > - Table_read_cursor cursor; > > +protected: > + Table_read_cursor cursor; > Group_bound_tracker bound_tracker; > + > public: > void init(THD *thd, READ_RECORD *info, SQL_I_List<ORDER> *partition_list, > SQL_I_List<ORDER> *order_list) > @@ -910,6 +913,35 @@ class Frame_unbounded_following : public Frame_cursor > }; > > > +class Frame_unbounded_following_set_count : public Frame_unbounded_following > +{ > + void next_partition(longlong rownum, Item_sum* item) > + { > + ulonglong num_rows_in_partition= 0; > + if (!rownum) > + { > + /* Read the first row */ > + if (cursor.get_next()) > + return; > + num_rows_in_partition++; > + } > + > + /* Remember which partition we are in */ > + bound_tracker.check_if_next_group(); > + /* Walk to the end of the partition, find how many rows there are. */ > + while (!cursor.get_next()) > + { > + if (bound_tracker.check_if_next_group()) > + break; > + num_rows_in_partition++; > + } > + > + Item_sum_window_with_row_count* item_with_row_count = > + static_cast<Item_sum_window_with_row_count *>(item); > + item_with_row_count->set_row_count(num_rows_in_partition); > + } > +}; > + > ///////////////////////////////////////////////////////////////////////////// > // ROWS-type frame bounds > ///////////////////////////////////////////////////////////////////////////// > @@ -1212,18 +1244,37 @@ Frame_cursor *get_frame_cursor(Window_frame *frame, bool is_top_bound) > return NULL; > } > > +void add_extra_frame_cursors(List<Frame_cursor> *cursors, > + const Item_sum *window_func) > +{ > + switch (window_func->sum_func()) > + { > + case Item_sum::CUME_DIST_FUNC: > + cursors->push_back(new Frame_unbounded_preceding); What is the reason for having this cursor at all? > + cursors->push_back(new Frame_range_current_row_bottom); > + break; This way to structure the code looks wrong. Why should add_extra_frame_cursors know that some functions (e.g. CUME_DIST) need a cursor, and some don't? I think it would be more natural if CUME_DIST knew that it needs certain kinds of cursor(s). Maybe, we should put off this question for now, and get back to it when we have more window functions. (If we implement LEAD and LAG, we will need to create Frame_rows_n_preceding and Frame_rows_n_following for them. Will this be done in this function too?) Another question (possibly wrong): why is CUME_DIST mentioned here, while PERCENT_RANK isn't? At the first glance they will need the same kind of cursors, wont they? > + default: > + cursors->push_back(new Frame_unbounded_preceding); > + cursors->push_back(new Frame_rows_current_row_bottom); > + } > +} > + > List<Frame_cursor> get_window_func_required_cursors( > const Item_window_func* item_win) > { > List<Frame_cursor> result; I don't think is's good idea to pass around List objects by value. - there is no need to do this. - List template is not fully specified w.r.t what happens when you copy-assign it. I don't consider the situation with two List objects sharing the list but not sharing e.g. the value of List::elements meaningful. because if that, I would pass a pointer or reference to the list and have the code populate the passed list. > > + if (item_win->requires_partition_size()) > + result.push_back(new Frame_unbounded_following_set_count); > + > /* > If it is not a regular window function that follows frame specifications, > specific cursors are required. > */ > if (item_win->is_frame_prohibited()) > { > - DBUG_ASSERT(0); // TODO-cvicentiu not-implemented yet. > + add_extra_frame_cursors(&result, item_win->window_func()); > + return result; > } > > /* A regular window function follows the frame specification. */ > @@ -1283,7 +1334,6 @@ bool compute_window_func_with_frames(Item_window_func *item_win, > > List<Frame_cursor> cursors= get_window_func_required_cursors(item_win); > > - > List_iterator_fast<Frame_cursor> it(cursors); > Frame_cursor *c; > while((c= it++)) > @@ -1362,107 +1412,6 @@ bool compute_window_func_with_frames(Item_window_func *item_win, > } > > > -bool compute_two_pass_window_functions(Item_window_func *item_win, > - TABLE *table, READ_RECORD *info) > -{ > - /* Perform first pass. */ > - > - // TODO-cvicentiu why not initialize the record for when we need, _in_ > - // this function. > - READ_RECORD *info2= new READ_RECORD(); > - int err; > - bool is_error = false; > - bool first_row= true; > - clone_read_record(info, info2); > - Item_sum_window_with_context *window_func= > - static_cast<Item_sum_window_with_context *>(item_win->window_func()); > - uchar *rowid_buf= (uchar*) my_malloc(table->file->ref_length, MYF(0)); > - > - is_error= window_func->create_window_context(); > - /* Unable to allocate a new context. */ > - if (is_error) > - return true; > - > - Window_context *context = window_func->get_window_context(); > - /* > - The two pass algorithm is as follows: > - We have a sorted table according to the partition and order by clauses. > - 1. Scan through the table till we reach a partition boundary. > - 2. For each row that we scan, add it to the context. > - 3. Once the partition boundary is met, do a second scan through the > - current partition and use the context information to compute the value for > - the window function for that partition. > - 4. Reset the context. > - 5. Repeat from 1 till end of table. > - */ > - > - bool done = false; > - longlong rows_in_current_partition = 0; > - // TODO handle end of table updating. > - while (!done) > - { > - > - if ((err= info->read_record(info))) > - { > - done = true; > - } > - > - bool partition_changed= done || item_win->check_if_partition_changed(); > - // The first time we always have a partition changed. Ignore it. > - if (first_row) > - { > - partition_changed= false; > - first_row= false; > - } > - > - if (partition_changed) > - { > - /* > - We are now looking at the first row for the next partition, or at the > - end of the table. Either way, we must remember this position for when > - we finish doing the second pass. > - */ > - table->file->position(table->record[0]); > - memcpy(rowid_buf, table->file->ref, table->file->ref_length); > - > - for (longlong row_number = 0; row_number < rows_in_current_partition; > - row_number++) > - { > - if ((err= info2->read_record(info2))) > - { > - is_error= true; > - break; > - } > - window_func->add(); > - // Save the window function into the table. > - item_win->save_in_field(item_win->result_field, true); > - err= table->file->ha_update_row(table->record[1], table->record[0]); > - if (err && err != HA_ERR_RECORD_IS_THE_SAME) > - { > - is_error= true; > - break; > - } > - } > - > - if (is_error) > - break; > - > - rows_in_current_partition= 0; > - window_func->clear(); > - context->reset(); > - > - // Return to the beginning of the new partition. > - table->file->ha_rnd_pos(table->record[0], rowid_buf); > - } > - rows_in_current_partition++; > - context->add_field_to_context(item_win->result_field); > - } > - > - window_func->delete_window_context(); > - delete info2; > - my_free(rowid_buf); > - return is_error; > -} > > > /* Make a list that is a concation of two lists of ORDER elements */ > @@ -1527,16 +1476,12 @@ bool Window_func_runner::setup(THD *thd) > compute_func= compute_window_func_values; > break; > } > - case Item_sum::PERCENT_RANK_FUNC: > - case Item_sum::CUME_DIST_FUNC: > - { > - compute_func= compute_two_pass_window_functions; > - break; > - } > case Item_sum::COUNT_FUNC: > case Item_sum::SUM_BIT_FUNC: > case Item_sum::SUM_FUNC: > case Item_sum::AVG_FUNC: > + case Item_sum::PERCENT_RANK_FUNC: > + case Item_sum::CUME_DIST_FUNC: > { > /* > Frame-aware window function computation. It does one pass, but > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] Fwd: Gsoc 2016 Mdev 371 Unique index for blob
by Sachin Setia 29 Mar '16

29 Mar '16

Hi Sergei Did you see the patch.Actually the patch works for every combination of blob and varchars table definition. Can you please tell what extra modification I should make Regards sachin On Mon, Mar 28, 2016 at 9:34 PM, Sachin Setia <sachinsetia1001(a)gmail.com> wrote: > > ---------- Forwarded message ---------- > From: Sachin Setia <sachinsetia1001(a)gmail.com> > Date: Mon, Mar 28, 2016 at 9:13 PM > Subject: Re: [Maria-developers] Gsoc 2016 Mdev 371 Unique index for blob > To: maria-developers(a)lists.launchpad.net > > > I am Sorry I did not see this mail.So I did not send the prototype link in > proposal > > On Thu, Mar 24, 2016 at 10:41 PM, Sergei Golubchik <serg(a)mariadb.org> > wrote: > >> Hi, Sachin! >> >> As you might have noticed on the mailing list, there is another student >> who wants to do this project. >> >> It would help us choose, if you'd put your prototype code on github and >> put the link to the repository into your final proposal. >> >> Thanks! >> >> On Mar 15, Sachin Setia wrote: >> > Dear Developers, >> > >> > I was doing prototype for this project as Mr Sergei Golubchik suggested >> > >> >> Regards, >> Sergei >> Chief Architect MariaDB >> and security(a)mariadb.org >> > > >

2 1

[Maria-developers] Getting restarted with mariadb and a port to AIX
by Michael Felt 28 Mar '16

28 Mar '16

Hi all, This week I added a few issues that I felt were clear enough to report directly. This one I am not so sure of as it concerns two areas I know too little about: a) cmake (how to get an equivalent of GNU autotools ./configure --help) b) cmake (how to make it show the commands it is actually running (so I could add -bnoquiet and/or -bloadmap:XXX.map) c) 'strategic' grepping through C++ sources to find where an undefined label is calling something. After I 'patch' aka bandaid mariadb-5.5.48.0/sql-common/mysql_async.c (see https://jira.mariadb.org/browse/MDEV-9814) I get loader errors - and I cannot, in ways I could with an autotools generated 'make' see how to debug. Q) are there variables (e.g., V=1) I can pass to make so that the commands it performs are more verbose Q) have any ideas why these symbols are undefined? Thanks, Michael

2 1

[Maria-developers] Unique Index for blob
by Sachin Setia 28 Mar '16

28 Mar '16

Hello Devlopers Hi this is sachin. I applied for gsoc 2016(applied by name sachin setiya) My topic was unique index for blob(MyISAM) But i already completed some part of it .it is working for blobs varchar for definitions like create table tbl(abc varchar(2000) , xyz blob ,unique(abc,xyz)); I have corrected the table2myisam function . For documentation you can refer to my gsoc proposal If am doing wrong please let me know. I am attaching the patch file TODO Second thing which i need to do is to tweak check_definition function to correctly check for definition currently I am commenting the function code Regards sachin

1 1

Re: [Maria-developers] Gsoc 2016 Mdev 371 Unique index for blob
by Sachin Setia 28 Mar '16

28 Mar '16

I am Sorry I did not see this mail.So I did not send the prototype link in proposal On Thu, Mar 24, 2016 at 10:41 PM, Sergei Golubchik <serg(a)mariadb.org> wrote: > Hi, Sachin! > > As you might have noticed on the mailing list, there is another student > who wants to do this project. > > It would help us choose, if you'd put your prototype code on github and > put the link to the repository into your final proposal. > > Thanks! > > On Mar 15, Sachin Setia wrote: > > Dear Developers, > > > > I was doing prototype for this project as Mr Sergei Golubchik suggested > > > > Regards, > Sergei > Chief Architect MariaDB > and security(a)mariadb.org >

1 0

[Maria-developers] Please review MDEV-6353 my_ismbchar() and my_mbcharlen() refactoring
by Alexander Barkov 28 Mar '16

28 Mar '16

Hi Sergei, Please review a partial patch for MDEV-6353. It removes my_mbcharlen() in all remaining pieces of the code, except LOAD DATA and SELECT INTO OUTFILE. I'll do LOAD DATA and SELECT INTO OUTFILE in a separate patch. Thanks.

1 0

Re: [Maria-developers] Google Summer of Code 2016
by Soham Mankad 26 Mar '16

26 Mar '16

Respected sir, I am Soham Mankad and thanks Sergei sir for directing to me this developers list. As I had mailed you before I have gone through the basic requirements and goals of the project "Aggregate Stored Functions" and also thinking on varoius syntax and languages that can be used like hsql, Postgresql, or mysql. Also as you had mentioned in a reply to Vikas Gupta still i have not found any data base with this facility existing uptill now. Also I have seen the syntax and possible ways given by you in knowledge base of the project. The most basic form that you have mentioned i.e using nested commands i.e returning sum adding two numbers. I am also learning about various aspects of create function commands in different languages loke read only or returning which rows or coloumns so that optimum result will be given. I have submitted my proposal for this project on Gsoc'16 website. I would definitely appreciate an opportunity to work on this project. Regards, Soham Mankad.

1 0

[Maria-developers] Window functions: why some functions are not in select_lex->window_funcs?
by Sergey Petrunia 24 Mar '16

24 Mar '16

Hello Igor, I was trying to move JOIN::process_window_functions() from using join->fields_list to using join->select_lex->window_funcs. And I discovered that join->select_lex->window_funcs does not contain window function items that use window definition. Is this intentional? Example: create table t0 (a int); insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t1 (pk int, c int); insert into t1 select a+1,1 from t0; update t1 set c=2 where pk not in (1,2,3,4); select pk, c, count(*) over (partition by c order by pk rows between 2 preceding and 2 following) as CNT from t1 Here: #1 0x0000555555af5db6 in AGGR_OP::end_send (this=0x7fff4c0086c8) (gdb) p join->select_lex->window_funcs $69 = {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x7fff4c005d18, last = 0x7fff4c005d18, elements = 1}, <No data fields>} But: select pk, c, count(*) over w1 as CNT from t1 window w1 as (partition by c order by pk rows between 2 preceding and 2 following); (gdb) p join->select_lex->window_funcs $74 = {<base_list> = {<Sql_alloc> = {<No data fields>}, first = 0x555556d48820, last = 0x55555ab10b70, elements = 0}, <No data fields>} BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 1

Re: [Maria-developers] Gsoc 2016 Mdev 371 Unique index for blob
by Sachin Setia 23 Mar '16

23 Mar '16

Hello Sergei Actually I was prototyping for blob and varchar for aria and myisam storage engine. My prototype worked for complex definations like craete table(abc int primary key, blob_col blob unique, varchar_col varchar(1000) unique) engine=myisam; Solved the issue of frm file incosistance. As you suggested for doing it for innodb i am current working on it.Innodb does not natively support hash based index. when we run select distinct column from tbl; it use create_internal_tmp_table() which uses maria storage engine for creating tmp table. But query like this works MariaDB [sachin]> create table iu2(abc blob unique); Query OK, 0 rows affected (0.04 sec) MariaDB [sachin]> insert into iu2 values(1); Query OK, 1 row affected (0.03 sec) MariaDB [sachin]> insert into iu2 values(1); ERROR 1062 (23000): Duplicate entry '1' for key 'abc' this query does not use hash but it simply compares values Will write a proposal shortly. On Tue, Mar 22, 2016 at 4:20 PM, Sachin Setia <sachinsetia1001(a)gmail.com> wrote: > Hello Sergi > Actually I was prototyping for blob and varchar for aria and myisam > storage engine. > My prototype worked for complex definations like > craete table(abc int primary key, blob_col blob unique, varchar_col > varchar(1000) unique) engine=myisam; > Solved the issue of frm file incosistance. > > As you suggested for doing it for innodb i am current working on it.Innodb > does not natively support hash based index. > when we run select distinct column from tbl; > it use create_internal_tmp_table() which uses maria storage engine for > creating tmp table. > But query like this works > MariaDB [sachin]> create table iu2(abc blob unique); > Query OK, 0 rows affected (0.04 sec) > > MariaDB [sachin]> insert into iu2 values(1); > Query OK, 1 row affected (0.03 sec) > > MariaDB [sachin]> insert into iu2 values(1); > ERROR 1062 (23000): Duplicate entry '1' for key 'abc' > this query does not use hash but it simply compares values > Will write a proposal shortly. > > > Regards > sachin > > On Sat, Mar 19, 2016 at 1:52 AM, Sergei Golubchik <serg(a)mariadb.org> > wrote: > >> Hi, Sachin! >> >> On Mar 18, Sachin Setia wrote: >> > >> > ERROR 1030 (HY000): Got error 190 "Incompatible key or row definition >> > between the MariaDB .frm file and the information in the storage engine. >> > You have to dump an" from storage engine MyISAM >> > >> > We are getting this becuase in mi_create for each unique_def it creates >> one >> > keydef and writes it.And this creates two problem >> > 1. frm keydef algorithm is different from saved kefdef algorithm(always >> > zero) for the time I have solved this problem . >> > >> > 2.Mismatch between keydef's keysegs the reason for this is when >> mi_create >> > creates keyseg for unique_def it did not keeps the orignal uniquedef's >> > keyseg parameters in mind like language start length which creates >> problem >> > in check_definition function in ha_myisam.cc.I am currently working on >> it >> > Once again sorry for this foolish mistake. >> > Regars >> > sachin >> >> No problem, everyone makes mistakes :) >> >> It's a prototype, after all. It's much more important that you >> understand how the code works and why it doesn't work. >> >> Regards, >> Sergei >> Chief Architect MariaDB >> and security(a)mariadb.org >> > >

2 3