revision-id: 38d7dc6171cba4d5d3cdadd424f1d4b6a2d43f75 (mariadb-10.5.0-571-g38d7dc6171c) parent(s): 139117528affc89b6e174231b86048ec2b03f686 author: Varun Gupta committer: Varun Gupta timestamp: 2020-04-06 13:30:17 +0530 message: MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field overflow happening when we add suffix length to sortlength. Make sure that sortlength is at max UINT_MAX32 --- mysql-test/main/order_by.result | 16 ++++++++++++++++ mysql-test/main/order_by.test | 13 +++++++++++++ sql/filesort.cc | 9 +++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result index 49aa1dd32a0..b4e508f1bda 100644 --- a/mysql-test/main/order_by.result +++ b/mysql-test/main/order_by.result @@ -3850,3 +3850,19 @@ ANALYZE } } drop table t1; +# +# MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field +# +create table t1 ( a longblob); +insert into t1 select repeat('a', 256); +insert into t1 select repeat('b', 256); +insert into t1 select repeat('c', 256); +insert into t1 select repeat('d', 256); +SELECT IF( 0, NULL, a ) AS f FROM t1 GROUP BY f WITH ROLLUP; +f +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd +NULL +DROP TABLE t1; diff --git a/mysql-test/main/order_by.test b/mysql-test/main/order_by.test index 816389a78a6..f685707bd10 100644 --- a/mysql-test/main/order_by.test +++ b/mysql-test/main/order_by.test @@ -2400,3 +2400,16 @@ select * from t1 order by a; --source include/analyze-format.inc analyze format=json select * from t1 order by a; drop table t1; + +--echo # +--echo # MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field +--echo # + +create table t1 ( a longblob); +insert into t1 select repeat('a', 256); +insert into t1 select repeat('b', 256); +insert into t1 select repeat('c', 256); +insert into t1 select repeat('d', 256); + +SELECT IF( 0, NULL, a ) AS f FROM t1 GROUP BY f WITH ROLLUP; +DROP TABLE t1; diff --git a/sql/filesort.cc b/sql/filesort.cc index 1f491df82eb..9b75d088289 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -2105,6 +2105,7 @@ Type_handler_string_result::sort_length(THD *thd, { CHARSET_INFO *cs; sortorder->length= item->max_length; + set_if_smaller(sortorder->length, thd->variables.max_sort_length); sortorder->original_length= item->max_length; if (use_strnxfrm((cs= item->collation.collation))) @@ -2114,9 +2115,13 @@ Type_handler_string_result::sort_length(THD *thd, else if (cs == &my_charset_bin) { /* Store length last to be able to sort blob/varbinary */ - sortorder->suffix_length= suffix_length(sortorder->length); + sortorder->suffix_length= suffix_length(item->max_length); + DBUG_ASSERT(sortorder->length <= UINT_MAX32 - sortorder->suffix_length); sortorder->length+= sortorder->suffix_length; - sortorder->original_length+= sortorder->suffix_length; + if (sortorder->original_length >= UINT_MAX32 - sortorder->suffix_length) + sortorder->original_length= UINT_MAX32; + else + sortorder->original_length+= sortorder->suffix_length; } }