revision-id: 8ba01caefee392f51b5ed7f5635c10dff07825bd (mariadb-10.0.36-27-g8ba01caefee) parent(s): 3a4242fd57b3a2235d2478ed080941b67a82ad1b author: Varun Gupta committer: Varun Gupta timestamp: 2018-09-13 22:56:23 +0530 message: MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion This assert is hit when we do filesort using the priority queue and try to insert elements in the queue. The compare function used for the priority queue should handle the case for zerolength sortkey. --- mysql-test/r/order_by_zerolength-4285.result | 20 ++++++++++++++++++++ mysql-test/t/order_by_zerolength-4285.test | 13 +++++++++++++ mysys/ptr_cmp.c | 3 +++ 3 files changed, 36 insertions(+) diff --git a/mysql-test/r/order_by_zerolength-4285.result b/mysql-test/r/order_by_zerolength-4285.result index f60ce7d90c7..e4c117b26af 100644 --- a/mysql-test/r/order_by_zerolength-4285.result +++ b/mysql-test/r/order_by_zerolength-4285.result @@ -24,3 +24,23 @@ Warning 1292 Truncated incorrect CHAR(0) value: '8' Warning 1292 Truncated incorrect CHAR(0) value: '9' Warning 1292 Truncated incorrect CHAR(0) value: '10' drop table t1; +# +# MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +# +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 4 NULL 2 Using index; Using filesort +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +pk +1 +2 +Warnings: +Warning 1292 Truncated incorrect CHAR(0) value: '1' +Warning 1292 Truncated incorrect CHAR(0) value: '2' +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysql-test/t/order_by_zerolength-4285.test b/mysql-test/t/order_by_zerolength-4285.test index 2fb58edd36d..f03d528320c 100644 --- a/mysql-test/t/order_by_zerolength-4285.test +++ b/mysql-test/t/order_by_zerolength-4285.test @@ -6,3 +6,16 @@ insert into t1 values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); select * from t1 order by now(), cast(pk as char(0)); drop table t1; +--echo # +--echo # MDEV-17020: Assertion `length > 0' failed in ptr_compare upon ORDER BY with bad conversion +--echo # + +set @save_sql_mode= @@sql_mode; +SET @@sql_mode= ''; +CREATE TABLE t1 (pk INT PRIMARY KEY); +INSERT INTO t1 VALUES (1),(2); +explain +SELECT * FROM t1 ORDER BY 'foo', CONVERT(pk, CHAR(0)) LIMIT 2; +SELECT * FROM t1 ORDER BY 'foo', Cast(pk as CHAR(0)) LIMIT 2; +set @@sql_mode= @save_sql_mode; +drop table t1; diff --git a/mysys/ptr_cmp.c b/mysys/ptr_cmp.c index 6e373e98972..9007265816b 100644 --- a/mysys/ptr_cmp.c +++ b/mysys/ptr_cmp.c @@ -91,6 +91,9 @@ static int ptr_compare(size_t *compare_length, uchar **a, uchar **b) reg3 int length= *compare_length; reg1 uchar *first,*last; + if (length == 0) + return 0; + DBUG_ASSERT(length > 0); first= *a; last= *b; while (--length)