commit baec8cae34591c92c9309333280f85f81d852f8d
author Joe McDonnell <joemcdonnell@cloudera.com> Thu Feb 15 13:10:21 2018 -0800
committer Impala Public Jenkins <impala-public-jenkins@gerrit.cloudera.org> Wed Feb 21 03:17:57 2018 +0000
tree d825352fa07781c510429b344b4fd41f901157cf
parent f562be8c29a080549fa15e689f795b7a57003154

IMPALA-4874: Increase maximum KRPC message size

The default value for rpc_max_message_size is 50MB.
Impala currently requires support for messages of
up to 2GB. This changes the value of rpc_max_message_size
to INT_MAX for Impala.

Testing:
- Added a test to test_very_large_strings that generates
  a row with multiple large strings. This row requires
  that the RPC framework successfully transmit over
  400MB. This works for both KRPC and Thrift.
  This query operates under the same amount of memory
  as other queries in large_strings.test.
- Tested separately that larger row sizes also work,
  including tests up to almost 2GB.

Change-Id: I876bba0536e1d85e41eacd9c0aeccfe5c2126e58
Reviewed-on: http://gerrit.cloudera.org:8080/9337
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
Tested-by: Impala Public Jenkins

be/src/rpc/rpc-mgr.cc

diff --git a/be/src/rpc/rpc-mgr.cc b/be/src/rpc/rpc-mgr.cc
index d723280..7e05f38 100644
--- a/be/src/rpc/rpc-mgr.cc
+++ b/be/src/rpc/rpc-mgr.cc
@@ -57,6 +57,9 @@
 // Defined in kudu/rpc/rpcz_store.cc
 DECLARE_int32(rpc_duration_too_long_ms);
 
+// Defined in kudu/rpc/transfer.cc
+DECLARE_int32(rpc_max_message_size);
+
 DEFINE_int32(num_acceptor_threads, 2,
     "Number of threads dedicated to accepting connection requests for RPC services");
 DEFINE_int32(num_reactor_threads, 0,
@@ -75,6 +78,10 @@
   // Log any RPCs which take longer than 2 minutes.
   FLAGS_rpc_duration_too_long_ms = 2 * 60 * 1000;
 
+  // IMPALA-4874: Impala requires support for messages up to 2GB. Override KRPC's default
+  //              maximum of 50MB.
+  FLAGS_rpc_max_message_size = numeric_limits<int32_t>::max();
+
   MessengerBuilder bld("impala-server");
   const scoped_refptr<MetricEntity> entity(
       METRIC_ENTITY_server.Instantiate(&registry_, "krpc-metrics"));

testdata/workloads/functional-query/queries/QueryTest/large_strings.test

diff --git a/testdata/workloads/functional-query/queries/QueryTest/large_strings.test b/testdata/workloads/functional-query/queries/QueryTest/large_strings.test
index 4419953..1d930db 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/large_strings.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/large_strings.test
@@ -209,3 +209,20 @@
 ---- CATCH
 String length larger than allowed limit of 1 GB character data
 =====
+---- QUERY
+# IMPALA-4874: Generate a large row made up of multiple large strings to test RPC
+#              transmission. This uses hashing to make this difficult to compress,
+#              which results in a larger row batch.
+select length(group_concat(h, "!")),
+       length(group_concat(h, "-"))
+from (
+select cast(fnv_hash(concat(l_comment, 'a')) as string) as h from tpch_parquet.lineitem union all
+select cast(fnv_hash(concat(l_comment, 'b')) as string) as h from tpch_parquet.lineitem union all
+select cast(fnv_hash(concat(l_comment, 'c')) as string) as h from tpch_parquet.lineitem union all
+select cast(fnv_hash(concat(l_comment, 'd')) as string) as h from tpch_parquet.lineitem union all
+select cast(fnv_hash(concat(l_comment, 'e')) as string) as h from tpch_parquet.lineitem) a;
+---- TYPES
+INT,INT
+---- RESULTS
+611468161,611468161
+=====
\ No newline at end of file