| <?xml version="1.0" encoding="UTF-8"?> |
| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| --> |
| <!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd"> |
| <concept id="hints"> |
| |
| <title>Optimizer Hints</title> |
| |
| <titlealts audience="PDF"> |
| |
| <navtitle>Optimizer Hints</navtitle> |
| |
| </titlealts> |
| |
| <prolog> |
| <metadata> |
| <data name="Category" value="Impala"/> |
| <data name="Category" value="SQL"/> |
| <data name="Category" value="Querying"/> |
| <data name="Category" value="Performance"/> |
| <data name="Category" value="Troubleshooting"/> |
| <data name="Category" value="Developers"/> |
| <data name="Category" value="Data Analysts"/> |
| </metadata> |
| </prolog> |
| |
| <conbody> |
| |
| <p> |
| <indexterm audience="hidden">hints</indexterm> |
| The Impala SQL supports query hints, for fine-tuning the inner workings of queries. |
| Specify hints as a temporary workaround for expensive queries, where missing statistics or |
| other factors cause inefficient performance. |
| </p> |
| |
| <p> |
| Hints are most often used for the resource-intensive Impala queries, such as: |
| </p> |
| |
| <ul> |
| <li> |
| Join queries involving large tables, where intermediate result sets are transmitted |
| across the network to evaluate the join conditions. |
| </li> |
| |
| <li> |
| Inserting into partitioned Parquet tables, where many memory buffers could be allocated |
| on each host to hold intermediate results for each partition. |
| </li> |
| </ul> |
| |
| <p conref="../shared/impala_common.xml#common/syntax_blurb"/> |
| |
| <p rev="2.0.0"> |
| In <keyword keyref="impala20_full"/> and higher, you can specify the hints inside comments |
| that use either the <codeph>/* */</codeph> or <codeph>--</codeph> notation. Specify a |
| <codeph>+</codeph> symbol immediately before the hint name. Recently added hints are only |
| available using the <codeph>/* */</codeph> and <codeph>--</codeph> notation. For clarity, |
| the <codeph>/* */</codeph> and <codeph>--</codeph> styles are used in the syntax and |
| examples throughout this section. With the <codeph>/* */</codeph> or <codeph>--</codeph> |
| notation for hints, specify a <codeph>+</codeph> symbol immediately before the first hint |
| name. Multiple hints can be specified separated by commas, for example <codeph>/* |
| +clustered,shuffle */</codeph> |
| </p> |
| |
| <codeblock rev="2.0.0">SELECT STRAIGHT_JOIN <varname>select_list</varname> FROM |
| <varname>join_left_hand_table</varname> |
| JOIN /* +BROADCAST|SHUFFLE */ |
| <varname>join_right_hand_table</varname> |
| <varname>remainder_of_query</varname>; |
| |
| SELECT <varname>select_list</varname> FROM |
| <varname>join_left_hand_table</varname> |
| JOIN -- +BROADCAST|SHUFFLE |
| <varname>join_right_hand_table</varname> |
| <varname>remainder_of_query</varname>; |
| |
| INSERT <varname>insert_clauses</varname> |
| /* +SHUFFLE|NOSHUFFLE */ |
| SELECT <varname>remainder_of_query</varname>; |
| |
| INSERT <varname>insert_clauses</varname> |
| -- +SHUFFLE|NOSHUFFLE |
| SELECT <varname>remainder_of_query</varname>; |
| |
| <ph rev="2.12.0 IMPALA-4168"> |
| INSERT /* +SHUFFLE|NOSHUFFLE */ |
| <varname>insert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="2.12.0 IMPALA-4168"> |
| INSERT -- +SHUFFLE|NOSHUFFLE |
| <varname>insert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="2.12.0 IMPALA-4168"> |
| UPSERT /* +SHUFFLE|NOSHUFFLE */ |
| <varname>upsert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="2.12.0 IMPALA-4168"> |
| UPSERT -- +SHUFFLE|NOSHUFFLE |
| <varname>upsert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="IMPALA-2924">SELECT <varname>select_list</varname> FROM |
| <varname>table_ref</varname> |
| /* +{SCHEDULE_CACHE_LOCAL | SCHEDULE_DISK_LOCAL | SCHEDULE_REMOTE} |
| [,RANDOM_REPLICA] */ |
| <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="IMPALA-2522 2.8.0">INSERT <varname>insert_clauses</varname> |
| -- +CLUSTERED |
| SELECT <varname>remainder_of_query</varname>; |
| |
| INSERT <varname>insert_clauses</varname> |
| /* +CLUSTERED */ |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| <ph rev="2.12.0 IMPALA-4168">INSERT -- +CLUSTERED |
| <varname>insert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>; |
| |
| INSERT /* +CLUSTERED */ |
| <varname>insert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>; |
| |
| UPSERT -- +CLUSTERED |
| <varname>upsert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>; |
| |
| UPSERT /* +CLUSTERED */ |
| <varname>upsert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| |
| CREATE /* +SHUFFLE|NOSHUFFLE */ |
| <varname>table_clauses</varname> |
| AS SELECT <varname>remainder_of_query</varname>; |
| |
| CREATE -- +SHUFFLE|NOSHUFFLE |
| <varname>table_clauses</varname> |
| AS SELECT <varname>remainder_of_query</varname>; |
| |
| CREATE /* +CLUSTERED|NOCLUSTERED */ |
| <varname>table_clauses</varname> |
| AS SELECT <varname>remainder_of_query</varname>; |
| |
| CREATE -- +CLUSTERED|NOCLUSTERED |
| <varname>table_clauses</varname> |
| AS SELECT <varname>remainder_of_query</varname>; |
| </codeblock> |
| |
| <p> |
| The square bracket style hints are supported for backward compatibility, but the syntax is |
| deprecated and will be removed in a future release. For that reason, any newly added hints |
| are not available with the square bracket syntax. |
| </p> |
| |
| <codeblock>SELECT STRAIGHT_JOIN <varname>select_list</varname> FROM |
| <varname>join_left_hand_table</varname> |
| JOIN [{ /* +BROADCAST */ | /* +SHUFFLE */ }] |
| <varname>join_right_hand_table</varname> |
| <varname>remainder_of_query</varname>; |
| |
| INSERT <varname>insert_clauses</varname> |
| [{ /* +SHUFFLE */ | /* +NOSHUFFLE */ }] |
| [<ph rev="IMPALA-2522 2.8.0">/* +CLUSTERED */</ph>] |
| SELECT <varname>remainder_of_query</varname>; |
| |
| <ph rev="2.12.0 IMPALA-4168"> |
| UPSERT [{ /* +SHUFFLE */ | /* +NOSHUFFLE */ }] |
| [<ph rev="IMPALA-2522 2.8.0">/* +CLUSTERED */</ph>] |
| <varname>upsert_clauses</varname> |
| SELECT <varname>remainder_of_query</varname>;</ph> |
| </codeblock> |
| |
| <p conref="../shared/impala_common.xml#common/usage_notes_blurb"/> |
| |
| <p> |
| With both forms of hint syntax, include the <codeph>STRAIGHT_JOIN</codeph> keyword |
| immediately after the <codeph>SELECT</codeph> and any <codeph>DISTINCT</codeph> or |
| <codeph>ALL</codeph> keywords to prevent Impala from reordering the tables in a way that |
| makes the join-related hints ineffective. |
| </p> |
| |
| <p conref="../shared/impala_common.xml#common/straight_join_nested_queries"/> |
| |
| <p> |
| To reduce the need to use hints, run the <codeph>COMPUTE STATS</codeph> statement against |
| all tables involved in joins, or used as the source tables for <codeph>INSERT ... |
| SELECT</codeph> operations where the destination is a partitioned Parquet table. Do this |
| operation after loading data or making substantial changes to the data within each table. |
| Having up-to-date statistics helps Impala choose more efficient query plans without the |
| need for hinting. See <xref href="impala_perf_stats.xml#perf_stats"/> for details and |
| examples. |
| </p> |
| |
| <p> |
| To see which join strategy is used for a particular query, examine the |
| <codeph>EXPLAIN</codeph> output for that query. See |
| <xref href="impala_explain_plan.xml#perf_explain"/> for details and examples. |
| </p> |
| |
| <p> |
| <b>Hints for join queries:</b> |
| </p> |
| |
| <p> |
| The <codeph>/* +BROADCAST */</codeph> and <codeph>/* +SHUFFLE */</codeph> hints control |
| the execution strategy for join queries. Specify one of the following constructs |
| immediately after the <codeph>JOIN</codeph> keyword in a query: |
| </p> |
| |
| <ul> |
| <li> |
| <codeph>/* +SHUFFLE */</codeph> makes that join operation use the <q>partitioned</q> |
| technique, which divides up corresponding rows from both tables using a hashing |
| algorithm, sending subsets of the rows to other nodes for processing. (The keyword |
| <codeph>SHUFFLE</codeph> is used to indicate a <q>partitioned join</q>, because that |
| type of join is not related to <q>partitioned tables</q>.) Since the alternative |
| <q>broadcast</q> join mechanism is the default when table and index statistics are |
| unavailable, you might use this hint for queries where broadcast joins are unsuitable; |
| typically, partitioned joins are more efficient for joins between large tables of |
| similar size. |
| </li> |
| |
| <li> |
| <codeph>/* +BROADCAST */</codeph> makes that join operation use the <q>broadcast</q> |
| technique that sends the entire contents of the right-hand table to all nodes involved |
| in processing the join. This is the default mode of operation when table and index |
| statistics are unavailable, so you would typically only need it if stale metadata caused |
| Impala to mistakenly choose a partitioned join operation. Typically, broadcast joins are |
| more efficient in cases where one table is much smaller than the other. (Put the smaller |
| table on the right side of the <codeph>JOIN</codeph> operator.) |
| </li> |
| </ul> |
| |
| <p> |
| <b>Hints for INSERT ... SELECT and CREATE TABLE AS SELECT (CTAS):</b> |
| </p> |
| |
| <p id="insert_hints"> |
| When inserting into partitioned tables, such as using the Parquet file format, you can |
| include a hint in the <codeph>INSERT</codeph> or <codeph>CREATE TABLE AS |
| SELECT(CTAS)</codeph> statements to fine-tune the overall performance of the operation and |
| its resource usage. |
| </p> |
| |
| <p> |
| You would only use hints if an <codeph>INSERT</codeph> or <codeph>CTAS</codeph> into a |
| partitioned table was failing due to capacity limits, or if such an operation was |
| succeeding but with less-than-optimal performance. |
| </p> |
| |
| <ul> |
| <li> |
| <codeph>/* +SHUFFLE */</codeph> and <codeph>/* +NOSHUFFLE */</codeph> Hints |
| <ul> |
| <li> |
| <codeph>/* +SHUFFLE */</codeph> adds an exchange node, before writing the data, |
| which re-partitions the result of the <codeph>SELECT</codeph> based on the |
| partitioning columns of the target table. With this hint, only one node writes to a |
| partition at a time, minimizing the global number of simultaneous writes and the |
| number of memory buffers holding data for individual partitions. This also reduces |
| fragmentation, resulting in fewer files. Thus it reduces overall resource usage of |
| the <codeph>INSERT</codeph> or <codeph>CTAS</codeph> operation and allows some |
| operations to succeed that otherwise would fail. It does involve some data transfer |
| between the nodes so that the data files for a particular partition are all written |
| on the same node. |
| <p> |
| Use <codeph>/* +SHUFFLE */</codeph> in cases where an <codeph>INSERT</codeph> or |
| <codeph>CTAS</codeph> statement fails or runs inefficiently due to all nodes |
| attempting to write data for all partitions. |
| </p> |
| |
| <p> |
| If the table is unpartitioned or every partitioning expression is constant, then |
| <codeph>/* +SHUFFLE */</codeph> will cause every write to happen on the |
| coordinator node. |
| </p> |
| </li> |
| |
| <li> |
| <codeph>/* +NOSHUFFLE */</codeph> does not add exchange node before inserting to |
| partitioned tables and disables re-partitioning. So the selected execution plan |
| might be faster overall, but might also produce a larger number of small data files |
| or exceed capacity limits, causing the <codeph>INSERT</codeph> or |
| <codeph>CTAS</codeph> operation to fail. |
| <p> |
| Impala automatically uses the <codeph>/* +SHUFFLE */</codeph> method if any |
| partition key column in the source table, mentioned in the <codeph>SELECT</codeph> |
| clause, does not have column statistics. In this case, use the <codeph>/* |
| +NOSHUFFLE */</codeph> hint if you want to override this default behavior. |
| </p> |
| </li> |
| |
| <li> |
| If column statistics are available for all partition key columns in the source table |
| mentioned in the <codeph>INSERT ... SELECT</codeph> or <codeph>CTAS</codeph> query, |
| Impala chooses whether to use the <codeph>/* +SHUFFLE */</codeph> or <codeph>/* |
| +NOSHUFFLE */</codeph> technique based on the estimated number of distinct values in |
| those columns and the number of nodes involved in the operation. In this case, you |
| might need the <codeph>/* +SHUFFLE */</codeph> or the <codeph>/* +NOSHUFFLE |
| */</codeph> hint to override the execution plan selected by Impala. |
| </li> |
| </ul> |
| </li> |
| |
| <li> |
| <codeph>/* +CLUSTERED */</codeph> and <codeph>/* +NOCLUSTERED */</codeph> Hints |
| <ul> |
| <li> |
| <codeph>/* +CLUSTERED */</codeph> sorts data by the partition columns before |
| inserting to ensure that only one partition is written at a time per node. Use this |
| hint to reduce the number of files kept open and the number of buffers kept in |
| memory simultaneously. This technique is primarily useful for inserts into Parquet |
| tables, where the large block size requires substantial memory to buffer data for |
| multiple output files at once. This hint is available in |
| <keyword keyref="impala28_full"/> or higher. |
| <p> |
| Starting in <keyword keyref="impala30_full"/>, <codeph>/* +CLUSTERED */</codeph> |
| is the default behavior for HDFS tables. |
| </p> |
| </li> |
| |
| <li> |
| <codeph>/* +NOCLUSTERED */</codeph> does not sort by primary key before insert. This |
| hint is available in <keyword |
| keyref="impala28_full"/> or higher. |
| <p> |
| Use this hint when inserting to Kudu tables. |
| </p> |
| |
| <p> |
| In the versions lower than <keyword keyref="impala30_full"/>, <codeph>/* |
| +NOCLUSTERED */</codeph> is the default in HDFS tables. |
| </p> |
| </li> |
| </ul> |
| </li> |
| </ul> |
| <p><b>Kudu consideration:</b></p> |
| <p conref="../shared/impala_common.xml#common/kudu_hints"/> |
| |
| <p rev="IMPALA-2924"> |
| <b>Hints for scheduling of scan ranges (HDFS data blocks or Kudu |
| tablets)</b> |
| </p> |
| |
| <p rev="IMPALA-2924"> The hints <codeph>/* +SCHEDULE_CACHE_LOCAL |
| */</codeph>, <codeph>/* +SCHEDULE_DISK_LOCAL */</codeph>, and <codeph>/* |
| +SCHEDULE_REMOTE */</codeph> have the same effect as specifying the |
| <codeph>REPLICA_PREFERENCE</codeph> query option with the respective |
| option settings of <codeph>CACHE_LOCAL</codeph>, |
| <codeph>DISK_LOCAL</codeph>, or <codeph>REMOTE</codeph>. </p> |
| <p rev="IMPALA-2924"> Specifying the replica preference as a query hint |
| always overrides the query option setting. </p> |
| <p rev="IMPALA-2924">The hint <codeph>/* +RANDOM_REPLICA */</codeph> is the |
| same as enabling the <codeph>SCHEDULE_RANDOM_REPLICA</codeph> query |
| option. </p> |
| |
| <p rev="IMPALA-2924"> You can use these hints in combination by separating |
| them with commas, for example, <codeph>/* |
| +SCHEDULE_CACHE_LOCAL,RANDOM_REPLICA */</codeph>. See <xref |
| keyref="replica_preference"/> and <xref keyref="schedule_random_replica" |
| /> for information about how these settings influence the way Impala |
| processes HDFS data blocks or Kudu tablets. </p> |
| <p rev="IMPALA-2924">Specifying either the |
| <codeph>SCHEDULE_RANDOM_REPLICA</codeph> query option or the |
| corresponding <codeph>RANDOM_REPLICA</codeph> query hint enables the |
| random tie-breaking behavior when processing data blocks during the query. </p> |
| |
| <p> |
| <b>Suggestions versus directives:</b> |
| </p> |
| |
| <p> |
| In early Impala releases, hints were always obeyed and so acted more like directives. Once |
| Impala gained join order optimizations, sometimes join queries were automatically |
| reordered in a way that made a hint irrelevant. Therefore, the hints act more like |
| suggestions in Impala 1.2.2 and higher. |
| </p> |
| |
| <p> |
| To force Impala to follow the hinted execution mechanism for a join query, include the |
| <codeph>STRAIGHT_JOIN</codeph> keyword in the <codeph>SELECT</codeph> statement. See |
| <xref href="impala_perf_joins.xml#straight_join"/> for details. When you use this |
| technique, Impala does not reorder the joined tables at all, so you must be careful to |
| arrange the join order to put the largest table (or subquery result set) first, then the |
| smallest, second smallest, third smallest, and so on. This ordering lets Impala do the |
| most I/O-intensive parts of the query using local reads on the DataNodes, and then reduce |
| the size of the intermediate result set as much as possible as each subsequent table or |
| subquery result set is joined. |
| </p> |
| |
| <p conref="../shared/impala_common.xml#common/restrictions_blurb"/> |
| |
| <p> |
| Queries that include subqueries in the <codeph>WHERE</codeph> clause can be rewritten |
| internally as join queries. Currently, you cannot apply hints to the joins produced by |
| these types of queries. |
| </p> |
| |
| <p> |
| Because hints can prevent queries from taking advantage of new metadata or improvements in |
| query planning, use them only when required to work around performance issues, and be |
| prepared to remove them when they are no longer required, such as after a new Impala |
| release or bug fix. |
| </p> |
| |
| <p> |
| In particular, the <codeph>/* +BROADCAST */</codeph> and <codeph>/* +SHUFFLE */</codeph> |
| hints are expected to be needed much less frequently in Impala 1.2.2 and higher, because |
| the join order optimization feature in combination with the <codeph>COMPUTE STATS</codeph> |
| statement now automatically choose join order and join mechanism without the need to |
| rewrite the query and add hints. See <xref href="impala_perf_joins.xml#perf_joins"/> for |
| details. |
| </p> |
| |
| <p conref="../shared/impala_common.xml#common/compatibility_blurb"/> |
| |
| <p rev="2.0.0"> |
| The hints embedded within <codeph>--</codeph> comments are compatible with Hive queries. |
| The hints embedded within <codeph>/* */</codeph> comments or <codeph>[ ]</codeph> square |
| brackets are not recognized by or not compatible with Hive. For example, Hive raises an |
| error for Impala hints within <codeph>/* */</codeph> comments because it does not |
| recognize the Impala hint names. |
| </p> |
| |
| <p conref="../shared/impala_common.xml#common/view_blurb"/> |
| |
| <p rev="2.0.0"> |
| If you use a hint in the query that defines a view, the hint is preserved when you query |
| the view. Impala internally rewrites all hints in views to use the <codeph>--</codeph> |
| comment notation, so that Hive can query such views without errors due to unrecognized |
| hint names. |
| </p> |
| |
| <p conref="../shared/impala_common.xml#common/example_blurb"/> |
| |
| <p> |
| For example, this query joins a large customer table with a small lookup table of less |
| than 100 rows. The right-hand table can be broadcast efficiently to all nodes involved in |
| the join. Thus, you would use the <codeph>/* +broadcast */</codeph> hint to force a |
| broadcast join strategy: |
| </p> |
| |
| <codeblock>select straight_join customer.address, state_lookup.state_name |
| from customer join <b>/* +broadcast */</b> state_lookup |
| on customer.state_id = state_lookup.state_id;</codeblock> |
| |
| <p> |
| This query joins two large tables of unpredictable size. You might benchmark the query |
| with both kinds of hints and find that it is more efficient to transmit portions of each |
| table to other nodes for processing. Thus, you would use the <codeph>/* +shuffle |
| */</codeph> hint to force a partitioned join strategy: |
| </p> |
| |
| <codeblock>select straight_join weather.wind_velocity, geospatial.altitude |
| from weather join <b>/* +shuffle */</b> geospatial |
| on weather.lat = geospatial.lat and weather.long = geospatial.long;</codeblock> |
| |
| <p> |
| For joins involving three or more tables, the hint applies to the tables on either side of |
| that specific <codeph>JOIN</codeph> keyword. The <codeph>STRAIGHT_JOIN</codeph> keyword |
| ensures that joins are processed in a predictable order from left to right. For example, |
| this query joins <codeph>t1</codeph> and <codeph>t2</codeph> using a partitioned join, |
| then joins that result set to <codeph>t3</codeph> using a broadcast join: |
| </p> |
| |
| <codeblock>select straight_join t1.name, t2.id, t3.price |
| from t1 join <b>/* +shuffle */</b> t2 join <b>/* +broadcast */</b> t3 |
| on t1.id = t2.id and t2.id = t3.id;</codeblock> |
| |
| <!-- To do: This is a good place to add more sample output showing before and after EXPLAIN plans. --> |
| |
| <p conref="../shared/impala_common.xml#common/related_info"/> |
| |
| <p> |
| For more background information about join queries, see |
| <xref href="impala_joins.xml#joins"/>. For performance considerations, see |
| <xref href="impala_perf_joins.xml#perf_joins"/>. |
| </p> |
| |
| </conbody> |
| |
| </concept> |