TAJO-1906: Decrease the default size of hash map and array list for testing.
diff --git a/CHANGES b/CHANGES
index b8d9ecf..1435422 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,21 @@
Tajo Change Log
-Release 0.11.0 - unreleased
+Release 0.11.1 - unreleased
+
+ NEW FEATURES
+
+ IMPROVEMENT
+
+ BUG FIXES
+
+ TASKS
+
+ TAJO-1906: Decrease the default size of hash map and array list for testing.
+ (jihoon)
+
+ SUB TASKS
+
+Release 0.11.0 - Released
NEW FEATURES
diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
index 7e419f0..2834e14 100644
--- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@ -130,6 +130,9 @@
Long.class, Validators.min("0")),
NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT),
CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT),
+ AGG_HASH_TABLE_SIZE(ConfVars.$AGG_HASH_TABLE_SIZE, "Aggregation hash table size", DEFAULT),
+ SORT_HASH_TABLE_SIZE(ConfVars.$SORT_HASH_TABLE_SIZE, "Sort hash table size", DEFAULT),
+ JOIN_HASH_TABLE_SIZE(ConfVars.$JOIN_HASH_TABLE_SIZE, "Join hash table size", DEFAULT),
// for index
INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT),
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index f53ada9..18e5534 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -344,6 +344,9 @@
Validators.min("0")),
$MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite
$CODEGEN("tajo.executor.codegen.enabled", false), // Runtime code generation (todo this is broken)
+ $AGG_HASH_TABLE_SIZE("tajo.executor.aggregate.hash-table.size", 10000),
+ $SORT_HASH_TABLE_SIZE("tajo.executor.sort.hash-table.size", 100000),
+ $JOIN_HASH_TABLE_SIZE("tajo.executor.join.hash-table.size", 100000),
// for index
$INDEX_ENABLED("tajo.query.index.enabled", false),
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index e796bad..4ee2c9c 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -138,17 +138,6 @@
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
ProjectionExec proj = (ProjectionExec) exec;
-
- // TODO - should be planed with user's optimization hint
- ExternalSortExec extSort;
- if (!(proj.getChild() instanceof ExternalSortExec)) {
- UnaryPhysicalExec sortExec = proj.getChild();
- SeqScanExec scan = sortExec.getChild();
-
- extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan);
- proj.setChild(extSort);
- }
-
Tuple tuple;
Tuple preVal = null;
Tuple curVal;
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index ae28912..61e35e1 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -34,14 +34,14 @@
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.engine.function.FunctionLoader;
-import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.engine.planner.PhysicalPlanner;
import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.exception.*;
+import org.apache.tajo.exception.TajoException;
+import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
@@ -70,7 +70,6 @@
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME;
import static org.apache.tajo.plan.serder.PlanProto.ColumnPartitionEnforcer.ColumnPartitionAlgorithm;
-import static org.apache.tajo.plan.serder.PlanProto.SortEnforce.SortAlgorithm;
import static org.junit.Assert.*;
public class TestPhysicalPlanner {
@@ -1042,57 +1041,6 @@
};
@Test
- public final void testSortEnforcer() throws IOException, TajoException {
- FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(),
- new Path(employee.getUri()), Integer.MAX_VALUE);
-
- Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testSortEnforcer");
- Expr context = analyzer.parse(SORT_QUERY[0]);
- LogicalPlan plan = planner.createPlan(defaultContext, context);
- optimizer.optimize(plan);
- LogicalNode rootNode = plan.getRootBlock().getRoot();
-
- SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
-
- Enforcer enforcer = new Enforcer();
- enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.IN_MEMORY_SORT);
- TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
- new FileFragment[] {frags[0]}, workDir);
- ctx.setEnforcer(enforcer);
-
- PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf);
- PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
- exec.init();
- exec.next();
- exec.close();
-
- assertTrue(exec instanceof MemSortExec);
-
- context = analyzer.parse(SORT_QUERY[0]);
- plan = planner.createPlan(defaultContext, context);
- optimizer.optimize(plan);
- rootNode = plan.getRootBlock().getRoot();
-
- sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
-
- enforcer = new Enforcer();
- enforcer.enforceSortAlgorithm(sortNode.getPID(), SortAlgorithm.MERGE_SORT);
- ctx = new TaskAttemptContext(new QueryContext(conf),
- LocalTajoTestingUtility.newTaskAttemptId(masterPlan),
- new FileFragment[] {frags[0]}, workDir);
- ctx.setEnforcer(enforcer);
-
- phyPlanner = new PhysicalPlannerImpl(conf);
- exec = phyPlanner.createPlan(ctx, rootNode);
- exec.init();
- exec.next();
- exec.close();
-
- assertTrue(exec instanceof ExternalSortExec);
- }
-
- @Test
public final void testGroupByEnforcer() throws IOException, TajoException {
FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()),
Integer.MAX_VALUE);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index ec41b0d..349aec0 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -32,14 +32,14 @@
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.parser.sql.SQLAnalyzer;
-import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.engine.planner.PhysicalPlanner;
import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
-import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.parser.sql.SQLAnalyzer;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
@@ -153,16 +153,6 @@
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
ProjectionExec proj = (ProjectionExec) exec;
-
- // TODO - should be planed with user's optimization hint
- if (!(proj.getChild() instanceof ExternalSortExec)) {
- UnaryPhysicalExec sortExec = proj.getChild();
- SeqScanExec scan = sortExec.getChild();
-
- ExternalSortExec extSort = new ExternalSortExec(ctx, ((MemSortExec)sortExec).getPlan(), scan);
- proj.setChild(extSort);
- }
-
Tuple tuple;
Tuple preVal = null;
Tuple curVal;
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 58c2a98..15708b1 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -27,6 +27,7 @@
import org.apache.tajo.catalog.*;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.parser.sql.SQLAnalyzer;
@@ -74,6 +75,7 @@
public static void setUp() throws Exception {
conf = new TajoConf();
conf.setBoolVar(TajoConf.ConfVars.$TEST_MODE, true);
+ conf.setIntVar(ConfVars.$SORT_HASH_TABLE_SIZE, 100);
util = TpchTestBase.getInstance().getTestingCluster();
catalog = util.getMaster().getCatalog();
workDir = CommonTestingUtil.getTestDir(TEST_PATH);
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
index 0fe5e0e..0fce97e 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
@@ -18,8 +18,6 @@
package org.apache.tajo.engine.query;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.tajo.*;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.common.TajoDataTypes.Type;
@@ -44,7 +42,6 @@
@Category(IntegrationTest.class)
@RunWith(Parameterized.class)
public class TestGroupByQuery extends QueryTestCaseBase {
- private static final Log LOG = LogFactory.getLog(TestGroupByQuery.class);
public TestGroupByQuery(String groupByOption) throws Exception {
super(TajoConstants.DEFAULT_DATABASE_NAME);
@@ -55,6 +52,7 @@
} else {
variables.put(SessionVars.GROUPBY_MULTI_LEVEL_ENABLED.keyname(), "false");
}
+ variables.put(SessionVars.AGG_HASH_TABLE_SIZE.keyname(), "100");
client.updateSessionVariables(variables);
}
@@ -80,11 +78,11 @@
}
@Test
+ @Option(sort = true)
+ @SimpleTest
public final void testGroupBy2() throws Exception {
// select count(1) as unique_key from lineitem group by l_linenumber;
- ResultSet res = executeQuery();
- assertResultSet(res);
- cleanupQuery(res);
+ runSimpleTests();
}
@Test
@@ -797,10 +795,10 @@
}
@Test
+ @Option(sort = true)
+ @SimpleTest
public final void testPythonUdaf2() throws Exception {
- ResultSet res = executeQuery();
- assertResultSet(res);
- cleanupQuery(res);
+ runSimpleTests();
}
@Test
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
index 90a004b..7835f99 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java
@@ -68,6 +68,8 @@
ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
+ testingCluster.setAllTajoDaemonConfValue(ConfVars.$JOIN_HASH_TABLE_SIZE.keyname(), "100");
+
testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.varname,
ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD.defaultVal);
testingCluster.setAllTajoDaemonConfValue(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD.varname,
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
index 6d6a44c..6022672 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java
@@ -28,6 +28,8 @@
import org.junit.experimental.categories.Category;
import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
@@ -36,6 +38,10 @@
public TestSortQuery() {
super(TajoConstants.DEFAULT_DATABASE_NAME);
+
+ Map<String, String> variables = new HashMap<>();
+ variables.put(SessionVars.SORT_HASH_TABLE_SIZE.keyname(), "100");
+ client.updateSessionVariables(variables);
}
@Test
diff --git a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result
index 6afdd23..1fda90c 100644
--- a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result
+++ b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testGroupBy2.result
@@ -1,4 +1,4 @@
unique_key
-------------------------------
2
-3
\ No newline at end of file
+3
diff --git a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result
index 2852167..c2a8888 100644
--- a/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result
+++ b/tajo-core-tests/src/test/resources/results/TestGroupByQuery/testPythonUdaf2.result
@@ -1,4 +1,4 @@
?countpy,?count_1
-------------------------------
2,2
-3,3
\ No newline at end of file
+3,3
diff --git a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
index 2d87b56..4d4cc2b 100644
--- a/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
+++ b/tajo-core-tests/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
@@ -36,6 +36,9 @@
\set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite.
\set NULL_CHAR [text value] - null char of text file output
\set CODEGEN [true or false] - Runtime code generation enabled (experiment)
+\set AGG_HASH_TABLE_SIZE [int value] - Aggregation hash table size
+\set SORT_HASH_TABLE_SIZE [int value] - Sort hash table size
+\set JOIN_HASH_TABLE_SIZE [int value] - Join hash table size
\set INDEX_ENABLED [true or false] - index scan enabled
\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan
\set PARTITION_NO_RESULT_OVERWRITE_ENABLED [true or false] - If True, a partitioned table is overwritten even if a sub query leads to no result. Otherwise, the table data will be kept if there is no result
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index f471e45..2a1717e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -1165,22 +1165,6 @@
}
}
- Enforcer enforcer = context.getEnforcer();
- EnforceProperty property = getAlgorithmEnforceProperty(enforcer, sortNode);
- if (property != null) {
- SortEnforce.SortAlgorithm algorithm = property.getSort().getAlgorithm();
- if (algorithm == SortEnforce.SortAlgorithm.IN_MEMORY_SORT) {
- return new MemSortExec(context, sortNode, child);
- } else {
- return new ExternalSortExec(context, sortNode, child);
- }
- }
-
- return createBestSortPlan(context, sortNode, child);
- }
-
- public SortExec createBestSortPlan(TaskAttemptContext context, SortNode sortNode,
- PhysicalExec child) throws IOException {
return new ExternalSortExec(context, sortNode, child);
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java
index 62af4e1..192709e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BasicPhysicalExecutorVisitor.java
@@ -55,8 +55,6 @@
return visitHaving(context, (HavingExec) exec, stack);
} else if (exec instanceof LimitExec) {
return visitLimit(context, (LimitExec) exec, stack);
- } else if (exec instanceof MemSortExec) {
- return visitMemSort(context, (MemSortExec) exec, stack);
} else if (exec instanceof MergeFullOuterJoinExec) {
return visitMergeFullOuterJoin(context, (MergeFullOuterJoinExec) exec, stack);
} else if (exec instanceof MergeJoinExec) {
@@ -180,12 +178,6 @@
}
@Override
- public RESULT visitMemSort(CONTEXT context, MemSortExec exec, Stack<PhysicalExec> stack) throws
- PhysicalPlanningException {
- return visitUnaryExecutor(context, exec, stack);
- }
-
- @Override
public RESULT visitMergeFullOuterJoin(CONTEXT context, MergeFullOuterJoinExec exec, Stack<PhysicalExec> stack)
throws PhysicalPlanningException {
return visitBinaryExecutor(context, exec, stack);
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
index a248d52..92a68bd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner.physical;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.engine.planner.KeyProjector;
@@ -158,7 +159,7 @@
protected TupleMap<TupleList> buildRightToHashTableForNonCrossJoin() throws IOException {
Tuple tuple;
- TupleMap<TupleList> map = new TupleMap<TupleList>(100000);
+ TupleMap<TupleList> map = new TupleMap<>(context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE));
KeyProjector keyProjector = new KeyProjector(rightSchema, rightKeyList);
while (!context.isStopped() && (tuple = rightChild.next()) != null) {
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java
index 2d1fa4b..aaade21 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner.physical;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -148,7 +149,7 @@
// Groupby_Key2 | Distinct1_Column_V3 | | |
//--------------------------------------------------------------------------------------
- List<TupleList> tupleSlots = new ArrayList<TupleList>();
+ List<TupleList> tupleSlots = new ArrayList<>();
// aggregation with single grouping key
for (int i = 0; i < hashAggregators.length; i++) {
@@ -359,9 +360,9 @@
public HashAggregator(GroupbyNode groupbyNode, Schema schema) throws IOException {
- hashTable = new TupleMap<TupleMap<FunctionContext[]>>(10000);
+ hashTable = new TupleMap<>(context.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE));
- List<Column> groupingKeyColumnList = new ArrayList<Column>(distinctGroupingKeyColumnSet);
+ List<Column> groupingKeyColumnList = new ArrayList<>(distinctGroupingKeyColumnSet);
Column[] keyColumns = groupbyNode.getGroupingColumns();
Column col;
@@ -399,7 +400,7 @@
TupleMap<FunctionContext[]> distinctEntry = hashTable.get(outerKeyTuple);
if (distinctEntry == null) {
- distinctEntry = new TupleMap<FunctionContext[]>();
+ distinctEntry = new TupleMap<>();
hashTable.put(outerKeyTuple, distinctEntry);
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
index 42d99bb..6ed22c8 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
@@ -118,7 +118,7 @@
this.sortBufferBytesNum = context.getQueryContext().getLong(SessionVars.EXTSORT_BUFFER_SIZE) * StorageUnit.MB;
this.allocatedCoreNum = context.getConf().getIntVar(ConfVars.EXECUTOR_EXTERNAL_SORT_THREAD_NUM);
this.executorService = Executors.newFixedThreadPool(this.allocatedCoreNum);
- this.inMemoryTable = new TupleList(100000);
+ this.inMemoryTable = new TupleList(context.getQueryContext().getInt(SessionVars.SORT_HASH_TABLE_SIZE));
this.sortTmpDir = getExecutorTmpDir();
localDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname);
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java
index b657622..9741982 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner.physical;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.engine.planner.KeyProjector;
import org.apache.tajo.plan.function.FunctionContext;
import org.apache.tajo.plan.logical.GroupbyNode;
@@ -42,7 +43,7 @@
public HashAggregateExec(TaskAttemptContext ctx, GroupbyNode plan, PhysicalExec subOp) throws IOException {
super(ctx, plan, subOp);
hashKeyProjector = new KeyProjector(inSchema, plan.getGroupingColumns());
- hashTable = new TupleMap<FunctionContext []>(10000);
+ hashTable = new TupleMap<>(ctx.getQueryContext().getInt(SessionVars.AGG_HASH_TABLE_SIZE));
this.tuple = new VTuple(plan.getOutSchema().size());
}
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java
deleted file mode 100644
index 029592a..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.engine.planner.physical;
-
-import org.apache.tajo.plan.logical.SortNode;
-import org.apache.tajo.storage.Tuple;
-import org.apache.tajo.worker.TaskAttemptContext;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-public class MemSortExec extends SortExec {
- private SortNode plan;
- private TupleList tupleSlots;
- private boolean sorted = false;
- private Iterator<Tuple> iterator;
-
- public MemSortExec(final TaskAttemptContext context,
- SortNode plan, PhysicalExec child) {
- super(context, plan.getInSchema(), plan.getOutSchema(), child, plan.getSortKeys());
- this.plan = plan;
- }
-
- public void init() throws IOException {
- super.init();
- this.tupleSlots = new TupleList(10000);
- }
-
- @Override
- public Tuple next() throws IOException {
-
- if (!sorted) {
- Tuple tuple;
- while (!context.isStopped() && (tuple = child.next()) != null) {
- tupleSlots.add(tuple);
- }
- iterator = getSorter(tupleSlots).sort().iterator();
- sorted = true;
- }
-
- if (iterator.hasNext()) {
- return this.iterator.next();
- } else {
- return null;
- }
- }
-
- @Override
- public void rescan() throws IOException {
- super.rescan();
- this.iterator = tupleSlots.iterator();
- sorted = true;
- }
-
- @Override
- public void close() throws IOException {
- super.close();
- tupleSlots.clear();
- tupleSlots = null;
- iterator = null;
- plan = null;
- }
-
- public SortNode getPlan() {
- return this.plan;
- }
-}
\ No newline at end of file
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
index 41e3648..824fb0e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import com.google.common.base.Preconditions;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.storage.NullTuple;
@@ -45,8 +46,6 @@
private JoinTupleComparator joincomparator = null;
private TupleComparator[] tupleComparator = null;
- private final static int INITIAL_TUPLE_SLOT = 10000;
-
private boolean end = false;
private int rightNumCols;
@@ -64,6 +63,8 @@
super(context, plan, leftChild, rightChild);
Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " +
"but there is no join condition");
+
+ final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE);
this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
this.rightTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
SortSpec[][] sortSpecs = new SortSpec[2][];
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java
index 3d8c108..ab831b5 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import com.google.common.base.Preconditions;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.storage.Tuple;
@@ -46,8 +47,6 @@
private JoinTupleComparator joincomparator = null;
private TupleComparator [] tupleComparator = null;
- private final static int INITIAL_TUPLE_SLOT = 10000;
-
private boolean end = false;
public MergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer,
@@ -56,6 +55,8 @@
Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " +
"but there is no join condition");
+ final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE);
+
this.outerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
SortSpec[][] sortSpecs = new SortSpec[2][];
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java
index 554c31e..fe86539 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalExecutorVisitor.java
@@ -64,9 +64,6 @@
RESULT visitLimit(CONTEXT context, LimitExec exec, Stack<PhysicalExec> stack)
throws PhysicalPlanningException;
- RESULT visitMemSort(CONTEXT context, MemSortExec exec, Stack<PhysicalExec> stack)
- throws PhysicalPlanningException;
-
RESULT visitMergeFullOuterJoin(CONTEXT context, MergeFullOuterJoinExec exec, Stack<PhysicalExec> stack)
throws PhysicalPlanningException;
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
index 239c6ab..706ec3e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
@@ -19,6 +19,7 @@
package org.apache.tajo.engine.planner.physical;
import com.google.common.base.Preconditions;
+import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.SortSpec;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.storage.NullTuple;
@@ -44,8 +45,6 @@
private JoinTupleComparator joinComparator = null;
private TupleComparator [] tupleComparator = null;
- private final static int INITIAL_TUPLE_SLOT = 10000;
-
private boolean end = false;
private int leftNumCols;
@@ -59,6 +58,8 @@
super(context, plan, outer, inner);
Preconditions.checkArgument(plan.hasJoinQual(), "Sort-merge join is only used for the equi-join, " +
"but there is no join condition");
+
+ final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE);
this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
this.innerTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
SortSpec[][] sortSpecs = new SortSpec[2][];
