KYLIN-6034 fix columns of OlapContext when top node is OlapWindowRel
Co-authored-by: Xuecheng Shan <xuecheng.shan@kyligence.io>
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java
new file mode 100644
index 0000000..1f8244f
--- /dev/null
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/WindowFunctionPlannerTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.engine;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
+import org.apache.kylin.query.relnode.ContextUtil;
+import org.apache.kylin.query.relnode.OlapContext;
+import org.apache.kylin.query.relnode.OlapLimitRel;
+import org.apache.kylin.query.relnode.OlapWindowRel;
+import org.apache.kylin.query.rules.CalciteRuleTestBase;
+import org.apache.kylin.query.util.QueryContextCutter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class WindowFunctionPlannerTest extends CalciteRuleTestBase {
+
+ String project = "default";
+
+ QueryExec queryExec;
+
+ @Before
+ public void setup() {
+ createTestMetadata();
+ queryExec = new QueryExec(project, getTestConfig());
+ }
+
+ private void prepareOlapContext(String sql) throws SqlParseException {
+ RelNode relNode = queryExec.parseAndOptimize(sql);
+ List<RelNode> optimizedRelNodes = queryExec.postOptimize(relNode);
+
+ try {
+ QueryContextCutter.selectRealization(project, optimizedRelNodes.get(0),
+ QueryContext.current().isForModeling());
+ } catch (NoRealizationFoundException e) {
+ log.info("No realization found for sql: {}", sql);
+ }
+ }
+
+ private String getSql(String fileName) throws IOException {
+ Pair<String, String> sql = readOneSQL(getTestConfig(), project, "query/sql_window_olap_context", fileName);
+ return sql.getSecond();
+ }
+
+ @Test
+ public void testSelectStarAndWindow() throws SqlParseException, IOException {
+ prepareOlapContext(getSql("query00.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Assert.assertEquals(17, olapContext.getAllColumns().size());
+ Assert.assertTrue(olapContext.getTopNode() instanceof OlapWindowRel);
+ }
+
+ @Test
+ public void testSelectStarAndWindowWithLimit() throws SqlParseException, IOException {
+ prepareOlapContext(getSql("query01.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Assert.assertEquals(17, olapContext.getAllColumns().size());
+ Assert.assertTrue(olapContext.getTopNode() instanceof OlapLimitRel);
+ }
+
+ @Test
+ public void testSelectOnlyWindow() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query02.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(2, allColumns.size());
+ Set<String> expectedColumns = Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' + tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSelectWindowAndSpecifiedColumn() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query03.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(3, allColumns.size());
+ Set<String> expectedColumns = Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_CUSTKEY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' + tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSelectWindowAndFilter() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query04.sql"));
+
+ Assert.assertEquals(1, ContextUtil.listContexts().size());
+ OlapContext olapContext = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns = olapContext.getAllColumns();
+ Assert.assertEquals(4, allColumns.size());
+ Set<String> expectedColumns = Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_CUSTKEY", "SSB.LINEORDER.LO_PARTKEY");
+ allColumns.forEach(tblColRef -> expectedColumns
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' + tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns.isEmpty());
+ }
+
+ @Test
+ public void testSubQueryWithWidow() throws IOException, SqlParseException {
+ prepareOlapContext(getSql("query05.sql"));
+
+ Assert.assertEquals(2, ContextUtil.listContexts().size());
+
+ OlapContext olapContext1 = ContextUtil.listContexts().get(0);
+ Set<TblColRef> allColumns1 = olapContext1.getAllColumns();
+ Assert.assertEquals(4, allColumns1.size());
+ Set<String> expectedColumns1 = Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_ORDERKEY", "SSB.LINEORDER.LO_PARTKEY");
+ allColumns1.forEach(tblColRef -> expectedColumns1
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' + tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns1.isEmpty());
+
+ OlapContext olapContext2 = ContextUtil.listContexts().get(1);
+ Set<TblColRef> allColumns2 = olapContext2.getAllColumns();
+ Assert.assertEquals(5, allColumns2.size());
+ Set<String> expectedColumns2 = Sets.newHashSet("SSB.LINEORDER.LO_ORDERDATE", "SSB.LINEORDER.LO_QUANTITY",
+ "SSB.LINEORDER.LO_ORDERKEY", "SSB.LINEORDER.LO_PARTKEY", "SSB.LINEORDER.LO_CUSTKEY");
+ allColumns2.forEach(tblColRef -> expectedColumns2
+ .remove(tblColRef.getTableRef().getTableIdentity() + '.' + tblColRef.getColumnDesc().getName()));
+ Assert.assertTrue(expectedColumns2.isEmpty());
+ }
+}
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql
new file mode 100644
index 0000000..c00f0ab
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query00.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql
new file mode 100644
index 0000000..f248deb
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query01.sql
@@ -0,0 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
+LIMIT 100
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql
new file mode 100644
index 0000000..832667e
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query02.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql
new file mode 100644
index 0000000..3cdacbb
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query03.sql
@@ -0,0 +1,20 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT LO_ORDERDATE, LO_CUSTKEY, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql
new file mode 100644
index 0000000..d00a6c4
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query04.sql
@@ -0,0 +1,21 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT LO_ORDERDATE, LO_CUSTKEY, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN
+FROM SSB.LINEORDER
+WHERE LO_PARTKEY < 1000
\ No newline at end of file
diff --git a/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql
new file mode 100644
index 0000000..7872fce
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_window_olap_context/query05.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select t1.LO_ORDERDATE, t1.LO_ORDERKEY, t2.LO_CUSTKEY, t1.RN_1, t2.RN_2
+from
+(SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN_1
+ FROM SSB.LINEORDER
+where LO_PARTKEY<1000) t1
+left join
+(SELECT *, ROW_NUMBER() OVER (PARTITION BY LO_ORDERDATE ORDER BY LO_QUANTITY) AS RN_2
+ FROM SSB.LINEORDER
+where LO_PARTKEY>2000) t2
+on t1.LO_ORDERKEY=t2.LO_ORDERKEY
\ No newline at end of file
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
index 661a30c..6d81957 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/ContextUtil.java
@@ -105,8 +105,12 @@
((OlapRel) rel).getColumnRowType().getAllColumns().stream().filter(context::isOriginAndBelongToCtxTables)
.forEach(context.getAllColumns()::add);
} else if (rel instanceof OlapWindowRel) {
- ((OlapWindowRel) rel).getGroupingColumns().stream().filter(context::isOriginAndBelongToCtxTables)
+ OlapWindowRel olapWindowRel = (OlapWindowRel) rel;
+ olapWindowRel.getGroupingColumns().stream().filter(context::isOriginAndBelongToCtxTables)
.forEach(context.getAllColumns()::add);
+ if (!olapWindowRel.isExistParentProjectNeedPushInfo()) {
+ amendAllColsIfNoAgg(olapWindowRel.getInput());
+ }
} else if (rel instanceof OlapJoinRel) {
amendAllColsIfNoAgg(rel.getInput(0));
amendAllColsIfNoAgg(rel.getInput(1));
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
index 6d7afe0..48cd6d7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OlapWindowRel.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Deque;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -55,6 +56,8 @@
@Setter
private Set<OlapContext> subContexts = Sets.newHashSet();
+ private boolean existParentProjectNeedPushInfo;
+
public OlapWindowRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, List<RexLiteral> constants,
RelDataType rowType, List<Window.Group> groups) {
super(cluster, traitSet, input, constants, rowType, groups);
@@ -87,12 +90,33 @@
if (context != null) {
this.context.setHasWindow(true);
if (this == context.getTopNode() && !context.isHasAgg())
- ContextUtil.amendAllColsIfNoAgg(this);
+ amendContextColumns(olapImpl);
} else {
ContextUtil.updateSubContexts(getGroupingColumns(), subContexts);
}
}
+ private void amendContextColumns(OlapImpl olapImpl) {
+ this.existParentProjectNeedPushInfo = checkParentProjectNeedPushInfo(olapImpl.getParentNodeStack());
+ ContextUtil.amendAllColsIfNoAgg(this);
+ }
+
+ public boolean checkParentProjectNeedPushInfo(Deque<RelNode> allParents) {
+ for (RelNode parent : allParents) {
+ if (parent instanceof OlapProjectRel) {
+ OlapProjectRel parentProject = (OlapProjectRel) parent;
+ if (parentProject.isNeedPushInfoToSubCtx()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public boolean isExistParentProjectNeedPushInfo() {
+ return existParentProjectNeedPushInfo;
+ }
+
protected ColumnRowType buildColumnRowType() {
OlapRel olapChild = (OlapRel) getInput(0);
ColumnRowType inputColumnRowType = olapChild.getColumnRowType();
