tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestUnionQuery.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tajo.engine.query;

import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.*;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.jdbc.TajoResultSetBase;
import org.apache.tajo.master.QueryInfo;
import org.junit.AfterClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import java.util.Optional;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

/*
 * Notations
 * - S - select
 * - SA - select *
 * - U - union
 * - G - group by
 * - O - order by
 */
@Category(IntegrationTest.class)
public class TestUnionQuery extends QueryTestCaseBase {

  public TestUnionQuery() {
    super(TajoConstants.DEFAULT_DATABASE_NAME);
    conf.setBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED, true);
  }

  @AfterClass
  public static void tearDown() throws Exception {
    conf.setBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED, false);
  }

  /**
   * S (SA U SA) O
   */
  @Test
  @SimpleTest
  public final void testUnionAll1() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 14L);
  }

  /**
   * S (S U S) O
   */
  @Test
  @SimpleTest
  public final void testUnionAll2() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 16L);
  }

  /**
   * S O ((S G) U (S G))
   */
  @Test
  @SimpleTest
  public final void testUnionAll3() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 2L);
  }

  /**
   * S G (S G)
   */
  @Test
  @SimpleTest
  public final void testUnionAll4() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S G (S F G)
   */
  @Test
  @SimpleTest
  public final void testUnionAll5() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S G (SA)
   */
  @Test
  @SimpleTest
  public final void testUnionAll6() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S (SA)
   */
  @Test
  @SimpleTest
  public final void testUnionAll7() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 16L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll8() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll9() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll10() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 20L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll11() throws Exception {
    // test filter pushdown
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll12() throws Exception {
    // test filter pushdown
    // with stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll13() throws Exception {
    // test filter pushdown
    // with stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 8L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll14() throws Exception {
    // test filter pushdown
    // with group by stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 9L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll15() throws Exception {
    // test filter pushdown
    // with group by out of union query and join in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 3L);
  }

  @Test
  @SimpleTest
  public final void testUnionAll16() throws Exception {
    // test filter pushdown
    // with count distinct out of union query and join in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 3L);
  }

  /**
   * S (SA U SA) O
   */
  @Test
  @SimpleTest
  public final void testUnion1() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 6L);
  }

  /**
   * S (S U S) O
   */
  @Test
  @SimpleTest
  public final void testUnion2() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 4L);
  }

  /**
   * S O ((S G) U (S G))
   */
  @Test
  @SimpleTest
  public final void testUnion3() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 2L);
  }

  /**
   * S G (S G)
   */
  @Test
  @SimpleTest
  public final void testUnion4() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S G (S F G)
   */
  @Test
  @SimpleTest
  public final void testUnion5() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S G (SA)
   */
  @Test
  @SimpleTest
  public final void testUnion6() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  /**
   * S (SA)
   */
  @Test
  @SimpleTest
  public final void testUnion7() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 4L);
  }

  @Test
  @SimpleTest
  public final void testUnion8() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  @Test
  @SimpleTest
  public final void testUnion9() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnion10() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnion11() throws Exception {
    // test filter pushdown
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 1L);
  }

  @Test
  @SimpleTest
  public final void testUnion12() throws Exception {
    // test filter pushdown
    // with stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnion13() throws Exception {
    // test filter pushdown
    // with stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 6L);
  }

  @Test
  @SimpleTest
  public final void testUnion14() throws Exception {
    // test filter pushdown
    // with group by stage in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 8L);
  }

  @Test
  @SimpleTest
  public final void testUnion15() throws Exception {
    // test filter pushdown
    // with group by out of union query and join in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 3L);
  }

  @Test
  @SimpleTest
  public final void testUnion16() throws Exception {
    // test filter pushdown
    // with count distinct out of union query and join in union query
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 3L);
  }

  @Test
  @SimpleTest
  public final void testUnionAllWithSameAliasNames() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 16L);
  }

  @Test
  @SimpleTest
  public final void testUnionAllWithDifferentAlias() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 2L);
  }

  @Test
  @SimpleTest
  public final void testUnionAllWithDifferentAliasAndFunction() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testUnionWithSameAliasNames() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 4L);
  }

  @Test
  @SimpleTest
  public final void testUnionWithDifferentAlias() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 2L);
  }

  @Test
  @SimpleTest
  public final void testUnionWithDifferentAliasAndFunction() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public final void testLeftUnionWithJoin() throws Exception {
    // https://issues.apache.org/jira/browse/TAJO-881
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 8L);
  }

  @Test
  @SimpleTest
  public final void testRightUnionWithJoin() throws Exception {
    // https://issues.apache.org/jira/browse/TAJO-881
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 8L);
  }

  @Test
  @SimpleTest
  public final void testAllUnionWithJoin() throws Exception {
    // https://issues.apache.org/jira/browse/TAJO-881
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 16L);
  }

  @Test
  @SimpleTest
  public final void testUnionWithCrossJoin() throws Exception {
    // https://issues.apache.org/jira/browse/TAJO-881
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 112L);
  }

  @Test
  @SimpleTest
  public final void testThreeJoinInUnion() throws Exception {
    // https://issues.apache.org/jira/browse/TAJO-881
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 33L);
  }

  @Test
  @SimpleTest
  public void testUnionCaseOfFirstEmptyAndJoin() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @SimpleTest
  public void testTajo1368Case1() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 10L);
  }

  @Test
  @SimpleTest
  public void testTajo1368Case2() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 16L);
  }

  @Test
  @Option(withExplain = true, withExplainGlobal = true)
  @SimpleTest
  public void testComplexUnion1() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 4L);
  }

  @Test
  @Option(withExplain = true, withExplainGlobal = true, sort = true)
  @SimpleTest
  public void testComplexUnion2() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  @Test
  @Option(withExplain =  true, sort = true)
  @SimpleTest
  public void testUnionAndFilter() throws Exception {
    Optional<TajoResultSetBase[]> existing = runSimpleTests();
    verifyResultStats(existing, 5L);
  }

  private void verifyResultStats(Optional<TajoResultSetBase[]> existing, long numRows) throws Exception {
    assertTrue(existing.isPresent());

    // Get TableStats using TajoResultSetBase.
    TajoResultSetBase[] resultSet = existing.get();
    QueryId qid = resultSet[0].getQueryId();
    QueryInfo queryInfo = testingCluster.getMaster().getContext().getQueryJobManager().getFinishedQuery(qid);
    TableDesc desc = queryInfo.getResultDesc();
    TableStats stats = desc.getStats();

    // Compare specified number of rows to the number of rows in TableStats.
    assertEquals(numRows, stats.getNumRows().longValue());

    // Compare the volume number of directRaw to the number of rows in TableStats.
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(desc.getUri());
    assertTrue(fs.exists(path));
    ContentSummary summary = fs.getContentSummary(path);
    assertEquals(summary.getLength(), stats.getNumBytes().longValue());

    closeResultSets(resultSet);
  }

}