test/org/apache/pig/tez/TestTezLauncher.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pig.tez;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.util.Arrays;
import java.util.Iterator;

import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
import org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.test.MiniGenericCluster;
import org.apache.pig.test.Util;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.tez.dag.api.TezConfiguration;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

/**
 * Test cases to test the TezLauncher.
 */
public class TestTezLauncher {
    private static PigContext pc;
    private static PigServer pigServer;
    private static MiniGenericCluster cluster;

    private static final String INPUT_FILE = "TestTezLauncherInput";
    private static final String[] INPUT_RECORDS = {
        "100\tapple",
        "200\torange",
        "300\tstrawberry",
        "300\tpear",
        "100\tapple",
        "300\tpear",
        "400\tapple",
    };

    private static final String OUTPUT_FILE = "TestTezLauncherOutput";
    private static final String[] OUTPUT_RECORDS = new String[] {
        "(apple)", "(pear)", "(pear)", "(strawberry)", "(orange)"
    };

    @BeforeClass
    public static void setUpBeforeClass() throws Exception {
        cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
        pc = new PigContext(new TezExecType(), cluster.getProperties());
        Util.createInputFile(cluster, INPUT_FILE, INPUT_RECORDS);
    }

    @AfterClass
    public static void tearDownAfterClass() throws Exception {
        cluster.shutDown();
    }

    @Before
    public void setUp() throws Exception {
        pigServer = new PigServer(pc);
    }

    @After
    public void tearDown() throws Exception {
        Util.deleteFile(cluster, OUTPUT_FILE);
    }

    @Test
    public void testRun1() throws Exception {
        String query =
                "a = load '" + INPUT_FILE + "' as (x:int, y:chararray);" +
                "b = filter a by x > 100;" +
                "c = foreach b generate y;" +
                "d = group c all;" +
                "store d into '" + OUTPUT_FILE + "';";

        PhysicalPlan pp = Util.buildPp(pigServer, query);
        TezLauncher launcher = new TezLauncher();
        PigStats pigStats = launcher.launchPig(pp, "testRun1", pc);
        assertTrue(pigStats.isSuccessful());

        assertEquals(1, pigStats.getInputStats().size());
        assertEquals(INPUT_FILE, pigStats.getInputStats().get(0).getName());

        assertEquals(1, pigStats.getOutputStats().size());
        assertEquals(OUTPUT_FILE, pigStats.getOutputStats().get(0).getName());

        query = "m = load '" + OUTPUT_FILE + "' as (a:chararray, b:{(y:chararray)});";
        pigServer = new PigServer(pc);
        pigServer.registerQuery(query);
        Iterator<Tuple> iter = pigServer.openIterator("m");
        Tuple result = iter.next();
        assertEquals(result.get(0).toString(), "all");
        Iterator<Tuple> innerIter = ((DataBag)result.get(1)).iterator();
        int count = 0;
        while (innerIter.hasNext()) {
            assertTrue(Arrays.asList(OUTPUT_RECORDS).contains(innerIter.next().toString()));
            count++;
        }
        assertEquals(count, OUTPUT_RECORDS.length);
    }

    @Test
    public void testQueueName() throws Exception {
        TezConfiguration conf = new TezConfiguration();
        conf.set("tez.queue.name", "special");
        MRToTezHelper.translateMRSettingsForTezAM(conf);
        assertEquals(conf.get("tez.queue.name"), "special");

    }
}