src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezPlanContainer.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.pig.backend.hadoop.executionengine.tez.plan;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.operator.POStoreTez;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.OperatorPlan;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.JarManager;

public class TezPlanContainer extends OperatorPlan<TezPlanContainerNode> {
    private static final long serialVersionUID = 1L;
    private PigContext pigContext;
    private String jobName;
    private long dagId = 0;
    //Incrementing static counter if multiple pig scripts have same name
    private static long scopeId = 0;

    public TezPlanContainer(PigContext pigContext) {
        this.pigContext = pigContext;
        this.jobName = pigContext.getProperties().getProperty(PigContext.JOB_NAME, "pig");
    }

    // Add the Pig jar and the UDF jars as AM resources (all DAG's in the planContainer
    // will use them for simplicity). This differs from MR Pig, where they are added to
    // the job jar.
    public Map<String, LocalResource> getLocalResources() throws Exception {
        Set<URI> jarLists = new HashSet<URI>();

        for (String jarFile : JarManager.getDefaultJars()) {
            jarLists.add(new File(jarFile).toURI());
        }

        // In MR Pig the extra jars and script jars get put in Distributed Cache, but
        // in Tez we'll add them as local resources.
        for (URL jarUrl : pigContext.extraJars) {
            jarLists.add(jarUrl.toURI());
        }

        for (String jarFile : pigContext.scriptJars) {
            jarLists.add(new File(jarFile).toURI());
        }

        // Script files for non-Java UDF's are added to the Job.jar by the JarManager class,
        // except for Groovy files, which need to be explicitly added as local resources due
        // to the GroovyScriptEngine (see JarManager.java for comments).
        for (Map.Entry<String, File> scriptFile : pigContext.getScriptFiles().entrySet()) {
            if (scriptFile.getKey().endsWith(".groovy")) {
                jarLists.add(scriptFile.getValue().toURI());
            }
        }

        TezPlanContainerUDFCollector tezPlanContainerUDFCollector = new TezPlanContainerUDFCollector(this);
        tezPlanContainerUDFCollector.visit();
        Set<String> udfs = tezPlanContainerUDFCollector.getUdfs();

        for (String func : udfs) {
            Class<?> clazz = pigContext.getClassForAlias(func);
            if (clazz != null) {
                String jarName = JarManager.findContainingJar(clazz);
                if (jarName == null) {
                    // It's possible that clazz is not registered by an external
                    // jar. For eg, in unit tests, UDFs that are not in default
                    // packages may be used. In that case, we should skip it to
                    // avoid NPE.
                    continue;
                }
                URI jarUri = new File(jarName).toURI();
                jarLists.add(jarUri);
            }
        }

        File scriptUDFJarFile = JarManager.createPigScriptUDFJar(pigContext);
        if (scriptUDFJarFile != null) {
            jarLists.add(scriptUDFJarFile.toURI());
        }

        return TezResourceManager.getInstance().addTezResources(jarLists);
    }

    public TezPlanContainerNode getNextPlan(List<TezOperPlan> processedPlans) {
        synchronized(this) {
            while (getRoots()!=null && !getRoots().isEmpty()) {
                TezPlanContainerNode currentPlan = null;
                for (TezPlanContainerNode plan : getRoots()) {
                    if (!processedPlans.contains(plan.getTezOperPlan())) {
                        currentPlan = plan;
                        break;
                    }
                }
                if (currentPlan!=null) {
                    return currentPlan;
                } else {
                    try {
                        wait();
                    } catch (InterruptedException e) {
                    }
                }
            }
        }
        return null;
    }

    public void addPlan(TezOperPlan plan) throws PlanException {
        TezPlanContainerNode node = new TezPlanContainerNode(generateNodeOperatorKey(), plan);
        this.add(node);
        this.split(node);
    }

    public void updatePlan(TezOperPlan plan, boolean succ) {
        TezPlanContainerNode tezPlanContainerNode = new TezPlanContainerNode(
                generateNodeOperatorKey(), plan);
        synchronized(this) {
            if (succ) {
                remove(tezPlanContainerNode);
            } else {
                // job fail
                trimBelow(tezPlanContainerNode);
                remove(tezPlanContainerNode);
            }
            notify();
        }
    }

    public void split(TezPlanContainerNode planNode) throws PlanException {

        TezOperPlan tezOperPlan = planNode.getTezOperPlan();
        if (tezOperPlan.size() == 1) {
            // Nothing to split. Only one operator in the plan
            return;
        }

        List<TezOperator> opersToSegment = null;
        try {
            // Split top down from root to leaves
            // Get list of operators closer to the root that can be segmented together
            FirstLevelSegmentOperatorsFinder finder = new FirstLevelSegmentOperatorsFinder(tezOperPlan);
            finder.visit();
            opersToSegment = finder.getOperatorsToSegment();
        } catch (VisitorException e) {
            throw new PlanException(e);
        }
        if (!opersToSegment.isEmpty()) {
            Set<TezOperator> commonSplitterPredecessors = new HashSet<>();
            for (TezOperator operToSegment : opersToSegment) {
                for (TezOperator succ : tezOperPlan.getSuccessors(operToSegment)) {
                    commonSplitterPredecessors
                            .addAll(getCommonSplitterPredecessors(tezOperPlan,
                                    operToSegment, succ));
                }
            }

            if (commonSplitterPredecessors.isEmpty()) {
                List<TezOperator> allSuccs = new ArrayList<TezOperator>();
                // Disconnect all the successors and move them to a new plan
                for (TezOperator operToSegment : opersToSegment) {
                    List<TezOperator> succs = new ArrayList<TezOperator>();
                    succs.addAll(tezOperPlan.getSuccessors(operToSegment));
                    allSuccs.addAll(succs);
                    for (TezOperator succ : succs) {
                        tezOperPlan.disconnect(operToSegment, succ);
                    }
                }
                TezOperPlan newOperPlan = new TezOperPlan();
                for (TezOperator succ : allSuccs) {
                    tezOperPlan.moveTree(succ, newOperPlan);
                }
                TezPlanContainerNode newPlanNode = new TezPlanContainerNode(
                        generateNodeOperatorKey(), newOperPlan);
                add(newPlanNode);
                connect(planNode, newPlanNode);
                split(newPlanNode);
            } else {
                // If there is a common splitter predecessor between operToSegment and the successor,
                // we have to separate out that split to be able to segment.
                // So we store the output of split to a temp store and then change the
                // splittees to load from it.
                String scope = opersToSegment.get(0).getOperatorKey().getScope();
                for (TezOperator splitter : commonSplitterPredecessors) {
                    try {
                        List<TezOperator> succs = new ArrayList<TezOperator>();
                        succs.addAll(tezOperPlan.getSuccessors(splitter));
                        FileSpec fileSpec = TezCompiler.getTempFileSpec(pigContext);
                        POStore tmpStore = getTmpStore(scope, fileSpec);
                        // Replace POValueOutputTez with POStore
                        splitter.plan.remove(splitter.plan.getLeaves().get(0));
                        splitter.plan.addAsLeaf(tmpStore);
                        splitter.segmentBelow = true;
                        splitter.setSplitter(false);
                        for (TezOperator succ : succs) {
                            // Replace POValueInputTez with POLoad
                            POLoad tmpLoad = getTmpLoad(scope, fileSpec);
                            succ.plan.replace(succ.plan.getRoots().get(0), tmpLoad);
                        }
                    } catch (Exception e) {
                        throw new PlanException(e);
                    }
                }
            }
            split(planNode);
        }
    }

    private static class FirstLevelSegmentOperatorsFinder extends TezOpPlanVisitor {

        private List<TezOperator> opersToSegment = new ArrayList<>();

        public FirstLevelSegmentOperatorsFinder(TezOperPlan plan) {
            super(plan, new DependencyOrderWalker<TezOperator, TezOperPlan>(plan));
        }

        public List<TezOperator> getOperatorsToSegment() {
            return opersToSegment;
        }

        @Override
        public void visitTezOp(TezOperator tezOp) throws VisitorException {
            if (tezOp.needSegmentBelow() && getPlan().getSuccessors(tezOp) != null) {
                if (opersToSegment.isEmpty()) {
                    opersToSegment.add(tezOp);
                } else {
                    // If the operator does not have dependency on previous
                    // operators chosen for segmenting then add it to the
                    // operators to be segmented together
                    if (!hasPredecessor(tezOp, opersToSegment)) {
                        opersToSegment.add(tezOp);
                    }
                }
            }
        }

        /**
         * Check if the tezOp has one of the opsToCheck as a predecessor.
         * It can be a immediate predecessor or multiple levels up.
         */
        private boolean hasPredecessor(TezOperator tezOp, List<TezOperator> opsToCheck) {
            List<TezOperator> predecessors = getPlan().getPredecessors(tezOp);
            if (predecessors != null) {
                for (TezOperator pred : predecessors) {
                    if (opersToSegment.contains(pred)) {
                        return true;
                    } else {
                        if (hasPredecessor(pred, opsToCheck)) {
                            return true;
                        }
                    }
                }
            }
            return false;
        }

    }

    private Set<TezOperator> getCommonSplitterPredecessors(TezOperPlan plan, TezOperator operToSegment, TezOperator successor) {
        Set<TezOperator> splitters1 = new HashSet<>();
        Set<TezOperator> splitters2 = new HashSet<>();
        Set<TezOperator> processedPredecessors = new HashSet<>();
        // Find predecessors which are splitters
        fetchSplitterPredecessors(plan, operToSegment, processedPredecessors, splitters1);
        if (!splitters1.isEmpty()) {
            // For the successor, traverse rest of the plan below it and
            // search the predecessors of its successors to find any predecessor that might be a splitter.
            Set<TezOperator> allSuccs = new HashSet<>();
            getAllSuccessors(plan, successor, allSuccs);
            processedPredecessors.clear();
            processedPredecessors.add(successor);
            for (TezOperator succ : allSuccs) {
                fetchSplitterPredecessors(plan, succ, processedPredecessors, splitters2);
            }
            // Find the common ones
            splitters1.retainAll(splitters2);
        }
        return splitters1;
    }

    private void fetchSplitterPredecessors(TezOperPlan plan, TezOperator tezOp,
            Set<TezOperator> processedPredecessors, Set<TezOperator> splitters) {
        List<TezOperator> predecessors = plan.getPredecessors(tezOp);
        if (predecessors != null) {
            for (TezOperator pred : predecessors) {
                // Skip processing already processed predecessor to avoid loops
                if (processedPredecessors.contains(pred)) {
                    continue;
                }
                if (pred.isSplitter()) {
                    splitters.add(pred);
                } else if (!pred.needSegmentBelow()) {
                    processedPredecessors.add(pred);
                    fetchSplitterPredecessors(plan, pred, processedPredecessors, splitters);
                }
            }
        }
    }

    private void getAllSuccessors(TezOperPlan plan, TezOperator tezOp, Set<TezOperator> allSuccs) {
        List<TezOperator> successors = plan.getSuccessors(tezOp);
        if (successors != null) {
            for (TezOperator succ : successors) {
                if (!allSuccs.contains(succ)) {
                    allSuccs.add(succ);
                    getAllSuccessors(plan, succ, allSuccs);
                }
            }
        }
    }

    private synchronized OperatorKey generateNodeOperatorKey() {
        OperatorKey opKey = new OperatorKey(jobName + "-" + dagId + "_scope", scopeId);
        scopeId++;
        dagId++;
        return opKey;
    }

    public static void resetScope() {
        scopeId = 0;
    }

    private POLoad getTmpLoad(String scope, FileSpec fileSpec){
        POLoad ld = new POLoad(new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        ld.setPc(pigContext);
        ld.setIsTmpLoad(true);
        ld.setLFile(fileSpec);
        return ld;
    }

    private POStore getTmpStore(String scope, FileSpec fileSpec){
        POStore st = new POStore(new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)));
        st.setIsTmpStore(true);
        st.setSFile(fileSpec);
        return new POStoreTez(st);
    }

    @Override
    public String toString() {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintStream ps = new PrintStream(baos);
        TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, this);
        printer.setVerbose(true);
        try {
            printer.visit();
        } catch (VisitorException e) {
            throw new RuntimeException(e);
        }
        return baos.toString();
    }
}