src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.pig.tools.pigstats.mapreduce;

import java.io.IOException;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NativeMapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import org.apache.pig.classification.InterfaceAudience.Private;
import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
import org.apache.pig.tools.pigstats.PigStatsUtil;



/**
 * A utility class for Pig Statistics
 */
public class MRPigStatsUtil extends PigStatsUtil {

    public static final String TASK_COUNTER_GROUP
            = "org.apache.hadoop.mapred.Task$Counter";
    public static final String FS_COUNTER_GROUP
            = "org.apache.hadoop.mapreduce.FileSystemCounter";

    private static final Log LOG = LogFactory.getLog(MRPigStatsUtil.class);

    /**
     * Returns the count for the given counter name in the counter group
     * 'MultiStoreCounters'
     *
     * @param job the MR job
     * @param jobClient the Hadoop job client
     * @param counterName the counter name
     * @return the count of the given counter name
     */
    public static long getMultiStoreCount(Job job, JobClient jobClient,
            String counterName) {
        long value = -1;
        try {
            RunningJob rj = jobClient.getJob(job.getAssignedJobID());
            if (rj != null) {
                Counters.Counter counter = rj.getCounters().getGroup(
                        MULTI_STORE_COUNTER_GROUP).getCounterForName(counterName);
                value = counter.getValue();
            }
        } catch (IOException e) {
            LOG.warn("Failed to get the counter for " + counterName, e);
        }
        return value;
    }

    /**
     * Starts collecting statistics for the given MR plan
     *
     * @param pc the Pig context
     * @param client the Hadoop job client
     * @param jcc the job compiler
     * @param plan the MR plan
     */
    public static void startCollection(PigContext pc, JobClient client,
            JobControlCompiler jcc, MROperPlan plan) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        ps.initialize(pc, client, jcc, plan);

        MRScriptState.get().emitInitialPlanNotification(plan);
        MRScriptState.get().emitLaunchStartedNotification(plan.size());
    }

    /**
     * Stops collecting statistics for a MR plan
     *
     * @param display if true, log collected statistics in the Pig log
     *      file at INFO level
     */
    public static void stopCollection(boolean display) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        ps.finish();
        if (!ps.isSuccessful()) {
            LOG.error(ps.getNumberFailedJobs() + " map reduce job(s) failed!");
            String errMsg = ps.getErrorMessage();
            if (errMsg != null) {
                LOG.error("Error message: " + errMsg);
            }
        }
        MRScriptState.get().emitLaunchCompletedNotification(
                ps.getNumberSuccessfulJobs());
        if (display) ps.display();
    }

    /**
     * Add stats for a new Job, which doesn't yet need to be completed.
     *
     * @param job the job being run
     * @return JobStats for the job
     */
    public static JobStats addJobStats(Job job) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        return ps.addMRJobStats(job);
    }

    /**
     * Logs the statistics in the Pig log file at INFO level
     */
    public static void displayStatistics() {
        ((SimplePigStats)PigStats.get()).display();
    }

    /**
     * Updates the {@link JobGraph} of the {@link PigStats}. The initial
     * {@link JobGraph} is created without job ids using {@link MROperPlan},
     * before any job is submitted for execution. The {@link JobGraph} then
     * is updated with job ids after jobs are executed.
     *
     * @param jobMroMap the map that maps {@link Job}s to {@link MapReduceOper}s
     */
    public static void updateJobMroMap(Map<Job, MapReduceOper> jobMroMap) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        for (Map.Entry<Job, MapReduceOper> entry : jobMroMap.entrySet()) {
            MapReduceOper mro = entry.getValue();
            ps.mapMROperToJob(mro, entry.getKey());
        }
    }

    /**
     * Updates the statistics after a patch of jobs is done
     *
     * @param jc the job control
     */
    public static void accumulateStats(JobControl jc) {
        SimplePigStats ps = (SimplePigStats)PigStats.get();
        MRScriptState ss = MRScriptState.get();

        for (Job job : jc.getSuccessfulJobs()) {
            MRJobStats js = addSuccessJobStats(ps, job);
            if (js != null) {
                ss.emitjobFinishedNotification(js);
            }
        }

        for (Job job : jc.getFailedJobs()) {
            MRJobStats js = addFailedJobStats(ps, job);
            if (js != null) {
                js.setErrorMsg(job.getMessage());
                ss.emitJobFailedNotification(js);
            }
        }
    }

    @Private
    public static void setBackendException(Job job, Exception e) {
        JobID jobId = job.getAssignedJobID();
        if (jobId == null) {
            return;
        }
        PigStats.get().setBackendException(jobId.toString(), e);
    }

    private static MRJobStats addFailedJobStats(SimplePigStats ps, Job job) {
        if (ps.isJobSeen(job)) return null;

        MRJobStats js = ps.addMRJobStats(job);
        if (js == null) {
            LOG.warn("unable to add failed job stats");
        } else {
            js.setSuccessful(false);
            js.addOutputStatistics();
            js.addInputStatistics();
        }
        return js;
    }

    public static MRJobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr,
            boolean success) {
        return addNativeJobStats(ps, mr, success, null);
    }

    public static MRJobStats addNativeJobStats(PigStats ps, NativeMapReduceOper mr,
            boolean success, Exception e) {
        if (ps.isEmbedded()) {
            throw new IllegalArgumentException();
        }
        MRJobStats js = ((SimplePigStats)ps).addMRJobStatsForNative(mr);
        if(js == null) {
            LOG.warn("unable to add native job stats");
        } else {
            js.setSuccessful(success);
            if(e != null)
                js.setBackendException(e);
        }
        return js;
    }

    private static MRJobStats addSuccessJobStats(SimplePigStats ps, Job job) {
        if (ps.isJobSeen(job)) return null;

        MRJobStats js = ps.addMRJobStats(job);
        if (js == null) {
            LOG.warn("unable to add job stats");
        } else {
            js.setSuccessful(true);

            js.addMapReduceStatistics(job);

            js.addCounters(job);

            js.addOutputStatistics();

            js.addInputStatistics();
        }
        return js;
    }

}