| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.pig.backend.hadoop20; |
| |
| import java.lang.reflect.Field; |
| import java.lang.reflect.Method; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.mapred.jobcontrol.JobControl; |
| import org.apache.pig.ExecType; |
| import org.apache.pig.impl.PigContext; |
| |
| /** |
| * extends the hadoop JobControl to remove the hardcoded sleep(5000) |
| * as most of this is private we have to use reflection |
| * |
| * See {@link https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/jobcontrol/JobControl.java} |
| * |
| */ |
| public class PigJobControl extends JobControl { |
| private static final Log log = LogFactory.getLog(PigJobControl.class); |
| |
| private static Field runnerState; |
| |
| private static Method checkRunningJobs; |
| private static Method checkWaitingJobs; |
| private static Method startReadyJobs; |
| |
| private static boolean initSuccesful; |
| |
| static { |
| try { |
| |
| runnerState = JobControl.class.getDeclaredField("runnerState"); |
| runnerState.setAccessible(true); |
| |
| checkRunningJobs = JobControl.class.getDeclaredMethod("checkRunningJobs"); |
| checkRunningJobs.setAccessible(true); |
| checkWaitingJobs = JobControl.class.getDeclaredMethod("checkWaitingJobs"); |
| checkWaitingJobs.setAccessible(true); |
| startReadyJobs = JobControl.class.getDeclaredMethod("startReadyJobs"); |
| startReadyJobs.setAccessible(true); |
| initSuccesful = true; |
| } catch (Exception e) { |
| log.warn("falling back to default JobControl (not using hadoop 0.20 ?)", e); |
| initSuccesful = false; |
| } |
| } |
| |
| // The thread can be in one of the following state |
| private static final int RUNNING = 0; |
| private static final int SUSPENDED = 1; |
| private static final int STOPPED = 2; |
| private static final int STOPPING = 3; |
| private static final int READY = 4; |
| |
| private int timeToSleep; |
| |
| /** |
| * Construct a job control for a group of jobs. |
| * @param groupName a name identifying this group |
| * @param pigContext |
| * @param conf |
| */ |
| public PigJobControl(String groupName, int timeToSleep) { |
| super(groupName); |
| this.timeToSleep = timeToSleep; |
| } |
| |
| public int getTimeToSleep() { |
| return timeToSleep; |
| } |
| |
| public void setTimeToSleep(int timeToSleep) { |
| this.timeToSleep = timeToSleep; |
| } |
| |
| private void setRunnerState(int state) { |
| try { |
| runnerState.set(this, state); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| } |
| |
| |
| private int getRunnerState() { |
| try { |
| return (Integer)runnerState.get(this); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| } |
| |
| /** |
| * The main loop for the thread. |
| * The loop does the following: |
| * Check the states of the running jobs |
| * Update the states of waiting jobs |
| * Submit the jobs in ready state |
| */ |
| public void run() { |
| if (!initSuccesful) { |
| super.run(); |
| return; |
| } |
| setRunnerState(PigJobControl.RUNNING); |
| while (true) { |
| while (getRunnerState() == PigJobControl.SUSPENDED) { |
| try { |
| Thread.sleep(timeToSleep); |
| } |
| catch (Exception e) { |
| |
| } |
| } |
| mainLoopAction(); |
| if (getRunnerState() != PigJobControl.RUNNING && |
| getRunnerState() != PigJobControl.SUSPENDED) { |
| break; |
| } |
| try { |
| Thread.sleep(timeToSleep); |
| } |
| catch (Exception e) { |
| |
| } |
| if (getRunnerState() != PigJobControl.RUNNING && |
| getRunnerState() != PigJobControl.SUSPENDED) { |
| break; |
| } |
| } |
| setRunnerState(PigJobControl.STOPPED); |
| } |
| |
| private void mainLoopAction() { |
| try { |
| checkRunningJobs.invoke(this); |
| checkWaitingJobs.invoke(this); |
| startReadyJobs.invoke(this); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| } |
| |
| } |