| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.hadoop.mapred; |
| |
| import java.io.IOException; |
| import java.util.Collection; |
| import java.util.Map; |
| |
| |
| /** |
| Test the dynamic scheduler. |
| Use the System Property test.build.data to drive the test run |
| */ |
| public class TestDynamicScheduler extends BaseSchedulerTest { |
| |
| private DynamicPriorityScheduler scheduler; |
| |
| |
| /** |
| * Create the test queues |
| * @throws Exception |
| */ |
| @Override |
| protected void setUp() throws Exception { |
| super.setUp(); |
| conf.set(PrioritySchedulerOptions.DYNAMIC_SCHEDULER_SCHEDULER, |
| "org.apache.hadoop.mapred.FakeDynamicScheduler"); |
| scheduler = new DynamicPriorityScheduler(); |
| scheduler.setTimer(timer); |
| scheduler.setConf(conf); |
| scheduler.setTaskTrackerManager(taskTracker); |
| taskTracker.addQueues(QUEUES); |
| scheduler.start(); |
| } |
| |
| /** |
| * Remove the queues |
| * @throws Exception |
| */ |
| @Override |
| protected void tearDown() throws Exception { |
| super.tearDown(); |
| removeQueues(QUEUES); |
| } |
| |
| |
| private void setSpending(String queue, float spending) throws IOException { |
| scheduler.allocations.setSpending(queue, spending); |
| } |
| |
| private void setBudgets(String[] queue, float[] budget) throws IOException { |
| for (int i = 0; i < queue.length; i++) { |
| scheduler.allocations.addBudget(queue[i], budget[i]); |
| } |
| } |
| |
| private void addQueues(String[] queue) throws IOException { |
| for (String aQueue : queue) { |
| scheduler.allocations.addQueue(aQueue); |
| } |
| } |
| private void removeQueues(String[] queue) throws IOException { |
| for (String aQueue : queue) { |
| scheduler.allocations.removeQueue(aQueue); |
| } |
| } |
| |
| |
| public void testAllocation() throws IOException { |
| addQueues(QUEUES); |
| setSpending("queue1", 1.0f); |
| setSpending("queue2", 2.0f); |
| setBudgets(QUEUES, new float[] {100.0f, 100.0f}); |
| scheduler.allocations.setUsage("queue1",2,0); |
| scheduler.allocations.setUsage("queue2",3,0); |
| timer.runTask(); |
| assertNotNull(scheduler.allocations); |
| assertNotNull(scheduler.allocations.allocation); |
| assertNotNull(scheduler.allocations.allocation.get("queue1")); |
| assertNotNull(scheduler.allocations.allocation.get("queue2")); |
| Collection<BudgetQueue> budgetQueues = |
| scheduler.allocations.store.getQueues(); |
| assertNotNull(budgetQueues); |
| assertEquals(2, budgetQueues.size()); |
| BudgetQueue queue1Budget = null; |
| BudgetQueue queue2Budget = null; |
| for (BudgetQueue queue: budgetQueues) { |
| if (queue.name.equals("queue1")) { |
| queue1Budget = queue; |
| } else { |
| queue2Budget = queue; |
| } |
| } |
| assertNotNull(queue1Budget); |
| assertNotNull(queue2Budget); |
| |
| assertEquals(98.0f, queue1Budget.budget, 0.1f); |
| assertEquals(94.0f, queue2Budget.budget, 0.1f); |
| assertEquals(1.0f, queue1Budget.spending, 0.1f); |
| assertEquals(2.0f, queue2Budget.spending, 0.1f); |
| |
| Map<String,QueueAllocation> shares = scheduler.allocations.getAllocation(); |
| assertNotNull(shares); |
| assertEquals(2, shares.size()); |
| assertNotNull(shares.get("queue1")); |
| assertNotNull(shares.get("queue2")); |
| assertEquals(1.0f/3.0f, shares.get("queue1").getShare(), 0.1f); |
| assertEquals(2.0f/3.0f, shares.get("queue2").getShare(), 0.1f); |
| } |
| |
| public void testBudgetUpdate() throws IOException { |
| addQueues(QUEUES); |
| setSpending("queue1", 1.0f); |
| setSpending("queue2", 2.0f); |
| setBudgets(QUEUES, new float[] {100.0f, 200.0f}); |
| timer.runTask(); |
| Collection<BudgetQueue> budgetQueues = |
| scheduler.allocations.store.getQueues(); |
| BudgetQueue queue1Budget = null; |
| BudgetQueue queue2Budget = null; |
| for (BudgetQueue queue: budgetQueues) { |
| if (queue.name.equals("queue1")) { |
| queue1Budget = queue; |
| } else { |
| queue2Budget = queue; |
| } |
| } |
| assertNotNull(queue1Budget); |
| assertNotNull(queue2Budget); |
| assertEquals(100.0f, queue1Budget.budget, 0.1f); |
| assertEquals(200.0f, queue2Budget.budget, 0.1f); |
| setBudgets(QUEUES, new float[] {200.0f, 300.0f}); |
| timer.runTask(); |
| budgetQueues = scheduler.allocations.store.getQueues(); |
| for (BudgetQueue queue: budgetQueues) { |
| if (queue.name.equals("queue1")) { |
| queue1Budget = queue; |
| } else { |
| queue2Budget = queue; |
| } |
| } |
| assertEquals(300.0f, queue1Budget.budget, 0.1f); |
| assertEquals(500.0f, queue2Budget.budget, 0.1f); |
| removeQueues(QUEUES); |
| } |
| |
| public void testSpendingUpdate() throws IOException { |
| addQueues(QUEUES); |
| setSpending("queue1", 1.0f); |
| setSpending("queue2", 2.0f); |
| setBudgets(QUEUES, new float[] {100.0f, 100.0f}); |
| scheduler.allocations.setUsage("queue1", 1, 0); |
| scheduler.allocations.setUsage("queue2", 1, 0); |
| timer.runTask(); |
| Map<String,QueueAllocation> shares = |
| scheduler.allocations.getAllocation(); |
| assertEquals(1.0f/3.0f, shares.get("queue1").getShare(), 0.1f); |
| assertEquals(2.0f/3.0f, shares.get("queue2").getShare(), 0.1f); |
| setSpending("queue1", 5.0f); |
| setSpending("queue2", 1.0f); |
| timer.runTask(); |
| shares = scheduler.allocations.getAllocation(); |
| assertEquals(5.0f/6.0f, shares.get("queue1").getShare(), 0.1f); |
| assertEquals(1.0f/6.0f, shares.get("queue2").getShare(), 0.1f); |
| } |
| } |