branch-2.0.4-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestFailoverController.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.ha;

import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;

import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HAServiceProtocol.RequestSource;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer;
import org.apache.hadoop.ha.TestNodeFencer.AlwaysFailFencer;
import static org.apache.hadoop.ha.TestNodeFencer.setupFencer;
import org.apache.hadoop.security.AccessControlException;

import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.stubbing.answers.ThrowsException;

import static org.junit.Assert.*;

public class TestFailoverController {
  private InetSocketAddress svc1Addr = new InetSocketAddress("svc1", 1234); 
  private InetSocketAddress svc2Addr = new InetSocketAddress("svc2", 5678);
  
  private Configuration conf = new Configuration();

  HAServiceStatus STATE_NOT_READY = new HAServiceStatus(HAServiceState.STANDBY)
      .setNotReadyToBecomeActive("injected not ready");

  @Test
  public void testFailoverAndFailback() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    AlwaysSucceedFencer.fenceCalled = 0;
    doFailover(svc1, svc2, false, false);
    assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.ACTIVE, svc2.state);

    AlwaysSucceedFencer.fenceCalled = 0;
    doFailover(svc2, svc1, false, false);
    assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testFailoverFromStandbyToStandby() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    doFailover(svc1, svc2, false, false);
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.ACTIVE, svc2.state);
  }

  @Test
  public void testFailoverFromActiveToActive() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Can't failover to an already active service");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.ACTIVE, svc2.state);
  }

  @Test
  public void testFailoverWithoutPermission() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    Mockito.doThrow(new AccessControlException("Access denied"))
        .when(svc1.proxy).getServiceStatus();
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new AccessControlException("Access denied"))
        .when(svc2.proxy).getServiceStatus();
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Can't failover when access is denied");
    } catch (FailoverFailedException ffe) {
      assertTrue(ffe.getCause().getMessage().contains("Access denied"));
    }
  }


  @Test
  public void testFailoverToUnreadyService() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
        .getServiceStatus();
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Can't failover to a service that's not ready");
    } catch (FailoverFailedException ffe) {
      // Expected
      if (!ffe.getMessage().contains("injected not ready")) {
        throw ffe;
      }
    }

    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);

    // Forcing it means we ignore readyToBecomeActive
    doFailover(svc1, svc2, false, true);
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.ACTIVE, svc2.state);
  }

  @Test
  public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new HealthCheckFailedException("Failed!"))
        .when(svc2.proxy).monitorHealth();
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failover to unhealthy service");
    } catch (FailoverFailedException ffe) {
      // Expected
    }
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testFailoverFromFaultyServiceSucceeds() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc1.proxy).transitionToStandby(anyReqInfo());

    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    AlwaysSucceedFencer.fenceCalled = 0;
    try {
      doFailover(svc1, svc2, false, false);
    } catch (FailoverFailedException ffe) {
      fail("Faulty active prevented failover");
    }

    // svc1 still thinks it's active, that's OK, it was fenced
    assertEquals(1, AlwaysSucceedFencer.fenceCalled);
    assertSame(svc1, AlwaysSucceedFencer.fencedSvc);
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.ACTIVE, svc2.state);
  }

  @Test
  public void testFailoverFromFaultyServiceFencingFailure() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc1.proxy).transitionToStandby(anyReqInfo());

    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

    AlwaysFailFencer.fenceCalled = 0;
    try {
      doFailover(svc1, svc2, false, false);
      fail("Failed over even though fencing failed");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    assertEquals(1, AlwaysFailFencer.fenceCalled);
    assertSame(svc1, AlwaysFailFencer.fencedSvc);
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testFencingFailureDuringFailover() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

    AlwaysFailFencer.fenceCalled = 0;
    try {
      doFailover(svc1, svc2, true, false);
      fail("Failed over even though fencing requested and failed");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    // If fencing was requested and it failed we don't try to make
    // svc2 active anyway, and we don't failback to svc1.
    assertEquals(1, AlwaysFailFencer.fenceCalled);
    assertSame(svc1, AlwaysFailFencer.fencedSvc);
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }
  
  @Test
  public void testFailoverFromNonExistantServiceWithFencer() throws Exception {
    DummyHAService svc1 = spy(new DummyHAService(null, svc1Addr));
    // Getting a proxy to a dead server will throw IOException on call,
    // not on creation of the proxy.
    HAServiceProtocol errorThrowingProxy = Mockito.mock(HAServiceProtocol.class,
        Mockito.withSettings()
          .defaultAnswer(new ThrowsException(
              new IOException("Could not connect to host")))
          .extraInterfaces(Closeable.class));
    Mockito.doNothing().when((Closeable)errorThrowingProxy).close();

    Mockito.doReturn(errorThrowingProxy).when(svc1).getProxy(
        Mockito.<Configuration>any(),
        Mockito.anyInt());
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
    } catch (FailoverFailedException ffe) {
      fail("Non-existant active prevented failover");
    }
    // Verify that the proxy created to try to make it go to standby
    // gracefully used the right rpc timeout
    Mockito.verify(svc1).getProxy(
        Mockito.<Configuration>any(),
        Mockito.eq(
          CommonConfigurationKeys.HA_FC_GRACEFUL_FENCE_TIMEOUT_DEFAULT));
        
    // Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
    assertEquals(HAServiceState.ACTIVE, svc2.state);
  }

  @Test
  public void testFailoverToNonExistantServiceFails() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
    Mockito.doThrow(new IOException("Failed to connect"))
      .when(svc2).getProxy(Mockito.<Configuration>any(),
          Mockito.anyInt());
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failed over to a non-existant standby");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    assertEquals(HAServiceState.ACTIVE, svc1.state);
  }

  @Test
  public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
    DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc2.proxy).transitionToActive(anyReqInfo());
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failover to already active service");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    // svc1 went standby then back to active
    verify(svc1.proxy).transitionToStandby(anyReqInfo());
    verify(svc1.proxy).transitionToActive(anyReqInfo());
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testWeDontFailbackIfActiveWasFenced() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc2.proxy).transitionToActive(anyReqInfo());
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, true, false);
      fail("Failed over to service that won't transition to active");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    // We failed to failover and did not failback because we fenced
    // svc1 (we forced it), therefore svc1 and svc2 should be standby.
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testWeFenceOnFailbackIfTransitionToActiveFails() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc2.proxy).transitionToActive(anyReqInfo());
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
    AlwaysSucceedFencer.fenceCalled = 0;

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failed over to service that won't transition to active");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    // We failed to failover. We did not fence svc1 because it cooperated
    // and we didn't force it, so we failed back to svc1 and fenced svc2.
    // Note svc2 still thinks it's active, that's OK, we fenced it.
    assertEquals(HAServiceState.ACTIVE, svc1.state);
    assertEquals(1, AlwaysSucceedFencer.fenceCalled);
    assertSame(svc2, AlwaysSucceedFencer.fencedSvc);
  }

  private StateChangeRequestInfo anyReqInfo() {
    return Mockito.<StateChangeRequestInfo>any();
  }

  @Test
  public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new IOException("Failed!"))
        .when(svc2.proxy).transitionToActive(anyReqInfo());
    svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
    AlwaysFailFencer.fenceCalled = 0;

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failed over to service that won't transition to active");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    // We did not fence svc1 because it cooperated and we didn't force it, 
    // we failed to failover so we fenced svc2, we failed to fence svc2
    // so we did not failback to svc1, ie it's still standby.
    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(1, AlwaysFailFencer.fenceCalled);
    assertSame(svc2, AlwaysFailFencer.fencedSvc);
  }

  @Test
  public void testFailbackToFaultyServiceFails() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc1.proxy).transitionToActive(anyReqInfo());
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    Mockito.doThrow(new ServiceFailedException("Failed!"))
        .when(svc2.proxy).transitionToActive(anyReqInfo());

    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

    try {
      doFailover(svc1, svc2, false, false);
      fail("Failover to already active service");
    } catch (FailoverFailedException ffe) {
      // Expected
    }

    assertEquals(HAServiceState.STANDBY, svc1.state);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }

  @Test
  public void testSelfFailoverFails() throws Exception {
    DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
    DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
    svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());
    AlwaysSucceedFencer.fenceCalled = 0;

    try {
      doFailover(svc1, svc1, false, false);
      fail("Can't failover to yourself");
    } catch (FailoverFailedException ffe) {
      // Expected
    }
    assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
    assertEquals(HAServiceState.ACTIVE, svc1.state);

    try {
      doFailover(svc2, svc2, false, false);
      fail("Can't failover to yourself");
    } catch (FailoverFailedException ffe) {
      // Expected
    }
    assertEquals(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled);
    assertEquals(HAServiceState.STANDBY, svc2.state);
  }
  
  private void doFailover(HAServiceTarget tgt1, HAServiceTarget tgt2,
      boolean forceFence, boolean forceActive) throws FailoverFailedException {
    FailoverController fc = new FailoverController(conf, 
        RequestSource.REQUEST_BY_USER);
    fc.failover(tgt1, tgt2, forceFence, forceActive);
  }

}