bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestBookieHealthCheck.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package org.apache.bookkeeper.client;

import java.util.concurrent.TimeUnit;

import org.apache.bookkeeper.client.AsyncCallback.AddCallback;
import org.apache.bookkeeper.net.BookieId;
import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Test the BookieHealthCheck.
 */
public class TestBookieHealthCheck extends BookKeeperClusterTestCase {
    private static final Logger LOG = LoggerFactory.getLogger(TestBookieHealthCheck.class);

    public TestBookieHealthCheck() {
        super(4);
        baseClientConf.setAddEntryTimeout(1);
        baseClientConf.enableBookieHealthCheck();
        baseClientConf.setBookieHealthCheckInterval(1, TimeUnit.SECONDS);
        baseClientConf.setBookieErrorThresholdPerInterval(1);
        baseClientConf.setBookieQuarantineTime(5, TimeUnit.SECONDS);
    }

    @Test
    public void testBkQuarantine() throws Exception {
        LedgerHandle lh = bkc.createLedger(2, 2, 2, BookKeeper.DigestType.CRC32, new byte[] {});

        final int numEntries = 10;
        for (int i = 0; i < numEntries; i++) {
            byte[] msg = ("msg-" + i).getBytes();
            lh.addEntry(msg);
        }

        BookieId bookieToQuarantine = lh.getLedgerMetadata().getEnsembleAt(numEntries).get(0);
        sleepBookie(bookieToQuarantine, baseClientConf.getAddEntryTimeout() * 2).await();

        byte[] tempMsg = "temp-msg".getBytes();
        lh.asyncAddEntry(tempMsg, new AddCallback() {

            @Override
            public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
                // no-op
            }
        }, null);

        // make sure the add entry timeouts
        Thread.sleep(baseClientConf.getAddEntryTimeout() * 2 * 1000);

        // make sure the health check runs once after the timeout
        Thread.sleep(baseClientConf.getBookieHealthCheckIntervalSeconds() * 2 * 1000);

        // the bookie watcher should contain the bookieToQuarantine in the quarantine set
        Assert.assertTrue(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookieToQuarantine));

        // the bookie to be left out of the ensemble should always be the quarantined bookie
        LedgerHandle lh1 = bkc.createLedger(2, 2, 2, BookKeeper.DigestType.CRC32, new byte[] {});
        LedgerHandle lh2 = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, new byte[] {});
        Assert.assertFalse(lh1.getLedgerMetadata().getEnsembleAt(0).contains(bookieToQuarantine));
        Assert.assertFalse(lh2.getLedgerMetadata().getEnsembleAt(0).contains(bookieToQuarantine));

        // the quarantined bookie can still be in the ensemble if we do not have enough healthy bookies
        LedgerHandle lh3 = bkc.createLedger(4, 4, 4, BookKeeper.DigestType.CRC32, new byte[] {});
        Assert.assertTrue(lh3.getLedgerMetadata().getEnsembleAt(0).contains(bookieToQuarantine));

        // make sure faulty bookie is out of quarantine
        Thread.sleep(baseClientConf.getBookieQuarantineTimeSeconds() * 1000);

        // the bookie should not be quarantined anymore
        Assert.assertFalse(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookieToQuarantine));
    }

    @Test
    public void testNoQuarantineOnBkRestart() throws Exception {
        final LedgerHandle lh = bkc.createLedger(2, 2, 2, BookKeeper.DigestType.CRC32, new byte[] {});
        final int numEntries = 20;
        BookieId bookieToRestart = lh.getLedgerMetadata().getEnsembleAt(0).get(0);

        // we add entries on a separate thread so that we can restart a bookie on the current thread
        Thread addEntryThread = new Thread() {
            public void run() {
                for (int i = 0; i < numEntries; i++) {
                    byte[] msg = ("msg-" + i).getBytes();
                    try {
                        lh.addEntry(msg);
                        // we add sufficient sleep to make sure all entries are not added before we restart the bookie
                        Thread.sleep(100);
                    } catch (Exception e) {
                        LOG.error("Error sending msg");
                    }
                }
            }
        };
        addEntryThread.start();
        restartBookie(bookieToRestart);

        // make sure the health check runs once
        Thread.sleep(baseClientConf.getBookieHealthCheckIntervalSeconds() * 2 * 1000);

        // the bookie watcher should not contain the bookieToRestart in the quarantine set
        Assert.assertFalse(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookieToRestart));
    }

    @Test
    public void testNoQuarantineOnExpectedBkErrors() throws Exception {
        final LedgerHandle lh = bkc.createLedger(2, 2, 2, BookKeeper.DigestType.CRC32, new byte[] {});
        final int numEntries = 10;
        for (int i = 0; i < numEntries; i++) {
            byte[] msg = ("msg-" + i).getBytes();
            lh.addEntry(msg);
        }
        BookieId bookie1 = lh.getLedgerMetadata().getEnsembleAt(0).get(0);
        BookieId bookie2 = lh.getLedgerMetadata().getEnsembleAt(0).get(1);
        try {
            // we read an entry that is not added
            lh.readEntries(10, 10);
        } catch (BKException e) {
            // ok
        }

        // make sure the health check runs once
        Thread.sleep(baseClientConf.getBookieHealthCheckIntervalSeconds() * 2 * 1000);

        // the bookie watcher should not contain the bookieToRestart in the quarantine set
        Assert.assertFalse(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookie1));
        Assert.assertFalse(bkc.bookieWatcher.quarantinedBookies.asMap().containsKey(bookie2));
    }

}