library/src/test/java/org/apache/apex/malhar/lib/io/jms/JMSOutputOperatorTest.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.datatorrent.lib.io.jms;

import java.io.File;
import java.io.IOException;
import java.util.Random;

import javax.jms.JMSException;
import javax.jms.Message;

import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestWatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.io.FileUtils;

import com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap;
import com.datatorrent.api.Context.OperatorContext;
import com.datatorrent.api.DAG;
import com.datatorrent.api.DefaultInputPort;
import com.datatorrent.api.Operator.ProcessingMode;
import com.datatorrent.lib.util.ActiveMQMultiTypeMessageListener;

import static com.datatorrent.lib.helper.OperatorContextTestHelper.mockOperatorContext;

/**
 * Test to verify JMS output operator adapter.
 */
public class JMSOutputOperatorTest extends JMSTestBase
{
  private static final Logger logger = LoggerFactory.getLogger(JMSOutputOperatorTest.class);
  public static int tupleCount = 0;
  public static final transient int maxTuple = 20;

  public static final String CLIENT_ID = "Client1";
  public static final String APP_ID = "appId";
  public static final int OPERATOR_ID = 1;
  public static JMSStringSinglePortOutputOperator outputOperator;
  public static OperatorContext testOperatorContext;
  public static OperatorContext testOperatorContextAMO;
  public static final int HALF_BATCH_SIZE = 5;
  public static final int BATCH_SIZE = HALF_BATCH_SIZE * 2;
  public static final Random random = new Random();

  public static class TestMeta extends TestWatcher
  {
    @Override
    protected void starting(org.junit.runner.Description description)
    {
      logger.debug("Starting test {}", description.getMethodName());
      DefaultAttributeMap attributes = new DefaultAttributeMap();
      attributes.put(DAG.APPLICATION_ID, APP_ID);
      testOperatorContext = mockOperatorContext(OPERATOR_ID, attributes);

      attributes = new DefaultAttributeMap();
      attributes.put(OperatorContext.PROCESSING_MODE, ProcessingMode.AT_MOST_ONCE);
      testOperatorContextAMO = mockOperatorContext(OPERATOR_ID, attributes);

      try {
        FileUtils.deleteDirectory(new File(FSPsuedoTransactionableStore.DEFAULT_RECOVERY_DIRECTORY));
      } catch (IOException ex) {
        throw new RuntimeException(ex);
      }
    }

    @Override
    protected void finished(org.junit.runner.Description description)
    {
      try {
        FileUtils.deleteDirectory(new File(FSPsuedoTransactionableStore.DEFAULT_RECOVERY_DIRECTORY));
      } catch (IOException ex) {
        throw new RuntimeException(ex);
      }
    }
  }

  @Rule
  public TestMeta testMeta = new TestMeta();

  /**
   * Concrete class of JMSStringSinglePortOutputOperator for testing.
   */
  public static class JMSStringSinglePortOutputOperator extends AbstractJMSSinglePortOutputOperator<String>
  {
    public JMSStringSinglePortOutputOperator()
    {
      this.setBatch(10);
    }

    @Override
    protected Message createMessage(Object tuple)
    {
      Message msg;
      try {
        msg = getSession().createTextMessage(tuple.toString());
      } catch (JMSException ex) {
        throw new RuntimeException("Failed to create message.", ex);
      }

      return msg;
    }
  } // End of JMSStringSinglePortOutputOperator

  /**
   * This is a helper method to create the output operator. Note this cannot be
   * put in test watcher because because JMS connection issues occur when this code
   * is run from a test watcher.
   */
  private void createOperator(boolean topic, OperatorContext context)
  {
    createOperator(topic, context, 0);
  }

  private void createOperator(boolean topic, OperatorContext context, @SuppressWarnings("unused") int maxMessages)
  {
    outputOperator = new JMSStringSinglePortOutputOperator();
    outputOperator.getConnectionFactoryProperties().put("userName", "");
    outputOperator.getConnectionFactoryProperties().put("password", "");
    outputOperator.getConnectionFactoryProperties().put("brokerURL", "tcp://localhost:61617");
    outputOperator.setAckMode("CLIENT_ACKNOWLEDGE");
    outputOperator.setClientId(CLIENT_ID);
    outputOperator.setSubject("TEST.FOO");
    outputOperator.setMessageSize(255);
    outputOperator.setBatch(BATCH_SIZE);
    outputOperator.setTopic(topic);
    outputOperator.setDurable(false);
    outputOperator.setVerbose(true);
    outputOperator.setup(context);
  }

  /**
   * Test AbstractJMSOutputOperator (i.e. an output adapter for JMS, aka producer).
   * This module sends data into an JMS message bus.
   *
   * [Generate tuple] ==> [send tuple through JMS output adapter(i.e. producer) into JMS message bus]
   * ==> [receive data in outside JMS listener]
   *
   * @throws Exception
   */
  //@Ignore
  @Test
  public void testJMSOutputOperator1() throws Exception
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setupConnection();
    listener.run();

    createOperator(false, testOperatorContext, 15);

    outputOperator.beginWindow(1);

    int i = 0;
    while (i < maxTuple) {
      logger.debug("Emitting tuple {}", i);
      String tuple = "testString " + (++i);
      outputOperator.inputPort.process(tuple);
      tupleCount++;
    }

    outputOperator.endWindow();
    outputOperator.teardown();

    Thread.sleep(500);

    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", maxTuple, listener.receivedData.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.receivedData.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.receivedData.get(1));

    listener.closeConnection();
  }

  /**
   * This test is same as prior one except maxMessage and topic setting is different.
   *
   * @throws Exception
   */
  ////@Ignore
  @Test
  public void testJMSOutputOperator2() throws Exception
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(true);
    listener.setupConnection();
    listener.run();

    createOperator(true, testOperatorContext, 15);

    outputOperator.beginWindow(1);

    // produce data and process

    int i = 0;
    while (i < maxTuple) {
      String tuple = "testString " + (++i);
      outputOperator.inputPort.process(tuple);
      tupleCount++;
    }

    outputOperator.endWindow();
    outputOperator.teardown();

    Thread.sleep(500);

    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", maxTuple, listener.receivedData.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.receivedData.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.receivedData.get(1));

    listener.closeConnection();
  }

  //@Ignore
  @Test
  public void testBatch() throws JMSException, InterruptedException
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(false);

    listener.setupConnection();
    listener.run();

    createOperator(false, testOperatorContext);

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();
    Thread.sleep(200);

    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);

    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();
    Thread.sleep(200);

    Assert.assertEquals("Batch should not be written", BATCH_SIZE + HALF_BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(2);

    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();
    Thread.sleep(200);

    Assert.assertEquals("Batch should not be written", 2 * BATCH_SIZE, listener.receivedData.size());

    outputOperator.teardown();
    listener.closeConnection();
  }

  @Test
  public void testAtLeastOnceFullBatch() throws JMSException, InterruptedException
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(false);

    listener.setupConnection();
    listener.run();

    createOperator(false, testOperatorContext);

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    Thread.sleep(200);

    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    Thread.sleep(200);

    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.teardown();

    ////

    outputOperator.setup(testOperatorContext);

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", 2 * BATCH_SIZE, listener.receivedData.size());

    listener.closeConnection();
  }

  //@Ignore
  @Test
  public void testAtLeastOnceHalfBatch() throws JMSException, InterruptedException
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(false);

    listener.setupConnection();
    listener.run();

    createOperator(false, testOperatorContext);

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);
    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.teardown();

    ////

    outputOperator.setup(testOperatorContext);

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);
    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();
    Thread.sleep(200);
    Assert.assertEquals("Batch should be written", BATCH_SIZE + HALF_BATCH_SIZE, listener.receivedData.size());
    listener.closeConnection();
  }

  //@Ignore
  @Test
  public void testAtMostOnceFullBatch()
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(false);

    try {
      listener.setupConnection();
    } catch (JMSException ex) {
      throw new RuntimeException(ex);
    }

    listener.run();

    createOperator(false, testOperatorContextAMO);

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.teardown();

    ////

    outputOperator.setup(testOperatorContext);

    outputOperator.beginWindow(2);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", 2 * BATCH_SIZE, listener.receivedData.size());

    listener.closeConnection();
  }

  //@Ignore
  @Test
  public void testAtMostOnceHalfBatch()
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setTopic(false);

    try {
      listener.setupConnection();
    } catch (JMSException ex) {
      throw new RuntimeException(ex);
    }

    listener.run();

    createOperator(false, testOperatorContextAMO);

    outputOperator.beginWindow(0);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(1);

    for (int batchCounter = 0; batchCounter < HALF_BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.teardown();

    ////

    outputOperator.setup(testOperatorContext);

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", BATCH_SIZE, listener.receivedData.size());

    outputOperator.beginWindow(2);

    for (int batchCounter = 0; batchCounter < BATCH_SIZE; batchCounter++) {
      outputOperator.inputPort.put(Integer.toString(random.nextInt()));
    }

    outputOperator.endWindow();

    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
      throw new RuntimeException(ex);
    }
    Assert.assertEquals("Batch should be written", 2 * BATCH_SIZE, listener.receivedData.size());
    listener.closeConnection();
  }

  /**
   * Concrete class of JMSMultiPortOutputOperator for testing.
   */
  public static class JMSMultiPortOutputOperator extends AbstractJMSOutputOperator
  {
    /**
     * Two input ports.
     */
    public final transient DefaultInputPort<String> inputPort1 = new DefaultInputPort<String>()
    {
      @Override
      public void process(String tuple)
      {
        sendMessage(tuple);
      }
    };
    public final transient DefaultInputPort<Integer> inputPort2 = new DefaultInputPort<Integer>()
    {
      @Override
      public void process(Integer tuple)
      {
        sendMessage(tuple);
      }
    };

    @Override
    protected Message createMessage(Object tuple)
    {
      Message msg;

      try {
        msg = getSession().createTextMessage(tuple.toString());
      } catch (JMSException ex) {
        throw new RuntimeException(ex);
      }

      return msg;
    }
  }

  @Test
  public void testJMSMultiPortOutputOperator() throws Exception
  {
    // Setup a message listener to receive the message
    final ActiveMQMultiTypeMessageListener listener = new ActiveMQMultiTypeMessageListener();
    listener.setMaximumReceiveMessages(0);
    listener.setupConnection();
    listener.run();

    // Malhar module to send message
    JMSMultiPortOutputOperator node = new JMSMultiPortOutputOperator();
    // Set configuration parameters for JMS

    node.getConnectionFactoryProperties().put("userName", "");
    node.getConnectionFactoryProperties().put("password", "");
    node.getConnectionFactoryProperties().put("brokerURL", "tcp://localhost:61617");
    node.setAckMode("CLIENT_ACKNOWLEDGE");
    node.setClientId("Client1");
    node.setSubject("TEST.FOO");
    node.setMessageSize(255);
    node.setBatch(10);
    node.setTopic(false);
    node.setDurable(false);
    logger.debug("Before set store");
    node.setStore(new JMSTransactionableStore());
    logger.debug("After set store");
    logger.debug("Store class {}", node.getStore());
    node.setVerbose(true);

    node.setup(testOperatorContext);
    node.beginWindow(1);

    int i = 0;
    while (i < maxTuple) {
      String tuple = "testString " + (++i);
      node.inputPort1.process(tuple);
      node.inputPort2.process(i);
      tupleCount++;
    }

    node.endWindow();
    node.teardown();

    final long emittedCount = 40;

    Thread.sleep(500);

    // Check values send vs received
    Assert.assertEquals("Number of emitted tuples", emittedCount, listener.receivedData.size());
    logger.debug(String.format("Number of emitted tuples: %d", listener.receivedData.size()));
    Assert.assertEquals("First tuple", "testString 1", listener.receivedData.get(1));

    listener.closeConnection();
  }
}