eagle-jpm/eagle-jpm-mr-running/src/main/java/org/apache/eagle/jpm/mr/running/parser/MRJobEntityCreationHandler.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.eagle.jpm.mr.running.parser;

import org.apache.eagle.jpm.mr.running.MRRunningJobConfig;
import org.apache.eagle.jpm.mr.running.parser.metrics.JobExecutionMetricsCreationListener;
import org.apache.eagle.jpm.mr.running.parser.metrics.TaskExecutionMetricsCreationListener;
import org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity;
import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
import org.apache.eagle.log.entity.GenericMetricEntity;
import org.apache.eagle.service.client.IEagleServiceClient;
import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;

public class MRJobEntityCreationHandler {
    private static final Logger LOG = LoggerFactory.getLogger(MRJobEntityCreationHandler.class);

    private List<TaggedLogAPIEntity> entities = new ArrayList<>();
    private MRRunningJobConfig.EagleServiceConfig eagleServiceConfig;
    private JobExecutionMetricsCreationListener jobMetricsListener;
    private TaskExecutionMetricsCreationListener taskMetricsListener;
    private static final int MAX_FLUSH_NUM = 1000;
    private static final int MAX_RETRY_COUNT = 3;

    public MRJobEntityCreationHandler(MRRunningJobConfig.EagleServiceConfig eagleServiceConfig) {
        this.eagleServiceConfig = eagleServiceConfig;
        jobMetricsListener = new JobExecutionMetricsCreationListener();
        taskMetricsListener = new TaskExecutionMetricsCreationListener();
    }

    public void add(TaggedLogAPIEntity entity) {
        entities.add(entity);
        List<GenericMetricEntity> metricEntities;
        /*if (entity instanceof TaskExecutionAPIEntity) {
            metricEntities = taskMetricsListener.generateMetrics((TaskExecutionAPIEntity) entity);
            entities.addAll(metricEntities);
        } else if (entity instanceof JobExecutionAPIEntity) {
            metricEntities = jobMetricsListener.generateMetrics((JobExecutionAPIEntity) entity);
            entities.addAll(metricEntities);
        }*/
        if (entity instanceof JobExecutionAPIEntity) {
            metricEntities = jobMetricsListener.generateMetrics((JobExecutionAPIEntity) entity);
            entities.addAll(metricEntities);
        }
        if (entities.size() >= MAX_FLUSH_NUM) {
            this.flush();
        }
    }

    public boolean flush() {
        //need flush right now
        if (entities.size() == 0) {
            return true;
        }
        IEagleServiceClient client = new EagleServiceClientImpl(
            eagleServiceConfig.eagleServiceHost,
            eagleServiceConfig.eagleServicePort,
            eagleServiceConfig.username,
            eagleServiceConfig.password);
        //client.getJerseyClient().setReadTimeout(eagleServiceConfig.readTimeoutSeconds * 1000);
        client.setReadTimeout(eagleServiceConfig.readTimeoutSeconds * 1000);
        try {
            return createEntities(client);
        } catch (Exception e) {
            LOG.warn("exception found when flush entities, {}", e);
            return false;
        } finally {
            try {
                client.close();
            } catch (Exception e) {
                // ignored exception
            }
        }
    }

    private boolean createEntities(IEagleServiceClient client) throws Exception {
        int count = 0;
        boolean success = false;
        while (count++ < MAX_RETRY_COUNT && !success) {
            try {
                LOG.info("start to flush mr job entities, size {}", entities.size());
                client.create(entities);
                success = true;
                LOG.info("finish flushing mr job entities, size {}", entities.size());
                entities.clear();

            } catch (Exception e) {
                LOG.warn("exception found when flush entities", e);
                if (!success && count < MAX_RETRY_COUNT) {
                    LOG.info("Sleep for a while before retrying");
                    Thread.sleep(10 * 1000);
                }
            }

        }
        if (!success) {
            LOG.warn("Fail flushing entities after tries {} times", MAX_RETRY_COUNT);
        }
        return success;
    }
}