| // Copyright 2017 Twitter. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| package com.twitter.heron.api.windowing.triggers; |
| |
| import java.util.logging.Logger; |
| |
| import com.twitter.heron.api.windowing.DefaultEvictionContext; |
| import com.twitter.heron.api.windowing.Event; |
| import com.twitter.heron.api.windowing.EvictionPolicy; |
| import com.twitter.heron.api.windowing.TriggerHandler; |
| import com.twitter.heron.api.windowing.TriggerPolicy; |
| import com.twitter.heron.api.windowing.WindowManager; |
| |
| /** |
| * Handles watermark events and triggers {@link TriggerHandler#onTrigger()} for each window |
| * interval that has events to be processed up to the watermark ts. |
| */ |
| public class WatermarkTimeTriggerPolicy<T> implements TriggerPolicy<T> { |
| private static final Logger LOG = Logger.getLogger(WatermarkTimeTriggerPolicy.class.getName()); |
| private final long slidingIntervalMs; |
| private final TriggerHandler handler; |
| private final EvictionPolicy<T> evictionPolicy; |
| private final WindowManager<T> windowManager; |
| private volatile long nextWindowEndTs; |
| private boolean started; |
| |
| public WatermarkTimeTriggerPolicy(long slidingIntervalMs, TriggerHandler handler, |
| EvictionPolicy<T> evictionPolicy, WindowManager<T> |
| windowManager) { |
| this.slidingIntervalMs = slidingIntervalMs; |
| this.handler = handler; |
| this.evictionPolicy = evictionPolicy; |
| this.windowManager = windowManager; |
| this.started = false; |
| } |
| |
| @Override |
| public void track(Event<T> event) { |
| if (started && event.isWatermark()) { |
| handleWaterMarkEvent(event); |
| } |
| } |
| |
| @Override |
| public void reset() { |
| // NOOP |
| } |
| |
| @Override |
| public void start() { |
| started = true; |
| } |
| |
| @Override |
| public void shutdown() { |
| // NOOP |
| } |
| |
| /** |
| * Invokes the trigger all pending windows up to the |
| * watermark timestamp. The end ts of the window is set |
| * in the eviction policy context so that the events falling |
| * within that window can be processed. |
| */ |
| private void handleWaterMarkEvent(Event<T> event) { |
| long watermarkTs = event.getTimestamp(); |
| long windowEndTs = nextWindowEndTs; |
| LOG.fine(String.format("Window end ts %d Watermark ts %d", windowEndTs, watermarkTs)); |
| while (windowEndTs <= watermarkTs) { |
| long currentCount = windowManager.getEventCount(windowEndTs); |
| evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); |
| if (handler.onTrigger()) { |
| windowEndTs += slidingIntervalMs; |
| } else { |
| /* |
| * No events were found in the previous window interval. |
| * Scan through the events in the queue to find the next |
| * window intervals based on event ts. |
| */ |
| long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); |
| LOG.fine(String.format("Next aligned window end ts %d", ts)); |
| if (ts == Long.MAX_VALUE) { |
| LOG.fine(String.format("No events to process between %d and watermark ts %d", |
| windowEndTs, watermarkTs)); |
| break; |
| } |
| windowEndTs = ts; |
| } |
| } |
| nextWindowEndTs = windowEndTs; |
| } |
| |
| /** |
| * Computes the next window by scanning the events in the window and |
| * finds the next aligned window between the startTs and endTs. Return the end ts |
| * of the next aligned window, i.e. the ts when the window should fire. |
| * |
| * @param startTs the start timestamp (excluding) |
| * @param endTs the end timestamp (including) |
| * @return the aligned window end ts for the next window or Long.MAX_VALUE if there |
| * are no more events to be processed. |
| */ |
| private long getNextAlignedWindowTs(long startTs, long endTs) { |
| long nextTs = windowManager.getEarliestEventTs(startTs, endTs); |
| if (nextTs == Long.MAX_VALUE || (nextTs % slidingIntervalMs == 0)) { |
| return nextTs; |
| } |
| return nextTs + (slidingIntervalMs - (nextTs % slidingIntervalMs)); |
| } |
| |
| @Override |
| public String toString() { |
| return "WatermarkTimeTriggerPolicy{" + "slidingIntervalMs=" + slidingIntervalMs |
| + ", nextWindowEndTs=" + nextWindowEndTs + ", started=" + started + '}'; |
| } |
| } |