flink-tests/src/test/java/org/apache/flink/test/manual/ReducePerformance.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.test.manual;

import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.operators.base.ReduceOperatorBase.CombineHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.util.SplittableIterator;

import java.io.Serializable;
import java.util.Iterator;
import java.util.Random;

/**
 * This is for testing the performance of reduce, with different execution strategies. (See also
 * http://peel-framework.org/2016/04/07/hash-aggregations-in-flink.html)
 */
public class ReducePerformance {

    public static void main(String[] args) throws Exception {

        final int numElements = 40_000_000;
        final int keyRange = 4_000_000;

        // warm up JIT
        testReducePerformance(
                new TupleIntIntIterator(1000),
                TupleTypeInfo.<Tuple2<Integer, Integer>>getBasicTupleTypeInfo(
                        Integer.class, Integer.class),
                CombineHint.SORT,
                10000,
                false);

        testReducePerformance(
                new TupleIntIntIterator(1000),
                TupleTypeInfo.<Tuple2<Integer, Integer>>getBasicTupleTypeInfo(
                        Integer.class, Integer.class),
                CombineHint.HASH,
                10000,
                false);

        // TupleIntIntIterator
        testReducePerformance(
                new TupleIntIntIterator(keyRange),
                TupleTypeInfo.<Tuple2<Integer, Integer>>getBasicTupleTypeInfo(
                        Integer.class, Integer.class),
                CombineHint.SORT,
                numElements,
                true);

        testReducePerformance(
                new TupleIntIntIterator(keyRange),
                TupleTypeInfo.<Tuple2<Integer, Integer>>getBasicTupleTypeInfo(
                        Integer.class, Integer.class),
                CombineHint.HASH,
                numElements,
                true);

        // TupleStringIntIterator
        testReducePerformance(
                new TupleStringIntIterator(keyRange),
                TupleTypeInfo.<Tuple2<String, Integer>>getBasicTupleTypeInfo(
                        String.class, Integer.class),
                CombineHint.SORT,
                numElements,
                true);

        testReducePerformance(
                new TupleStringIntIterator(keyRange),
                TupleTypeInfo.<Tuple2<String, Integer>>getBasicTupleTypeInfo(
                        String.class, Integer.class),
                CombineHint.HASH,
                numElements,
                true);
    }

    private static <T, B extends CopyableIterator<T>> void testReducePerformance(
            B iterator,
            TypeInformation<T> typeInfo,
            CombineHint hint,
            int numRecords,
            boolean print)
            throws Exception {

        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().enableObjectReuse();

        @SuppressWarnings("unchecked")
        DataSet<T> output =
                env.fromParallelCollection(
                                new SplittableRandomIterator<T, B>(numRecords, iterator), typeInfo)
                        .groupBy("0")
                        .reduce(new SumReducer())
                        .setCombineHint(hint);

        long start = System.currentTimeMillis();

        System.out.println(output.count());

        long end = System.currentTimeMillis();
        if (print) {
            System.out.println(
                    "=== Time for "
                            + iterator.getClass().getSimpleName()
                            + " with hint "
                            + hint.toString()
                            + ": "
                            + (end - start)
                            + "ms ===");
        }
    }

    private static final class SplittableRandomIterator<T, B extends CopyableIterator<T>>
            extends SplittableIterator<T> implements Serializable {

        private int numElements;
        private final B baseIterator;

        public SplittableRandomIterator(int numElements, B baseIterator) {
            this.numElements = numElements;
            this.baseIterator = baseIterator;
        }

        @Override
        public boolean hasNext() {
            return numElements > 0;
        }

        @Override
        public T next() {
            numElements--;
            return baseIterator.next();
        }

        @SuppressWarnings("unchecked")
        @Override
        public SplittableRandomIterator<T, B>[] split(int numPartitions) {
            int splitSize = numElements / numPartitions;
            int rem = numElements % numPartitions;
            SplittableRandomIterator<T, B>[] res = new SplittableRandomIterator[numPartitions];
            for (int i = 0; i < numPartitions; i++) {
                res[i] =
                        new SplittableRandomIterator<T, B>(
                                i < rem ? splitSize : splitSize + 1, (B) baseIterator.copy());
            }
            return res;
        }

        @Override
        public int getMaximumNumberOfSplits() {
            return numElements;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }

    private interface CopyableIterator<T> extends Iterator<T> {
        CopyableIterator<T> copy();
    }

    private static final class TupleIntIntIterator
            implements CopyableIterator<Tuple2<Integer, Integer>>, Serializable {

        private final int keyRange;
        private Tuple2<Integer, Integer> reuse = new Tuple2<Integer, Integer>();

        private int rndSeed = 11;
        private Random rnd;

        public TupleIntIntIterator(int keyRange) {
            this.keyRange = keyRange;
            this.rnd = new Random(this.rndSeed);
        }

        public TupleIntIntIterator(int keyRange, int rndSeed) {
            this.keyRange = keyRange;
            this.rndSeed = rndSeed;
            this.rnd = new Random(rndSeed);
        }

        @Override
        public boolean hasNext() {
            return true;
        }

        @Override
        public Tuple2<Integer, Integer> next() {
            reuse.f0 = rnd.nextInt(keyRange);
            reuse.f1 = 1;
            return reuse;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

        @Override
        public CopyableIterator<Tuple2<Integer, Integer>> copy() {
            return new TupleIntIntIterator(keyRange, rndSeed + rnd.nextInt(10000));
        }
    }

    private static final class TupleStringIntIterator
            implements CopyableIterator<Tuple2<String, Integer>>, Serializable {

        private final int keyRange;
        private Tuple2<String, Integer> reuse = new Tuple2<>();

        private int rndSeed = 11;
        private Random rnd;

        public TupleStringIntIterator(int keyRange) {
            this.keyRange = keyRange;
            this.rnd = new Random(this.rndSeed);
        }

        public TupleStringIntIterator(int keyRange, int rndSeed) {
            this.keyRange = keyRange;
            this.rndSeed = rndSeed;
            this.rnd = new Random(rndSeed);
        }

        @Override
        public boolean hasNext() {
            return true;
        }

        @Override
        public Tuple2<String, Integer> next() {
            reuse.f0 = String.valueOf(rnd.nextInt(keyRange));
            reuse.f1 = 1;
            return reuse;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

        @Override
        public CopyableIterator<Tuple2<String, Integer>> copy() {
            return new TupleStringIntIterator(keyRange, rndSeed + rnd.nextInt(10000));
        }
    }

    private static final class SumReducer<K> implements ReduceFunction<Tuple2<K, Integer>> {
        @Override
        public Tuple2<K, Integer> reduce(Tuple2<K, Integer> a, Tuple2<K, Integer> b)
                throws Exception {
            if (!a.f0.equals(b.f0)) {
                throw new RuntimeException(
                        "SumReducer was called with two record that have differing keys.");
            }
            a.f1 = a.f1 + b.f1;
            return a;
        }
    }
}