lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
// 
//   http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

using System.Collections.Generic;
using NSubstitute;
using Org.Apache.REEF.Client.Yarn;
using Org.Apache.REEF.Client.YARN;
using Org.Apache.REEF.Client.YARN.Parameters;
using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Util;
using Xunit;

namespace Org.Apache.REEF.Client.Tests
{
    public class WindowsYarnJobCommandProviderTests
    {
        private static readonly List<string> AnyClassPathItems = new List<string>
        {
            "%HADOOP_CONF_DIR%",
            "%HADOOP_HOME%/*",
            "%HADOOP_HOME%/lib/*",
            "%HADOOP_COMMON_HOME%/*",
            "%HADOOP_COMMON_HOME%/lib/*",
            "%HADOOP_YARN_HOME%/*",
            "%HADOOP_YARN_HOME%/lib/*",
            "%HADOOP_HDFS_HOME%/*",
            "%HADOOP_HDFS_HOME%/lib/*",
            "%HADOOP_MAPRED_HOME%/*",
            "%HADOOP_MAPRED_HOME%/lib/*",
            "%HADOOP_HOME%/etc/hadoop",
            "%HADOOP_HOME%/share/hadoop/common/*",
            "%HADOOP_HOME%/share/hadoop/common/lib/*",
            "%HADOOP_HOME%/share/hadoop/yarn/*",
            "%HADOOP_HOME%/share/hadoop/yarn/lib/*",
            "%HADOOP_HOME%/share/hadoop/hdfs/*",
            "%HADOOP_HOME%/share/hadoop/hdfs/lib/*",
            "%HADOOP_HOME%/share/hadoop/mapreduce/*",
            "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*"
        };

        [Fact]
        public void YarnJobCommandBuilderIsDefaultImplementationOfIYarnJobCommandBuilder()
        {
            Assert.IsType(typeof(WindowsYarnJobCommandProvider), TangFactory.GetTang().NewInjector().GetInstance<IYarnJobCommandProvider>());
        }

        [Fact]
        public void GetJobSubmissionCommandGeneratesCorrectCommand()
        {
            var testContext = new TestContext();
            const string expectedCommand = @"reef\Org.Apache.REEF.Bridge.exe" +
                                           " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx512m " +
                                           "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
                                           "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
                                           "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
                                           "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
                                           "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
                                           "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
                                           "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
                                           "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
                                           "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
                                           "che.reef.bridge.client.YarnBootstrapREEFLauncher job-submission-params" +
                                           ".json reef/local/app-submission-params.json 1> <LOG_DIR>/driver.stdout" +
                                           " 2> <LOG_DIR>/driver.stderr";

            var commandBuilder = testContext.GetCommandBuilder();
            var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
            Assert.Equal(expectedCommand, jobSubmissionCommand);
        }

        [Fact]
        public void GetJobSubmissionCommandLoggingEnabledGeneratesCorrectCommand()
        {
            var testContext = new TestContext();
            const string expectedCommand = @"reef\Org.Apache.REEF.Bridge.exe" +
                                           " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx512m " +
                                           "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
                                           "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
                                           "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
                                           "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
                                           "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
                                           "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
                                           "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
                                           "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
                                           "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef -Djava." +
                                           "util.logging.config.class=org.apache.reef.util.logging.Config org.apac" +
                                           "he.reef.bridge.client.YarnBootstrapREEFLauncher job-submission-params" +
                                           ".json reef/local/app-submission-params.json 1> <LOG_DIR>/driver.stdout" +
                                           " 2> <LOG_DIR>/driver.stderr";
            var commandBuilder = testContext.GetCommandBuilder(true);
            var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
            Assert.Equal(expectedCommand, jobSubmissionCommand);
        }

        [Fact]
        public void GetJobSubmissionCommandSetsCorrectDriverMemoryAllocationSize()
        {
            var testContext = new TestContext();
            const int sizeMB = 256;
            const string expectedCommandFormat = @"reef\Org.Apache.REEF.Bridge.exe" +
                                                 " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx{0}m " +
                                                 "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
                                                 "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
                                                 "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
                                                 "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
                                                 "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
                                                 "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
                                                 "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
                                                 "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
                                                 "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
                                                 "che.reef.bridge.client.YarnBootstrapREEFLauncher job-submission-params" +
                                                 ".json reef/local/app-submission-params.json 1> <LOG_DIR>/driver.stdout" +
                                                 " 2> <LOG_DIR>/driver.stderr";
            string expectedCommand = string.Format(expectedCommandFormat, sizeMB);
            var commandBuilder = testContext.GetCommandBuilder(maxMemAllocPoolSize: sizeMB);
            var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
            Assert.Equal(expectedCommand, jobSubmissionCommand);
        }

        [Fact]
        public void GetJobSubmissionCommandSetsCorrectMaxPermSize()
        {
            var testContext = new TestContext();
            const int sizeMB = 256;
            const string expectedCommandFormat = @"reef\Org.Apache.REEF.Bridge.exe" +
                                                 " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize={0}m -Xmx512m " +
                                                 "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
                                                 "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
                                                 "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
                                                 "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
                                                 "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
                                                 "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
                                                 "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
                                                 "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
                                                 "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
                                                 "che.reef.bridge.client.YarnBootstrapREEFLauncher job-submission-params" +
                                                 ".json reef/local/app-submission-params.json 1> <LOG_DIR>/driver.stdout" +
                                                 " 2> <LOG_DIR>/driver.stderr";
            string expectedCommand = string.Format(expectedCommandFormat, sizeMB);
            
            var commandBuilder = testContext.GetCommandBuilder(maxPermSize: sizeMB);
            var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
            
            Assert.Equal(expectedCommand, jobSubmissionCommand);
        }

        private class TestContext
        {
            public WindowsYarnJobCommandProvider GetCommandBuilder(bool enableLogging = false, 
                int? maxPermSize = null,
                int? maxMemAllocPoolSize = null)
            {
                var injector = TangFactory.GetTang().NewInjector();
         
                IYarnCommandLineEnvironment yarnCommandLineEnvironment = Substitute.For<IYarnCommandLineEnvironment>();
                yarnCommandLineEnvironment.GetYarnClasspathList().Returns(AnyClassPathItems);
                
                injector.BindVolatileInstance(GenericType<IYarnCommandLineEnvironment>.Class, yarnCommandLineEnvironment);
                injector.BindVolatileParameter(GenericType<EnableDebugLogging>.Class, enableLogging);
                if (maxPermSize.HasValue)
                {
                    injector.BindVolatileParameter<DriverMaxPermSizeMB, int>(maxPermSize.Value);
                }
                if (maxMemAllocPoolSize.HasValue)
                {
                    injector.BindVolatileParameter<DriverMaxMemoryAllicationPoolSizeMB, int>(maxMemAllocPoolSize.Value);
                }

                return injector.GetInstance<WindowsYarnJobCommandProvider>();
            }
        }
    }
}