[REEF-227]Add implementation of the REEF Client for HDInsight
* Minor fixes for FileSystemJobResourceUploader
* Create HDInsightClientConfiguration which provides correct bindings
* Add HDI submission to HelloREEF
JIRA:
[REEF-227](https://issues.apache.org/jira/browse/REEF-227)
Pull Request:
Closes #850
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs
index 5f75d59..cb163f7 100644
--- a/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/JobResourceUploaderTests.cs
@@ -41,7 +41,7 @@
private const string AnyJobFileResourceAbsoluteUri = AnyScheme + AnyHost + AnyJobFileResourcePath;
private const string AnyLocalArchivePath = @"Any\Local\Archive\Path.zip";
private const string AnyLocalJobFilePath = @"Any\Local\Folder\Path\job-submission-params.json";
- private const long AnyModificationTime = 1447413621;
+ private const long AnyModificationTime = 1447413621000;
private const long AnyResourceSize = 53092;
private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0);
@@ -116,9 +116,9 @@
{
var injector = TangFactory.GetTang().NewInjector();
FileSystem.GetFileStatus(new Uri(AnyUploadedResourceAbsoluteUri))
- .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize));
+ .Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize));
FileSystem.GetFileStatus(new Uri(AnyJobFileResourceAbsoluteUri))
- .Returns(new FileStatus(Epoch + TimeSpan.FromSeconds(AnyModificationTime), AnyResourceSize));
+ .Returns(new FileStatus(Epoch + TimeSpan.FromMilliseconds(AnyModificationTime), AnyResourceSize));
ResourceArchiveFileGenerator.CreateArchiveToUpload(AnyDriverLocalFolderPath)
.Returns(AnyLocalArchivePath);
FileSystem.CreateUriForPath(AnyDriverResourceUploadPath)
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
index babf263..aefc5fe 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
@@ -99,6 +99,14 @@
<Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" />
<Compile Include="Local\Parameters\NumberOfEvaluators.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="YARN\HDI\HDInsightClientConfiguration.cs" />
+ <Compile Include="YARN\HDI\HDInsightCommandLineEnvironment.cs" />
+ <Compile Include="YARN\HDI\HDInsightCredential.cs" />
+ <Compile Include="YARN\HDI\HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs" />
+ <Compile Include="YARN\HDI\HDInsightUrlProvider.cs" />
+ <Compile Include="YARN\HDI\NamedParameters.cs" />
+ <Compile Include="YARN\IdentityResourceFileRemoteUrlToClusterUrlConverter.cs" />
+ <Compile Include="YARN\IResourceFileRemoteUrlToClusterUrlConverter.cs" />
<Compile Include="YARN\IYarnCommandLineEnvironment.cs" />
<Compile Include="YARN\IYarnJobCommandProvider.cs" />
<Compile Include="YARN\IJobResourceUploader.cs" />
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
new file mode 100644
index 0000000..4b91146
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Security;
+using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Client.Yarn.RestClient;
+using Org.Apache.REEF.Client.YARN.Parameters;
+using Org.Apache.REEF.Client.YARN.RestClient;
+using Org.Apache.REEF.IO.FileSystem.AzureBlob;
+using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Util;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ public class HDInsightClientConfiguration : ConfigurationModuleBuilder
+ {
+ public static readonly RequiredParameter<string> HDInsightUsernameParameter = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> HDInsightUrlParameter = new RequiredParameter<string>();
+
+ // Required because user needs to provide the correct path which includes the container name for their
+ // blob storage account where they want to upload the resource files. This container should be the container that
+ // HDInsight is using.
+ public static readonly RequiredParameter<string> JobSubmissionDirectoryPrefix = new RequiredParameter<string>();
+
+ public static readonly OptionalParameter<string> HDInsightPasswordParameter = new OptionalParameter<string>();
+
+ public static readonly OptionalParameter<SecureString> HDInsightSecurePasswordParameter =
+ new OptionalParameter<SecureString>();
+
+ /// <summary>
+ /// Provides <see cref="ConfigurationModule"/> to be used to obtain <see cref="IREEFClient"/> to
+ /// submit REEF application to HDInsight.
+ /// Sample usage:
+ /// <code>
+ /// <![CDATA[
+ /// const string connectionString = "ConnString";
+ /// var config = HDInsightClientConfiguration.GetConfigurationModule(azureBlockBlobConfig)
+ /// .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString)
+ /// .Set(HDInsightClientConfiguration.HDInsightUsernameParameter, @"foo")
+ /// .Set(HDInsightClientConfiguration.HDInsightUrlParameter, @"https://bar.azurehdinsight.net/")
+ /// ...;
+ /// var client = TangFactory.GetTang().NewInjector(config).GetInstance<IREEFClient>();
+ /// ]]>
+ /// </code>
+ /// </summary>
+ public static ConfigurationModule ConfigurationModule = new HDInsightClientConfiguration()
+ .BindNamedParameter(GenericType<HDInsightUrl>.Class, HDInsightUrlParameter)
+ .BindNamedParameter(GenericType<HDInsightUserName>.Class, HDInsightUsernameParameter)
+ .BindNamedParameter(GenericType<HDInsightPasswordSecureString>.Class,
+ HDInsightSecurePasswordParameter)
+ .BindNamedParameter(GenericType<JobSubmissionDirectoryPrefixParameter>.Class,
+ JobSubmissionDirectoryPrefix)
+ .BindNamedParameter(GenericType<HDInsightPasswordString>.Class, HDInsightPasswordParameter)
+ .BindImplementation(GenericType<IREEFClient>.Class, GenericType<YarnREEFDotNetClient>.Class)
+ .BindImplementation(GenericType<IYarnRestClientCredential>.Class,
+ GenericType<HDInsightCredential>.Class)
+ .BindImplementation(GenericType<IUrlProvider>.Class, GenericType<HDInsightUrlProvider>.Class)
+ .BindImplementation(GenericType<IJobResourceUploader>.Class,
+ GenericType<FileSystemJobResourceUploader>.Class)
+ .BindImplementation(GenericType<IYarnCommandLineEnvironment>.Class,
+ GenericType<HDInsightCommandLineEnvironment>.Class)
+ .BindImplementation(GenericType<IResourceFileRemoteUrlToClusterUrlConverter>.Class,
+ GenericType<HDInsightResourceFileRemoteUrlToClusterUrlConverter>.Class)
+ .Merge(AzureBlockBlobFileSystemConfiguration.ConfigurationModule)
+ .Build();
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs
new file mode 100644
index 0000000..e7852c6
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCommandLineEnvironment.cs
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ /// <summary>
+ /// Provides YARN classpath for HDInsight environment
+ /// </summary>
+ public sealed class HDInsightCommandLineEnvironment : IYarnCommandLineEnvironment
+ {
+ [Inject]
+ private HDInsightCommandLineEnvironment()
+ {
+ }
+
+ public IReadOnlyList<string> GetYarnClasspathList()
+ {
+ return new List<string>
+ {
+ "%HADOOP_HOME%/etc/hadoop",
+ "%HADOOP_HOME%/share/hadoop/common/lib/*",
+ "%HADOOP_HOME%/share/hadoop/common/*",
+ "%HADOOP_HOME%/share/hadoop/hdfs",
+ "%HADOOP_HOME%/share/hadoop/hdfs/lib/*",
+ "%HADOOP_HOME%/share/hadoop/hdfs/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/lib/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/*",
+ "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*",
+ "%HADOOP_HOME%/share/hadoop/mapreduce/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/lib/*"
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs
new file mode 100644
index 0000000..6f723cf
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightCredential.cs
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Net;
+using System.Security;
+using Org.Apache.REEF.Client.YARN.RestClient;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ /// <summary>
+ /// Provides a container around ICredential to capture HDInsight credential
+ /// </summary>
+ internal sealed class HDInsightCredential : IYarnRestClientCredential
+ {
+ private readonly ICredentials _credentials;
+
+ [Inject]
+ private HDInsightCredential(
+ [Parameter(typeof(HDInsightUserName))] string userName,
+ [Parameter(typeof(HDInsightPasswordSecureString))] SecureString password)
+ {
+ _credentials = new NetworkCredential(userName, password);
+ }
+
+ [Inject]
+ private HDInsightCredential(
+ [Parameter(typeof(HDInsightUserName))] string userName,
+ [Parameter(typeof(HDInsightPasswordString))] string password)
+ {
+ _credentials = new NetworkCredential(userName, password);
+ }
+
+ public ICredentials Credentials
+ {
+ get { return _credentials; }
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs
new file mode 100644
index 0000000..0332e81
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightResourceFileRemoteUrlToClusterUrlConverter.cs
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using System.Linq;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ internal class HDInsightResourceFileRemoteUrlToClusterUrlConverter : IResourceFileRemoteUrlToClusterUrlConverter
+ {
+ [Inject]
+ private HDInsightResourceFileRemoteUrlToClusterUrlConverter()
+ {
+ }
+
+ public string ConvertToLocalUrl(Uri uri)
+ {
+ // [uri]"https://reefhdi.blob.core.windows.net/reefdev/tmp/foo.json"
+ // Authority : reefhdi.blob.core.windows.net
+ // Segments : {/, reefdev/, tmp/, foo.json}
+ var container = uri.Segments[1].Trim('/');
+ var path = string.Join(string.Empty, uri.Segments.Skip(2));
+ return string.Format("wasb://{0}@{1}/{2}", container, uri.Authority, path);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs
new file mode 100644
index 0000000..d8647ee
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightUrlProvider.cs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Org.Apache.REEF.Client.Yarn.RestClient;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ internal sealed class HDInsightUrlProvider : IUrlProvider
+ {
+ private readonly string _url;
+
+ [Inject]
+ private HDInsightUrlProvider([Parameter(typeof(HDInsightUrl))] string url)
+ {
+ _url = url;
+ }
+
+ public Task<IEnumerable<Uri>> GetUrlAsync()
+ {
+ return Task.FromResult(Enumerable.Repeat(new Uri(_url), 1));
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs
new file mode 100644
index 0000000..1f18691
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/NamedParameters.cs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Security;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.HDI
+{
+ [NamedParameter("User name for HDInsight cluster.")]
+ public sealed class HDInsightUserName : Name<string>
+ {
+ }
+
+ [NamedParameter("Password for HDInsight cluster.")]
+ public sealed class HDInsightPasswordString : Name<string>
+ {
+ }
+
+ [NamedParameter("Password for HDInsight cluster.")]
+ public sealed class HDInsightPasswordSecureString : Name<SecureString>
+ {
+ }
+
+ [NamedParameter("URL for HDInsight cluster like https://foo.azurehdinsight.net/.")]
+ public sealed class HDInsightUrl : Name<string>
+ {
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs
new file mode 100644
index 0000000..dc60d05
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IResourceFileRemoteUrlToClusterUrlConverter.cs
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+ /// <summary>
+ /// For certain filesystem there might be two different URLs for the same resouce,
+ /// depending on access using an external endpoint or from within the system.
+ /// One such example is AzureBlockBlobStorageFileSystem
+ /// https://reefhdi.blob.core.windows.net/reefdev/tmp/foo.txt is the external URL whereas
+ /// when accessing this file from within HDInsight the URL is
+ /// wasb://reefdev@reefhdi.blob.core.windows.net/tmp/foo.txt
+ /// This creates a problem with Job resource upload where SubmitApplication request to YARN
+ /// needs to use the local path whereas resource upload uses external URL.
+ /// This interface allows users to specify logic for the conversion between external URL
+ /// and local URL
+ /// </summary>
+ [DefaultImplementation(typeof(IdentityResourceFileRemoteUrlToClusterUrlConverter))]
+ internal interface IResourceFileRemoteUrlToClusterUrlConverter
+ {
+ string ConvertToLocalUrl(Uri url);
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs
new file mode 100644
index 0000000..a1fe2a9
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IdentityResourceFileRemoteUrlToClusterUrlConverter.cs
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+ internal class IdentityResourceFileRemoteUrlToClusterUrlConverter : IResourceFileRemoteUrlToClusterUrlConverter
+ {
+ [Inject]
+ private IdentityResourceFileRemoteUrlToClusterUrlConverter()
+ {
+ }
+
+ public string ConvertToLocalUrl(Uri url)
+ {
+ return url.AbsoluteUri;
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs
index 83e5ff1..8414475 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/FileSystemJobResourceUploader.cs
@@ -16,7 +16,6 @@
// under the License.
using System;
-using System.Collections.Generic;
using System.IO;
using Org.Apache.REEF.Client.Common;
using Org.Apache.REEF.Client.Yarn;
@@ -40,19 +39,22 @@
private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0);
private readonly IResourceArchiveFileGenerator _resourceArchiveFileGenerator;
private readonly IFileSystem _fileSystem;
- private readonly REEFFileNames _reefFileNames;
private readonly IFile _file;
+ private readonly REEFFileNames _reefFileNames;
+ private readonly IResourceFileRemoteUrlToClusterUrlConverter _urlConverter;
[Inject]
private FileSystemJobResourceUploader(
IResourceArchiveFileGenerator resourceArchiveFileGenerator,
IFileSystem fileSystem,
REEFFileNames reefFileNames,
- IFile file)
+ IFile file,
+ IResourceFileRemoteUrlToClusterUrlConverter urlConverter)
{
+ _urlConverter = urlConverter;
+ _reefFileNames = reefFileNames;
_fileSystem = fileSystem;
_resourceArchiveFileGenerator = resourceArchiveFileGenerator;
- _reefFileNames = reefFileNames;
_file = file;
}
@@ -66,7 +68,7 @@
_fileSystem.CreateDirectory(parentDirectoryUri);
var archivePath = _resourceArchiveFileGenerator.CreateArchiveToUpload(driverLocalFolderPath);
- return GetJobResource(archivePath, ResourceType.ARCHIVE, driverUploadPath);
+ return GetJobResource(archivePath, ResourceType.ARCHIVE, driverUploadPath, _reefFileNames.GetReefFolderName());
}
public JobResource UploadFileResource(string fileLocalPath, string remoteUploadDirectoryPath)
@@ -78,7 +80,7 @@
return GetJobResource(fileLocalPath, ResourceType.FILE, remoteUploadDirectoryPath);
}
- private JobResource GetJobResource(string filePath, ResourceType resourceType, string driverUploadPath)
+ private JobResource GetJobResource(string filePath, ResourceType resourceType, string driverUploadPath, string localizedName = null)
{
if (!_file.Exists(filePath))
{
@@ -97,9 +99,9 @@
return new JobResource
{
- Name = Path.GetFileNameWithoutExtension(filePath),
+ Name = localizedName ?? Path.GetFileName(filePath),
LastModificationUnixTimestamp = DateTimeToUnixTimestamp(fileStatus.ModificationTime),
- RemoteUploadPath = remoteFileUri.AbsoluteUri,
+ RemoteUploadPath = _urlConverter.ConvertToLocalUrl(remoteFileUri),
ResourceSize = fileStatus.LengthBytes,
ResourceType = resourceType
};
@@ -107,7 +109,7 @@
private static long DateTimeToUnixTimestamp(DateTime dateTime)
{
- return (long)(dateTime - Epoch).TotalSeconds;
+ return (long)(dateTime - Epoch).TotalMilliseconds;
}
}
}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs
index dd1e8ab..7fae0c4 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/RESTClient/IUrlProvider.cs
@@ -26,7 +26,10 @@
public interface IUrlProvider
{
/// <summary>
- /// Returns available Yarn resourcemanager web address for the environment
+ /// Returns list of available Yarn resourcemanager addresses for the environment
+ /// For environments with single endpoint/single RM this will be list of size 1.
+ /// For environments with multiple RMs with different endpoints this is list of all
+ /// RMs. Client will try all endpoints until successful/the list is exhausted.
/// </summary>
/// <returns></returns>
Task<IEnumerable<Uri>> GetUrlAsync();
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
index 8c63953..575a13f 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
@@ -19,12 +19,13 @@
using Org.Apache.REEF.Client.API;
using Org.Apache.REEF.Client.Local;
using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Client.YARN.HDI;
using Org.Apache.REEF.Driver;
+using Org.Apache.REEF.IO.FileSystem.AzureBlob;
using Org.Apache.REEF.Tang.Annotations;
using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Tang.Util;
-using Org.Apache.REEF.Utilities.Logging;
namespace Org.Apache.REEF.Examples.HelloREEF
{
@@ -33,10 +34,10 @@
/// </summary>
public sealed class HelloREEF
{
- private static readonly Logger Logger = Logger.GetLogger(typeof(HelloREEF));
private const string Local = "local";
private const string YARN = "yarn";
private const string YARNRest = "yarnrest";
+ private const string HDInsight = "hdi";
private readonly IREEFClient _reefClient;
private readonly JobRequestBuilder _jobRequestBuilder;
@@ -84,6 +85,19 @@
return YARNClientConfiguration.ConfigurationModule.Build();
case YARNRest:
return YARNClientConfiguration.ConfigurationModuleYARNRest.Build();
+ case HDInsight:
+ // To run against HDInsight please replace placeholders below, with actual values for
+ // connection string, container name (available at Azure portal) and HDInsight
+ // credentials (username and password)
+ const string connectionString = "ConnString";
+ const string continerName = "foo";
+ return HDInsightClientConfiguration.ConfigurationModule
+ .Set(HDInsightClientConfiguration.HDInsightPasswordParameter, @"pwd")
+ .Set(HDInsightClientConfiguration.HDInsightUsernameParameter, @"foo")
+ .Set(HDInsightClientConfiguration.HDInsightUrlParameter, @"https://foo.azurehdinsight.net/")
+ .Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix, string.Format(@"/{0}/tmp", continerName))
+ .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString)
+ .Build();
default:
throw new Exception("Unknown runtime: " + name);
}
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj
index 825170f..c044163 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.csproj
@@ -66,6 +66,10 @@
<Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project>
<Name>Org.Apache.REEF.Wake</Name>
</ProjectReference>
+ <ProjectReference Include="..\Org.Apache.REEF.IO\Org.Apache.REEF.IO.csproj">
+ <Project>{DEC0F0A8-DBEF-4EBF-B69C-E2369C15ABF1}</Project>
+ <Name>Org.Apache.REEF.IO</Name>
+ </ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
@@ -76,4 +80,4 @@
</PropertyGroup>
<Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
</Target>
-</Project>
+</Project>
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.sln.DotSettings b/lang/cs/Org.Apache.REEF.sln.DotSettings
index 00610d9..c937155 100644
--- a/lang/cs/Org.Apache.REEF.sln.DotSettings
+++ b/lang/cs/Org.Apache.REEF.sln.DotSettings
@@ -49,12 +49,14 @@
specific language governing permissions and limitations
under the License.
</s:String>
+ <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=HD/@EntryIndexedValue">HDI</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MB/@EntryIndexedValue">MB</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MB/@EntryIndexedValue">GB</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=REEF/@EntryIndexedValue">REEF</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=REST/@EntryIndexedValue">REST</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RM/@EntryIndexedValue">RM</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=URL/@EntryIndexedValue">URL</s:String>
+ <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARN/@EntryIndexedValue">YARN</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNNM/@EntryIndexedValue">YARNNM</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNREST/@EntryIndexedValue">YARNREST</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YARNRM/@EntryIndexedValue">YARNRM</s:String>
