[REEF-1540] Release 0.15.1
Backport the following fixes from master to branch-0.15.1:
* [REEF-1539] Fix memory leak in InteropUtil.cpp
* [REEF-1528] Refactor dependency between yarn runtime and reef-webserver projects
* [REEF-1529] Fixing reef-webserver for multi runtime
Pull Request: Closes #1106
diff --git a/lang/cs/Org.Apache.REEF.Bridge/InteropUtil.cpp b/lang/cs/Org.Apache.REEF.Bridge/InteropUtil.cpp
index 66e1a01..b7dd871 100644
--- a/lang/cs/Org.Apache.REEF.Bridge/InteropUtil.cpp
+++ b/lang/cs/Org.Apache.REEF.Bridge/InteropUtil.cpp
@@ -101,6 +101,7 @@
for (int i = 0; i < len; i++) {
managedByteArray[i] = bytes[i];
}
+ env->ReleaseByteArrayElements(javaByteArray, (jbyte*)bytes, JNI_ABORT);
return managedByteArray;
}
return nullptr;
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
index 58f5a8a..b2faf98 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
@@ -47,6 +47,7 @@
/**
* @param resourceCatalog
* @param resourceRequestHandler
+ * @param loggingScopeFactory
*/
@Inject
public EvaluatorRequestorImpl(final ResourceCatalog resourceCatalog,
@@ -98,7 +99,7 @@
relaxLocality = false;
}
- try (LoggingScope ls = loggingScopeFactory.evaluatorSubmit(req.getNumber())) {
+ try (LoggingScope ls = this.loggingScopeFactory.evaluatorSubmit(req.getNumber())) {
final ResourceRequestEvent request = ResourceRequestEventImpl
.newBuilder()
.setResourceCount(req.getNumber())
diff --git a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java
index bfbc037..62646d2 100644
--- a/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java
+++ b/lang/java/reef-runtime-multi/src/main/java/org/apache/reef/runtime/multi/driver/RuntimesHost.java
@@ -40,16 +40,20 @@
import org.apache.reef.wake.EventHandler;
import org.apache.reef.wake.time.runtime.event.RuntimeStart;
import org.apache.reef.wake.time.runtime.event.RuntimeStop;
+import org.apache.reef.webserver.HttpServer;
import javax.inject.Inject;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
/**
* Hosts the actual runtime implementations and delegates invocations to them.
*/
final class RuntimesHost {
+ private static final Logger LOG = Logger.getLogger(RuntimesHost.class.getName());
private final AvroMultiRuntimeDefinition runtimeDefinition;
private final Injector originalInjector;
private final String defaultRuntimeName;
@@ -127,12 +131,23 @@
runtimeInjector.bindVolatileParameter(
RuntimeParameters.RuntimeStatusHandler.class,
runtimeStatusEventHandler);
+ HttpServer httpServer = null;
+ try {
+ httpServer = this.originalInjector.getInstance(HttpServer.class);
+ } catch (final InjectionException e) {
+ LOG.log(Level.INFO, "Http Server is not configured for the runtime", e);
+ }
+
+ if (httpServer != null) {
+ runtimeInjector.bindVolatileInstance(HttpServer.class, httpServer);
+ LOG.log(Level.INFO, "Binding http server for the runtime implementation");
+ }
}
/**
* Retrieves requested runtime, if requested name is empty a default runtime will be used.
* @param requestedRuntimeName the requested runtime name
- * @return
+ * @return The runtime
*/
private Runtime getRuntime(final String requestedRuntimeName) {
String runtimeName = requestedRuntimeName;
diff --git a/lang/java/reef-runtime-yarn/pom.xml b/lang/java/reef-runtime-yarn/pom.xml
index 8f3e933..556d784 100644
--- a/lang/java/reef-runtime-yarn/pom.xml
+++ b/lang/java/reef-runtime-yarn/pom.xml
@@ -42,6 +42,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>reef-webserver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>reef-utils-hadoop</artifactId>
<version>${project.version}</version>
</dependency>
@@ -65,6 +70,11 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpTrackingURLProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/HttpTrackingURLProvider.java
similarity index 86%
rename from lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpTrackingURLProvider.java
rename to lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/HttpTrackingURLProvider.java
index 2835421..d7829bb 100644
--- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpTrackingURLProvider.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/HttpTrackingURLProvider.java
@@ -16,9 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.reef.webserver;
+package org.apache.reef.runtime.yarn.driver;
-import org.apache.reef.runtime.yarn.driver.TrackingURLProvider;
+import org.apache.reef.webserver.NoOpHttpServerImpl;
+import org.apache.reef.webserver.HttpServer;
import javax.inject.Inject;
import java.net.InetAddress;
@@ -47,7 +48,7 @@
*/
@Inject
public HttpTrackingURLProvider(final HttpServer httpServer) {
- this.httpServer = httpServer;
+ this.httpServer = httpServer instanceof NoOpHttpServerImpl ? null : httpServer;
}
/**
@@ -57,6 +58,10 @@
*/
@Override
public String getTrackingUrl() {
+ if (this.httpServer == null) {
+ return "";
+ }
+
try {
return InetAddress.getLocalHost().getHostAddress() + ":" + httpServer.getPort();
} catch (final UnknownHostException e) {
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/TrackingURLProvider.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/TrackingURLProvider.java
index 068abd6..47d1e40 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/TrackingURLProvider.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/TrackingURLProvider.java
@@ -23,7 +23,7 @@
/**
* Implement this interface to set the tracking URL reported to YARN.
*/
-@DefaultImplementation(DefaultTrackingURLProvider.class)
+@DefaultImplementation(HttpTrackingURLProvider.class)
public interface TrackingURLProvider {
String getTrackingUrl();
}
diff --git a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestTrackingUri.java b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java
similarity index 95%
rename from lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestTrackingUri.java
rename to lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java
index a2ac62e..c97c02c 100644
--- a/lang/java/reef-webserver/src/test/java/org/apache/reef/webserver/TestTrackingUri.java
+++ b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/TestTrackingUri.java
@@ -16,15 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.reef.webserver;
+package org.apache.reef.runtime.yarn.driver;
-import org.apache.reef.runtime.yarn.driver.TrackingURLProvider;
import org.apache.reef.tang.Injector;
import org.apache.reef.tang.JavaConfigurationBuilder;
import org.apache.reef.tang.Tang;
import org.apache.reef.tang.exceptions.BindException;
import org.apache.reef.tang.exceptions.InjectionException;
import org.apache.reef.wake.remote.ports.parameters.TcpPortRangeBegin;
+import org.apache.reef.webserver.HttpHandlerConfiguration;
+import org.apache.reef.webserver.HttpServer;
+import org.apache.reef.webserver.HttpServerImpl;
import org.junit.Assert;
import org.junit.Test;
diff --git a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java
index 66a8b16..a7a9d59 100644
--- a/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java
+++ b/lang/java/reef-runtime-yarn/src/main/test/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandlerTest.java
@@ -23,6 +23,9 @@
import org.apache.reef.driver.evaluator.EvaluatorRequestor;
import org.apache.reef.runtime.common.driver.EvaluatorRequestorImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.exceptions.InjectionException;
+import org.apache.reef.util.logging.LoggingScopeFactory;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -35,7 +38,6 @@
private final MockContainerRequestHandler containerRequestHandler = new MockContainerRequestHandler();
private final YarnResourceRequestHandler resourceRequestHandler = new YarnResourceRequestHandler(containerRequestHandler, applicationMasterRegistration);
private final ResourceCatalog resourceCatalog = Mockito.mock(ResourceCatalog.class);
- private final EvaluatorRequestor evaluatorRequestor = new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler);
private class MockContainerRequestHandler implements YarnContainerRequestHandler {
private AMRMClient.ContainerRequest[] requests;
@@ -54,7 +56,10 @@
* Tests whether the amount of memory is transferred correctly.
*/
@Test
- public void testDifferentMemory() {
+ public void testDifferentMemory() throws InjectionException {
+ final LoggingScopeFactory loggingScopeFactory = Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class);
+ final EvaluatorRequestor evaluatorRequestor = new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory);
+
final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder()
.setNumber(1)
.setMemory(64)
@@ -65,31 +70,30 @@
.setMemory(128)
.setNumberOfCores(2)
.build();
- {
+
evaluatorRequestor.submit(requestOne);
Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory",
requestOne.getMegaBytes(),
containerRequestHandler.getRequests()[0].getCapability().getMemory()
);
- }
- {
+
evaluatorRequestor.submit(requestTwo);
Assert.assertEquals("Request in REEF and YARN form should have the same amount of memory",
requestTwo.getMegaBytes(),
containerRequestHandler.getRequests()[0].getCapability().getMemory()
);
- }
- {
+
evaluatorRequestor.submit(requestOne);
Assert.assertNotEquals("Request in REEF and YARN form should have the same amount of memory",
requestTwo.getMegaBytes(),
containerRequestHandler.getRequests()[0].getCapability().getMemory()
);
- }
}
@Test
- public void testEvaluatorCount() {
+ public void testEvaluatorCount() throws InjectionException {
+ final LoggingScopeFactory loggingScopeFactory = Tang.Factory.getTang().newInjector().getInstance(LoggingScopeFactory.class);
+ final EvaluatorRequestor evaluatorRequestor = new EvaluatorRequestorImpl(resourceCatalog, resourceRequestHandler, loggingScopeFactory);
final EvaluatorRequest requestOne = EvaluatorRequest.newBuilder()
.setNumber(1)
.setMemory(64)
@@ -100,28 +104,23 @@
.setMemory(128)
.setNumberOfCores(2)
.build();
- {
+
evaluatorRequestor.submit(requestOne);
Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators",
requestOne.getNumber(),
containerRequestHandler.getRequests().length
);
- }
- {
+
evaluatorRequestor.submit(requestTwo);
Assert.assertEquals("Request in REEF and YARN form should have the same number of Evaluators",
requestTwo.getNumber(),
containerRequestHandler.getRequests().length
);
- }
- {
+
evaluatorRequestor.submit(requestTwo);
Assert.assertNotEquals("Request in REEF and YARN form should have the same number of Evaluators",
requestOne.getNumber(),
containerRequestHandler.getRequests().length
);
- }
}
-
-
}
diff --git a/lang/java/reef-webserver/pom.xml b/lang/java/reef-webserver/pom.xml
index be554cb..1809494 100644
--- a/lang/java/reef-webserver/pom.xml
+++ b/lang/java/reef-webserver/pom.xml
@@ -107,11 +107,6 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>reef-runtime-yarn</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>${project.groupId}</groupId>
<artifactId>reef-common</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpHandlerConfiguration.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpHandlerConfiguration.java
index 778e202..7c72681 100644
--- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpHandlerConfiguration.java
+++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpHandlerConfiguration.java
@@ -18,7 +18,6 @@
*/
package org.apache.reef.webserver;
-import org.apache.reef.runtime.yarn.driver.TrackingURLProvider;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
import org.apache.reef.tang.formats.OptionalParameter;
@@ -38,6 +37,5 @@
*/
public static final ConfigurationModule CONF = new HttpHandlerConfiguration().merge(HttpRuntimeConfiguration.CONF)
.bindSetEntry(HttpEventHandlers.class, HTTP_HANDLERS)
- .bindImplementation(TrackingURLProvider.class, HttpTrackingURLProvider.class)
.build();
}
diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServer.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServer.java
index d49d774..43c3e8f 100644
--- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServer.java
+++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServer.java
@@ -18,9 +18,12 @@
*/
package org.apache.reef.webserver;
+import org.apache.reef.tang.annotations.DefaultImplementation;
+
/**
* HttpServer interface.
*/
+@DefaultImplementation(NoOpHttpServerImpl.class)
public interface HttpServer {
/**
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/DefaultTrackingURLProvider.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/NoOpHttpServerImpl.java
similarity index 63%
rename from lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/DefaultTrackingURLProvider.java
rename to lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/NoOpHttpServerImpl.java
index af2442a..9958b73 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/DefaultTrackingURLProvider.java
+++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/NoOpHttpServerImpl.java
@@ -16,18 +16,34 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.reef.runtime.yarn.driver;
+
+package org.apache.reef.webserver;
import javax.inject.Inject;
-final class DefaultTrackingURLProvider implements TrackingURLProvider {
-
+/**
+ * Empty implementation for the HttpServer.
+ * It has no functionality and serves only as an injectable placeholder.
+ */
+public final class NoOpHttpServerImpl implements HttpServer {
@Inject
- DefaultTrackingURLProvider() {
+ NoOpHttpServerImpl() {
}
@Override
- public String getTrackingUrl() {
- return "";
+ public void start() throws Exception {
+ }
+
+ @Override
+ public void stop() throws Exception {
+ }
+
+ @Override
+ public int getPort() {
+ return 0;
+ }
+
+ @Override
+ public void addHttpHandler(final HttpHandler httpHandler) {
}
}
