Merge pull request #3639 from agresch/agresch_worker_cpu_metric
STORM-4054 -Add Worker CPU Metric
diff --git a/DEPENDENCY-LICENSES b/DEPENDENCY-LICENSES
index e34b9b1..21bfd45 100644
--- a/DEPENDENCY-LICENSES
+++ b/DEPENDENCY-LICENSES
@@ -54,6 +54,8 @@
* Apache Hadoop Annotations (org.apache.hadoop:hadoop-annotations:3.3.6 - no url defined)
* Apache Hadoop Archives (org.apache.hadoop:hadoop-archives:3.3.6 - no url defined)
* Apache Hadoop Auth (org.apache.hadoop:hadoop-auth:3.3.6 - no url defined)
+ * Apache Hadoop Client API (org.apache.hadoop:hadoop-client-api:3.3.6 - no url defined)
+ * Apache Hadoop Client Runtime (org.apache.hadoop:hadoop-client-runtime:3.3.6 - no url defined)
* Apache Hadoop Common (org.apache.hadoop:hadoop-common:3.3.6 - no url defined)
* Apache Hadoop HDFS (org.apache.hadoop:hadoop-hdfs:3.3.6 - no url defined)
* Apache Hadoop HDFS Client (org.apache.hadoop:hadoop-hdfs-client:3.3.6 - no url defined)
@@ -421,7 +423,6 @@
Apache License, Version 2.0, Eclipse Public License - Version 1.0
- * Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:9.4.53.v20231009 - https://eclipse.org/jetty/jetty-client)
* Jetty :: Continuation (org.eclipse.jetty:jetty-continuation:9.4.53.v20231009 - https://eclipse.org/jetty/jetty-continuation)
* Jetty :: Http Utility (org.eclipse.jetty:jetty-http:9.4.53.v20231009 - https://eclipse.org/jetty/jetty-http)
* Jetty :: IO Utility (org.eclipse.jetty:jetty-io:9.4.53.v20231009 - https://eclipse.org/jetty/jetty-io)
diff --git a/examples/storm-elasticsearch-examples/pom.xml b/examples/storm-elasticsearch-examples/pom.xml
index 4e9ecf4..90831c0 100644
--- a/examples/storm-elasticsearch-examples/pom.xml
+++ b/examples/storm-elasticsearch-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-hdfs-examples/pom.xml b/examples/storm-hdfs-examples/pom.xml
index 9f9baf4..1fa2d4c 100644
--- a/examples/storm-hdfs-examples/pom.xml
+++ b/examples/storm-hdfs-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-hive-examples/pom.xml b/examples/storm-hive-examples/pom.xml
index ac5a60a..599462a 100644
--- a/examples/storm-hive-examples/pom.xml
+++ b/examples/storm-hive-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-jdbc-examples/pom.xml b/examples/storm-jdbc-examples/pom.xml
index 71b51f4..d849223 100644
--- a/examples/storm-jdbc-examples/pom.xml
+++ b/examples/storm-jdbc-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-jms-examples/pom.xml b/examples/storm-jms-examples/pom.xml
index 44d2ceb..ce49ca4 100644
--- a/examples/storm-jms-examples/pom.xml
+++ b/examples/storm-jms-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-kafka-client-examples/pom.xml b/examples/storm-kafka-client-examples/pom.xml
index b8e0e05..6e53b1f 100644
--- a/examples/storm-kafka-client-examples/pom.xml
+++ b/examples/storm-kafka-client-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-loadgen/pom.xml b/examples/storm-loadgen/pom.xml
index eaa22e7..7df932a 100644
--- a/examples/storm-loadgen/pom.xml
+++ b/examples/storm-loadgen/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>storm-loadgen</artifactId>
diff --git a/examples/storm-perf/pom.xml b/examples/storm-perf/pom.xml
index e6a90d7..0f97ccf 100644
--- a/examples/storm-perf/pom.xml
+++ b/examples/storm-perf/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-redis-examples/pom.xml b/examples/storm-redis-examples/pom.xml
index 8c2c6da..e8ea91c 100644
--- a/examples/storm-redis-examples/pom.xml
+++ b/examples/storm-redis-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index ace380d..5b61b86 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-autocreds/pom.xml b/external/storm-autocreds/pom.xml
index 5939965..fb73075 100644
--- a/external/storm-autocreds/pom.xml
+++ b/external/storm-autocreds/pom.xml
@@ -15,12 +15,11 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/external/storm-blobstore-migration/pom.xml b/external/storm-blobstore-migration/pom.xml
index a9f0be6..4ac02a2 100644
--- a/external/storm-blobstore-migration/pom.xml
+++ b/external/storm-blobstore-migration/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-elasticsearch/pom.xml b/external/storm-elasticsearch/pom.xml
index 4ebe701..92f678f 100644
--- a/external/storm-elasticsearch/pom.xml
+++ b/external/storm-elasticsearch/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-hdfs-blobstore/pom.xml b/external/storm-hdfs-blobstore/pom.xml
index 75b571e..4fa4dfc 100644
--- a/external/storm-hdfs-blobstore/pom.xml
+++ b/external/storm-hdfs-blobstore/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -29,40 +29,43 @@
<dependencies>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-client</artifactId>
<version>${project.version}</version>
<scope>${provided.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-autocreds</artifactId>
+ <version>${project.version}</version>
<exclusions>
- <!--log4j-over-slf4j must be excluded for hadoop-minicluster
- see: http://stackoverflow.com/q/20469026/3542091 -->
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -74,11 +77,21 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
diff --git a/external/storm-hdfs-oci/pom.xml b/external/storm-hdfs-oci/pom.xml
index 1ea95ab..449d1ec 100644
--- a/external/storm-hdfs-oci/pom.xml
+++ b/external/storm-hdfs-oci/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -28,6 +28,10 @@
<dependencies>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-server</artifactId>
<version>${project.version}</version>
@@ -43,26 +47,13 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <artifactId>hadoop-client-api</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>*</artifactId>
- <groupId>*</groupId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index a3dcdf7..7b35980 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -29,6 +29,10 @@
<dependencies>
<dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-client</artifactId>
<version>${project.version}</version>
@@ -61,34 +65,30 @@
<groupId>org.apache.storm</groupId>
<artifactId>storm-autocreds</artifactId>
<version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
<exclusions>
- <!-- bring in a lot of stuff which isn't needed here -->
<exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
- <groupId>com.fasterxml.woodstox</groupId>
- <artifactId>woodstox-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
@@ -110,11 +110,21 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
diff --git a/external/storm-hive/pom.xml b/external/storm-hive/pom.xml
index b8deced..35ab670 100644
--- a/external/storm-hive/pom.xml
+++ b/external/storm-hive/pom.xml
@@ -15,14 +15,13 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-jdbc/pom.xml b/external/storm-jdbc/pom.xml
index 14bc1f1..ec43c86 100644
--- a/external/storm-jdbc/pom.xml
+++ b/external/storm-jdbc/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-jms/pom.xml b/external/storm-jms/pom.xml
index b318b21..c00f00c 100644
--- a/external/storm-jms/pom.xml
+++ b/external/storm-jms/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-kafka-client/pom.xml b/external/storm-kafka-client/pom.xml
index b6a308a..53b3234 100644
--- a/external/storm-kafka-client/pom.xml
+++ b/external/storm-kafka-client/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-kafka-migration/pom.xml b/external/storm-kafka-migration/pom.xml
index 88322d3..fb1b85f 100644
--- a/external/storm-kafka-migration/pom.xml
+++ b/external/storm-kafka-migration/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-kafka-monitor/pom.xml b/external/storm-kafka-monitor/pom.xml
index 0b6bf6a..45a1aff 100644
--- a/external/storm-kafka-monitor/pom.xml
+++ b/external/storm-kafka-monitor/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/external/storm-metrics/pom.xml b/external/storm-metrics/pom.xml
index a36c2c3..b2ff537 100644
--- a/external/storm-metrics/pom.xml
+++ b/external/storm-metrics/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/external/storm-redis/pom.xml b/external/storm-redis/pom.xml
index 3aa0e1c..d4f3e4a 100644
--- a/external/storm-redis/pom.xml
+++ b/external/storm-redis/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/flux/flux-core/pom.xml b/flux/flux-core/pom.xml
index 2ace0ce..0d99a27 100644
--- a/flux/flux-core/pom.xml
+++ b/flux/flux-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>flux</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/flux/flux-examples/pom.xml b/flux/flux-examples/pom.xml
index 9465ba1..b6cc8a9 100644
--- a/flux/flux-examples/pom.xml
+++ b/flux/flux-examples/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>flux</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/flux/flux-wrappers/pom.xml b/flux/flux-wrappers/pom.xml
index 7d84ead..35972b2 100644
--- a/flux/flux-wrappers/pom.xml
+++ b/flux/flux-wrappers/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>flux</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/flux/pom.xml b/flux/pom.xml
index 5e71d4c..ccede5a 100644
--- a/flux/pom.xml
+++ b/flux/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 716755a..08dbe08 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/pom.xml b/pom.xml
index cf6b5e0..de54eef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
<groupId>org.apache.storm</groupId>
<artifactId>storm</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Storm</name>
<description>Distributed and fault-tolerant realtime computation</description>
@@ -59,7 +59,7 @@
<connection>git@github.com:apache/storm.git</connection>
<developerConnection>scm:git:https://gitbox.apache.org/repos/asf/storm.git</developerConnection>
<url>https://github.com/apache/storm</url>
- <tag>v2.6.1</tag>
+ <tag>v2.6.2</tag>
</scm>
<issueManagement>
@@ -226,6 +226,7 @@
<excludes>
<!-- exclude maven artifacts -->
<exclude>**/target/**</exclude>
+ <exclude>**/target/rat.txt</exclude>
<!-- exclude intellij projects -->
<exclude>**/*.iml</exclude>
<exclude>**/.idea/**</exclude>
diff --git a/sql/pom.xml b/sql/pom.xml
index 00119e1..0807ec5 100644
--- a/sql/pom.xml
+++ b/sql/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/sql/storm-sql-core/pom.xml b/sql/storm-sql-core/pom.xml
index 2b34474..1551ede 100644
--- a/sql/storm-sql-core/pom.xml
+++ b/sql/storm-sql-core/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/sql/storm-sql-external/storm-sql-hdfs/pom.xml b/sql/storm-sql-external/storm-sql-hdfs/pom.xml
index 5bec530..88e0406 100644
--- a/sql/storm-sql-external/storm-sql-hdfs/pom.xml
+++ b/sql/storm-sql-external/storm-sql-hdfs/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
@@ -86,26 +86,18 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
+ <artifactId>hadoop-client-minicluster</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- </exclusions>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/sql/storm-sql-external/storm-sql-kafka/pom.xml b/sql/storm-sql-external/storm-sql-kafka/pom.xml
index e433d27..b2b31ef 100644
--- a/sql/storm-sql-external/storm-sql-kafka/pom.xml
+++ b/sql/storm-sql-external/storm-sql-kafka/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
diff --git a/sql/storm-sql-external/storm-sql-redis/pom.xml b/sql/storm-sql-external/storm-sql-redis/pom.xml
index aad4d65..0a35c15 100644
--- a/sql/storm-sql-external/storm-sql-redis/pom.xml
+++ b/sql/storm-sql-external/storm-sql-redis/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
diff --git a/sql/storm-sql-runtime/pom.xml b/sql/storm-sql-runtime/pom.xml
index 3fd5103..e0f1c87 100644
--- a/sql/storm-sql-runtime/pom.xml
+++ b/sql/storm-sql-runtime/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
index 1070637..9887ed2 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
+++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-buildtools/storm-maven-plugins/pom.xml b/storm-buildtools/storm-maven-plugins/pom.xml
index 77a55a8..e63671b 100644
--- a/storm-buildtools/storm-maven-plugins/pom.xml
+++ b/storm-buildtools/storm-maven-plugins/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-checkstyle/pom.xml b/storm-checkstyle/pom.xml
index 3d125ad..0fc9fcd 100644
--- a/storm-checkstyle/pom.xml
+++ b/storm-checkstyle/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/storm-client/pom.xml b/storm-client/pom.xml
index 64b819b..b7ce59f 100644
--- a/storm-client/pom.xml
+++ b/storm-client/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java b/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java
index c4fd33f..6858228 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/VersionedStore.java
@@ -128,15 +128,11 @@
public void cleanup(int versionsToKeep) throws IOException {
List<Long> versions = getAllVersions();
if (versionsToKeep >= 0) {
- versions = versions.subList(0, Math.min(versions.size(), versionsToKeep));
+ versions = versions.subList(Math.min(versions.size(), versionsToKeep), versions.size());
}
- HashSet<Long> keepers = new HashSet<Long>(versions);
- for (String p : listDir(root)) {
- Long v = parseVersion(p);
- if (v != null && !keepers.contains(v)) {
- deleteVersion(v);
- }
+ for (Long v : versions) {
+ deleteVersion(v);
}
}
diff --git a/storm-clojure-test/pom.xml b/storm-clojure-test/pom.xml
index 9afa83d..17d61bd 100644
--- a/storm-clojure-test/pom.xml
+++ b/storm-clojure-test/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-clojure-test</artifactId>
diff --git a/storm-clojure/pom.xml b/storm-clojure/pom.xml
index 0383639..91053e6 100644
--- a/storm-clojure/pom.xml
+++ b/storm-clojure/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-clojure</artifactId>
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 525a1ad..43371a0 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-dist/binary/final-package/pom.xml b/storm-dist/binary/final-package/pom.xml
index 0a7b490..b5e375d 100644
--- a/storm-dist/binary/final-package/pom.xml
+++ b/storm-dist/binary/final-package/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-storm-bin</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml
index 102bcac..27a2f3c 100644
--- a/storm-dist/binary/pom.xml
+++ b/storm-dist/binary/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-dist/binary/storm-autocreds-bin/pom.xml b/storm-dist/binary/storm-autocreds-bin/pom.xml
index 2dbc107..e67890d 100644
--- a/storm-dist/binary/storm-autocreds-bin/pom.xml
+++ b/storm-dist/binary/storm-autocreds-bin/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-autocreds-bin</artifactId>
<packaging>pom</packaging>
diff --git a/storm-dist/binary/storm-client-bin/pom.xml b/storm-dist/binary/storm-client-bin/pom.xml
index d25c8a6..703b8cc 100644
--- a/storm-dist/binary/storm-client-bin/pom.xml
+++ b/storm-dist/binary/storm-client-bin/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-storm-bin</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml
index d1f827e..1b6a050 100644
--- a/storm-dist/binary/storm-kafka-monitor-bin/pom.xml
+++ b/storm-dist/binary/storm-kafka-monitor-bin/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-kafka-monitor-bin</artifactId>
<packaging>pom</packaging>
diff --git a/storm-dist/binary/storm-sql-core-bin/pom.xml b/storm-dist/binary/storm-sql-core-bin/pom.xml
index d5b6a5e..378e0cd 100644
--- a/storm-dist/binary/storm-sql-core-bin/pom.xml
+++ b/storm-dist/binary/storm-sql-core-bin/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-sql-core-bin</artifactId>
<packaging>pom</packaging>
diff --git a/storm-dist/binary/storm-sql-runtime-bin/pom.xml b/storm-dist/binary/storm-sql-runtime-bin/pom.xml
index db96ad9..610caa6 100644
--- a/storm-dist/binary/storm-sql-runtime-bin/pom.xml
+++ b/storm-dist/binary/storm-sql-runtime-bin/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-sql-runtime-bin</artifactId>
<packaging>pom</packaging>
diff --git a/storm-dist/binary/storm-submit-tools-bin/pom.xml b/storm-dist/binary/storm-submit-tools-bin/pom.xml
index 3c2f7e2..a442a09 100644
--- a/storm-dist/binary/storm-submit-tools-bin/pom.xml
+++ b/storm-dist/binary/storm-submit-tools-bin/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.storm</groupId>
<artifactId>apache-storm-bin</artifactId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
</parent>
<artifactId>storm-submit-tools-bin</artifactId>
<packaging>pom</packaging>
diff --git a/storm-dist/binary/storm-webapp-bin/pom.xml b/storm-dist/binary/storm-webapp-bin/pom.xml
index e2423ea..3c81bc8 100644
--- a/storm-dist/binary/storm-webapp-bin/pom.xml
+++ b/storm-dist/binary/storm-webapp-bin/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-storm-bin</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-dist/source/pom.xml b/storm-dist/source/pom.xml
index 5f85b23..cb0674a 100644
--- a/storm-dist/source/pom.xml
+++ b/storm-dist/source/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-multilang/javascript/pom.xml b/storm-multilang/javascript/pom.xml
index a4c077b..b752271 100644
--- a/storm-multilang/javascript/pom.xml
+++ b/storm-multilang/javascript/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-multilang/python/pom.xml b/storm-multilang/python/pom.xml
index 2d7d0d5..fc52dbe 100644
--- a/storm-multilang/python/pom.xml
+++ b/storm-multilang/python/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-multilang/ruby/pom.xml b/storm-multilang/ruby/pom.xml
index 19156ad..130d9e9 100644
--- a/storm-multilang/ruby/pom.xml
+++ b/storm-multilang/ruby/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/storm-server/pom.xml b/storm-server/pom.xml
index 6457e13..b0ad175 100644
--- a/storm-server/pom.xml
+++ b/storm-server/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
index 953e52a..89d6849 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
@@ -627,14 +627,12 @@
double cpuAvailable = resourcesAvailable.getTotalCpu();
if (cpuAdded > cpuAvailable) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Could not schedule {}:{} on {} not enough CPU {} > {}",
- td.getName(),
- exec,
- ws,
- cpuAdded,
- cpuAvailable);
- }
+ LOG.debug("Could not schedule {}:{} on {} not enough CPU {} > {}",
+ td.getName(),
+ exec,
+ ws,
+ cpuAdded,
+ cpuAvailable);
return false;
}
@@ -642,25 +640,21 @@
double memoryAvailable = resourcesAvailable.getTotalMemoryMb();
if (memoryAdded > memoryAvailable) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Could not schedule {}:{} on {} not enough Mem {} > {}",
- td.getName(),
- exec,
- ws,
- memoryAdded,
- memoryAvailable);
- }
+ LOG.debug("Could not schedule {}:{} on {} not enough Mem {} > {}",
+ td.getName(),
+ exec,
+ ws,
+ memoryAdded,
+ memoryAvailable);
return false;
}
if (afterOnHeap > maxHeap) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Could not schedule {}:{} on {} HEAP would be too large {} > {}",
- td.getName(),
- exec,
- ws,
- afterOnHeap,
- maxHeap);
- }
+ LOG.debug("Could not schedule {}:{} on {} HEAP would be too large {} > {}",
+ td.getName(),
+ exec,
+ ws,
+ afterOnHeap,
+ maxHeap);
return false;
}
return true;
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
index 6928eb1..a70ae86 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
@@ -708,9 +708,27 @@
double memoryRequirement = entry.getValue().getOnHeapMemoryMb();
totalMemoryRequired += memoryRequirement * parallelism;
}
+
+ double ackerMem = getTotalAckerExecutorMemoryUsageForTopo(topology, topoConf);
+ totalMemoryRequired += ackerMem;
+
return totalMemoryRequired;
}
+ private static double getTotalAckerExecutorMemoryUsageForTopo(
+ StormTopology topology, Map<String, Object> topologyConf)
+ throws InvalidTopologyException {
+ topology = StormCommon.systemTopology(topologyConf, topology);
+ Map<String, NormalizedResourceRequest> boltResources = ResourceUtils.getBoltsResources(topology, topologyConf);
+ NormalizedResourceRequest entry = boltResources.get(Acker.ACKER_COMPONENT_ID);
+ if (entry == null) {
+ return 0.0d;
+ }
+ Map<String, Integer> componentParallelism = getComponentParallelism(topologyConf, topology);
+ int parallelism = componentParallelism.getOrDefault(Acker.ACKER_COMPONENT_ID, 1);
+ return entry.getTotalMemoryMb() * parallelism;
+ }
+
public static Map<String, Integer> getComponentParallelism(Map<String, Object> topoConf, StormTopology topology)
throws InvalidTopologyException {
Map<String, Integer> ret = new HashMap<>();
@@ -1398,6 +1416,9 @@
topology = StormCommon.systemTopology(topologyConf, topology);
Map<String, NormalizedResourceRequest> boltResources = ResourceUtils.getBoltsResources(topology, topologyConf);
NormalizedResourceRequest entry = boltResources.get(Acker.ACKER_COMPONENT_ID);
+ if (entry == null) {
+ return 0.0d;
+ }
return entry.getTotalMemoryMb();
}
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
index bed28ce..abc5f25 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestBlacklistScheduler.java
@@ -20,10 +20,19 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.config.Configurator;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
+import org.apache.storm.daemon.StormCommon;
+import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.InvalidTopologyException;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StormTopology;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.DefaultScheduler;
+import org.apache.storm.scheduler.ExecutorDetails;
import org.apache.storm.scheduler.INimbus;
import org.apache.storm.scheduler.IScheduler;
import org.apache.storm.scheduler.SchedulerAssignmentImpl;
@@ -31,11 +40,13 @@
import org.apache.storm.scheduler.Topologies;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.resource.ResourceAwareScheduler;
+import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler;
import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy;
import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategyOld;
import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy;
import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategyOld;
import org.apache.storm.scheduler.resource.strategies.scheduling.RoundRobinResourceAwareStrategy;
+import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -74,6 +85,118 @@
}
@Test
+ public void testBlacklistResumeWhenAckersWontFit() throws InvalidTopologyException {
+ // 3 supervisors exist with 4 slots, 2 are blacklisted
+ // topology with given worker heap size would fit in 4 slots if ignoring ackers, needs 5 slots with ackers.
+ // verify that one of the supervisors will be resumed and topology will schedule.
+
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 300);
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 3);
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 1800);
+ config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 100);
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10);
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, false);
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 3);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 128);
+ config.put(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY, "org.apache.storm.scheduler.blacklist.strategies.RasBlacklistStrategy");
+ config.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, 1);
+ config.setNumWorkers(1);
+ config.put(Config.TOPOLOGY_ACKER_EXECUTORS, 4);
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, "org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy");
+ config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 512);
+ config.put(Config.WORKER_HEAP_MEMORY_MB, 768);
+ config.put(Config.TOPOLOGY_NAME, "testTopology");
+
+ INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
+ StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
+ ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
+ Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4, 400.0d, 4096.0d);
+ Topologies noTopologies = new Topologies();
+ Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), noTopologies, config);
+
+ scheduler = new BlacklistScheduler(new ResourceAwareScheduler());
+ scheduler.prepare(config, metricsRegistry);
+ scheduler.schedule(noTopologies, cluster);
+
+ Map<String, SupervisorDetails> removedSup0 = TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0");
+ Map<String, SupervisorDetails> removedSup0Sup1 = TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(removedSup0, "sup-1");
+
+ cluster = new Cluster(iNimbus, resourceMetrics, removedSup0Sup1,
+ TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), noTopologies, config);
+ scheduler.schedule(noTopologies, cluster);
+ scheduler.schedule(noTopologies, cluster);
+ scheduler.schedule(noTopologies, cluster);
+
+ // 2 supervisors blacklisted at this point. Let's schedule the topology.
+
+ Map<String, TopologyDetails> topoMap = new HashMap<>();
+ TopologyDetails topo1 = createResourceTopo(config);
+ topoMap.put(topo1.getId(), topo1);
+ Topologies topologies = new Topologies(topoMap);
+
+ cluster = new Cluster(iNimbus, resourceMetrics, supMap,
+ TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
+ boolean enableTraceLogging = false; // for scheduling debug
+ if (enableTraceLogging) {
+ Configurator.setAllLevels(LogManager.getRootLogger().getName(), Level.TRACE);
+ }
+ scheduler.schedule(topologies, cluster);
+
+ // topology should be fully scheduled with 1 host remaining blacklisted
+ String topoScheduleStatus = cluster.getStatus("testTopology-id");
+ assertTrue(topoScheduleStatus.contains("Running - Fully Scheduled"));
+ assertEquals(1, cluster.getBlacklistedHosts().size());
+ }
+
+ public TopologyDetails createResourceTopo(Config conf) throws InvalidTopologyException {
+ int spoutParallelism = 6;
+ int boltParallelism = 5;
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("word", new TestUtilsForResourceAwareScheduler.TestSpout(),
+ spoutParallelism);
+ builder.setBolt("exclaim1", new TestUtilsForResourceAwareScheduler.TestBolt(),
+ boltParallelism).shuffleGrouping("word");
+ builder.setBolt("exclaim2", new TestUtilsForResourceAwareScheduler.TestBolt(),
+ boltParallelism).shuffleGrouping("exclaim1");
+
+ StormTopology stormTopology = builder.createTopology();
+
+ TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormTopology, 0,
+ genExecsAndComps(StormCommon.systemTopology(conf, stormTopology)), currentTime, "user");
+ return topo;
+ }
+
+ public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology) {
+ Map<ExecutorDetails, String> retMap = new HashMap<>();
+ int startTask = 1;
+ int endTask = 1;
+ for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) {
+ SpoutSpec spout = entry.getValue();
+ String spoutId = entry.getKey();
+ int spoutParallelism = spout.get_common().get_parallelism_hint();
+ for (int i = 0; i < spoutParallelism; i++) {
+ retMap.put(new ExecutorDetails(startTask, endTask), spoutId);
+ startTask++;
+ endTask++;
+ }
+ }
+
+ for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
+ String boltId = entry.getKey();
+ Bolt bolt = entry.getValue();
+ int boltParallelism = bolt.get_common().get_parallelism_hint();
+ for (int i = 0; i < boltParallelism; i++) {
+ retMap.put(new ExecutorDetails(startTask, endTask), boltId);
+ startTask++;
+ endTask++;
+ }
+ }
+ return retMap;
+ }
+
+ @Test
public void TestBadSupervisor() {
INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java
index 7d2f3fe..403cb99 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/blacklist/TestUtilsForBlacklistScheduler.java
@@ -18,6 +18,7 @@
package org.apache.storm.scheduler.blacklist;
import org.apache.storm.Config;
+import org.apache.storm.Constants;
import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.SpoutSpec;
import org.apache.storm.generated.StormTopology;
@@ -112,6 +113,22 @@
return retList;
}
+ public static Map<String, SupervisorDetails> genSupervisors(int numSup, int numPorts, double cpu, double memory) {
+ Map<String, Double> totalResources = new HashMap<>();
+ totalResources.put(Constants.COMMON_CPU_RESOURCE_NAME, cpu);
+ totalResources.put(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, memory);
+ Map<String, SupervisorDetails> retList = new HashMap<>();
+ for (int i = 0; i < numSup; i++) {
+ List<Number> ports = new LinkedList<>();
+ for (int j = 0; j < numPorts; j++) {
+ ports.add(j);
+ }
+ SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, totalResources);
+ retList.put(sup.getId(), sup);
+ }
+ return retList;
+ }
+
public static TopologyDetails getTopology(String name, Map<String, Object> config, int numSpout, int numBolt,
int spoutParallelism, int boltParallelism, int launchTime, boolean blacklistEnable) {
diff --git a/storm-shaded-deps/pom.xml b/storm-shaded-deps/pom.xml
index 49f03fb..f7f423e 100644
--- a/storm-shaded-deps/pom.xml
+++ b/storm-shaded-deps/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/storm-submit-tools/pom.xml b/storm-submit-tools/pom.xml
index 46d8b1b..5708584 100644
--- a/storm-submit-tools/pom.xml
+++ b/storm-submit-tools/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/storm-webapp/pom.xml b/storm-webapp/pom.xml
index 01d50e3..7ff751f 100644
--- a/storm-webapp/pom.xml
+++ b/storm-webapp/pom.xml
@@ -21,7 +21,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
- <version>2.6.2-SNAPSHOT</version>
+ <version>2.6.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
