Merge pull request #3114 from dandsager1/STORM-3488
STORM-3488 Scheduling can cause RAS_Node resources to become negative
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 1a17e52..e555ac8 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -144,7 +144,7 @@
nimbus.blobstore.expiration.secs: 600
storm.blobstore.inputstream.buffer.size.bytes: 65536
-storm.blobstore.dependency.jar.upload.chuck.size.bytes: 1048576
+storm.blobstore.dependency.jar.upload.chunk.size.bytes: 1048576
client.blobstore.class: "org.apache.storm.blobstore.NimbusBlobStore"
storm.blobstore.replication.factor: 3
# For secure mode we would want to change this config to true
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 5d89fc9..b46c112 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -1418,12 +1418,12 @@
@IsInteger
public static final String STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES = "storm.blobstore.inputstream.buffer.size.bytes";
/**
- * What chuck size to use for storm client to upload dependency jars.
+ * What chunk size to use for storm client to upload dependency jars.
*/
@IsPositiveNumber
@IsInteger
- public static final String STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUCK_SIZE_BYTES =
- "storm.blobstore.dependency.jar.upload.chuck.size.bytes";
+ public static final String STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUNK_SIZE_BYTES =
+ "storm.blobstore.dependency.jar.upload.chunk.size.bytes";
/**
* FQCN of a class that implements {@code ISubmitterHook} @see ISubmitterHook for details.
*/
diff --git a/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java b/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java
index 1f6b4f4..85ab835 100644
--- a/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java
+++ b/storm-client/src/jvm/org/apache/storm/dependency/DependencyUploader.java
@@ -48,11 +48,11 @@
private final Map<String, Object> conf;
private ClientBlobStore blobStore;
- private int uploadChuckSize;
+ private final int uploadChunkSize;
public DependencyUploader() {
conf = Utils.readStormConfig();
- this.uploadChuckSize = ObjectReader.getInt(conf.get(Config.STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUCK_SIZE_BYTES), 1024 * 1024);
+ this.uploadChunkSize = ObjectReader.getInt(conf.get(Config.STORM_BLOBSTORE_DEPENDENCY_JAR_UPLOAD_CHUNK_SIZE_BYTES), 1024 * 1024);
}
public void init() {
@@ -164,7 +164,7 @@
try {
blob = getBlobStore().createBlob(key, new SettableBlobMeta(acls));
try (InputStream in = Files.newInputStream(dependency.toPath())) {
- IOUtils.copy(in, blob, this.uploadChuckSize);
+ IOUtils.copy(in, blob, this.uploadChunkSize);
}
blob.close();
blob = null;
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index ddb59c8..3c31d4e 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -81,10 +81,6 @@
public void cleanup(ClientBlobStore store) {
LOG.debug("cleanup target size: {} current size is: {}", targetSize, currentSize);
long bytesOver = currentSize - targetSize;
- if (bytesOver <= 0) { // no need to query remote files
- return;
- }
-
//First delete everything that no longer exists...
for (Iterator<Map.Entry<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>>> i = noReferences.entrySet().iterator();
i.hasNext(); ) {
