[STORM-3758] Ignore upto 10 consecutive exceptions within Worker#checkCredentialsChanged (#3389)
Co-authored-by: Bipin Prasad <bprasad@verizonmedia.com>
diff --git a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
index f7b352c..6196893 100644
--- a/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
+++ b/storm-client/src/jvm/org/apache/storm/daemon/worker/Worker.java
@@ -277,9 +277,25 @@
establishLogSettingCallback();
+ final int credCheckMaxAllowed = 10;
+ final int[] credCheckErrCnt = new int[1]; // consecutive-error-count
+
workerState.refreshCredentialsTimer.scheduleRecurring(0,
(Integer) conf.get(Config.TASK_CREDENTIALS_POLL_SECS), () -> {
- checkCredentialsChanged();
+ try {
+ checkCredentialsChanged();
+ credCheckErrCnt[0] = 0;
+ } catch (Exception ex) {
+ credCheckErrCnt[0]++;
+ if (credCheckErrCnt[0] <= credCheckMaxAllowed) {
+ LOG.warn("Ignoring {} of {} consecutive exceptions when checking for credential change",
+ credCheckErrCnt[0], credCheckMaxAllowed, ex);
+ } else {
+ LOG.error("Received {} consecutive exceptions, {} tolerated, when checking for credential change",
+ credCheckErrCnt[0], credCheckMaxAllowed, ex);
+ throw ex;
+ }
+ }
});
workerState.checkForUpdatedBlobsTimer.scheduleRecurring(0,
