[#4562] fixed dual-AZ engine disk abnormal scenario registration center intermittently delete instance problem (#4563)
diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
index 5b8c434..4d65465 100644
--- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
+++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
@@ -64,7 +64,8 @@
public void startConfigCenterManager() {
this.startTask(new PollConfigurationTask(0));
- this.startTask(new CheckConfigCenterAddressTask());
+ schedulerCheckAddressAvailable("cc-addr-check", new CheckConfigCenterAddressTask(),
+ configCenterConfiguration.getRefreshIntervalInMillis());
}
class PollConfigurationTask implements Task {
@@ -94,9 +95,9 @@
}
}
- class CheckConfigCenterAddressTask implements Task {
+ class CheckConfigCenterAddressTask implements Runnable {
@Override
- public void execute() {
+ public void run() {
List<String> isolationAddresses = configCenterAddressManager.getIsolationAddresses();
if (isolationAddresses.isEmpty()) {
return;
@@ -104,8 +105,6 @@
for (String address : isolationAddresses) {
configCenterClient.checkAddressAvailable(queryConfigurationsRequest, address);
}
- startTask(new BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(),
- new CheckConfigCenterAddressTask()));
}
}
}
diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
index e85a7c6..b44ed0b 100644
--- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
+++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
@@ -125,7 +125,7 @@
addressManager.recoverIsolatedAddress(address);
}
} catch (IOException e) {
- LOGGER.error("check kie config isolation address {} available error!", address, e);
+ LOGGER.error("check kie config isolation address {} available error!", address);
}
}
diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
index bd5ff87..5927431 100644
--- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
+++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
@@ -125,7 +125,8 @@
public void startConfigKieManager() {
this.configurationsRequests.forEach((t) ->
this.startTask(new PollConfigurationTask(0, t)));
- startTask(new CheckKieConfigAddressTask(configurationsRequests.get(0)));
+ schedulerCheckAddressAvailable("kie-addr-check", new CheckKieAddressTask(configurationsRequests.get(0)),
+ kieConfiguration.getRefreshIntervalInMillis());
}
class PollConfigurationTask implements Task {
@@ -161,15 +162,15 @@
}
}
- class CheckKieConfigAddressTask implements Task {
+ class CheckKieAddressTask implements Runnable {
ConfigurationsRequest configurationsRequest;
- public CheckKieConfigAddressTask(ConfigurationsRequest configurationsRequest) {
+ public CheckKieAddressTask(ConfigurationsRequest configurationsRequest) {
this.configurationsRequest = configurationsRequest;
}
@Override
- public void execute() {
+ public void run() {
List<String> isolationAddresses = kieAddressManager.getIsolationAddresses();
if (isolationAddresses.isEmpty()) {
return;
@@ -177,8 +178,6 @@
for (String address : isolationAddresses) {
configKieClient.checkAddressAvailable(this.configurationsRequest, address);
}
- startTask(new BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(),
- new CheckKieConfigAddressTask(this.configurationsRequest)));
}
}
}
diff --git a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
index 59cb3e5..c5587b6 100644
--- a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
+++ b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
@@ -198,7 +198,7 @@
recordSuccessState(address);
if (addressAutoRefreshed) {
if (isolationZoneAddress.remove(address)) {
- LOGGER.warn("restore default address [{}]", address);
+ LOGGER.warn("restore same region address [{}]", address);
if (eventBus != null && availableZone.isEmpty()) {
eventBus.post(new EngineConnectChangedEvent());
}
@@ -212,7 +212,7 @@
return;
}
if (defaultIsolationAddress.remove(address)) {
- LOGGER.warn("restore same region address [{}]", address);
+ LOGGER.warn("restore default address [{}]", address);
addresses.add(address);
}
}
diff --git a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java
index deba2d8..a252714 100644
--- a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java
+++ b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/task/AbstractTask.java
@@ -20,6 +20,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
@@ -63,6 +64,8 @@
private volatile boolean running = true;
+ private ScheduledExecutorService addrCheckExecutor;
+
protected AbstractTask(String taskName) {
initTaskPool(taskName);
Runtime.getRuntime().addShutdownHook(new Thread(AbstractTask.this::stop, taskName + "-shutdown-hook"));
@@ -73,6 +76,13 @@
new Thread(task, taskName));
}
+ protected void schedulerCheckAddressAvailable(String taskName, Runnable task, long delayTime) {
+ if (addrCheckExecutor == null) {
+ addrCheckExecutor = Executors.newScheduledThreadPool(1, (t) -> new Thread(t, taskName));
+ }
+ addrCheckExecutor.scheduleWithFixedDelay(task, delayTime, delayTime, TimeUnit.MILLISECONDS);
+ }
+
protected void startTask(Task task) {
if (!running) {
return;
@@ -96,6 +106,10 @@
running = false;
this.taskPool.shutdown();
this.taskPool.awaitTermination(10, TimeUnit.SECONDS);
+ if (addrCheckExecutor != null) {
+ this.addrCheckExecutor.shutdown();
+ this.addrCheckExecutor.awaitTermination(10, TimeUnit.SECONDS);
+ }
} catch (InterruptedException e) {
LOGGER.warn("tasks not shutdown in time {}", e.getMessage());
}
diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
index c994861..45c4c27 100755
--- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
+++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
@@ -555,14 +555,13 @@
}
@Override
- public void checkIsolationAddressAvailable(String serviceId, String instanceId) {
+ public void checkIsolationAddressAvailable() {
List<String> isolationAddresses = addressManager.getIsolationAddresses();
if (isolationAddresses.isEmpty()) {
return;
}
for (String address : isolationAddresses) {
- httpClient.checkServiceCenterAddressAvailable("/registry/microservices/" + serviceId + "/instances/" + instanceId +
- "/heartbeat", null, null, address);
+ httpClient.checkAddressAvailable("/registry/microservices", null, null, address);
}
}
}
diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
index cd91b24..f296e66 100644
--- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
+++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
@@ -196,9 +196,6 @@
/**
* Check serviceCenter isolation address available
- *
- * @param serviceId serviceId
- * @param instanceId instanceId
*/
- void checkIsolationAddressAvailable(String serviceId, String instanceId);
+ void checkIsolationAddressAvailable();
}
diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
index c58e5dc..e95240b 100755
--- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
+++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
@@ -93,15 +93,15 @@
}
}
- public void checkServiceCenterAddressAvailable(String url, Map<String, String> headers, String content,
+ public void checkAddressAvailable(String url, Map<String, String> headers, String content,
String address) {
String formatUrl = addressManager.formatUrl(url, false, address);
- HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, HttpRequest.PUT);
+ HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, HttpRequest.GET);
try {
httpTransport.doRequest(httpRequest);
addressManager.recoverIsolatedAddress(address);
} catch (IOException e) {
- LOGGER.error("check service center isolation address {} available error!", address, e);
+ LOGGER.error("check service center isolation address {} available error!", address);
}
}
diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
index 2a1296d..3321719 100644
--- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
+++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
@@ -104,6 +104,7 @@
public void startRegistration() {
startTask(new RegisterMicroserviceTask(0));
+ schedulerCheckAddressAvailable("sc-addr-check", new CheckAddressTask(), heartBeatInterval);
}
class RegisterMicroserviceTask implements Task {
@@ -248,7 +249,6 @@
microserviceInstance.getInstanceId());
eventBus.post(new MicroserviceInstanceRegistrationEvent(true, microservice, microserviceInstance));
startTask(new SendHeartBeatTask(0));
- startTask(new CheckServiceCenterAddressTask());
}
} catch (Exception e) {
LOGGER.error("register microservice instance failed, and will try again.", e);
@@ -294,13 +294,10 @@
}
}
- class CheckServiceCenterAddressTask implements Task {
+ class CheckAddressTask implements Runnable {
@Override
- public void execute() {
- serviceCenterClient.checkIsolationAddressAvailable(microservice.getServiceId(),
- microserviceInstance.getInstanceId());
- startTask(new BackOffSleepTask(Math.max(heartBeatInterval, heartBeatRequestTimeout),
- new CheckServiceCenterAddressTask()));
+ public void run() {
+ serviceCenterClient.checkIsolationAddressAvailable();
}
}
}
