[SCB-2330] add injectFault into governance (#3140)
diff --git a/governance/src/main/java/org/apache/servicecomb/governance/GovernanceConfiguration.java b/governance/src/main/java/org/apache/servicecomb/governance/GovernanceConfiguration.java
index 9679999..03eb521 100644
--- a/governance/src/main/java/org/apache/servicecomb/governance/GovernanceConfiguration.java
+++ b/governance/src/main/java/org/apache/servicecomb/governance/GovernanceConfiguration.java
@@ -21,6 +21,7 @@
import org.apache.servicecomb.governance.handler.BulkheadHandler;
import org.apache.servicecomb.governance.handler.CircuitBreakerHandler;
+import org.apache.servicecomb.governance.handler.FaultInjectionHandler;
import org.apache.servicecomb.governance.handler.InstanceIsolationHandler;
import org.apache.servicecomb.governance.handler.RateLimitingHandler;
import org.apache.servicecomb.governance.handler.RetryHandler;
@@ -36,6 +37,7 @@
import org.apache.servicecomb.governance.marker.operator.SuffixOperator;
import org.apache.servicecomb.governance.properties.BulkheadProperties;
import org.apache.servicecomb.governance.properties.CircuitBreakerProperties;
+import org.apache.servicecomb.governance.properties.FaultInjectionProperties;
import org.apache.servicecomb.governance.properties.InstanceIsolationProperties;
import org.apache.servicecomb.governance.properties.MatchProperties;
import org.apache.servicecomb.governance.properties.RateLimitProperties;
@@ -81,6 +83,11 @@
return new RetryProperties();
}
+ @Bean
+ public FaultInjectionProperties faultInjectionProperties() {
+ return new FaultInjectionProperties();
+ }
+
// handlers configuration
@Bean
public BulkheadHandler bulkheadHandler(BulkheadProperties bulkheadProperties) {
@@ -111,6 +118,11 @@
return new RetryHandler(retryProperties, retryExtension);
}
+ @Bean
+ public FaultInjectionHandler faultInjectionHandler(FaultInjectionProperties faultInjectionProperties) {
+ return new FaultInjectionHandler(faultInjectionProperties);
+ }
+
// request processor
@Bean
public RequestProcessor requestProcessor(Map<String, MatchOperator> operatorMap) {
diff --git a/governance/src/main/java/org/apache/servicecomb/governance/handler/FaultInjectionHandler.java b/governance/src/main/java/org/apache/servicecomb/governance/handler/FaultInjectionHandler.java
new file mode 100644
index 0000000..5f5e024
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/governance/handler/FaultInjectionHandler.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.governance.handler;
+
+import org.apache.servicecomb.governance.marker.GovernanceRequest;
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+import org.apache.servicecomb.governance.properties.FaultInjectionProperties;
+import org.apache.servicecomb.injection.Fault;
+import org.apache.servicecomb.injection.FaultInjectionUtil;
+
+public class FaultInjectionHandler extends AbstractGovernanceHandler<Fault, FaultInjectionPolicy> {
+
+ private final FaultInjectionProperties faultInjectionProperties;
+
+ public FaultInjectionHandler(FaultInjectionProperties faultInjectionProperties) {
+ this.faultInjectionProperties = faultInjectionProperties;
+ }
+
+ @Override
+ protected String createKey(GovernanceRequest governanceRequest, FaultInjectionPolicy policy) {
+ return FaultInjectionProperties.MATCH_FAULT_INJECTION_KEY + "." + policy.getName();
+ }
+
+ @Override
+ public FaultInjectionPolicy matchPolicy(GovernanceRequest governanceRequest) {
+ return matchersManager.match(governanceRequest, faultInjectionProperties.getParsedEntity());
+ }
+
+ @Override
+ protected Fault createProcessor(String key, GovernanceRequest governanceRequest, FaultInjectionPolicy policy) {
+ return FaultInjectionUtil.getFault(key, policy);
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/governance/policy/FaultInjectionPolicy.java b/governance/src/main/java/org/apache/servicecomb/governance/policy/FaultInjectionPolicy.java
new file mode 100644
index 0000000..07b853a
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/governance/policy/FaultInjectionPolicy.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.governance.policy;
+
+import java.time.Duration;
+
+import org.apache.servicecomb.injection.FaultInjectionConst;
+
+public class FaultInjectionPolicy extends AbstractPolicy {
+ public static final Duration DEFAULT_TIMEOUT_DURATION = Duration.ofMillis(0);
+
+ private String type = FaultInjectionConst.TYPE_DELAY;
+
+ private String delayTime = DEFAULT_TIMEOUT_DURATION.toString();
+
+ private int percentage = -1;
+
+ private int errorCode = -1;
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getDelayTime() {
+ return delayTime;
+ }
+
+ public void setDelayTime(String delayTime) {
+ this.delayTime = stringOfDuration(delayTime, Duration.ofMillis(-1));
+ }
+
+ public int getPercentage() {
+ return percentage;
+ }
+
+ public void setPercentage(int percentage) {
+ this.percentage = percentage;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+
+ public void setErrorCode(int errorCode) {
+ this.errorCode = errorCode;
+ }
+
+ public long getDelayTimeToMillis() {
+ return Duration.parse(delayTime).toMillis();
+ }
+
+ @Override
+ public boolean isValid() {
+ if (getDelayTimeToMillis() < 0 && FaultInjectionConst.TYPE_DELAY.equals(type)) {
+ return false;
+ }
+ if ((getErrorCode() < FaultInjectionConst.ERROR_CODE_MIN
+ || getErrorCode() > FaultInjectionConst.ERROR_CODE_MAX)
+ && FaultInjectionConst.TYPE_ABORT.equals(type)) {
+ return false;
+ }
+ return super.isValid();
+ }
+
+ @Override
+ public String toString() {
+ return "FaultInjectionPolicy{" +
+ "type=" + type +
+ ", delayTime=" + delayTime +
+ ", percentage=" + percentage +
+ ", errorCode=" + errorCode +
+ '}';
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/governance/properties/FaultInjectionProperties.java b/governance/src/main/java/org/apache/servicecomb/governance/properties/FaultInjectionProperties.java
new file mode 100644
index 0000000..3fe8af4
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/governance/properties/FaultInjectionProperties.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.governance.properties;
+
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+
+public class FaultInjectionProperties extends PolicyProperties<FaultInjectionPolicy> {
+ public static final String MATCH_FAULT_INJECTION_KEY = "servicecomb.faultInjection";
+
+ public FaultInjectionProperties() {
+ super(MATCH_FAULT_INJECTION_KEY);
+ }
+
+ @Override
+ public Class<FaultInjectionPolicy> getEntityClass() {
+ return FaultInjectionPolicy.class;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/AbortFault.java b/governance/src/main/java/org/apache/servicecomb/injection/AbortFault.java
new file mode 100644
index 0000000..dceebee
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/AbortFault.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AbortFault extends AbstractFault {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbortFault.class);
+
+ public static final String ABORTED_ERROR_MSG = "aborted by fault inject";
+
+ public AbortFault(String key, FaultInjectionPolicy policy) {
+ super(key, policy);
+ }
+
+ @Override
+ public void injectFault(FaultParam faultParam) {
+ if (!shouldAbort(faultParam, policy)) {
+ return;
+ }
+
+ // get the config values related to abort percentage.
+ int errorCode = policy.getErrorCode();
+ if (errorCode == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: Abort error code is not configured");
+ return;
+ }
+
+ throw new FaultInjectionException(FaultResponse.createFail(errorCode, ABORTED_ERROR_MSG));
+ }
+
+ private boolean shouldAbort(FaultParam param, FaultInjectionPolicy policy) {
+ // get the config values related to abort.
+ int abortPercent = policy.getPercentage();
+ if (abortPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: Abort percentage is not configured");
+ return false;
+ }
+
+ // check fault abort condition.
+ return FaultInjectionUtil.isFaultNeedToInject(param.getReqCount(), abortPercent);
+ }
+
+ @Override
+ public int getOrder() {
+ return 200;
+ }
+
+ @Override
+ public String getName() {
+ return FaultInjectionConst.TYPE_ABORT;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/AbstractFault.java b/governance/src/main/java/org/apache/servicecomb/injection/AbstractFault.java
new file mode 100644
index 0000000..9fd1b9f
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/AbstractFault.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+
+public abstract class AbstractFault implements Fault {
+ protected String key;
+
+ protected FaultInjectionPolicy policy;
+
+ public AbstractFault(String key, FaultInjectionPolicy policy) {
+ this.key = key;
+ this.policy = policy;
+ }
+
+ @Override
+ public void injectFault() {
+ FaultParam faultParam = FaultInjectionUtil.initFaultParam(key);
+ injectFault(faultParam);
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/DelayFault.java b/governance/src/main/java/org/apache/servicecomb/injection/DelayFault.java
new file mode 100644
index 0000000..d2d9d8f
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/DelayFault.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DelayFault extends AbstractFault {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DelayFault.class);
+
+ public DelayFault(String key, FaultInjectionPolicy policy) {
+ super(key, policy);
+ }
+
+ @Override
+ public int getOrder() {
+ return 100;
+ }
+
+ @Override
+ public void injectFault(FaultParam faultParam) {
+ if (!shouldDelay(faultParam, policy)) {
+ return;
+ }
+
+ LOGGER.debug("Fault injection: delay is added for the request by fault inject handler");
+ long delay = policy.getDelayTimeToMillis();
+ if (delay == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: delay is not configured");
+ return;
+ }
+
+ executeDelay(faultParam, delay);
+ }
+
+ private void executeDelay(FaultParam faultParam, long delay) {
+ Sleepable sleepable = faultParam.getSleepable();
+ if (sleepable != null) {
+ sleepable.sleep(delay);
+ }
+ }
+
+ private boolean shouldDelay(FaultParam param, FaultInjectionPolicy policy) {
+ int delayPercent = policy.getPercentage();
+ if (delayPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: delay percentage is not configured");
+ return false;
+ }
+ // check fault delay condition.
+ return FaultInjectionUtil.isFaultNeedToInject(param.getReqCount(), delayPercent);
+ }
+
+ @Override
+ public String getName() {
+ return FaultInjectionConst.TYPE_DELAY;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/Fault.java b/governance/src/main/java/org/apache/servicecomb/injection/Fault.java
new file mode 100644
index 0000000..84de7d8
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/Fault.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import io.vavr.CheckedFunction0;
+
+public interface Fault {
+ static <T> CheckedFunction0<T> decorateCheckedSupplier(Fault fault, CheckedFunction0<T> supplier) {
+ return () -> {
+ fault.injectFault();
+ return supplier.apply();
+ };
+ }
+
+ int getOrder();
+
+ String getName();
+
+ void injectFault();
+
+ void injectFault(FaultParam faultParam);
+
+ String getKey();
+
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionConst.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionConst.java
new file mode 100644
index 0000000..bf50fc5
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionConst.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+/**
+ * Handles the all constant values for fault injection.
+ */
+public class FaultInjectionConst {
+
+ public static final int FAULT_INJECTION_DEFAULT_VALUE = -1;
+
+ public static final String TYPE_DELAY = "delay";
+
+ public static final String TYPE_ABORT = "abort";
+
+ public static final int ERROR_CODE_MIN = 200;
+
+ public static final int ERROR_CODE_MAX = 600;
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionDecorators.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionDecorators.java
new file mode 100644
index 0000000..c499f3e
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionDecorators.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import io.vavr.CheckedFunction0;
+
+public interface FaultInjectionDecorators {
+ static <T> FaultInjectionDecorateCheckedSupplier<T> ofCheckedSupplier(CheckedFunction0<T> supplier) {
+ return new FaultInjectionDecorateCheckedSupplier<>(supplier);
+ }
+
+ class FaultInjectionDecorateCheckedSupplier<T> {
+
+ private CheckedFunction0<T> supplier;
+
+ protected FaultInjectionDecorateCheckedSupplier(CheckedFunction0<T> supplier) {
+ this.supplier = supplier;
+ }
+
+ public FaultInjectionDecorateCheckedSupplier<T> withFaultInjection(Fault fault) {
+ supplier = Fault.decorateCheckedSupplier(fault, supplier);
+ return this;
+ }
+
+ public T get() throws Throwable {
+ return supplier.apply();
+ }
+ }
+
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionException.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionException.java
new file mode 100644
index 0000000..c061c90
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionException.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+public class FaultInjectionException extends RuntimeException {
+ private static final long serialVersionUID = 1675558351029273343L;
+
+ private final FaultResponse faultResponse;
+
+ public FaultInjectionException(FaultResponse faultResponse) {
+ super(faultResponse.getErrorMsg());
+ this.faultResponse = faultResponse;
+ }
+
+ public FaultResponse getFaultResponse() {
+ return faultResponse;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionUtil.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionUtil.java
new file mode 100644
index 0000000..d95c10f
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultInjectionUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
+
+import io.vertx.core.Context;
+import io.vertx.core.Vertx;
+
+/**
+ * Handles the count for all request based key[transport + microservice qualified name].
+ */
+public final class FaultInjectionUtil {
+
+ private FaultInjectionUtil() {
+ }
+
+ /**
+ * key is transport+operQualifiedName
+ */
+ private static final Map<String, AtomicLong> REQUEST_COUNT = new ConcurrentHashMapEx<>();
+
+ /**
+ * Returns total requests per provider for operational level.
+ *
+ * @param key
+ * transport+operational name
+ * @return long total requests
+ */
+ public static AtomicLong getOperMetTotalReq(String key) {
+ return REQUEST_COUNT.computeIfAbsent(key, p -> new AtomicLong(1));
+ }
+
+ /**
+ * It will check the delay/abort condition based on request count and percentage
+ * received.
+ *
+ * @param reqCount total request count of the uri
+ * @param percentage the percentage of hitting fault injection
+ * @return true: delay/abort is needed. false: delay/abort is not needed.
+ */
+ public static boolean isFaultNeedToInject(long reqCount, int percentage) {
+ /*
+ * Example: delay/abort percentage configured is 10% and Get the count(suppose
+ * if it is 10th request) from map and calculate resultNew(10th request) and
+ * requestOld(9th request). Like this for every request it will calculate
+ * current request count and previous count. if both not matched need to add
+ * delay/abort otherwise no need to add.
+ */
+
+ // calculate the value with current request count.
+ long resultNew = (reqCount * percentage) / 100;
+
+ // calculate the value with previous count value.
+ long resultOld = ((reqCount - 1) * percentage) / 100;
+
+ // if both are not matching then delay/abort should be added.
+ return (resultNew != resultOld);
+ }
+
+ public static Fault getFault(String key, FaultInjectionPolicy policy) {
+ Fault fault = null;
+ if (FaultInjectionConst.TYPE_DELAY.equals(policy.getType())) {
+ fault = new DelayFault(key, policy);
+ } else if (FaultInjectionConst.TYPE_ABORT.equals(policy.getType())) {
+ fault = new AbortFault(key, policy);
+ }
+ return fault;
+ }
+
+ public static FaultParam initFaultParam(String key) {
+ AtomicLong reqCount = FaultInjectionUtil.getOperMetTotalReq(key);
+ // increment the request count here after checking the delay/abort condition.
+ long reqCountCurrent = reqCount.getAndIncrement();
+
+ FaultParam param = new FaultParam(reqCountCurrent);
+ Context currentContext = Vertx.currentContext();
+ if (currentContext != null && currentContext.owner() != null && currentContext.isEventLoopContext()) {
+ param.setSleepable(
+ (delay) -> currentContext.owner().setTimer(delay, timeId -> {}));
+ }
+ return param;
+ }
+
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultParam.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultParam.java
new file mode 100644
index 0000000..3e53a68
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultParam.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Fault injection parameters which decides the fault injection condition.
+ */
+public class FaultParam {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FaultParam.class);
+
+ private final long reqCount;
+
+ private Sleepable sleepable = (delay) -> {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+ LOGGER.info("Interrupted exception is received");
+ }
+ };
+
+ public long getReqCount() {
+ return reqCount;
+ }
+
+ public FaultParam(long reqCount) {
+ this.reqCount = reqCount;
+ }
+
+ public Sleepable getSleepable() {
+ return sleepable;
+ }
+
+ public void setSleepable(Sleepable sleepable) {
+ this.sleepable = sleepable;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/FaultResponse.java b/governance/src/main/java/org/apache/servicecomb/injection/FaultResponse.java
new file mode 100644
index 0000000..603451c
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/FaultResponse.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+public class FaultResponse {
+
+ private int errorCode;
+
+ private String errorMsg;
+
+ public static FaultResponse createFail(int errorCode, String errorMsg) {
+ FaultResponse faultResponse = new FaultResponse();
+ faultResponse.setErrorCode(errorCode);
+ faultResponse.setErrorMsg(errorMsg);
+ return faultResponse;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+
+ public void setErrorCode(int errorCode) {
+ this.errorCode = errorCode;
+ }
+
+ public String getErrorMsg() {
+ return errorMsg;
+ }
+
+ public void setErrorMsg(String errorMsg) {
+ this.errorMsg = errorMsg;
+ }
+}
diff --git a/governance/src/main/java/org/apache/servicecomb/injection/Sleepable.java b/governance/src/main/java/org/apache/servicecomb/injection/Sleepable.java
new file mode 100644
index 0000000..46b9873
--- /dev/null
+++ b/governance/src/main/java/org/apache/servicecomb/injection/Sleepable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.injection;
+
+public interface Sleepable {
+ /**
+ * sleep some time
+ * @param delay time unit is millisecond
+ */
+ void sleep(long delay);
+}
diff --git a/governance/src/test/java/org/apache/servicecomb/governance/FaultInjectionTest.java b/governance/src/test/java/org/apache/servicecomb/governance/FaultInjectionTest.java
new file mode 100644
index 0000000..148b0fb
--- /dev/null
+++ b/governance/src/test/java/org/apache/servicecomb/governance/FaultInjectionTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.governance;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.servicecomb.governance.handler.FaultInjectionHandler;
+import org.apache.servicecomb.governance.marker.GovernanceRequest;
+import org.apache.servicecomb.injection.Fault;
+import org.apache.servicecomb.injection.FaultInjectionDecorators;
+import org.apache.servicecomb.injection.FaultInjectionDecorators.FaultInjectionDecorateCheckedSupplier;
+import org.apache.servicecomb.injection.FaultInjectionException;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ContextConfiguration;
+
+@SpringBootTest
+@ContextConfiguration(classes = {GovernanceConfiguration.class, MockConfiguration.class})
+public class FaultInjectionTest {
+ private FaultInjectionHandler faultInjectionHandler;
+
+ @Autowired
+ public void setFaultInjectionHandler(FaultInjectionHandler faultInjectionHandler) {
+ this.faultInjectionHandler = faultInjectionHandler;
+ }
+
+ public FaultInjectionTest() {
+ }
+
+ @Test
+ public void test_delay_fault_injection_service_name_work() throws Throwable {
+ FaultInjectionDecorateCheckedSupplier<Object> ds =
+ FaultInjectionDecorators.ofCheckedSupplier(() -> "test");
+
+ GovernanceRequest request = new GovernanceRequest();
+ request.setUri("/faultInjectDelay");
+ request.setServiceName("srcService");
+
+ Fault fault = faultInjectionHandler.getActuator(request);
+ ds.withFaultInjection(fault);
+
+ Assertions.assertEquals("test", ds.get());
+
+ // flow control
+ CountDownLatch cd = new CountDownLatch(10);
+ AtomicBoolean expected = new AtomicBoolean(false);
+ AtomicBoolean notExpected = new AtomicBoolean(false);
+ for (int i = 0; i < 10; i++) {
+ new Thread(() -> {
+ try {
+ long startTime = System.currentTimeMillis();
+ Object result = ds.get();
+ if (!"test".equals(result)) {
+ notExpected.set(true);
+ }
+ // delayTime is 2S
+ if (System.currentTimeMillis() - startTime > 1000) {
+ expected.set(true);
+ }
+ } catch (Throwable e) {
+ notExpected.set(true);
+ }
+ cd.countDown();
+ }).start();
+ }
+ //timeout should be bigger than delayTime
+ cd.await(10, TimeUnit.SECONDS);
+ Assertions.assertFalse(notExpected.get());
+ Assertions.assertTrue(expected.get());
+ }
+
+ @Test
+ public void test_abort_fault_injection_service_name_work() throws Throwable {
+ FaultInjectionDecorateCheckedSupplier<Object> ds =
+ FaultInjectionDecorators.ofCheckedSupplier(() -> "test");
+
+ GovernanceRequest request = new GovernanceRequest();
+ request.setUri("/faultInjectAbort");
+ request.setServiceName("srcService");
+
+ Fault fault = faultInjectionHandler.getActuator(request);
+ ds.withFaultInjection(fault);
+
+ Assertions.assertEquals("test", ds.get());
+
+ // flow control
+ CountDownLatch cd = new CountDownLatch(10);
+ AtomicBoolean expected = new AtomicBoolean(false);
+ AtomicBoolean notExpected = new AtomicBoolean(false);
+ for (int i = 0; i < 10; i++) {
+ new Thread(() -> {
+ try {
+ Object result = ds.get();
+ if (!"test".equals(result)) {
+ notExpected.set(true);
+ }
+ } catch (FaultInjectionException e) {
+ expected.set(true);
+ } catch (Throwable e) {
+ notExpected.set(true);
+ }
+ cd.countDown();
+ }).start();
+ }
+ cd.await(1, TimeUnit.SECONDS);
+ Assertions.assertFalse(notExpected.get());
+ Assertions.assertTrue(expected.get());
+ }
+}
diff --git a/governance/src/test/java/org/apache/servicecomb/governance/GovernancePropertiesTest.java b/governance/src/test/java/org/apache/servicecomb/governance/GovernancePropertiesTest.java
index 21135ab..5cd5b25 100644
--- a/governance/src/test/java/org/apache/servicecomb/governance/GovernancePropertiesTest.java
+++ b/governance/src/test/java/org/apache/servicecomb/governance/GovernancePropertiesTest.java
@@ -31,15 +31,18 @@
import org.apache.servicecomb.governance.policy.AbstractPolicy;
import org.apache.servicecomb.governance.policy.BulkheadPolicy;
import org.apache.servicecomb.governance.policy.CircuitBreakerPolicy;
+import org.apache.servicecomb.governance.policy.FaultInjectionPolicy;
import org.apache.servicecomb.governance.policy.RateLimitingPolicy;
import org.apache.servicecomb.governance.policy.RetryPolicy;
import org.apache.servicecomb.governance.properties.BulkheadProperties;
import org.apache.servicecomb.governance.properties.CircuitBreakerProperties;
+import org.apache.servicecomb.governance.properties.FaultInjectionProperties;
import org.apache.servicecomb.governance.properties.GovernanceProperties;
import org.apache.servicecomb.governance.properties.InstanceIsolationProperties;
import org.apache.servicecomb.governance.properties.MatchProperties;
import org.apache.servicecomb.governance.properties.RateLimitProperties;
import org.apache.servicecomb.governance.properties.RetryProperties;
+import org.apache.servicecomb.injection.FaultInjectionConst;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -68,6 +71,8 @@
private RetryProperties retryProperties;
+ private FaultInjectionProperties faultInjectionProperties;
+
private Environment environment;
@Autowired
@@ -106,6 +111,12 @@
}
@Autowired
+ public void setFaultInjectionProperties(
+ FaultInjectionProperties faultInjectionProperties) {
+ this.faultInjectionProperties = faultInjectionProperties;
+ }
+
+ @Autowired
public void setEnvironment(Environment environment) {
this.environment = environment;
}
@@ -153,13 +164,13 @@
@Test
public void test_all_bean_is_loaded() {
- Assertions.assertEquals(5, propertiesList.size());
+ Assertions.assertEquals(6, propertiesList.size());
}
@Test
public void test_match_properties_successfully_loaded() {
Map<String, TrafficMarker> markers = matchProperties.getParsedEntity();
- Assertions.assertEquals(7, markers.size());
+ Assertions.assertEquals(9, markers.size());
TrafficMarker demoRateLimiting = markers.get("demo-rateLimiting");
List<Matcher> matchers = demoRateLimiting.getMatches();
Assertions.assertEquals(1, matchers.size());
@@ -177,17 +188,17 @@
@Test
public void test_match_properties_delete() {
Map<String, TrafficMarker> markers = matchProperties.getParsedEntity();
- Assertions.assertEquals(7, markers.size());
+ Assertions.assertEquals(9, markers.size());
dynamicValues.put("servicecomb.matchGroup.test", "matches:\n"
+ " - apiPath:\n"
+ " exact: \"/hello2\"\n"
+ " name: match0");
GovernanceEventManager.post(new GovernanceConfigurationChangedEvent(new HashSet<>(dynamicValues.keySet())));
markers = matchProperties.getParsedEntity();
- Assertions.assertEquals(8, markers.size());
+ Assertions.assertEquals(10, markers.size());
tearDown();
markers = matchProperties.getParsedEntity();
- Assertions.assertEquals(7, markers.size());
+ Assertions.assertEquals(9, markers.size());
}
@Test
@@ -204,7 +215,7 @@
GovernanceEventManager.post(new GovernanceConfigurationChangedEvent(new HashSet<>(dynamicValues.keySet())));
Map<String, TrafficMarker> markers = matchProperties.getParsedEntity();
- Assertions.assertEquals(8, markers.size());
+ Assertions.assertEquals(10, markers.size());
TrafficMarker demoRateLimiting = markers.get("demo-rateLimiting");
List<Matcher> matchers = demoRateLimiting.getMatches();
Assertions.assertEquals(1, matchers.size());
@@ -388,4 +399,19 @@
Assertions.assertEquals(2, policy.getMinimumNumberOfCalls());
Assertions.assertEquals("2", policy.getSlidingWindowSize());
}
+
+ @Test
+ public void test_fault_injection_properties_successfully_loaded() {
+ Map<String, FaultInjectionPolicy> policies = faultInjectionProperties.getParsedEntity();
+ Assertions.assertEquals(2, policies.size());
+ FaultInjectionPolicy policy = policies.get("demo-faultInjectDelay");
+ Assertions.assertEquals(FaultInjectionConst.TYPE_DELAY, policy.getType());
+ Assertions.assertEquals(2000, policy.getDelayTimeToMillis());
+ Assertions.assertEquals(100, policy.getPercentage());
+
+ policy = policies.get("demo-faultInjectAbort");
+ Assertions.assertEquals(FaultInjectionConst.TYPE_ABORT, policy.getType());
+ Assertions.assertEquals(50, policy.getPercentage());
+ Assertions.assertEquals(500, policy.getErrorCode());
+ }
}
diff --git a/governance/src/test/resources/application.yaml b/governance/src/test/resources/application.yaml
index eee3e84..8a617a1 100644
--- a/governance/src/test/resources/application.yaml
+++ b/governance/src/test/resources/application.yaml
@@ -58,6 +58,14 @@
- apiPath:
exact: "/bulkhead"
services: other:1.0
+ demo-faultInjectDelay: |
+ matches:
+ - apiPath:
+ contains: "/faultInjectDelay"
+ demo-faultInjectAbort: |
+ matches:
+ - apiPath:
+ contains: "/faultInjectAbort"
rateLimiting:
demo-rateLimiting: |
rate: 1
@@ -105,3 +113,15 @@
slidingWindowSize: 2
slidingWindowType: COUNT_BASED
waitDurationInOpenState: 1000
+ faultInjection:
+ demo-faultInjectDelay: |
+ delayTime: 2S
+ type: delay
+ percentage: 100
+ demo-faultInjectAbort: |
+ type: abort
+ percentage: 50
+ errorCode: 500
+ wrongIngored: |
+ delayTime: -1
+ type: ERROR
