GEODE-7341: Provide a way to avoid memory lock if over committed (#4210)
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
index b7f238c..aed3e68 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
@@ -136,11 +136,6 @@
public class InternalDistributedSystem extends DistributedSystem
implements LogConfigSupplier {
- /**
- * True if the user is allowed lock when memory resources appear to be overcommitted.
- */
- private static final boolean ALLOW_MEMORY_LOCK_WHEN_OVERCOMMITTED =
- Boolean.getBoolean(GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT");
private static final Logger logger = LogService.getLogger();
private static final String DISABLE_MANAGEMENT_PROPERTY =
@@ -149,6 +144,11 @@
public static final String ALLOW_MULTIPLE_SYSTEMS_PROPERTY =
GEMFIRE_PREFIX + "ALLOW_MULTIPLE_SYSTEMS";
+ public static final String ALLOW_MEMORY_OVERCOMMIT =
+ GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT";
+ public static final String AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT =
+ GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT";
+
/**
* If auto-reconnect is going on this will hold a reference to it
*/
@@ -174,6 +174,16 @@
private final StatisticsManager statisticsManager;
private MetricsService metricsService;
private final FunctionStatsManager functionStatsManager;
+ /**
+ * True if the user is allowed lock when memory resources appear to be overcommitted.
+ */
+ private final boolean allowMemoryLockWhenOvercommitted =
+ Boolean.getBoolean(ALLOW_MEMORY_OVERCOMMIT);
+ /**
+ * True if memory lock is avoided when memory resources appear to be overcommitted.
+ */
+ private final boolean avoidMemoryLockWhenOvercommitted =
+ Boolean.getBoolean(AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT);
/**
* The distribution manager that is used to communicate with the distributed system.
@@ -728,28 +738,7 @@
// included the available memory calculation.
long avail = LinuxProcFsStatistics.getAvailableMemory(logger);
long size = offHeapMemorySize + Runtime.getRuntime().totalMemory();
- if (avail < size) {
- if (ALLOW_MEMORY_LOCK_WHEN_OVERCOMMITTED) {
- logger.warn(
- "System memory appears to be over committed by {} bytes. You may experience "
- + "instability, performance issues, or terminated processes due to the Linux "
- + "OOM killer.",
- size - avail);
- } else {
- throw new IllegalStateException(
- String.format(
- "Insufficient free memory (%s) when attempting to lock %s bytes. Either "
- + "reduce the amount of heap or off-heap memory requested or free up "
- + "additional system memory. You may also specify -Dgemfire.Cache"
- + ".ALLOW_MEMORY_OVERCOMMIT=true on the command-line to override the "
- + "constraint check.",
- avail, size));
- }
- }
-
- logger.info("Locking memory. This may take a while...");
- GemFireCacheImpl.lockMemory();
- logger.info("Finished locking memory.");
+ lockMemory(avail, size);
}
try {
@@ -830,6 +819,34 @@
reconnectAttemptCounter.set(0);
}
+ void lockMemory(long avail, long size) {
+ if (avail < size) {
+ if (avoidMemoryLockWhenOvercommitted) {
+ logger.warn(
+ "System memory appears to be over committed by {} bytes. Memory will not be locked because -D{} is set to true.",
+ size - avail, AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT);
+ } else if (allowMemoryLockWhenOvercommitted) {
+ logger.warn(
+ "System memory appears to be over committed by {} bytes. Memory is locked anyway because -D{} is set to true. You may experience instability, performance issues, or terminated processes due to the Linux OOM killer.",
+ size - avail, ALLOW_MEMORY_OVERCOMMIT);
+ lockMemory();
+ } else {
+ throw new IllegalStateException(
+ String.format(
+ "Insufficient free memory (%s) when attempting to lock %s bytes. Either reduce the amount of heap or off-heap memory requested or free up additional system memory. You may also specify -D%s=true on the command-line to override the constraint check.",
+ avail, size, ALLOW_MEMORY_OVERCOMMIT));
+ }
+ } else {
+ lockMemory();
+ }
+ }
+
+ void lockMemory() {
+ logger.info("Locking memory. This may take a while...");
+ GemFireCacheImpl.lockMemory();
+ logger.info("Finished locking memory.");
+ }
+
private void startSampler() {
if (statsDisabled) {
return;
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java
new file mode 100644
index 0000000..ad68bb5
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.distributed.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Properties;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.RestoreSystemProperties;
+
+import org.apache.geode.metrics.internal.MetricsService;
+
+
+public class InternalDistributedSystemTest {
+ private MetricsService.Builder builder;
+
+ @Before
+ public void setup() {
+ builder = mock(MetricsService.Builder.class);
+ when(builder.build(any())).thenReturn(mock(MetricsService.class));
+ }
+
+ @Rule
+ public RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
+
+ @Test
+ public void lockMemoryAllowedIfAllowMemoryOverCommitIsSet() {
+ System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties(), builder).build());
+ doNothing().when(system).lockMemory();
+
+ system.lockMemory(100, 200);
+
+ verify(system).lockMemory();
+ }
+
+ @Test
+ public void lockMemoryAvoidedIfAvoidMemoryLockWhenOverCommitIsSet() {
+ System.setProperty(
+ DistributionConfig.GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties(), builder).build());
+
+ system.lockMemory(100, 200);
+
+ verify(system, never()).lockMemory();
+ }
+
+ @Test
+ public void lockMemoryAvoidedIfAvoidAndAllowMemoryLockWhenOverCommitBothSet() {
+ System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT", "true");
+ System.setProperty(
+ DistributionConfig.GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties(), builder).build());
+
+ system.lockMemory(100, 200);
+
+ verify(system, never()).lockMemory();
+ }
+
+
+ @Test
+ public void lockMemoryThrowsIfMemoryOverCommit() {
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties(), builder).build());
+
+ Throwable caughtException = catchThrowable(() -> system.lockMemory(100, 200));
+
+ assertThat(caughtException).isInstanceOf(IllegalStateException.class);
+ verify(system, never()).lockMemory();
+ }
+
+ @Test
+ public void locksMemoryIfMemoryNotOverCommit() {
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties(), builder).build());
+ doNothing().when(system).lockMemory();
+
+ system.lockMemory(200, 100);
+
+ verify(system).lockMemory();
+ }
+}
