IGNITE-13592 .NET: Respect cgroup limits in DataRegionConfiguration.DefaultMaxSize
* Read cgroup limits to determine available memory for the current process
* Cache retrieved values in static fields
* Tweak `IgniteConfigurationTest.TestSpringXml` to account for the fact that older JDK versions ignore cgroup limits
* Skip `MaxSize` property check in `IgnitionStartTest.TestIgniteStartsFromSpringXml`
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
index d3e8f46..0f67ed4 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
@@ -209,6 +209,7 @@
<Compile Include="Client\Services\TestServiceGenericMethods.cs" />
<Compile Include="Client\Services\TestServiceOverloads.cs" />
<Compile Include="Common\IgniteProductVersionTests.cs" />
+ <Compile Include="Common\MemoryInfoTest.cs" />
<Compile Include="Compute\ComputeApiTest.JavaTask.cs" />
<Compile Include="Compute\ComputeWithExecutorTest.cs" />
<Compile Include="Deployment\CacheGetFunc.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Common/MemoryInfoTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Common/MemoryInfoTest.cs
new file mode 100644
index 0000000..3f2052b
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Common/MemoryInfoTest.cs
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Tests.Common
+{
+ using Apache.Ignite.Core.Impl;
+ using Apache.Ignite.Core.Impl.Common;
+ using Apache.Ignite.Core.Impl.Unmanaged;
+ using NUnit.Framework;
+
+ /// <summary>
+ /// Tests for <see cref="MemoryInfo"/>.
+ /// </summary>
+ public class MemoryInfoTest
+ {
+ /// <summary>
+ /// Tests that cgroup limit can be always determined on Linux.
+ /// </summary>
+ [Test]
+ public void TestMemoryInfoReturnsNonNullLimitOnLinux()
+ {
+ if (Os.IsWindows)
+ {
+ return;
+ }
+
+ Assert.IsNotNull(MemoryInfo.TotalPhysicalMemory);
+ Assert.Greater(MemoryInfo.TotalPhysicalMemory, 655360);
+
+ Assert.IsNotNull(MemoryInfo.MemoryLimit);
+ Assert.Greater(MemoryInfo.MemoryLimit, 655360);
+
+ Assert.IsNotNull(CGroup.MemoryLimitInBytes);
+ Assert.Greater(CGroup.MemoryLimitInBytes, 655360);
+
+ if (CGroup.MemoryLimitInBytes > MemoryInfo.TotalPhysicalMemory)
+ {
+ Assert.AreEqual(MemoryInfo.TotalPhysicalMemory, MemoryInfo.MemoryLimit,
+ "When cgroup limit is not set, memory limit is equal to physical memory amount.");
+ }
+ else
+ {
+ Assert.AreEqual(CGroup.MemoryLimitInBytes, MemoryInfo.MemoryLimit,
+ "When cgroup limit is set, memory limit is equal to cgroup limit.");
+ }
+ }
+ }
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
index b4bc0fe..e043b7e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
@@ -40,6 +40,7 @@
using Apache.Ignite.Core.Discovery.Tcp.Static;
using Apache.Ignite.Core.Encryption.Keystore;
using Apache.Ignite.Core.Events;
+ using Apache.Ignite.Core.Impl;
using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.PersistentStore;
using Apache.Ignite.Core.Tests.Plugin;
@@ -611,11 +612,24 @@
Assert.AreEqual(DataRegionConfiguration.DefaultEmptyPagesPoolSize, cfg.EmptyPagesPoolSize);
Assert.AreEqual(DataRegionConfiguration.DefaultEvictionThreshold, cfg.EvictionThreshold);
Assert.AreEqual(DataRegionConfiguration.DefaultInitialSize, cfg.InitialSize);
- Assert.AreEqual(DataRegionConfiguration.DefaultMaxSize, cfg.MaxSize);
Assert.AreEqual(DataRegionConfiguration.DefaultPersistenceEnabled, cfg.PersistenceEnabled);
Assert.AreEqual(DataRegionConfiguration.DefaultMetricsRateTimeInterval, cfg.MetricsRateTimeInterval);
Assert.AreEqual(DataRegionConfiguration.DefaultMetricsSubIntervalCount, cfg.MetricsSubIntervalCount);
Assert.AreEqual(default(long), cfg.CheckpointPageBufferSize);
+
+ if (DataRegionConfiguration.DefaultMaxSize != cfg.MaxSize)
+ {
+ // Java respects cgroup limits only in recent JDK versions.
+ // We don't know which version is used for tests, so we should expect both variants
+ var physMem = MemoryInfo.TotalPhysicalMemory;
+ Assert.IsNotNull(physMem);
+
+ var expected = (long) physMem / 5;
+
+ Assert.AreEqual(expected, cfg.MaxSize,
+ string.Format("Expected max size with cgroup limit: '{0}', without: '{1}'",
+ DataRegionConfiguration.DefaultMaxSize, expected));
+ }
}
/// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgnitionStartTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgnitionStartTest.cs
index 7bcf32c..a0dfef1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgnitionStartTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgnitionStartTest.cs
@@ -18,6 +18,7 @@
namespace Apache.Ignite.Core.Tests
{
using System;
+ using System.Collections.Generic;
using System.IO;
using System.Linq;
using Apache.Ignite.Core.Configuration;
@@ -115,7 +116,9 @@
Name = "default"
}
};
- AssertExtensions.ReflectionEqual(dsCfg, resCfg.DataStorageConfiguration);
+
+ AssertExtensions.ReflectionEqual(dsCfg, resCfg.DataStorageConfiguration,
+ ignoredProperties: new HashSet<string> {"MaxSize"});
}
}
}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index 76b77fe..99bf6fd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -132,6 +132,7 @@
<Compile Include="Impl\Client\Services\ServicesClient.cs" />
<Compile Include="Impl\Client\SocketEndpoint.cs" />
<Compile Include="Impl\Client\Transactions\ClientCacheTransactionManager.cs" />
+ <Compile Include="Impl\Common\Cgroup.cs" />
<Compile Include="Impl\Common\PlatformType.cs" />
<Compile Include="Impl\Client\Transactions\TransactionClient.cs" />
<Compile Include="Impl\Client\Transactions\TransactionsClient.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs
index 0eedd48..00e6647 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs
@@ -51,7 +51,7 @@
/// The default maximum size, equals to 20% of total RAM.
/// </summary>
public static readonly long DefaultMaxSize =
- (long) ((long) MemoryInfo.GetTotalPhysicalMemory(2048L * 1024 * 1024) * 0.2);
+ (long) ((long) (MemoryInfo.MemoryLimit ?? 2048L * 1024 * 1024) * 0.2);
/// <summary>
/// The default sub intervals.
@@ -187,14 +187,14 @@
public TimeSpan RateTimeInterval { get; set; }
/// <summary>
- /// Gets or sets the number of sub intervals to split <see cref="RateTimeInterval"/> into to calculate
+ /// Gets or sets the number of sub intervals to split <see cref="RateTimeInterval"/> into to calculate
/// <see cref="IMemoryMetrics.AllocationRate"/> and <see cref="IMemoryMetrics.EvictionRate"/>.
/// <para />
/// Bigger value results in more accurate metrics.
/// </summary>
[DefaultValue(DefaultSubIntervals)]
- [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly",
+ [SuppressMessage("Microsoft.Naming", "CA1702:CompoundWordsShouldBeCasedCorrectly",
Justification = "Consistency with Java config")]
public int SubIntervals { get; set; }
}
-}
\ No newline at end of file
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/DataRegionConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/DataRegionConfiguration.cs
index 35417a7..c50e8a1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/DataRegionConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Configuration/DataRegionConfiguration.cs
@@ -27,7 +27,7 @@
/// <summary>
/// Defines custom data region configuration for Apache Ignite page memory
- /// (see <see cref="DataStorageConfiguration"/>).
+ /// (see <see cref="DataStorageConfiguration"/>).
/// <para />
/// For each configured data region Apache Ignite instantiates respective memory regions with different
/// parameters like maximum size, eviction policy, swapping options, etc.
@@ -60,7 +60,7 @@
/// The default maximum size, equals to 20% of total RAM.
/// </summary>
public static readonly long DefaultMaxSize =
- (long)((long)MemoryInfo.GetTotalPhysicalMemory(2048L * 1024 * 1024) * 0.2);
+ (long)((long) (MemoryInfo.MemoryLimit ?? 2048L * 1024 * 1024) * 0.2);
/// <summary>
/// The default sub intervals.
@@ -171,7 +171,7 @@
/// <summary>
/// Gets or sets the page eviction mode. If <see cref="DataPageEvictionMode.Disabled"/> is used (default)
- /// then an out of memory exception will be thrown if the memory region usage
+ /// then an out of memory exception will be thrown if the memory region usage
/// goes beyond <see cref="MaxSize"/>.
/// </summary>
public DataPageEvictionMode PageEvictionMode { get; set; }
@@ -212,7 +212,7 @@
public TimeSpan MetricsRateTimeInterval { get; set; }
/// <summary>
- /// Gets or sets the number of sub intervals to split <see cref="MetricsRateTimeInterval"/> into to calculate
+ /// Gets or sets the number of sub intervals to split <see cref="MetricsRateTimeInterval"/> into to calculate
/// <see cref="IDataRegionMetrics.AllocationRate"/> and <see cref="IDataRegionMetrics.EvictionRate"/>.
/// <para />
/// Bigger value results in more accurate metrics.
@@ -228,7 +228,7 @@
/// Default is <c>0</c>: Ignite will choose buffer size automatically.
/// </summary>
public long CheckpointPageBufferSize { get; set; }
-
+
/// <summary>
/// Gets or sets the lazy memory allocation flag.
/// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Cgroup.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Cgroup.cs
new file mode 100644
index 0000000..b797b09
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/Cgroup.cs
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Impl.Common
+{
+ using System;
+ using System.Collections.Generic;
+ using System.Diagnostics.CodeAnalysis;
+ using System.IO;
+ using System.Linq;
+ using Apache.Ignite.Core.Impl.Unmanaged;
+
+ /// <summary>
+ /// Reads cgroup limits for the current process.
+ /// <para />
+ /// Based on cgroup handling in CLR:
+ /// https://github.com/dotnet/runtime/blob/master/src/coreclr/src/gc/unix/cgroup.cpp
+ /// </summary>
+ internal static class CGroup
+ {
+ /// <summary>
+ /// Gets cgroup memory limit in bytes.
+ /// </summary>
+ public static readonly ulong? MemoryLimitInBytes = GetMemoryLimitInBytes();
+
+ /** */
+ private const string MemorySubsystem = "memory";
+
+ /** */
+ private const string MemoryLimitFileName = "memory.limit_in_bytes";
+
+ /** */
+ private const string ProcMountInfoFileName = "/proc/self/mountinfo";
+
+ /** */
+ private const string ProcCGroupFileName = "/proc/self/cgroup";
+
+ /// <summary>
+ /// Gets memory limit in bytes.
+ /// </summary>
+ [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
+ private static ulong? GetMemoryLimitInBytes()
+ {
+ if (Os.IsWindows)
+ {
+ return null;
+ }
+
+ try
+ {
+ var memMount = FindHierarchyMount(MemorySubsystem);
+ if (memMount == null)
+ {
+ return null;
+ }
+
+ var cgroupPathRelativeToMount = FindCGroupPath(MemorySubsystem);
+ if (cgroupPathRelativeToMount == null)
+ {
+ return null;
+ }
+
+ var hierarchyMount = memMount.Value.Key;
+ var hierarchyRoot = memMount.Value.Value;
+
+ // Host CGroup: append the relative path
+ // In Docker: root and relative path are the same
+ var groupPath =
+ string.Equals(hierarchyRoot, cgroupPathRelativeToMount, StringComparison.Ordinal)
+ ? hierarchyMount
+ : hierarchyMount + cgroupPathRelativeToMount;
+
+ var memLimitFilePath = Path.Combine(groupPath, MemoryLimitFileName);
+
+ var memLimitText = File.ReadAllText(memLimitFilePath);
+
+ ulong memLimit;
+ if (ulong.TryParse(memLimitText, out memLimit))
+ {
+ return memLimit;
+ }
+ }
+ catch (Exception e)
+ {
+ Console.Error.WriteLine("Failed to determine cgroup memory limit: " + e);
+ }
+
+ return null;
+ }
+
+ /// <summary>
+ /// Finds the hierarchy mount and root for the current process.
+ /// </summary>
+ private static KeyValuePair<string, string>? FindHierarchyMount(string subsystem)
+ {
+ foreach (var line in File.ReadAllLines(ProcMountInfoFileName))
+ {
+ var mount = GetHierarchyMount(line, subsystem);
+
+ if (mount != null)
+ {
+ return mount;
+ }
+ }
+
+ return null;
+ }
+
+ /// <summary>
+ /// Get the hierarchy mount and root.
+ /// </summary>
+ private static KeyValuePair<string, string>? GetHierarchyMount(string mountInfo, string subsystem)
+ {
+ // Example: 41 34 0:35 / /sys/fs/cgroup/memory rw,nosuid,nodev shared:17 - cgroup cgroup rw,memory
+ const string cGroup = " - cgroup ";
+
+ var cgroupIdx = mountInfo.IndexOf(cGroup, StringComparison.Ordinal);
+
+ if (cgroupIdx < 0)
+ return null;
+
+ var optionsIdx = mountInfo.LastIndexOf(" ", cgroupIdx + cGroup.Length, StringComparison.Ordinal);
+
+ if (optionsIdx < 0)
+ return null;
+
+ var memIdx = mountInfo.IndexOf(subsystem, optionsIdx + 1, StringComparison.Ordinal);
+
+ if (memIdx < 0)
+ return null;
+
+ var parts = mountInfo.Split(' ');
+
+ if (parts.Length < 5)
+ return null;
+
+ return new KeyValuePair<string, string>(parts[4], parts[3]);
+ }
+
+ /// <summary>
+ /// Finds the cgroup path for the current process.
+ /// </summary>
+ private static string FindCGroupPath(string subsystem)
+ {
+ var lines = File.ReadAllLines(ProcCGroupFileName);
+
+ foreach (var line in lines)
+ {
+ var parts = line.Split(new[] {':'}, 3);
+
+ if (parts.Length == 3 && parts[1].Split(',').Contains(subsystem, StringComparer.Ordinal))
+ {
+ return parts[2];
+ }
+ }
+
+ return null;
+ }
+ }
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/MemoryInfo.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/MemoryInfo.cs
index 7cfd019..062065e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/MemoryInfo.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/MemoryInfo.cs
@@ -17,11 +17,13 @@
namespace Apache.Ignite.Core.Impl
{
+ using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
+ using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Core.Impl.Unmanaged;
/// <summary>
@@ -30,19 +32,56 @@
internal static class MemoryInfo
{
/// <summary>
- /// Gets the total physical memory.
+ /// Gets total physical memory.
/// </summary>
- public static ulong GetTotalPhysicalMemory(ulong defaultValue)
+ public static readonly ulong? TotalPhysicalMemory = GetTotalPhysicalMemory();
+
+ /// <summary>
+ /// Gets memory limit (when set by cgroups) or the value of <see cref="TotalPhysicalMemory"/>.
+ /// </summary>
+ public static readonly ulong? MemoryLimit = GetMemoryLimit();
+
+ /// <summary>
+ /// Gets the memory limit.
+ /// <para />
+ /// When memory is limited with cgroups, returns that limit. Otherwise, returns total physical memory.
+ /// </summary>
+ private static ulong? GetMemoryLimit()
{
if (Os.IsWindows)
{
- return NativeMethodsWindows.GlobalMemoryStatusExTotalPhys();
+ return null;
}
- const string memInfo = "/proc/meminfo";
+ var physical = TotalPhysicalMemory;
- if (File.Exists(memInfo))
+ if (physical == null)
{
+ return null;
+ }
+
+ var limit = CGroup.MemoryLimitInBytes;
+
+ return limit != null && limit < physical
+ ? limit.Value
+ : physical.Value;
+ }
+
+ /// <summary>
+ /// Gets total physical memory.
+ /// </summary>
+ [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes")]
+ private static ulong? GetTotalPhysicalMemory()
+ {
+ try
+ {
+ if (Os.IsWindows)
+ {
+ return NativeMethodsWindows.GlobalMemoryStatusExTotalPhys();
+ }
+
+ const string memInfo = "/proc/meminfo";
+
var kbytes = File.ReadAllLines(memInfo).Select(x => Regex.Match(x, @"MemTotal:\s+([0-9]+) kB"))
.Where(x => x.Success)
.Select(x => x.Groups[1].Value).FirstOrDefault();
@@ -52,10 +91,15 @@
return ulong.Parse(kbytes) * 1024;
}
}
+ catch (Exception e)
+ {
+ Console.Error.WriteLine("Failed to determine physical memory size: " + e);
+ }
- return defaultValue;
+ return null;
}
+
/// <summary>
/// Native methods.
/// </summary>
@@ -107,4 +151,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln.DotSettings b/modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln.DotSettings
index 3bbc7bd..59bf2e5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln.DotSettings
+++ b/modules/platforms/dotnet/Apache.Ignite.DotNetCore.sln.DotSettings
@@ -8,6 +8,7 @@
<s:String x:Key="/Default/CodeInspection/Highlighting/InspectionSeverities/=ConvertClosureToMethodGroup/@EntryIndexedValue">DO_NOT_SHOW</s:String>
<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002EXml_002ECodeStyle_002EFormatSettingsUpgrade_002EXmlMoveToCommonFormatterSettingsUpgrade/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/Environment/UnitTesting/ShadowCopy/@EntryValue">False</s:Boolean>
+ <s:Boolean x:Key="/Default/UserDictionary/Words/=cgroup/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=failover/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Multithreaded/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Reentrancy/@EntryIndexedValue">True</s:Boolean>
