| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| #include "kudu/util/mem_tracker.h" |
| |
| #include <atomic> |
| #include <functional> |
| #include <memory> |
| #include <ostream> |
| #include <string> |
| #include <thread> |
| #include <type_traits> |
| #include <unordered_map> |
| #include <utility> |
| #include <vector> |
| |
| #include <glog/logging.h> |
| #include <gtest/gtest.h> |
| |
| #include "kudu/gutil/strings/substitute.h" |
| #include "kudu/util/jsonwriter.h" |
| #include "kudu/util/mem_tracker.pb.h" |
| #include "kudu/util/monotime.h" |
| #include "kudu/util/scoped_cleanup.h" |
| #include "kudu/util/test_util.h" |
| |
| namespace kudu { |
| |
| using std::equal_to; |
| using std::hash; |
| using std::pair; |
| using std::shared_ptr; |
| using std::string; |
| using std::unordered_map; |
| using std::vector; |
| using strings::Substitute; |
| |
| TEST(MemTrackerTest, SingleTrackerNoLimit) { |
| shared_ptr<MemTracker> t = MemTracker::CreateTracker(-1, "t"); |
| EXPECT_FALSE(t->has_limit()); |
| t->Consume(10); |
| EXPECT_EQ(t->consumption(), 10); |
| t->Consume(10); |
| EXPECT_EQ(t->consumption(), 20); |
| t->Release(15); |
| EXPECT_EQ(t->consumption(), 5); |
| EXPECT_FALSE(t->LimitExceeded()); |
| t->Release(5); |
| EXPECT_EQ(t->consumption(), 0); |
| } |
| |
| TEST(MemTrackerTest, SingleTrackerWithLimit) { |
| shared_ptr<MemTracker> t = MemTracker::CreateTracker(11, "t"); |
| EXPECT_TRUE(t->has_limit()); |
| t->Consume(10); |
| EXPECT_EQ(t->consumption(), 10); |
| EXPECT_FALSE(t->LimitExceeded()); |
| t->Consume(10); |
| EXPECT_EQ(t->consumption(), 20); |
| EXPECT_TRUE(t->LimitExceeded()); |
| t->Release(15); |
| EXPECT_EQ(t->consumption(), 5); |
| EXPECT_FALSE(t->LimitExceeded()); |
| t->Release(5); |
| } |
| |
| TEST(MemTrackerTest, TrackerHierarchy) { |
| shared_ptr<MemTracker> p = MemTracker::CreateTracker(100, "p"); |
| shared_ptr<MemTracker> c1 = MemTracker::CreateTracker(80, "c1", p); |
| shared_ptr<MemTracker> c2 = MemTracker::CreateTracker(50, "c2", p); |
| |
| // everything below limits |
| c1->Consume(60); |
| EXPECT_EQ(c1->consumption(), 60); |
| EXPECT_FALSE(c1->LimitExceeded()); |
| EXPECT_FALSE(c1->AnyLimitExceeded()); |
| EXPECT_EQ(c2->consumption(), 0); |
| EXPECT_FALSE(c2->LimitExceeded()); |
| EXPECT_FALSE(c2->AnyLimitExceeded()); |
| EXPECT_EQ(p->consumption(), 60); |
| EXPECT_FALSE(p->LimitExceeded()); |
| EXPECT_FALSE(p->AnyLimitExceeded()); |
| |
| // p goes over limit |
| c2->Consume(50); |
| EXPECT_EQ(c1->consumption(), 60); |
| EXPECT_FALSE(c1->LimitExceeded()); |
| EXPECT_TRUE(c1->AnyLimitExceeded()); |
| EXPECT_EQ(c2->consumption(), 50); |
| EXPECT_FALSE(c2->LimitExceeded()); |
| EXPECT_TRUE(c2->AnyLimitExceeded()); |
| EXPECT_EQ(p->consumption(), 110); |
| EXPECT_TRUE(p->LimitExceeded()); |
| |
| // c2 goes over limit, p drops below limit |
| c1->Release(20); |
| c2->Consume(10); |
| EXPECT_EQ(c1->consumption(), 40); |
| EXPECT_FALSE(c1->LimitExceeded()); |
| EXPECT_FALSE(c1->AnyLimitExceeded()); |
| EXPECT_EQ(c2->consumption(), 60); |
| EXPECT_TRUE(c2->LimitExceeded()); |
| EXPECT_TRUE(c2->AnyLimitExceeded()); |
| EXPECT_EQ(p->consumption(), 100); |
| EXPECT_FALSE(p->LimitExceeded()); |
| c1->Release(40); |
| c2->Release(60); |
| } |
| |
| TEST(MemTrackerTest, TestToPb) { |
| shared_ptr<MemTracker> p0 = MemTracker::CreateTracker(100, "p0"); |
| shared_ptr<MemTracker> c00 = MemTracker::CreateTracker(80, "c00", p0); |
| shared_ptr<MemTracker> gc000 = MemTracker::CreateTracker(40, "c000", c00); |
| shared_ptr<MemTracker> c01 = MemTracker::CreateTracker(40, "c01", p0); |
| shared_ptr<MemTracker> p1 = MemTracker::CreateTracker(120, "p1"); |
| shared_ptr<MemTracker> c10 = MemTracker::CreateTracker(70, "c10", p1); |
| |
| MemTrackerPB pb; |
| MemTracker::TrackersToPb(&pb); |
| const auto expected = R"({ |
| "id": "root", |
| "limit": -1, |
| "current_consumption": 0, |
| "peak_consumption": 110, |
| "child_trackers": [ |
| { |
| "id": "p0", |
| "parent_id": "root", |
| "limit": 100, |
| "current_consumption": 0, |
| "peak_consumption": 0, |
| "child_trackers": [ |
| { |
| "id": "c00", |
| "parent_id": "p0", |
| "limit": 80, |
| "current_consumption": 0, |
| "peak_consumption": 0, |
| "child_trackers": [ |
| { |
| "id": "c000", |
| "parent_id": "c00", |
| "limit": 40, |
| "current_consumption": 0, |
| "peak_consumption": 0 |
| } |
| ] |
| }, |
| { |
| "id": "c01", |
| "parent_id": "p0", |
| "limit": 40, |
| "current_consumption": 0, |
| "peak_consumption": 0 |
| } |
| ] |
| }, |
| { |
| "id": "p1", |
| "parent_id": "root", |
| "limit": 120, |
| "current_consumption": 0, |
| "peak_consumption": 0, |
| "child_trackers": [ |
| { |
| "id": "c10", |
| "parent_id": "p1", |
| "limit": 70, |
| "current_consumption": 0, |
| "peak_consumption": 0 |
| } |
| ] |
| } |
| ] |
| })"; |
| ASSERT_EQ(expected, JsonWriter::ToJson(pb, JsonWriter::Mode::PRETTY)); |
| |
| // Tickle the 'consumption' and 'peak_consumption' values a bit. |
| gc000->Consume(40); |
| gc000->Release(20); |
| |
| // Need to release all memory before the root MemTracker is destroyed, even |
| // if the test fails. |
| SCOPED_CLEANUP({ |
| gc000->Release(20); |
| }); |
| |
| pb.Clear(); |
| MemTracker::TrackersToPb(&pb); |
| const auto expected_post_tickling = R"({ |
| "id": "root", |
| "limit": -1, |
| "current_consumption": 20, |
| "peak_consumption": 110, |
| "child_trackers": [ |
| { |
| "id": "p0", |
| "parent_id": "root", |
| "limit": 100, |
| "current_consumption": 20, |
| "peak_consumption": 40, |
| "child_trackers": [ |
| { |
| "id": "c00", |
| "parent_id": "p0", |
| "limit": 80, |
| "current_consumption": 20, |
| "peak_consumption": 40, |
| "child_trackers": [ |
| { |
| "id": "c000", |
| "parent_id": "c00", |
| "limit": 40, |
| "current_consumption": 20, |
| "peak_consumption": 40 |
| } |
| ] |
| }, |
| { |
| "id": "c01", |
| "parent_id": "p0", |
| "limit": 40, |
| "current_consumption": 0, |
| "peak_consumption": 0 |
| } |
| ] |
| }, |
| { |
| "id": "p1", |
| "parent_id": "root", |
| "limit": 120, |
| "current_consumption": 0, |
| "peak_consumption": 0, |
| "child_trackers": [ |
| { |
| "id": "c10", |
| "parent_id": "p1", |
| "limit": 70, |
| "current_consumption": 0, |
| "peak_consumption": 0 |
| } |
| ] |
| } |
| ] |
| })"; |
| ASSERT_EQ(expected_post_tickling, |
| JsonWriter::ToJson(pb, JsonWriter::Mode::PRETTY)); |
| } |
| |
| class GcFunctionHelper { |
| public: |
| static const int kNumReleaseBytes = 1; |
| |
| explicit GcFunctionHelper(MemTracker* tracker) : tracker_(tracker) { } |
| |
| void GcFunc() { tracker_->Release(kNumReleaseBytes); } |
| |
| private: |
| MemTracker* tracker_; |
| }; |
| |
| TEST(MemTrackerTest, STLContainerAllocator) { |
| shared_ptr<MemTracker> t = MemTracker::CreateTracker(-1, "t"); |
| MemTrackerAllocator<int> vec_alloc(t); |
| MemTrackerAllocator<pair<const int, int>> map_alloc(t); |
| |
| // Simple test: use the allocator in a vector. |
| { |
| vector<int, MemTrackerAllocator<int> > v(vec_alloc); |
| ASSERT_EQ(0, t->consumption()); |
| v.reserve(5); |
| ASSERT_EQ(5 * sizeof(int), t->consumption()); |
| v.reserve(10); |
| ASSERT_EQ(10 * sizeof(int), t->consumption()); |
| } |
| ASSERT_EQ(0, t->consumption()); |
| |
| // Complex test: use it in an unordered_map, where it must be rebound in |
| // order to allocate the map's buckets. |
| { |
| unordered_map<int, int, hash<int>, equal_to<int>, MemTrackerAllocator<pair<const int, int>>> um( |
| 10, |
| hash<int>(), |
| equal_to<int>(), |
| map_alloc); |
| |
| // Don't care about the value (it depends on map internals). |
| ASSERT_GT(t->consumption(), 0); |
| } |
| ASSERT_EQ(0, t->consumption()); |
| } |
| |
| TEST(MemTrackerTest, FindFunctionsTakeOwnership) { |
| // In each test, ToString() would crash if the MemTracker is destroyed when |
| // 'm' goes out of scope. |
| |
| shared_ptr<MemTracker> ref; |
| { |
| shared_ptr<MemTracker> m = MemTracker::CreateTracker(-1, "test"); |
| ASSERT_TRUE(MemTracker::FindTracker(m->id(), &ref)); |
| } |
| LOG(INFO) << ref->ToString(); |
| ref.reset(); |
| |
| { |
| shared_ptr<MemTracker> m = MemTracker::CreateTracker(-1, "test"); |
| ref = MemTracker::FindOrCreateGlobalTracker(-1, m->id()); |
| } |
| LOG(INFO) << ref->ToString(); |
| ref.reset(); |
| |
| vector<shared_ptr<MemTracker> > refs; |
| { |
| shared_ptr<MemTracker> m = MemTracker::CreateTracker(-1, "test"); |
| MemTracker::ListTrackers(&refs); |
| } |
| for (const shared_ptr<MemTracker>& r : refs) { |
| LOG(INFO) << r->ToString(); |
| } |
| refs.clear(); |
| } |
| |
| TEST(MemTrackerTest, ScopedTrackedConsumption) { |
| shared_ptr<MemTracker> m = MemTracker::CreateTracker(-1, "test"); |
| ASSERT_EQ(0, m->consumption()); |
| { |
| ScopedTrackedConsumption consumption(m, 1); |
| ASSERT_EQ(1, m->consumption()); |
| |
| consumption.Reset(3); |
| ASSERT_EQ(3, m->consumption()); |
| } |
| ASSERT_EQ(0, m->consumption()); |
| } |
| |
| TEST(MemTrackerTest, CollisionDetection) { |
| shared_ptr<MemTracker> p = MemTracker::CreateTracker(-1, "parent"); |
| shared_ptr<MemTracker> c = MemTracker::CreateTracker(-1, "child", p); |
| vector<shared_ptr<MemTracker>> all; |
| |
| // Three trackers: root, parent, and child. |
| MemTracker::ListTrackers(&all); |
| ASSERT_EQ(3, all.size()); |
| |
| // Now only two because the child has been destroyed. |
| c.reset(); |
| MemTracker::ListTrackers(&all); |
| ASSERT_EQ(2, all.size()); |
| shared_ptr<MemTracker> not_found; |
| ASSERT_FALSE(MemTracker::FindTracker("child", ¬_found, p)); |
| |
| // Let's duplicate the parent. It's not recommended, but it's allowed. |
| shared_ptr<MemTracker> p2 = MemTracker::CreateTracker(-1, "parent"); |
| ASSERT_EQ(p->ToString(), p2->ToString()); |
| |
| // Only when we do a Find() operation do we crash. |
| #ifndef NDEBUG |
| const string kDeathMsg = "Multiple memtrackers with same id"; |
| EXPECT_DEATH({ |
| shared_ptr<MemTracker> found; |
| MemTracker::FindTracker("parent", &found); |
| }, kDeathMsg); |
| EXPECT_DEATH({ |
| MemTracker::FindOrCreateGlobalTracker(-1, "parent"); |
| }, kDeathMsg); |
| #endif |
| } |
| |
| TEST(MemTrackerTest, TestMultiThreadedRegisterAndDestroy) { |
| std::atomic<bool> done(false); |
| vector<std::thread> threads; |
| for (int i = 0; i < 10; i++) { |
| threads.emplace_back([&done]{ |
| while (!done.load()) { |
| shared_ptr<MemTracker> t = MemTracker::FindOrCreateGlobalTracker( |
| 1000, "foo"); |
| } |
| }); |
| } |
| |
| SleepFor(MonoDelta::FromSeconds(AllowSlowTests() ? 5 : 1)); |
| done.store(true); |
| for (auto& t : threads) { |
| t.join(); |
| } |
| } |
| |
| TEST(MemTrackerTest, TestMultiThreadedCreateFind) { |
| shared_ptr<MemTracker> p = MemTracker::CreateTracker(-1, "p"); |
| shared_ptr<MemTracker> c1 = MemTracker::CreateTracker(-1, "c1", p); |
| std::atomic<bool> done(false); |
| vector<std::thread> threads; |
| threads.emplace_back([&]{ |
| while (!done.load()) { |
| shared_ptr<MemTracker> c1_copy; |
| CHECK(MemTracker::FindTracker(c1->id(), &c1_copy, p)); |
| } |
| }); |
| for (int i = 0; i < 5; i++) { |
| threads.emplace_back([&, i]{ |
| while (!done.load()) { |
| shared_ptr<MemTracker> c2 = |
| MemTracker::CreateTracker(-1, Substitute("ci-$0", i), p); |
| } |
| }); |
| } |
| |
| SleepFor(MonoDelta::FromMilliseconds(500)); |
| done.store(true); |
| for (auto& t : threads) { |
| t.join(); |
| } |
| } |
| |
| } // namespace kudu |