be/src/runtime/mem-tracker-test.cc

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include <functional>
#include <string>

#include "runtime/mem-tracker.h"
#include "testutil/gtest-util.h"
#include "util/metrics.h"

#include "common/names.h"

namespace impala {

TEST(MemTestTest, SingleTrackerNoLimit) {
  MemTracker t;
  EXPECT_FALSE(t.has_limit());
  t.Consume(10);
  EXPECT_EQ(t.consumption(), 10);
  t.Consume(10);
  EXPECT_EQ(t.consumption(), 20);
  t.Release(15);
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  // Clean up.
  t.Release(5);
}

TEST(MemTestTest, SingleTrackerWithLimit) {
  MemTracker t(11);
  EXPECT_TRUE(t.has_limit());
  t.Consume(10);
  EXPECT_EQ(t.consumption(), 10);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  t.Consume(10);
  EXPECT_EQ(t.consumption(), 20);
  EXPECT_TRUE(t.LimitExceeded(MemLimit::HARD));
  t.Release(15);
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  // Clean up.
  t.Release(5);
}

/// Exercise individual functions that take MemLimit::SOFT option.
TEST(MemTestTest, SoftLimit) {
  MemTracker t(100);
  MemTracker child(-1, "", &t);

  // Exercise functions that return limits.
  EXPECT_EQ(90, t.soft_limit());
  EXPECT_EQ(90, t.GetLimit(MemLimit::SOFT));
  EXPECT_EQ(100, t.GetLimit(MemLimit::HARD));
  EXPECT_EQ(-1, child.soft_limit());
  EXPECT_EQ(90, t.GetLowestLimit(MemLimit::SOFT));
  EXPECT_EQ(90, child.GetLowestLimit(MemLimit::SOFT));
  EXPECT_EQ(100, t.GetLowestLimit(MemLimit::HARD));

  // Test SpareCapacity()
  EXPECT_EQ(100, t.SpareCapacity(MemLimit::HARD));
  EXPECT_EQ(90, t.SpareCapacity(MemLimit::SOFT));
  EXPECT_EQ(100, child.SpareCapacity(MemLimit::HARD));
  EXPECT_EQ(90, child.SpareCapacity(MemLimit::SOFT));

  // Test TryConsume() within soft limit.
  EXPECT_TRUE(t.TryConsume(90, MemLimit::SOFT));
  EXPECT_FALSE(t.LimitExceeded(MemLimit::SOFT));
  EXPECT_FALSE(child.AnyLimitExceeded(MemLimit::SOFT));

  // Test TryConsume() going over soft limit.
  EXPECT_FALSE(t.TryConsume(1, MemLimit::SOFT));
  EXPECT_FALSE(child.TryConsume(1, MemLimit::SOFT));
  EXPECT_TRUE(t.TryConsume(1, MemLimit::HARD));
  EXPECT_TRUE(t.LimitExceeded(MemLimit::SOFT));
  EXPECT_FALSE(child.LimitExceeded(MemLimit::SOFT));
  EXPECT_TRUE(t.AnyLimitExceeded(MemLimit::SOFT));
  EXPECT_TRUE(child.AnyLimitExceeded(MemLimit::SOFT));
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(child.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(9, child.SpareCapacity(MemLimit::HARD));
  EXPECT_EQ(-1, child.SpareCapacity(MemLimit::SOFT));

  // Test Consume() going over hard limit.
  child.Consume(10);
  EXPECT_TRUE(t.LimitExceeded(MemLimit::SOFT));
  EXPECT_TRUE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(child.LimitExceeded(MemLimit::SOFT));
  EXPECT_FALSE(child.LimitExceeded(MemLimit::HARD));
  EXPECT_TRUE(t.AnyLimitExceeded(MemLimit::SOFT));
  EXPECT_TRUE(t.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_TRUE(child.AnyLimitExceeded(MemLimit::SOFT));
  EXPECT_TRUE(child.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(-1, child.SpareCapacity(MemLimit::HARD));
  EXPECT_EQ(-11, child.SpareCapacity(MemLimit::SOFT));

  t.Release(91);
  child.Release(10);
}

TEST(MemTestTest, ConsumptionMetric) {
  TMetricDef md;
  md.__set_key("test");
  md.__set_units(TUnit::BYTES);
  md.__set_kind(TMetricKind::GAUGE);
  IntGauge metric(md, 0);
  EXPECT_EQ(metric.GetValue(), 0);

  TMetricDef neg_md;
  neg_md.__set_key("neg_test");
  neg_md.__set_units(TUnit::BYTES);
  neg_md.__set_kind(TMetricKind::GAUGE);
  NegatedGauge neg_metric(neg_md, &metric);

  MemTracker t(&metric, 100, "");
  MemTracker neg_t(&neg_metric, 100, "");
  EXPECT_TRUE(t.has_limit());
  EXPECT_EQ(t.consumption(), 0);
  EXPECT_EQ(neg_t.consumption(), 0);

  // Consume()/Release() arguments have no effect
  t.Consume(150);
  EXPECT_EQ(t.consumption(), 0);
  EXPECT_EQ(t.peak_consumption(), 0);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), 0);
  t.Release(5);
  EXPECT_EQ(t.consumption(), 0);
  EXPECT_EQ(t.peak_consumption(), 0);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), 0);

  metric.Increment(10);
  // consumption_ is only updated with consumption_metric_ after calls to
  // Consume()/Release() with a non-zero value
  t.Consume(1);
  neg_t.Consume(1);
  EXPECT_EQ(t.consumption(), 10);
  EXPECT_EQ(t.peak_consumption(), 10);
  EXPECT_EQ(neg_t.consumption(), -10);
  metric.Increment(-5);
  t.Release(1);
  neg_t.Consume(1);
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_EQ(t.peak_consumption(), 10);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), -5);
  metric.Increment(150);
  t.Consume(1);
  neg_t.Consume(1);
  EXPECT_EQ(t.consumption(), 155);
  EXPECT_EQ(t.peak_consumption(), 155);
  EXPECT_TRUE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), -155);
  metric.Increment(-150);
  t.Release(1);
  neg_t.Consume(1);
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_EQ(t.peak_consumption(), 155);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), -5);
  // consumption_ is not updated when Consume()/Release() is called with a zero value
  metric.Increment(10);
  t.Consume(0);
  neg_t.Release(0);
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_EQ(t.peak_consumption(), 155);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), -5);
  // consumption_ is not updated when TryConsume() is called with a zero value
  EXPECT_TRUE(t.TryConsume(0));
  EXPECT_TRUE(neg_t.TryConsume(0));
  EXPECT_EQ(t.consumption(), 5);
  EXPECT_EQ(t.peak_consumption(), 155);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(neg_t.consumption(), -5);
  // Clean up.
  metric.Increment(-15);
  t.Release(1);
  neg_t.Release(1);
}

TEST(MemTestTest, TrackerHierarchy) {
  MemTracker p(100);
  MemTracker c1(80, "", &p);
  MemTracker c2(50, "", &p);

  // everything below limits
  c1.Consume(60);
  EXPECT_EQ(c1.consumption(), 60);
  EXPECT_FALSE(c1.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(c1.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(c2.consumption(), 0);
  EXPECT_FALSE(c2.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(c2.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(p.consumption(), 60);
  EXPECT_FALSE(p.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(p.AnyLimitExceeded(MemLimit::HARD));

  // p goes over limit
  c2.Consume(50);
  EXPECT_EQ(c1.consumption(), 60);
  EXPECT_FALSE(c1.LimitExceeded(MemLimit::HARD));
  EXPECT_TRUE(c1.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(c2.consumption(), 50);
  EXPECT_FALSE(c2.LimitExceeded(MemLimit::HARD));
  EXPECT_TRUE(c2.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(p.consumption(), 110);
  EXPECT_TRUE(p.LimitExceeded(MemLimit::HARD));

  // c2 goes over limit, p drops below limit
  c1.Release(20);
  c2.Consume(10);
  EXPECT_EQ(c1.consumption(), 40);
  EXPECT_FALSE(c1.LimitExceeded(MemLimit::HARD));
  EXPECT_FALSE(c1.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(c2.consumption(), 60);
  EXPECT_TRUE(c2.LimitExceeded(MemLimit::HARD));
  EXPECT_TRUE(c2.AnyLimitExceeded(MemLimit::HARD));
  EXPECT_EQ(p.consumption(), 100);
  EXPECT_FALSE(p.LimitExceeded(MemLimit::HARD));

  // Clean up.
  c1.Release(40);
  c2.Release(60);
}

// Test that we can transfer between MemTrackers without temporary double-counting
// in ancestors
TEST(MemTestTest, TransferTo) {
  MemTracker root(100);
  MemTracker parent(-1, "", &root);
  MemTracker uncle(-1, "", &root);
  MemTracker child1(-1, "", &parent);
  MemTracker child2(-1, "", &parent);

  child1.Consume(100);
  // To self.
  child1.TransferTo(&child1, 100);
  EXPECT_EQ(child1.consumption(), 100);
  EXPECT_EQ(child1.peak_consumption(), 100);
  EXPECT_EQ(parent.consumption(), 100);
  EXPECT_EQ(parent.peak_consumption(), 100);

  // Child to parent.
  child1.TransferTo(&parent, 100);
  EXPECT_EQ(child1.consumption(), 0);
  EXPECT_EQ(child1.peak_consumption(), 100);
  EXPECT_EQ(parent.consumption(), 100);
  EXPECT_EQ(parent.peak_consumption(), 100);

  // Parent to child
  parent.TransferTo(&child1, 100);
  EXPECT_EQ(child1.consumption(), 100);
  EXPECT_EQ(child1.peak_consumption(), 100);
  EXPECT_EQ(parent.consumption(), 100);
  EXPECT_EQ(parent.peak_consumption(), 100);

  // Child to child.
  child1.TransferTo(&child2, 50);
  EXPECT_EQ(child1.consumption(), 50);
  EXPECT_EQ(child2.consumption(), 50);
  EXPECT_EQ(child2.peak_consumption(), 50);
  EXPECT_EQ(parent.consumption(), 100);
  EXPECT_EQ(parent.peak_consumption(), 100);

  // Child to uncle.
  child1.TransferTo(&uncle, 50);
  EXPECT_EQ(child1.consumption(), 0);
  EXPECT_EQ(uncle.consumption(), 50);
  EXPECT_EQ(uncle.peak_consumption(), 50);
  EXPECT_EQ(parent.consumption(), 50);
  EXPECT_EQ(parent.peak_consumption(), 100);
  EXPECT_EQ(root.consumption(), 100);
  EXPECT_EQ(root.peak_consumption(), 100);

  // Child to root
  child2.TransferTo(&root, 50);
  EXPECT_EQ(child2.consumption(), 0);
  EXPECT_EQ(parent.consumption(), 0);
  EXPECT_EQ(parent.peak_consumption(), 100);
  EXPECT_EQ(root.consumption(), 100);
  EXPECT_EQ(root.peak_consumption(), 100);

  uncle.Release(50);
  root.Release(50);
}

class GcFunctionHelper {
 public:
  static const int NUM_RELEASE_BYTES = 1;

  GcFunctionHelper(MemTracker* tracker) : tracker_(tracker) { }

  void GcFunc() { tracker_->Release(NUM_RELEASE_BYTES); }

 private:
  MemTracker* tracker_;
};

TEST(MemTestTest, GcFunctions) {
  // By default, a MemTracker will try to free extra memory when performing GC. To make
  // things predictable, turn this off.
  MemTracker t(/* byte_limit */ 10, /* extra_bytes_to_gc */ 0);
  ASSERT_TRUE(t.has_limit());

  t.Consume(9);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));

  // Test TryConsume()
  EXPECT_FALSE(t.TryConsume(2));
  EXPECT_EQ(t.consumption(), 9);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));

  // Attach GcFunction that releases 1 byte
  GcFunctionHelper gc_func_helper(&t);
  t.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_func_helper));
  EXPECT_TRUE(t.TryConsume(2));
  EXPECT_EQ(t.consumption(), 10);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));

  // GcFunction will be called even though TryConsume() fails
  EXPECT_FALSE(t.TryConsume(2));
  EXPECT_EQ(t.consumption(), 9);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));

  // GcFunction won't be called
  EXPECT_TRUE(t.TryConsume(1));
  EXPECT_EQ(t.consumption(), 10);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));

  // Test LimitExceeded(MemLimit::HARD)
  t.Consume(1);
  EXPECT_EQ(t.consumption(), 11);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(t.consumption(), 10);

  // Add more GcFunctions, test that we only call them until the limit is no longer
  // exceeded
  GcFunctionHelper gc_func_helper2(&t);
  t.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_func_helper2));
  GcFunctionHelper gc_func_helper3(&t);
  t.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_func_helper3));
  t.Consume(1);
  EXPECT_EQ(t.consumption(), 11);
  EXPECT_FALSE(t.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(t.consumption(), 10);

  // Clean up.
  t.Release(10);

  // Create a new MemTracker to test that GC frees extra memory properly.
  MemTracker t_extra(/* byte_limit */ 10, /* extra_bytes_to_gc */ 1);
  ASSERT_TRUE(t_extra.has_limit());

  // Attach three GcFunctions that each release 1 byte
  GcFunctionHelper gc_extra_func_helper(&t_extra);
  t_extra.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_extra_func_helper));
  GcFunctionHelper gc_extra_func_helper2(&t_extra);
  t_extra.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_extra_func_helper2));
  GcFunctionHelper gc_extra_func_helper3(&t_extra);
  t_extra.AddGcFunction(std::bind(&GcFunctionHelper::GcFunc, &gc_extra_func_helper3));

  // Freeing extra bytes for GC doesn't change when we call GC. We can go up to the limit
  // and it will not call GC.
  t_extra.Consume(10);
  EXPECT_FALSE(t_extra.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(t_extra.consumption(), 10);

  // Go one byte over the limit. Test that we only call the GC functions until we
  // free the necessary amount plus the extra byte. In this case, it will call the
  // first two GC functions, but not the third.
  t_extra.Consume(1);
  EXPECT_EQ(t_extra.consumption(), 11);
  EXPECT_FALSE(t_extra.LimitExceeded(MemLimit::HARD));
  EXPECT_EQ(t_extra.consumption(), 9);

  // Clean up
  t_extra.Release(9);
}

// Test that we can compute topN queries from a hierarchy of mem trackers. These
// queries are represented by 100 query mem trackers.
TEST(MemTestTest, TopN) {
  MemTracker root;
  root.Consume(10);

  static const int NUM_QUERY_MEM_TRACKERS = 100;
  // Populate these many query mem trackers with some memory consumptions.
  std::vector<MemTracker*> trackers;
  for (int i = 0; i < NUM_QUERY_MEM_TRACKERS; i++) {
    MemTracker* tracker = new MemTracker(-1, "", &root);
    tracker->query_id_.hi = 0;
    tracker->query_id_.lo = i;
    tracker->Consume(int64_t(i + 1));
    tracker->is_query_mem_tracker_ = true;
    trackers.push_back(tracker);
  }
  // Ready to compute top 5 queries which should be the last 5 of those
  // populated above. The result is to be saved in pool_stats.heavy_memory_queries.
  TPoolStats pool_stats;
  root.UpdatePoolStatsForQueries(5, pool_stats);
  // Validate the top entries
  for (int i = 0; i < 5; i++) {
    EXPECT_EQ(pool_stats.heavy_memory_queries[i].queryId.hi, 0);
    EXPECT_EQ(
        pool_stats.heavy_memory_queries[i].queryId.lo, NUM_QUERY_MEM_TRACKERS - i - 1);
    EXPECT_EQ(
        pool_stats.heavy_memory_queries[i].memory_consumed, NUM_QUERY_MEM_TRACKERS - i);
  }
  // Delete the allocated query mem trackers.
  for (int i = 0; i < NUM_QUERY_MEM_TRACKERS; i++) {
    trackers[i]->Release(int64_t(i + 1));
    delete trackers[i];
  }
  root.Release(10);
}
}