src/kudu/master/catalog_manager-test.cc - kudu - Git at Google

 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
 // to you under the Apache License, Version 2.0 (the
 // "License"); you may not use this file except in compliance
 // with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing,
 // software distributed under the License is distributed on an
 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.

 #include "kudu/master/catalog_manager.h"

 #include <numeric>
 #include <ostream>
 #include <string>
 #include <vector>

 #include <glog/logging.h>
 #include <gflags/gflags_declare.h>
 #include <gtest/gtest.h>

 #include "kudu/common/common.pb.h"
 #include "kudu/common/row_operations.pb.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/ts_descriptor.h"
 #include "kudu/util/cow_object.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"

 DECLARE_bool(require_new_spec_for_custom_hash_schema_range_bound);

 using std::iota;
 using std::string;
 using std::vector;
 using strings::Substitute;

 namespace kudu {
 namespace master {

 // Test of the tablet assignment algo for splits done at table creation time.
 // This tests that when we define a split, the tablet lands on the expected
 // side of the split, i.e. it's a closed interval on the start key and an open
 // interval on the end key (non-inclusive).
 TEST(TableInfoTest, TestAssignmentRanges) {
   const string table_id = CURRENT_TEST_NAME();
   scoped_refptr<TableInfo> table(new TableInfo(table_id));
   vector<scoped_refptr<TabletInfo>> tablets;

   // Define & create the splits.
   const int kNumSplits = 3;
   string split_keys[kNumSplits] = { "a", "b", "c" };  // The keys we split on.
   for (int i = 0; i <= kNumSplits; i++) {
     const string& start_key = (i == 0) ? "" : split_keys[i - 1];
     const string& end_key = (i == kNumSplits) ? "" : split_keys[i];
     string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

     scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
     {
       TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
       PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
       partition->set_partition_key_start(start_key);
       partition->set_partition_key_end(end_key);
       meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
       meta_lock.Commit();
     }

     TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
     table->AddRemoveTablets({ tablet }, {});
     tablets.push_back(tablet);
   }

   // Ensure they give us what we are expecting.
   for (int i = 0; i <= kNumSplits; i++) {
     // Calculate the tablet id and start key.
     const string& start_key = (i == 0) ? "" : split_keys[i - 1];
     const string& end_key = (i == kNumSplits) ? "" : split_keys[i];
     string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

     // Query using the start key.
     GetTableLocationsRequestPB req;
     req.set_max_returned_locations(1);
     req.mutable_table()->mutable_table_name()->assign(table_id);
     req.mutable_partition_key_start()->assign(start_key);
     vector<scoped_refptr<TabletInfo>> tablets_in_range;
     ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));

     // Only one tablet should own this key.
     ASSERT_EQ(1, tablets_in_range.size());
     // The tablet with range start key matching 'start_key' should be the owner.
     ASSERT_EQ(tablet_id, (*tablets_in_range.begin())->id());
     LOG(INFO) << "Key " << start_key << " found in tablet " << tablet_id;
   }
 }

 TEST(TableInfoTest, GetTableLocationsLegacyCustomHashSchemas) {
   const string table_id = CURRENT_TEST_NAME();
   scoped_refptr<TableInfo> table(new TableInfo(table_id));

   {
     TableMetadataLock meta_lock(table.get(), LockMode::WRITE);
     auto* ps = meta_lock.mutable_data()->pb.mutable_partition_schema();
     // It's not really necessary to fill everything in the scope of this test.
     auto* range = ps->add_custom_hash_schema_ranges();
     range->mutable_range_bounds()->set_rows("a");
     auto* hash_dimension = range->add_hash_schema();
     hash_dimension->add_columns()->set_name("b");
     hash_dimension->set_num_buckets(2);
     meta_lock.Commit();
   }

   const string start_key = "a";
   const string end_key = "";
   string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

   scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
   {
     TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
     PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
     partition->set_partition_key_start(start_key);
     partition->set_partition_key_end(end_key);
     meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
     meta_lock.Commit();
   }

   TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
   table->AddRemoveTablets({ tablet }, {});

   // Query by specifying the start of the partition via the partition_key_start
   // field: it should pass even if the table has a range with custom hash schema
   // since as of now all the range partitions must have the number of hash
   // dimenstions fixed across all the ranges in a table.
   {
     GetTableLocationsRequestPB req;
     req.set_max_returned_locations(1);
     req.mutable_table()->mutable_table_name()->assign(table_id);
     req.mutable_partition_key_start()->assign("a");
     vector<scoped_refptr<TabletInfo>> tablets_in_range;
     ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));
     ASSERT_EQ(1, tablets_in_range.size());
   }

   // Query by specifying the start of the partition via the partition_key_start
   // field: it should fail since the table has a range with custom hash schema
   // and --require_new_spec_for_custom_hash_schema_range_bound=true.
   {
     FLAGS_require_new_spec_for_custom_hash_schema_range_bound = true;
     GetTableLocationsRequestPB req;
     req.set_max_returned_locations(1);
     req.mutable_table()->mutable_table_name()->assign(table_id);
     req.mutable_partition_key_start()->assign("a");
     vector<scoped_refptr<TabletInfo>> tablets_in_range;
     auto s = table->GetTabletsInRange(&req, &tablets_in_range);
     ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
     ASSERT_STR_CONTAINS(s.ToString(),
                         "for a table with custom per-range hash schemas "
                         "the range must be specified using partition_key_range "
                         "field, not partition_key_{start,end} fields");
   }

   // Query by specifying the start of the partition via the partition_key_range
   // field: it should succeed.
   {
     GetTableLocationsRequestPB req;
     req.set_max_returned_locations(1);
     req.mutable_table()->mutable_table_name()->assign(table_id);
     req.mutable_key_start()->set_hash_key(string("\0\0\0\0", 4));
     req.mutable_key_start()->set_range_key("a");
     vector<scoped_refptr<TabletInfo>> tablets_in_range;
     ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));
     ASSERT_EQ(1, tablets_in_range.size());
   }
 }

 TEST(TestTSDescriptor, TestReplicaCreationsDecay) {
   TSDescriptor ts("test");
   ASSERT_EQ(0, ts.RecentReplicaCreations());
   ts.IncrementRecentReplicaCreations();

   // The load should start at close to 1.0.
   double val_a = ts.RecentReplicaCreations();
   ASSERT_NEAR(1.0, val_a, 0.05);

   // After 10ms it should have dropped a bit, but still be close to 1.0.
   SleepFor(MonoDelta::FromMilliseconds(10));
   double val_b = ts.RecentReplicaCreations();
   ASSERT_LT(val_b, val_a);
   ASSERT_NEAR(0.99, val_a, 0.05);

   if (AllowSlowTests()) {
     // After 10 seconds, we should have dropped to 0.5^(10/60) = 0.891
     SleepFor(MonoDelta::FromSeconds(10));
     ASSERT_NEAR(0.891, ts.RecentReplicaCreations(), 0.05);
   }
 }
 TEST(TableInfoTest, MaxReturnedLocationsNotSpecified) {
   const string table_id = CURRENT_TEST_NAME();
   scoped_refptr<TableInfo> table(new TableInfo(table_id));
   vector<scoped_refptr<TabletInfo>> tablets;

   vector<string> ranges(128);
   std::iota(ranges.begin(), ranges.end(), '\0');
   for (auto it = ranges.begin(); it != ranges.end(); ++it) {
     auto next_it = it;
     ++next_it;
     const string& start_key = *it;
     const string& end_key = (next_it == ranges.end()) ? "" : *next_it;

     string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

     scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
     {
       TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
       PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
       partition->set_partition_key_start(start_key);
       if (next_it != ranges.end()) {
         partition->set_partition_key_end(end_key);
       }
       meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
       meta_lock.Commit();
     }

     TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
     table->AddRemoveTablets({ tablet }, {});
     tablets.push_back(tablet);
   }

   // Fetch all the available tablets.
   {
     GetTableLocationsRequestPB req;
     req.clear_max_returned_locations(); // the default is 10 in protobuf
     req.mutable_table()->mutable_table_name()->assign(table_id);
     // Query using the start key of the first tablet's range.
     req.mutable_partition_key_start()->assign("\0");
     vector<scoped_refptr<TabletInfo>> tablets_in_range;
     ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));

     // The response should contain information on every tablet created.
     ASSERT_EQ(ranges.size(), tablets_in_range.size());
   }
 }

 } // namespace master
 } // namespace kudu
	// Licensed to the Apache Software Foundation (ASF) under one
	// or more contributor license agreements. See the NOTICE file
	// distributed with this work for additional information
	// regarding copyright ownership. The ASF licenses this file
	// to you under the Apache License, Version 2.0 (the
	// "License"); you may not use this file except in compliance
	// with the License. You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing,
	// software distributed under the License is distributed on an
	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	// KIND, either express or implied. See the License for the
	// specific language governing permissions and limitations
	// under the License.

	#include "kudu/master/catalog_manager.h"

	#include <numeric>
	#include <ostream>
	#include <string>
	#include <vector>

	#include <glog/logging.h>
	#include <gflags/gflags_declare.h>
	#include <gtest/gtest.h>

	#include "kudu/common/common.pb.h"
	#include "kudu/common/row_operations.pb.h"
	#include "kudu/gutil/ref_counted.h"
	#include "kudu/gutil/strings/substitute.h"
	#include "kudu/master/master.pb.h"
	#include "kudu/master/ts_descriptor.h"
	#include "kudu/util/cow_object.h"
	#include "kudu/util/monotime.h"
	#include "kudu/util/status.h"
	#include "kudu/util/test_macros.h"
	#include "kudu/util/test_util.h"

	DECLARE_bool(require_new_spec_for_custom_hash_schema_range_bound);

	using std::iota;
	using std::string;
	using std::vector;
	using strings::Substitute;

	namespace kudu {
	namespace master {

	// Test of the tablet assignment algo for splits done at table creation time.
	// This tests that when we define a split, the tablet lands on the expected
	// side of the split, i.e. it's a closed interval on the start key and an open
	// interval on the end key (non-inclusive).
	TEST(TableInfoTest, TestAssignmentRanges) {
	const string table_id = CURRENT_TEST_NAME();
	scoped_refptr<TableInfo> table(new TableInfo(table_id));
	vector<scoped_refptr<TabletInfo>> tablets;

	// Define & create the splits.
	const int kNumSplits = 3;
	string split_keys[kNumSplits] = { "a", "b", "c" }; // The keys we split on.
	for (int i = 0; i <= kNumSplits; i++) {
	const string& start_key = (i == 0) ? "" : split_keys[i - 1];
	const string& end_key = (i == kNumSplits) ? "" : split_keys[i];
	string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

	scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
	{
	TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
	PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
	partition->set_partition_key_start(start_key);
	partition->set_partition_key_end(end_key);
	meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
	meta_lock.Commit();
	}

	TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
	table->AddRemoveTablets({ tablet }, {});
	tablets.push_back(tablet);
	}

	// Ensure they give us what we are expecting.
	for (int i = 0; i <= kNumSplits; i++) {
	// Calculate the tablet id and start key.
	const string& start_key = (i == 0) ? "" : split_keys[i - 1];
	const string& end_key = (i == kNumSplits) ? "" : split_keys[i];
	string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

	// Query using the start key.
	GetTableLocationsRequestPB req;
	req.set_max_returned_locations(1);
	req.mutable_table()->mutable_table_name()->assign(table_id);
	req.mutable_partition_key_start()->assign(start_key);
	vector<scoped_refptr<TabletInfo>> tablets_in_range;
	ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));

	// Only one tablet should own this key.
	ASSERT_EQ(1, tablets_in_range.size());
	// The tablet with range start key matching 'start_key' should be the owner.
	ASSERT_EQ(tablet_id, (*tablets_in_range.begin())->id());
	LOG(INFO) << "Key " << start_key << " found in tablet " << tablet_id;
	}
	}

	TEST(TableInfoTest, GetTableLocationsLegacyCustomHashSchemas) {
	const string table_id = CURRENT_TEST_NAME();
	scoped_refptr<TableInfo> table(new TableInfo(table_id));

	{
	TableMetadataLock meta_lock(table.get(), LockMode::WRITE);
	auto* ps = meta_lock.mutable_data()->pb.mutable_partition_schema();
	// It's not really necessary to fill everything in the scope of this test.
	auto* range = ps->add_custom_hash_schema_ranges();
	range->mutable_range_bounds()->set_rows("a");
	auto* hash_dimension = range->add_hash_schema();
	hash_dimension->add_columns()->set_name("b");
	hash_dimension->set_num_buckets(2);
	meta_lock.Commit();
	}

	const string start_key = "a";
	const string end_key = "";
	string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

	scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
	{
	TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
	PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
	partition->set_partition_key_start(start_key);
	partition->set_partition_key_end(end_key);
	meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
	meta_lock.Commit();
	}

	TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
	table->AddRemoveTablets({ tablet }, {});

	// Query by specifying the start of the partition via the partition_key_start
	// field: it should pass even if the table has a range with custom hash schema
	// since as of now all the range partitions must have the number of hash
	// dimenstions fixed across all the ranges in a table.
	{
	GetTableLocationsRequestPB req;
	req.set_max_returned_locations(1);
	req.mutable_table()->mutable_table_name()->assign(table_id);
	req.mutable_partition_key_start()->assign("a");
	vector<scoped_refptr<TabletInfo>> tablets_in_range;
	ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));
	ASSERT_EQ(1, tablets_in_range.size());
	}

	// Query by specifying the start of the partition via the partition_key_start
	// field: it should fail since the table has a range with custom hash schema
	// and --require_new_spec_for_custom_hash_schema_range_bound=true.
	{
	FLAGS_require_new_spec_for_custom_hash_schema_range_bound = true;
	GetTableLocationsRequestPB req;
	req.set_max_returned_locations(1);
	req.mutable_table()->mutable_table_name()->assign(table_id);
	req.mutable_partition_key_start()->assign("a");
	vector<scoped_refptr<TabletInfo>> tablets_in_range;
	auto s = table->GetTabletsInRange(&req, &tablets_in_range);
	ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString();
	ASSERT_STR_CONTAINS(s.ToString(),
	"for a table with custom per-range hash schemas "
	"the range must be specified using partition_key_range "
	"field, not partition_key_{start,end} fields");
	}

	// Query by specifying the start of the partition via the partition_key_range
	// field: it should succeed.
	{
	GetTableLocationsRequestPB req;
	req.set_max_returned_locations(1);
	req.mutable_table()->mutable_table_name()->assign(table_id);
	req.mutable_key_start()->set_hash_key(string("\0\0\0\0", 4));
	req.mutable_key_start()->set_range_key("a");
	vector<scoped_refptr<TabletInfo>> tablets_in_range;
	ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));
	ASSERT_EQ(1, tablets_in_range.size());
	}
	}

	TEST(TestTSDescriptor, TestReplicaCreationsDecay) {
	TSDescriptor ts("test");
	ASSERT_EQ(0, ts.RecentReplicaCreations());
	ts.IncrementRecentReplicaCreations();

	// The load should start at close to 1.0.
	double val_a = ts.RecentReplicaCreations();
	ASSERT_NEAR(1.0, val_a, 0.05);

	// After 10ms it should have dropped a bit, but still be close to 1.0.
	SleepFor(MonoDelta::FromMilliseconds(10));
	double val_b = ts.RecentReplicaCreations();
	ASSERT_LT(val_b, val_a);
	ASSERT_NEAR(0.99, val_a, 0.05);

	if (AllowSlowTests()) {
	// After 10 seconds, we should have dropped to 0.5^(10/60) = 0.891
	SleepFor(MonoDelta::FromSeconds(10));
	ASSERT_NEAR(0.891, ts.RecentReplicaCreations(), 0.05);
	}
	}
	TEST(TableInfoTest, MaxReturnedLocationsNotSpecified) {
	const string table_id = CURRENT_TEST_NAME();
	scoped_refptr<TableInfo> table(new TableInfo(table_id));
	vector<scoped_refptr<TabletInfo>> tablets;

	vector<string> ranges(128);
	std::iota(ranges.begin(), ranges.end(), '\0');
	for (auto it = ranges.begin(); it != ranges.end(); ++it) {
	auto next_it = it;
	++next_it;
	const string& start_key = *it;
	const string& end_key = (next_it == ranges.end()) ? "" : *next_it;

	string tablet_id = Substitute("tablet-$0-$1", start_key, end_key);

	scoped_refptr<TabletInfo> tablet = new TabletInfo(table, tablet_id);
	{
	TabletMetadataLock meta_lock(tablet.get(), LockMode::WRITE);
	PartitionPB* partition = meta_lock.mutable_data()->pb.mutable_partition();
	partition->set_partition_key_start(start_key);
	if (next_it != ranges.end()) {
	partition->set_partition_key_end(end_key);
	}
	meta_lock.mutable_data()->pb.set_state(SysTabletsEntryPB::RUNNING);
	meta_lock.Commit();
	}

	TabletMetadataLock meta_lock(tablet.get(), LockMode::READ);
	table->AddRemoveTablets({ tablet }, {});
	tablets.push_back(tablet);
	}

	// Fetch all the available tablets.
	{
	GetTableLocationsRequestPB req;
	req.clear_max_returned_locations(); // the default is 10 in protobuf
	req.mutable_table()->mutable_table_name()->assign(table_id);
	// Query using the start key of the first tablet's range.
	req.mutable_partition_key_start()->assign("\0");
	vector<scoped_refptr<TabletInfo>> tablets_in_range;
	ASSERT_OK(table->GetTabletsInRange(&req, &tablets_in_range));

	// The response should contain information on every tablet created.
	ASSERT_EQ(ranges.size(), tablets_in_range.size());
	}
	}

	} // namespace master
	} // namespace kudu