src/kudu/tools/testdata/sample-diagnostics-log.txt

I0314 11:54:20.737790 symbols 1521053660737790 {"0x1caba0d":"kudu::StackTraceCollector::TriggerAsync()","0x1caef51":"kudu::StackTraceSnapshot::SnapshotAllStacks()","0xa30202":"kudu::server::DiagnosticsLog::LogStacks()","0xa339c0":"kudu::server::DiagnosticsLog::RunThread()","0x1d46ca1":"kudu::Thread::SuperviseThread()","0x1d8a80c":"ev_run","0x1ba7f69":"kudu::rpc::ReactorThread::RunThread()","0x1d2205a":"kudu::Socket::Accept()","0x1b9e1b3":"kudu::rpc::AcceptorPool::RunThread()","0x1bccad8":"kudu::rpc::LifoServiceQueue::BlockingGet()","0x1bcb161":"kudu::rpc::ServicePool::RunThread()","0x1d50b04":"kudu::ThreadPool::DispatchThread()","0x1caaf69":"kudu::ConditionVariable::WaitUntil()","0x97867c":"kudu::tserver::TabletCopyServiceImpl::EndExpiredSessions()","0xa06f8c":"kudu::server::ServerBase::ExcessLogFileDeleterThread()","0x1bb3594":"kudu::rpc::ResultTracker::RunGCThread()","0x1ce34e4":"kudu::FileCache<>::RunDescriptorExpiry()","0x1cf3d5f":"kudu::KernelStackWatchdog::RunThread()","0x1caafd7":"kudu::ConditionVariable::WaitFor()","0x919e01":"kudu::master::CatalogManagerBgTasks::Run()","0x1cfb5db":"kudu::MaintenanceManager::RunSchedulerThread()","0x1d60c8f":"kudu::AsyncLogger::RunThread()","0x1d83474":"base::SleepForNanoseconds()","0x1d12523":"kudu::MinidumpExceptionHandler::RunUserSignalHandlerThread()"}
I0314 11:54:20.737790 stacks 1521053660737790 {"reason":"periodic","groups":[{"tids":[37480],"stack":["0x3f5ec0f710","0x3f5e8df0d3","0xa52153","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37274],"stack":["0x3f5ec0f710","0x3f5e8e5159","0x1caba0d","0x1caef51","0xa30202","0xa339c0","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37272,37273,37271,37270,37269,37268,37267,37266,37265,37264,37263,37261,37260,37259,37258,37262],"stack":["0x3f5ec0f710","0x3f5e8e8ef3","0x1d87173","0x1d8a80c","0x1ba7f69","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37479],"stack":["0x3f5ec0f710","0x3f5e8eac2d","0x1d2205a","0x1b9e1b3","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37411,37399,37410,37409,37408,37407,37406,37405,37404,37403,37402,37401,37400,37412,37413,37414,37415,37416,37417,37418,37373,37423,37422,37421,37420,37419,37385,37475,37374,37375,37376,37377,37378,37379,37380,37381,37382,37383,37384,37398,37386,37387,37388,37389,37390,37391,37392,37393,37394,37395,37396,37397,37462,37450,37451,37452,37453,37454,37455,37456,37457,37458,37459,37460,37461,37449,37463,37464,37465,37466,37467,37468,37469,37470,37471,37472,37473,37474,37437,37425,37426,37427,37428,37429,37430,37431,37432,37433,37434,37435,37436,37424,37438,37439,37440,37441,37442,37443,37444,37445,37446,37447,37448,37312,37300,37301,37302,37303,37304,37305,37306,37307,37308,37309,37310,37311,37299,37313,37314,37315,37316,37317,37318,37319,37320,37321,37322,37323,37287,37478,37477,37476,37279,37280,37281,37282,37283,37284,37285,37286,37371,37288,37289,37290,37291,37292,37293,37294,37295,37296,37297,37298,37360,37349,37350,37351,37352,37353,37354,37355,37356,37357,37358,37359,37325,37361,37362,37363,37364,37365,37366,37367,37368,37369,37370,37372,37347,37324,37326,37327,37328,37329,37330,37331,37332,37333,37334,37335,37337,37338,37339,37340,37341,37342,37343,37344,37336,37345,37348,37346],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1bccad8","0x1bcb161","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37255],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1d50b04","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37278],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x97867c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37276],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0xa06f8c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37275],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1bb3594","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37257],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1ce34e4","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37253],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1cf3d5f","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37488],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x919e01","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37277],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1cfb5db","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37252,37251,37250],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d60c8f","0x3f620b6470","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37153],"stack":["0x3f5ec0f710","0x3f5ec0ef3d","0x1d83474","0x8c4590","0x3f5e81ed5d","0x8c4141"]},{"tids":[37254],"stack":["0x3f5ec0f710","0x3f5ec0f4b5","0x1d12523","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]}]}
I0314 11:55:06.805381 symbols 1521053706805381 {"0x1abf81f":"kudu::log::Log::AppendThread::DoWork()","0x1d50444":"kudu::ThreadPool::DispatchThread()","0x1d50bbc":"kudu::ThreadPool::DispatchThread()"}
I0314 11:55:06.805381 stacks 1521053706805381 {"reason":"periodic","groups":[{"tids":[37480],"stack":["0x3f5ec0f710","0x3f5e8df0d3","0xa52153","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37274],"stack":["0x3f5ec0f710","0x3f5e8e5159","0x1caba0d","0x1caef51","0xa30202","0xa339c0","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37273,37272,37271,37270,37269,37268,37267,37266,37265,37264,37263,37261,37260,37259,37258,37262],"stack":["0x3f5ec0f710","0x3f5e8e8ef3","0x1d87173","0x1d8a80c","0x1ba7f69","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37479],"stack":["0x3f5ec0f710","0x3f5e8eac2d","0x1d2205a","0x1b9e1b3","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37413,37401,37412,37411,37410,37409,37408,37407,37406,37405,37404,37403,37402,37414,37415,37416,37417,37418,37419,37420,37421,37426,37375,37425,37424,37423,37422,37387,37365,37376,37377,37378,37379,37380,37381,37382,37383,37384,37385,37386,37400,37388,37389,37390,37391,37392,37393,37394,37395,37396,37397,37398,37399,37466,37454,37455,37456,37457,37458,37459,37460,37461,37462,37463,37464,37465,37453,37467,37468,37469,37470,37471,37472,37473,37474,37475,37476,37477,37478,37440,37428,37429,37430,37431,37432,37433,37434,37435,37436,37437,37438,37439,37427,37441,37442,37443,37444,37445,37446,37447,37448,37449,37450,37451,37452,37314,37303,37304,37305,37306,37307,37308,37309,37310,37311,37312,37313,37302,37315,37316,37317,37318,37319,37320,37321,37322,37323,37324,37325,37290,37279,37280,37281,37282,37283,37284,37285,37286,37287,37288,37289,37373,37291,37292,37293,37294,37295,37296,37297,37298,37299,37300,37301,37361,37350,37351,37352,37353,37354,37355,37356,37357,37358,37359,37360,37327,37362,37363,37364,37366,37367,37368,37369,37370,37371,37372,37374,37348,37326,37328,37329,37330,37331,37332,37333,37334,37335,37336,37337,37339,37349,37347,37346,37345,37344,37342,37341,37340,37338,37343],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1bccad8","0x1bcb161","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37255],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1d50b04","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37278],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x97867c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37276],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0xa06f8c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[38724],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1abf81f","0x1d50444","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37275],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1bb3594","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37257],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1ce34e4","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37253],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1cf3d5f","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37488],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x919e01","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37277],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1cfb5db","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[38846,38844,38845],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d50bbc","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37252,37251,37250],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d60c8f","0x3f620b6470","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37153],"stack":["0x3f5ec0f710","0x3f5ec0ef3d","0x1d83474","0x8c4590","0x3f5e81ed5d","0x8c4141"]},{"tids":[37254],"stack":["0x3f5ec0f710","0x3f5ec0f4b5","0x1d12523","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]}]}
I0314 11:55:15.220449 metrics 1521053715220449 [{"type":"tablet","id":"00000000000000000000000000000000","metrics":[{"name":"memrowset_size","value":265},{"name":"on_disk_data_size","value":54961885},{"name":"rows_updated","value":36},{"name":"rows_deleted","value":1},{"name":"scans_started","value":6},{"name":"tablet_active_scanners","value":0},{"name":"bloom_lookups","value":74},{"name":"key_file_lookups","value":74},{"name":"delta_file_lookups","value":72},{"name":"bloom_lookups_per_op","total_count":37,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[37],"total_sum":74},{"name":"key_file_lookups_per_op","total_count":37,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[37],"total_sum":74},{"name":"delta_file_lookups_per_op","total_count":37,"min":0,"mean":1.94595,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[0,2],"counts":[1,36],"total_sum":72},{"name":"flush_dms_running","value":0},{"name":"flush_mrs_running","value":0},{"name":"compact_rs_running","value":0},{"name":"undo_delta_block_gc_running","value":0},{"name":"write_op_duration_client_propagated_consistency","total_count":37,"min":361,"mean":609.514,"percentile_75":784,"percentile_95":920,"percentile_99":996,"percentile_99_9":996,"percentile_99_99":996,"max":998,"values":[361,375,383,385,393,403,413,417,419,421,423,463,473,505,523,547,559,611,631,679,699,751,779,787,835,843,851,875,883,887,923,983,999],"counts":[1,1,1,1,1,2,1,1,2,1,1,1,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":22552},{"name":"delta_minor_compact_rs_running","value":0},{"name":"delta_major_compact_rs_running","value":0},{"name":"undo_delta_block_estimated_retained_bytes","value":0},{"name":"state","value":"RUNNING"},{"name":"op_prepare_queue_length","total_count":37,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[37],"total_sum":0},{"name":"op_prepare_queue_time","total_count":37,"min":36,"mean":74.6757,"percentile_75":107,"percentile_95":125,"percentile_99":133,"percentile_99_9":133,"percentile_99_99":133,"max":133,"values":[36,38,39,40,41,42,43,44,47,48,49,84,88,96,99,101,105,107,109,112,113,114,125,129,133],"counts":[1,1,4,3,2,1,2,1,1,1,1,2,1,2,2,1,1,2,1,1,1,2,1,1,1],"total_sum":2763},{"name":"op_prepare_run_time","total_count":37,"min":74,"mean":114.865,"percentile_75":141,"percentile_95":189,"percentile_99":219,"percentile_99_9":219,"percentile_99_99":219,"max":219,"values":[74,78,79,80,81,82,83,84,86,89,90,95,96,97,107,109,110,116,120,122,136,140,141,142,150,154,156,158,184,189,195,219],"counts":[1,2,1,2,1,2,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":4250},{"name":"on_disk_size","value":69705682},{"name":"all_transactions_inflight","value":0},{"name":"write_transactions_inflight","value":0},{"name":"alter_schema_transactions_inflight","value":0},{"name":"log_gc_running","value":0},{"name":"majority_done_ops","value":0},{"name":"in_progress_ops","value":0},{"name":"ops_behind_leader","value":0},{"name":"log_cache_num_ops","value":1},{"name":"log_cache_size","value":1090},{"name":"raft_term","value":184},{"name":"log_bytes_logged","value":2787244},{"name":"log_sync_latency","total_count":8167,"min":0,"mean":0.00232644,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":4,"max":15,"values":[0,4,15],"counts":[8165,1,1],"total_sum":19},{"name":"log_append_latency","total_count":8204,"min":2,"mean":3.33155,"percentile_75":3,"percentile_95":6,"percentile_99":14,"percentile_99_9":46,"percentile_99_99":60,"max":61,"values":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,19,20,21,22,23,24,25,26,27,28,29,30,32,33,35,36,37,38,41,42,44,46,47,50,51,60,61],"counts":[3460,2963,668,290,462,120,56,26,22,24,16,12,9,6,7,3,3,3,2,1,4,4,6,3,2,2,3,2,4,2,2,1,1,2,2,1,1,2,1,3,1,1,1],"total_sum":27332},{"name":"log_group_commit_latency","total_count":76,"min":15,"mean":68.5526,"percentile_75":98,"percentile_95":143,"percentile_99":176,"percentile_99_9":184,"percentile_99_99":184,"max":184,"values":[15,16,17,18,20,22,23,25,26,27,28,29,30,31,32,33,34,35,36,37,38,41,42,49,50,53,64,67,73,74,76,78,79,80,85,86,92,93,94,98,105,106,111,114,118,121,123,124,131,133,134,136,138,140,143,145,149,176,184],"counts":[1,1,3,1,1,2,1,1,3,2,3,1,1,2,1,1,1,1,1,3,1,1,2,1,1,1,1,1,3,1,2,1,2,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":5210},{"name":"log_entry_batches_per_group","total_count":76,"min":1,"mean":1,"percentile_75":1,"percentile_95":1,"percentile_99":1,"percentile_99_9":1,"percentile_99_99":1,"max":1,"values":[1],"counts":[76],"total_sum":76}]},{"type":"server","id":"kudu.master","metrics":[{"name":"tablet_copy_open_source_sessions","value":0},{"name":"op_apply_queue_length","total_count":37,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[37],"total_sum":0},{"name":"op_apply_queue_time","total_count":37,"min":10,"mean":76.5405,"percentile_75":117,"percentile_95":241,"percentile_99":266,"percentile_99_9":266,"percentile_99_99":266,"max":266,"values":[10,12,13,14,16,17,22,32,36,38,41,60,63,66,71,75,82,89,95,113,117,119,120,139,147,218,241,247,267],"counts":[1,1,3,1,3,3,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1],"total_sum":2832},{"name":"op_apply_run_time","total_count":37,"min":86,"mean":148.162,"percentile_75":171,"percentile_95":196,"percentile_99":214,"percentile_99_9":214,"percentile_99_99":214,"max":214,"values":[86,94,100,103,108,116,118,120,121,122,129,144,150,151,154,159,160,161,163,165,168,171,172,178,180,182,189,196,208,214],"counts":[1,1,1,2,1,1,1,2,1,1,2,2,1,1,1,1,1,3,1,1,1,1,1,1,1,2,1,1,1,1],"total_sum":5482},{"name":"handler_latency_kudu_master_MasterService_TSHeartbeat","total_count":587,"min":9,"mean":198.216,"percentile_75":65,"percentile_95":792,"percentile_99":4800,"percentile_99_9":6464,"percentile_99_99":8512,"max":8575,"values":[9,14,15,16,17,18,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,74,76,77,78,79,80,81,82,83,84,85,89,90,91,92,93,94,95,96,97,98,100,102,103,104,106,108,109,110,112,115,117,118,120,122,124,127,129,136,138,146,583,607,619,631,639,651,659,675,679,703,707,719,723,751,763,783,795,839,895,915,963,987,995,1031,1071,1119,1135,1143,1167,1183,1191,1199,1215,1263,3599,4223,4639,4831,4863,4991,5855,6111,6495,8575],"counts":[1,8,2,4,1,1,2,6,3,4,9,6,2,1,6,2,5,2,6,2,6,10,4,3,9,25,15,5,4,4,6,5,4,1,2,3,8,3,4,4,5,11,24,28,21,38,29,31,21,24,11,9,10,1,5,2,3,2,1,4,1,2,2,2,1,2,1,1,1,4,2,3,2,3,1,3,1,1,1,1,2,1,3,2,2,4,1,1,1,1,1,1,2,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":116353},{"name":"handler_latency_kudu_master_MasterService_Ping","total_count":5,"min":11,"mean":15.2,"percentile_75":18,"percentile_95":19,"percentile_99":19,"percentile_99_9":19,"percentile_99_99":19,"max":19,"values":[11,14,18,19],"counts":[1,2,1,1],"total_sum":76},{"name":"generic_current_allocated_bytes","value":313516592},{"name":"tcmalloc_pageheap_unmapped_bytes","value":0},{"name":"tcmalloc_max_total_thread_cache_bytes","value":536870912},{"name":"tcmalloc_current_total_thread_cache_bytes","value":1939336},{"name":"generic_heap_size","value":322920448},{"name":"tcmalloc_pageheap_free_bytes","value":598016},{"name":"glog_info_messages","value":835},{"name":"code_cache_hits","value":4},{"name":"code_cache_queries","value":6},{"name":"hybrid_clock_timestamp","value":6230236017547149312},{"name":"hybrid_clock_error","value":1092763},{"name":"data_dirs_failed","value":0},{"name":"data_dirs_full","value":0},{"name":"log_block_manager_bytes_under_management","value":61263138},{"name":"log_block_manager_blocks_under_management","value":2222},{"name":"log_block_manager_full_containers","value":0},{"name":"log_block_manager_containers","value":30},{"name":"block_manager_blocks_open_reading","value":2222},{"name":"block_manager_blocks_open_writing","value":0},{"name":"block_manager_total_readable_blocks","value":2222},{"name":"block_manager_total_bytes_read","value":54857719},{"name":"rpc_connections_accepted","value":5},{"name":"reactor_load_percent","total_count":9584,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[9584],"total_sum":0},{"name":"reactor_active_latency_us","total_count":20603,"min":0,"mean":3.25894,"percentile_75":2,"percentile_95":23,"percentile_99":41,"percentile_99_9":67,"percentile_99_99":99,"max":110,"values":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,61,62,63,64,65,66,67,68,69,70,71,75,76,77,78,87,91,92,97,99,104,110],"counts":[9931,1445,6159,584,267,169,125,53,28,30,24,25,30,33,78,190,118,66,48,28,33,22,27,65,80,80,94,74,57,59,44,47,25,34,38,20,23,33,39,34,23,29,33,21,23,11,4,13,9,6,4,6,6,1,3,2,6,2,4,3,1,2,3,2,4,1,1,2,1,2,3,1,2,1,2,1,1,1,1,1,1,1],"total_sum":67144},{"name":"rpc_incoming_queue_time","total_count":592,"min":9,"mean":44.353,"percentile_75":53,"percentile_95":60,"percentile_99":67,"percentile_99_9":76,"percentile_99_99":83,"max":83,"values":[9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,65,67,68,72,74,75,76,83],"counts":[1,1,5,10,8,10,3,1,4,2,6,4,7,7,4,5,4,7,5,6,4,8,4,3,2,4,8,4,7,3,4,7,9,9,10,7,18,14,12,42,70,37,36,19,18,18,17,17,17,16,12,6,9,6,4,3,2,1,1,1,1,1,1],"total_sum":26257},{"name":"block_cache_inserts","value":1898},{"name":"block_cache_lookups","value":16233},{"name":"block_cache_misses_caching","value":1898},{"name":"block_cache_hits_caching","value":14335},{"name":"block_cache_usage","value":55202104},{"name":"spinlock_contention_time","value":0},{"name":"threads_started","value":307},{"name":"threads_running","value":231},{"name":"cpu_utime","value":1690},{"name":"cpu_stime","value":668},{"name":"involuntary_context_switches","value":316},{"name":"voluntary_context_switches","value":23147}]}]
I0314 11:55:54.652815 stacks 1521053754652815 {"reason":"periodic","groups":[{"tids":[37480],"stack":["0x3f5ec0f710","0x3f5e8df0d3","0xa52153","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37274],"stack":["0x3f5ec0f710","0x3f5e8e5159","0x1caba0d","0x1caef51","0xa30202","0xa339c0","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37273,37272,37271,37270,37269,37268,37267,37266,37265,37264,37263,37261,37260,37259,37258,37262],"stack":["0x3f5ec0f710","0x3f5e8e8ef3","0x1d87173","0x1d8a80c","0x1ba7f69","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37479],"stack":["0x3f5ec0f710","0x3f5e8eac2d","0x1d2205a","0x1b9e1b3","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37413,37401,37412,37411,37410,37409,37408,37407,37406,37405,37404,37403,37402,37414,37415,37416,37417,37418,37419,37420,37421,37426,37375,37425,37424,37423,37422,37387,37365,37376,37377,37378,37379,37380,37381,37382,37383,37384,37385,37386,37400,37388,37389,37390,37391,37392,37393,37394,37395,37396,37397,37398,37399,37466,37454,37455,37456,37457,37458,37459,37460,37461,37462,37463,37464,37465,37453,37467,37468,37469,37470,37471,37472,37473,37474,37475,37476,37477,37478,37440,37428,37429,37430,37431,37432,37433,37434,37435,37436,37437,37438,37439,37427,37441,37442,37443,37444,37445,37446,37447,37448,37449,37450,37451,37452,37314,37303,37304,37305,37306,37307,37308,37309,37310,37311,37312,37313,37302,37315,37316,37317,37318,37319,37320,37321,37322,37323,37324,37325,37290,37279,37280,37281,37282,37283,37284,37285,37286,37287,37288,37289,37373,37291,37292,37293,37294,37295,37296,37297,37298,37299,37300,37301,37361,37350,37351,37352,37353,37354,37355,37356,37357,37358,37359,37360,37327,37362,37363,37364,37366,37367,37368,37369,37370,37371,37372,37374,37348,37326,37328,37329,37330,37331,37332,37333,37334,37335,37336,37337,37339,37349,37347,37346,37345,37344,37342,37341,37340,37338,37343],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1bccad8","0x1bcb161","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37255],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1d50b04","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37278],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x97867c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37276],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0xa06f8c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[44449],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1abf81f","0x1d50444","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37275],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1bb3594","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37257],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1ce34e4","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37253],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1cf3d5f","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37488],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x919e01","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37277],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1cfb5db","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[44683,44681,44682],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d50bbc","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37252,37251,37250],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d60c8f","0x3f620b6470","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37153],"stack":["0x3f5ec0f710","0x3f5ec0ef3d","0x1d83474","0x8c4590","0x3f5e81ed5d","0x8c4141"]},{"tids":[37254],"stack":["0x3f5ec0f710","0x3f5ec0f4b5","0x1d12523","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]}]}
I0314 11:56:15.220513 metrics 1521053775220513 [{"type":"tablet","id":"00000000000000000000000000000000","metrics":[{"name":"memrowset_size","value":265},{"name":"on_disk_data_size","value":54961885},{"name":"rows_updated","value":321},{"name":"rows_deleted","value":1},{"name":"scans_started","value":8},{"name":"bloom_lookups","value":644},{"name":"key_file_lookups","value":644},{"name":"delta_file_lookups","value":642},{"name":"bloom_lookups_per_op","total_count":322,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[322],"total_sum":644},{"name":"key_file_lookups_per_op","total_count":322,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[322],"total_sum":644},{"name":"delta_file_lookups_per_op","total_count":322,"min":0,"mean":1.99379,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[0,2],"counts":[1,321],"total_sum":642},{"name":"write_op_duration_client_propagated_consistency","total_count":322,"min":178,"mean":458.078,"percentile_75":480,"percentile_95":748,"percentile_99":944,"percentile_99_9":3856,"percentile_99_99":3856,"max":3857,"values":[178,241,249,251,255,261,275,281,285,291,297,299,303,307,313,317,321,323,327,329,331,333,335,337,339,341,343,347,349,351,355,357,359,361,363,365,367,369,371,373,375,377,379,381,383,385,387,389,391,393,395,397,399,403,405,407,409,411,413,415,417,419,421,423,425,427,429,431,433,435,437,439,441,443,445,447,449,451,453,455,457,459,463,465,467,471,473,475,477,479,481,483,485,487,489,491,493,499,503,505,511,515,519,523,527,531,535,539,543,547,551,559,563,567,575,579,587,591,611,615,631,651,671,679,683,687,695,699,719,751,755,763,779,787,803,835,843,851,875,883,887,923,947,983,999,3871],"counts":[1,1,1,1,1,1,1,2,1,1,2,3,1,3,2,1,1,2,2,1,3,1,1,1,1,3,1,1,1,4,1,3,2,4,2,2,3,3,2,3,4,1,1,1,4,3,3,3,5,8,4,7,3,5,9,4,5,6,3,6,3,5,9,5,1,3,2,4,6,7,1,3,1,2,1,3,4,3,3,1,1,3,1,2,1,1,3,3,1,1,3,1,1,2,2,1,1,3,6,2,2,1,3,2,1,2,3,1,3,1,1,1,1,1,1,1,2,1,3,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":147501},{"name":"state","value":"RUNNING"},{"name":"op_prepare_queue_length","total_count":322,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[322],"total_sum":0},{"name":"op_prepare_queue_time","total_count":322,"min":7,"mean":37.5683,"percentile_75":41,"percentile_95":105,"percentile_99":118,"percentile_99_9":133,"percentile_99_99":133,"max":133,"values":[7,8,9,10,11,12,13,14,15,16,17,18,19,23,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,52,54,57,58,65,71,74,77,78,81,82,83,84,88,96,97,99,100,101,103,105,107,108,109,112,113,114,118,125,129,133],"counts":[2,3,6,7,10,9,6,16,10,9,12,9,5,1,5,4,1,3,7,6,11,2,4,6,6,9,15,17,19,11,16,6,5,7,3,2,3,2,1,1,1,1,1,1,1,1,1,1,1,3,1,1,2,2,2,1,2,1,2,2,1,3,2,2,2,1,2,1,1,1,1],"total_sum":12097},{"name":"op_prepare_run_time","total_count":322,"min":44,"mean":98.4161,"percentile_75":115,"percentile_95":151,"percentile_99":195,"percentile_99_9":240,"percentile_99_99":240,"max":240,"values":[44,46,47,52,56,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,134,135,136,138,139,140,141,142,145,146,149,150,151,152,153,154,156,157,158,162,163,167,184,189,195,196,219,240],"counts":[1,1,1,2,2,1,1,3,4,1,3,1,2,5,5,6,4,3,6,5,2,7,5,7,6,8,2,13,5,8,6,4,10,2,4,5,4,3,6,3,2,6,9,7,4,2,5,4,4,2,1,3,3,3,2,5,2,3,2,5,5,4,1,1,1,2,3,2,1,3,1,2,3,4,1,2,3,2,2,4,1,2,4,1,1,2,1,1,2,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1],"total_sum":31690},{"name":"on_disk_size","value":69705682},{"name":"all_transactions_inflight","value":0},{"name":"write_transactions_inflight","value":0},{"name":"log_cache_num_ops","value":1},{"name":"log_cache_size","value":1090},{"name":"log_bytes_logged","value":2958219},{"name":"log_sync_latency","total_count":8452,"min":0,"mean":0.00224799,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":4,"max":15,"values":[0,4,15],"counts":[8450,1,1],"total_sum":19},{"name":"log_append_latency","total_count":8774,"min":2,"mean":4.43355,"percentile_75":4,"percentile_95":13,"percentile_99":36,"percentile_99_9":54,"percentile_99_99":83,"max":118,"values":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,54,55,59,60,61,62,69,83,118],"counts":[3461,2980,683,299,527,151,84,47,31,36,32,27,18,14,19,10,3,11,12,9,11,22,32,21,17,15,18,25,10,10,16,9,10,12,11,9,6,7,6,9,3,3,4,1,2,2,3,4,4,6,2,1,1,1,2,1,1,1,1,1],"total_sum":38900},{"name":"log_group_commit_latency","total_count":646,"min":7,"mean":55.1533,"percentile_75":83,"percentile_95":126,"percentile_99":145,"percentile_99_9":176,"percentile_99_99":184,"max":184,"values":[7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,47,49,50,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,114,115,116,118,119,120,121,123,124,125,126,127,128,130,131,132,133,134,135,136,137,138,140,143,145,149,152,164,176,184],"counts":[2,2,3,9,8,11,13,16,23,26,19,17,10,18,11,9,7,5,13,7,10,9,7,5,9,7,6,6,4,4,8,6,3,2,2,2,2,2,2,3,4,4,2,3,2,2,3,5,5,4,7,7,6,8,5,1,10,7,10,7,4,5,6,7,11,5,6,3,6,5,8,4,3,4,4,3,3,8,2,4,7,3,3,4,2,1,4,2,3,5,3,4,6,5,3,3,1,4,2,2,2,1,2,1,1,3,3,1,1,1,1,3,1,4,1,4,1,1,2,2,1,1,3,1,1,2,1,1,1],"total_sum":35629},{"name":"log_entry_batches_per_group","total_count":646,"min":1,"mean":1,"percentile_75":1,"percentile_95":1,"percentile_99":1,"percentile_99_9":1,"percentile_99_99":1,"max":1,"values":[1],"counts":[646],"total_sum":646}]},{"type":"server","id":"kudu.master","metrics":[{"name":"op_apply_queue_length","total_count":322,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[322],"total_sum":0},{"name":"op_apply_queue_time","total_count":322,"min":5,"mean":32.9348,"percentile_75":38,"percentile_95":97,"percentile_99":218,"percentile_99_9":266,"percentile_99_99":266,"max":266,"values":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,46,47,49,50,53,54,58,60,63,66,67,69,71,74,75,77,78,81,82,84,89,90,95,97,102,109,113,117,119,120,136,139,143,147,218,241,247,267],"counts":[1,1,3,2,6,14,13,18,26,27,13,13,11,4,2,2,1,1,2,5,4,7,6,7,4,7,2,4,2,8,10,7,9,11,7,7,9,3,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,2,2,1,1,1,1,1,1,1,1],"total_sum":10605},{"name":"op_apply_run_time","total_count":322,"min":58,"mean":134.472,"percentile_75":154,"percentile_95":189,"percentile_99":214,"percentile_99_9":352,"percentile_99_99":352,"max":352,"values":[58,59,66,67,69,72,76,79,81,83,84,85,86,87,91,93,94,97,98,100,101,102,103,104,105,106,107,108,109,110,111,113,114,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,174,175,176,177,178,179,180,182,186,187,189,190,191,194,196,198,208,209,214,257,269,353],"counts":[1,1,1,1,1,1,1,2,2,1,1,2,2,1,2,2,3,3,5,2,3,1,8,5,2,5,4,5,4,5,6,3,6,6,5,6,4,6,4,7,1,2,6,5,4,3,6,4,6,1,6,4,2,2,4,4,2,3,4,3,8,4,4,1,2,2,6,4,4,4,3,2,2,2,3,1,3,6,4,6,2,3,3,3,2,1,1,2,2,1,1,1,1,1,1,1,2,1,1,3,1,2,3,2,1,2,1,1,1,1,1],"total_sum":43300},{"name":"handler_latency_kudu_master_MasterService_TSHeartbeat","total_count":3122,"min":6,"mean":140.836,"percentile_75":76,"percentile_95":684,"percentile_99":980,"percentile_99_9":5824,"percentile_99_99":8512,"max":8575,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,122,123,124,125,127,128,129,130,131,132,133,134,135,136,138,140,141,142,143,146,147,151,157,159,161,164,168,315,389,409,413,427,435,439,443,445,457,471,477,479,503,505,515,519,527,531,535,543,547,551,555,559,563,567,571,575,579,583,587,591,595,599,603,607,611,615,619,623,627,631,635,639,643,647,651,655,659,663,667,671,675,679,683,687,691,695,699,703,707,711,715,719,723,727,731,735,739,743,747,751,755,759,763,767,771,775,783,787,791,795,799,807,815,819,823,827,831,839,843,847,855,859,863,867,871,875,879,887,891,895,903,911,915,923,931,951,963,979,983,987,995,1031,1055,1063,1071,1087,1103,1119,1135,1143,1167,1183,1191,1199,1215,1263,1631,3599,4127,4223,4639,4831,4863,4991,5855,6111,6495,8575],"counts":[1,1,2,4,2,3,9,8,23,17,17,8,19,22,28,34,31,40,39,28,25,15,34,29,31,29,42,32,34,37,33,26,33,66,50,31,23,26,19,14,18,9,13,14,18,25,33,49,52,44,60,78,96,102,113,105,109,62,61,59,42,34,37,38,12,19,18,16,17,17,15,23,16,9,12,10,15,10,10,16,15,10,5,11,18,9,13,15,8,9,13,8,6,12,10,9,4,13,5,4,8,5,7,7,8,5,6,5,2,3,4,4,4,1,5,3,2,5,2,2,2,1,2,1,2,2,2,2,1,3,1,2,2,1,3,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,2,1,3,1,3,2,3,1,1,7,3,3,2,3,2,1,1,3,5,3,5,6,5,5,1,6,6,2,6,2,3,5,9,5,5,6,3,9,7,5,7,5,5,7,4,7,2,1,4,1,5,7,3,1,2,2,5,1,1,2,3,5,1,4,2,1,3,2,1,1,1,2,2,1,6,1,1,3,1,1,2,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":439691},{"name":"handler_latency_kudu_master_MasterService_ConnectToMaster","total_count":14,"min":3447,"mean":4965.21,"percentile_75":5280,"percentile_95":5472,"percentile_99":6112,"percentile_99_9":6112,"percentile_99_99":6112,"max":6132,"values":[3455,3599,4799,4863,4895,4959,4991,5119,5247,5311,5503,6143],"counts":[1,1,1,1,1,1,1,1,1,2,2,1],"total_sum":69513},{"name":"handler_latency_kudu_master_MasterService_GetTableSchema","total_count":23,"min":10,"mean":46.6957,"percentile_75":55,"percentile_95":88,"percentile_99":105,"percentile_99_9":105,"percentile_99_99":105,"max":105,"values":[10,11,14,15,18,34,41,42,45,51,52,54,55,61,62,69,76,88,105],"counts":[1,1,1,1,2,1,1,1,1,4,1,1,1,1,1,1,1,1,1],"total_sum":1074},{"name":"generic_current_allocated_bytes","value":315310808},{"name":"tcmalloc_pageheap_unmapped_bytes","value":0},{"name":"tcmalloc_max_total_thread_cache_bytes","value":536870912},{"name":"tcmalloc_current_total_thread_cache_bytes","value":2256320},{"name":"generic_heap_size","value":326066176},{"name":"tcmalloc_pageheap_free_bytes","value":876544},{"name":"glog_info_messages","value":1120},{"name":"code_cache_hits","value":6},{"name":"code_cache_queries","value":8},{"name":"hybrid_clock_timestamp","value":6230236263308177408},{"name":"hybrid_clock_error","value":1122763},{"name":"block_manager_total_bytes_read","value":56064459},{"name":"rpc_connections_accepted","value":11},{"name":"reactor_load_percent","total_count":19184,"min":0,"mean":0.000104254,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":1,"values":[0,1],"counts":[19182,2],"total_sum":2},{"name":"reactor_active_latency_us","total_count":48917,"min":0,"mean":5.63487,"percentile_75":3,"percentile_95":30,"percentile_99":46,"percentile_99_9":71,"percentile_99_99":113,"max":757,"values":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,84,85,87,89,90,91,92,93,94,97,99,104,110,113,739,743,759],"counts":[19434,4932,10832,1849,1179,687,533,287,218,162,171,151,208,327,458,764,467,331,237,211,176,201,319,343,442,385,326,287,231,228,201,198,143,159,159,137,118,147,146,130,93,94,94,81,57,55,58,50,47,34,37,28,24,15,18,16,19,16,18,14,8,6,11,8,8,12,5,7,6,5,3,9,3,2,2,4,3,5,4,2,1,1,1,2,1,3,1,1,1,1,1,1,1,1,2,1,2],"total_sum":275641},{"name":"rpc_incoming_queue_time","total_count":3164,"min":6,"mean":37.4716,"percentile_75":49,"percentile_95":61,"percentile_99":70,"percentile_99_9":91,"percentile_99_99":204,"max":204,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,74,75,76,77,78,83,89,91,98,204],"counts":[2,12,12,24,43,44,52,52,62,42,50,41,63,63,49,50,55,68,60,58,60,51,55,54,46,43,35,46,38,56,35,35,42,44,54,58,49,60,68,78,78,66,156,178,130,102,59,43,51,53,48,49,34,27,20,27,20,19,9,8,11,11,10,7,11,5,6,4,1,2,1,1,3,1,2,1,1],"total_sum":118560},{"name":"block_cache_inserts","value":2190},{"name":"block_cache_lookups","value":21171},{"name":"block_cache_misses_caching","value":2190},{"name":"block_cache_hits_caching","value":18981},{"name":"block_cache_usage","value":56406696},{"name":"spinlock_contention_time","value":0},{"name":"threads_started","value":381},{"name":"threads_running","value":231},{"name":"cpu_utime","value":2831},{"name":"cpu_stime","value":1053},{"name":"involuntary_context_switches","value":423},{"name":"voluntary_context_switches","value":52316}]}]
I0314 11:57:11.020710 stacks 1521053831020710 {"reason":"periodic","groups":[{"tids":[37480],"stack":["0x3f5ec0f710","0x3f5e8df0d3","0xa52153","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37274],"stack":["0x3f5ec0f710","0x3f5e8e5159","0x1caba0d","0x1caef51","0xa30202","0xa339c0","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37273,37272,37271,37270,37269,37268,37267,37266,37265,37264,37263,37261,37260,37259,37258,37262],"stack":["0x3f5ec0f710","0x3f5e8e8ef3","0x1d87173","0x1d8a80c","0x1ba7f69","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37479],"stack":["0x3f5ec0f710","0x3f5e8eac2d","0x1d2205a","0x1b9e1b3","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37413,37401,37412,37411,37410,37409,37408,37407,37406,37405,37404,37403,37402,37414,37415,37416,37417,37418,37419,37420,37421,37426,37375,37425,37424,37423,37422,37387,37365,37376,37377,37378,37379,37380,37381,37382,37383,37384,37385,37386,37400,37388,37389,37390,37391,37392,37393,37394,37395,37396,37397,37398,37399,37466,37454,37455,37456,37457,37458,37459,37460,37461,37462,37463,37464,37465,37453,37467,37468,37469,37470,37471,37472,37473,37474,37475,37476,37477,37478,37440,37428,37429,37430,37431,37432,37433,37434,37435,37436,37437,37438,37439,37427,37441,37442,37443,37444,37445,37446,37447,37448,37449,37450,37451,37452,37314,37303,37304,37305,37306,37307,37308,37309,37310,37311,37312,37313,37302,37315,37316,37317,37318,37319,37320,37321,37322,37323,37324,37325,37290,37279,37280,37281,37282,37283,37284,37285,37286,37287,37288,37289,37373,37291,37292,37293,37294,37295,37296,37297,37298,37299,37300,37301,37361,37350,37351,37352,37353,37354,37355,37356,37357,37358,37359,37360,37327,37362,37363,37364,37366,37367,37368,37369,37370,37371,37372,37374,37348,37326,37328,37329,37330,37331,37332,37333,37334,37335,37336,37337,37339,37349,37347,37346,37345,37344,37342,37341,37340,37338,37343],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1bccad8","0x1bcb161","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37255],"stack":["0x3f5ec0f710","0x3f5ec0b5ba","0x1d50b04","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37278],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x97867c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37276],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0xa06f8c","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[45532],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1abf81f","0x1d50444","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37275],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1bb3594","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37257],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1ce34e4","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37253],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caaf69","0x1cf3d5f","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37488],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x919e01","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37277],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1cfb5db","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[45545,45543,45544],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d50bbc","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37252,37251,37250],"stack":["0x3f5ec0f710","0x3f5ec0b98e","0x1caafd7","0x1d60c8f","0x3f620b6470","0x3f5ec079d1","0x3f5e8e88fd"]},{"tids":[37153],"stack":["0x3f5ec0f710","0x3f5ec0ef3d","0x1d83474","0x8c4590","0x3f5e81ed5d","0x8c4141"]},{"tids":[37254],"stack":["0x3f5ec0f710","0x3f5ec0f4b5","0x1d12523","0x1d46ca1","0x3f5ec079d1","0x3f5e8e88fd"]}]}
I0314 11:57:15.220611 metrics 1521053835220611 [{"type":"tablet","id":"00000000000000000000000000000000","metrics":[{"name":"memrowset_size","value":265},{"name":"on_disk_data_size","value":54961885},{"name":"rows_updated","value":656},{"name":"rows_deleted","value":1},{"name":"scans_started","value":10},{"name":"bloom_lookups","value":1314},{"name":"key_file_lookups","value":1314},{"name":"delta_file_lookups","value":1314},{"name":"bloom_lookups_per_op","total_count":657,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[657],"total_sum":1314},{"name":"key_file_lookups_per_op","total_count":657,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[657],"total_sum":1314},{"name":"delta_file_lookups_per_op","total_count":657,"min":0,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":4,"max":4,"values":[0,2,4],"counts":[1,655,1],"total_sum":1314},{"name":"flush_dms_running","value":0},{"name":"flush_dms_duration","total_count":1,"min":250,"mean":250,"percentile_75":248,"percentile_95":248,"percentile_99":248,"percentile_99_9":248,"percentile_99_99":248,"max":250,"values":[255],"counts":[1],"total_sum":250},{"name":"write_op_duration_client_propagated_consistency","total_count":657,"min":178,"mean":449.807,"percentile_75":484,"percentile_95":684,"percentile_99":884,"percentile_99_9":1576,"percentile_99_99":3856,"max":3857,"values":[178,212,217,223,227,241,249,251,252,254,255,261,263,267,269,271,273,275,277,279,281,283,285,287,289,291,297,299,303,307,311,313,315,317,321,323,325,327,329,331,333,335,337,339,341,343,345,347,349,351,353,355,357,359,361,363,365,367,369,371,373,375,377,379,381,383,385,387,389,391,393,395,397,399,401,403,405,407,409,411,413,415,417,419,421,423,425,427,429,431,433,435,437,439,441,443,445,447,449,451,453,455,457,459,461,463,465,467,469,471,473,475,477,479,481,483,485,487,489,491,493,495,497,499,501,503,505,507,509,511,515,519,523,527,531,535,539,543,547,551,555,559,563,567,571,575,579,583,587,591,599,603,611,615,619,631,651,655,667,671,675,679,683,687,691,695,699,715,719,739,751,755,759,763,779,787,803,815,835,843,851,875,883,887,923,947,983,999,1519,1583,3871],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,4,2,2,1,3,1,3,1,1,1,2,2,2,4,1,3,1,3,3,2,2,8,3,3,2,7,3,3,2,1,4,3,2,3,2,9,2,2,5,3,4,2,3,4,4,6,4,7,4,3,5,7,5,8,8,9,10,7,10,5,2,9,11,7,10,8,6,9,10,10,11,8,1,8,7,7,9,11,3,7,2,6,5,4,7,6,5,3,4,6,2,3,5,4,3,4,3,7,4,3,5,4,4,4,3,6,2,2,2,5,2,7,3,1,3,5,3,3,4,3,6,5,4,6,4,3,4,2,1,2,2,2,2,2,4,1,2,2,4,1,1,2,1,1,1,1,1,3,2,3,1,2,2,2,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":295523},{"name":"delta_minor_compact_rs_running","value":0},{"name":"delta_minor_compact_rs_duration","total_count":1,"min":91,"mean":91,"percentile_75":88,"percentile_95":88,"percentile_99":88,"percentile_99_9":88,"percentile_99_99":88,"max":91,"values":[91],"counts":[1],"total_sum":91},{"name":"state","value":"RUNNING"},{"name":"op_prepare_queue_length","total_count":657,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[657],"total_sum":0},{"name":"op_prepare_queue_time","total_count":657,"min":7,"mean":37.239,"percentile_75":41,"percentile_95":101,"percentile_99":118,"percentile_99_9":182,"percentile_99_99":1120,"max":1126,"values":[7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,56,57,58,65,70,71,74,77,78,79,81,82,83,84,88,91,94,95,96,97,99,100,101,103,105,106,107,108,109,110,112,113,114,116,117,118,123,125,129,133,134,182,1127],"counts":[5,4,6,16,18,22,16,25,21,23,21,15,7,6,3,4,7,3,10,13,9,7,22,14,16,8,11,13,9,19,26,30,31,22,21,10,16,16,11,8,8,7,2,3,2,2,1,1,1,2,1,1,1,1,1,2,1,1,3,2,1,3,3,1,1,1,2,1,2,1,3,2,1,1,3,6,2,1,2,1,2,1,1,1,1,1,1,1,1,1,1],"total_sum":24466},{"name":"op_prepare_run_time","total_count":657,"min":29,"mean":96.2938,"percentile_75":113,"percentile_95":150,"percentile_99":195,"percentile_99_9":240,"percentile_99_99":316,"max":317,"values":[29,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,162,163,167,172,175,180,184,188,189,195,196,205,212,219,240,317],"counts":[1,1,2,1,2,3,1,4,1,4,3,1,1,3,5,3,2,5,3,9,2,5,4,6,10,11,8,14,6,14,14,7,14,14,9,12,13,7,19,17,12,14,9,16,4,8,14,6,6,13,5,8,7,12,10,8,6,7,4,7,6,2,4,7,7,5,4,8,4,5,6,8,8,8,2,5,2,2,3,4,4,3,3,3,3,6,2,3,3,3,2,3,5,1,2,5,9,2,1,4,2,1,4,2,2,3,2,5,2,1,2,2,1,1,1,1,1,2,2,2,1,1,1,1,1,1,1,1,1,2,1,1,1],"total_sum":63265},{"name":"on_disk_size","value":69712809},{"name":"all_transactions_inflight","value":0},{"name":"write_transactions_inflight","value":0},{"name":"log_cache_num_ops","value":1},{"name":"log_cache_size","value":1090},{"name":"log_bytes_logged","value":3159581},{"name":"log_sync_latency","total_count":8789,"min":0,"mean":0.00352714,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":12,"max":15,"values":[0,4,12,15],"counts":[8786,1,1,1],"total_sum":31},{"name":"log_append_latency","total_count":9444,"min":2,"mean":5.73401,"percentile_75":4,"percentile_95":27,"percentile_99":46,"percentile_99_9":69,"percentile_99_99":118,"max":167,"values":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,65,67,69,70,72,75,83,88,93,103,118,167],"counts":[3461,3014,705,323,590,201,110,69,41,46,45,35,27,32,29,16,10,15,24,14,20,33,41,31,36,23,33,37,22,17,20,16,25,23,23,19,16,21,13,20,18,9,13,7,11,7,5,7,13,9,5,4,2,5,3,1,2,4,4,2,3,2,1,2,1,1,1,1,1,1,1,1,1],"total_sum":54152},{"name":"log_group_commit_latency","total_count":1317,"min":6,"mean":56.5672,"percentile_75":88,"percentile_95":130,"percentile_99":152,"percentile_99_9":184,"percentile_99_99":342,"max":343,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,145,147,148,149,150,151,152,154,156,160,161,164,166,167,168,176,178,183,184,343],"counts":[1,4,13,12,18,20,27,34,37,41,47,35,35,27,35,24,21,19,11,16,14,14,15,13,10,12,13,9,12,12,8,13,9,4,5,4,5,2,1,2,5,4,3,3,4,4,6,5,5,5,7,6,4,10,8,7,12,10,10,10,6,7,17,11,13,10,8,9,9,12,14,8,14,8,8,9,14,10,6,12,11,6,6,12,6,7,12,7,6,9,7,8,5,7,6,11,8,7,14,10,7,7,7,5,7,6,6,4,3,3,5,2,6,6,1,3,1,4,4,1,5,3,4,6,4,6,2,2,3,4,3,1,4,1,2,3,2,1,1,1,2,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":74499},{"name":"log_entry_batches_per_group","total_count":1317,"min":1,"mean":1.00076,"percentile_75":1,"percentile_95":1,"percentile_99":1,"percentile_99_9":1,"percentile_99_99":2,"max":2,"values":[1,2],"counts":[1316,1],"total_sum":1318}]},{"type":"server","id":"kudu.master","metrics":[{"name":"op_apply_queue_length","total_count":657,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[657],"total_sum":0},{"name":"op_apply_queue_time","total_count":657,"min":5,"mean":30.2907,"percentile_75":38,"percentile_95":81,"percentile_99":139,"percentile_99_9":247,"percentile_99_99":266,"max":266,"values":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,49,50,52,53,54,57,58,59,60,63,66,67,68,69,70,71,74,75,77,78,80,81,82,84,86,89,90,91,92,94,95,97,98,102,109,110,113,117,119,120,136,139,143,147,201,218,241,247,267],"counts":[1,3,11,12,25,30,30,30,46,39,20,23,17,9,3,3,7,5,4,7,9,7,14,17,12,10,12,9,5,9,15,14,16,23,23,26,13,16,8,1,2,5,4,3,1,1,2,1,1,1,1,1,2,1,2,1,2,1,1,1,1,1,2,1,1,1,2,1,1,1,1,1,1,2,2,1,1,1,1,1,1,2,3,1,1,1,1,1,1,1,1,1],"total_sum":19901},{"name":"op_apply_run_time","total_count":657,"min":58,"mean":133.773,"percentile_75":153,"percentile_95":193,"percentile_99":256,"percentile_99_9":352,"percentile_99_99":458,"max":459,"values":[58,59,61,62,64,65,66,67,69,70,71,72,74,75,76,77,78,79,80,81,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,184,186,187,188,189,190,191,193,194,196,198,199,200,201,203,205,207,208,209,213,214,218,234,242,257,265,269,275,281,313,353,459],"counts":[2,1,1,3,1,2,1,2,2,2,1,1,1,1,1,1,2,4,1,4,1,3,3,3,2,1,4,2,6,3,3,6,5,2,4,9,1,11,5,6,8,5,5,6,7,11,7,8,10,4,6,8,5,9,8,7,10,10,12,10,4,7,12,11,10,6,14,12,1,9,5,10,7,4,3,5,7,6,5,10,6,12,9,6,3,4,5,8,7,7,9,8,5,2,6,4,4,8,6,5,9,3,4,5,4,4,2,3,3,6,2,4,2,2,1,1,1,3,3,3,1,2,2,1,4,3,2,2,5,2,2,1,1,1,1,1,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":87889},{"name":"handler_latency_kudu_master_MasterService_TSHeartbeat","total_count":6584,"min":6,"mean":130.509,"percentile_75":78,"percentile_95":684,"percentile_99":896,"percentile_99_9":4608,"percentile_99_99":6464,"max":8575,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,150,151,152,153,154,155,156,157,158,159,160,161,163,164,166,167,168,176,196,206,226,231,315,341,359,389,409,413,415,417,419,427,433,435,437,439,443,445,447,451,457,459,467,469,471,473,477,479,481,483,487,499,501,503,505,507,509,511,515,519,523,527,531,535,543,547,551,555,559,563,567,571,575,579,583,587,591,595,599,603,607,611,615,619,623,627,631,635,639,643,647,651,655,659,663,667,671,675,679,683,687,691,695,699,703,707,711,715,719,723,727,731,735,739,743,747,751,755,759,763,767,771,775,779,783,787,791,795,799,803,807,811,815,819,823,827,831,835,839,843,847,851,855,859,863,867,871,875,879,887,891,895,899,903,911,915,919,923,931,935,951,963,967,979,983,987,995,1019,1023,1031,1055,1063,1071,1087,1095,1103,1119,1135,1143,1151,1167,1183,1191,1199,1215,1263,1287,1319,1415,1575,1631,1831,2207,3599,4127,4223,4639,4831,4863,4991,5855,6111,6495,8575],"counts":[1,2,3,7,7,12,19,19,32,35,43,16,39,51,57,69,70,72,74,54,70,43,69,67,73,84,103,79,79,106,95,72,84,118,87,62,52,49,41,35,29,25,34,29,34,40,65,102,100,98,117,151,195,212,204,205,175,137,129,99,68,60,72,58,34,45,40,43,41,33,40,45,29,20,20,24,29,21,22,27,32,22,11,22,32,11,28,24,20,26,29,22,11,23,20,21,12,27,15,10,15,14,18,16,22,14,15,13,12,13,12,11,7,3,9,6,11,10,11,11,9,11,15,6,7,6,4,9,7,4,3,3,6,2,4,5,3,3,4,4,6,4,1,2,3,2,1,2,1,1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,3,3,5,2,2,4,3,1,8,3,4,4,3,1,4,9,4,3,4,6,6,3,2,6,8,5,9,9,6,7,4,8,8,9,8,7,10,11,18,8,10,11,5,12,10,12,15,12,7,9,9,13,5,7,10,5,11,13,7,3,6,4,8,7,4,4,5,7,4,1,7,2,5,4,5,1,4,2,2,4,4,3,2,5,7,3,2,3,3,3,1,6,3,5,5,3,2,3,1,2,1,3,3,1,2,2,1,1,1,1,1,3,1,1,1,6,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":859273},{"name":"handler_latency_kudu_master_MasterService_ListTables","total_count":2,"min":51,"mean":55.5,"percentile_75":60,"percentile_95":60,"percentile_99":60,"percentile_99_9":60,"percentile_99_99":60,"max":60,"values":[51,60],"counts":[1,1],"total_sum":111},{"name":"handler_latency_kudu_master_MasterService_GetTableLocations","total_count":160,"min":23,"mean":70.8375,"percentile_75":82,"percentile_95":104,"percentile_99":125,"percentile_99_9":153,"percentile_99_99":153,"max":153,"values":[23,24,25,26,27,28,29,31,35,36,37,38,39,51,54,55,56,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,94,97,98,101,102,103,104,109,111,114,115,122,125,146,153],"counts":[1,2,2,3,3,4,1,1,2,2,1,1,1,1,1,1,1,1,1,3,5,12,3,9,7,12,9,3,6,2,2,3,2,2,4,3,1,2,5,3,4,1,1,2,1,1,1,1,2,1,1,5,1,1,1,1,1,1,1,1,1,1,1],"total_sum":11334},{"name":"handler_latency_kudu_master_MasterService_GetTableSchema","total_count":79,"min":8,"mean":24.3797,"percentile_75":32,"percentile_95":62,"percentile_99":88,"percentile_99_9":105,"percentile_99_99":105,"max":105,"values":[8,9,10,11,12,13,14,15,16,17,18,21,24,30,32,34,35,40,41,42,45,46,51,52,54,55,61,62,69,76,88,105],"counts":[3,4,11,10,6,3,3,4,3,2,3,2,1,3,1,1,1,1,1,1,1,1,4,1,1,1,1,1,1,1,1,1],"total_sum":1926},{"name":"handler_latency_kudu_master_MasterService_ListTabletServers","total_count":2,"min":28,"mean":29.5,"percentile_75":31,"percentile_95":31,"percentile_99":31,"percentile_99_9":31,"percentile_99_99":31,"max":31,"values":[28,31],"counts":[1,1],"total_sum":59},{"name":"handler_latency_kudu_master_MasterService_GetMasterRegistration","total_count":4,"min":13,"mean":24.75,"percentile_75":23,"percentile_95":41,"percentile_99":41,"percentile_99_9":41,"percentile_99_99":41,"max":41,"values":[13,22,23,41],"counts":[1,1,1,1],"total_sum":99},{"name":"generic_current_allocated_bytes","value":316756064},{"name":"tcmalloc_pageheap_unmapped_bytes","value":0},{"name":"tcmalloc_max_total_thread_cache_bytes","value":536870912},{"name":"tcmalloc_current_total_thread_cache_bytes","value":2909072},{"name":"generic_heap_size","value":329211904},{"name":"tcmalloc_pageheap_free_bytes","value":1581056},{"name":"glog_info_messages","value":1466},{"name":"code_cache_hits","value":8},{"name":"code_cache_queries","value":10},{"name":"hybrid_clock_timestamp","value":6230236509067182080},{"name":"hybrid_clock_error","value":113659},{"name":"log_block_manager_bytes_under_management","value":61270254},{"name":"log_block_manager_blocks_under_management","value":2222},{"name":"log_block_manager_holes_punched","value":2},{"name":"block_manager_total_disk_sync","value":4},{"name":"block_manager_blocks_open_reading","value":2222},{"name":"block_manager_blocks_open_writing","value":0},{"name":"block_manager_total_writable_blocks","value":2},{"name":"block_manager_total_readable_blocks","value":2224},{"name":"block_manager_total_blocks_created","value":2},{"name":"block_manager_total_blocks_deleted","value":2},{"name":"block_manager_total_bytes_written","value":41868},{"name":"block_manager_total_bytes_read","value":56892443},{"name":"rpc_connections_accepted","value":15},{"name":"reactor_load_percent","total_count":28784,"min":0,"mean":0.000382157,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":1,"max":2,"values":[0,1,2],"counts":[28775,7,2],"total_sum":11},{"name":"reactor_active_latency_us","total_count":81391,"min":0,"mean":6.78854,"percentile_75":6,"percentile_95":33,"percentile_99":48,"percentile_99_9":75,"percentile_99_99":113,"max":757,"values":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,99,104,105,109,110,113,117,153,154,739,743,759],"counts":[28931,9060,15247,3102,2341,1445,1084,708,541,563,436,371,451,712,1029,1615,980,691,470,439,391,401,565,667,917,886,658,546,441,493,409,377,286,316,305,286,278,281,331,270,173,183,171,162,129,118,128,106,109,73,80,71,51,42,31,36,33,38,30,21,14,22,26,17,18,22,14,10,13,9,6,15,6,4,6,9,3,11,6,2,2,3,3,2,2,3,4,2,2,3,1,3,2,1,1,3,1,1,1,1,1,1,1,1,1,1,1,2,1,2],"total_sum":552526},{"name":"rpc_incoming_queue_time","total_count":6850,"min":6,"mean":37.4261,"percentile_75":49,"percentile_95":62,"percentile_99":72,"percentile_99_9":91,"percentile_99_99":204,"max":5179,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,86,87,89,91,92,93,98,150,204,5179],"counts":[12,29,85,107,104,106,104,124,142,97,116,92,119,127,120,111,107,126,110,115,123,104,109,102,97,103,93,118,100,120,88,97,106,105,132,127,113,120,116,173,181,180,324,306,242,167,108,103,102,104,92,81,63,54,47,50,43,29,37,29,29,27,20,24,18,12,15,9,11,7,7,4,3,2,1,2,2,4,1,1,1,1,2,1,1,1,1,1,1],"total_sum":256369},{"name":"block_cache_inserts","value":2393},{"name":"block_cache_lookups","value":26326},{"name":"block_cache_misses","value":2},{"name":"block_cache_misses_caching","value":2393},{"name":"block_cache_hits","value":9},{"name":"block_cache_hits_caching","value":23922},{"name":"block_cache_usage","value":57528604},{"name":"spinlock_contention_time","value":0},{"name":"threads_started","value":446},{"name":"threads_running","value":231},{"name":"cpu_utime","value":4115},{"name":"cpu_stime","value":1587},{"name":"involuntary_context_switches","value":541},{"name":"voluntary_context_switches","value":84984}]}]
I0314 11:58:15.220710 metrics 1521053895220710 [{"type":"tablet","id":"00000000000000000000000000000000","metrics":[{"name":"memrowset_size","value":265},{"name":"on_disk_data_size","value":54961885},{"name":"rows_updated","value":781},{"name":"rows_deleted","value":1},{"name":"scans_started","value":12},{"name":"bloom_lookups","value":1564},{"name":"key_file_lookups","value":1564},{"name":"delta_file_lookups","value":1564},{"name":"bloom_lookups_per_op","total_count":782,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[782],"total_sum":1564},{"name":"key_file_lookups_per_op","total_count":782,"min":2,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":2,"max":2,"values":[2],"counts":[782],"total_sum":1564},{"name":"delta_file_lookups_per_op","total_count":782,"min":0,"mean":2,"percentile_75":2,"percentile_95":2,"percentile_99":2,"percentile_99_9":2,"percentile_99_99":4,"max":4,"values":[0,2,4],"counts":[1,780,1],"total_sum":1564},{"name":"write_op_duration_client_propagated_consistency","total_count":719,"min":178,"mean":466.474,"percentile_75":498,"percentile_95":696,"percentile_99":944,"percentile_99_9":3856,"percentile_99_99":5792,"max":5798,"values":[178,212,217,223,227,241,249,251,252,254,255,261,263,267,269,271,273,275,277,279,281,283,285,287,289,291,297,299,303,307,311,313,315,317,321,323,325,327,329,331,333,335,337,339,341,343,345,347,349,351,353,355,357,359,361,363,365,367,369,371,373,375,377,379,381,383,385,387,389,391,393,395,397,399,401,403,405,407,409,411,413,415,417,419,421,423,425,427,429,431,433,435,437,439,441,443,445,447,449,451,453,455,457,459,461,463,465,467,469,471,473,475,477,479,481,483,485,487,489,491,493,495,497,499,501,503,505,507,509,511,515,519,523,527,531,535,539,543,547,551,555,559,563,567,571,575,579,583,587,591,595,599,603,611,615,619,631,643,647,651,655,667,671,675,679,683,687,691,695,699,715,719,723,727,731,739,743,747,751,755,759,763,779,787,803,815,835,843,851,875,883,887,923,947,983,999,1519,1583,2223,3871,5823],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,4,2,2,1,3,1,3,1,2,1,2,2,2,4,1,3,1,3,3,2,2,8,3,3,2,7,3,3,2,1,4,3,2,3,3,9,2,3,5,3,4,2,3,4,4,7,4,7,5,3,5,7,5,8,8,9,11,8,10,6,2,9,11,7,10,9,7,9,11,10,11,8,2,9,7,7,9,11,3,8,2,6,6,4,7,7,5,4,4,6,2,3,5,6,4,4,4,7,5,3,5,4,5,5,4,6,2,3,2,6,3,9,3,1,3,5,3,4,4,3,7,9,6,7,4,3,4,3,1,2,2,4,2,2,4,1,1,2,2,7,1,1,2,1,1,2,2,2,1,1,3,2,4,1,3,2,2,1,1,1,1,1,1,1,2,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":335395},{"name":"state","value":"RUNNING"},{"name":"op_prepare_queue_length","total_count":719,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[719],"total_sum":0},{"name":"op_prepare_queue_time","total_count":719,"min":6,"mean":45.7455,"percentile_75":42,"percentile_95":104,"percentile_99":125,"percentile_99_9":1120,"percentile_99_99":5248,"max":5253,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,56,57,58,65,68,70,71,74,77,78,79,81,82,83,84,85,86,88,89,91,93,94,95,96,97,99,100,101,103,104,105,106,107,108,109,110,112,113,114,116,117,118,123,125,126,129,133,134,182,1127,5279],"counts":[1,5,4,6,16,19,22,18,26,23,23,22,16,9,7,3,6,7,4,10,14,9,8,22,14,17,9,11,13,10,19,27,32,35,24,23,12,17,18,12,11,9,7,2,4,2,2,1,1,1,2,1,1,1,1,1,2,2,1,1,3,2,1,3,1,1,5,2,2,1,1,1,2,1,2,1,3,3,2,1,1,3,7,3,2,2,2,2,1,1,2,1,1,1,1,1,1,1,1,1],"total_sum":32891},{"name":"op_prepare_run_time","total_count":719,"min":29,"mean":98.3408,"percentile_75":115,"percentile_95":153,"percentile_99":205,"percentile_99_9":316,"percentile_99_99":342,"max":342,"values":[29,43,44,45,46,47,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,162,163,164,167,170,172,175,180,184,188,189,195,196,203,205,212,219,231,240,317,343],"counts":[1,1,2,1,2,3,1,4,1,4,3,1,1,3,5,3,2,6,3,9,3,5,4,6,10,11,9,15,6,14,14,8,14,14,12,12,13,8,20,17,14,14,9,18,5,8,15,7,6,13,6,8,8,12,10,8,6,7,7,7,6,3,4,8,7,5,4,8,5,5,10,8,8,8,2,6,5,5,5,5,4,4,3,3,3,7,2,5,4,5,2,3,6,1,3,6,11,3,1,4,2,1,4,2,2,4,2,6,2,1,2,2,1,1,2,1,1,2,2,1,2,1,2,2,1,1,1,1,1,1,1,1,2,1,1,1,1,1],"total_sum":70707},{"name":"on_disk_size","value":69712809},{"name":"all_transactions_inflight","value":0},{"name":"write_transactions_inflight","value":0},{"name":"log_cache_num_ops","value":1},{"name":"log_cache_size","value":35617},{"name":"log_bytes_logged","value":3224123},{"name":"log_sync_latency","total_count":8851,"min":0,"mean":0.00440628,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":12,"max":15,"values":[0,4,8,12,15],"counts":[8847,1,1,1,1],"total_sum":39},{"name":"log_append_latency","total_count":9568,"min":2,"mean":6.03135,"percentile_75":4,"percentile_95":28,"percentile_99":48,"percentile_99_9":70,"percentile_99_99":175,"max":225,"values":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,65,67,69,70,72,75,83,88,93,103,118,167,175,225],"counts":[3461,3017,708,330,600,206,114,71,46,48,46,38,28,37,30,16,12,16,25,14,22,34,41,33,40,25,35,37,24,17,22,16,27,27,26,21,18,24,16,24,20,10,14,10,13,9,5,10,13,12,5,5,3,5,3,2,3,4,5,4,3,1,2,1,2,2,1,1,1,1,1,1,1,1,1,1],"total_sum":57708},{"name":"log_group_commit_latency","total_count":1441,"min":6,"mean":57.9188,"percentile_75":89,"percentile_95":132,"percentile_99":164,"percentile_99_9":342,"percentile_99_99":350,"max":350,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,145,147,148,149,150,151,152,154,155,156,160,161,164,166,167,168,176,178,181,183,184,194,199,241,343,351],"counts":[1,4,13,12,19,23,29,36,41,46,51,36,40,28,36,25,21,20,13,17,17,15,15,14,12,14,14,10,12,12,10,15,10,5,6,5,6,2,2,2,5,4,3,4,4,5,8,5,7,5,7,6,4,10,8,8,12,10,11,13,6,9,17,12,14,10,9,9,12,12,14,9,15,8,9,9,15,10,6,13,12,6,6,12,6,8,13,7,7,9,7,8,6,8,6,11,11,8,15,10,8,7,8,5,8,7,7,6,3,4,6,2,7,7,3,5,1,4,4,2,5,5,5,6,4,7,4,3,3,4,3,1,5,2,2,3,2,1,2,1,2,1,3,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,2,1,1],"total_sum":83461},{"name":"log_entry_batches_per_group","total_count":1441,"min":1,"mean":1.00069,"percentile_75":1,"percentile_95":1,"percentile_99":1,"percentile_99_9":1,"percentile_99_99":2,"max":2,"values":[1,2],"counts":[1440,1],"total_sum":1442}]},{"type":"server","id":"kudu.master","metrics":[{"name":"op_apply_queue_length","total_count":719,"min":0,"mean":0,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":0,"max":0,"values":[0],"counts":[719],"total_sum":0},{"name":"op_apply_queue_time","total_count":719,"min":5,"mean":31.9722,"percentile_75":39,"percentile_95":88,"percentile_99":147,"percentile_99_9":247,"percentile_99_99":266,"max":266,"values":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,52,53,54,55,57,58,59,60,62,63,64,65,66,67,68,69,70,71,72,74,75,77,78,80,81,82,84,86,87,88,89,90,91,92,94,95,97,98,100,102,109,110,113,117,119,120,131,136,139,143,147,181,201,205,218,241,247,267],"counts":[1,3,11,13,27,30,31,31,47,40,21,24,17,9,3,4,7,5,4,7,10,7,16,19,14,11,12,10,5,9,15,16,22,24,28,27,14,17,10,2,4,5,4,1,3,2,1,2,1,1,1,1,1,2,1,2,1,1,3,2,1,3,2,1,1,1,1,1,2,1,1,1,2,1,1,1,1,1,2,1,1,3,2,1,1,1,1,2,1,1,2,3,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":22988},{"name":"op_apply_run_time","total_count":719,"min":58,"mean":135.433,"percentile_75":154,"percentile_95":193,"percentile_99":268,"percentile_99_9":458,"percentile_99_99":1144,"max":1148,"values":[58,59,61,62,64,65,66,67,69,70,71,72,74,75,76,77,78,79,80,81,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,184,186,187,188,189,190,191,192,193,194,196,198,199,200,201,203,205,207,208,209,213,214,218,228,234,242,257,265,269,275,281,299,313,353,459,1151],"counts":[2,1,1,3,1,2,1,2,2,2,1,1,1,1,1,1,2,4,1,4,2,4,4,3,2,1,4,2,6,3,5,7,5,5,5,9,2,11,5,6,8,7,5,6,10,12,8,8,10,4,9,8,5,9,9,7,10,10,12,10,6,7,14,13,10,6,14,12,3,10,7,10,8,5,6,5,8,6,5,11,6,12,9,6,4,4,5,8,7,7,9,8,6,2,6,5,6,9,6,5,10,3,4,6,6,4,3,4,3,6,2,5,2,2,1,2,1,5,3,3,1,3,2,1,4,3,2,1,3,5,2,2,1,1,1,1,1,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":97376},{"name":"handler_latency_kudu_master_MasterService_TSHeartbeat","total_count":8204,"min":6,"mean":122.703,"percentile_75":78,"percentile_95":676,"percentile_99":900,"percentile_99_9":4608,"percentile_99_99":6464,"max":8575,"values":[6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,150,151,152,153,154,155,156,157,158,159,160,161,163,164,166,167,168,170,176,196,200,206,226,231,315,341,359,389,409,413,415,417,419,421,427,433,435,437,439,443,445,447,451,457,459,467,469,471,473,477,479,481,483,487,499,501,503,505,507,509,511,515,519,523,527,531,535,543,547,551,555,559,563,567,571,575,579,583,587,591,595,599,603,607,611,615,619,623,627,631,635,639,643,647,651,655,659,663,667,671,675,679,683,687,691,695,699,703,707,711,715,719,723,727,731,735,739,743,747,751,755,759,763,767,771,775,779,783,787,791,795,799,803,807,811,815,819,823,827,831,835,839,843,847,851,855,859,863,867,871,875,879,883,887,891,895,899,903,911,915,919,923,927,931,935,951,963,967,979,983,987,995,999,1007,1019,1023,1031,1047,1055,1063,1071,1087,1095,1103,1119,1135,1143,1151,1159,1167,1183,1191,1199,1215,1263,1287,1319,1415,1503,1575,1631,1831,2207,3599,4127,4223,4639,4831,4863,4991,5855,6111,6143,6495,8575],"counts":[1,3,3,10,10,14,21,24,41,43,50,21,45,61,74,85,79,83,92,63,82,56,77,81,100,102,131,105,110,129,113,100,113,140,114,78,65,60,53,42,33,29,41,38,38,51,77,125,132,137,147,195,256,257,270,252,213,168,158,123,83,78,90,68,46,57,52,57,55,44,49,58,40,37,32,33,38,29,29,35,38,27,17,27,33,17,35,29,29,31,37,31,21,28,23,30,17,31,20,14,20,23,21,23,23,17,24,21,15,16,18,15,12,4,12,8,12,11,17,12,11,17,16,8,7,6,5,12,9,7,5,3,10,4,4,5,4,6,4,4,7,4,1,2,3,3,1,2,1,1,2,2,1,1,1,1,3,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,3,3,5,2,2,4,3,1,8,3,4,5,4,1,5,9,4,3,5,6,6,3,2,6,8,6,9,9,6,7,4,8,8,11,9,7,10,11,19,8,10,11,5,12,10,12,16,13,7,9,9,13,5,7,13,5,12,14,9,4,6,5,8,7,4,5,5,9,6,1,9,2,5,7,6,1,5,2,2,7,4,3,2,5,8,5,3,3,3,3,2,6,3,5,5,1,4,2,3,1,3,1,4,3,2,1,3,2,1,2,1,1,1,3,1,2,1,3,1,7,2,1,2,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"total_sum":1006659},{"name":"handler_latency_kudu_master_MasterService_DeleteTable","total_count":1,"min":7932,"mean":7932,"percentile_75":7904,"percentile_95":7904,"percentile_99":7904,"percentile_99_9":7904,"percentile_99_99":7904,"max":7932,"values":[7935],"counts":[1],"total_sum":7932},{"name":"handler_latency_kudu_master_MasterService_ListTables","total_count":8,"min":26,"mean":41.125,"percentile_75":48,"percentile_95":60,"percentile_99":60,"percentile_99_9":60,"percentile_99_99":60,"max":60,"values":[26,32,33,47,48,51,60],"counts":[1,2,1,1,1,1,1],"total_sum":329},{"name":"handler_latency_kudu_master_MasterService_GetTableLocations","total_count":560,"min":17,"mean":70.6661,"percentile_75":84,"percentile_95":105,"percentile_99":136,"percentile_99_9":153,"percentile_99_99":174,"max":174,"values":[17,18,19,21,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,45,46,47,48,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,108,109,111,114,115,116,117,118,119,122,123,125,130,136,139,146,148,151,153,174],"counts":[1,1,1,1,2,7,8,5,3,6,4,7,4,1,2,4,5,4,8,3,2,6,2,2,2,2,4,2,1,5,1,2,2,3,1,4,1,3,1,3,6,8,11,11,16,20,11,16,22,28,19,19,11,7,4,10,8,3,7,10,3,12,18,15,17,11,9,7,6,6,5,5,1,5,6,3,7,4,3,2,7,2,3,1,3,1,2,3,1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1],"total_sum":39573},{"name":"handler_latency_kudu_master_MasterService_GetTableSchema","total_count":219,"min":7,"mean":17.9224,"percentile_75":19,"percentile_95":51,"percentile_99":76,"percentile_99_9":105,"percentile_99_99":105,"max":105,"values":[7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,29,30,32,33,34,35,40,41,42,45,46,51,52,54,55,61,62,69,76,88,105],"counts":[3,13,25,28,22,19,14,8,8,8,6,7,4,5,3,4,3,3,2,3,2,4,1,2,2,2,1,1,1,1,1,4,1,1,1,1,1,1,1,1,1],"total_sum":3925},{"name":"handler_latency_kudu_master_MasterService_ListTabletServers","total_count":7,"min":15,"mean":25.5714,"percentile_75":27,"percentile_95":31,"percentile_99":31,"percentile_99_9":31,"percentile_99_99":31,"max":31,"values":[15,26,27,28,31],"counts":[1,3,1,1,1],"total_sum":179},{"name":"handler_latency_kudu_master_MasterService_GetMasterRegistration","total_count":16,"min":7,"mean":23.0625,"percentile_75":28,"percentile_95":41,"percentile_99":42,"percentile_99_9":42,"percentile_99_99":42,"max":42,"values":[7,9,10,13,20,22,23,24,28,31,34,41,42],"counts":[1,1,1,1,2,2,2,1,1,1,1,1,1],"total_sum":369},{"name":"generic_current_allocated_bytes","value":317165072},{"name":"tcmalloc_pageheap_unmapped_bytes","value":0},{"name":"tcmalloc_max_total_thread_cache_bytes","value":536870912},{"name":"tcmalloc_current_total_thread_cache_bytes","value":3034160},{"name":"generic_heap_size","value":330260480},{"name":"tcmalloc_pageheap_free_bytes","value":1826816},{"name":"glog_info_messages","value":1969},{"name":"code_cache_hits","value":10},{"name":"code_cache_queries","value":12},{"name":"hybrid_clock_timestamp","value":6230236754827210752},{"name":"hybrid_clock_error","value":143659},{"name":"block_manager_total_bytes_read","value":57020737},{"name":"rpc_connections_accepted","value":27},{"name":"reactor_load_percent","total_count":38384,"min":0,"mean":0.000651313,"percentile_75":0,"percentile_95":0,"percentile_99":0,"percentile_99_9":0,"percentile_99_99":2,"max":4,"values":[0,1,2,3,4],"counts":[38369,8,5,1,1],"total_sum":25},{"name":"reactor_active_latency_us","total_count":108096,"min":0,"mean":6.65955,"percentile_75":6,"percentile_95":32,"percentile_99":48,"percentile_99_9":81,"percentile_99_99":205,"max":757,"values":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,99,101,102,103,104,105,106,108,109,110,111,112,113,116,117,118,122,150,153,154,161,168,171,172,177,205,263,277,307,347,375,493,739,743,759],"counts":[38243,13085,19327,4434,3150,1973,1488,1052,840,816,715,554,588,925,1359,1988,1217,879,621,578,504,523,711,853,1147,1102,855,698,574,600,537,477,372,412,397,360,358,339,416,354,229,245,220,192,155,150,161,134,127,91,104,81,67,59,42,47,40,48,38,27,26,27,30,24,20,26,15,14,16,11,10,16,9,5,6,14,5,12,8,5,7,4,6,4,2,4,5,3,2,3,6,6,2,2,1,3,3,2,2,2,3,2,1,2,4,2,2,2,2,3,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,2],"total_sum":719871},{"name":"rpc_incoming_queue_time","total_count":9034,"min":5,"mean":36.477,"percentile_75":49,"percentile_95":62,"percentile_99":72,"percentile_99_9":89,"percentile_99_99":204,"max":5179,"values":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,89,91,92,93,98,99,150,204,5179],"counts":[12,42,98,155,182,194,170,149,171,199,132,138,122,145,142,145,133,136,153,135,145,150,131,124,126,126,127,120,144,132,150,125,123,133,141,180,174,153,156,152,220,228,226,389,376,287,200,139,133,129,138,119,105,85,72,69,62,56,44,59,46,40,40,29,30,23,19,22,14,12,9,10,5,3,3,1,3,3,4,1,2,2,2,1,2,1,1,1,1,1,1,1],"total_sum":329533},{"name":"block_cache_inserts","value":2424},{"name":"block_cache_lookups","value":30244},{"name":"block_cache_misses_caching","value":2424},{"name":"block_cache_hits_caching","value":27809},{"name":"block_cache_usage","value":57656654},{"name":"spinlock_contention_time","value":0},{"name":"threads_started","value":520},{"name":"threads_running","value":227},{"name":"cpu_utime","value":5077},{"name":"cpu_stime","value":1835},{"name":"involuntary_context_switches","value":628},{"name":"voluntary_context_switches","value":111741}]}]