support clean metrics of OrphanFilesCleaner (#118)
diff --git a/include/paimon/orphan_files_cleaner.h b/include/paimon/orphan_files_cleaner.h index 411314a..e9c6859 100644 --- a/include/paimon/orphan_files_cleaner.h +++ b/include/paimon/orphan_files_cleaner.h
@@ -184,6 +184,11 @@ /// files. virtual Result<std::set<std::string>> Clean() = 0; + /// Retrieve metrics related to orphan files cleaning operations. + /// + /// @return A shared pointer to a `Metrics` object containing cleaning metrics. + virtual std::shared_ptr<Metrics> GetMetrics() const = 0; + protected: OrphanFilesCleaner() = default; };
diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 109bc37..a9c4a4d 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp
@@ -62,6 +62,7 @@ #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/fs/file_system.h" @@ -379,7 +380,7 @@ int32_t attempt = 0; int32_t generated_snapshot = 0; - const auto started = std::chrono::high_resolution_clock::now(); + Duration duration; if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) { PAIMON_ASSIGN_OR_RAISE(int32_t cnt, TryCommit(append_table_files, append_table_index_files, @@ -403,10 +404,7 @@ compaction_input_file_size += entry.File()->file_size; } } - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::high_resolution_clock::now() - started) - .count()); + metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get()); metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt); metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added); metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted);
diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 34987d1..9fb9ed3 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp
@@ -39,6 +39,7 @@ #include "paimon/core/operation/metrics/scan_metrics.h" #include "paimon/core/partition/partition_info.h" #include "paimon/core/stats/simple_stats.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/predicate/literal.h" @@ -94,7 +95,7 @@ } Result<std::shared_ptr<FileStoreScan::RawPlan>> FileStoreScan::CreatePlan() const { - const auto started = std::chrono::high_resolution_clock::now(); + Duration duration; std::optional<Snapshot> snapshot; std::vector<ManifestFileMeta> all_manifest_file_metas; std::vector<ManifestFileMeta> filtered_manifest_file_metas; @@ -133,10 +134,7 @@ [](const int64_t sum, const ManifestFileMeta& manifest_file_meta) { return sum + manifest_file_meta.NumAddedFiles() - manifest_file_meta.NumDeletedFiles(); }); - metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION, - std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::high_resolution_clock::now() - started) - .count()); + metrics_->SetCounter(ScanMetrics::LAST_SCAN_DURATION, duration.Get()); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID, snapshot.has_value() ? snapshot.value().Id() : int64_t{0}); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size());
diff --git a/src/paimon/core/operation/metrics/clean_metrics.h b/src/paimon/core/operation/metrics/clean_metrics.h new file mode 100644 index 0000000..e898783 --- /dev/null +++ b/src/paimon/core/operation/metrics/clean_metrics.h
@@ -0,0 +1,35 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace paimon { + +/// Metrics to measure clean operation. +class CleanMetrics { + public: + static constexpr char CLEAN_DURATION[] = "cleanDuration"; + static constexpr char CLEAN_LIST_DIRECTORIES_DURATION[] = "listDirectoriesDuration"; + static constexpr char CLEAN_LIST_DIRECTORIES[] = "listDirectories"; + static constexpr char CLEAN_LIST_FILE_STATUS_DURATION[] = "listFileStatusDuration"; + static constexpr char CLEAN_LIST_FILE_STATUS_TASKS[] = "listFileStatusTasks"; + static constexpr char CLEAN_LIST_USED_FILES_DURATION[] = "listUsedFilesDuration"; + static constexpr char CLEAN_USED_FILES[] = "usedFiles"; + static constexpr char CLEAN_SCAN_ORPHAN_FILES_DURATION[] = "scanOrphanFilesDuration"; + static constexpr char CLEAN_ORPHAN_FILES[] = "orphanFiles"; +}; + +} // namespace paimon
diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp index 740a629..874a8c5 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.cpp
@@ -25,6 +25,7 @@ #include "fmt/format.h" #include "paimon/common/executor/future.h" +#include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -32,7 +33,9 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/metrics/clean_metrics.h" #include "paimon/core/snapshot.h" +#include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/status.h" @@ -61,7 +64,8 @@ manifest_file_(manifest_file), manifest_list_(manifest_list), older_than_ms_(older_than_ms), - should_be_retained_(should_be_retained) {} + should_be_retained_(should_be_retained), + metrics_(std::make_shared<MetricsImpl>()) {} bool OrphanFilesCleanerImpl::SupportToClean(const std::string& file_name) { static std::vector<std::pair<std::string, std::string>> supported_pattern = { @@ -83,6 +87,7 @@ } Result<std::set<std::string>> OrphanFilesCleanerImpl::Clean() { + Duration main_duration; if (!MinimalTryBestListingDirs(PathUtil::JoinPath(root_path_, "tag")).empty()) { return Status::NotImplemented("OrphanFilesCleaner do not support cleaning table with tag"); } @@ -98,9 +103,11 @@ } PAIMON_ASSIGN_OR_RAISE(std::set<std::string> used_file_names, GetUsedFiles()); + Duration duration; std::set<std::string> need_to_deletes; std::vector<std::future<void>> futures; ScopeGuard guard([&futures]() { Wait(futures); }); + uint64_t file_statuses_duration = duration.Reset(); for (const auto& file_statuses : CollectAll(file_statuses_futures)) { for (const auto& file_status : file_statuses) { if (file_status->IsDir()) { @@ -129,10 +136,18 @@ } } } + metrics_->SetCounter(CleanMetrics::CLEAN_DURATION, main_duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_SCAN_ORPHAN_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_DURATION, file_statuses_duration); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_FILE_STATUS_TASKS, + static_cast<uint64_t>(file_statuses_futures.size())); + metrics_->SetCounter(CleanMetrics::CLEAN_ORPHAN_FILES, + static_cast<uint64_t>(need_to_deletes.size())); return need_to_deletes; } Result<std::set<std::string>> OrphanFilesCleanerImpl::ListPaimonFileDirs() const { + Duration duration; std::set<std::string> paimon_file_dirs; paimon_file_dirs.insert(snapshot_manager_->SnapshotDirectory()); paimon_file_dirs.insert(FileStorePathFactory::ManifestPath(root_path_)); @@ -156,6 +171,9 @@ // ListFileDirs(external_path, partition_keys_.size()); // paimon_file_dirs.insert(external_file_dirs.begin(), external_file_dirs.end()); // } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_DIRECTORIES, + static_cast<uint64_t>(paimon_file_dirs.size())); return paimon_file_dirs; } @@ -225,6 +243,7 @@ // TODO(jinli.zjw): consider changelog(add tests), stats used_files.insert(SnapshotManager::EARLIEST); used_files.insert(SnapshotManager::LATEST); + Duration duration; PAIMON_ASSIGN_OR_RAISE(std::vector<Snapshot> snapshots, snapshot_manager_->GetAllSnapshots()); for (const auto& snapshot : snapshots) { used_files.insert(SnapshotManager::SNAPSHOT_PREFIX + std::to_string(snapshot.Id())); @@ -257,6 +276,8 @@ } } } + metrics_->SetCounter(CleanMetrics::CLEAN_LIST_USED_FILES_DURATION, duration.Get()); + metrics_->SetCounter(CleanMetrics::CLEAN_USED_FILES, static_cast<uint64_t>(used_files.size())); return used_files; }
diff --git a/src/paimon/core/operation/orphan_files_cleaner_impl.h b/src/paimon/core/operation/orphan_files_cleaner_impl.h index 14be18d..f499182 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_impl.h +++ b/src/paimon/core/operation/orphan_files_cleaner_impl.h
@@ -28,6 +28,7 @@ #include "paimon/core/snapshot.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/metrics.h" #include "paimon/orphan_files_cleaner.h" #include "paimon/result.h" @@ -62,6 +63,10 @@ Result<std::set<std::string>> Clean() override; + std::shared_ptr<Metrics> GetMetrics() const override { + return metrics_; + } + private: Result<std::set<std::string>> ListPaimonFileDirs() const; std::vector<std::unique_ptr<FileStatus>> TryBestListingDirs(const std::string& path) const; @@ -86,5 +91,7 @@ std::shared_ptr<ManifestList> manifest_list_; int64_t older_than_ms_; std::function<bool(const std::string&)> should_be_retained_; + + std::shared_ptr<Metrics> metrics_; }; } // namespace paimon
diff --git a/src/paimon/core/utils/duration.h b/src/paimon/core/utils/duration.h new file mode 100644 index 0000000..d5b64db --- /dev/null +++ b/src/paimon/core/utils/duration.h
@@ -0,0 +1,44 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <chrono> + +namespace paimon { + +// Calculate operation duration. +class Duration { + public: + Duration() : start_(std::chrono::high_resolution_clock::now()) {} + + uint64_t Get() { + return std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::high_resolution_clock::now() - start_) + .count(); + } + + uint64_t Reset() { + uint64_t dura = Get(); + start_ = std::chrono::high_resolution_clock::now(); + return dura; + } + + private: + std::chrono::high_resolution_clock::time_point start_; +}; + +} // namespace paimon