be/src/common/phdr_cache.cpp

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/ClickHouse/ClickHouse/blob/master/src/base/phdr_cache.cpp
// and modified by Doris

#include "common/phdr_cache.h"

#if defined(__clang__)
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif

/// This code was based on the code by Fedor Korotkiy https://www.linkedin.com/in/fedor-korotkiy-659a1838/

#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL)
#define USE_PHDR_CACHE 1
#endif

/// Thread Sanitizer uses dl_iterate_phdr function on initialization and fails if we provide our own.
#ifdef USE_PHDR_CACHE

#if defined(__clang__)
#pragma clang diagnostic ignored "-Wreserved-id-macro"
#pragma clang diagnostic ignored "-Wunused-macros"
#endif

#define __msan_unpoison(X, Y) // NOLINT
#if defined(__clang__) && defined(__has_feature)
#if __has_feature(memory_sanitizer)
#undef __msan_unpoison
#include <sanitizer/msan_interface.h>
#endif
#endif

#include <dlfcn.h>
#include <link.h>

#include <algorithm>
#include <array>
#include <atomic>
#include <cstddef>
#include <cstring>
#include <limits>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>

#if defined(USE_UNWIND) && USE_UNWIND
#ifndef UNW_LOCAL_ONLY
#define UNW_LOCAL_ONLY
#endif
#include <libunwind.h>
#endif

namespace {

// This is adapted from
// https://github.com/scylladb/seastar/blob/master/core/exception_hacks.hh
// https://github.com/scylladb/seastar/blob/master/core/exception_hacks.cc

constexpr size_t MAX_PHDR_CACHE_LOADED_OBJECTS = 4096;
constexpr size_t MAX_PHDR_CACHE_PROGRAM_HEADERS = 128;
constexpr size_t MAX_PHDR_CACHE_OBJECT_NAME = 4096;

using DLIterateFunction = int (*)(int (*callback)(dl_phdr_info* info, size_t size, void* data),
                                  void* data);

DLIterateFunction getOriginalDLIteratePHDR() {
    void* func = dlsym(RTLD_NEXT, "dl_iterate_phdr");
    if (!func) {
        throw std::runtime_error("Cannot find dl_iterate_phdr function with dlsym");
    }
    return reinterpret_cast<DLIterateFunction>(func);
}

struct RawPHDRCacheEntry {
    dl_phdr_info info {};
    std::array<ElfW(Phdr), MAX_PHDR_CACHE_PROGRAM_HEADERS> phdrs {};
    size_t phdr_count = 0;
    std::array<char, MAX_PHDR_CACHE_OBJECT_NAME> name {};
    uintptr_t address_begin = std::numeric_limits<uintptr_t>::max();
    uintptr_t address_end = 0;
};

struct RawPHDRCacheSnapshot {
    std::array<RawPHDRCacheEntry, MAX_PHDR_CACHE_LOADED_OBJECTS> entries {};
    size_t size = 0;
    bool overflow = false;
    bool phdr_truncated = false;
    bool name_truncated = false;
};

struct PHDRCacheEntry {
    dl_phdr_info info {};
    std::vector<ElfW(Phdr)> phdrs;
    std::string name;
    uintptr_t address_begin = std::numeric_limits<uintptr_t>::max();
    uintptr_t address_end = 0;

    bool contains(uintptr_t ip) const {
        return ip == 0 || (address_begin <= ip && ip < address_end);
    }
};

using PHDRCache = std::vector<PHDRCacheEntry>;
std::atomic<PHDRCache*> phdr_cache {};
// This flag is flipped inside the stack-trace signal handler. Force a static TLS access model so
// reading it from our dl_iterate_phdr interposer does not call into the dynamic loader's TLS path.
__thread bool use_phdr_cache __attribute__((tls_model("initial-exec"))) = false;

uintptr_t saturated_segment_end(uintptr_t begin, uintptr_t size) {
    const uintptr_t max_address = std::numeric_limits<uintptr_t>::max();
    return begin > max_address - size ? max_address : begin + size;
}

void copy_object_name(const char* source, RawPHDRCacheEntry* entry,
                      RawPHDRCacheSnapshot* snapshot) {
    if (source == nullptr) {
        entry->name[0] = '\0';
        return;
    }

    size_t length = 0;
    while (length + 1 < entry->name.size() && source[length] != '\0') {
        entry->name[length] = source[length];
        ++length;
    }
    entry->name[length] = '\0';
    if (source[length] != '\0') {
        snapshot->name_truncated = true;
    }
}

int collectPHDRCacheEntry(dl_phdr_info* info, size_t /*size*/, void* data) {
    auto* snapshot = reinterpret_cast<RawPHDRCacheSnapshot*>(data);
    if (snapshot->size >= snapshot->entries.size()) {
        snapshot->overflow = true;
        return 0;
    }

    auto& entry = snapshot->entries[snapshot->size++];
    entry.info = *info;
    copy_object_name(info->dlpi_name, &entry, snapshot);

    const size_t phdr_count = std::min<size_t>(info->dlpi_phnum, entry.phdrs.size());
    if (phdr_count < info->dlpi_phnum) {
        snapshot->phdr_truncated = true;
    }
    entry.phdr_count = phdr_count;
    entry.info.dlpi_phnum = static_cast<ElfW(Half)>(phdr_count);
    entry.info.dlpi_name = nullptr;
    entry.info.dlpi_phdr = nullptr;

    if (info->dlpi_phdr == nullptr) {
        return 0;
    }
    std::memcpy(entry.phdrs.data(), info->dlpi_phdr, phdr_count * sizeof(ElfW(Phdr)));

    for (size_t i = 0; i < phdr_count; ++i) {
        const auto& phdr = entry.phdrs[i];
        if (phdr.p_type != PT_LOAD || phdr.p_memsz == 0) {
            continue;
        }
        const auto begin = static_cast<uintptr_t>(info->dlpi_addr + phdr.p_vaddr);
        const auto end = saturated_segment_end(begin, static_cast<uintptr_t>(phdr.p_memsz));
        entry.address_begin = std::min(entry.address_begin, begin);
        entry.address_end = std::max(entry.address_end, end);
    }
    return 0;
}

PHDRCache* buildPHDRCache(const RawPHDRCacheSnapshot& snapshot) {
    auto* cache = new PHDRCache;
    cache->reserve(snapshot.size);
    for (size_t i = 0; i < snapshot.size; ++i) {
        const auto& raw_entry = snapshot.entries[i];
        PHDRCacheEntry entry;
        entry.info = raw_entry.info;
        entry.phdrs.assign(raw_entry.phdrs.begin(), raw_entry.phdrs.begin() + raw_entry.phdr_count);
        entry.name = raw_entry.name.data();
        entry.address_begin = raw_entry.address_begin;
        entry.address_end = raw_entry.address_end;
        cache->emplace_back(std::move(entry));
    }
    for (auto& entry : *cache) {
        entry.info.dlpi_phdr = entry.phdrs.data();
        entry.info.dlpi_name = entry.name.c_str();
    }
    return cache;
}

int iteratePHDRCache(int (*callback)(dl_phdr_info* info, size_t size, void* data), void* data,
                     uintptr_t ip) {
    auto* current_phdr_cache = phdr_cache.load(std::memory_order_acquire);
    if (current_phdr_cache == nullptr) {
        return 0;
    }

    int result = 0;
    for (auto& entry : *current_phdr_cache) {
        if (!entry.contains(ip)) {
            continue;
        }
        result = callback(&entry.info, sizeof(dl_phdr_info), data);
        if (result != 0) {
            break;
        }
    }
    return result;
}

} // namespace

extern "C"
#ifndef __clang__
        [[gnu::visibility("default")]] [[gnu::externally_visible]]
#endif
        int
        dl_iterate_phdr(int (*callback)(dl_phdr_info* info, size_t size, void* data), void* data) {
    if (!use_phdr_cache) {
        return getOriginalDLIteratePHDR()(callback, data);
    }

    return iteratePHDRCache(callback, data, 0);
}

extern "C"
#ifndef __clang__
        [[gnu::visibility("default")]] [[gnu::externally_visible]]
#endif
        int
        doris_unwind_iterate_phdr(int (*callback)(dl_phdr_info* info, size_t size, void* data),
                                  void* data, uintptr_t ip) {
    return iteratePHDRCache(callback, data, ip);
}

#include "util/debug/leak_annotations.h"

void updatePHDRCache() {
    // Fill out ELF header cache for access without locking.
    // Old snapshots are intentionally kept alive because another thread may already be unwinding
    // through the previous cache when a Doris-controlled dlopen/dlclose refreshes this one.

    auto raw_snapshot = std::make_unique<RawPHDRCacheSnapshot>();
    getOriginalDLIteratePHDR()(
            [](dl_phdr_info* info, size_t size, void* data) {
                // `info` is created by dl_iterate_phdr, which is a non-instrumented
                // libc function, so we have to unpoison it manually.
                __msan_unpoison(info, sizeof(*info));

                return collectPHDRCacheEntry(info, size, data);
            },
            raw_snapshot.get());

    PHDRCache* new_phdr_cache = buildPHDRCache(*raw_snapshot);
    phdr_cache.store(new_phdr_cache, std::memory_order_release);

    /// Memory is intentionally leaked.
    ANNOTATE_LEAKING_OBJECT_PTR(new_phdr_cache);
}

bool hasPHDRCache() {
    return phdr_cache.load(std::memory_order_acquire) != nullptr;
}

void configureLibunwindPHDRCache() {
#if defined(USE_UNWIND) && USE_UNWIND
    static std::once_flag once;
    std::call_once(once, [] {
        unw_context_t context;
        unw_cursor_t cursor;
        (void)unw_getcontext(&context);
        (void)unw_init_local(&cursor, &context);
        // Doris-patched libunwind gets FDEs from the PHDR snapshot. Disable the global DWARF
        // register-state cache so a signal handler cannot self-deadlock on libunwind's cache mutex
        // after interrupting a thread that was already unwinding.
        (void)unw_set_caching_policy(unw_local_addr_space, UNW_CACHE_NONE);
    });
#endif
}

ScopedPHDRCacheRead::ScopedPHDRCacheRead() : _previous(use_phdr_cache) {
    use_phdr_cache = true;
}

ScopedPHDRCacheRead::~ScopedPHDRCacheRead() {
    use_phdr_cache = _previous;
}

#else

void updatePHDRCache() {}

void configureLibunwindPHDRCache() {}

#if defined(USE_MUSL)
/// With statically linked with musl, dl_iterate_phdr is immutable.
bool hasPHDRCache() {
    return true;
}
#else
bool hasPHDRCache() {
    return false;
}
#endif

ScopedPHDRCacheRead::ScopedPHDRCacheRead() = default;

ScopedPHDRCacheRead::~ScopedPHDRCacheRead() = default;

#endif