chDB 内核升级全记录：ClickHouse 升级至 v25.8.2.29

chdb/├── chdb/                    # Python package│   ├── __init__.py          # Main query interface│   ├── session/             # Session management│   ├── dbapi/               # DB-API 2.0 implementation│   └── udf/                 # User-defined functions├── programs/local/          # Local query engine│   ├── LocalChdb.cpp        # chDB main entry│   ├── PythonSource.cpp     # Python Table Engine│   └── PandasDataFrame.cpp  # Pandas integration├── src/                     # ClickHouse core source└── contrib/                 # Third-party dependencies

# Build independent binding libraries for Python 3.8-3.13for version in 3.8 3.9 3.10 3.11 3.12 3.13; do    cmake -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${version} ..    ninja pybind11nonlimitedapi_chdb_${version}done

Program received signal SIGSEGV, Segmentation fault.0x00007ffff7a9e123 in je_free () from /path/to/_chdb.abi3.so(gdb) bt#0  je_free (ptr=0x7fffe8001000)#1  __wrap_free (ptr=0x7fffe8001000) at AllocationInterceptors.cpp:451#2  0x00007ffff7e8a456 in PyMem_Free (ptr=0x7fffe8001000)#3  0x00007ffff7dab234 in list_dealloc ()

// Scenario 1: Objects returned from Python C APIPyObject* obj = PyList_New(10);  // Python uses malloc// ... used in C++delete obj;  // chDB's operator delete uses je_free → crash!// Scenario 2: Memory allocated by glibc functionschar* cwd = getcwd(NULL, 0);  // glibc internally uses malloc// ...free(cwd);  // wrapped to je_free → crash!// Scenario 3: Third-party libraries (e.g., libhdfs)hdfsFile file = hdfsOpenFile(...);  // libhdfs uses system mallochdfsCloseFile(fs, file);  // internally calls free, wrapped → crash!

-DENABLE_JEMALLOC=0

LD_PRELOAD=/usr/lib/libjemalloc.so python chdb_example.py

# Add wrap parameters at link time-Wl,-wrap,malloc-Wl,-wrap,free-Wl,-wrap,calloc-Wl,-wrap,realloc# ... other memory allocation functions

Application code calls:  free(ptr)      ↓Linker redirects:  __wrap_free(ptr)  ← Our implementation      ↓Callback when needed:  __real_free(ptr)  ← Original glibc implementation

# ClickHouse default config: Rename jemalloc symbols--with-jemalloc-prefix=je_# Effect: malloc → je_malloc, free → je_free# chDB config: Disable renaming-DJEMALLOC_PREFIX=""# Effect: Preserve je_malloc, je_free and other original symbol names

inline bool isJemallocMemory(void * ptr) {    // Query which arena this memory block belongs to    int arena_ind;    size_t sz = sizeof(arena_ind);    int ret = je_mallctl("arenas.lookup",                         &arena_ind, &sz,     // Output: arena index                         &ptr, sizeof(ptr));  // Input: memory pointer    // arena_ind == 0: special value, indicates memory doesn't belong to jemalloc    // arena_ind > 0:  memory belongs to a jemalloc arena    return ret == 0 && arena_ind != 0;}

inline ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) {    if (unlikely(ptr == nullptr))        return true;    // Check memory source    int arena_ind;    size_t sz = sizeof(arena_ind);    int ret = je_mallctl("arenas.lookup", &arena_ind, &sz, &ptr, sizeof(ptr));    if (ret == 0 && arena_ind == 0) {        // This memory doesn't belong to jemalloc, use system free        __real_free(ptr);        return true;  // Handled, no need to continue    }    // This memory belongs to jemalloc, or query failed (conservative handling)    return false;  // Continue with jemalloc release process}extern "C" void __wrap_free(void * ptr) {    if (tryFreeNonJemallocMemory(ptr))        return;    // Use jemalloc to release    AllocationTrace trace;    size_t actual_size = Memory::untrackMemory(ptr, trace);    trace.onFree(ptr, actual_size);    je_free(ptr);}

// Before: Assumes pointer is valid, directly accesses metadata → invalid pointer crashes// After: First checks pointer validity, safely returns resultif (ptr == NULL || !isValidPointer(ptr)) {    return EINVAL;  // Return error code instead of crashing}

chDB's wrap (introduced in Section 2):  free(ptr) → __wrap_free() (chDB implementation)            → Check memory source (je_mallctl)            → Route to je_free or __real_freeClickHouse new version's wrap:  free(ptr) → __wrap_free() (ClickHouse implementation)            → Update MemoryTracker statistics            → Call je_freeConflict:  Two __wrap_free implementations cannot coexist!

extern "C" void __wrap_free(void * ptr) // NOLINT{#if USE_JEMALLOC    // chDB logic: Check if it's non-jemalloc memory    if (tryFreeNonJemallocMemory(ptr))        return;  // Already handled glibc-allocated memory via __real_free#endif    // ClickHouse logic: Update MemoryTracker statistics    size_t actual_size = Memory::untrackMemory(ptr, trace);#if USE_JEMALLOC    // Use jemalloc to release    je_free(ptr);#else    // If jemalloc not enabled, use system free    __real_free(ptr);#endif}

libpybind11nonlimitedapi_chdb_3.8.so:  - Links to glibc at compile time  - Calls malloc → Bound to glibc malloc (compile-time symbol binding)  - Calls delete during object destruction_chdb.abi3.so:  - Contains jemalloc and operator delete implementation  - libpybind11*.so loads _chdb.abi3.so at runtime  - delete call → Resolves to definition in _chdb.abi3.so at runtime

In libpybind11*.so:  char* obj = malloc(100);  // malloc() → glibc malloc (compile-time binding)  delete obj;            // operator delete → _chdb.abi3.so implementation (runtime binding)                         // Tries to use jemalloc to free glibc-allocated memory → crash!

void operator delete(void * ptr) noexcept{#if USE_JEMALLOC    // Detect memory source, handle non-jemalloc memory early    if (tryFreeNonJemallocMemory(ptr))        return;#endif    // ClickHouse memory tracking    Memory::untrackMemory(ptr, trace);    // Actual release (jemalloc memory)    Memory::deleteImpl(ptr);}

SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\\\\.)?([^/]+)/.*$', '\\\\1') AS k,       AVG(length(Referer)) AS l,       COUNT(*) AS c,       MIN(Referer)FROM clickbench.hitsWHERE Referer <> ''GROUP BY kHAVING COUNT(*) > 100000ORDER BY l DESCLIMIT 25;

// Pre-optimization codeinline bool tryFreeNonJemallocMemory(void * ptr){    if (unlikely(ptr == nullptr))        return true;    // Key bottleneck: je_mallctl lock contention    int arena_ind = je_mallctl("arenas.lookup", nullptr, nullptr, &ptr, sizeof(ptr));    if (unlikely(arena_ind != 0))    {        __real_free(ptr);        return true;    }    return false;}

int je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,    size_t newlen) {		...    check_entry_exit_locking(tsd_tsdn(tsd));  // Lock contention point!    ...    return ret;}

operator delete  └─ tryFreeNonJemallocMemory      └─ je_mallctl          └─ check_entry_exit_locking  ← Hotspot!              └─ pthread_mutex_lock

namespace Memory {    thread_local bool disable_memory_check{false};  // Disable checking by default inside engine}// Version with conditional checkinginline ALWAYS_INLINE bool tryFreeNonJemallocMemoryConditional(void * ptr) {    if (unlikely(ptr == nullptr))        return true;    // Fast path: Skip checking directly inside engine    if (likely(Memory::disable_memory_check))        return false;  // Continue normal jemalloc release process    ...}// Updated operator deletevoid operator delete(void * ptr) noexcept{#if USE_JEMALLOC    if (tryFreeNonJemallocMemoryConditional(ptr))  // Use conditional version        return;#endif		...}

// RAII helper classstruct MemoryCheckScope {    MemoryCheckScope() {        Memory::disable_memory_check = false;  // Enable checking    }    ~MemoryCheckScope() {        Memory::disable_memory_check = true;   // Restore disabled    }};// Use at Python interaction pointsvoid convertPandasDataFrame(...) {    MemoryCheckScope scope;  // Enter Python boundary, enable checking    // Call Python C API, may use Python's memory allocator    PyObject* obj = PyList_GetItem(...);    // ...    // Leave scope, automatically restore disabled state}

// Before optimization: Every delete calls je_mallctlif (tryFreeNonJemallocMemory(ptr))  // ~500 CPU cycles    return;// After optimization: Most deletes directly skipif (likely(Memory::disable_memory_check))  // ~2 CPU cycles    return false;

After optimization (Q29):  89.2%  Regular expression engine    ← As expected  7.1%   String operations and memory allocation  2.3%   Aggregation and sorting  1.4%   Other

# ClickHouse Local$ time clickhouse local -q "SELECT COUNT(*) FROM file('data.parquet', Parquet)"# 1.734 seconds# chDB$ time python -c "import chdb; chdb.query('SELECT COUNT(*) FROM file(\\\\"data.parquet\\\\", Parquet)')"# 2.203 seconds

import chdbimport timet0 = time.time()import chdb  # Load libraryt1 = time.time()result = chdb.query("SELECT COUNT(*) FROM file('data.parquet', Parquet)")t2 = time.time()print(f"Import time: {t1-t0:.2f}s")      # 0.58sprint(f"Query time:  {t2-t1:.2f}s")      # 1.62sprint(f"Total time:  {t2-t0:.2f}s")      # 2.20sprint(f"Query elapsed: {result.elapsed():.2f}s")  # 1.60s

$ ldd _chdb.abi3.so    linux-vdso.so.1 (0x00007fff)    libpybind11nonlimitedapi_chdb_3.8.so => ./libpybind11...    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6$ ls -lh _chdb.abi3.so-rwxr-xr-x 1 user user 642M _chdb.abi3.so$ nm -D _chdb.abi3.so | wc -l540000  # 540,000 exported symbols

# Create session once, query multiple timessess = chdb.session.Session()# First query bears startup overheadresult1 = sess.query("SELECT * FROM file('data1.parquet', Parquet)")  # 2.2s# Subsequent queries have no startup overheadresult2 = sess.query("SELECT * FROM file('data2.parquet', Parquet)")  # 1.6sresult3 = sess.query("SELECT * FROM file('data3.parquet', Parquet)")  # 1.5s