Parquet at the
speed of C.

The simplest path: clone, make, install. You get the static library, the headers, the carquet CLI, a pkg-config file and find_package(carquet) support.

$ git clone https://github.com/Vitruves/carquet.git && cd carquet
$ make                    # optimized build (make native for -march=native)
$ sudo make install       # /usr/local, or make install PREFIX=/opt/carquet

$ cc app.c $(pkg-config --cflags --libs carquet)

Prebuilt wheels bundle the library for Linux, macOS and Windows. Tables come back as pyarrow.Table through the Arrow C Data Interface.

$ pip install carquet

Pull a pinned release straight into your CMake project, no install step.

include(FetchContent)
set(CARQUET_BUILD_CLI OFF CACHE BOOL "" FORCE)
FetchContent_Declare(carquet
    GIT_REPOSITORY https://github.com/Vitruves/carquet.git
    GIT_TAG        v0.7.3)
FetchContent_MakeAvailable(carquet)
target_link_libraries(app PRIVATE carquet)

Drive CMake directly for custom options: shared library, -march=native, Brotli off, your own zstd/zlib/lz4 targets.

$ cmake -B build -DCMAKE_BUILD_TYPE=Release -DCARQUET_BUILD_SHARED=ON
$ cmake --build build -j
$ sudo cmake --install build

An equivalent xmake build with the same options and defaults.

$ xmake                      # static library + CLI, release
$ xmake f --native_arch=y    # optional: -march=native
$ xmake install -o /usr/local

Presets for MSVC, clang-cl and MinGW-w64, static or shared. Run from a Developer PowerShell.

PS> cmake --preset msvc
PS> cmake --build --preset msvc
PS> ctest --preset msvc

Define a schema, then hand each column a plain C array. Nullable columns take a definition level per row and only the present values.

#include <carquet/carquet.h>
#include <stdio.h>

#define TRY(x) do { if ((x) != CARQUET_OK) return 1; } while (0)

int main(void) {
    carquet_error_t err = CARQUET_ERROR_INIT;
    carquet_logical_type_t utf8 = { .id = CARQUET_LOGICAL_STRING };

    carquet_schema_t* schema = carquet_schema_create(&err);
    TRY(carquet_schema_add_column(schema, "id",    CARQUET_PHYSICAL_INT64,      NULL,  CARQUET_REPETITION_REQUIRED, 0, 0));
    TRY(carquet_schema_add_column(schema, "name",  CARQUET_PHYSICAL_BYTE_ARRAY, &utf8, CARQUET_REPETITION_REQUIRED, 0, 0));
    TRY(carquet_schema_add_column(schema, "score", CARQUET_PHYSICAL_DOUBLE,     NULL,  CARQUET_REPETITION_OPTIONAL, 0, 0));

    carquet_writer_options_t opts;
    carquet_writer_options_init(&opts);
    opts.compression      = CARQUET_COMPRESSION_ZSTD;
    opts.write_page_index = true;               /* lets readers skip pages */

    carquet_writer_t* w = carquet_writer_create("people.parquet", schema, &opts, &err);
    if (!w) { fprintf(stderr, "%s\n", err.message); return 1; }

    int64_t ids[] = {1, 2, 3};
    carquet_byte_array_t names[] = {{(uint8_t*)"ada", 3}, {(uint8_t*)"linus", 5}, {(uint8_t*)"grace", 5}};
    double  scores[] = {9.5, 7.25};              /* present values only */
    int16_t defs[]   = {1, 0, 1};                /* row 2 is NULL */

    TRY(carquet_writer_write_batch(w, 0, ids,    3, NULL, NULL));
    TRY(carquet_writer_write_batch(w, 1, names,  3, NULL, NULL));
    TRY(carquet_writer_write_batch(w, 2, scores, 3, defs, NULL));
    TRY(carquet_writer_close(w));
    carquet_schema_free(schema);
    return 0;
}

The batch reader decodes only the columns you ask for, in parallel, and hands you typed pointers plus an Arrow-style validity bitmap.

#include <carquet/carquet.h>
#include <stdio.h>

#define TRY(x) do { if ((x) != CARQUET_OK) return 1; } while (0)

int main(void) {
    carquet_error_t err = CARQUET_ERROR_INIT;
    carquet_reader_t* r = carquet_reader_open("people.parquet", NULL, &err);
    if (!r) { fprintf(stderr, "%s\n", err.message); return 1; }

    carquet_batch_reader_config_t cfg;
    carquet_batch_reader_config_init(&cfg);
    const char* cols[] = {"name", "score"};     /* only these are decoded */
    cfg.column_names     = cols;
    cfg.num_column_names = 2;
    cfg.batch_size       = 65536;

    carquet_batch_reader_t* br = carquet_batch_reader_create(r, &cfg, &err);
    carquet_row_batch_t* batch = NULL;
    while (carquet_batch_reader_next(br, &batch) == CARQUET_OK && batch) {
        const void *names, *scores; const uint8_t *nn, *valid; int64_t n;
        TRY(carquet_row_batch_column(batch, 0, &names,  &nn,    &n));
        TRY(carquet_row_batch_column(batch, 1, &scores, &valid, &n));
        const carquet_byte_array_t* s = names;
        for (int64_t i = 0; i < n; i++) {
            if (valid && !(valid[i / 8] & (1u << (i % 8))))
                printf("%.*s  NULL\n", (int)s[i].length, s[i].data);
            else
                printf("%.*s  %.2f\n", (int)s[i].length, s[i].data, ((const double*)scores)[i]);
        }
        carquet_row_batch_free(batch);
        batch = NULL;
    }
    carquet_batch_reader_free(br);
    carquet_reader_close(r);
    return 0;
}

ada 9.50 · linus NULL · grace 7.25

Predicates are pushed down: row groups are pruned by statistics and bloom filters, pages by the page index, before anything is decompressed.

#include <carquet/carquet.h>
#include <stdio.h>

#define TRY(x) do { if ((x) != CARQUET_OK) return 1; } while (0)

int main(void) {
    carquet_error_t err = CARQUET_ERROR_INIT;
    carquet_reader_t* r = carquet_reader_open("people.parquet", NULL, &err);
    if (!r) { fprintf(stderr, "%s\n", err.message); return 1; }

    carquet_batch_reader_config_t cfg;
    carquet_batch_reader_config_init(&cfg);
    carquet_batch_reader_t* br = carquet_batch_reader_create(r, &cfg, &err);

    /* WHERE id >= 2: row groups and pages whose min/max cannot match are
       never decompressed (statistics, bloom filters, page index). */
    int64_t lo = 2;
    carquet_filter_clause_t where = {
        .column_index = 0, .op = CARQUET_FILTER_GE,
        .value = &lo, .value_size = sizeof lo,
    };
    TRY(carquet_batch_reader_set_page_filter(br, &where, 1));

    carquet_row_batch_t* batch = NULL;
    int64_t rows = 0;
    while (carquet_batch_reader_next(br, &batch) == CARQUET_OK && batch) {
        rows += carquet_row_batch_num_rows(batch);
        carquet_row_batch_free(batch);
        batch = NULL;
    }
    printf("read %lld rows, skipped %lld\n",
           (long long)rows, (long long)carquet_batch_reader_rows_skipped(br));
    carquet_batch_reader_free(br);
    carquet_reader_close(r);
    return 0;
}

Read and write through the standard Arrow C Data Interface, with struct, list and map at any depth. No Arrow library needed on either side of the call.

#include <carquet/carquet.h>
#include <stdio.h>

int main(void) {
    carquet_error_t err = CARQUET_ERROR_INIT;
    carquet_reader_t* r = carquet_reader_open("people.parquet", NULL, &err);
    if (!r) { fprintf(stderr, "%s\n", err.message); return 1; }

    /* One row group as a standard Arrow C Data struct array: hand it to
       Arrow, DuckDB, polars, nanoarrow... with no Arrow dependency here. */
    struct ArrowSchema schema;
    struct ArrowArray  array;
    if (carquet_reader_read_arrow(r, 0, &schema, &array, &err) == CARQUET_OK) {
        printf("%lld rows, %lld fields\n", (long long)array.length, (long long)schema.n_children);
        array.release(&array);
        schema.release(&schema);
    }
    carquet_reader_close(r);
    return 0;
}

The same engine from Python: pyarrow tables, pandas, streaming batches and filters, with the GIL released in every C call.

import carquet as cq

# Projection and predicate pushdown, straight into a pyarrow.Table
table = cq.read_table("people.parquet", columns=["id", "score"],
                      filters=[("id", ">=", 2)])

df = cq.read_pandas("people.parquet")             # pandas.DataFrame

with cq.ParquetFile("people.parquet") as pf:      # streaming, GIL released
    for batch in pf.iter_batches(batch_size=65_536, columns=["name"]):
        ...                                       # pyarrow.RecordBatch
    print(pf.count_rows(filters=[("score", "is null")]))    # 1

cq.write_table(table, "out.parquet", compression="zstd")