Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions internal/duckdb/policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,12 @@ func ConfigurePolicy(ctx context.Context, db *sql.DB, directories ...string) err
"SET allow_community_extensions=false",
"SET allowed_directories=[" + strings.Join(paths, ",") + "]",
"SET enable_external_access=false",
// UTC is set on each newly pooled connection. Public SQL cannot SET
// options; memory and worker limits remain available to the operator.
"SET allowed_configs=['memory_limit','threads','TimeZone']",
// UTC and parquet_metadata_cache are set on each newly pooled connection,
// so both must stay settable after the lock. Public SQL cannot change any
// other option; memory and worker limits remain available to the operator.
// parquet_metadata_cache only caches immutable Parquet footers, so allowing
// it does not widen file access or the configuration lockdown.
"SET allowed_configs=['memory_limit','threads','TimeZone','parquet_metadata_cache']",
"SET lock_configuration=true",
}
for _, statement := range statements {
Expand Down
11 changes: 11 additions & 0 deletions internal/engine/duck.go
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,17 @@ func openDuckDB(dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, *
if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil {
return fmt.Errorf("set timezone: %w", err)
}
// parquet_metadata_cache is a GLOBAL-scoped option, but duckdb-go applies a
// SET to the issuing connection only — it does not reach the other pooled
// connections, so it must be set on each one here rather than once at boot.
// Every read statement binds an explicit per-file Parquet list and a panel
// issues several, so caching footers by path+mtime+size avoids re-reading
// them per statement. Our batch files are immutable with unique paths, so a
// cached footer is never stale. The policy keeps it in allowed_configs so
// this SET still succeeds on connections opened after lock_configuration.
if _, err := execer.ExecContext(ctx, "SET parquet_metadata_cache=true", nil); err != nil {
return fmt.Errorf("set parquet_metadata_cache: %w", err)
}
tempDirOnce.Do(func() {
_, tempDirErr = execer.ExecContext(ctx, "SET temp_directory="+sqlLiteral(tempDir), nil)
})
Expand Down
41 changes: 41 additions & 0 deletions internal/engine/duck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,47 @@ func TestNewDuckReconnectsInUTCAfterStartupContextCancelled(t *testing.T) {
}
}

// Every pooled read connection must enable parquet_metadata_cache so repeated
// statements over the same immutable files reuse their footers instead of
// re-reading them. The setting is applied per connection and kept in the policy
// allowed_configs, so it must also hold on a connection opened after the
// configuration lock, and the lock must still refuse other configuration.
func TestReadConnectionsEnableParquetMetadataCache(t *testing.T) {
ctx := context.Background()
cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB", DuckDBThreads: 2, DuckDBMaxConns: 4}
repository, err := telemetrystore.Open(cfg.TelemetryDir())
if err != nil {
t.Fatal(err)
}
defer repository.Close()
d, err := NewDuck(ctx, cfg, repository)
if err != nil {
t.Fatalf("NewDuck() error = %v", err)
}
defer d.Close()

for name, db := range map[string]*sql.DB{"read": d.DB, "write": d.writeDB, "memory": d.memoryDB, "parse": d.parseDB} {
t.Run(name, func(t *testing.T) {
// Retire idle connections so this query runs on a freshly opened one,
// whose init hook runs after the configuration was locked.
db.SetMaxIdleConns(0)
var enabled bool
if err := db.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&enabled); err != nil {
t.Fatal(err)
}
if !enabled {
t.Fatalf("parquet_metadata_cache = false on a %s connection, want true", name)
}
})
}

// The lock still refuses configuration outside allowed_configs, so public SQL
// cannot re-open file or external access.
if _, err := d.DB.ExecContext(ctx, "SET enable_external_access=true"); err == nil {
t.Fatal("expected locked configuration to refuse SET enable_external_access")
}
}

// A cancelled statement's interrupt can reach the ROLLBACK that ends its read
// transaction, leaving the transaction open. The read must roll back itself
// after cancellation, and a connection whose rollback failed must leave the
Expand Down
254 changes: 254 additions & 0 deletions internal/engine/metadata_cache_bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,254 @@
//go:build filebench

package engine

import (
"context"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"sort"
"testing"
"time"

duckdb "github.com/duckdb/duckdb-go/v2"
"github.com/labstack/pitot/internal/config"
"github.com/labstack/pitot/internal/queryrows"
"github.com/labstack/pitot/internal/telemetry"
telemetrystore "github.com/labstack/pitot/internal/telemetry/store"
)

// A/B for parquet_metadata_cache (#484). The engine passes an explicit per-file
// Parquet list on every statement, and a panel issues several. With the footer
// cache OFF, each statement re-reads every file footer; with it ON, footers are
// cached by path+mtime+size. Both arms read the SAME immutable fixtures and run
// the same explicit-list statement; the only difference is the per-connection
// cache SET. Both engines are opened and warmed up front, then sampled with
// AB/BA alternation so OS page-cache and time drift affect both arms equally.
// This is an engine-internal control, not a product code path and not capacity
// evidence.
//
// go test -tags=filebench ./internal/engine -run '^TestParquetMetadataCacheComparisonBenchmark$' -count=1 -v
func TestParquetMetadataCacheComparisonBenchmark(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute)
defer cancel()

cfg := config.Config{DataDir: t.TempDir()}
repo, err := telemetrystore.Open(cfg.TelemetryDir())
if err != nil {
t.Fatal(err)
}
defer repo.Close()

base := time.Date(2026, 11, 1, 12, 0, 0, 123, time.UTC)
w := queryrows.Window{Start: base, End: base.Add(time.Minute)}
signal, _ := telemetry.LookupSignal("spans")

published := 0
for _, files := range []int{128, 512, 1024} {
for ; published < files; published++ {
batch := telemetrystore.Batch{ID: fmt.Sprintf("mdc_%05d", published)}
for row := range 32 {
service := []string{"api", "worker", "db"}[row%3]
n := base.Add(time.Duration(row) * time.Millisecond).UnixNano()
if row == 0 {
n = w.Start.UnixNano()
}
if row == 31 {
n = w.End.UnixNano() - 1
}
status := "STATUS_CODE_OK"
if row%7 == 0 {
status = "STATUS_CODE_ERROR"
}
span := telemetry.Span{Namespace: "prod", ServiceName: service, TraceID: fmt.Sprintf("mdc_trace_%05d_%02d", published, row/2), SpanID: fmt.Sprint(row), StartUnixNanos: n, EndUnixNanos: n + int64(row+1)*int64(time.Millisecond), DurationMS: float64(row + 1), StatusCode: status, IngestedAt: base.UnixNano() + int64(published)}
batch.Spans = append(batch.Spans, span)
}
if err := repo.Commit(ctx, batch); err != nil {
t.Fatal(err)
}
}
batches := repo.Parquet.BatchMetadata()
if len(batches) != files {
t.Fatalf("captured files=%d expected=%d", len(batches), files)
}

sort.Slice(batches, func(i, j int) bool { return batches[i].ID < batches[j].ID })
paths := make([]string, 0, len(batches))
for _, b := range batches {
paths = append(paths, sqlLiteral(filepath.ToSlash(filepath.Join(repo.Parquet.Dir(), "batches", b.ID+".batch", "spans.parquet"))))
}
physical := "SELECT " + rawProjection(signal) + " FROM read_parquet([" + joinCSV(paths) + "], union_by_name=false, hive_partitioning=false)"
statement := `SELECT service,count(*)::BIGINT,count(DISTINCT trace_id)::BIGINT,
sum(end_unix_nano-start_unix_nano)::BIGINT,
sum(CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END)::BIGINT
FROM (` + cleanSource("spans", physical) + `) WHERE start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<?::TIMESTAMP_NS::TIMESTAMPTZ_NS GROUP BY service ORDER BY service`

off := newMetadataCacheArm(t, ctx, statement, w, false)
on := newMetadataCacheArm(t, ctx, statement, w, true)

// Warm both arms (populates the footer cache for the ON arm, and the OS
// page cache for both).
for range 5 {
off.exec()
on.exec()
}

// AB/BA alternation: each arm sampled 100 times, order flips per sample.
for sample := range 100 {
for offset := range 2 {
if (sample+offset)%2 == 0 {
off.sample()
} else {
on.sample()
}
}
}

for _, arm := range []*metadataCacheArm{off, on} {
result := arm.report()
result["experiment"] = "parquet_metadata_cache"
result["files"] = files
encoded, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
t.Log("RESULT", string(encoded))
arm.close()
}
}
}

func joinCSV(items []string) string {
out := ""
for i, s := range items {
if i > 0 {
out += ","
}
out += s
}
return out
}

type mdcConnector struct {
*duckdb.Connector
cache bool
}

func (c *mdcConnector) Connect(ctx context.Context) (driver.Conn, error) {
conn, err := c.Connector.Connect(ctx)
if err != nil {
return nil, err
}
ex := conn.(driver.ExecerContext)
stmts := []string{"LOAD parquet", "SET TimeZone='UTC'"}
if c.cache {
stmts = append(stmts, "SET parquet_metadata_cache=true")
}
for _, stmt := range stmts {
if _, err := ex.ExecContext(ctx, stmt, nil); err != nil {
return nil, errors.Join(err, conn.Close())
}
}
return conn, nil
}

type metadataCacheArm struct {
t *testing.T
ctx context.Context
pool *sql.DB
conn *sql.Conn
tx *sql.Tx
statement string
w queryrows.Window
cache bool
timings []float64
}

func newMetadataCacheArm(t *testing.T, ctx context.Context, statement string, w queryrows.Window, cache bool) *metadataCacheArm {
t.Helper()
dsn, err := duckDSN(filepath.Join(t.TempDir(), "arm.duckdb"), "4GB", 4)
if err != nil {
t.Fatal(err)
}
native, err := duckdb.NewConnector(dsn, nil)
if err != nil {
t.Fatal(err)
}
pool := sql.OpenDB(&mdcConnector{Connector: native, cache: cache})
pool.SetMaxOpenConns(1)
pool.SetMaxIdleConns(1)
conn, err := pool.Conn(ctx)
if err != nil {
t.Fatal(err)
}
var got bool
if err := conn.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&got); err != nil {
t.Fatal(err)
}
if got != cache {
t.Fatalf("arm cache=%v but connection reports %v", cache, got)
}
tx, err := conn.BeginTx(context.WithoutCancel(ctx), nil)
if err != nil {
t.Fatal(err)
}
return &metadataCacheArm{t: t, ctx: ctx, pool: pool, conn: conn, tx: tx, statement: statement, w: w, cache: cache}
}

func (a *metadataCacheArm) exec() {
rows, err := a.tx.QueryContext(a.ctx, a.statement, a.w.Start, a.w.End)
if err != nil {
a.t.Fatal(err)
}
for rows.Next() {
var s string
var x, y, z, q int64
if err := rows.Scan(&s, &x, &y, &z, &q); err != nil {
a.t.Fatal(err)
}
}
if err := rows.Err(); err != nil {
a.t.Fatal(err)
}
rows.Close()
}

func (a *metadataCacheArm) sample() {
started := time.Now()
a.exec()
a.timings = append(a.timings, float64(time.Since(started))/float64(time.Millisecond))
}

func (a *metadataCacheArm) report() map[string]any {
sort.Float64s(a.timings)
var sum float64
for _, v := range a.timings {
sum += v
}
var objectCacheBytes int64
_ = a.tx.QueryRowContext(a.ctx, "SELECT coalesce(sum(memory_usage_bytes),0)::BIGINT FROM duckdb_memory() WHERE tag='OBJECT_CACHE'").Scan(&objectCacheBytes)
n := len(a.timings)
return map[string]any{
"arm": map[bool]string{false: "cache_off", true: "cache_on"}[a.cache],
"cache_enabled": a.cache,
"samples": n,
"mean_ms": sum / float64(n),
"median_ms": (a.timings[n/2-1] + a.timings[n/2]) / 2,
"p95_ms": a.timings[int(float64(n)*0.95)-1],
"p99_ms": a.timings[int(float64(n)*0.99)-1],
"min_ms": a.timings[0],
"max_ms": a.timings[n-1],
"object_cache_bytes": objectCacheBytes,
"unit": "native_execute_scan_same_tx_ab_ba",
}
}

func (a *metadataCacheArm) close() {
_ = a.tx.Rollback()
_ = a.conn.Close()
_ = a.pool.Close()
}
Loading