From 2efea17bf55189fc5505f9fee812c8321ab0304a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 25 Jul 2026 08:07:59 +0200 Subject: [PATCH] perf(writer): raise default global-dict retained budget 1GB -> 2GB (#303) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a wide, high-cardinality file the per-column code buffers (~37 MB each, ADR 0021) sum past the 1 GB budget, and eviction demotes the largest-retained — i.e. the highest-cardinality — columns to per-chunk dictionaries, repeating their values pool every chunk. nyc-311 (~30 admitted string columns ≈ 1.15 GB retained) lost its three Cross Street / Street Name columns this way. Raise the default to 2 GB, which fits that file with headroom (measured: 2 GB and 3 GB both match the 8 GB result, so retained tops out ~1.15 GB) while still bounding the pathological many-wide-columns risk. Constrained-heap writers can still lower it via withGlobalDictMaxRetainedBytes(...). nyc-311 re-encode: 1934.23 MB -> 1879.34 MB (1.10x -> 1.07x vortex-jni). This overlaps #305 (both improve the street columns, via keeping-global vs bitpacked codes): stacked on #305 the budget adds ~35 MB net (1862 -> 1827 MB). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../io/github/dfa1/vortex/writer/WriteOptions.java | 13 ++++++++----- .../github/dfa1/vortex/writer/WriteOptionsTest.java | 12 ++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java index 730a8841..b99740df 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java @@ -37,12 +37,15 @@ public record WriteOptions( boolean enableZstd, long globalDictMaxRetainedBytes ) { - /// Default aggregate retention budget (1 GB) for the buffered per-chunk code arrays of global + /// Default aggregate retention budget (2 GB) for the buffered per-chunk code arrays of global /// -dictionary candidate columns. Raised from 256 MB when buffering became cardinality-bounded - /// (ADR 0021): codes are ~35–45× smaller than the raw values the old budget guarded, so a 1 GB - /// default bounds the same pathological many-wide-columns risk while letting normal wide, - /// low-cardinality files (e.g. NYC 311, ~38 string columns) keep all their shared dictionaries. - private static final long DEFAULT_GLOBAL_DICT_MAX_RETAINED_BYTES = 1024L * 1024 * 1024; + /// (ADR 0021), then from 1 GB (#303): a wide, high-cardinality file (NYC 311, ~30 admitted string + /// columns × ~37 MB of buffered codes ≈ 1.15 GB) crossed the 1 GB budget and evicted its + /// highest-cardinality columns to per-chunk dictionaries, repeating their values pool each chunk + /// (~35 MB larger). 2 GB fits that file with headroom while still bounding the pathological + /// many-wide-columns risk. Constrained-heap writers can lower it via + /// [#withGlobalDictMaxRetainedBytes(long)]. + private static final long DEFAULT_GLOBAL_DICT_MAX_RETAINED_BYTES = 2L * 1024 * 1024 * 1024; /// Default options: global dictionary encoding enabled, no cascading compression, Zstd disabled. /// diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java index ff881416..904f6480 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java @@ -7,14 +7,14 @@ /// Unit tests for [WriteOptions] factories and copy-methods. class WriteOptionsTest { - // The hardcoded default; both factories must keep supplying it so the budget stays 1 GB unless a + // The hardcoded default; both factories must keep supplying it so the budget stays 2 GB unless a // caller overrides it via withGlobalDictMaxRetainedBytes(...). Raised from 256 MB when buffering - // became cardinality-bounded (ADR 0021): the budget now guards ~2 B/row code arrays, not raw - // values, so a larger budget bounds the same risk while keeping wide low-cardinality files dicted. - private static final long DEFAULT_BUDGET = 1024L * 1024 * 1024; + // became cardinality-bounded (ADR 0021), then from 1 GB (#303) so wide high-cardinality files + // keep their high-cardinality columns globally dictionaried instead of evicting them to per-chunk. + private static final long DEFAULT_BUDGET = 2L * 1024 * 1024 * 1024; @Test - void defaults_globalDictMaxRetainedBytes_is1Gb() { + void defaults_globalDictMaxRetainedBytes_is2Gb() { // Given / When WriteOptions result = WriteOptions.defaults(); @@ -23,7 +23,7 @@ void defaults_globalDictMaxRetainedBytes_is1Gb() { } @Test - void cascading_globalDictMaxRetainedBytes_is1Gb() { + void cascading_globalDictMaxRetainedBytes_is2Gb() { // Given / When WriteOptions result = WriteOptions.cascading(3);