diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java index 730a8841..b99740df 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteOptions.java @@ -37,12 +37,15 @@ public record WriteOptions( boolean enableZstd, long globalDictMaxRetainedBytes ) { - /// Default aggregate retention budget (1 GB) for the buffered per-chunk code arrays of global + /// Default aggregate retention budget (2 GB) for the buffered per-chunk code arrays of global /// -dictionary candidate columns. Raised from 256 MB when buffering became cardinality-bounded - /// (ADR 0021): codes are ~35–45× smaller than the raw values the old budget guarded, so a 1 GB - /// default bounds the same pathological many-wide-columns risk while letting normal wide, - /// low-cardinality files (e.g. NYC 311, ~38 string columns) keep all their shared dictionaries. - private static final long DEFAULT_GLOBAL_DICT_MAX_RETAINED_BYTES = 1024L * 1024 * 1024; + /// (ADR 0021), then from 1 GB (#303): a wide, high-cardinality file (NYC 311, ~30 admitted string + /// columns × ~37 MB of buffered codes ≈ 1.15 GB) crossed the 1 GB budget and evicted its + /// highest-cardinality columns to per-chunk dictionaries, repeating their values pool each chunk + /// (~35 MB larger). 2 GB fits that file with headroom while still bounding the pathological + /// many-wide-columns risk. Constrained-heap writers can lower it via + /// [#withGlobalDictMaxRetainedBytes(long)]. + private static final long DEFAULT_GLOBAL_DICT_MAX_RETAINED_BYTES = 2L * 1024 * 1024 * 1024; /// Default options: global dictionary encoding enabled, no cascading compression, Zstd disabled. /// diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java index ff881416..904f6480 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriteOptionsTest.java @@ -7,14 +7,14 @@ /// Unit tests for [WriteOptions] factories and copy-methods. class WriteOptionsTest { - // The hardcoded default; both factories must keep supplying it so the budget stays 1 GB unless a + // The hardcoded default; both factories must keep supplying it so the budget stays 2 GB unless a // caller overrides it via withGlobalDictMaxRetainedBytes(...). Raised from 256 MB when buffering - // became cardinality-bounded (ADR 0021): the budget now guards ~2 B/row code arrays, not raw - // values, so a larger budget bounds the same risk while keeping wide low-cardinality files dicted. - private static final long DEFAULT_BUDGET = 1024L * 1024 * 1024; + // became cardinality-bounded (ADR 0021), then from 1 GB (#303) so wide high-cardinality files + // keep their high-cardinality columns globally dictionaried instead of evicting them to per-chunk. + private static final long DEFAULT_BUDGET = 2L * 1024 * 1024 * 1024; @Test - void defaults_globalDictMaxRetainedBytes_is1Gb() { + void defaults_globalDictMaxRetainedBytes_is2Gb() { // Given / When WriteOptions result = WriteOptions.defaults(); @@ -23,7 +23,7 @@ void defaults_globalDictMaxRetainedBytes_is1Gb() { } @Test - void cascading_globalDictMaxRetainedBytes_is1Gb() { + void cascading_globalDictMaxRetainedBytes_is2Gb() { // Given / When WriteOptions result = WriteOptions.cascading(3);