Spec conformance: zstd.zig implements the Zstandard 1.6.0 specification natively in Zig - every algorithm, frame element, and default table in this document follows that version.

Dictionaries

Dictionary compression achieves significantly better compression ratios when compressing many similar small buffers (e.g., database records, JSON objects, log entries).

Dictionary Type

Dictionary is defined:

pub const Dictionary = struct {
    data: []u8, // includes 8-byte header (magic + dictId)
    dictId: u32,
    allocator: std.mem.Allocator,

    pub fn deinit(self: *Dictionary) void
    pub fn dictId(self: *const Dictionary) u32
    pub fn content(self: *const Dictionary) []const u8 // raw content without header
};

Top-level helpers:

pub fn loadDictionary(allocator: std.mem.Allocator, data: []const u8) ZstdError!Dictionary
pub fn createDictionaryFromData(allocator: std.mem.Allocator, data: []const u8, dictId: u32) ZstdError!Dictionary

const zstd = @import("zstd");

// Create a dictionary from raw bytes with an explicit ID
var dict = try zstd.createDictionaryFromData(allocator, raw_bytes, 12345);
defer dict.deinit();
std.debug.print("ID={d} size={d}\n", .{ dict.dictId(), dict.data.len });
std.debug.print("content len={d}\n", .{ dict.content().len });

// Load an existing dictionary blob (preserves its stored dictId)
var loaded = try zstd.loadDictionary(allocator, dict.data);
defer loaded.deinit();
std.debug.assert(loaded.dictId() == dict.dictId());

Compressing with a Dictionary ID

Dictionary-aware compression currently stores the dictId in the frame header via CompressionOptions:

const opts = zstd.CompressionOptions{ .dictId = dict.dictId() };
const compressed = try zstd.compressWithOptions(allocator, data, opts);
defer allocator.free(compressed);

// Verify via header
const hdr = try zstd.getFrameHeader(compressed);
std.debug.assert(hdr.dictId == dict.dictId());

const decompressed = try zstd.decompress(allocator, compressed);
defer allocator.free(decompressed);

DictionaryBuilder

DictionaryBuilder is defined:

pub const DictBuilderParams = struct {
    dictSize: usize = 112640,
    dictId: u32 = 0,
    level: u32 = 3,
};

pub const DictionaryBuilder = struct {
    pub fn init(allocator: std.mem.Allocator, params: DictBuilderParams) DictionaryBuilder
    pub fn train(self: *DictionaryBuilder, samples: []const []const u8) anyerror!Dictionary
    pub fn trainCover(self: *DictionaryBuilder, samples: []const []const u8, k: usize, d: usize) anyerror!Dictionary
    pub fn trainFastCover(self: *DictionaryBuilder, samples: []const []const u8, k: usize, d: usize, f: u32, accel: u32) anyerror!Dictionary
};

Helper functions (same file):

pub fn trainFromSamples(allocator, samples, params) !Dictionary
pub fn trainCoverImpl(allocator, samples, params, k, d) !Dictionary
pub fn trainFastCoverImpl(allocator, samples, params, k, d, f, accel) !Dictionary

Training a Dictionary

const samples = &[_][]const u8{
    sample1, sample2, sample3, sample4, sample5,
};

var builder = zstd.DictionaryBuilder.init(allocator, .{ .dictSize = 8192, .dictId = 42 });
var dict = try builder.train(samples);
defer dict.deinit();

// Cover variants (k = capacity, d = dict bits)
var cdict = try builder.trainCover(samples, 6, 8);
defer cdict.deinit();

var fdict = try builder.trainFastCover(samples, 6, 8, 6, 2);
defer fdict.deinit();

Training requires at least one non-empty sample; content is synthesized from the samples in the current implementation.

Full Example (mirrors examples/dictionary_training.zig)

var samples: std.ArrayList([]const u8) = .empty;
defer samples.deinit(allocator);
for (0..100) |i| {
    const s = try std.fmt.allocPrint(allocator, "sample {d}: common header payload {d}", .{ i, i % 10 });
    try samples.append(allocator, s);
}
defer for (samples.items) |s| allocator.free(s);

var builder = zstd.DictionaryBuilder.init(allocator, .{ .dictSize = 8192 });
var d = try builder.train(samples.items);
defer d.deinit();

var cd = try builder.trainCover(samples.items, 6, 8);
defer cd.deinit();

var fd = try builder.trainFastCover(samples.items, 6, 8, 6, 2);
defer fd.deinit();

Running the Dictionary Examples

zig build run-dictionary_compression
zig build run-dictionary_training