Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;

import static org.assertj.core.api.Assertions.assertThat;

import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
Expand Down Expand Up @@ -53,6 +55,8 @@ void profileFullScan() throws Exception {
}
double ms = (System.nanoTime() - t0) / 1e6 / iterations;
System.out.printf("full scan x%d: %.2f ms/query | count=%,d%n", iterations, ms, count);
// A wrong count here means the profile below is timing a broken scan.
assertThat(count).isEqualTo(rows);
}
} finally {
Files.deleteIfExists(file);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,15 @@ private static final class Loop {
/// ASCII spinner frames; cycled by render tick.
private static final char[] SPINNER = {'|', '/', '-', '\\'};

/// Suffix appended after the spinner frame while a background fetch is still pending.
private static final String LOADING_SUFFIX = " loading...";

/// Format for one numbered preview line (dictionary entry, zone-stats row, or data row).
private static final String ENTRY_FORMAT = " [%2d] %s";

/// Label prefix for the selected column's data preview.
private static final String DATA_COLUMN_PREFIX = "Data (column '";

private final Terminal term;
private final InspectorTree tree;
private final VortexHandle handle;
Expand Down Expand Up @@ -508,13 +517,13 @@ private List<String> detailLines(InspectorTree.Node node) {
lines.add("");
switch (dictState) {
case DataState.Pending _ ->
lines.add("Dictionary: " + SPINNER[(int) (tick % SPINNER.length)] + " loading...");
lines.add("Dictionary: " + SPINNER[(int) (tick % SPINNER.length)] + LOADING_SUFFIX);
case DataState.Failed(String msg) ->
lines.add("Dictionary: ! " + msg);
case DataState.Loaded(List<String> values) -> {
lines.add("Dictionary (" + values.size() + " entries):");
for (int i = 0; i < values.size(); i++) {
lines.add(String.format(" [%2d] %s", i, values.get(i)));
lines.add(String.format(ENTRY_FORMAT, i, values.get(i)));
}
}
}
Expand All @@ -529,13 +538,13 @@ private List<String> detailLines(InspectorTree.Node node) {
switch (zoneState) {
case DataState.Pending _ ->
lines.add("Per-chunk stats: "
+ SPINNER[(int) (tick % SPINNER.length)] + " loading...");
+ SPINNER[(int) (tick % SPINNER.length)] + LOADING_SUFFIX);
case DataState.Failed(String msg) ->
lines.add("Per-chunk stats: ! " + msg);
case DataState.Loaded(List<String> rows) -> {
lines.add("Per-chunk stats (" + rows.size() + " chunks):");
for (int i = 0; i < rows.size(); i++) {
lines.add(String.format(" [%2d] %s", i, rows.get(i)));
lines.add(String.format(ENTRY_FORMAT, i, rows.get(i)));
}
}
}
Expand All @@ -545,14 +554,14 @@ private List<String> detailLines(InspectorTree.Node node) {
lines.add("");
switch (state) {
case DataState.Pending _ ->
lines.add("Data (column '" + col + "'): "
+ SPINNER[(int) (tick % SPINNER.length)] + " loading...");
lines.add(DATA_COLUMN_PREFIX + col + "'): "
+ SPINNER[(int) (tick % SPINNER.length)] + LOADING_SUFFIX);
case DataState.Failed(String msg) ->
lines.add("Data (column '" + col + "'): ! " + msg);
lines.add(DATA_COLUMN_PREFIX + col + "'): ! " + msg);
case DataState.Loaded(List<String> values) -> {
lines.add("Data (column '" + col + "', first " + values.size() + " rows):");
lines.add(DATA_COLUMN_PREFIX + col + "', first " + values.size() + " rows):");
for (int i = 0; i < values.size(); i++) {
lines.add(String.format(" [%2d] %s", i, values.get(i)));
lines.add(String.format(ENTRY_FORMAT, i, values.get(i)));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ final class LossyPerfectHashTable {
/// Low three bytes of the input word — the only bytes the hash keys on.
private static final long PREFIX_MASK = 0x00FF_FFFFL;

/// Slot table with two adjacent longs per slot (the FSST paper's C layout): `slots[2 * s]` is
/// the candidate's packed symbol bytes (LSB-first, [Symbol] convention) and `slots[2 * s + 1]`
/// Slot table with two adjacent longs per slot (the FSST paper's C layout): `table[2 * s]` is
/// the candidate's packed symbol bytes (LSB-first, [Symbol] convention) and `table[2 * s + 1]`
/// is its metadata `ignoredBits << 16 | code << 8 | length` (`ignoredBits = 64 - 8 * length`).
/// A lookup derives the keep-mask from the ignored-bits with one shift instead of loading a
/// separate mask array, so the whole 16-byte slot lands in a single cache line — one memory
Expand All @@ -46,10 +46,10 @@ final class LossyPerfectHashTable {
/// input word of exactly 0 can "hit" an empty slot — harmlessly, because the returned low 16
/// bits (`code << 8 | length`) are then 0, which is precisely the "no match" answer. No
/// occupancy flag or sentinel is needed.
private final long[] slots;
private final long[] table;

private LossyPerfectHashTable(long[] slots) {
this.slots = slots;
private LossyPerfectHashTable(long[] table) {
this.table = table;
}

/// Builds the table from the trained symbols in descending-gain order, keeping only those of
Expand All @@ -67,21 +67,21 @@ private LossyPerfectHashTable(long[] slots) {
/// @param symbolsByGainDescending the trained symbols, code = list index, gain-descending
/// @return a hash table resolving 3-8 byte matches with first-writer-wins on collision
static LossyPerfectHashTable of(List<Symbol> symbolsByGainDescending) {
long[] slots = new long[2 * SLOTS];
long[] table = new long[2 * SLOTS];
for (int code = 0; code < symbolsByGainDescending.size(); code++) {
Symbol symbol = symbolsByGainDescending.get(code);
if (symbol.length() < 3) {
continue; // Length 1-2 belongs to ShortCodeTable.
}
int slot = slotFor(symbol.packedBytes());
if (slots[2 * slot + 1] != 0) {
if (table[2 * slot + 1] != 0) {
continue; // First writer (higher gain) wins; skip the collision.
}
long ignoredBits = 64L - 8 * symbol.length();
slots[2 * slot] = symbol.packedBytes();
slots[2 * slot + 1] = ignoredBits << 16 | (long) code << 8 | symbol.length();
table[2 * slot] = symbol.packedBytes();
table[2 * slot + 1] = ignoredBits << 16 | (long) code << 8 | symbol.length();
}
return new LossyPerfectHashTable(slots);
return new LossyPerfectHashTable(table);
}

/// Looks up `word` and returns the matched symbol as `code << 8 | length`, or 0 when no stored
Expand All @@ -99,8 +99,8 @@ static LossyPerfectHashTable of(List<Symbol> symbolsByGainDescending) {
/// @return the match as `code << 8 | length`, or 0 when there is no match
int lookup(long word) {
int slot = slotFor(word) << 1;
long symbol = slots[slot];
long meta = slots[slot + 1];
long symbol = table[slot];
long meta = table[slot + 1];
return (word & (~0L >>> (int) (meta >>> 16))) == symbol ? (int) (meta & 0xFFFF) : 0;
}

Expand Down
20 changes: 10 additions & 10 deletions fsst/src/main/java/io/github/dfa1/vortex/fsst/ShortCodeTable.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ final class ShortCodeTable {
private static final int NO_MATCH = NO_CODE << 8;

/// Packed `code << 8 | length` per 16-bit key. A zero length marks "no match" ([#NO_MATCH]).
private final int[] slots;
private final int[] table;

private ShortCodeTable(int[] slots) {
this.slots = slots;
private ShortCodeTable(int[] table) {
this.table = table;
}

/// Builds the table from symbols in descending-gain order, keeping only the length-1 and
Expand All @@ -51,17 +51,17 @@ private ShortCodeTable(int[] slots) {
/// @param symbolsByGainDescending the trained symbols, code = list index, gain-descending
/// @return a table resolving 0/1/2-byte matches for any two-byte input prefix
static ShortCodeTable of(List<Symbol> symbolsByGainDescending) {
int[] slots = new int[SLOTS];
Arrays.fill(slots, NO_MATCH);
int[] table = new int[SLOTS];
Arrays.fill(table, NO_MATCH);
for (int code = 0; code < symbolsByGainDescending.size(); code++) {
Symbol symbol = symbolsByGainDescending.get(code);
if (symbol.length() == 1) {
int low = symbol.byteAt(0) & 0xFF;
int packed = code << 8 | 1;
for (int high = 0; high < 256; high++) {
int key = high << 8 | low;
if (length(slots[key]) == 0) {
slots[key] = packed;
if (length(table[key]) == 0) {
table[key] = packed;
}
}
}
Expand All @@ -70,10 +70,10 @@ static ShortCodeTable of(List<Symbol> symbolsByGainDescending) {
Symbol symbol = symbolsByGainDescending.get(code);
if (symbol.length() == 2) {
int key = (int) (symbol.packedBytes() & 0xFFFF);
slots[key] = code << 8 | 2;
table[key] = code << 8 | 2;
}
}
return new ShortCodeTable(slots);
return new ShortCodeTable(table);
}

/// Returns the match for the low two bytes of `word` as `code << 8 | length`, or
Expand All @@ -84,7 +84,7 @@ static ShortCodeTable of(List<Symbol> symbolsByGainDescending) {
/// @param word an input word; only its low 16 bits (first two input bytes) are consulted
/// @return the match as `code << 8 | length`; length 0 (and code [#NO_CODE]) means no match
int packedFor(long word) {
return slots[(int) (word & 0xFFFF)];
return table[(int) (word & 0xFFFF)];
}

/// Returns the code matched by the low two bytes of `word`, or [#NO_CODE] if none.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ class RaincloudConformanceIntegrationTest {
Path.of(System.getProperty("user.home"), ".cache", "raincloud", "corpus-manifest.tsv");

@Test
@SuppressWarnings("java:S2699") // assumeTrue IS this test's check: a visible skip marker,
// not a pass/fail assertion — there is nothing else to assert
void corpusIsHydrated() {
// Given / When / Then — visible skip marker when the corpus is absent; the
// factory below yields zero tests in that case, which would otherwise pass silently
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ class RaincloudSizeComparisonIntegrationTest {
Path.of(System.getProperty("user.home"), ".cache", "raincloud", "corpus-manifest.tsv");

@Test
@SuppressWarnings("java:S2699") // assumeTrue IS this test's check: a visible skip marker,
// not a pass/fail assertion — there is nothing else to assert
void corpusIsHydrated() {
// Given / When / Then — visible skip marker when the corpus is absent
assumeTrue(Files.exists(manifestPath()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ public sealed interface Predicate
Predicate.Gte, Predicate.Between, Predicate.IsNull, Predicate.IsNotNull,
Predicate.And, Predicate.Or {

/// `Objects.requireNonNull` message naming the rejected component, shared by every leaf
/// below whose sole component is named `value`.
String VALUE = "value";

/// Matches rows whose value equals `value`.
///
/// Equality is value-domain equality (the kernel decides how that maps onto the encoded form);
Expand All @@ -44,7 +48,7 @@ record Eq(Object value) implements Predicate {
///
/// @param value the value to compare against, must be non-null
public Eq {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand All @@ -61,7 +65,7 @@ record Neq(Object value) implements Predicate {
///
/// @param value the value to compare against, must be non-null
public Neq {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand All @@ -74,7 +78,7 @@ record Lt(Comparable<?> value) implements Predicate {
///
/// @param value the exclusive upper bound, must be non-null
public Lt {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand All @@ -87,7 +91,7 @@ record Gt(Comparable<?> value) implements Predicate {
///
/// @param value the exclusive lower bound, must be non-null
public Gt {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand All @@ -100,7 +104,7 @@ record Lte(Comparable<?> value) implements Predicate {
///
/// @param value the inclusive upper bound, must be non-null
public Lte {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand All @@ -113,7 +117,7 @@ record Gte(Comparable<?> value) implements Predicate {
///
/// @param value the inclusive lower bound, must be non-null
public Gte {
Objects.requireNonNull(value, "value");
Objects.requireNonNull(value, VALUE);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@
/// Read-only decoder for `vortex.sparse`.
public final class SparseEncodingDecoder implements EncodingDecoder {

/// `role` argument naming the patch indices child, for error messages.
private static final String ROLE_INDICES = "indices";

/// `role` argument naming the patch values child, for error messages.
private static final String ROLE_VALUES = "values";

@Override
public EncodingId encodingId() {
return EncodingId.VORTEX_SPARSE;
Expand Down Expand Up @@ -119,10 +125,10 @@ public Array decode(DecodeContext ctx) {
valData = m.inner();
patchValidity = m.validity();
}
checkPatchChild(idxData, numPatches, "indices");
checkPatchChild(valData, numPatches, "values");
checkPatchChild(idxData, numPatches, ROLE_INDICES);
checkPatchChild(valData, numPatches, ROLE_VALUES);
boolean fillValue = Boolean.TRUE.equals(fillScalar.bool_value());
BoolArray boolValues = checkedCast(valData, BoolArray.class, "values");
BoolArray boolValues = checkedCast(valData, BoolArray.class, ROLE_VALUES);
Array result = new LazySparseBoolArray(ctx.dtype(), n, fillValue, boolValues, idxData, offset);
return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset);
}
Expand All @@ -146,26 +152,26 @@ public Array decode(DecodeContext ctx) {
valData = m.inner();
patchValidity = m.validity();
}
checkPatchChild(idxData, numPatches, "indices");
checkPatchChild(valData, numPatches, "values");
checkPatchChild(idxData, numPatches, ROLE_INDICES);
checkPatchChild(valData, numPatches, ROLE_VALUES);

Array result = switch (valuePtype) {
case I64, U64 -> new LazySparseLongArray(ctx.dtype(), n, fillBits,
checkedCast(valData, LongArray.class, "values"), idxData, offset);
checkedCast(valData, LongArray.class, ROLE_VALUES), idxData, offset);
case I32, U32 -> new LazySparseIntArray(ctx.dtype(), n, (int) fillBits,
checkedCast(valData, IntArray.class, "values"), idxData, offset);
checkedCast(valData, IntArray.class, ROLE_VALUES), idxData, offset);
case F64 -> new LazySparseDoubleArray(ctx.dtype(), n, Double.longBitsToDouble(fillBits),
checkedCast(valData, DoubleArray.class, "values"), idxData, offset);
checkedCast(valData, DoubleArray.class, ROLE_VALUES), idxData, offset);
case F32 -> new LazySparseFloatArray(ctx.dtype(), n, Float.intBitsToFloat((int) fillBits),
checkedCast(valData, FloatArray.class, "values"), idxData, offset);
checkedCast(valData, FloatArray.class, ROLE_VALUES), idxData, offset);
case I16 -> new LazySparseShortArray(ctx.dtype(), n, (short) fillBits, (short) fillBits,
checkedCast(valData, ShortArray.class, "values"), idxData, offset);
checkedCast(valData, ShortArray.class, ROLE_VALUES), idxData, offset);
case U16 -> new LazySparseShortArray(ctx.dtype(), n, (short) fillBits, (int) (fillBits & 0xFFFFL),
checkedCast(valData, ShortArray.class, "values"), idxData, offset);
checkedCast(valData, ShortArray.class, ROLE_VALUES), idxData, offset);
case I8 -> new LazySparseByteArray(ctx.dtype(), n, (byte) fillBits, (byte) fillBits,
checkedCast(valData, ByteArray.class, "values"), idxData, offset);
checkedCast(valData, ByteArray.class, ROLE_VALUES), idxData, offset);
case U8 -> new LazySparseByteArray(ctx.dtype(), n, (byte) fillBits, (int) (fillBits & 0xFFL),
checkedCast(valData, ByteArray.class, "values"), idxData, offset);
checkedCast(valData, ByteArray.class, ROLE_VALUES), idxData, offset);
default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype " + valuePtype);
};
return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset);
Expand Down Expand Up @@ -275,7 +281,7 @@ private static Array decodeVarBin(
DType indicesDtype = new DType.Primitive(indicesPtype, false);
Array patchIndices = ctx.decodeChild(0, indicesDtype, numPatches);
Array idxData = patchIndices instanceof MaskedArray m ? m.inner() : patchIndices;
checkPatchChild(idxData, numPatches, "indices");
checkPatchChild(idxData, numPatches, ROLE_INDICES);
byte[] fill = fillBytes(fillScalar, fillValid);

if (numPatches == 0) {
Expand All @@ -294,7 +300,7 @@ private static Array decodeVarBin(
valData = m.inner();
patchValidity = m.validity();
}
VarBinArray values = checkedCast(valData, VarBinArray.class, "values");
VarBinArray values = checkedCast(valData, VarBinArray.class, ROLE_VALUES);
Array result = new VarBinSparseArray(ctx.dtype(), n, fill, values, idxData, offset);
return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset);
}
Expand Down
Loading