Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion sqlite_utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,20 @@ def _extra_key_strategy(
yield row_out


# Delimiters rows_from_file() looks for when it detects the format. Without a list
# csv.Sniffer takes any character that repeats on the sampled lines, so a single-column
# file was split on a letter, or raised "Could not determine delimiter".
_SNIFF_DELIMITERS = ",;\t|"


def _sniff_csv_dialect(sample: str) -> type[csv.Dialect]:
try:
return csv.Sniffer().sniff(sample, delimiters=_SNIFF_DELIMITERS)
except csv.Error:
# None of them: a single-column file, read as plain CSV
return csv.excel


def rows_from_file(
fp: BinaryIO,
format: Format | None = None,
Expand Down Expand Up @@ -389,7 +403,7 @@ class Format(enum.Enum):
with buffered:
return rows_from_file(buffered, format=Format.JSON)
else:
dialect = csv.Sniffer().sniff(
dialect = _sniff_csv_dialect(
first_bytes.decode(encoding or "utf-8-sig", "ignore")
)
rows, _ = rows_from_file(
Expand Down
11 changes: 11 additions & 0 deletions tests/test_cli_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,17 @@ def test_memory_csv_encoding(tmpdir, use_stdin):
}


def test_memory_csv_single_column(tmpdir):
csv_path = str(tmpdir / "names.csv")
with open(csv_path, "w") as fp:
fp.write("name\nAnn\nBob\n")
result = CliRunner().invoke(
cli.cli, ["memory", csv_path, "select * from names", "--nl"]
)
assert result.exit_code == 0, result.output
assert result.output.strip() == '{"name": "Ann"}\n{"name": "Bob"}'


def test_memory_csv_headers_only(tmpdir):
csv_path = str(tmpdir / "headers_only.csv")
with open(csv_path, "w") as fp:
Expand Down
19 changes: 19 additions & 0 deletions tests/test_rows_from_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"input,expected_format",
(
(b"id,name\n1,Cleo", Format.CSV),
(b"id;name\n1;Cleo", Format.CSV),
(b"id\tname\n1\tCleo", Format.TSV),
(b'[{"id": "1", "name": "Cleo"}]', Format.JSON),
),
Expand All @@ -22,6 +23,24 @@ def test_rows_from_file_detect_format(input, expected_format):
assert rows_list == [{"id": "1", "name": "Cleo"}]


@pytest.mark.parametrize(
"input,expected",
(
# csv.Sniffer could not determine a delimiter for this one
(b"name\nAnn\nBob\n", [{"name": "Ann"}, {"name": "Bob"}]),
# and took "l", which every line holds twice, for this one
(
b"url\nhttps://example.com/a\nhttps://example.org/b\n",
[{"url": "https://example.com/a"}, {"url": "https://example.org/b"}],
),
),
)
def test_rows_from_file_detect_format_single_column(input, expected):
rows, format = rows_from_file(BytesIO(input))
assert format == Format.CSV
assert list(rows) == expected


@pytest.mark.parametrize("input", (b"", b" \n\t"))
def test_rows_from_file_empty_input(input):
rows, format = rows_from_file(BytesIO(input))
Expand Down
Loading