Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions tests/test_store_read_lines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""Store.read_lines: only JSON-object records are returned, and `limit` counts those."""
import os

from tokeneconomizer.plugin import accounting
from tokeneconomizer.plugin.store import Store


def _write(store, name, lines):
os.makedirs(store.base, exist_ok=True)
with open(store.path(name), "w", encoding="utf-8") as fh:
fh.write("\n".join(lines) + "\n")


def test_read_lines_skips_non_object_json(tmp_path):
s = Store(str(tmp_path / "st"))
_write(s, "ledger.jsonl", ['{"n": 1}', "null", "[]", '"str"', "3", '{"n": 2}', "null"])
assert s.read_lines("ledger.jsonl") == [{"n": 1}, {"n": 2}]


def test_read_lines_limit_counts_only_object_records(tmp_path):
s = Store(str(tmp_path / "st"))
_write(s, "ledger.jsonl", ['{"n": 1}', '{"n": 2}', "null"])
assert s.read_lines("ledger.jsonl", limit=1) == [{"n": 2}]


def test_summarize_tolerates_non_object_records(tmp_path):
s = Store(str(tmp_path / "st"))
valid = {"kind": "session", "session_id": "a", "imported": False}
_write(s, "ledger.jsonl", ["null", "[]", '"str"', '{"kind": "session", "session_id": "a", '
'"imported": false}'])
assert accounting.summarize(s.read_lines("ledger.jsonl")) == accounting.summarize([valid])
assert accounting.summarize([None, [], "str", valid]) == accounting.summarize([valid])
1 change: 1 addition & 0 deletions tokeneconomizer/plugin/accounting.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def summarize(records):

A session imported from the agent's history (`imported: true`) ran before the plugin: it is
counted on its own (`imported_sessions`) and left out of every other number."""
records = [r for r in records if isinstance(r, dict)]
imported = [r for r in records if r.get("kind") == "session" and r.get("imported")]
records = [r for r in records if not r.get("imported")]
sessions = [r for r in records if r.get("kind") == "session"]
Expand Down
6 changes: 4 additions & 2 deletions tokeneconomizer/plugin/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,15 +103,17 @@ def append(self, name, record):
fh.write(json.dumps(record, sort_keys=True) + "\n")

def read_lines(self, name, limit=None):
"""Records of a JSONL log, oldest first; the last `limit` when given."""
"""Object records of a JSONL log, oldest first; the last `limit` when given."""
out = []
try:
with open(self.path(name), encoding="utf-8") as fh:
for line in fh:
try:
out.append(json.loads(line))
rec = json.loads(line)
except ValueError:
continue
if isinstance(rec, dict):
out.append(rec)
except OSError:
return []
return out[-limit:] if limit else out
Expand Down
Loading