Repository navigation
Expand file tree
/
Copy pathdatabase.py
More file actions
359 lines (328 loc) · 13.6 KB
/
Copy pathdatabase.py
File metadata and controls
359 lines (328 loc) · 13.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
"""JSON database utilities for the GUI.
"""
import json
import os
import time
from typing import Any, Dict, List
class JSONDatabase:
"""Load and manage multiple JSON tables stored as files."""
def __init__(self, directory: str) -> None:
self.directory = directory
self.tables: Dict[str, List[Dict[str, Any]]] = {}
self.types: Dict[str, Dict[str, type]] = {}
self.next_id: Dict[str, int] = {}
self.changed_tables: set[str] = set()
self.links: List[Dict[str, str]] = []
self.links_changed: bool = False
self.meta: Dict[str, Dict[str, Any]] = {}
self.load()
def load(self) -> None:
"""Load all JSON files in the directory."""
for name in os.listdir(self.directory):
if not name.endswith('.json') or name == 'links.json':
continue
path = os.path.join(self.directory, name)
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
table = data.get('table') or os.path.splitext(name)[0]
rows = data.get('rows', [])
if not isinstance(rows, list):
continue
meta = {k: v for k, v in data.items() if k not in {'table', 'rows'}}
for key in ['license', 'last_updated', 'persistent_identifier', 'info']:
if key not in meta:
meta[key] = None
self.changed_tables.add(table)
self.meta[table] = meta
self.tables[table] = rows
col_types: Dict[str, type] = {}
for row in rows:
for col, value in row.items():
if col in col_types or value is None:
continue
col_types[col] = type(value)
if rows:
all_cols = {c for r in rows for c in r.keys()}
for col in all_cols:
col_types.setdefault(col, str)
self.types[table] = col_types
if 'id' in self.types[table]:
try:
max_id = max(r.get('id', 0) or 0 for r in rows)
except ValueError:
max_id = 0
self.next_id[table] = max_id + 1
else:
self.types[table] = {}
links_path = os.path.join(self.directory, 'links.json')
try:
with open(links_path, 'r', encoding='utf-8') as f:
data = json.load(f)
self.links = data.get('links', [])
except FileNotFoundError:
self.links = []
def save(self) -> None:
"""Write only changed tables back to their files."""
for table in list(self.changed_tables):
rows = self.tables.get(table)
path = os.path.join(self.directory, f'{table}.json')
if rows is None:
if os.path.exists(path):
os.remove(path)
continue
meta = self.meta.get(table, {})
meta['last_updated'] = int(time.time())
data = {'table': table, **meta, 'rows': rows}
with open(path, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=4, ensure_ascii=False)
self.changed_tables.clear()
if self.links_changed:
path = os.path.join(self.directory, 'links.json')
with open(path, 'w', encoding='utf-8') as f:
json.dump({'links': self.links}, f, indent=4, ensure_ascii=False)
self.links_changed = False
# Table operations -------------------------------------------------
def add_table(self, name: str) -> None:
if name in self.tables:
raise ValueError('Table already exists')
self.tables[name] = []
self.types[name] = {}
self.next_id[name] = 1
self.meta[name] = {
'license': None,
'last_updated': None,
'persistent_identifier': None,
'info': None,
}
self.changed_tables.add(name)
def remove_table(self, name: str) -> None:
self.tables.pop(name, None)
self.types.pop(name, None)
self.next_id.pop(name, None)
self.meta.pop(name, None)
path = os.path.join(self.directory, f'{name}.json')
if os.path.exists(path):
os.remove(path)
self.changed_tables.add(name)
# Column operations -----------------------------------------------
def add_column(self, table: str, column: str, default: Any = None, dtype: type = str) -> None:
if column in self.types.get(table, {}):
raise ValueError('Column already exists')
self.types[table][column] = dtype
for row in self.tables.get(table, []):
row[column] = default
if column == 'id':
try:
max_id = max(r.get('id', 0) or 0 for r in self.tables[table])
except ValueError:
max_id = 0
self.next_id[table] = max_id + 1
self.changed_tables.add(table)
def remove_column(self, table: str, column: str) -> None:
self.types.get(table, {}).pop(column, None)
for row in self.tables.get(table, []):
row.pop(column, None)
if column == 'id':
self.next_id.pop(table, None)
self.changed_tables.add(table)
# Metadata operations ----------------------------------------------
def update_metadata(
self, table: str, metadata: Dict[str, Any], apply_all: bool = False
) -> None:
targets = self.meta.keys() if apply_all else [table]
for t in targets:
self.meta.setdefault(t, {}).update(metadata)
self.changed_tables.add(t)
def add_metadata_field(
self,
field: str,
default: Any = None,
table: str | None = None,
apply_all: bool = False,
) -> None:
if apply_all:
targets = self.meta.keys()
else:
if table is None:
raise ValueError('table is required when apply_all is False')
targets = [table]
for t in targets:
if field not in self.meta.setdefault(t, {}):
self.meta[t][field] = default
self.changed_tables.add(t)
def remove_metadata_field(
self,
field: str,
table: str | None = None,
apply_all: bool = False,
) -> None:
"""Remove ``field`` from metadata."""
if apply_all:
targets = list(self.meta.keys())
else:
if table is None:
raise ValueError("table is required when apply_all is False")
targets = [table]
for t in targets:
if field in self.meta.get(t, {}):
del self.meta[t][field]
self.changed_tables.add(t)
# Link operations -------------------------------------------------
def add_link(
self,
source_table: str,
source_column: str,
target_table: str,
target_column: str,
) -> None:
if source_table not in self.tables:
raise ValueError(f'source table {source_table} does not exist')
if source_column not in self.types.get(source_table, {}):
raise ValueError(
f'column {source_column} does not exist in {source_table}'
)
if target_table not in self.tables:
raise ValueError(f'target table {target_table} does not exist')
if target_column not in self.types.get(target_table, {}):
raise ValueError(
f'column {target_column} does not exist in {target_table}'
)
src_type = self.types[source_table][source_column]
tgt_type = self.types[target_table][target_column]
if src_type is not tgt_type:
raise ValueError('Source and target column types must match')
link = {
'source_table': source_table,
'source_column': source_column,
'target_table': target_table,
'target_column': target_column,
}
if link in self.links:
raise ValueError('Link already exists')
self.links.append(link)
self.links_changed = True
def remove_link(self, index: int) -> None:
if 0 <= index < len(self.links):
self.links.pop(index)
self.links_changed = True
def validate_links(self) -> List[tuple[int, Dict[str, str]]]:
"""Return a list of (index, link) tuples that reference missing tables
or columns."""
broken: List[tuple[int, Dict[str, str]]] = []
for idx, link in enumerate(self.links):
if (
link['source_table'] not in self.tables
or link['source_column'] not in self.types.get(link['source_table'], {})
or link['target_table'] not in self.tables
or link['target_column'] not in self.types.get(link['target_table'], {})
):
broken.append((idx, link))
return broken
def prune_links(self) -> List[Dict[str, str]]:
"""Remove links that reference missing tables or columns.
Returns a list with the removed link dictionaries.
"""
broken = self.validate_links()
removed: List[Dict[str, str]] = []
for idx, link in reversed(broken):
removed.append(self.links.pop(idx))
if removed:
self.links_changed = True
return removed
def get_link(self, table: str, column: str) -> Dict[str, str] | None:
for link in self.links:
if link['source_table'] == table and link['source_column'] == column:
return link
return None
# Row operations ---------------------------------------------------
def add_row(self, table: str, row: Dict[str, Any]) -> None:
self._ensure_unique_id(table, row)
self._validate_row(table, row)
self._check_links(table, row)
self.tables[table].append(row)
self.changed_tables.add(table)
def update_row(self, table: str, index: int, row: Dict[str, Any]) -> None:
self._ensure_unique_id(table, row, index)
self._validate_row(table, row)
self._check_links(table, row)
self.tables[table][index] = row
self.changed_tables.add(table)
def delete_row(self, table: str, index: int) -> None:
del self.tables[table][index]
self.changed_tables.add(table)
# Helpers ----------------------------------------------------------
def check_row(self, table: str, row: Dict[str, Any], index: int | None = None) -> None:
"""Validate row without modifying internal state."""
temp = dict(row)
self._validate_row(table, temp)
self._check_links(table, temp)
if 'id' in self.types.get(table, {}):
existing = {
r.get('id')
for i, r in enumerate(self.tables[table])
if i != (index if index is not None else -1)
}
if temp.get('id') is None:
raise ValueError('id column is required')
if temp['id'] in existing:
raise ValueError(f'id {temp["id"]} already exists')
def _validate_row(self, table: str, row: Dict[str, Any]) -> None:
"""Validate row values against known column types."""
for col, coltype in self.types.get(table, {}).items():
if col not in row:
continue
value = row[col]
if value is None:
continue
if isinstance(value, coltype):
continue
try:
if coltype is int:
row[col] = int(value)
elif coltype is float:
row[col] = float(value)
elif coltype is bool:
if isinstance(value, str):
lv = value.lower()
if lv in {'1', 'true', 'yes'}:
row[col] = True
elif lv in {'0', 'false', 'no'}:
row[col] = False
else:
raise ValueError
else:
row[col] = bool(value)
elif coltype is str:
row[col] = str(value)
else:
raise ValueError
except Exception as exc:
raise ValueError(f'Column {col} expects {coltype.__name__}') from exc
def _ensure_unique_id(self, table: str, row: Dict[str, Any], index: int | None = None) -> None:
if 'id' not in self.types.get(table, {}):
return
existing = {
r.get('id')
for i, r in enumerate(self.tables[table])
if i != (index if index is not None else -1)
}
if row.get('id') is None:
raise ValueError('id column is required')
if row['id'] in existing:
raise ValueError(f'id {row["id"]} already exists')
if row['id'] >= self.next_id.get(table, 1):
self.next_id[table] = row['id'] + 1
def _check_links(self, table: str, row: Dict[str, Any]) -> None:
for link in self.links:
if link['source_table'] != table:
continue
src_col = link['source_column']
tgt_table = link['target_table']
tgt_col = link['target_column']
if src_col not in row or row[src_col] is None:
continue
val = row[src_col]
if val not in {r.get(tgt_col) for r in self.tables.get(tgt_table, [])}:
raise ValueError(
f"Value {val!r} for {src_col} not found in {tgt_table}.{tgt_col}"
)