Skip to content

Commit e09dc4e

Browse files
TonisOrmissonTonisOrmisson
committed
Add basic variable transformations
1 parent f726e06 commit e09dc4e

8 files changed

Lines changed: 361 additions & 21 deletions

File tree

‎src/openstatspec/__init__.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
from .sql.workflow import TransformationError
1616
from .transform import (
1717
AssignOperation, BooleanExpression, ComparisonExpression,
18-
ConditionalAssignOperation, ExecuteOperation, Operand, PredicateExpression,
18+
ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation,
19+
ExecuteOperation, Operand, PredicateExpression,
1920
RecodeMatch, RecodeOperation, RecodeResult, RecodeRule,
2021
ReplaceValueLabelsOperation, SetFormatOperation,
2122
SetMeasurementLevelOperation, SetVariableLabelOperation,
@@ -26,7 +27,7 @@
2627
__all__ = [
2728
"AssignOperation", "BooleanExpression", "ComparisonExpression",
2829
"ConditionalAssignOperation", "ExecuteOperation", "Operand",
29-
"PredicateExpression",
30+
"ConditionalAssignOperation", "CreateVariableOperation", "DeleteVariableOperation",
3031
"CapabilityDeclaration", "DoltConformanceSource", "LossReport",
3132
"SpssFrontendCompilation", "TransformationError",
3233
"TransformationFrontendError",

‎src/openstatspec/frontends/spss/binding.py‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
from ...transform.errors import SourceSpan, frontend_error
99
from ...transform.plan import (
1010
AssignOperation, BooleanExpression, ComparisonExpression,
11-
ConditionalAssignOperation, ExecuteOperation, Operand, PlanOperation,
11+
ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation,
12+
ExecuteOperation, Operand, PlanOperation,
1213
PredicateExpression, RecodeMatch, RecodeOperation, RecodeResult, RecodeRule,
1314
ReplaceValueLabelsOperation, SetFormatOperation,
1415
SetMeasurementLevelOperation, SetVariableLabelOperation,
@@ -21,10 +22,11 @@
2122
from ...transform.validation import bind_transformation_plan
2223
from .syntax import (
2324
BooleanSyntax, ComparisonSyntax, ComputeCommandSyntax, ExecuteCommandSyntax,
24-
FormatsCommandSyntax, IfCommandSyntax, OperandSyntax, PredicateSyntax,
25+
DeleteVariablesCommandSyntax, FormatsCommandSyntax, IfCommandSyntax,
26+
OperandSyntax, PredicateSyntax,
2527
RecodeCommandSyntax, RecodeMatchSyntax, RecodeResultSyntax,
2628
SpssSyntaxProgram, SyntaxLiteral, ValueLabelsCommandSyntax,
27-
VariableLabelsCommandSyntax, VariableLevelCommandSyntax,
29+
StringCommandSyntax, VariableLabelsCommandSyntax, VariableLevelCommandSyntax,
2830
)
2931

3032

@@ -290,8 +292,6 @@ def _bind_recode(
290292
# replace intentionally preserves the existing variable metadata. A later
291293
# VALUE LABELS command replaces value labels explicitly.
292294
return operations, spans
293-
294-
295295
def bind_spss_syntax(
296296
program: SpssSyntaxProgram, schema: VariableSchema, *, input_alias: str = "parent",
297297
) -> BoundTransformation:
@@ -305,6 +305,50 @@ def bind_spss_syntax(
305305
operations: list[PlanOperation] = []
306306
spans: list[SourceSpan] = []
307307
for command in program.commands:
308+
if isinstance(command, StringCommandSyntax):
309+
for variable_token in command.variables:
310+
if variable_token.text.startswith("__"):
311+
raise frontend_error(
312+
"reserved_target_name",
313+
f"Target name {variable_token.text!r} is reserved.",
314+
span=variable_token.span,
315+
target=variable_token.text,
316+
)
317+
if any(
318+
variable.name.casefold() == variable_token.text.casefold()
319+
for variable in variables
320+
):
321+
raise frontend_error(
322+
"target_already_exists",
323+
f"Target name {variable_token.text!r} already exists.",
324+
span=variable_token.span,
325+
target=variable_token.text,
326+
)
327+
operations.append(CreateVariableOperation(
328+
variable_token.text, "string", command.width,
329+
))
330+
spans.append(command.span)
331+
variables.append(VariableDefinition(
332+
variable_token.text, "string",
333+
declared_string_width=command.width,
334+
))
335+
continue
336+
if isinstance(command, DeleteVariablesCommandSyntax):
337+
for variable_token in command.variables:
338+
index, variable = _resolve(
339+
variables, variable_token.text, variable_token.span,
340+
)
341+
if len(variables) == 1:
342+
raise frontend_error(
343+
"cannot_delete_last_variable",
344+
"DELETE VARIABLES cannot remove the final dataset variable.",
345+
span=variable_token.span,
346+
variable=variable.name,
347+
)
348+
operations.append(DeleteVariableOperation(variable.name))
349+
spans.append(command.span)
350+
del variables[index]
351+
continue
308352
if isinstance(command, RecodeCommandSyntax):
309353
recodes, recode_spans = _bind_recode(command, variables)
310354
operations.extend(recodes)

‎src/openstatspec/frontends/spss/syntax.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,19 @@ class ExecuteCommandSyntax:
137137
span: SourceSpan
138138

139139

140+
@dataclass(frozen=True)
141+
class StringCommandSyntax:
142+
variables: tuple[Token, ...]
143+
width: int
144+
span: SourceSpan
145+
146+
147+
@dataclass(frozen=True)
148+
class DeleteVariablesCommandSyntax:
149+
variables: tuple[Token, ...]
150+
span: SourceSpan
151+
152+
140153

141154
@dataclass(frozen=True)
142155
class VariableLabelSyntax:
@@ -175,6 +188,7 @@ class ValueLabelsCommandSyntax:
175188
RecodeCommandSyntax | ComputeCommandSyntax | IfCommandSyntax
176189
| VariableLabelsCommandSyntax | ValueLabelsCommandSyntax
177190
| FormatsCommandSyntax | VariableLevelCommandSyntax | ExecuteCommandSyntax
191+
| StringCommandSyntax | DeleteVariablesCommandSyntax
178192
)
179193

180194

@@ -549,6 +563,43 @@ def execute(self, start: Token) -> ExecuteCommandSyntax:
549563
end = self.expects("period", "Expected '.' after EXECUTE.")
550564
return ExecuteCommandSyntax(_joined_span(start.span, end.span))
551565

566+
def string(self, start: Token) -> StringCommandSyntax:
567+
variables = self.variable_list(
568+
stop_kinds=frozenset({"left_paren", "period", "eof"}),
569+
)
570+
self.expects("left_paren", "Expected '(' before a STRING width.")
571+
width_token = self.expects(
572+
"identifier", "STRING requires a width such as A20.",
573+
)
574+
match = re.fullmatch(r"A([0-9]+)", width_token.text, re.IGNORECASE)
575+
if match is None:
576+
raise frontend_error(
577+
"spss_syntax_error",
578+
"STRING supports only character widths such as A20.",
579+
span=width_token.span,
580+
)
581+
width = int(match.group(1))
582+
if not 1 <= width <= 32767:
583+
raise frontend_error(
584+
"invalid_string_width",
585+
"STRING width must be between 1 and 32767.",
586+
span=width_token.span,
587+
width=width,
588+
)
589+
self.expects("right_paren", "Expected ')' after a STRING width.")
590+
end = self.expects("period", "Expected '.' after STRING.")
591+
return StringCommandSyntax(
592+
variables, width, _joined_span(start.span, end.span),
593+
)
594+
595+
def delete_variables(self, start: Token) -> DeleteVariablesCommandSyntax:
596+
self.expects_keyword("VARIABLES")
597+
variables = self.variable_list(stop_kinds=frozenset({"period", "eof"}))
598+
end = self.expects("period", "Expected '.' after DELETE VARIABLES.")
599+
return DeleteVariablesCommandSyntax(
600+
variables, _joined_span(start.span, end.span),
601+
)
602+
552603

553604
def recode_result(self) -> RecodeResultSyntax:
554605
if (token := self.accepts_keyword("SYSMIS")) is not None:
@@ -692,6 +743,10 @@ def parse(self) -> SpssSyntaxProgram:
692743
commands.append(self.formats(start))
693744
elif command == "execute":
694745
commands.append(self.execute(start))
746+
elif command == "string":
747+
commands.append(self.string(start))
748+
elif command == "delete":
749+
commands.append(self.delete_variables(start))
695750
elif command == "variable":
696751
if self.current.kind == "identifier" and self.current.text.casefold() == "level":
697752
commands.append(self.variable_level(start))

‎src/openstatspec/sql/inplace_transform.py‎

Lines changed: 104 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
from ..transform import (
1818
AssignOperation, BooleanExpression, ComparisonExpression,
19+
CreateVariableOperation, DeleteVariableOperation,
1920
ConditionalAssignOperation, ExecuteOperation, Operand,
2021
RecodeOperation, RecodeResult, ReplaceValueLabelsOperation,
2122
SetFormatOperation, SetMeasurementLevelOperation,
@@ -57,7 +58,8 @@ def in_place_transformation_capabilities() -> dict[str, Any]:
5758
"parent_kinds": ["core"],
5859
"mutation": "same_dataset_same_physical_wide_table",
5960
"commands": ["RECODE", "COMPUTE", "IF", "VARIABLE LABELS",
60-
"VALUE LABELS", "FORMATS", "VARIABLE LEVEL", "EXECUTE"],
61+
"VALUE LABELS", "FORMATS", "VARIABLE LEVEL",
62+
"STRING", "DELETE VARIABLES", "EXECUTE"],
6163
"new_target_column": {
6264
"sqlite": True,
6365
"postgresql": True,
@@ -211,6 +213,10 @@ def _input_schema(
211213
and str(row["storage_kind"]) == "numeric" else None
212214
),
213215
measurement_level=row["measurement_level"],
216+
declared_string_width=(
217+
int(row["declared_string_width"])
218+
if row["declared_string_width"] is not None else None
219+
),
214220
)
215221
for row in variables
216222
))
@@ -301,6 +307,53 @@ def _replace_value_labels(
301307
label=item.label,
302308
))
303309

310+
def _delete_variable_metadata(
311+
connection: Any,
312+
*,
313+
core: Any,
314+
variable: dict[str, Any],
315+
) -> None:
316+
"""Delete one variable and every normative catalog row owned by it."""
317+
variable_id = str(variable["variable_id"])
318+
label_set_id = connection.execute(
319+
select(core.variable_value_label_set.c.value_label_set_id).where(
320+
core.variable_value_label_set.c.variable_id == variable_id
321+
)
322+
).scalar_one_or_none()
323+
connection.execute(delete(core.dataset_weight_variable).where(
324+
core.dataset_weight_variable.c.variable_id == variable_id
325+
))
326+
connection.execute(delete(core.variable_set_member).where(
327+
core.variable_set_member.c.variable_id == variable_id
328+
))
329+
connection.execute(delete(core.multiple_response_member).where(
330+
core.multiple_response_member.c.variable_id == variable_id
331+
))
332+
connection.execute(delete(core.variable_attribute).where(
333+
core.variable_attribute.c.variable_id == variable_id
334+
))
335+
connection.execute(delete(core.missing_rule).where(
336+
core.missing_rule.c.variable_id == variable_id
337+
))
338+
connection.execute(delete(core.variable_value_label_set).where(
339+
core.variable_value_label_set.c.variable_id == variable_id
340+
))
341+
if label_set_id is not None:
342+
still_used = connection.execute(
343+
select(core.variable_value_label_set.c.variable_id).where(
344+
core.variable_value_label_set.c.value_label_set_id == label_set_id
345+
)
346+
).first()
347+
if still_used is None:
348+
connection.execute(delete(core.value_label).where(
349+
core.value_label.c.value_label_set_id == label_set_id
350+
))
351+
connection.execute(delete(core.value_label_set).where(
352+
core.value_label_set.c.value_label_set_id == label_set_id
353+
))
354+
connection.execute(delete(core.variable).where(
355+
core.variable.c.variable_id == variable_id
356+
))
304357

305358
def _failure_boundary(_name: str) -> None:
306359
"""Synthetic-test hook for schema/data/catalog/audit failure boundaries."""
@@ -382,17 +435,25 @@ def _apply_plan_on_connection(
382435
)
383436
create_operations = [
384437
operation for operation in plan.operations
385-
if isinstance(operation, (RecodeOperation, AssignOperation))
386-
and operation.target_mode == "create"
438+
if isinstance(operation, CreateVariableOperation)
439+
or (
440+
isinstance(operation, (RecodeOperation, AssignOperation))
441+
and operation.target_mode == "create"
442+
)
443+
]
444+
schema_operations = create_operations + [
445+
operation for operation in plan.operations
446+
if isinstance(operation, DeleteVariableOperation)
387447
]
388-
if create_operations and not allow_schema_change:
448+
if schema_operations and not allow_schema_change:
389449
raise TransformationError(
390450
"schema_change_not_atomic",
391451
"This database profile has no coherent new-target strategy.",
392452
)
393453
unsupported_targets = [
394454
operation.target for operation in create_operations
395-
if output_by_name[operation.target.casefold()].storage_kind != "numeric"
455+
if not isinstance(operation, CreateVariableOperation)
456+
and output_by_name[operation.target.casefold()].storage_kind != "numeric"
396457
]
397458
if unsupported_targets:
398459
raise TransformationError(
@@ -449,20 +510,37 @@ def _apply_plan_on_connection(
449510
# Compensation is journal-bounded to target columns and catalog identities
450511
# created by this apply, without resetting unrelated database state.
451512
for operation in create_operations:
452-
target_physical = physical_name(operation.target, used_physical)
513+
target_name = (
514+
operation.variable
515+
if isinstance(operation, CreateVariableOperation)
516+
else operation.target
517+
)
518+
target_physical = physical_name(target_name, used_physical)
519+
storage_kind = (
520+
operation.storage_kind
521+
if isinstance(operation, CreateVariableOperation)
522+
else "numeric"
523+
)
524+
string_width = (
525+
operation.declared_string_width
526+
if isinstance(operation, CreateVariableOperation)
527+
else None
528+
)
529+
column_type = numeric_type if storage_kind == "numeric" else f"VARCHAR({string_width})"
453530
connection.exec_driver_sql(
454531
f"ALTER TABLE {qualified_table} ADD COLUMN "
455-
f"{quote(target_physical)} {numeric_type} NULL"
532+
f"{quote(target_physical)} {column_type} NULL"
456533
)
457534
if mutation_journal is not None:
458535
mutation_journal["added_columns"].append(target_physical)
459536
target_row = {
460537
"variable_id": str(uuid4()),
461538
"dataset_id": dataset_id,
462539
"source_ordinal": next_ordinal,
463-
"source_name": operation.target,
540+
"source_name": target_name,
464541
"physical_name": target_physical,
465-
"storage_kind": "numeric",
542+
"storage_kind": storage_kind,
543+
"declared_string_width": string_width,
466544
"variable_label": None,
467545
}
468546
next_ordinal += 1
@@ -549,6 +627,23 @@ def _apply_plan_on_connection(
549627
core.variable.c.variable_id == variable["variable_id"]
550628
).values(measurement_level=operation.level))
551629
_failure_boundary("catalog")
630+
elif isinstance(operation, DeleteVariableOperation):
631+
variable = by_name[operation.variable.casefold()]
632+
connection.exec_driver_sql(
633+
f"ALTER TABLE {qualified_table} DROP COLUMN "
634+
f"{quote(variable['physical_name'])}"
635+
)
636+
_delete_variable_metadata(connection, core=core, variable=variable)
637+
by_name.pop(operation.variable.casefold(), None)
638+
variables = [
639+
row for row in variables
640+
if row["variable_id"] != variable["variable_id"]
641+
]
642+
relation = Table(
643+
table_name, MetaData(), schema=dataset.get("physical_table_schema"),
644+
autoload_with=connection,
645+
)
646+
_failure_boundary("schema")
552647
elif isinstance(operation, ExecuteOperation):
553648
continue
554649
else: # pragma: no cover

‎src/openstatspec/transform/__init__.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
from .errors import SourcePosition, SourceSpan, TransformationFrontendError
44
from .plan import (
5+
TRANSFORMATION_PLAN_SCHEMA_CHANGE_CONTRACT,
56
TRANSFORMATION_PLAN_CONTRACT,
67
AssignOperation, BooleanExpression, ComparisonExpression,
7-
ConditionalAssignOperation, ExecuteOperation, Operand, PredicateExpression,
8+
ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation,
9+
ExecuteOperation, Operand, PredicateExpression,
810
RecodeMatch, RecodeOperation, RecodeResult, RecodeRule,
911
ReplaceValueLabelsOperation, SetFormatOperation,
1012
SetMeasurementLevelOperation, SetVariableLabelOperation, TransformationPlan,
@@ -17,7 +19,7 @@
1719
from .validation import bind_transformation_plan
1820
__all__ = [
1921
"AssignOperation", "BooleanExpression", "ComparisonExpression",
20-
"ConditionalAssignOperation", "ExecuteOperation", "Operand",
22+
"ConditionalAssignOperation", "CreateVariableOperation", "DeleteVariableOperation",
2123
"PredicateExpression",
2224
"BoundTransformation",
2325
"RecodeMatch", "RecodeOperation", "RecodeResult",

0 commit comments

Comments
 (0)