diff --git a/fern/versions/latest/pages/concepts/columns.mdx b/fern/versions/latest/pages/concepts/columns.mdx index 8930e4803..7b25f6e20 100644 --- a/fern/versions/latest/pages/concepts/columns.mdx +++ b/fern/versions/latest/pages/concepts/columns.mdx @@ -251,6 +251,10 @@ Template capabilities: - **Conditional logic**: if/elif/else support - **Arithmetic**: Add, subtract, multiply, divide +Values from other columns keep their type in templates, so a text cell such as `"1.10"` or `"true"` renders exactly as written. Cast numbers stored as text before doing math with them, for example `{{ quantity | int + 1 }}` or `{{ price | float * 2 }}`. Columns holding a JSON object or array still support nested access such as `{{ result.score }}`. + +The same applies to `skip.when`. A text cell holding `"false"` or `"null"` is a non-empty string, so `when="{{ flag }}"` skips the cell and `when="{{ not flag }}"` does not. For flags stored as text, compare explicitly, for example `when="{{ flag == 'true' }}"`. + ### 🔍 Validation Columns Validation columns check generated content against rules and return structured pass/fail results. diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/embedding.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/embedding.py index e59cfc9bd..188d7a2e6 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/embedding.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/embedding.py @@ -7,7 +7,7 @@ from data_designer.config.column_configs import EmbeddingColumnConfig from data_designer.engine.column_generators.generators.base import ColumnGeneratorWithModel, GenerationStrategy -from data_designer.engine.processing.utils import deserialize_json_values, parse_list_string +from data_designer.engine.processing.utils import parse_list_string class EmbeddingGenerationResult(BaseModel): @@ -29,18 +29,15 @@ def get_generation_strategy() -> GenerationStrategy: def _prepare_embedding_inputs(self, data: dict) -> list[str]: cell_value = data[self.config.target_column] - deserialized_value = deserialize_json_values(data)[self.config.target_column] - if isinstance(deserialized_value, list): - return [str(text).strip() for text in deserialized_value] - if isinstance(deserialized_value, str): - return parse_list_string(deserialized_value) - if not isinstance(cell_value, str): - raise ValueError( - f"Embedding target column {self.config.target_column!r} must contain text or a list of texts, " - f"got {type(cell_value).__name__}" - ) - # A JSON scalar such as "123" or "true" is embedded as the user wrote it. - return [cell_value.strip()] + if isinstance(cell_value, list): + return [str(text).strip() for text in cell_value] + if isinstance(cell_value, str): + # A JSON scalar such as "123" or "true" is embedded as the user wrote it. + return parse_list_string(cell_value) + raise ValueError( + f"Embedding target column {self.config.target_column!r} must contain text or a list of texts, " + f"got {type(cell_value).__name__}" + ) def generate(self, data: dict) -> dict: input_texts = self._prepare_embedding_inputs(data) diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py index 57bf12d71..d93596fa2 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py @@ -14,7 +14,7 @@ from data_designer.engine.context import format_row_group_tag from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering from data_designer.engine.processing.ginja.exceptions import EmptyTemplateRenderError, UserTemplateError -from data_designer.engine.processing.utils import deserialize_json_values +from data_designer.engine.processing.utils import prepare_template_context if TYPE_CHECKING: import pandas as pd @@ -59,7 +59,7 @@ def generate(self, data: pd.DataFrame) -> pd.DataFrame: row_records = [{} for _ in row_indexes] for row_index, record in zip(row_indexes, row_records, strict=True): - prepared_record = deserialize_json_values(record) + prepared_record = prepare_template_context(record) try: rendered_value = self.render_template(prepared_record) except EmptyTemplateRenderError: diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py index 2060afb64..29b0f6192 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py @@ -9,7 +9,7 @@ from data_designer.config.column_configs import ImageColumnConfig from data_designer.engine.column_generators.generators.base import ColumnGeneratorWithModel, GenerationStrategy from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering -from data_designer.engine.processing.utils import deserialize_json_values +from data_designer.engine.processing.utils import prepare_template_context if TYPE_CHECKING: from data_designer.engine.storage.media_storage import MediaStorage @@ -34,7 +34,7 @@ def get_generation_strategy() -> GenerationStrategy: def _prepare_image_inputs(self, data: dict) -> tuple[str, list[dict] | None]: """Validate inputs and render prompt for image generation.""" - deserialized_record = deserialize_json_values(data) + deserialized_record = prepare_template_context(data) missing_columns = list(set(self.config.required_columns) - set(data.keys())) if len(missing_columns) > 0: raise ValueError( diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py index f0dbec4d9..a1cadf0ac 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py @@ -23,7 +23,7 @@ ) from data_designer.engine.configurable_task import TaskConfigT from data_designer.engine.models.recipes.base import ResponseRecipe -from data_designer.engine.processing.utils import deserialize_json_values +from data_designer.engine.processing.utils import deserialize_json_string, prepare_template_context if TYPE_CHECKING: from data_designer.engine.models.utils import ChatMessage @@ -78,7 +78,7 @@ def _prepare_generation_kwargs(self, data: dict) -> dict[str, Any]: # Deserialize input data from previous columns so Jinja2 templates can access nested fields # Example: If prev column stored '{"key": "value"}', templates can use {{ prev_column.key }} # Note: This creates a new dict and doesn't mutate the original `data` argument - deserialized_record = deserialize_json_values(data) + deserialized_record = prepare_template_context(data) multi_modal_context = self._build_multi_modal_context(deserialized_record) @@ -155,9 +155,9 @@ class LLMCodeCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMCodeColumnC class LLMStructuredCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMStructuredColumnConfig]): def _process_serialized_output(self, serialized_output: str) -> dict | list: - return deserialize_json_values(serialized_output) + return deserialize_json_string(serialized_output) class LLMJudgeCellGenerator(ColumnGeneratorWithModelChatCompletion[LLMJudgeColumnConfig]): def _process_serialized_output(self, serialized_output: str) -> dict | list: - return deserialize_json_values(serialized_output) + return deserialize_json_string(serialized_output) diff --git a/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py b/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py index 0a3d90725..771cc67de 100644 --- a/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py +++ b/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py @@ -15,7 +15,7 @@ from jinja2.sandbox import SandboxedEnvironment from data_designer.engine.dataset_builders.utils.skip_tracker import SKIPPED_COLUMNS_RECORD_KEY -from data_designer.engine.processing.utils import deserialize_json_values +from data_designer.engine.processing.utils import prepare_template_context if TYPE_CHECKING: from jinja2 import Template @@ -47,7 +47,7 @@ def evaluate_skip_when(expression: str, record: dict) -> bool: """ try: template = _compile_skip_template(expression) - deserialized = deserialize_json_values(record) + deserialized = prepare_template_context(record) result = template.render(deserialized) return bool(result) except (UndefinedError, SecurityError, TemplateSyntaxError, TypeError, ValueError): diff --git a/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py b/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py index 84539affe..c43d17ea7 100644 --- a/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py +++ b/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py @@ -12,7 +12,7 @@ from data_designer.config.utils.storage_schema import JsonStoragePlan from data_designer.engine.processing.ginja.environment import WithJinja2UserTemplateRendering from data_designer.engine.processing.processors.base import Processor -from data_designer.engine.processing.utils import deserialize_json_values +from data_designer.engine.processing.utils import prepare_template_context from data_designer.engine.storage.artifact_storage import BatchStage if TYPE_CHECKING: @@ -83,7 +83,7 @@ def _transform(self, data: pd.DataFrame) -> pd.DataFrame: ) formatted_records = [] for record in data.to_dict(orient="records"): - deserialized = deserialize_json_values(record) + deserialized = prepare_template_context(record) rendered = self.render_template(deserialized) formatted_records.append(json.loads(rendered)) return create_logical_dataframe(formatted_records, self.config.json_storage_columns) diff --git a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py index afdff5a49..0a5c910ec 100644 --- a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py +++ b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py @@ -8,7 +8,7 @@ import json import logging import re -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeVar import data_designer.lazy_heavy_imports as lazy @@ -18,8 +18,6 @@ logger = logging.getLogger(__name__) T = TypeVar("T") -K = TypeVar("K") -V = TypeVar("V") def concat_datasets(datasets: list[pd.DataFrame]) -> pd.DataFrame: @@ -30,87 +28,38 @@ def concat_datasets(datasets: list[pd.DataFrame]) -> pd.DataFrame: return lazy.pd.concat([df for df in datasets], axis=1) -# Overloads to help static type checker better understand -# the input/output types of the deserialize_json_values function. -@overload -def deserialize_json_values(data: str) -> dict[str, Any] | list[Any] | Any: ... - - -@overload -def deserialize_json_values(data: list[T]) -> list[Any]: ... - - -@overload -def deserialize_json_values(data: dict[K, V]) -> dict[K, Any]: ... - - -@overload -def deserialize_json_values(data: T) -> T: ... +def deserialize_json_string(value: str) -> Any: + """Decode a JSON document, returning the original string if it is not valid JSON.""" + try: + return json.loads(value) + except json.JSONDecodeError: + return value -def deserialize_json_values(data): - """De-serialize JSON strings in various input formats. +def prepare_template_context(data: T) -> Any: + """Decode JSON objects and arrays stored as strings so templates can reach their fields. - This function creates a deep copy of the input data and does not mutate the original. + Native dictionaries and lists are processed recursively. A string is replaced only when it holds a JSON + object or array, so `{{ column.key }}` works on a column that stored a JSON object, while scalar strings + like "1.10", "true" or "null" stay text. The input is not mutated. Args: - data: Input data in one of four formats: - - Single string (JSON string to deserialize) - - List of strings (list of JSON strings to deserialize) - - Dictionary (potentially with nested JSON strings to deserialize) - - Some other object that can't be deserialized. + data: A record, a list, a string, or any other value. Returns: - Deserialized data in the corresponding format: - - Dictionary (when input is a single string) - - List of dictionaries (when input is a list of strings) - - Dictionary (when input is a dictionary, with nested JSON strings deserialized) - - The original object (if there is no deserialization to perform) - + A copy of `data` with embedded JSON objects and arrays decoded. """ - # Create a deep copy to avoid mutating the original data - data_copy = copy.deepcopy(data) - - # Case 1: Single string input - if isinstance(data_copy, str): - try: - return json.loads(data_copy) - except json.JSONDecodeError: - return data_copy - - # Case 2: List of strings input - elif isinstance(data_copy, list): - result = [] - for item in data_copy: - if isinstance(item, str): - try: - result.append(json.loads(item)) - except json.JSONDecodeError: - result.append(item) - else: - # If list contains non-string items, recursively process them - result.append(deserialize_json_values(item)) - return result - - # Case 3: Dictionary input with potential nested JSON strings - elif isinstance(data_copy, dict): - result = {} - for key, value in data_copy.items(): - if isinstance(value, str): - try: - result[key] = json.loads(value) - except json.JSONDecodeError: - result[key] = value - elif isinstance(value, (dict, list)): - # Recursively process nested dictionaries and lists - result[key] = deserialize_json_values(value) - else: - result[key] = value - return result - - # Fallback for other data types - else: - return data_copy + if isinstance(data, str): + loaded = deserialize_json_string(data) + return loaded if isinstance(loaded, (dict, list)) else data + + if isinstance(data, list): + return [prepare_template_context(item) for item in data] + + if isinstance(data, dict): + return {key: prepare_template_context(value) for key, value in data.items()} + + return copy.deepcopy(data) def parse_list_string(text: str) -> list[str]: @@ -131,13 +80,17 @@ def parse_list_string(text: str) -> list[str]: # Try JSON again with cleaned text try: - return _clean_whitespace(json.loads(text_cleaned)) + list_obj = json.loads(text_cleaned) + if isinstance(list_obj, list): + return _clean_whitespace(list_obj) except json.JSONDecodeError: pass # Try Python literal eval (handles single quotes) try: - return _clean_whitespace(ast.literal_eval(text_cleaned)) + list_obj = ast.literal_eval(text_cleaned) + if isinstance(list_obj, list): + return _clean_whitespace(list_obj) except (ValueError, SyntaxError): pass @@ -145,8 +98,8 @@ def parse_list_string(text: str) -> list[str]: return [text.strip()] -def _clean_whitespace(texts: list[str]) -> list[str]: - return [text.strip() for text in texts] +def _clean_whitespace(texts: list[Any]) -> list[str]: + return [str(text).strip() for text in texts] def _verify_columns_are_unique(datasets: list[pd.DataFrame]) -> None: diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_embedding.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_embedding.py index b7d759f0e..139a961e8 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_embedding.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_embedding.py @@ -54,6 +54,11 @@ def test_embedding_cell_generator_generate(stub_embedding_column_config, stub_re ("123", ["123"]), ("true", ["true"]), ("null", ["null"]), + ('"quoted text"', ['"quoted text"']), + (["true", "1.10"], ["true", "1.10"]), + ('["true", "1.10"]', ["true", "1.10"]), + ("['true', '1.10']", ["true", "1.10"]), + ([" padded ", '{"a": 1}'], ["padded", '{"a": 1}']), ] diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py index a27f9f7bb..bbce70d0b 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py @@ -46,7 +46,7 @@ def test_generator_creation(): "data_designer.engine.column_generators.generators.expression.WithJinja2UserTemplateRendering.render_template", autospec=True, ) -@patch("data_designer.engine.column_generators.generators.expression.deserialize_json_values", autospec=True) +@patch("data_designer.engine.column_generators.generators.expression.prepare_template_context", autospec=True) def test_generate_with_dataframe(mock_deserialize, mock_render, mock_prepare): config = _create_test_config("sum_column", "{{ col1 + col2 }}", "int") generator = _create_test_generator(config) @@ -75,7 +75,7 @@ def test_generate_with_dataframe(mock_deserialize, mock_render, mock_prepare): "data_designer.engine.column_generators.generators.expression.WithJinja2UserTemplateRendering.render_template", autospec=True, ) -@patch("data_designer.engine.column_generators.generators.expression.deserialize_json_values", autospec=True) +@patch("data_designer.engine.column_generators.generators.expression.prepare_template_context", autospec=True) def test_generate_with_different_dtypes(mock_deserialize, mock_render, mock_prepare): config = _create_test_config("result_column", "{{ col1 }}", "float") generator = _create_test_generator(config) @@ -100,7 +100,7 @@ def test_generate_with_different_dtypes(mock_deserialize, mock_render, mock_prep "data_designer.engine.column_generators.generators.expression.WithJinja2UserTemplateRendering.render_template", autospec=True, ) -@patch("data_designer.engine.column_generators.generators.expression.deserialize_json_values", autospec=True) +@patch("data_designer.engine.column_generators.generators.expression.prepare_template_context", autospec=True) def test_generate_with_bool_dtype_numeric(mock_deserialize, mock_render, mock_prepare): config = _create_test_config("bool_column", "{{ col1 }}", "bool") generator = _create_test_generator(config) @@ -125,7 +125,7 @@ def test_generate_with_bool_dtype_numeric(mock_deserialize, mock_render, mock_pr "data_designer.engine.column_generators.generators.expression.WithJinja2UserTemplateRendering.render_template", autospec=True, ) -@patch("data_designer.engine.column_generators.generators.expression.deserialize_json_values", autospec=True) +@patch("data_designer.engine.column_generators.generators.expression.prepare_template_context", autospec=True) def test_generate_with_bool_dtype_string(mock_deserialize, mock_render, mock_prepare): config = _create_test_config("bool_column", "{{ col1 }}", "bool") generator = _create_test_generator(config) @@ -221,6 +221,28 @@ def test_generate_int_dtype_drops_only_the_out_of_range_row() -> None: assert result["number"].tolist() == [1, 3] +def test_generate_keeps_scalar_strings_from_previous_columns() -> None: + config = _create_test_config("label", "v{{ version }} {{ flag }} {{ flag == 'true' }} {{ meta.score }}", "str") + generator = _create_test_generator(config) + df = lazy.pd.DataFrame( + {"version": ["1.10", "1e3"], "flag": ["true", "null"], "meta": ['{"score": 95}', '{"score": 42}']} + ) + + result = generator.generate(df) + + assert result["label"].tolist() == ["v1.10 true True 95", "v1e3 null False 42"] + + +def test_generate_casts_numeric_strings_explicitly() -> None: + config = _create_test_config("next", "{{ n | int + 1 }}", "int") + generator = _create_test_generator(config) + df = lazy.pd.DataFrame({"n": ["5", "6"]}) + + result = generator.generate(df) + + assert result["next"].tolist() == [6, 7] + + def test_generate_with_missing_columns(): config = _create_test_config("test_column", "{{ col1 }}", "str") generator = _create_test_generator(config) diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py index bc5dc0c51..514e09dc8 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py @@ -114,6 +114,21 @@ def test_image_cell_generator_generate_in_dataframe_mode( mock_storage.save_base64_image.assert_any_call("base64_image_2", subfolder_name="test_image") +def test_image_cell_generator_keeps_scalar_strings_in_prompt(stub_resource_provider: None) -> None: + config = ImageColumnConfig( + name="test_image", prompt="Logo for v{{ version }}, beta={{ flag }}, {{ meta.style }}", model_alias="test_model" + ) + stub_resource_provider.artifact_storage.media_storage = Mock() + + with patch.object( + stub_resource_provider.model_registry.get_model.return_value, "generate_image", return_value=["base64_image"] + ) as mock_generate: + generator = ImageCellGenerator(config=config, resource_provider=stub_resource_provider) + generator.generate(data={"version": "1.10", "flag": "true", "meta": '{"style": "flat"}'}) + + mock_generate.assert_called_once_with(prompt="Logo for v1.10, beta=true, flat", multi_modal_context=None) + + def test_image_cell_generator_missing_columns_error(stub_image_column_config, stub_resource_provider): """Test that missing required columns raises ValueError.""" generator = ImageCellGenerator(config=stub_image_column_config, resource_provider=stub_resource_provider) diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py index 974c6c016..ff53e2ddb 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py @@ -402,6 +402,16 @@ def test_generate_with_json_deserialization(): assert result["test_column"] == {"result": "json_output"} +def test_generate_keeps_scalar_strings_in_prompt_record() -> None: + generator, _, mock_model, _, _, mock_prompt_renderer, mock_response_recipe = _create_generator_with_mocks() + _setup_generate_mocks(mock_prompt_renderer, mock_response_recipe, mock_model) + + generator.generate({"version": "1.10", "flag": "true", "meta": '{"score": 95}'}) + + record = mock_prompt_renderer.render.call_args_list[0].kwargs["record"] + assert record == {"version": "1.10", "flag": "true", "meta": {"score": 95}} + + def test_generate_passes_tool_alias() -> None: generator, mock_resource_provider, mock_model, _, _, mock_prompt_renderer, mock_response_recipe = ( _create_generator_with_mocks(tool_alias="search") @@ -444,6 +454,18 @@ def test_generate_passes_tool_alias() -> None: '{"field": "value", "nested": {"key": "val"}}', {"field": "value", "nested": {"key": "val"}}, ), + ( + LLMStructuredCellGenerator, + LLMStructuredColumnConfig, + { + "name": "struct_col", + "prompt": "Generate struct", + "model_alias": "test_model", + "output_format": {"type": "object", "properties": {"field": {"type": "string"}}}, + }, + '{"field": "1.10", "payload": "{\\"key\\": \\"val\\"}"}', + {"field": "1.10", "payload": '{"key": "val"}'}, + ), ( LLMJudgeCellGenerator, LLMJudgeColumnConfig, diff --git a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py index 5eb05e464..18cfbfbe8 100644 --- a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py +++ b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py @@ -32,6 +32,13 @@ def test_native_sandboxed_environment_returns_native_types() -> None: pytest.param("{{ x }}", {"x": 0}, False, id="native-zero"), pytest.param("{{ x }}", {"x": ""}, False, id="native-empty-string"), pytest.param('{{ x.key == "val" }}', {"x": '{"key": "val"}'}, True, id="deserializes-json"), + pytest.param("{{ x == 'true' }}", {"x": "true"}, True, id="keeps-scalar-string"), + pytest.param("{{ x == '1.10' }}", {"x": "1.10"}, True, id="keeps-numeric-string"), + pytest.param("{{ x | int > 5 }}", {"x": "7"}, True, id="numeric-string-with-cast"), + pytest.param("{{ x }}", {"x": "false"}, True, id="text-false-is-truthy"), + pytest.param("{{ x }}", {"x": "null"}, True, id="text-null-is-truthy"), + pytest.param("{{ not x }}", {"x": "false"}, False, id="not-text-false"), + pytest.param("{{ x == 'true' }}", {"x": "false"}, False, id="explicit-compare-text-false"), ], ) def test_evaluate_skip_when(expression: str, record: dict, expected: bool) -> None: diff --git a/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py b/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py index 20c01479b..1428b7ed0 100644 --- a/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py +++ b/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py @@ -218,6 +218,12 @@ def test_process_after_batch_with_mixed_special_characters(stub_processor: Schem [{"first_item": "alpha"}], id="list-indexing", ), + pytest.param( + {"version": ["1.10", "007"], "flag": ["true", "null"]}, + {"version": "v{{ version }}", "flag": "{{ flag }}"}, + [{"version": "v1.10", "flag": "true"}, {"version": "v007", "flag": "null"}], + id="scalar-strings-kept", + ), ], ) def test_process_after_batch_with_nested_field_access( diff --git a/packages/data-designer-engine/tests/engine/processing/test_utils.py b/packages/data-designer-engine/tests/engine/processing/test_utils.py index 7ca3bfce2..52da35fbc 100644 --- a/packages/data-designer-engine/tests/engine/processing/test_utils.py +++ b/packages/data-designer-engine/tests/engine/processing/test_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations +from typing import Any from unittest.mock import patch import pytest @@ -10,8 +11,9 @@ import data_designer.lazy_heavy_imports as lazy from data_designer.engine.processing.utils import ( concat_datasets, - deserialize_json_values, + deserialize_json_string, parse_list_string, + prepare_template_context, ) @@ -85,40 +87,86 @@ def test_concat_datasets_logging(mock_logger, stub_sample_dataframes): @pytest.mark.parametrize( - "test_case,input_data,expected_result", + "value,expected_result", [ - ("single_string_valid_json", '{"key": "value", "number": 42}', {"key": "value", "number": 42}), - ("single_string_invalid_json", '{"key": "value", "number": 42', '{"key": "value", "number": 42'), - ("list_of_strings", ['{"a": 1}', '{"b": 2}', "invalid_json"], [{"a": 1}, {"b": 2}, "invalid_json"]), - ("list_with_nested_structures", ['{"a": 1}', [2, 3], {"c": "d"}], [{"a": 1}, [2, 3], {"c": "d"}]), - ( - "dict_with_json_strings", - {"json_str": '{"nested": "value"}', "regular_str": "not_json", "number": 42}, - {"json_str": {"nested": "value"}, "regular_str": "not_json", "number": 42}, + pytest.param('{"key": "value", "number": 42}', {"key": "value", "number": 42}, id="object"), + pytest.param('[{"a": 1}, 2]', [{"a": 1}, 2], id="array"), + pytest.param('{"key": "value", "number": 42', '{"key": "value", "number": 42', id="invalid-json"), + pytest.param( + '{"inner": "{\\"a\\": 1}", "version": "1.10"}', {"inner": '{"a": 1}', "version": "1.10"}, id="one-pass" ), - ( - "dict_with_nested_structures", + pytest.param("1.10", 1.1, id="number"), + pytest.param("true", True, id="bool"), + pytest.param("null", None, id="null"), + pytest.param('"text"', "text", id="quoted-string"), + pytest.param("", "", id="empty"), + ], +) +def test_deserialize_json_string(value: str, expected_result: Any) -> None: + assert deserialize_json_string(value) == expected_result + + +@pytest.mark.parametrize( + "input_data,expected_result", + [ + pytest.param('{"key": 1.10}', {"key": 1.1}, id="object-string"), + pytest.param("[1, 2]", [1, 2], id="array-string"), + pytest.param("1.10", "1.10", id="number-string"), + pytest.param("true", "true", id="bool-string"), + pytest.param("null", "null", id="null-string"), + pytest.param('"text"', '"text"', id="quoted-string"), + pytest.param("not_json", "not_json", id="plain-string"), + pytest.param("", "", id="empty-string"), + pytest.param(42, 42, id="number"), + pytest.param(None, None, id="none"), + pytest.param([], [], id="empty-list"), + pytest.param({}, {}, id="empty-dict"), + pytest.param( + ["true", "1.10", '["a", "b"]', {"n": "007"}, 3], + ["true", "1.10", ["a", "b"], {"n": "007"}, 3], + id="list", + ), + pytest.param( { - "json_str": '{"nested": "value"}', - "nested_dict": {"inner": '{"deep": "value"}'}, - "nested_list": ['{"item": 1}', 2, 3], + "version": "1.10", + "flag": "true", + "missing": "null", + "big": "1e3", + "quoted": '"text"', + "padded": "007", + "obj": '{"score": 95}', + "arr": "[1, 2]", + "nested": {"inner": "false", "deep": '{"ok": true}', "items": ["null", '{"x": 1}']}, + "number": 42, }, { - "json_str": {"nested": "value"}, - "nested_dict": {"inner": {"deep": "value"}}, - "nested_list": [{"item": 1}, 2, 3], + "version": "1.10", + "flag": "true", + "missing": "null", + "big": "1e3", + "quoted": '"text"', + "padded": "007", + "obj": {"score": 95}, + "arr": [1, 2], + "nested": {"inner": "false", "deep": {"ok": True}, "items": ["null", {"x": 1}]}, + "number": 42, }, + id="dict", ), - ("non_string_non_dict_non_list", 42, 42), - ("none", None, None), - ("empty_string", "", ""), - ("empty_list", [], []), - ("empty_dict", {}, {}), ], ) -def test_deserialize_json_values_scenarios(test_case, input_data, expected_result): - result = deserialize_json_values(input_data) - assert result == expected_result +def test_prepare_template_context(input_data: Any, expected_result: Any) -> None: + assert prepare_template_context(input_data) == expected_result + + +def test_prepare_template_context_does_not_mutate_input() -> None: + record = {"obj": '{"a": 1}', "nested": {"items": ['{"b": 2}']}, "native": {"c": [1]}} + result = prepare_template_context(record) + + result["nested"]["items"][0]["b"] = 3 + result["native"]["c"].append(2) + + assert record == {"obj": '{"a": 1}', "nested": {"items": ['{"b": 2}']}, "native": {"c": [1]}} @pytest.mark.parametrize( @@ -130,6 +178,13 @@ def test_deserialize_json_values_scenarios(test_case, input_data, expected_resul ("['a', 'b', 'c']", ["a", "b", "c"]), # valid python-style list with single quotes ("['a', 'b', 'c', ]", ["a", "b", "c"]), # valid python-style list with trailing comma ("simple string ", ["simple string"]), # simple string with whitespace + ("[1, 2.5]", ["1", "2.5"]), # non-string items become text, as in a native list + ("123", ["123"]), # JSON number stays the text the user wrote + ("true", ["true"]), # JSON boolean + ("null", ["null"]), # JSON null + ('"text"', ['"text"']), # JSON-quoted text is not split into characters + ('{"a": 1,}', ['{"a": 1,}']), # object with a trailing comma is not a list + ("'text'", ["'text'"]), # Python string literal is not split into characters ], ) def test_parse_list_string_scenarios(input_string, expected_result):