From 25cddc86f1cb725619621776c40afae21070b1b6 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Fri, 30 Oct 2026 16:58:07 +0300 Subject: [PATCH 1/3] fix: keep scalar strings as text in template contexts Template contexts ran every string cell through json.loads, so "1.10" rendered as 1.1, "true" as True and "null" as None. Only replace strings that hold a JSON object or array there, so nested access keeps working and text renders as written. Embedding inputs and structured-response parsing still use the full conversion. Closes #1088 Signed-off-by: Dmitry Voropaev --- .../latest/pages/concepts/columns.mdx | 2 + .../generators/expression.py | 2 +- .../column_generators/generators/image.py | 2 +- .../generators/llm_completion.py | 2 +- .../dataset_builders/utils/skip_evaluator.py | 2 +- .../processing/processors/schema_transform.py | 2 +- .../data_designer/engine/processing/utils.py | 42 ++++++++++--------- .../generators/test_expression.py | 30 +++++++++++-- .../generators/test_image.py | 15 +++++++ .../test_llm_completion_generators.py | 10 +++++ .../utils/test_skip_evaluator.py | 3 ++ .../processors/test_schema_transform.py | 6 +++ .../tests/engine/processing/test_utils.py | 39 +++++++++++++++++ 13 files changed, 129 insertions(+), 28 deletions(-) diff --git a/fern/versions/latest/pages/concepts/columns.mdx b/fern/versions/latest/pages/concepts/columns.mdx index 8930e4803..78af58fd3 100644 --- a/fern/versions/latest/pages/concepts/columns.mdx +++ b/fern/versions/latest/pages/concepts/columns.mdx @@ -251,6 +251,8 @@ Template capabilities: - **Conditional logic**: if/elif/else support - **Arithmetic**: Add, subtract, multiply, divide +Values from other columns keep their type in templates, so a text cell such as `"1.10"` or `"true"` renders exactly as written. Cast numbers stored as text before doing math with them, for example `{{ quantity | int + 1 }}` or `{{ price | float * 2 }}`. Columns holding a JSON object or array still support nested access such as `{{ result.score }}`. + ### 🔍 Validation Columns Validation columns check generated content against rules and return structured pass/fail results. diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py index 57bf12d71..5b9e1fb38 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/expression.py @@ -59,7 +59,7 @@ def generate(self, data: pd.DataFrame) -> pd.DataFrame: row_records = [{} for _ in row_indexes] for row_index, record in zip(row_indexes, row_records, strict=True): - prepared_record = deserialize_json_values(record) + prepared_record = deserialize_json_values(record, keep_scalar_strings=True) try: rendered_value = self.render_template(prepared_record) except EmptyTemplateRenderError: diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py index 2060afb64..64db976ab 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/image.py @@ -34,7 +34,7 @@ def get_generation_strategy() -> GenerationStrategy: def _prepare_image_inputs(self, data: dict) -> tuple[str, list[dict] | None]: """Validate inputs and render prompt for image generation.""" - deserialized_record = deserialize_json_values(data) + deserialized_record = deserialize_json_values(data, keep_scalar_strings=True) missing_columns = list(set(self.config.required_columns) - set(data.keys())) if len(missing_columns) > 0: raise ValueError( diff --git a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py index f0dbec4d9..276803485 100644 --- a/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py +++ b/packages/data-designer-engine/src/data_designer/engine/column_generators/generators/llm_completion.py @@ -78,7 +78,7 @@ def _prepare_generation_kwargs(self, data: dict) -> dict[str, Any]: # Deserialize input data from previous columns so Jinja2 templates can access nested fields # Example: If prev column stored '{"key": "value"}', templates can use {{ prev_column.key }} # Note: This creates a new dict and doesn't mutate the original `data` argument - deserialized_record = deserialize_json_values(data) + deserialized_record = deserialize_json_values(data, keep_scalar_strings=True) multi_modal_context = self._build_multi_modal_context(deserialized_record) diff --git a/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py b/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py index 0a3d90725..60e858550 100644 --- a/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py +++ b/packages/data-designer-engine/src/data_designer/engine/dataset_builders/utils/skip_evaluator.py @@ -47,7 +47,7 @@ def evaluate_skip_when(expression: str, record: dict) -> bool: """ try: template = _compile_skip_template(expression) - deserialized = deserialize_json_values(record) + deserialized = deserialize_json_values(record, keep_scalar_strings=True) result = template.render(deserialized) return bool(result) except (UndefinedError, SecurityError, TemplateSyntaxError, TypeError, ValueError): diff --git a/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py b/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py index 84539affe..3c82b4f7d 100644 --- a/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py +++ b/packages/data-designer-engine/src/data_designer/engine/processing/processors/schema_transform.py @@ -83,7 +83,7 @@ def _transform(self, data: pd.DataFrame) -> pd.DataFrame: ) formatted_records = [] for record in data.to_dict(orient="records"): - deserialized = deserialize_json_values(record) + deserialized = deserialize_json_values(record, keep_scalar_strings=True) rendered = self.render_template(deserialized) formatted_records.append(json.loads(rendered)) return create_logical_dataframe(formatted_records, self.config.json_storage_columns) diff --git a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py index afdff5a49..e1f9a1295 100644 --- a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py +++ b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py @@ -33,22 +33,22 @@ def concat_datasets(datasets: list[pd.DataFrame]) -> pd.DataFrame: # Overloads to help static type checker better understand # the input/output types of the deserialize_json_values function. @overload -def deserialize_json_values(data: str) -> dict[str, Any] | list[Any] | Any: ... +def deserialize_json_values(data: str, *, keep_scalar_strings: bool = False) -> dict[str, Any] | list[Any] | Any: ... @overload -def deserialize_json_values(data: list[T]) -> list[Any]: ... +def deserialize_json_values(data: list[T], *, keep_scalar_strings: bool = False) -> list[Any]: ... @overload -def deserialize_json_values(data: dict[K, V]) -> dict[K, Any]: ... +def deserialize_json_values(data: dict[K, V], *, keep_scalar_strings: bool = False) -> dict[K, Any]: ... @overload -def deserialize_json_values(data: T) -> T: ... +def deserialize_json_values(data: T, *, keep_scalar_strings: bool = False) -> T: ... -def deserialize_json_values(data): +def deserialize_json_values(data, *, keep_scalar_strings=False): """De-serialize JSON strings in various input formats. This function creates a deep copy of the input data and does not mutate the original. @@ -59,6 +59,9 @@ def deserialize_json_values(data): - List of strings (list of JSON strings to deserialize) - Dictionary (potentially with nested JSON strings to deserialize) - Some other object that can't be deserialized. + keep_scalar_strings: Only replace strings that hold a JSON object or array, and keep strings like + "1.10", "true" or "null" as they are. Template contexts use this so a text cell renders as written + while `{{ column.key }}` still works on a column that stored a JSON object. Returns: Deserialized data in the corresponding format: @@ -73,23 +76,17 @@ def deserialize_json_values(data): # Case 1: Single string input if isinstance(data_copy, str): - try: - return json.loads(data_copy) - except json.JSONDecodeError: - return data_copy + return _loads_json_string(data_copy, keep_scalar_strings) # Case 2: List of strings input elif isinstance(data_copy, list): result = [] for item in data_copy: if isinstance(item, str): - try: - result.append(json.loads(item)) - except json.JSONDecodeError: - result.append(item) + result.append(_loads_json_string(item, keep_scalar_strings)) else: # If list contains non-string items, recursively process them - result.append(deserialize_json_values(item)) + result.append(deserialize_json_values(item, keep_scalar_strings=keep_scalar_strings)) return result # Case 3: Dictionary input with potential nested JSON strings @@ -97,13 +94,10 @@ def deserialize_json_values(data): result = {} for key, value in data_copy.items(): if isinstance(value, str): - try: - result[key] = json.loads(value) - except json.JSONDecodeError: - result[key] = value + result[key] = _loads_json_string(value, keep_scalar_strings) elif isinstance(value, (dict, list)): # Recursively process nested dictionaries and lists - result[key] = deserialize_json_values(value) + result[key] = deserialize_json_values(value, keep_scalar_strings=keep_scalar_strings) else: result[key] = value return result @@ -113,6 +107,16 @@ def deserialize_json_values(data): return data_copy +def _loads_json_string(value: str, keep_scalar_strings: bool) -> Any: + try: + loaded = json.loads(value) + except json.JSONDecodeError: + return value + if keep_scalar_strings and not isinstance(loaded, (dict, list)): + return value + return loaded + + def parse_list_string(text: str) -> list[str]: """Parse a list from a string, handling JSON arrays, Python lists, and trailing commas.""" text = text.strip() diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py index a27f9f7bb..9280b09bf 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_expression.py @@ -54,7 +54,7 @@ def test_generate_with_dataframe(mock_deserialize, mock_render, mock_prepare): df = lazy.pd.DataFrame({"col1": [1, 2, 3], "col2": [4, 5, 6]}) mock_prepare.return_value = None - mock_deserialize.side_effect = lambda x: x + mock_deserialize.side_effect = lambda x, **_: x mock_render.side_effect = ["5", "7", "9"] result = generator.generate(df) @@ -83,7 +83,7 @@ def test_generate_with_different_dtypes(mock_deserialize, mock_render, mock_prep df = lazy.pd.DataFrame({"col1": [1, 2, 3]}) mock_prepare.return_value = None - mock_deserialize.side_effect = lambda x: x + mock_deserialize.side_effect = lambda x, **_: x mock_render.side_effect = ["1.5", "2.7", "3.9"] result = generator.generate(df) @@ -108,7 +108,7 @@ def test_generate_with_bool_dtype_numeric(mock_deserialize, mock_render, mock_pr df = lazy.pd.DataFrame({"col1": [1, 0, 1]}) mock_prepare.return_value = None - mock_deserialize.side_effect = lambda x: x + mock_deserialize.side_effect = lambda x, **_: x mock_render.side_effect = ["1", "0", "1.0"] result = generator.generate(df) @@ -133,7 +133,7 @@ def test_generate_with_bool_dtype_string(mock_deserialize, mock_render, mock_pre df = lazy.pd.DataFrame({"col1": ["true", "false", "True"]}) mock_prepare.return_value = None - mock_deserialize.side_effect = lambda x: x + mock_deserialize.side_effect = lambda x, **_: x mock_render.side_effect = ["true", "false", "True"] result = generator.generate(df) @@ -221,6 +221,28 @@ def test_generate_int_dtype_drops_only_the_out_of_range_row() -> None: assert result["number"].tolist() == [1, 3] +def test_generate_keeps_scalar_strings_from_previous_columns() -> None: + config = _create_test_config("label", "v{{ version }} {{ flag }} {{ flag == 'true' }} {{ meta.score }}", "str") + generator = _create_test_generator(config) + df = lazy.pd.DataFrame( + {"version": ["1.10", "1e3"], "flag": ["true", "null"], "meta": ['{"score": 95}', '{"score": 42}']} + ) + + result = generator.generate(df) + + assert result["label"].tolist() == ["v1.10 true True 95", "v1e3 null False 42"] + + +def test_generate_casts_numeric_strings_explicitly() -> None: + config = _create_test_config("next", "{{ n | int + 1 }}", "int") + generator = _create_test_generator(config) + df = lazy.pd.DataFrame({"n": ["5", "6"]}) + + result = generator.generate(df) + + assert result["next"].tolist() == [6, 7] + + def test_generate_with_missing_columns(): config = _create_test_config("test_column", "{{ col1 }}", "str") generator = _create_test_generator(config) diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py index bc5dc0c51..1a0ae0863 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py @@ -114,6 +114,21 @@ def test_image_cell_generator_generate_in_dataframe_mode( mock_storage.save_base64_image.assert_any_call("base64_image_2", subfolder_name="test_image") +def test_image_cell_generator_keeps_scalar_strings_in_prompt(stub_resource_provider): + config = ImageColumnConfig( + name="test_image", prompt="Logo for v{{ version }}, beta={{ flag }}, {{ meta.style }}", model_alias="test_model" + ) + stub_resource_provider.artifact_storage.media_storage = Mock() + + with patch.object( + stub_resource_provider.model_registry.get_model.return_value, "generate_image", return_value=["base64_image"] + ) as mock_generate: + generator = ImageCellGenerator(config=config, resource_provider=stub_resource_provider) + generator.generate(data={"version": "1.10", "flag": "true", "meta": '{"style": "flat"}'}) + + mock_generate.assert_called_once_with(prompt="Logo for v1.10, beta=true, flat", multi_modal_context=None) + + def test_image_cell_generator_missing_columns_error(stub_image_column_config, stub_resource_provider): """Test that missing required columns raises ValueError.""" generator = ImageCellGenerator(config=stub_image_column_config, resource_provider=stub_resource_provider) diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py index 974c6c016..887961861 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_llm_completion_generators.py @@ -402,6 +402,16 @@ def test_generate_with_json_deserialization(): assert result["test_column"] == {"result": "json_output"} +def test_generate_keeps_scalar_strings_in_prompt_record() -> None: + generator, _, mock_model, _, _, mock_prompt_renderer, mock_response_recipe = _create_generator_with_mocks() + _setup_generate_mocks(mock_prompt_renderer, mock_response_recipe, mock_model) + + generator.generate({"version": "1.10", "flag": "true", "meta": '{"score": 95}'}) + + record = mock_prompt_renderer.render.call_args_list[0].kwargs["record"] + assert record == {"version": "1.10", "flag": "true", "meta": {"score": 95}} + + def test_generate_passes_tool_alias() -> None: generator, mock_resource_provider, mock_model, _, _, mock_prompt_renderer, mock_response_recipe = ( _create_generator_with_mocks(tool_alias="search") diff --git a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py index 5eb05e464..9fe824eca 100644 --- a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py +++ b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py @@ -32,6 +32,9 @@ def test_native_sandboxed_environment_returns_native_types() -> None: pytest.param("{{ x }}", {"x": 0}, False, id="native-zero"), pytest.param("{{ x }}", {"x": ""}, False, id="native-empty-string"), pytest.param('{{ x.key == "val" }}', {"x": '{"key": "val"}'}, True, id="deserializes-json"), + pytest.param("{{ x == 'true' }}", {"x": "true"}, True, id="keeps-scalar-string"), + pytest.param("{{ x == '1.10' }}", {"x": "1.10"}, True, id="keeps-numeric-string"), + pytest.param("{{ x | int > 5 }}", {"x": "7"}, True, id="numeric-string-with-cast"), ], ) def test_evaluate_skip_when(expression: str, record: dict, expected: bool) -> None: diff --git a/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py b/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py index 20c01479b..1428b7ed0 100644 --- a/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py +++ b/packages/data-designer-engine/tests/engine/processing/processors/test_schema_transform.py @@ -218,6 +218,12 @@ def test_process_after_batch_with_mixed_special_characters(stub_processor: Schem [{"first_item": "alpha"}], id="list-indexing", ), + pytest.param( + {"version": ["1.10", "007"], "flag": ["true", "null"]}, + {"version": "v{{ version }}", "flag": "{{ flag }}"}, + [{"version": "v1.10", "flag": "true"}, {"version": "v007", "flag": "null"}], + id="scalar-strings-kept", + ), ], ) def test_process_after_batch_with_nested_field_access( diff --git a/packages/data-designer-engine/tests/engine/processing/test_utils.py b/packages/data-designer-engine/tests/engine/processing/test_utils.py index 7ca3bfce2..4705b0d4b 100644 --- a/packages/data-designer-engine/tests/engine/processing/test_utils.py +++ b/packages/data-designer-engine/tests/engine/processing/test_utils.py @@ -121,6 +121,45 @@ def test_deserialize_json_values_scenarios(test_case, input_data, expected_resul assert result == expected_result +@pytest.mark.parametrize( + "input_data,expected_result", + [ + pytest.param("1.10", "1.10", id="single-scalar-string"), + pytest.param('{"key": 1.10}', {"key": 1.1}, id="single-object-string"), + pytest.param(["true", '["a", "b"]'], ["true", ["a", "b"]], id="list"), + pytest.param( + { + "version": "1.10", + "flag": "true", + "missing": "null", + "big": "1e3", + "quoted": '"text"', + "padded": "007", + "obj": '{"score": 95}', + "arr": "[1, 2]", + "nested": {"inner": "false", "deep": '{"ok": true}'}, + "number": 42, + }, + { + "version": "1.10", + "flag": "true", + "missing": "null", + "big": "1e3", + "quoted": '"text"', + "padded": "007", + "obj": {"score": 95}, + "arr": [1, 2], + "nested": {"inner": "false", "deep": {"ok": True}}, + "number": 42, + }, + id="dict", + ), + ], +) +def test_deserialize_json_values_keep_scalar_strings(input_data, expected_result): + assert deserialize_json_values(input_data, keep_scalar_strings=True) == expected_result + + @pytest.mark.parametrize( "input_string,expected_result", [ From 04cb2202599802fb11752ffe2edc85d2a5532f52 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Fri, 30 Oct 2026 21:54:01 +0300 Subject: [PATCH 2/3] test: pin skip.when text flags and default-mode scalar parsing Document that text "false"/"null" are truthy in skip.when and that explicit comparisons are the safe pattern for text flags. Signed-off-by: Dmitry Voropaev --- fern/versions/latest/pages/concepts/columns.mdx | 2 ++ .../dataset_builders/utils/test_skip_evaluator.py | 4 ++++ .../tests/engine/processing/test_utils.py | 10 ++++++++++ 3 files changed, 16 insertions(+) diff --git a/fern/versions/latest/pages/concepts/columns.mdx b/fern/versions/latest/pages/concepts/columns.mdx index 78af58fd3..7b25f6e20 100644 --- a/fern/versions/latest/pages/concepts/columns.mdx +++ b/fern/versions/latest/pages/concepts/columns.mdx @@ -253,6 +253,8 @@ Template capabilities: Values from other columns keep their type in templates, so a text cell such as `"1.10"` or `"true"` renders exactly as written. Cast numbers stored as text before doing math with them, for example `{{ quantity | int + 1 }}` or `{{ price | float * 2 }}`. Columns holding a JSON object or array still support nested access such as `{{ result.score }}`. +The same applies to `skip.when`. A text cell holding `"false"` or `"null"` is a non-empty string, so `when="{{ flag }}"` skips the cell and `when="{{ not flag }}"` does not. For flags stored as text, compare explicitly, for example `when="{{ flag == 'true' }}"`. + ### 🔍 Validation Columns Validation columns check generated content against rules and return structured pass/fail results. diff --git a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py index 9fe824eca..18cfbfbe8 100644 --- a/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py +++ b/packages/data-designer-engine/tests/engine/dataset_builders/utils/test_skip_evaluator.py @@ -35,6 +35,10 @@ def test_native_sandboxed_environment_returns_native_types() -> None: pytest.param("{{ x == 'true' }}", {"x": "true"}, True, id="keeps-scalar-string"), pytest.param("{{ x == '1.10' }}", {"x": "1.10"}, True, id="keeps-numeric-string"), pytest.param("{{ x | int > 5 }}", {"x": "7"}, True, id="numeric-string-with-cast"), + pytest.param("{{ x }}", {"x": "false"}, True, id="text-false-is-truthy"), + pytest.param("{{ x }}", {"x": "null"}, True, id="text-null-is-truthy"), + pytest.param("{{ not x }}", {"x": "false"}, False, id="not-text-false"), + pytest.param("{{ x == 'true' }}", {"x": "false"}, False, id="explicit-compare-text-false"), ], ) def test_evaluate_skip_when(expression: str, record: dict, expected: bool) -> None: diff --git a/packages/data-designer-engine/tests/engine/processing/test_utils.py b/packages/data-designer-engine/tests/engine/processing/test_utils.py index 4705b0d4b..91b5622a7 100644 --- a/packages/data-designer-engine/tests/engine/processing/test_utils.py +++ b/packages/data-designer-engine/tests/engine/processing/test_utils.py @@ -114,6 +114,16 @@ def test_concat_datasets_logging(mock_logger, stub_sample_dataframes): ("empty_string", "", ""), ("empty_list", [], []), ("empty_dict", {}, {}), + ("scalar_number_string", "1.10", 1.1), + ("scalar_bool_string", "true", True), + ("scalar_null_string", "null", None), + ("scalar_quoted_string", '"text"', "text"), + ("list_of_scalar_strings", ["1.10", "true", "null", '"text"'], [1.1, True, None, "text"]), + ( + "dict_with_nested_scalar_strings", + {"version": "1.10", "nested": {"flag": "false", "deep": {"missing": "null", "quoted": '"x"'}}}, + {"version": 1.1, "nested": {"flag": False, "deep": {"missing": None, "quoted": "x"}}}, + ), ], ) def test_deserialize_json_values_scenarios(test_case, input_data, expected_result): From 9105c1dd6df7a196b678ec4170acc8c900a3e343 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Sat, 31 Oct 2026 00:47:41 +0300 Subject: [PATCH 3/3] chore: annotate new tests and move private helper below public functions Signed-off-by: Dmitry Voropaev --- .../data_designer/engine/processing/utils.py | 20 +++++++++---------- .../generators/test_image.py | 2 +- .../tests/engine/processing/test_utils.py | 3 ++- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py index e1f9a1295..5b373a899 100644 --- a/packages/data-designer-engine/src/data_designer/engine/processing/utils.py +++ b/packages/data-designer-engine/src/data_designer/engine/processing/utils.py @@ -107,16 +107,6 @@ def deserialize_json_values(data, *, keep_scalar_strings=False): return data_copy -def _loads_json_string(value: str, keep_scalar_strings: bool) -> Any: - try: - loaded = json.loads(value) - except json.JSONDecodeError: - return value - if keep_scalar_strings and not isinstance(loaded, (dict, list)): - return value - return loaded - - def parse_list_string(text: str) -> list[str]: """Parse a list from a string, handling JSON arrays, Python lists, and trailing commas.""" text = text.strip() @@ -149,6 +139,16 @@ def parse_list_string(text: str) -> list[str]: return [text.strip()] +def _loads_json_string(value: str, keep_scalar_strings: bool) -> Any: + try: + loaded = json.loads(value) + except json.JSONDecodeError: + return value + if keep_scalar_strings and not isinstance(loaded, (dict, list)): + return value + return loaded + + def _clean_whitespace(texts: list[str]) -> list[str]: return [text.strip() for text in texts] diff --git a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py index 1a0ae0863..514e09dc8 100644 --- a/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py +++ b/packages/data-designer-engine/tests/engine/column_generators/generators/test_image.py @@ -114,7 +114,7 @@ def test_image_cell_generator_generate_in_dataframe_mode( mock_storage.save_base64_image.assert_any_call("base64_image_2", subfolder_name="test_image") -def test_image_cell_generator_keeps_scalar_strings_in_prompt(stub_resource_provider): +def test_image_cell_generator_keeps_scalar_strings_in_prompt(stub_resource_provider: None) -> None: config = ImageColumnConfig( name="test_image", prompt="Logo for v{{ version }}, beta={{ flag }}, {{ meta.style }}", model_alias="test_model" ) diff --git a/packages/data-designer-engine/tests/engine/processing/test_utils.py b/packages/data-designer-engine/tests/engine/processing/test_utils.py index 91b5622a7..b480850ae 100644 --- a/packages/data-designer-engine/tests/engine/processing/test_utils.py +++ b/packages/data-designer-engine/tests/engine/processing/test_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations +from typing import Any from unittest.mock import patch import pytest @@ -166,7 +167,7 @@ def test_deserialize_json_values_scenarios(test_case, input_data, expected_resul ), ], ) -def test_deserialize_json_values_keep_scalar_strings(input_data, expected_result): +def test_deserialize_json_values_keep_scalar_strings(input_data: Any, expected_result: Any) -> None: assert deserialize_json_values(input_data, keep_scalar_strings=True) == expected_result