Skip to content

Commit daa773f

Browse files
committed
gh-159672: Release trial identifiers with their arena
Skip immortal interning for replay-only identifiers and dotted names. The parser can then release those strings after each trial.
1 parent 05fb3a1 commit daa773f

3 files changed

Lines changed: 34 additions & 5 deletions

File tree

‎Lib/test/test_traceback.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1951,6 +1951,30 @@ def test_keyword_trials_do_not_call_normalizer(self):
19511951
normalize.assert_not_called()
19521952
self.assertIsNone(caught.exception._metadata)
19531953

1954+
@cpython_only
1955+
@requires_subprocess()
1956+
def test_keyword_trials_do_not_intern_names(self):
1957+
assert_python_ok('-c', textwrap.dedent('''
1958+
import sys
1959+
1960+
def check(index, suffix):
1961+
source = (f"impor math\\nidentifier_after_typo_{index} = 1\\n"
1962+
f"import module_after_typo_{index}.child" + suffix)
1963+
try:
1964+
compile(source, "<test>", "exec")
1965+
except SyntaxError as error:
1966+
assert (error._metadata is not None) == (not suffix)
1967+
else:
1968+
raise AssertionError("expected SyntaxError")
1969+
1970+
for suffix in ("", "\\nother ="):
1971+
check(0, suffix)
1972+
before = sys.getunicodeinternedsize(_only_immortal=True)
1973+
for index in range(1, 20):
1974+
check(index, suffix)
1975+
assert sys.getunicodeinternedsize(_only_immortal=True) == before
1976+
'''))
1977+
19541978
@cpython_only
19551979
def test_keyword_replay_source_limit(self):
19561980
with tempfile.TemporaryDirectory() as script_dir:

‎Parser/action_helpers.c‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,10 @@ _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
128128
if (!uni) {
129129
return NULL;
130130
}
131-
PyInterpreterState *interp = _PyInterpreterState_GET();
132-
_PyUnicode_InternImmortal(interp, &uni);
131+
if (p->keyword_trial == NULL) {
132+
PyInterpreterState *interp = _PyInterpreterState_GET();
133+
_PyUnicode_InternImmortal(interp, &uni);
134+
}
133135
if (_PyArena_AddPyObject(p->arena, uni) < 0) {
134136
Py_DECREF(uni);
135137
return NULL;

‎Parser/pegen.c‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,8 +662,11 @@ _PyPegen_new_identifier(Parser *p, const char *n)
662662
goto error;
663663
}
664664
}
665-
PyInterpreterState *interp = _PyInterpreterState_GET();
666-
_PyUnicode_InternImmortal(interp, &id);
665+
// Trial-only identifiers must die with the trial arena.
666+
if (p->keyword_trial == NULL) {
667+
PyInterpreterState *interp = _PyInterpreterState_GET();
668+
_PyUnicode_InternImmortal(interp, &id);
669+
}
667670
if (_PyArena_AddPyObject(p->arena, id) < 0)
668671
{
669672
Py_DECREF(id);
@@ -687,7 +690,7 @@ get_cached_identifier(Parser *p, PyObject *bytes)
687690
}
688691
// Identifiers repeat constantly; a small span-keyed cache skips the
689692
// UTF-8 decode + intern for repeated occurrences. Keys point into
690-
// arena-owned token bytes and values are arena-owned interned strings,
693+
// arena-owned token bytes and values are arena-owned strings,
691694
// so borrowed references are valid for the lifetime of the parse
692695
// (including the second error pass, which reuses parser and arena).
693696
Py_ssize_t len = PyBytes_GET_SIZE(bytes);

0 commit comments

Comments
 (0)