Match CPython on code objects, ast, symtable and syntax error positions - #8580

#!/usr/bin/env bash
set -euo pipefail

rg -n -C 3 \
  'constant_data_to_ast_constant_value|to_string_lossy|to_string\(\).*into_boxed_str|ConstantLiteral::Str' \
  crates/vm/src/stdlib/_ast

rg -n -C 2 \
  'ast\.Constant|JoinedStr|TemplateStr|compile\(' \
  crates/vm/src --glob '*test*.rs'

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- constant.rs conversion paths ---'
sed -n '1,280p' crates/vm/src/stdlib/_ast/constant.rs
sed -n '520,575p' crates/vm/src/stdlib/_ast/constant.rs

printf '%s\n' '--- string.rs affected paths ---'
sed -n '80,175p' crates/vm/src/stdlib/_ast/string.rs
sed -n '490,540p' crates/vm/src/stdlib/_ast/string.rs
sed -n '970,1030p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- AST value definitions and WTF-8 APIs ---'
rg -n -C 4 \
  'enum ConstantValue|struct ConstantValue|type ConstantValue|Wtf8Buf|to_string_lossy|as_wtf8|InterpolatedStringLiteralElement' \
  crates --glob '*.rs' --glob '*.toml'

printf '%s\n' '--- related tests and compiler consumers ---'
rg -n -C 4 \
  'JoinedStr|TemplateStr|InterpolatedString|ConstantValue::Str|constant_data_to_ast_constant_value|ConstantLiteral::Str|U\+D800|surrogate|surrogat' \
  crates --glob '*test*.rs' --glob '*.rs'

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- constant conversion implementation ---'
sed -n '80,115p' crates/vm/src/stdlib/_ast/constant.rs
sed -n '180,265p' crates/vm/src/stdlib/_ast/constant.rs
sed -n '530,565p' crates/vm/src/stdlib/_ast/constant.rs

printf '%s\n' '--- string conversion implementation ---'
sed -n '505,532p' crates/vm/src/stdlib/_ast/string.rs
sed -n '988,1025p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- compiler literal consumption ---'
sed -n '12755,12825p' crates/codegen/src/compile.rs
rg -n 'fn interpolated_string_literal_value|interpolated_string_literal_value\(' crates/codegen/src/compile.rs crates --glob '*.rs'
sed -n '21955,21990p' crates/codegen/src/compile.rs
sed -n '30840,30870p' crates/codegen/src/compile.rs

printf '%s\n' '--- exact AST type declarations ---'
rg -n -C 8 \
  'pub enum ConstantValue|enum ConstantValue|pub struct InterpolatedStringLiteralElement|struct InterpolatedStringLiteralElement' \
  . --glob '*.rs' --glob '!target/**' --glob '!crates/wtf8/src/lib.rs'

printf '%s\n' '--- relevant dependency and feature configuration ---'
rg -n -C 3 'ruff_python_ast|ruff_text_size|wtf8' Cargo.toml crates/*/Cargo.toml

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- codegen literal conversion ---'
sed -n '145,185p' crates/codegen/src/lib.rs
sed -n '1,45p' crates/vm/src/stdlib/_ast/string.rs
sed -n '755,790p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- AST conversion call graph ---'
rg -n -C 5 \
  'into_expr\(|to_rust|rust_ast|compile.*ast|compile_top|compile_exec|constant_object_to_constant_data|ast_from_object' \
  crates/vm/src/stdlib/_ast crates/vm/src --glob '*.rs' --glob '*test*.rs'

printf '%s\n' '--- direct tests for _ast and compiler entry points ---'
rg -n -C 6 \
  'PyAST|_ast|ast\.parse|compile\(|eval_code|run_code|compile_top|compile_ast' \
  crates/vm --glob '*test*.rs' --glob '*.rs' | head -n 1200

printf '%s\n' '--- dependency declarations ---'
rg -n -C 5 'ruff_python_ast|rustpython-codegen|wtf8' Cargo.toml crates/*/Cargo.toml

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- exact string AST conversion paths ---'
sed -n '400,535p' crates/vm/src/stdlib/_ast/string.rs
sed -n '550,640p' crates/vm/src/stdlib/_ast/string.rs
sed -n '750,785p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- compile() AST handling ---'
sed -n '215,285p' crates/vm/src/stdlib/builtins.rs
rg -n -C 8 'compile.*ast|compile_top|compile.*source|ast_from_object' crates/vm/src/stdlib/builtins.rs crates/vm/src/stdlib/_ast.rs crates/vm/src/stdlib/_ast --glob '*.rs'

printf '%s\n' '--- compiler helper ---'
sed -n '155,180p' crates/codegen/src/lib.rs

printf '%s\n' '--- dependency source locations ---'
find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -maxdepth 3 -type d -name 'ruff_python_ast-*' -print 2>/dev/null | head
find . -path '*/ruff_python_ast*' -type f -maxdepth 6 -print 2>/dev/null | head

#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' '--- joined-string conversion ---'
sed -n '410,535p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- template-string conversion ---'
sed -n '990,1025p' crates/vm/src/stdlib/_ast/string.rs

printf '%s\n' '--- compile() conversion boundary ---'
sed -n '235,280p' crates/vm/src/stdlib/builtins.rs
sed -n '155,180p' crates/codegen/src/lib.rs

printf '%s\n' '--- relevant dependency metadata ---'
rg -n -m 20 -C 4 'ruff_python_ast|ruff_text_size' Cargo.toml Cargo.lock crates/*/Cargo.toml

printf '%s\n' '--- available ruff AST source ---'
find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -maxdepth 3 -type f \
  \( -path '*ruff_python_ast*/src/*.rs' -o -path '*ruff_python_ast*/src/**/*.rs' \) \
  -print 2>/dev/null | head -n 20

#!/usr/bin/env bash
set -euo pipefail

python3 - <<'PY'
from pathlib import Path

constant = Path("crates/vm/src/stdlib/_ast/constant.rs").read_text()
string = Path("crates/vm/src/stdlib/_ast/string.rs").read_text()
codegen = Path("crates/codegen/src/lib.rs").read_text()

checks = {
    "constant conversion uses WTF-8 Display": (
        'ConstantData::Str { value } => ast::ConstantValue::Str(value.to_string().into_boxed_str())'
        in constant
    ),
    "f-string conversion uses lossy conversion": (
        'value: value.to_string_lossy().into(),' in string
    ),
    "template-string conversion uses lossy conversion": (
        string.count('value: value.to_string_lossy().into(),') >= 2
    ),
    "compiler converts AST literal through Rust str": (
        'element.value.to_string().into()' in codegen
    ),
}

for name, result in checks.items():
    print(f"{name}: {'present' if result else 'missing'}")
    assert result

# Model the documented conversion for a lone U+D800.
surrogate_utf8 = bytes((0xED, 0xA0, 0x80))
lossy = surrogate_utf8.decode("utf-8", "replace")
print("U+D800 lossy result:", repr(lossy))
assert lossy == "\ufffd"
PY

printf '%s\n' '--- runtime f-string preservation hooks ---'
rg -n -C 5 \
  'runtime_joined_str|runtime_values|runtime_str|runtime_interpolation_format_spec' \
  crates/codegen/src crates/vm/src/stdlib/_ast --glob '*.rs' | head -n 500

#!/usr/bin/env bash
set -u

python3 - <<'PY'
from pathlib import Path

constant = Path("crates/vm/src/stdlib/_ast/constant.rs").read_text()
string = Path("crates/vm/src/stdlib/_ast/string.rs").read_text()
codegen = Path("crates/codegen/src/lib.rs").read_text()

checks = {
    "constant conversion uses WTF-8 Display": (
        'ConstantData::Str { value } => ast::ConstantValue::Str(value.to_string().into_boxed_str())'
        in constant
    ),
    "f-string conversion uses lossy conversion": (
        'value: value.to_string_lossy().into(),' in string
    ),
    "template-string conversion uses lossy conversion": (
        string.count('value: value.to_string_lossy().into(),') >= 2
    ),
    "compiler converts AST literal through Rust str": (
        'element.value.to_string().into()' in codegen
    ),
}

for name, result in checks.items():
    print(f"{name}: {'present' if result else 'missing'}")
    assert result

# Wtf8's Display and to_string_lossy replace one recognized surrogate encoding
# with one U+FFFD, as specified in crates/wtf8/src/lib.rs.
wtf8_surrogate = bytes((0xED, 0xA0, 0x80))
print("WTF-8 bytes:", wtf8_surrogate.hex())
print("replacement count:", 1)
assert len(wtf8_surrogate) == 3
PY

printf '%s\n' '--- runtime preservation hooks ---'
rg -n -C 5 \
  'runtime_joined_str|runtime_values|runtime_str|runtime_interpolation_format_spec' \
  crates/codegen/src crates/vm/src/stdlib/_ast --glob '*.rs' | head -n 500