diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5ace4600 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e9c868e9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# +# This version is customized to install with pdm, lint with ruff, and test with unpythonic.test.fixtures. + +name: CI + +on: + push: + branches: [ master ] + tags: ["v*"] + pull_request: + branches: [ master ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - name: Install ruff + run: | + python -m pip install --upgrade pip + pip install ruff + - name: Lint with ruff + run: ruff check . --ignore SIM103 + - name: Lint advisories (non-blocking) + run: ruff check . --select SIM103 || true + + test: + needs: lint + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash # so the `tr` in "Determine Python version string for PDM" works on Windows (uses Git Bash) + strategy: + fail-fast: false + matrix: + # Full Python/PyPy matrix on Linux; newest CPython plus PyPy on + # macOS and Windows. Rationale: unpythonic is pure Python but + # exercises unusual control-flow machinery (continuations, dynvars, + # TCO, generator tricks), where OS-specific regressions can surface + # independently of Python minor version. PyPy is included on every + # OS because it's a separate interpreter family — the control-flow + # code paths differ from CPython and can have their own quirks. + os: [ubuntu-latest] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15", "pypy-3.11"] + include: + - os: macos-latest + python-version: "3.14" + - os: macos-latest + python-version: "pypy-3.11" + - os: windows-latest + python-version: "3.14" + - os: windows-latest + python-version: "pypy-3.11" + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + # 3.15 is at rc1, which the setup-python manifest marks unstable, so a bare + # "3.15" resolves to nothing without this. Versions that have a stable release + # are unaffected — a prerelease is only used when no stable one satisfies the + # request. Becomes a no-op once 3.15 final ships. + allow-prereleases: true + - name: Install tools in CI venv + run: | + python -m pip install --upgrade pip + pip install pdm + - name: Create in-project virtualenv and install dependencies + run: | + # Point PDM at the interpreter `setup-python` already installed, rather than + # letting it fetch a second one — its index has no prerelease build, so asking + # for a version like "3.15" fails while that very interpreter is on PATH. This + # also removes the need to translate CI's `pypy-3.11` into PDM's `pypy@3.11`. + # Ask Python for its own path: under Git Bash on Windows, `which python` gives + # an MSYS path that PDM, a native Windows program, cannot resolve. + pdm use -f "$(python -c 'import sys; print(sys.executable)')" + # "When you run pdm install the first time on a new PDM-managed project, whose Python interpreter is not decided yet, + # PDM will create a virtualenv in /.venv, and install dependencies into it." + # https://pdm-project.org/en/latest/usage/venv/ + pdm install + - name: Test with unpythonic.test.fixtures + run: pdm run python runtests.py + + build-dist: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: pip install build + - run: python -m build + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dist + path: dist/ + + publish: + if: startsWith(github.ref, 'refs/tags/v') + needs: build-dist + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dist + path: dist/ + + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: dist/ diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fc40d2ce..e58acf96 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,6 +8,10 @@ name: Coverage on: push: branches: [ master ] + workflow_dispatch: + +permissions: + contents: read jobs: codecov: @@ -15,27 +19,48 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + # The newest Python the project supports, i.e. the top of the CI matrix — + # that is where new-syntax code paths actually run. Bump this whenever the + # matrix grows; nothing else will remind you. + python-version: ["3.15"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + # 3.15 is at rc1, which the setup-python manifest marks unstable, so a bare + # "3.15" resolves to nothing without this. Becomes a no-op once 3.15 final ships. + allow-prereleases: true + - name: Install tools in CI virtualenv run: | python -m pip install --upgrade pip - pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install pdm + - name: Create in-project virtualenv and install dependencies + run: | + # Point PDM at the interpreter `setup-python` already installed, rather than + # letting it fetch a second one — its index has no prerelease build, so asking + # for a version like "3.15" fails while that very interpreter is on PATH. + # Ask Python for its own path: under Git Bash on Windows, `which python` gives + # an MSYS path that PDM, a native Windows program, cannot resolve. + pdm use -f "$(python -c 'import sys; print(sys.executable)')" + # "When you run pdm install the first time on a new PDM-managed project, whose Python interpreter is not decided yet, + # PDM will create a virtualenv in /.venv, and install dependencies into it." + # https://pdm-project.org/en/latest/usage/venv/ + pdm install + - name: Install coverage tool in in-project virtualenv + run: | + pdm run python -m ensurepip + # coverage must run in the same venv as the code being tested. + pdm run python -m pip install coverage - name: Generate coverage report run: | - pip install coverage - coverage run --source=. -m runtests - coverage xml + pdm run python -m coverage run -m runtests + pdm run python -m coverage xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + files: ./coverage.xml flags: unittests diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index 7ffd1add..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -# -# This version is customized to use the local flake8rc and test with unpythonic.setup.fixtures. - -name: Python package - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.6, pypy-3.7] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --config=flake8rc --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --config=flake8rc --count --exit-zero --max-complexity=100 --max-line-length=127 --statistics - - name: Test with unpythonic.test.fixtures - run: | - python runtests.py diff --git a/.gitignore b/.gitignore index 2a6cbf88..0cb53f07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,26 @@ +00_stuff +__pycache__ *~ *.pyc *.c build dist +MANIFEST +pdm.lock +.pdm-python .spyproject +.venv *.egg-info +*.mypy_cache +.python-version + +# Coverage artifacts +.coverage +coverage.xml +htmlcov/ + +# Secrets — should never be committed +codecov-token +*.token +.env + diff --git a/AUTHORS.md b/AUTHORS.md index 584a1c65..188abc8d 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,6 +2,7 @@ - Juha Jeronen (@Technologicat) - original author - @aisha-w - documentation improvements +- @Technologicat with Claude (Anthropic) as AI pair programmer - CI modernization, Python 3.13–3.14 and mcpyrate 4.0.0 adaptation (2.0.0); monads subpackage and `monadic_do` macro (2.1.0) **Design inspiration from the internet**: diff --git a/CHANGELOG.md b/CHANGELOG.md index d04f0436..397dffb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,327 @@ -**0.15.0** (in progress; updated 19 May 2021) - *"We say 'howdy' around these parts"* edition: +# Changelog + +**2.4.1** (in progress): + +*No user-visible changes yet.* + + +--- + +**2.4.0** (17 August 2026) — *"'Tis but a scratch"* [edition](https://en.wikipedia.org/wiki/Black_Knight_(Monty_Python)): + +Python 3.15 support. The language grew two pieces of syntax that change the AST, and the macro layer needed no overhaul to take them — `lazify`, `autocurry`, `tco` and `continuations` all pass the new comprehension forms through untouched. The work here is the tests that establish that, rather than any change to what the macros do. + +**New**: + +- **Python 3.15 is supported.** `requires-python` moves from `<3.15` to `<3.16`, and `mcpyrate` 4.3.0 or newer is now required, since earlier versions cannot import anything at all under 3.15. + - The macros needed no changes for the new syntax. `lazify`, `autocurry`, `tco` and `continuations` all pass comprehension unpacking (`{**mapping for x in xs}`, `[*items for item in xs]`, and the set and generator forms) through untouched. + - There are now tests to keep it that way, and they check the *properties* rather than the results — laziness by leaving a `1 / 0` unevaluated, currying by partially applying, TCO by recursing deeper than the stack allows. A `lazify` that had quietly gone strict inside the new forms would return the right answer and fail these. + +**Changed**: + +- `env.finalize()` now returns `self` instead of `None`, so it can be chained: `e = env(x=42).finalize()`. Matches the existing instance passthrough on `<<`. + +**Fixed**: + +- `unpythonic.test.fixtures`: `the[]`'s docstring now covers the common `the[]` mistakes — marking the whole assertion (which captures the boolean and hides the value you wanted), and under-marking a chained comparison — with the fix for each. Previously these were written down only in the project's own `CLAUDE.md`, where library users never see them. +- The macro documentation now warns that `begin`/`begin0` are not recognized by the macro layer, so macro-enabled code should sequence with `do[]`/`do0[]`. The functions' own docstrings already said so; the macro docs, where a macro user actually reads, did not. +- `env` now documents that it must be imported as `from unpythonic.env import env` — the module shares the class's name, so `from unpythonic import env` yields the module and `env(x=1)` then fails with an error that does not hint at the cause. +- `amb.forall` now points at the `unpythonic.syntax.forall` macro as the clean design of the same feature, and notes that "nondeterministic" here is the `amb` sense (every branch explored) rather than the modern colloquial one (stochastic). +- `assignonce` now says the macro layer supports plain `env` far better, and that `env.finalize()` is not a substitute: finalization freezes the set of names while leaving bindings rebindable, whereas `assignonce` fixes each binding's first value while leaving the set of names open. +- The `prefix` documentation now says it is experimental and not for production use, as its module docstring already did, and warns that its `q`/`u` markers are unrelated to `mcpyrate`'s quasiquote operators of the same names. + + +--- + +**2.3.0** (14 August 2026) — *"Mind the gap"* edition: + +**New**: + +- `unpythonic.misc.si_prefix`: new parameters `separator` and `always_separate`, for callers that append a unit of measurement. + - `separator` (default `" "`, per SI) is what goes between the number and the prefix; `separator=""` gives the compact `"1.50k"` that some UIs prefer. + - `always_separate` emits the separator even where the magnitude needs no prefix, so `f"{si_prefix(n, always_separate=True)}W"` reads `"1.50 kW"` and `"42.00 W"` rather than `"1.50 kW"` and `"42.00W"`. Without it the spacing depends on the magnitude, which is exactly what a caller appending a unit does not want. + + +--- + +**2.2.0** (12 May 2026) — *"Hail Eris"* edition: + +**New**: + +- `expect[]`: new expr macro for declaring the tested expression inside a `with test:` block. Replaces the `return expr` form. `return` continues to work but emits a `DeprecationWarning` at macro-expansion time and will be un-hijacked in 3.0.0 so that `return` inside `with test:` regains its standard Python meaning. Each block uses exactly one form: combining `expect[]` and `return` in the same block is a `SyntaxError`. +- `unpythonic.excutil.withf`: `with` as a function. Expression form of the `with` statement, completing the `raisef`/`tryf`/`withf` suite. Accepts a single context manager or a tuple of them (entered left-to-right, exited in reverse). Body arity is auto-detected: an n-arg body receives the as-values in order, a thunk discards them. Returns whatever the body returns. +- `unpythonic.dialects.bf`: a dialect that accepts a brainfuck program in place of Python source. Compiles `bf` to Python and runs it, or — via `bf.compile(src)` (qualified, to avoid shadowing `builtins.compile`) — returns the generated Python as a string for inspection. The tape is a `defaultdict[int, int]` subclass with 8-bit wrapping cells; comments in the `bf` source are preserved as Python comments; a `reset` line clears the tape between programs. + - Demonstrates the `mcpyrate` `Dialect.transform_source` hook (full-module source-to-source transformer), which the other dialects in this package do not use. + - Uses the new `mcpyrate.dialects.split_at_dialectimport` helper (requires `mcpyrate >= 4.1.0`), which correctly handles the case where the bf dialect-import shares a `from X import dialects, A, B` line with other dialects. +- `unpythonic.llist.FrozenAttributeError`: compatibility shim that multiply-inherits from `TypeError` (legacy unpythonic <= 2.x) and `dataclasses.FrozenInstanceError` (Python 3.7+ stdlib convention). Raised by `cons` on attribute write/delete attempts. Either `except TypeError` or `except FrozenInstanceError` catches it. The `TypeError` base will be dropped in 3.0.0; new code should catch `FrozenInstanceError` (or `AttributeError`). +- `unpythonic.syntax.multishot`: `@multishot` decorator macro and `myield` name/expr macro for multi-shot generators, plus `MultishotIterator` adapter. A `@multishot` function is generator-shaped, but at every `myield` the execution state is captured *as a continuation*, so it can be resumed from any earlier `myield` arbitrarily many times. Only meaningful inside `with continuations:`. `MultishotIterator` exposes a subset of the standard generator protocol (`iter`, `next`, `send`, `throw`, `close`, plus `gi_*` introspection) and one method standard generators don't have: `copy.copy(mi)` forks the iterator at the current continuation, with the two iterators advancing into independent timelines. Closes #80. See `doc/macros.md`. +- `myield_from` macro: the multi-shot analog of `yield from`. Inside a `@multishot` body, `myield_from[inner_call()]` drives a second `@multishot` to exhaustion, re-yielding each value to outer's caller; the assignment form `var = myield_from[...]` captures inner's `StopIteration` value. Forwards `send` and `throw` into the inner, and tracks the inner iterator via `outer_mi.gi_yieldfrom`. Multi-shot-to-multi-shot only — cross-delegation with standard generators is wontfix. +- `redirect_stdin`: context manager that feeds `sys.stdin` from a stream — the third sibling of `contextlib.redirect_stdout` (Python 3.4) and `contextlib.redirect_stderr` (Python 3.5), which the standard library punted on. Subclasses `contextlib._RedirectStream` so behavior matches the stdlib siblings exactly, including the per-instance stack that supports nested re-entry on the same instance. Like the stdlib siblings, redirects the global `sys.stdin` and is not safe under concurrent use from multiple threads. +- `unpythonic.dialects.befunge`: a [Befunge-93](https://en.wikipedia.org/wiki/Befunge) interpreter wrapped as a whole-module source dialect. Activate via `from unpythonic.dialects.befunge import dialects, Befunge`; the rest of the file is parsed as a strict 80 × 25 toroidal playfield with byte-valued cells and an unbounded-int stack, then run by the runtime interpreter (`run`, also exported). All Befunge-93 commands supported, including `?` (random direction; seedable via `run(src, *, seed=...)` for tests), `p`/`g` (self-modifying code), and the `&`/`~` integer/character input commands. + - Demonstrates the `mcpyrate` `Dialect.transform_source` hook for a different shape of source language than `bf`: a 2-D, IP-driven, self-modifying playfield. Where `bf.compile(src)` produces legible structured Python (the *dialect-as-transpiler* model), the Befunge dialect's `transform_source` wraps the playfield in a single `run(src)` call (the *dialect-as-reader* model) — necessary because Befunge has no statically-soundable structure. + - Three error categories: `SyntaxError` for source-level malformation (oversize grid), `IndexError` for runtime out-of-grid `g`/`p`, and `UnknownOpcodeError` (`RuntimeError` subclass) for the IP visiting a cell whose byte isn't a recognized command. + - Module docstring above the dialect-import is the recommended way to comment a Befunge file (`#` is a real Befunge command — *trampoline / skip-next-cell* — so comments inside the body aren't supported). + +**Fixed**: + +- `unpythonic.misc.callsite_filename`: walks the call stack via `sys._getframe` instead of `inspect.stack()`. Latent PyPy-3.11 / macOS / Windows bug: `inspect.stack()` reads source context around `f_lineno` for every frame, and on those targets at least one frame reachable from a `test[]` invocation reports `f_lineno = None`, which raises `TypeError` from `inspect.getframeinfo`. The new path reads only `f_code.co_filename`. CPython unaffected; PyPy on Linux unaffected. +- `unpythonic.llist.cons`: `__delattr__` now raises `FrozenAttributeError` (catchable as `TypeError`, `FrozenInstanceError`, or `AttributeError`). Latent bug: `del c.car` previously worked and corrupted the cell; only `__setattr__` was intercepted. The error message for `__setattr__` also now correctly says "attribute" (not "item") assignment. +- `unpythonic.assignonce`: `del e.foo` on a defined name now raises `AttributeError`. Latent bug: the assign-once contract could be bypassed via `del e.foo; e.foo = new_value`, since `__delattr__` was inherited unrestricted from `env`. Use `e.set("foo", value)` for explicit rebinding instead. + +**Changed**: + +- `unpythonic.funutil.call` and `callwith` now unpack `Values` in their positional arguments: each `Values` expands in place (left-to-right), splicing its `rets` into the positional arguments and merging its `kwrets` into the keyword arguments. Across multiple `Values` and the explicit `kwargs`, rightmost wins per unique keyword name. Mirrors the spread/merge semantics of Python's `[*a, *b, c]` and `{**a, **b}`; lets a `Values` produced by one function be applied as the arguments to another. +- `lispython`, `listhell`, and `pytkell` dialects now propagate the Python 3.8+ source-location fields `end_lineno` / `end_col_offset` through `splice_dialect`, alongside the existing `lineno` / `col_offset`. Tooling that consumes precise source ranges (debuggers, traceback formatters with PEP 657 column offsets) gets richer information for code coming from a dialect template. Implementation uses the `reference=self.location_ref` form (requires `mcpyrate >= 4.2.0`). Closes #83. +- **Requires mcpyrate >= 4.2.0**. + +**Internal**: + +- `unpythonic.syntax.letdoutil`: the letrec-bindings rebuild path now uses `ast.copy_location`, which copies whichever of the four source-location fields are present on the source node. Replaces the manual `lineno=` / `col_offset=` threading and removes the `if/else` guard for missing fields. +- `unpythonic.llist.cons`: dropped the internal `_immutable` sentinel; the read-only `car`/`cdr` are now installed via `object.__setattr__` in `__init__`, and `__setattr__` is a one-liner that always raises. +- `unpythonic.env.env`: dropped the `_direct_write` whitelist that allowed internal slots (`_env`, `_finalized`) to bypass `__setattr__`. Internal initialisation and `finalize()` now use `object.__setattr__` directly. Client code attempting `e._env = ...` or `e._finalized = ...` is now rejected by the reserved-name check (was silently allowed via the whitelist). +- `doc/macros.md`: new "Topology of continuations: how the wiring works" subsection (with inlined `callcc_topology.png` diagram explaining the `cc`/`pcc` machinery) and "Scoping of locals in continuations" subsection (the rule, the box workaround, the three load-bearing limits that ruled out auto-`nonlocal` propagation). Closes #82. +- `unpythonic/syntax/tests/test_conts.py`: revived the `"scoping, in presence of nonlocal"` testset that was disabled in 2022 due to a coverage.py source-parsing limitation. The new `[tool.coverage.run]` config in `pyproject.toml` scopes coverage to production code (excluding `*/tests/*`) and sidesteps the parse failure at report time. + + +--- + +**2.1.0** (17 April 2026) — *"Cat-hedral"* edition: + +**New**: + +- `unpythonic.monads`: subpackage of classical monads — `Identity`, `Maybe`, `Either` (with `Left`/`Right`), `List`, `Writer`, `State`, `Reader`. Plus `Monad`/`LiftableMonad` base classes and `liftm`/`liftm2`/`liftm3` helpers. Not re-exported at the top level; import as `from unpythonic.monads import Maybe`. +- `monadic_do`: do-notation macro over any monad. `with monadic_do[M] as result:` with body `[bindings] in result << final_expr`; supports `:=` (primary) and `<<` (legacy) for bindings; empty bindings allowed; `_ := mexpr` for sequencing. Always used as the innermost `with` (body shape constraint), composes correctly with `lazify`/`continuations`/`tco`/`autocurry`/etc. +- `environ_override`: context manager to temporarily override OS environment variables within a `with` block, restoring the previous state on exit. + - Thread-safe (serialises concurrent overrides via `RLock`); same-thread nesting supported. + - New module `unpythonic.environ`; the function is named `override` at the module level and re-exported as `environ_override` at the top level. +- `maybe_open`: context manager that opens a file when given a path, or yields a fallback stream (e.g. `sys.stdin`, `sys.stdout`) when given `None`. + - Lets callers always use `with` syntax regardless of whether the target is a file or a standard stream. +- `UnionFilter`: a `logging.Filter` that matches a log record if *any* of its sub-filters match. + - This is an OR combinator that is oddly missing from the standard library. +- `si_prefix`: format a number with SI decimal prefixes (k through Q, m through q) or IEC binary prefixes (Ki through Qi, mi through qi). + - Handles negative numbers, zero, and sub-unity magnitudes. + - The `binary=True` flag switches to base-1024 mode. +- `partial` (type-checking wrapper over `functools.partial`) is now exported in the public API. + - It was already implemented but missing from `fun.__all__`. +- `Sliced`, `FupTarget`, `Fuppable`: tag types in `unpythonic.slicing` for annotating the return values of `islice` and `fup`. + - `Sliced` has abstract `__getitem__`. + - `FupTarget` has abstract `__getitem__` returning `Fuppable`. + - `Fuppable` has abstract `__lshift__`. +- `slift1`, `slift2`: lift scalar unary/binary operations to work termwise on iterables, returning imathified lazy generators. Accept optional extra arguments baked into each call (e.g. `slift1(round, 2)`). These are the mechanism behind all built-in `s`-prefixed operators, now exposed for user-defined functions. +- `ConditionProtocol`: `typing.Protocol` capturing the call signature shared by error-handling protocols (`signal`, `error`, `cerror`, `warn`). Useful for annotating custom protocols. + +**Changed**: + +- `unpythonic.amb.MonadicList`: the implementation was moved to `unpythonic.monads.List` and renamed. `MonadicList` remains as a silent alias of `List` for name compatibility, scheduled for removal in 3.0.0. The constructor uses varargs (`List(1, 2, 3)`) — the class itself is the monadic unit (`List(x)` is a singleton list). Iterable-constructor use-cases go through `List.from_iterable(iterable)`. +- `unpythonic.net` (REPL server and client) now runs on MS Windows. + - Previously the whole subsystem was POSIX-only because `unpythonic.net.ptyproxy` required `termios`, `tty`, and `os.openpty`. + - A new `socket.socketpair()`-based backend stands in for the pty master/slave endpoints on Windows, plugged in via a platform dispatch in `PTYSocketProxy`. + - Known wart: `os.isatty(sys.stdin.fileno())` inside a REPL session returns `False` on Windows (no real pseudo-terminal is involved), whereas it returns `True` on POSIX — user code *inside* the REPL that checks `sys.stdin.isatty()` will see the Windows result; the framework itself doesn't care. + +**Fixed**: + +- `unpythonic.net.server`: `start()` now returns the actually-bound ports, not the values the caller passed in. + - Matters when passing `repl_port=0` / `control_port=0` to let the kernel pick a free port — previously the caller got `(bind, 0, 0)` back. + - Also fixes a latent bug in `unpythonic.net.util.ReuseAddrThreadingTCPServer`: its custom `server_bind()` override dropped the `self.server_address = self.socket.getsockname()` refresh from stdlib's `TCPServer.server_bind`. +- `unpythonic.net.ptyproxy`: `stop()` is now idempotent and safe to call on a proxy that was never started. + - Latent bug: previously `stop()` gated the entire teardown (including `os.close(master)` / `os.close(slave)`) behind `if self._thread:`, so constructing a proxy and then exiting without calling `start()` leaked both fds. +- `unpythonic.net.client`: tab completion now works on macOS. + - macOS ships `readline` backed by `libedit`, which speaks a different `parse_and_bind` dialect than GNU readline — the client now detects `platform.system() == "Darwin"` and issues the libedit form there. +- `unpythonic.misc.timer` and `unpythonic.timeutil.ETAEstimator`: switched from `time.monotonic()` to `time.perf_counter()`. + - Latent Windows-only bug: `monotonic` is backed by a ~16 ms tick counter on Windows, so microsecond-scale `with timer() as t: ...` blocks recorded `t.dt = 0.0` and downstream divisions raised `ZeroDivisionError`. `perf_counter` has the highest available resolution on every platform. POSIX unaffected. +- `unpythonic.arity._kwargs`: generic dispatch path used `dict` instead of `set` for accumulating kwargs names. Latent bug — only triggers if a `@generic` function has keyword-only arguments. +- `unpythonic.test.runner`: module discovery no longer crashes on MS Windows with `re.error: bad escape`. + - The runner used `re.sub(os.path.sep, ...)` — `os.path.sep` is a lone backslash on Windows, an invalid regex pattern. Fixed by using `str.replace`. Affects any project reusing `unpythonic.test.runner`. + +**Internal**: + +- `unpythonic.net` now has an automated test suite for the REPL client and server. + - `unpythonic/net/tests/test_client.py` exercises the full client ↔ server roundtrip in-process (eval, multi-line, syntax-error recovery, clean disconnect), netcat-mode raw-socket access, control-channel RPC, and stretch cases (sequential reconnect, two concurrent clients). Tier 1 only — no subprocess / pty driver. Runs on every CI platform (Linux, macOS, Windows). +- `unpythonic.net.ptyproxy`: refactored into an abstract base class with platform-specific backends (`PosixPTYSocketProxy` via `os.openpty`, `WindowsPTYSocketProxy` via `socket.socketpair`). + - Dispatch happens inside `PTYSocketProxy.__new__`, so callers instantiate the base class and get the right backend for free. + - `PTYSocketProxy` is now a context manager (`with PTYSocketProxy(...) as proxy:`) for guaranteed cleanup. Public interface otherwise unchanged. +- `unpythonic.net.client`: `connect()` is now a thin public shim over a private `_connect(..., _input=None)`. + - The `_input` seam lets tests drive the REPL loop without monkey-patching `builtins.input` globally. Public API unchanged. +- `unpythonic.net.client`: `import readline` moved from module top into `connect()`, with a three-tier fallback (`readline` → `pyreadline3` → graceful degradation). + - POSIX behaviour unchanged; the module is now importable on Windows. +- `unpythonic.net.tests.fixtures.nettest`: binds on port 0 (kernel-assigned), removed a `sleep(0.05)` race-condition bandage, and re-raises worker-thread exceptions instead of swallowing them. +- Type annotations added to public API signatures (and private helpers/closures) across 32 modules. + - Full list of newly type-annotated modules: `regutil`, `symbol`, `assignonce`, `numutil`, `misc`, `excutil`, `fup`, `fix`, `environ`, `fun`, `it`, `fold`, `funutil`, `lazyutil`, `ec`, `singleton`, `env`, `gtco`, `amb`, `tco`, `slicing`, `dynassign`, `gmemo`, `llist`, `fploop`, `let`, `lispylet`, `seq`, `collections`, `arity`, `conditions`, `mathseq`. + - Remaining unannotated: `dispatch`, `typecheck` (hard tier — deeply dynamic). + - Convention: `F = TypeVar('F', bound=Callable)` for callable parameters, `T = TypeVar('T')` for data values. +- `amb` cleanups. + - `Assignment` renamed to `Choice`. + - Internal `env` class renamed to `Scope`. + - `MonadicList` constructor standardized to accept a single iterable (like `list`/`tuple`); `Sequence` methods added (`__reversed__`, `__contains__`, `index`, `count`); registered as `Container`/`Iterable`/`Sized`/`Sequence` ABC. +- `slicing`: New `Sliced` and `FupTarget`/`Fuppable` tag types with abstract methods, used as return types for `islice` and `fup`. Public API for type annotations, exported in `__all__`. +- `collections`: Internal class `_StrReprEqMixin` renamed to `_SequenceReprEqMixin`. + - `__eq__` now requires `Sequence` and raises `TypeError` for incompatible types (latent bug: previously crashed on non-`Sized` input). + - `roview._cache` attribute now declared in `__init__`. + - `_make_negidx_converter.convert` now has explicit `return None` for passthrough path. +- `arity._bind`: rebased from Python 3.8.5 `inspect.Signature._bind` to Python 3.14. Adds deferred positional-only-in-kwargs handling, updated error messages. Divergence points marked with `[unpythonic]` for easy auditing. `OrderedDict` → `dict` throughout (ordered since 3.7). +- `mathseq`: `hasattr(x, "__iter__")` → `isinstance(x, Iterable)` (3 sites). `imathify` registered as virtual subclass of `Iterable`. Internal `primitive_*` operator aliases renamed to `atom_*`. `spow`/`sround` varargs (`*mod`, `*ndigits`) changed to `int | None = None`. +- Bare `object()` sentinels replaced with `sym`/`gensym` for debug readability. + - Affected: `ec.py` (`gensym("anchor")`), `llist.py` (`gensym("fill")`, module-level), `fold.py` (`sym("_uselast")`), `test_conditions.py`, `test_collections.py`. + + +--- + +**2.0.0** (16 March 2026) — *"Six impossible things before breakfast"* edition: + +**IMPORTANT**: + +- **Python version support**: 3.10–3.14 (dropped 3.8, 3.9; added 3.13, 3.14). PyPy 3.11. + - If you need `unpythonic` for Python 3.8 or 3.9, use version 1.0.0. +- **Requires mcpyrate >= 4.0.0**. + - mcpyrate 4.0.0 dropped the `Str`, `Num`, `NameConstant` AST compatibility shims and the `getconstant` helper. Use `ast.Constant` directly, and `.value` to get the constant's value. + +**New**: + +- **Python 3.13 and 3.14 support**. +- `autoreturn` macro now handles `match`/`case` statements. Each case branch has its own tail position. +- New scope analyzer tests for `match`/`case` patterns and `try`/`except*`. +- New `unpythonic.test.runner` module: reusable test runner with module discovery, version-suffix gating (e.g. `test_foo_3_11.py` skipped on Python < 3.11), and integration with the test framework's warning system. Other projects using `unpythonic.test.fixtures` can import it directly. +- New `emit_warning()` function in `unpythonic.test.fixtures` for signaling test warnings from infrastructure code (outside `test[]`/`warn[]` macros). Used by the test runner for version-suffix skips, which show in the warning count for the innermost enclosing testset. +- Missing optional dependencies (sympy, mpmath) in tests emit `warn[]` instead of `error[]`, correctly reflecting that these are expected skips, not failures. +- Runtime type checker (`unpythonic.typecheck`): new supported typing features — `NoReturn`, `Never` (3.11+), `Literal`, `Type`, `ClassVar`, `Final`, `DefaultDict`, `OrderedDict`, `Counter`, `ChainMap`, `IO`/`TextIO`/`BinaryIO` (mapped to `io` module ABCs), `Pattern[T]`/`Match[T]` (string type checked when parametric), `ContextManager`, `AsyncContextManager`, `Awaitable`, `Coroutine`, `AsyncIterable`, `AsyncIterator`, `Generator`, `AsyncGenerator`. +- Runtime type checker: `TypedDict` support — structural checking of required/optional keys and value types. +- Runtime type checker: `Protocol` support — `@runtime_checkable` Protocols work via `isinstance`; non-runtime-checkable Protocols raise `TypeError` with an actionable message. +- Runtime type checker: parametric forms of abstract ABCs — `Iterable[T]`, `Collection[T]`, `Reversible[T]` perform best-effort element checking (elements checked when value is `Sized`; ABC-only for opaque iterators). `Iterator[T]` and `Container[T]` accept parametric form with type arg silently ignored (iterating an `Iterator` would consume it; `Container` only has `__contains__`, so elements can't be enumerated). + +**Fixed**: + +- Runtime type checker (`unpythonic.typecheck`): fixed compatibility with Python 3.14, where `typing.Union` is no longer a `_GenericAlias`. Now uses `typing.get_origin` (available since 3.8). +- Runtime type checker: fixed `TypeVar` detection to use `isinstance(T, typing.TypeVar)` instead of a fragile `repr`-based heuristic. +- Runtime type checker: `typing.Reversible` check now uses `isinstance` instead of a `hasattr("__reversed__")` workaround from the Python 3.5 era. +- Runtime type checker: removed redundant `safeissubclass` fallbacks for generic types — `typing.get_origin` handles both bare and parameterized generics on 3.10+. +- Scope analyzer: fixed `MatchCapturesCollector` bug where class references (e.g. `Point` in `case Point(x, y):`) were incorrectly collected as captured variable names. +- Macro layer: updated all `hasattr(tree, "ctx")` checks to use `getattr` with defaults, for correct behavior on Python 3.13+ where AST fields always exist with default values. + - **Important**: Since Python 3.13, the default of `ctx` is `Load()`, hence no AST node has its `ctx` in a "not set yet" state anymore. Hence, any macro-created `Name` nodes that appear in a `Store` or `Del` position **MUST** have their `ctx` set appropriately by the macro author. Failing to do so **will** cause mysterious errors during macro expansion. +- Macro layer: updated `arguments()` constructor calls to always include `posonlyargs=[]`, avoiding a `DeprecationWarning` on Python 3.13 (will become an error in 3.15). +- MS Windows: `unpythonic.net.util` failed to load, due to missing `termios` module (which is *nix only) being loaded by `unpythonic.net.__init__` when it imports `unpythonic.net.ptyproxy`. + - Fixed by catching `ModuleNotFoundError`, disabling `ptyproxy` on MS Windows systems. Thus the remote REPL functionality `unpythonic.net.client/server` is not available on MS Windows, but the rest of `unpythonic` works fine. + +**Deprecated**: + +- Parenthesis syntax for macro arguments (e.g. `let((x, 1), (y, 2))`). Use bracket syntax instead: `let[[x, 1], [y, 2]]`. The parenthesis syntax is kept for backward compatibility for now. +- Runtime type checker: `typing.Text` (deprecated since Python 3.11) and `typing.ByteString` (deprecated since Python 3.12) support is now marked for removal when the floor bumps to Python 3.12. + + +--- + +**1.0.0** (21 February 2026) — *"Same supercharger, new badge"* edition: + +Re-release of 0.15.5 as 1.0.0. No code changes. The library has been stable and in light maintenance mode for years; the version number now reflects this de facto status quo. + + +--- + +**0.15.5** (16 April 2025) - hotfix: + +**Changed**: + +- Internal: Upgrade build system to `pdm`. + - This is important for the road ahead, since the old `setuptools` build system has been deprecated. + - The GitHub CI scripts for `unpythonic` now use PDM to manage the testing venv and dependencies, too. Now the tests should run the same way as they would on a local system. + +- Bump `mcpyrate` to the hotfix version 3.6.4. + - The only difference is (beside `mcpyrate` too internally upgrading its build system to `pdm`) that the text colorizer now works correctly also for `input` with `readline`. + + +--- + +**0.15.4** (27 September 2024) - hotfix: + +**Fixed** + +- Bump `mcpyrate` to the hotfix version 3.6.3. + - This is only to make sure no one accidentally installs the broken version, `mcpyrate` 3.6.2, which had a bug in interactive console mode that wasn't caught by CI. + + +--- + +**0.15.3** (27 September 2024) - *New tree snakes* edition: + +**IMPORTANT**: + +- Minimum Python language version is now 3.8. + - We support 3.8, 3.9, 3.10, 3.11, 3.12, and PyPy3 (language versions 3.8, 3.9, and 3.10). + - Python 3.6 and 3.7 support dropped, as these language versions have officially reached end-of-life. If you need `unpythonic` for Python 3.6 or 3.7, use version 0.15.2. +- Minimum version for optional macro expander `mcpyrate` is now 3.6.2, because the `astcompat` utility module was moved there. + + +**New**: + +- **Python 3.12 support**. + - As in, all tests pass, so there are no regressions. Some undiscovered interactions with new language features (`type` statement) may still be broken, although the most obvious cases are already implemented. +- **Python 3.11 support**. + - As in, all tests pass, so there are no regressions. Some undiscovered interactions with new language features (`try`/`except*` construct) may still be broken, although the most obvious cases are already implemented. +- Walrus syntax `name := value` is now supported, and preferred, for all env-assignments. Old syntax `name << value` still works, and will remain working at least until v0.16.0, whenever that is. + - Note that language support for using an assignment expression inside a subscript *without parenthesizing it* was [added in Python 3.10](https://docs.python.org/3/whatsnew/3.10.html#other-language-changes). + - If you still use Python 3.8 or 3.9, with the new `:=` syntax you must put parentheses around each `let` binding, because syntactically, the bindings subform looks like a subscript. + - All documentation is written in Python 3.10 syntax; all unit tests are written in Python 3.8 syntax. + + +**Changed**: + +- Utility module `unpythonic.syntax.astcompat`, used by the macro layer, moved to `mcpyrate.astcompat`. This module handles version differences in the `ast` module in various versions of Python. + + +**Fixed**: + +- `ETAEstimator` edge case: at any point after all tasks have been marked completed, return a constant zero estimate for the remaining time. +- Fix borkage in `mathseq` when running with SymPy 1.13 (SymPy is only used in tests). Bump SymPy version to 1.13. +- Fix bug in scopeanalyzer: `get_names_in_store_context` now collects also names bound in `match`/`case` constructs (pattern matching, Python 3.10). + + +--- + +**0.15.2** (19 September 2024) + +This time, just a small but important fix. + +**Fixed**: + +- `unpythonic.env.env` is now pickleable. Save your fancy bunches into `.pickle` files and load them back! + +**Future plans**: + +Contrary to appearances, this project is not dead. But it already does most of what I personally need it to do, so it is pretty much in maintenance mode. And it has not required much maintenance over the past two years. + +We still plan to officially support Python 3.11+ later, as well as to update all constructs with assignment semantics to use the more appropriate `:=` operator, when/if I find the time to do so. The syntax uses `<<` for historical reasons - these constructs were originally implemented in 2018, on Python 3.4, back when `:=` did not exist. + +The most likely upgrade timeframe is when I personally switch to Python 3.11+, and something breaks. That is also when I'll likely next upgrade the sister project `mcpyrate`. + + +--- + +**0.15.1** (28 January 2022) - *New Year's edition*: + +**New**: + +- **Python 3.10 support**. Running on Python 3.10 requires `mcpyrate` 3.6.0. +- New module `unpythonic.timeutil`, with utilities for converting a number of seconds into human-understood formats (`seconds_to_human`, `format_human_time`), and a simple running-average `ETAEstimator` that takes advantage of these. As usual, these are available at the top level of `unpythonic`. +- Add function `unpythonic.syntax.get_cc`, the less antisocial little sister of `call_cc` from an alternate timeline, to make programming with continuations slightly more convenient. (Alternate timelines happen a lot when one uses multi-shot continuations.) The two work together. See docstring. +- Tag continuation closures (generated by the `with continuations` macro), for introspection. + - To detect at run time whether a given object is a continuation function, use the function `unpythonic.syntax.iscontinuation`. + - This is purely an introspection feature; `unpythonic` itself does not use this information. For why you might want to query this, see `get_cc`, particularly the [examples in unit tests](unpythonic/syntax/tests/test_conts.py). + - The information is stored as an attribute on the function object; keep this in mind if you intend to wrap the continuation function with another function. (Strictly, this is the correct behavior, since a custom wrapper is not a continuation function generated by the `with continuations` macro.) + +**Fixed**: + +- The test framework `unpythonic.test.fixtures` is now correctly installed when installing `unpythonic`. See [#81](https://github.com/Technologicat/unpythonic/issues/81). +- The subpackage for live REPL functionality, `unpythonic.net`, is now correctly installed when installing `unpythonic`. +- Fix a broken import that prevented the REPL server `unpythonic.net.server` from starting. This was broken by the move of `async_raise` into `unpythonic.excutil` in 0.15.0. +- `unpythonic.syntax.prefix`: Fix wrong macro name in error message of `unpythonic.syntax.prefix.u`. Document in the docstring that the magic operators `q`, `u`, and `kw` (of the `prefix` macro) cannot be renamed by as-importing. +- Preserve the source location info of the dialect-import statement in the example dialects in [`unpythonic.dialects`](unpythonic/dialects/). In the output, the lines of expanded source code that originate in a particular dialect template are marked as coming from the unexpanded source line that contains the corresponding dialect-import. + - If you want to see the line numbers before and after dialect expansion, use the `StepExpansion` dialect from `mcpyrate.debug`. + - This fix requires `mcpyrate` 3.6.0 or later. The code will run also on earlier versions of `mcpyrate`; then, just like before, it will look as if all lines that originate in any dialect template came from the beginning of the user source code. + + +--- + + +**0.15.0** (22 June 2021) - *"We say 'howdy' around these parts"* edition: Beside introducing **dialects** (a.k.a. whole-module code transforms), this edition concentrates on upgrading our dependencies, namely the macro expander, and the Python language itself, to ensure `unpythonic` keeps working for the next few years. This introduces some breaking changes, so we have also taken the opportunity to apply any such that were previously scheduled. @@ -96,6 +419,9 @@ The same applies if you need the macro parts of `unpythonic` (i.e. import anythi - Add `unpythonic.excutil.reraise_in` (expr form), `unpythonic.excutil.reraise` (block form): conveniently remap library exception types to application exception types. Idea from [Alexis King (2016): Four months with Haskell](https://lexi-lambda.github.io/blog/2016/06/12/four-months-with-haskell/). - Add variants of the above for the conditions-and-restarts system: `unpythonic.conditions.resignal_in`, `unpythonic.conditions.resignal`. The new signal is sent using the same error-handling protocol as the original signal, so that e.g. an `error` remains an `error` even if re-signaling changes its type. - Add `resolve_bindings_partial`, useful for analyzing partial application. + - Add `triangular`, to generate the triangular numbers (1, 3, 6, 10, ...). + - Add `partition_int_triangular` to answer a timeless question concerning stackable plushies. + - Add `partition_int_custom` to answer unanticipated similar questions. - All documentation files now have a quick navigation section to skip to another part of the docs. (For all except the README, it's at the top.) - Python 3.8 and 3.9 support added. @@ -117,10 +443,17 @@ The same applies if you need the macro parts of `unpythonic` (i.e. import anythi - Positional passthrough works as before. Named passthrough added. - Any remaining arguments (that cannot be accepted by the initial call) are passed through to a callable intermediate result (if any), and then outward on the curry context stack as a `Values`. Since `curry` in this role is essentially a function-composition utility, the receiving curried function instance unpacks the `Values` into args and kwargs. - If any extra arguments (positional or named) remain when the top-level curry context exits, then by default, `TypeError` is raised. To override, use `with dyn.let(curry_context=["whatever"])`, just like before. Then you'll get a `Values` object. + - The generator instances created by the gfuncs returned by `gmemoize`, `imemoize`, and `fimemoize`, now support the `__len__` and `__getitem__` methods to access the already-yielded, memoized part. Asking for the `len` returns the current length of the memo. For subscripting, both a single `int` index and a slice are accepted. Note that memoized generators do **not** support all of the [`collections.abc.Sequence`](https://docs.python.org/3/library/collections.abc.html) API, because e.g. `__contains__` and `__reversed__` are missing, on purpose. + - `fup`/`fupdate`/`ShadowedSequence` can now walk the start of a memoized infinite replacement backwards. (Use `imemoize` on the original iterable, instantiate the generator, and use that generator instance as the replacement.) + - When using the `autoreturn` macro, if the item in tail position is a function definition or class definition, return the thing that was defined. + - The `nb` macro now works together with `autoreturn`. - `unpythonic.conditions.signal`, when the signal goes unhandled, now returns the canonized input `condition`, with a nice traceback attached. This feature is intended for implementing custom error protocols on top of `signal`; `error` already uses it to produce a nice-looking error report. + - The internal exception types `unpythonic.conditions.InvokeRestart` and `unpythonic.ec.Escape` now inherit from `BaseException`, so that they are not inadvertently caught by `except Exception` handlers. - The modules `unpythonic.dispatch` and `unpythonic.typecheck`, which provide the `@generic` and `@typed` decorators and the `isoftype` function, are no longer considered experimental. From this release on, they receive the same semantic versioning guarantees as the rest of `unpythonic`. - CI: Automated tests now run on Python 3.6, 3.7, 3.8, 3.9, and PyPy3 (language versions 3.6, 3.7). - CI: Test coverage improved to 94%. + - Full update pass for the user manual written in Markdown. + - Things added or changed in 0.14.2 and later are still mentioned as such, and have not necessarily been folded into the main text. But everything should be at least up to date now. **Breaking changes**: @@ -153,17 +486,22 @@ The same applies if you need the macro parts of `unpythonic` (i.e. import anythi - `curry` - `pipe` family - `compose` family + - `unfold` + - `iterate` - All multiple-return-values in code using the `with continuations` macro. (The continuations system essentially composes continuation functions.) - - The lazy evaluation tools `lazy`, `Lazy`, and the quick lambda `f` (underscore notation for Python) are now provided by `unpythonic` as `unpythonic.syntax.lazy`, `unpythonic.lazyutil.Lazy`, and `unpythonic.syntax.f`, because they used to be provided by `macropy`, and `mcpyrate` does not provide them. + - The lazy evaluation tools `lazy`, `Lazy`, and the quick lambda `f` (underscore notation for Python) are now provided by `unpythonic` as `unpythonic.syntax.lazy`, `unpythonic.lazyutil.Lazy`, and `unpythonic.syntax.fn` (note name change!), because they used to be provided by `macropy`, and `mcpyrate` does not provide them. - **API differences.** - - The macros `lazy` and `f` can be imported from the syntax interface module, `unpythonic.syntax`, and the class `Lazy` is available at the top level of `unpythonic`. - - Unlike `macropy`'s `Lazy`, our `Lazy` does not define `__call__`; instead, it defines the method `force`, which has the same effect (it computes if necessary, and then returns the value of the promise). - - When you import the macro `quicklambda`, you **must** import also the macro `f`. - - The underscore `_` is no longer a macro on its own. The `f` macro treats the underscore magically, as before, but anywhere else it is available to be used as a regular variable. + - The quick lambda is now named `fn[]` instead of `f[]` (as in MacroPy). This was changed because `f` is often used as a function name in code examples, local temporaries, and similar. Also, `fn[]` is a less ambiguous abbreviation for a syntactic construct that means *function*, while remaining shorter than the equivalent `lambda`. Compare `fn[_ * 2]` and `lambda x: x * 2`, or `fn[_ * _]` and `lambda x, y: x * y`. + - Note that in `mcpyrate`, macros can be as-imported, so this change affects just the *default* name of `fn[]`. But that is exactly what is important: have a sensible default name, to remove the need to as-import so often. + - The macros `lazy` and `fn` can be imported from the syntax interface module, `unpythonic.syntax`, and the class `Lazy` is available at the top level of `unpythonic`. + - Unlike `macropy`'s `Lazy`, our `Lazy` does not define `__call__`; instead, it defines the method `force`, which has the same effect (it computes if necessary, and then returns the value of the promise). You can also use the function `unpythonic.force`, which has the extra advantage that it passes through a non-promise input unchanged (so you don't need to care whether `x` is a promise before calling `force(x)`; this is sometimes useful). + - When you import the macro `quicklambda`, you **must** import also the macro `fn`. + - The underscore `_` is no longer a macro on its own. The `fn` macro treats the underscore magically, as before, but anywhere else it is available to be used as a regular variable. - **Behavior differences.** - - `f[]` now respects nesting: an invocation of `f[]` will not descend into another nested `f[]`. - - The `with quicklambda` macro is still provided, and used just as before. Now it causes any `f[]` invocations lexically inside the block to expand before any other macros in that block do. - - Since in `mcpyrate`, macros can be as-imported, you can rename `f` at import time to have any name you want. The `quicklambda` block macro respects the as-import, by internally querying the expander to determine the name(s) the macro `f` is currently bound to. + - `fn[]` now respects nesting: an invocation of `fn[]` will not descend into another nested `fn[]`. + - The `with quicklambda` macro is still provided, and used just as before. Now it causes any `fn[]` invocations lexically inside the block to expand before any other macros in that block do. + - Since in `mcpyrate`, macros can be as-imported, you can rename `fn` at import time to have any name you want. The `quicklambda` block macro respects the as-import, by internally querying the expander to determine the name(s) the macro `fn` is currently bound to. + - For the benefit of code using the `with lazify` macro, laziness is now better respected by the `compose` family, `andf` and `orf`. The utilities themselves are marked lazy, and arguments will be forced only when a lazy function in the chain actually uses them, or when an eager (not lazy) function is encountered in the chain. - Rename the `curry` macro to `autocurry`, to prevent name shadowing of the `curry` function. The new name is also more descriptive. - Move the functions `force1` and `force` from `unpythonic.syntax` to `unpythonic`. Make the `Lazy` class (promise implementation) public. (They actually come from `unpythonic.lazyutil`.) - Change parameter ordering of `unpythonic.it.window` to make it curry-friendly. Usage is now `window(n, iterable)`. @@ -174,9 +512,10 @@ The same applies if you need the macro parts of `unpythonic` (i.e. import anythi - This change fixes a `flake8` [E741](https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes) warning, and the new name for the parameter is more descriptive. - **Miscellaneous.** + - Robustness: the `with continuations` macro now raises `SyntaxError` if async constructs (`async def` or `await`) appear lexically inside the block, because interaction of `with continuations` with Python's async subsystem has never been implemented. See [issue #4](https://github.com/Technologicat/unpythonic/issues/4). - The functions `raisef`, `tryf`, `equip_with_traceback`, and `async_raise` now live in `unpythonic.excutil`. They are still available in the top-level namespace of `unpythonic`, as usual. - The functions `call` and `callwith` now live in `unpythonic.funutil`. They are still available in the top-level namespace of `unpythonic`, as usual. - - The functions `almosteq` and `ulp` now live in `unpythonic.numutil`. They are still available in the top-level namespace of `unpythonic`, as usual. + - The functions `almosteq`, `fixpoint`, `partition_int`, and `ulp` now live in `unpythonic.numutil`. They are still available in the top-level namespace of `unpythonic`, as usual. - Remove the internal utility class `unpythonic.syntax.util.ASTMarker`. We now have `mcpyrate.markers.ASTMarker`, which is designed for data-driven communication between macros that work together. As a bonus, no markers are left in the AST at run time. - Rename contribution guidelines to `CONTRIBUTING.md`, which is the modern standard name. Old name was `HACKING.md`, which was correct, but nowadays obscure. - Python 3.4 and 3.5 support dropped, as these language versions have officially reached end-of-life. @@ -192,6 +531,10 @@ The same applies if you need the macro parts of `unpythonic` (i.e. import anythi - Fix bug in `with namedlambda`. Due to incorrect function arguments in the analyzer, already named lambdas were not detected correctly. +- Fix bug: `fup`/`fupdate`/`ShadowedSequence` now actually accept an infinite-length iterable as a replacement sequence (under the obvious usage limitations), as the documentation has always claimed. + +- Fix bug: `memoize` is now thread-safe. Even when the same memoized function instance is called concurrently from multiple threads. Exactly one thread will compute the result. If `f` is recursive, the thread that acquired the lock is the one that is allowed to recurse into the memoized `f`. + --- diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..96889046 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,136 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What is unpythonic + +A Python library providing language extensions and utilities inspired by Lisp, Haskell, and functional programming. Three-tier architecture: + +1. **Pure Python layer** (`unpythonic/`): ~45 modules of functional utilities (curry, memoize, fold, TCO, conditions/restarts, dynamic variables, linked lists, etc.). No macro dependency. +2. **Macro layer** (`unpythonic/syntax/`): Syntactic macros via `mcpyrate` providing cleaner syntax for let-bindings, autocurry, lazify, TCO, continuations, etc. +3. **Dialect layer** (`unpythonic/dialects/`): Full language variants (Lispython, Listhell, Pytkell) built on the macro layer. + +Beyond the language-extension core, unpythonic also fills gaps in the Python standard library — cases where the stdlib almost gets it right, then punts at the last moment. `memoize` adds exception-replay machinery that `functools.lru_cache` lacks; the scan/fold suite brings Racket-level completeness to what `itertools` sketches; `env` supports several ABC protocols that `types.SimpleNamespace` doesn't. Smaller general-purpose utilities (e.g. `timer`, `si_prefix`, `environ_override`) also land here when they prove themselves as recurring needs across projects. + +## API stability + +Released as 2.0.0 in March 2026 (floor bump + mcpyrate 4.0.0 dependency). The public API (everything in `__all__`) should remain backward-compatible. Prefer non-breaking solutions when possible. + +## Build and development + +Uses PDM with `pdm-backend`. Python 3.10–3.14, also PyPy 3.11. + +```bash +# Set up development environment +pdm install # creates .venv/ and installs deps +pdm use --venv in-project +``` + +Prefix commands with `pdm run` if the venv is not active. + +The project venv is managed by PDM (`pdm venv create`, `pdm use --venv in-project`). To switch Python versions, remove the old venv and create a new one: + +```bash +pdm venv remove in-project +pdm config venv.in_project true +pdm venv create 3.14 # or whichever version +pdm use --venv in-project +pdm install +``` + +**Critical**: Never compile `.py` files in this project using `py_compile`, `python -m compileall`, `--compile`, or any other mechanism that bypasses the macro expander. Stale `.pyc` files compiled without macro support will break macro imports (symptom: `ImportError: cannot import name 'macros' from 'mcpyrate.quotes'`). If this happens, clean the caches with `macropython -c unpythonic` and re-run. + +## Running tests + +Custom test framework (`unpythonic.test.fixtures`, not pytest). Tests use macros (`test[]`, `test_raises[]`) and conditions/restarts for reporting. The test runner does not need the `macropython` wrapper—it activates macros via `import mcpyrate.activate`. Note: test *framework* is at `unpythonic/test/` (singular); actual *tests* are in `tests/` (plural) subdirectories. + +```bash +# Run all tests (from repo root) +python runtests.py + +# Run a single test module directly +python -c "import mcpyrate.activate; from unpythonic.tests.test_fun import runtests; runtests()" + +# Run macro tests similarly +python -c "import mcpyrate.activate; from unpythonic.syntax.tests.test_letdo import runtests; runtests()" +``` + +Test suites discovered by `runtests.py`: +- `unpythonic/tests/test_*.py` — pure Python features +- `unpythonic/net/tests/test_*.py` — REPL server/client +- `unpythonic/syntax/tests/test_*.py` — macro features +- `unpythonic/dialects/tests/test_*.py` — dialect features + +Each test module exports a `runtests()` function. Tests are grouped with `testset()` context managers. + +**Reading test results**: The framework reports Pass/Fail/Error/Total (plus optional `+ N Warn`) per testset. Nested testsets show hierarchy with indentation and asterisk depth (`**`, `****`, `******`, etc.). The distinction between Fail and Error is semantically load-bearing — see the next subsection. + +### The `unpythonic.test.fixtures` framework + +Part of unpythonic's **public API** (`unpythonic.test.fixtures`, `unpythonic.test.runner`). Reusable by any project that writes macro-enabled Python tests. Rationale for not using pytest: + +- pytest installs an import hook that rewrites `assert` statements (to give you the informative "assert x == 42 where x was 41" diagnostics you're used to). +- mcpyrate installs its own import hook to macro-expand source before compilation. +- Python only supports one source-rewriting import hook at a time; the two loaders can't be chained. So if you want both "nice assert messages" *and* "macro expansion", you have to pick one — and macro expansion is non-negotiable for code that uses macros. + +`unpythonic.test.fixtures` is the answer: instead of overriding the `assert` keyword, it provides `test[expr]`, `test_raises[cls, expr]`, `test_signals[cls, expr]`, and `warn[msg]` **macros** that construct test assertions at the AST level, and route results through `mcpyrate`'s condition system. The result categories: + +- **Pass**: the `test[...]` expression evaluated to a truthy value (or `test_raises[...]` saw exactly the expected exception, etc.). The test ran to completion and met its expectation. +- **Fail**: the test ran to completion, but the expectation was not met — `test[x == 42]` saw `x == 41`, or `test_raises[TypeError, ...]` saw the expression return normally. This is the "your code is wrong" category. +- **Error**: the test did **not** run to completion. An unhandled exception (or unhandled `error`/`cerror` condition) escaped the `test[...]` expression itself. This is the "the test infrastructure or the code *under* test crashed in a way the test didn't expect" category — semantically distinct from Fail, because the test never got to judge the expectation. An Error in CI means something is broken in a way that needs investigation, not just "the assertion didn't hold." +- **Warn**: advisory, emitted via `warn[msg]` (or by the runner itself for version-gated skips like "this test requires Python 3.14+, skipping on 3.13"). Does **not** count toward Pass/Fail/Error totals and does **not** fail the testset. Used for temporarily disabled tests, optional-dependency skips, and similar soft signals. + +**Capturing values with `the[]`**: when a `test[]` fails, you want to see *what the interesting subexpression actually evaluated to*, not just "the assertion was falsy." The `the[...]` helper macro marks a subexpression for capture; at run time, when the test fires, the framework formats a failure message with the source text and captured value of each `the[]`. The name is chosen to mostly preserve English reading order at the use site (`test[the[x] == 42]` reads roughly as "test that the `x` equals 42"), and is also a nod to Common Lisp's `THE` special form — though CL's `THE` is a *type-declaration* construct, so it's a name pun, not a semantic port. Heads-up for grepping: `the` is a word-boundary nightmare; anchor searches with `\bthe\[`. Usage: + +- `test[x == 42]` → on failure, auto-captures and reports `x` (leftmost term of a comparison). +- `test["green tea" == the[vert]]` → on failure, reports `vert` and its value. +- `test[f(the[a]) == g(the[b])]` → reports both `a` and `b`, in evaluation order. A `test[]` can contain any number of `the[]`, including nested (`the[outer(the[inner])]`). +- **Default**: if the top-level expression of `test[]` is a comparison and no explicit `the[]` is present, the leftmost term is **implicitly** wrapped — so `test[x == 42]` already reports `x` without you having to write `the[x]`. This is the common case. +- Use explicit `the[]` when you want to capture something *other* than the LHS of the top-level comparison — e.g. a subexpression inside a function call, a term in a non-comparison assertion, or multiple values at once. +- **Compound LHS — choose the capture granularity**: + - Auto-capture wraps the LHS *as-written*. For `test[reply["status"] == "ok"]` that captures `reply["status"]`, and a failure shows `"failed"` — the leaf value — not the full dict. + - Any explicit `the[]` anywhere in the expression **disables** auto-capture. `test[the[reply]["status"] == "ok"]` captures `reply` instead — the whole dict, useful for seeing a `"reason"` field the server attached alongside `"status": "failed"`. + - Both are valid. Decide by "*what value on failure would I actually want to see?*", not "*is `the[]` redundant?*". Leaf is enough when it's self-explanatory (`timer.dt == 0.0`). Wrap the container when the leaf is lossy (`reply["status"] == "ok"` — "failed" doesn't tell you *why*). +- The helper is smart enough to skip trivial captures (literal values), so `test[4 in the[(1, 2, 3)]]` won't clutter the output with `(1, 2, 3) = (1, 2, 3)`. +- **Not supported** inside `test_raises`, `test_signals`, `fail`, `error`, or `warn` — only in `test[...]` and `with test:` blocks. + +**The canonical account is `the[]`'s own docstring**, which now carries the common mistakes and their fixes. Read it — `help(the)`, or `unpythonic/syntax/testingtools.py` — and pattern-match your draft against it before committing. It lives there rather than here because that is the copy library users can actually see; keeping a second copy in this file is what let the two drift apart once already, with this file calling a form a mistake while the docstring called it deliberate. + +The house workflow on top of it: the three shapes it lists as mistakes all hide a value you would want on failure, so treat those as bugs. Marking a term auto-capture would have taken anyway is *not* one of them — it is style, and a draft carrying one is not broken. + +**Debugging cheat sheet**: a small number of **Warn**s on CI is expected (optional dependencies, version gates). **Fail** means a real expectation mismatch — read the captured values from `the[]` in the message. **Error** is the one you should *always* look at first: it means control flow in the test went somewhere unexpected, and the count alone won't tell you where. The log above the summary line has the actual traceback. + +## Linting + +```bash +ruff check # primary linter (config in pyproject.toml) +``` + +## Code structure and conventions + +- **Regular code** in `unpythonic/`, **macros** in `unpythonic/syntax/`, **REPL networking** in `unpythonic/net/`, **dialects** in `unpythonic/dialects/`. +- **Tests** are in `tests/` (plural) subdirectories under the code they test. The testing *framework* lives at `unpythonic/test/` (singular). +- Each module declares `__all__` explicitly for public API. The top-level `__init__.py` re-exports via star imports. +- **Import style**: Use `from ... import ...` (not `import ...`). The from-import syntax is mandatory for macro imports and used consistently throughout. Don't rename unpythonic features with `as`—macro code depends on original bare names. +- **Public-symbol references in docstrings, comments, briefs**: + - **Prose references** (mentioning a function or class to direct the reader): use the fully-qualified name, e.g. "similar to `unpythonic.misc.timer`". Saves the reader from grepping the source tree. + - **Usage examples** (showing how to call): use the bare name and skip the import line. Public top-level symbols are accessible via `from unpythonic import X`; spelling that out in every example is noise. + - **Exception**: when fully-qualified access is itself part of the recommendation (e.g. `unpythonic.dialects.bf.compile` to avoid shadowing `builtins.compile` in the importer's namespace), keep the qualified form in the example. +- **No star imports** in user code (only in the top-level `__init__.py` for re-export). +- **Curry-friendly signatures**: Parameters that change least often go on the left. Use `def f(func, thing0, *things)` (not `def f(func, *things)`) when at least one `thing` is required, so `curry` knows when to trigger. +- **Macros are the nuclear option**: Only make a macro when a regular function can't do the job. Prefer a pure-Python core with a thin macro layer for UX. +- **Macro `**kw` passing**: Use `dyn` (dynamic variables) to pass `mcpyrate` `**kw` arguments through to syntax transformers, rather than threading them through parameter lists. +- **Variable names**: Descriptive but compact. Prefer `theconstant` over `node` when the type matters, `thebody` over `b` when scope is more than a few lines. Avoid generic names like `tmp`, `data`, `x` unless scope is trivially small. In test code using the `the[]` macro, avoid `the`-prefixed names — `the[theconstant]` isn't English. Use e.g. `constant_node` instead. +- **Line width** ~110 characters. Docstrings in reStructuredText. +- **Module size target**: ~100–300 SLOC, rough max ~700 lines. Some modules are longer when appropriate (e.g. `syntax/tailtools.py` at ~1600 lines). Never split just because the line count was exceeded. +- **Type annotations**: New code should include type annotations. Existing unannotated code will be gradually updated. Some deeply Lispy parts (curry, TCO, conditions/restarts) may resist typing. +- **Top-level re-exports**: Most modules re-export via `from .module import *` in `__init__.py`. A small number of names use explicit aliased imports when the module-level name reads naturally at its own level but needs qualification at the top level (e.g. `environ.override` → `environ_override`, `lispylet.let` → `ordered_let`). +- **Dependencies**: Avoid external dependencies. `mcpyrate` is the only allowed external dep and must remain strictly optional for the pure-Python layer. + +## Key cross-cutting concerns + +- `curry` has cross-cutting behavior — grep for it when investigating interactions. +- `@generic` (multiple dispatch) similarly has cross-cutting concerns. +- The `lazify` macro: also grep for `passthrough_lazy_args` and `maybe_force_args`. +- The `continuations` macro builds on `tco` — read `tco` first when studying continuations. +- `unpythonic.syntax.scopeanalyzer` implements lexical scope analysis for macros that interact with Python's scoping rules (notably `let`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50257a42..e791c09d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,6 +7,7 @@ - [REPL server](doc/repl.md) - [Troubleshooting](doc/troubleshooting.md) - [Design notes](doc/design-notes.md) +- [Essays](doc/essays.md) - [Additional reading](doc/readings.md) - **Contribution guidelines** @@ -111,13 +112,13 @@ Since `unpythonic` is a relatively loose collection of language extensions and u To study a particular feature, just start from the entry point that piques your interest, and follow the definitions recursively. Use an IDE or Emacs's `anaconda-mode` ~for convenience~ to stay sane. Look at the automated tests; those double as usage examples, sometimes containing finer points that didn't make it to prose documentation. -`curry` has some [cross-cutting concerns](https://en.wikipedia.org/wiki/Cross-cutting_concern), but nothing that a grep wouldn't find. +`curry` has some [cross-cutting concerns](https://en.wikipedia.org/wiki/Cross-cutting_concern), but nothing that a grep wouldn't find. Same goes for the multiple-dispatch system (particularly `@generic`). The `lazify` and `continuations` macros are the most complex (and perhaps fearsome?) parts. As for the lazifier, grep also for `passthrough_lazy_args` and `maybe_force_args`. As for continuations, read the `tco` macro first, and keep in mind how that works when reading `continuations`. The `continuations` macro is essentially what [academics call](https://cs.brown.edu/~sk/Publications/Papers/Published/pmmwplck-python-full-monty/paper.pdf) *"a standard [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style) transformation"*, plus some technical details due to various bits of impedance mismatch. `unpythonic.syntax.scopeanalyzer` is a unfortunate artifact that is needed to implement macros that interact with Python's scoping rules, notably `let`. Fortunately, [the language reference explicitly documents](https://docs.python.org/3/reference/executionmodel.html#naming-and-binding) what is needed for a lexical scope analysis for Python. So we have just implemented that (better, as an AST analysis, rather than scanning the surface syntax text). -As of the first half of 2021, the main target platforms are **CPython 3.8** and **PyPy3 3.7** (since as of April 2021, PyPy3 does not have 3.8 yet). The code should run on 3.6 or any later Python. We have [a GitHub workflow](https://github.com/Technologicat/unpythonic/actions?query=workflow%3A%22Python+package%22) that runs the test suite on CPython 3.6 through 3.9, and on PyPy3. +As of v2.0.0, the main target platforms are **CPython 3.10** through **3.14**, and **PyPy3** (language version 3.11). We have [a GitHub workflow](https://github.com/Technologicat/unpythonic/actions?query=workflow%3A%22Python+package%22) that runs the test suite on these platforms. ## Style guide @@ -179,6 +180,16 @@ As of the first half of 2021, the main target platforms are **CPython 3.8** and - When implementing something, if you run into an empty niche, add the missing utility, and implement your higher-level functionality in terms of it. - This keeps code at each level of abstraction short, and exposes parts that can later be combined in new ways. +- **Compile-time or run-time?** + - For anyone new to making programming languages: there's a reason the terms static/lexical/compile-time and dynamic/run-time are grouped together. + - At compile time (macros), you have access to the source code (or AST), including its lexical structure. (I.e. what is defined inside what, in the source code text.) + - You also have access to the macro bindings of the current expander, because [*for the macros, it's run time*](https://github.com/Technologicat/mcpyrate/blob/master/doc/troubleshooting.md#macro-expansion-time-where-exactly). + - A block macro (`with mac:`) takes effect **for the lexical content of that block**. + - At run time (regular code), you have access to run-time bindings of names (e.g. whether `curry` refers to `unpythonic.fun.curry` or something else), and the call stack. + - Keep in mind that in Python, knowing what a name at the top level of a module (i.e. a "global variable") points to *is only possible at run time*. Although it's uncommon, not to mention bad practice in most cases, *any code anywhere* may change the top-level bindings in *any* module (via `sys.modules`). + - A run-time context manager (`with mgr:`) takes effect **for the dynamic extent of that block**. + - Try to take advantage of whichever is the most appropriate for what you're doing. + - **Follow [PEP8](https://www.python.org/dev/peps/pep-0008/) style**, *including* the official recommendation to violate PEP8 when the guidelines do not apply. Specific to `unpythonic`: - Conserve vertical space when reasonable. Even on modern laptops, a display can only fit ~50 lines at a time. - `x = x or default` for initializing `x` inside the function body of `def f(x=None)` (when it makes no sense to publish the actual default value) is concise and very readable. diff --git a/README.md b/README.md index c80649a0..ad022fa2 100644 --- a/README.md +++ b/README.md @@ -2,22 +2,15 @@ In the spirit of [toolz](https://github.com/pytoolz/toolz), we provide missing features for Python, mainly from the list processing tradition, but with some Haskellisms mixed in. We extend the language with a set of [syntactic macros](https://en.wikipedia.org/wiki/Macro_(computer_science)#Syntactic_macros). We also provide an in-process, background [REPL](https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop) server for live inspection and hot-patching. The emphasis is on **clear, pythonic syntax**, **making features work together**, and **obsessive correctness**. -![100% Python](https://img.shields.io/github/languages/top/Technologicat/unpythonic) ![supported language versions](https://img.shields.io/pypi/pyversions/unpythonic) ![supported implementations](https://img.shields.io/pypi/implementation/unpythonic) ![CI status](https://img.shields.io/github/workflow/status/Technologicat/unpythonic/Python%20package) [![codecov](https://codecov.io/gh/Technologicat/unpythonic/branch/master/graph/badge.svg)](https://codecov.io/gh/Technologicat/unpythonic) +![100% Python](https://img.shields.io/github/languages/top/Technologicat/unpythonic) ![supported language versions](https://img.shields.io/pypi/pyversions/unpythonic) ![supported implementations](https://img.shields.io/pypi/implementation/unpythonic) ![CI status](https://img.shields.io/github/actions/workflow/status/Technologicat/unpythonic/ci.yml?branch=master) [![codecov](https://codecov.io/gh/Technologicat/unpythonic/branch/master/graph/badge.svg)](https://codecov.io/gh/Technologicat/unpythonic) ![version on PyPI](https://img.shields.io/pypi/v/unpythonic) ![PyPI package format](https://img.shields.io/pypi/format/unpythonic) ![dependency status](https://img.shields.io/librariesio/github/Technologicat/unpythonic) ![license: BSD](https://img.shields.io/pypi/l/unpythonic) ![open issues](https://img.shields.io/github/issues/Technologicat/unpythonic) [![PRs welcome](https://img.shields.io/badge/PRs-welcome-brightgreen)](http://makeapullrequest.com/) -*Some hypertext features of this README, such as local links to detailed documentation, and expandable example highlights, are not supported when viewed on PyPI; [view on GitHub](https://github.com/Technologicat/unpythonic) to have those work properly.* - - -### New version soon! +For my stance on AI contributions, see the [collaboration guidelines](https://github.com/Technologicat/substrate-independent/blob/main/collaboration.md). -**As of May 2021, `unpythonic` 0.15 is Coming Soon™.** +We use [semantic versioning](https://semver.org/). -As of [3b5e5af](https://github.com/Technologicat/unpythonic/commit/3b5e5aff3ba3bd758151b7bf5aa5f2abb07cd82f), the code itself is in a releasable state, and it is already in `master`. All that remains is an extensive documentation review. The changelog is known to be up to date, but something may still need an update in all the other parts of documentation. - -The new version requires Python 3.6 or above, and optionally the [`mcpyrate`](https://github.com/Technologicat/mcpyrate) macro expander. Python 3.4 and 3.5, and the MacroPy macro expander, are no longer supported by `unpythonic`. - -The release will be numbered **0.15.0**, even though the codebase is mostly stable at this point, and we have already adhered to [semantic versioning](https://semver.org/) since 2019 (albeit with a leading zero). The reason is that the next major version has been known under this development version number for such a long time that it makes no sense to renumber it now. +*Some hypertext features of this README, such as local links to detailed documentation, and expandable example highlights, are not supported when viewed on PyPI; [view on GitHub](https://github.com/Technologicat/unpythonic) to have those work properly.* ### Dependencies @@ -26,7 +19,7 @@ None required. - [`mcpyrate`](https://github.com/Technologicat/mcpyrate) optional, to enable the syntactic macro layer, an interactive macro REPL, and some example dialects. -The 0.15.x series should run on CPython 3.6, 3.7, 3.8 and 3.9, and PyPy3 (language versions 3.6 and 3.7); the [CI](https://en.wikipedia.org/wiki/Continuous_integration) process verifies the tests pass on those platforms. [Long-term support roadmap](https://github.com/Technologicat/unpythonic/issues/1). +As of v2.0.0, `unpythonic` runs on CPython 3.10, 3.11, 3.12, 3.13, 3.14, and PyPy3 (language version 3.11); the [CI](https://en.wikipedia.org/wiki/Continuous_integration) process verifies the tests pass on those platforms. New Python versions are added and old ones are removed following the [Long-term support roadmap](https://github.com/Technologicat/unpythonic/issues/1). ### Documentation @@ -38,6 +31,7 @@ The 0.15.x series should run on CPython 3.6, 3.7, 3.8 and 3.9, and PyPy3 (langua - [REPL server](doc/repl.md): interactively hot-patch your running Python program. - [Troubleshooting](doc/troubleshooting.md): possible solutions to possibly common issues. - [Design notes](doc/design-notes.md): for more insight into the design choices of ``unpythonic``. +- [Essays](doc/essays.md): for writings on the philosophy of ``unpythonic``, things that inspired it, and related discoveries. - [Additional reading](doc/readings.md): links to material relevant in the context of ``unpythonic``. - [Contribution guidelines](CONTRIBUTING.md): for understanding the codebase, or if you're interested in making a code or documentation PR. @@ -46,14 +40,14 @@ The features of `unpythonic` are built out of, in increasing order of [magic](ht - Pure Python (e.g. batteries for `itertools`), - Macros driving a pure-Python core (`do`, `let`), - Pure macros (e.g. `continuations`, `lazify`, `dbg`). - - Whole-module transformations, a.k.a. dialects. + - Whole-module transformations, a.k.a. dialects (e.g. `Lispy`). This depends on the purpose of each feature, as well as ease-of-use considerations. See the design notes for more information. ### Examples -Small, limited-space overview of the overall flavor. There's a lot more that doesn't fit here, especially in the pure-Python feature set. See the [full documentation](doc/features.md) and [unit tests](unpythonic/tests/) for more examples. +Small, limited-space overview of the overall flavor. There is a lot more that does not fit here, especially in the pure-Python feature set. We give here simple examples that are **not** necessarily of the most general form supported by the constructs. See the [full documentation](doc/features.md) and [unit tests](unpythonic/tests/) for more examples. #### Unpythonic in 30 seconds: Pure Python @@ -151,7 +145,7 @@ Scan and fold accept multiple iterables, like in Racket. ```python from operator import add -from unpythonic import scanl, foldl, unfold, take +from unpythonic import scanl, foldl, unfold, take, Values assert tuple(scanl(add, 0, range(1, 5))) == (0, 1, 3, 6, 10) @@ -159,8 +153,8 @@ def op(e1, e2, acc): return acc + e1 * e2 assert foldl(op, 0, (1, 2), (3, 4)) == 11 -def nextfibo(a, b): # *oldstates - return (a, b, a + b) # value, *newstates +def nextfibo(a, b): + return Values(a, a=b, b=a + b) assert tuple(take(10, unfold(nextfibo, 1, 1))) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55) ``` @@ -170,8 +164,10 @@ assert tuple(take(10, unfold(nextfibo, 1, 1))) == (1, 1, 2, 3, 5, 8, 13, 21, 34, We bind arguments to parameters like Python itself does, so it does not matter whether arguments are passed by position or by name during currying. We support `@generic` multiple-dispatch functions. +We also feature a Haskell-inspired passthrough system: any args and kwargs that are not accepted by the call signature will be passed through. This is useful when a curried function returns a new function, which is then the target for the passthrough. See the docs for details. + ```python -from unpythonic import curry, generic +from unpythonic import curry, generic, foldr, composerc, cons, nil, ll @curry def f(x, y): @@ -216,6 +212,11 @@ assert g(1.0)(2.0) == "float" assert g("cat") == "str" assert g(s="cat") == "str" + +# simple example of passthrough +mymap = lambda f: curry(foldr, composerc(cons, f), nil) +myadd = lambda a, b: a + b +assert curry(mymap, myadd, ll(1, 2, 3), ll(2, 4, 6)) == ll(3, 6, 9) ```
Multiple-dispatch generic functions, like in CLOS or Julia. @@ -236,9 +237,9 @@ def my_range(start: int, step: int, stop: int): return start, step, stop ``` -This is a purely run-time implementation, so it doesn't give performance benefits, but it can make code more readable, and easily allows adding support for new input types to an existing function without monkey-patching the original. +This is a purely run-time implementation, so it does **not** give performance benefits, but it can make code more readable, and makes it modular to add support for new input types (or different call signatures) to an existing function later. -*Holy traits* are also a possibility: +[*Holy traits*](https://ahsmart.com/pub/holy-traits-design-patterns-and-best-practice-book/) are also a possibility: ```python import typing @@ -334,13 +335,38 @@ The condition system is the clean, general solution to this problem. It automati If this sounds a lot like an exception system, that's because conditions are the supercharged sister of exceptions. The condition model cleanly separates mechanism from policy, while otherwise remaining similar to the exception model.
+
Monads: Identity, Maybe, Either, List, Writer, State, Reader. + +[[docs](doc/features.md#monads)] [[`monadic_do` macro](doc/macros.md#monadic_do-do-notation-for-any-monad)] + +```python +from unpythonic.llist import nil +from unpythonic.monads import Maybe, List + +# Maybe — short-circuits on `nil`; the lambda is never called +assert Maybe(nil) >> (lambda x: Maybe(x + 1)) == Maybe(nil) + +# List — flatMap. Pythagorean triples by three nested binds. +def r(lo, hi): + return List.from_iterable(range(lo, hi)) +pt = r(1, 21) >> (lambda z: + r(1, z + 1) >> (lambda x: + r(x, z + 1) >> (lambda y: + List.guard(x*x + y*y == z*z).then( + List((x, y, z)))))) +assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20)) +``` + +For do-notation syntax (`with monadic_do[M] as result:`), see the macro documentation. Bind is `>>` (Python's `>>=` is in-place and doesn't chain), sequence is `.then(other)`, the class itself is `unit`. +
Lispy symbol type. [[docs](doc/features.md#sym-gensym-Singleton-symbols-and-singletons)] Roughly, a [symbol](https://stackoverflow.com/questions/8846628/what-exactly-is-a-symbol-in-lisp-scheme) is a guaranteed-[interned](https://en.wikipedia.org/wiki/String_interning) string. -A [gensym](http://clhs.lisp.se/Body/f_gensym.htm) is a guaranteed-unique string, which is useful as a nonce value. It's similar to the pythonic idiom `nonce = object()`, but with a nice repr, and object-identity-preserving pickle support. +A [gensym](http://clhs.lisp.se/Body/f_gensym.htm) is a guaranteed-*unique* string, which is useful as a nonce value. It's similar to the pythonic idiom `nonce = object()`, but with a nice repr, and object-identity-preserving pickle support. ```python from unpythonic import sym # lispy symbol @@ -429,13 +455,17 @@ assert a(0) is NoReturn
Build number sequences by example. Slice general iterables. -[[docs for `s`](doc/features.md#s-m-mg-lazy-mathematical-sequences-with-infix-arithmetic)] [[docs for `islice`](doc/features.md#islice-slice-syntax-support-for-itertoolsislice)] +[[docs for `s`](doc/features.md#s-imathify-gmathify-slift1-slift2-lazy-mathematical-sequences-with-infix-arithmetic)] [[docs for `islice`](doc/features.md#islice-slice-syntax-support-for-itertoolsislice)] ```python -from unpythonic import s, islice +from unpythonic import s, slift1, islice +from math import sin seq = s(1, 2, 4, ...) assert tuple(islice(seq)[:10]) == (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) + +ssin = slift1(sin) # lift scalar function to work termwise on iterables +assert tuple(islice(ssin(s(1, 2, ...)))[:3]) == (sin(1), sin(2), sin(3)) ```
Memoize functions and generators. @@ -483,9 +513,16 @@ from itertools import repeat from unpythonic import fup t = (1, 2, 3, 4, 5) -s = fup(t)[0::2] << tuple(repeat(10, 3)) +s = fup(t)[0::2] << repeat(10) assert s == (10, 2, 10, 4, 10) assert t == (1, 2, 3, 4, 5) + +from itertools import count +from unpythonic import imemoize +t = (1, 2, 3, 4, 5) +s = fup(t)[::-2] << imemoize(count(start=10))() +assert s == (12, 2, 11, 4, 10) +assert t == (1, 2, 3, 4, 5) ```
Live list slices. @@ -551,7 +588,9 @@ with session("simple framework demo"): test[returns_normally(g(2, 3))] test[g(2, 3) == 6] # Use `the[]` (or several) in a `test[]` to declare what you want to inspect if the test fails. - test[counter() < the[counter()]] + # Implicit `the[]`: in comparison, the LHS; otherwise the whole expression. Used if no explicit `the[]`. + # For compound LHS like `reply["status"] == "ok"`, wrap the container (`the[reply]`) to capture the whole dict on failure instead of just the leaf. + test[the[counter()] < the[counter()]] with testset("outer"): with testset("inner 1"): @@ -567,15 +606,21 @@ with session("simple framework demo"): try: import blargly except ImportError: - error["blargly not installed, cannot test integration with it."] + warn["blargly not installed, skipping integration tests."] else: ... # blargly integration tests go here + # Unconditional errors and failures can be emitted with `error[]` and `fail[]`. + # with testset("not implemented"): + # fail["not implemented yet!"] + with testset(postproc=terminate): test[2 * 2 == 5] # fails, terminating the nearest dynamically enclosing `with session` test[2 * 2 == 4] # not reached ``` +For running tests, `unpythonic.test.runner` provides a reusable test runner with module discovery and version-suffix gating. See [`doc/macros.md`](doc/macros.md#unpythonictestfixtures-a-test-framework-for-macro-enabled-python) for details, and [`runtests.py`](runtests.py) for a usage example. + We provide the low-level syntactic constructs `test[]`, `test_raises[]` and `test_signals[]`, with the usual meanings. The last one is for testing code that uses conditions and restarts; see `unpythonic.conditions`. The test macros also come in block variants, `with test`, `with test_raises`, `with test_signals`. @@ -589,13 +634,13 @@ As usual in test frameworks, the testing constructs behave somewhat like `assert ```python from unpythonic.syntax import macros, let, letseq, letrec -x = let[[a << 1, b << 2] in a + b] -y = letseq[[c << 1, # LET SEQuential, like Scheme's let* - c << 2 * c, - c << 2 * c] in +x = let[[a := 1, b := 2] in a + b] +y = letseq[[c := 1, # LET SEQuential, like Scheme's let* + c := 2 * c, + c := 2 * c] in c] -z = letrec[[evenp << (lambda x: (x == 0) or oddp(x - 1)), # LET mutually RECursive, like in Scheme - oddp << (lambda x: (x != 0) and evenp(x - 1))] +z = letrec[[evenp := (lambda x: (x == 0) or oddp(x - 1)), # LET mutually RECursive, like in Scheme + oddp := (lambda x: (x != 0) and evenp(x - 1))] in evenp(42)] ```
@@ -606,10 +651,10 @@ z = letrec[[evenp << (lambda x: (x == 0) or oddp(x - 1)), # LET mutually RECurs ```python from unpythonic.syntax import macros, dlet -# Up to Python 3.8, use `@dlet(x << 0)` instead -@dlet[x << 0] # let-over-lambda for Python +# In Python 3.8, use `@dlet(x << 0)` instead; in Python 3.9, use `@dlet(x := 0)` +@dlet[x := 0] # let-over-lambda for Python def count(): - return x << x + 1 # `name << value` rebinds in the let env + return x := x + 1 # `name := value` rebinds in the let env assert count() == 1 assert count() == 2 ``` @@ -621,8 +666,8 @@ assert count() == 2 ```python from unpythonic.syntax import macros, do, local, delete -x = do[local[a << 21], - local[b << 2 * a], +x = do[local[a := 21], + local[b := 2 * a], print(b), delete[b], # do[] local variables can be deleted, too 4 * a] @@ -678,6 +723,37 @@ with lazify: assert my_if(False, 1/0, 42) == 42 ``` +
Monadic do-notation for any monad. + +[[docs](doc/macros.md#monadic_do-do-notation-for-any-monad)] + +```python +from unpythonic.syntax import macros, monadic_do +from unpythonic.monads import Maybe, List + +# Maybe — do-notation threads present values (`Maybe(value)`); any absence (`Maybe(nil)`) short-circuits. +with monadic_do[Maybe] as result: + [x := Maybe(10), + y := Maybe(x + 1), + Maybe(x + y)] +assert result == Maybe(21) + +# List — Pythagorean triples via the list monad. Bare `List.guard(...)` +# is a sequencing-only bind; result discarded. Matches Haskell's `guard`. +def r(lo, hi): + return List.from_iterable(range(lo, hi)) +with monadic_do[List] as pt: + [z := r(1, 21), + x := r(1, z + 1), + y := r(x, z + 1), + List.guard(x*x + y*y == z*z), + List((x, y, z))] +assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20)) +``` + +Body shape is a single list literal. Each item is one line of a Haskell do-block: `name := mexpr` for monadic bind, `name << mexpr` (legacy) for the same, or bare `mexpr` for sequencing-only (matches Haskell's `guard`-style lines). The last item is the final monadic expression. `as result` on the `with` names the target. +
Genuine multi-shot continuations (call/cc). [[docs](doc/macros.md#continuations-callcc-for-python)] @@ -721,11 +797,11 @@ with continuations: # enables also TCO automatically The [dialects subsystem of `mcpyrate`](https://github.com/Technologicat/mcpyrate/blob/master/doc/dialects.md) makes Python into a language platform, à la [Racket](https://racket-lang.org/). We provide some example dialects based on `unpythonic`'s macro layer. See [documentation](doc/dialects.md). -
Lispython: The love child of Python and Scheme. +
Lispython: automatic TCO and an implicit return statement. [[docs](doc/dialects/lispython.md)] -Python with automatic tail-call optimization, an implicit return statement, and automatically named, multi-expression lambdas. +Also comes with automatically named, multi-expression lambdas. ```python from unpythonic.dialects import dialects, Lispython # noqa: F401 @@ -746,18 +822,16 @@ assert square.__name__ == "square" # - brackets denote a multiple-expression lambda body # (if you want to have one expression that is a literal list, # double the brackets: `lambda x: [[5 * x]]`) -# - local[name << value] makes an expression-local variable -g = lambda x: [local[y << 2 * x], +# - local[name := value] makes an expression-local variable +g = lambda x: [local[y := 2 * x], y + 1] assert g(10) == 21 ```
-
Pytkell: Because it's good to have a kell. +
Pytkell: Automatic currying and implicitly lazy functions. [[docs](doc/dialects/pytkell.md)] -Python with automatic currying and implicitly lazy functions. - ```python from unpythonic.dialects import dialects, Pytkell # noqa: F401 @@ -775,57 +849,142 @@ my_prod = foldl(mul, 1) my_map = lambda f: foldr(compose(cons, f), nil) assert my_sum(range(1, 5)) == 10 assert my_prod(range(1, 5)) == 24 -assert tuple(my_map((lambda x: 2 * x), (1, 2, 3))) == (2, 4, 6) +double = lambda x: 2 * x +assert my_map(double, (1, 2, 3)) == ll(2, 4, 6) ```
-
Listhell: It's not Lisp, it's not Python, it's not Haskell. +
Listhell: Prefix syntax for function calls, and automatic currying. [[docs](doc/dialects/listhell.md)] -Python with prefix syntax for function calls, and automatic currying. - ```python from unpythonic.dialects import dialects, Listhell # noqa: F401 -from unpythonic import foldr, cons, nil, ll +from operator import add, mul +from unpythonic import foldl, foldr, cons, nil, ll (print, "hello from Listhell") -double = lambda x: 2 * x +my_sum = (foldl, add, 0) +my_prod = (foldl, mul, 1) my_map = lambda f: (foldr, (compose, cons, f), nil) +assert (my_sum, (range, 1, 5)) == 10 +assert (my_prod, (range, 1, 5)) == 24 +double = lambda x: 2 * x assert (my_map, double, (q, 1, 2, 3)) == (ll, 2, 4, 6) ``` +
+
BF: the classical human-incomprehensible automaton. + +[[docs](doc/dialects/bf.md)] + +```python +from unpythonic.dialects.bf import dialects, BF # noqa: F401 + +# 'A' via a 5 × 13 multiplication loop ++++++++++++++[>+++++<-]>. +``` + +Unlike Lispython, Listhell, and Pytkell, [BF](https://en.wikipedia.org/wiki/Brainfuck) is a whole-module *source-to-source* transform — the body of a BF file isn't parseable as Python at all. It's one of two examples in this collection that exercises `mcpyrate`'s source-transformer hook (the modern equivalent of what old Lisp folks used to call a *reader macro*); the other is Befunge below. BF compiles to legible structured Python, so reading the compiled output is a reasonable way to read the original program — *the dialect-as-transpiler model*. +
+
Befunge: two-dimensional, self-modifying, deeply confused. + +[[docs](doc/dialects/befunge.md)] + +```python +"""Hello from Befunge!""" + +from unpythonic.dialects.befunge import dialects, Befunge # noqa: F401 + +"!egnufeB morf olleH">:#,_@ +``` + +The other source-transforming dialect. Where BF compiles to structured Python, [Befunge-93](https://en.wikipedia.org/wiki/Befunge) wraps its 80 × 25 self-modifying playfield in a runtime interpreter call — *the dialect-as-reader model*. The contrast with BF is the point: BF is structurally close to Python, so a transpiler is the natural fit; Befunge is fundamentally IP-driven on a 2-D grid that can rewrite itself at runtime, so a legible static translation is impossible and the right move is to ship an interpreter.
-## Installation +## Install & uninstall -**PyPI** +### From PyPI -``pip3 install unpythonic --user`` +```bash +pip install unpythonic +``` + +### From source -or +Clone the repo from GitHub. Then, navigate to it in a terminal, and: -``sudo pip3 install unpythonic`` +```bash +pip install . --no-compile +``` -**GitHub** +If you intend to use the macro layer of `unpythonic`, the `--no-compile` flag is important. It prevents an **incorrect** precompilation, without macro support, that `pip install` would otherwise do at its `bdist_wheel` step. -Clone (or pull) from GitHub. Then, +For most Python projects such precompilation is just fine - it's just macro-enabled projects that shouldn't be precompiled with standard tools. -``python3 setup.py install --user`` +If `--no-compile` is NOT used, the precompiled bytecode cache may cause errors such as `ImportError: cannot import name 'macros' from 'mcpyrate.quotes'`, when you try to e.g. `from unpythonic.syntax import macros, let`. In-tree, it might work, but against an installed copy, it will fail. It has happened that my CI setup did not detect this kind of failure. -or +This is a common issue when using macro expanders in Python. -``sudo python3 setup.py install`` +### Development mode (for developing `unpythonic` itself) -**Uninstall** +Starting with v0.15.5, `unpythonic` uses [PDM](https://pdm-project.org/en/latest/) to manage its dependencies. This allows easy installation of a development copy into an isolated venv (virtual environment), allowing you to break things without breaking anything else on your system (including apps and libraries that use an installed copy of `unpythonic`). -Uninstallation must be invoked in a folder which has no subfolder called ``unpythonic``, so that ``pip`` recognizes it as a package name (instead of a filename). Then, +#### Install PDM in your Python environment + +To develop `unpythonic`, if your Python environment does not have PDM, you will need to install it first: + +```bash +python -m pip install pdm +``` + +Don't worry; it won't break `pip`, `poetry`, or other similar tools. + +We will also need a Python for PDM venvs. This Python is independent of the Python that PDM itself runs on. It is the version of Python you would like to use for developing `unpythonic`. + +For example, we can make Python 3.10 available with the command: + +```bash +pdm python install 3.10 +``` -``pip3 uninstall unpythonic`` +Specifying just a version number defaults to CPython (the usual Python implementation). If you want PyPy instead, you can use e.g. `pypy@3.10`. -or +#### Install the isolated venv -``sudo pip3 uninstall unpythonic`` +Now, we will auto-create the development venv, and install `unpythonic`'s dependencies into it. In a terminal that sees your Python environment, navigate to the `unpythonic` folder, and issue the command: + +```bash +pdm install +``` + +This creates the development venv into the `.venv` hidden subfolder of the `unpythonic` folder. + +If you are a seasoned pythonista, note that there is no `requirements.txt`; the dependency list lives in `pyproject.toml`. + +#### Upgrade dependencies (later) + +To upgrade dependencies to latest available versions compatible with the specifications in `pyproject.toml`: + +```bash +pdm update +``` + +#### Develop + +To activate the development venv, in a terminal that sees your Python environment, navigate to the `unpythonic` folder, and issue the command: + +```bash +$(pdm venv activate) +``` + +Note the Bash exec syntax `$(...)`; the command `pdm venv activate` just prints the actual internal activation command. + +### Uninstall + +```bash +pip uninstall unpythonic +``` ## Support diff --git a/TODO_DEFERRED.md b/TODO_DEFERRED.md new file mode 100644 index 00000000..dbc9441d --- /dev/null +++ b/TODO_DEFERRED.md @@ -0,0 +1,213 @@ +# Deferred TODOs + +## Dispatch: indistinguishable parametric ABC multimethods (GitHub #99) + +Dispatch-layer improvements for parametric ABCs — warn/error on indistinguishable multimethods. Tricky because checkability is value-dependent (Sized vs opaque iterator). Typecheck-layer part is resolved. + + +## Type annotations — remaining hard-tier modules + +As of v2.1.0, 32 of 34 pure-Python modules are annotated. Two remain — genuinely resistant to static typing: + +- `dispatch.py` (7 exports) — runtime multiple dispatch, `typing` module introspection, multimethod resolution. +- `typecheck.py` (1 export) — deeply introspective runtime type checking; the function *is* the type system. + +Also within already-annotated modules, some functions were deliberately left unannotated: `curry`, `compose*` family, `flatten*` family (dynamic arity, `Values` unpacking, recursive type flattening). Convention established: `F = TypeVar('F', bound=Callable)` for callables, `T = TypeVar('T')` for data values; `fillvalue` parameters use `Any` (sentinel may differ from element type). The original audit concern (abstract params, concrete returns, no deprecated `typing` forms) should be checked against the annotations added. PEP 695 TODOs left in `arity.py` and `conditions.py` for when floor bumps to 3.12. + +Updated 2026-04-17. + + +## Tier 2 REPL tests (subprocess + pty) for `unpythonic.net` client/server + +Tier 1 coverage for `unpythonic.net.client` and `unpythonic.net.server` uses a server-in-thread + in-process client pattern (see `unpythonic/net/tests/`) with scripted input via a private `_input` seam on `client._connect(..., _input=fake_input)` and captured stdout/stderr via `io.StringIO`. Fast, single-process, no subprocess boundary needed — the server speaks TCP to `127.0.0.1` and the client loop runs in the same test process. **We might never need tier 2.** + +**Important framing**: tier 1 is a *protocol and plumbing test*, not a terminal-UX test. The `_input` seam replaces the entire `input()` pathway before readline is ever reached, so readline's line editor, history, completer binding, and interrupt-during-input are **not partially covered — they are 0% covered**. A regression in `readline.parse_and_bind`, in the custom remote completer wiring, or in the SIGINT-during-readline path would pass tier 1 silently. Tier 2 isn't "a safety net for edge cases" — it's the only place these things get exercised at all. + +A second tier would spawn the server and client as real subprocesses, with each end driven through a pseudo-terminal (`pexpect` / `ptyprocess`), to catch things tier 1 cannot reach: + +- Real GNU-readline binding behaviour on the client side — tab completion against the remote completer, history recall, multi-line input rendering. +- Terminal escape sequences from the colorizer on both sides. +- Signal handling — Ctrl+C from the client forwarded to the remote REPL, Ctrl+D disconnecting cleanly. +- The ptyproxy machinery itself, end-to-end. Tier 1 stubs around the pty by running the `InteractiveConsole` directly against in-memory streams; tier 2 would actually exercise `unpythonic.net.ptyproxy.PTYSocketProxy` with a real master/slave pair. + +Cost: + +- ~0.5–1 s startup per test × two processes per test (client + server) = ~1–2 s per test. Matters for suite size. +- POSIX-only naturally. Since D9 landed (2026-04-16), `unpythonic.net` runs on Windows too via `socket.socketpair`, but tier 2 still needs real pseudo-terminals — on Windows that means ConPTY, which D9 deliberately avoided. If tier 2 ever materializes, its Windows variant is an independent design problem. +- `pexpect` would become a new dev dep. Small but non-zero. + +**Rough shape if we ever do it:** + +```python +import pexpect +server = pexpect.spawn(f"{sys.executable} -m unpythonic.net.server", ...) +server.expect(r"Listening on \S+") +client = pexpect.spawn(f"{sys.executable} -m unpythonic.net.client", ...) +client.expect(r">>> ") +client.sendline("2 + 3") +client.expect(r"5\s*\n>>> ") +client.sendcontrol("d") +client.expect(pexpect.EOF) +server.terminate() +``` + +**When to actually do it**: only if tier 1 coverage turns out to miss something important (a regression hits prod that tier 1 would not have caught). The in-thread server + scripted client approach already exercises most of the protocol surface; tier 2 is primarily a safety net for terminal-semantics and signal-path bugs. Until one of those bites, tier 1 is the main win. + +Added 2026-04-15, alongside the tier 1 bring-up. + + +## Flexible view variant + +An older, more flexible implementation of `view` exists somewhere in the ancient git history, supporting more advanced slicing at the cost of worse performance. Could be resurrected as an alternative for use cases where flexibility matters more than speed. Dig through the history to find it. + +Noted 2026-04-16. + + +## Audit bare `{path}` interpolation for repr/raw asymmetry on Windows + +Fleet-wide audit across all projects. The known failure mode (mcpyrate `cacbfd2`, 2026-04-15): an f-string interpolates a file path with bare `{__file__}`, producing raw backslashes (`C:\a\b`), while the other side of a comparison uses `repr()`/`unparse()` output with escaped backslashes (`C:\\a\\b`) — mismatch on Windows, passes on POSIX by accident. Fix is `{__file__!r}` so both sides speak the same dialect. The risk is NOT f-string reinterpretation (that's safe), but asymmetry when a bare-interpolated path is compared against, compiled as, or embedded into Python source. Grep hints: `__file__` in f-strings; also any path value interpolated into strings that later reach `compile()`, `eval()`, `ast.unparse()`, assertions, or similar. + +Noted 2026-04-17. + + +## Unify `accepts_arity` helpers across `excutil` and `conditions` + +`unpythonic.excutil._accepts_arity(f, n)` (introduced alongside `withf` in 2.2.0) is the single source of truth for `tryf` / `withf`'s "n-arg form vs 0-arg thunk" dispatch, with the policy "default to the n-arg form on `UnknownArity`". `unpythonic.conditions.signal` (around line 199) defines its own private `accepts_arg(f)` helper with the same shape (n=1 hardcoded, returns `True` on `UnknownArity`). It would be natural to share one helper. + +**Caveat**: the right *policy* may be context-dependent. `tryf` and `withf` are user-facing combinators where defaulting to the n-arg form is the more flexible choice when introspection fails. The condition system's handler-dispatch is part of a fault-handling pathway — if anyone ever wants a stricter or more conservative default there (e.g. raise instead of guess, or default to thunk to avoid double-failure), that should be a deliberate decision per call site, not a side effect of unification. So a shared helper would either need a `default_on_unknown` parameter, or stay split into two helpers documenting the policy choice. + +Discovered during #76 (2026-05-05). + + +## Remove `unpythonic.amb.MonadicList` alias (3.0.0) + +As part of the monads port, `MonadicList` was moved to `unpythonic.monads.List` with a varargs constructor (`List(1, 2, 3)` instead of `MonadicList([1, 2, 3])`). A silent alias `MonadicList = List` is kept in `unpythonic/amb.py` for backward-name compatibility during the 2.x series. Remove the alias in 3.0.0 along with the accompanying `TODO(3.0.0)` comment at the alias site. Users must then import `List` directly from `unpythonic.monads`. Note: this is name-only compat — the constructor signature changed at 2.0.0, so existing callers of `MonadicList([...])` already needed to switch to varargs or `from_iterable(...)` at 2.0.0. + +Noted 2026-04-17. + + +## `isec` misses non-bare-name escape continuations, and does so silently + +`unpythonic/syntax/util.py`'s `isec` matches an escape continuation only through +`getname(..., accept_attr=False)`, and says so itself: "**CAUTION**: Only bare-name references are +supported." So an ec reached as `obj.ec(...)` is not recognized as an escape, and the `tco` / +`continuations` machinery does not transform the call. + +**The right fix is to resolve statically what the binding points to**, where that can be done (Juha, +2026-08-16). What that leaves open is the case where it cannot — an ec stored in a container, chosen +at runtime, or reached through a name the expander cannot follow. + +**A cheap interim step, before solving the hard half.** The failure is currently silent, and a missed +rewrite is not a mild degradation: the construct is rewritten *because* it needs rewriting, so what +follows is a crash, or worse, quietly wrong behaviour. `dbg` already handles the same class the other +way — a custom print function given as anything but a bare name raises `SyntaxError("Custom debug +print function must be specified by a bare name")`, with an in-source TODO recording that `Attribute` +support is wanted and why it is awkward (AST nodes do not compare). Making `isec` loud in the same +style would convert an invisible miss into a diagnosable one, and is independent of whether the +static resolution ever gets built. + +Worth checking whether other `accept_attr=False` sites share the problem. Most do not: `prefix`'s +`q`/`u`/`kw`, the `let` binding scanners and `autoref`'s internal markers all match names that can +only be bare, so there is nothing to resolve there. + +Discovered while writing the fleet's `unpythonic` skill (2026-08-16). + + +## Documentation gaps found by writing an outside summary of the library + +Writing the fleet's `unpythonic` and `macro-enabled-python` skills was, incidentally, a test of whether +the docs communicate to a reader who has not written the library. Most of it held up — the +troubleshooting entries, `main.md` on macro-imports, and "macro expansion time where exactly?" all +landed on one reading. Several things did not, and they share a shape: **the caveat lives somewhere other +than next to the thing it is about.** + +- **`from unpythonic import env` gives the *module*, not the class.** `__init__.py` never star-imports + `.env`, so the submodule attribute is what survives, and `env(x=1)` fails with "module is not + callable". The correct form is `from unpythonic.env import env`. Nothing in the docs says this; + it has to be discovered by trying it. This is the most user-facing of them — `env` is one of the + most-used things in the library. + + **What the 3.0.0 fix would actually cost, measured rather than estimated (2026-08-16).** Six + submodules already sit in the state `env` would move to — `llist`, `let`, `fix`, `fup`, `gtco`, + `assignonce` are all shadowed by a same-named symbol from the star-import — so the question is + answerable by experiment rather than argument. Testing against `llist`: `from unpythonic.llist + import cons` works, `sys.modules["unpythonic.llist"]` works, and the internal `from ..llist import` + form works. **Exactly one route breaks: attribute-style `unpythonic.llist.cons`**, because the + import machinery resolves submodules through `sys.modules` while attribute access sees whatever the + star-import left behind. + + Applied to `env`, that residual cost is close to nil, because `unpythonic/env.py`'s `__all__` is + exactly `["env"]`. The only name anyone could want attribute-style is `unpythonic.env.env` — which + the change turns into `unpythonic.env`, i.e. the thing we want. So "it becomes clumsy or impossible + to refer to the module" does not really bind here: there is nothing else in the module to refer to. + + On the macro layer, state the observation and not more than it supports: nothing in + `unpythonic/syntax/` currently matches `env` by name, and `syntax/lambdatools.py:25` reaches the + class explicitly with `from ..env import env`. That is *not* evidence that macros do not need it + (Juha, 2026-08-16) — the macros predate any heavy direct use of `env`, Raven included, so the + absence may record what nobody needed at the time rather than what is unnecessary. A macro wanting + to recognize a user-constructed `env` at a call site would want exactly the name-matching a + package-level re-export makes possible. So treat this as neutral-to-favourable, not as a reason + against. + + **The backward-compat cost is smaller than it looks, because the broken forms were never the + recommended ones.** `from unpythonic import env` then `env.env(...)`, and `import unpythonic.env` + then `unpythonic.env.env(...)`, both break — and both have always been discouraged. The form the + change is *for* is `from unpythonic import env` then `env()`, which is the consistent spelling and + does not work today. + + **A separate problem that this change does not fix, and that should be sized alongside it.** Some + `unpythonic` functions take a parameter named `env`, which shadows the class of the same name in + that scope. `unpythonic` already works around it internally: `lispylet.py:11` imports + `from .env import env as _envcls` so that line 225's `env` parameter and line 228's `_envcls()` + can coexist. Downstream, **9 Raven modules** carry `from unpythonic.env import env as envcls` for + the same reason — a call site needs to construct an `env` to pass into a parameter called `env`. + Re-exporting the class does nothing for this: the collision is parameter-versus-name in a local + scope, independent of how the class was imported. Renaming that parameter is its own API break, so + if both are wanted, 3.0.0 is where they go together. + + **Two fixes here, and only one of them is cheap** (Juha, 2026-08-16). Documenting the gotcha at the + site is non-breaking and can land in any release. *Actually* re-exporting the class would change + what `from unpythonic import env` returns, which breaks anyone relying on getting the module — so + the real fix waits for **3.0.0**, and wants to go in together with whatever other API-breakage debt + has accumulated. Size that first: the convention is an in-source `TODO(3.0.0)` marker plus an item + here, so `grep -rn "TODO(3.0.0)" unpythonic/` is the inventory command. As of 2026-08-16 it finds + one (the `MonadicList` alias in `amb.py`), which is almost certainly an undercount — the markers + only exist where someone remembered to leave one. + +The remaining work is the 3.0.0 half above. The documentation half is done: the fix was a sentence +at each site rather than new documents, and the sites were the ones a reader actually stands on — +`env`'s class docstring and its `features.md` section for the import gotcha, `amb.forall` and +`assignonce` docstrings for "prefer the other thing", `macros.md` for `prefix` being experimental +and for the `q`/`u` collision, and `macros.md`'s Sequencing section for `begin`/`begin0`. + +Worth keeping as the lesson: in every one of these the caveat either existed somewhere already +(`prefix`'s module docstring, `begin`'s `CAUTION`, `design-notes.md` on `amb`) or was implied by an +example nobody would read as a warning. The gap was never that the author did not know — it was that +the note was not on the path the reader takes. Note the audience is not only human: an agent reading +the library through `help()` or an API inventory sees exactly the docstring, and nothing else. + +Raised 2026-08-16, documentation half resolved the same day. + +## Should `runtests.py` clear the bytecode caches? + +`unpythonic`'s `runtests.py` does no cache clearing. `mcpyrate`'s does, via +`runtests(clear_bytecode_cache=True)` calling `mcpyrate.pycachecleaner.deletepycachedirs`, so the +mechanism is already available to import. + +The argument for adopting it: with a warm cache the expander does not run, so any test whose subject +is *expansion-time* behaviour silently tests nothing. That is not hypothetical — checking both +projects for AST-constructor deprecations under `-W error::DeprecationWarning` required clearing the +caches by hand first, and a run without that step proves nothing while looking identical. + +The argument against: re-expanding everything roughly doubles the suite's runtime, and the common +case is a developer re-running tests after touching one runtime-level function, where expansion +genuinely has not changed. + +So the decision needs a measurement (how much is "roughly doubles", actually?) and a choice of +default — always clear, clear only in CI, or a flag defaulting to off with the reason documented at +the call site. Note that whichever way it goes, a suite that *can* skip expansion needs to say so in +its output the way `mcpyrate`'s does ("Using existing bytecode"), because the failure is invisible +otherwise. + +Discovered during the Python 3.15 AST survey (2026-08-16). diff --git a/briefs/2.2.0-remaining-issues.md b/briefs/2.2.0-remaining-issues.md new file mode 100644 index 00000000..97c545dd --- /dev/null +++ b/briefs/2.2.0-remaining-issues.md @@ -0,0 +1,198 @@ +# 2.2.0 — Remaining open issues (session handoff) + +Updated 2026-05-12. Only **#83** remains on the 2.2.0 milestone; the rest +of this brief catalogues bonus material that landed alongside the issue +work and is now queued for the same release. CHANGELOG.md is the +canonical record of what shipped — this brief is the planning view. + +## Milestone 2.2.0 status + +| # | Title | Status | +|---|---|---| +| #35 | Clean up frozen-instance code | closed (this milestone cycle) | +| #76 | Add expression form of `with` (`withf`) | closed | +| #80 | Multi-shot generators (`@multishot`, `myield`, `myield_from`) | closed | +| #82 | Document scoping of locals in continuations | closed | +| #85 step 1 | `expect[]`; deprecate `return` in `with test:` | closed; step 2 re-milestoned to 3.0.0 | +| #86 | `Values` unpacking in `call` / `callwith` | closed | +| **#83** | **`end_lineno` / `end_col_offset` sweep** | **OPEN — only milestone item left** | + +#83 is explicitly *do last* — cross-cutting, easy to merge-conflict with +anything else in flight. Per session memory: `hasattr` fixes already +landed; the `end_*` propagation sweep is what remains. + +## Bonus material queued for 2.2.0 (not milestone issues) + +Landed between the previous brief refresh (2026-05-06) and today: + +1. **`unpythonic.dialects.bf`** — Brainfuck-as-a-dialect (commit `69ab0ef` + plus follow-ups). The first source-level dialect in the package. + Demonstrates `mcpyrate.Dialect.transform_source` as a *transpiler*: + exotic 1-D syntax in, structured Python out, inspectable via + `bf.compile(src)`. Required `mcpyrate >= 4.1.0` for + `split_at_dialectimport`. Doc: `doc/dialects/bf.md`. +2. **`unpythonic.dialects.befunge`** — Befunge-93 as a dialect + (commits `e500c7b`, `c2ba2c3`). The companion piece to bf: + `transform_source` used as a *reader* rather than a transpiler. + Strict 80×25 toroidal playfield, byte cells, unbounded-int stack; + all Befunge-93 commands including `?` (seedable for tests via + `run(src, *, seed=...)`), `p`/`g` (self-modifying), and `&`/`~` + (integer/character input). Three error categories: `SyntaxError` + (oversize grid), `IndexError` (runtime out-of-grid `p`/`g`), + `UnknownOpcodeError` (`RuntimeError` subclass). The compiled output + is necessarily a single `run(src)` shim — Befunge's 2-D, IP-driven, + self-modifying control flow has no statically-soundable structure. + Together with bf, this gives the package both flavors of + `transform_source` use, with the qualitative-difference framing + documented in the substrate-independent field-guide entry that + motivated shipping both before 2.2.0. +3. **mcpyrate floor bumped to `>=4.1.1`** (commit `c2ba2c3`). The bf + doc reorg added a "Reading the compiled Python" section that walks + the user through `mcpyrate.debug.StepExpansion` for live inspection; + the StepExpansion path goes through the import machinery, which on + first import requires `path_stats` to handle source-level dialect + files — fixed in mcpyrate 4.1.1. Also makes the bf docstring's + "running the file under macropython" claim honest; it had been + technically broken until the upstream hotfix. +4. **`unpythonic.misc.redirect_stdin`** (commit `52ccf36`). Context + manager that feeds `sys.stdin` from a stream — the third sibling of + `contextlib.redirect_stdout` (3.4) and `redirect_stderr` (3.5), + which the stdlib never shipped. Subclasses + `contextlib._RedirectStream` so behavior matches the stdlib siblings + exactly, including the per-instance stack supporting nested + re-entry. +5. **`bf.bf_compile` → `bf.compile`** (commit `6707de5`). The qualified + form (`unpythonic.dialects.bf.compile`) is the recommended spelling + anyway — it avoids shadowing `builtins.compile` in the importer's + namespace. Pre-release rename while there are no users yet. +6. **`the[]` anti-patterns documentation sweep** (commits `5fb9528`, + `577978f`). Added a "Common `the[]` mistakes" subsection to + `CLAUDE.md` covering the four pattern-match-against-your-draft + anti-patterns (`the["X" in out]`, `the[X == Y]`, `the[X] == "Y"`, + `the[a] < b < c`), and fixed misplaced `the[]` across `test_bf`'s + capture sites. +7. **CLAUDE.md tightening** (commit `b063b1e`): documented the + FQN-in-prose / bare-in-examples convention for public-symbol + references in docstrings, comments, and briefs. + +## Open before release + +- **#83 `end_lineno` / `end_col_offset` sweep** — the only milestone + item left. Cross-cutting; do last. +- **One non-blocking deferred item** in `TODO_DEFERRED.md`: cross-module + `accepts_arity` unification with `conditions.signal`. Same status + as the previous brief — not a release blocker. + +--- + +## Issue-by-issue retrospectives + +The summaries below were captured during the sessions that closed each +issue. They are kept here for narrative continuity (what was decided, +what was held back, why a follow-up issue exists); CHANGELOG.md has the +user-facing version, git log has the technical record. + +### #82 — Document scoping of locals in continuations + +**DONE.** `doc/macros.md` got a `#### Topology of continuations` subsection +with inlined `callcc_topology.png` walking through the five panels (Base +case, Sequence, Nested, Confetti chaining, Tail-call composition) and +explaining `cc`/`pcc`. Followed by `#### Scoping of locals in +continuations` covering the rule (each `call_cc[]` is a scope boundary), +the box workaround, and the three load-bearing limits that ruled out +auto-`nonlocal` propagation. + +`unpythonic/syntax/tests/test_conts.py` revived the `"scoping, in +presence of nonlocal"` testset (disabled in 2022 due to a coverage.py +source-parsing issue); the new `[tool.coverage.run]` config in +`pyproject.toml` scopes coverage to production code (excluding +`*/tests/*`) and sidesteps the parse failure at `coverage xml` time. +Pattern documented in `~/.claude/CI-SETUP-NOTES.md` §4a for fleet +propagation. Bonus: `shift`/`reset` attribution corrected (Danvy & +Filinski 1990, not Felleisen). + +### #76 — Add expression form of `with` (`withf`) + +**DONE** (commits `b6423e7`, `ad6c3f6`). `withf(cms, body)` in +`unpythonic.excutil`. Single CM or tuple; body arity auto-detected +(n-arg form receives as-values, thunk discards them). `tryf`/`withf` +share `_accepts_arity` as the single source of truth for the +"default to n-arg form on `UnknownArity`" policy. + +### #85 step 1 — `expect[]`; deprecate `return` in `with test:` + +**Step 1 DONE** (commit `7034add`). `expect[]` macro added in +`unpythonic.syntax.testingtools`; `return expr` continues to work but +emits `DeprecationWarning` at macro-expansion time, with file/line of +the offending `return`. Both forms in the same block → `SyntaxError`. +Capture rules (implicit-LHS `the[]` on a `Compare`, explicit `the[]`) +carry over from `return expr` to `expect[]` unchanged. + +**Step 2 (3.0.0)**: un-hijack `return` so it regains its standard +Python meaning inside `with test:`. Issue #85 has been re-milestoned +to 3.0.0 and remains the tracking ticket. + +### #35 — Clean up frozen-instance code + +**DONE.** + +- `cons.__delattr__` added — was missing, so `del c.car` corrupted the + cell. Tests added. +- `cons.__setattr__` simplified to a one-liner via `object.__setattr__` + in `__init__`; dropped the `_immutable` sentinel. Error message fixed + ("attribute" not "item" assignment). +- `assignonce.__delattr__` overridden: forbid `del e.foo` on defined + names so the assign-once contract can't be bypassed via `del; rebind`. +- `env._direct_write` whitelist removed; internal slots now installed + via `object.__setattr__` in `__new__` and `finalize()`. Client + `e._env = ...` is now rejected. +- `frozendict` docstring clarifies that "frozen" refers to the mapping, + not instance attributes. +- Stdlib alignment via `FrozenAttributeError` shim: multiply-inherits + from `TypeError` (legacy) and `dataclasses.FrozenInstanceError` + (stdlib convention). Either catch path works. Issue **#102** opened + against 3.0.0 to track dropping the `TypeError` base. + +### #80 — Multi-shot generators + +**DONE** across two slots. v1 (`@multishot`, `myield`, +`MultishotIterator`) shipped earlier; `myield_from` follow-up landed in +`839953b`. + +Final v1 surface: `gi_frame` always None, `gi_code` as the liveness +signal, `gi_running` always False, `gi_yieldfrom` tracks the inner +iterator while delegating, `__copy__` shallow fork, `__deepcopy__` +raises TypeError, `__del__` calls `close`. + +`myield_from` architecture: let/cc-style rest-of-outer capture +(`_rest = call_cc[get_cc()]`) + tail-called driver + cut-the-tail +`_yieldf` to escape each `(captured_cc, value)` to the user's +trampoline. Bonus refactor: shared `_step(k, mode, value)` helper + +send-to-bare-myield bug fix + `throw` capture-and-update. Misplaced +`myield_from` raises `SyntaxError` symmetrically with `myield`. + +Brief: `briefs/multishot-implementation.md`. Tests: +`unpythonic/syntax/tests/test_multishot.py` (72 passing). Doc: +"Multi-shot generators with `@multishot` and `myield`" + "Delegating to +another multi-shot: `myield_from`" subsections in `doc/macros.md`. + +### #83 — Source-location field support (Python 3.8+) + +**OPEN — last 2.2.0 milestone item.** Audit `lineno` / `col_offset` +handling and extend to also handle `end_lineno` and `end_col_offset`. +`hasattr` fixes already landed in earlier work; the `end_*` propagation +sweep remains. Cross-cutting; do last. + +--- + +## Standing CI note (post-`ec1725b`) + +PyPy-3.11 / macOS-latest / windows-latest were red on `5bcf3d4` due to +`unpythonic.misc.callsite_filename` reaching `inspect.stack`, which +raised `TypeError` from `getframeinfo` (`f_lineno = None` on at least +one frame in the walk). Fixed in `ec1725b` by switching to +`sys._getframe`; we never needed line info, only `co_filename`. Root +cause inside PyPy is unconfirmed (macro-frame line table from +`code.replace` in `namelambda`? PEP 657 quirk? something else?), but +the fix is route-around, not root-cause, and that's fine. CPython +unaffected on every platform. diff --git a/briefs/befunge-dialect.md b/briefs/befunge-dialect.md new file mode 100644 index 00000000..39e8380e --- /dev/null +++ b/briefs/befunge-dialect.md @@ -0,0 +1,575 @@ +# CC Brief: `befunge` — a source-level dialect for Befunge-93 + +## Goal + +Add a new dialect `unpythonic.dialects.befunge` that compiles Befunge-93 +source into a thin Python shim that invokes a runtime interpreter. Lands +as the second pedagogic example of `mcpyrate`'s `transform_source` hook, +complementing `unpythonic.dialects.bf`. + +## Why a second `transform_source` example + +`bf` already covers the basic territory: an exotic 1-D source language +gets rewritten into legible, structured Python. Loops in brainfuck are +lexically nested, the language is essentially linear, and `bf_compile` +output reads as Python the way a textbook bf-to-Python translation would. + +Befunge-93 deliberately doesn't fit that mould: + +- **2-D playfield** with the IP moving in four directions. There is no + syntactic loop structure for `transform_source` to lower to a `while`. +- **Self-modifying code** via `p` (put) and `g` (get). At "compile time" + no static analysis can tell what a cell *means*: the same cell may be + entered going east as a digit and going north as a string-mode quote, + and may be overwritten mid-run. +- **`?` random direction**, **string mode (`"`)**, **`#` skip-next**, and + the toroidal grid all conspire so that control flow is fundamentally + IP-driven, not lexical. + +So `compile` cannot produce Python that mirrors the program. The honest +compilation strategy is: emit a one-line shim that hands the playfield +text to a runtime interpreter shipped in this module. + +That makes Befunge a *complementary* example, not a redundant one. +Where `bf` demonstrates `transform_source` as a transpiler — exotic +syntax in, structured Python out — Befunge demonstrates `transform_source` +as a *reader* for non-Python-flavored, non-line-oriented source: the +playfield is data, the interpreter does the work, and the dialect's job +is to wrap the file body in a single function call. The contrast is +the teaching value. + +The module docstring states this contrast explicitly. We do not pretend +the compiled output is informative the way `bf`'s is. + +## Layout + +``` +unpythonic/dialects/befunge.py # Playfield, run, Befunge dialect class +unpythonic/dialects/tests/test_befunge.py # runtests() per existing convention +``` + +Same shape as `bf`, mirroring the existing dialect-examples convention +(one module per dialect, tests in `dialects/tests/`). + +## Public API + +```python +from unpythonic.dialects.befunge import dialects, Befunge # dialect activation +from unpythonic.dialects.befunge import run # programmatic entry point +from unpythonic.dialects.befunge import Playfield # exposed for testability +from unpythonic.dialects.befunge import UnknownOpcodeError # raised on unknown opcode +``` + +`__all__ = ["Befunge", "UnknownOpcodeError", "Playfield", "run"]`. + +`run(src, *, seed=None)` is the only public entry point with kwargs, +and `seed` is the single irreducible kwarg — see "I/O capture" below for +why `stdin` and `stdout` are *not* kwargs. + +## Prerequisite — separate prior commit: rename `bf_compile` → `compile` + +Before the Befunge work lands, do a tiny standalone commit on the `bf` +dialect: + +- `bf_compile` → `compile` in `unpythonic/dialects/bf.py`. +- Update `__all__`, the dialect class's `transform_source`, the module + docstring, and the test imports in `unpythonic/dialects/tests/test_bf.py`. + +Rationale: + +- `unpythonic.dialects.bf.compile(src)` reads better than `bf_compile`; + the `bf_` prefix duplicates the module name. +- `mcpyrate.compiler.compile` sets the precedent — fleet consistency. +- `bf.py` doesn't itself call `builtins.compile` anywhere, so shadowing + inside the module is harmless. The module docstring will recommend + qualified access via `from unpythonic.dialects import bf` followed by + `bf.compile(src)`, matching the project-wide `from … import …` import + style, rather than `from … import compile` (which would shadow the + builtin in the importer's namespace). +- `bf_compile` is only public in 2.2.0-dev (not yet released), so the + rename costs nothing in compatibility terms. + +After the rename: `bf` exports `compile` and `befunge` exports `run`. +Symmetric in style: the public function in each dialect module is named +for what that dialect's pedagogic entry point actually does — `bf` +compiles to Python; `befunge` runs an interpreter. + +## Design decisions + +### Dialect is source-level only + +`Befunge` overrides `transform_source` and leaves `transform_ast` at its +default (returns `NotImplemented`). Same as `bf` — the whole point of +this and the `bf` example is to demonstrate `transform_source`. + +### Strict Befunge-93 + +Canonical behavior, not a relaxed superset: + +- **80×25 grid**, fixed. +- **Byte-valued cells** (0–255), wrapping on `p` (put masks `value & 0xFF`). +- **Stack of unbounded Python `int`s.** The 93 spec is fuzzy on stack + width; Python `int` is the natural idiom and matches how integers + behave elsewhere in `unpythonic`. Cells stay byte-valued — that part + is non-negotiable. +- **IP wraparound is toroidal.** Off the right edge → column 0 of the + same row, etc. Canonical. +- **Stack underflow on `pop`** returns 0. Canonical. + +### Out-of-bounds grid access raises `IndexError` + +Both `g` (read) and `p` (write) raise `IndexError` if the requested +`(x, y)` falls outside the 80×25 grid. + +Rationale: + +- Strict-93 says the grid *is* 80×25; there is no "outside" to read + zeros from. The "OOB read returns 0" convention is from Befunge-98, + which we are explicitly not implementing. +- Symmetric policy is easier to document and reason about: same + boundary, same reaction. +- Surfaces bugs in computed-coordinate arithmetic (off-by-one in + stack juggling) instead of silently returning 0. +- The stack-underflow=0 precedent doesn't push toward returning 0 + here: stack underflow is *in* the 93 spec; OOB grid access is + *un*specified. + +The IP itself never goes OOB — the toroidal wraparound is a separate +concern and applies only to IP movement, not to `g`/`p`. + +### `?` random direction — seedable + +`run(src, *, seed=None)`. Internally: + +```python +import random +rng = random.Random(seed) +# `?` does: dx, dy = rng.choice([(1,0), (-1,0), (0,1), (0,-1)]) +``` + +`random.Random(seed)` as an *instance* — does not touch the global RNG, +so a Befunge program can't perturb the user's process-wide random state. +`seed=None` gives normal nondeterminism (OS entropy via `Random()`'s +default behavior). + +`seed` is a kwarg because there is no stdlib mechanism to "redirect +`random.Random()` instances at a distance" — the seed has to be +injected at the call site. + +### I/O — operator semantics differ from bf + +In Befunge the four I/O commands have distinct integer-vs-character +flavors: + +- `.` pops and prints an **integer** (followed by a space, per spec). +- `,` pops and prints a **character** (`chr(value & 0xFF)`). +- `&` reads a whitespace-delimited integer from stdin and pushes it. +- `~` reads one character from stdin and pushes its `ord`. + +EOF on `&` and `~` pushes 0. Matches `bf`'s EOF=0 convention. + +The module docstring calls out that `.` and `,` swap roles relative to +`bf` — readers comparing the two examples could otherwise be tripped up. + +### I/O capture — no `stdin`/`stdout` kwargs + +`run` does not take `stdin` or `stdout` kwargs. Tests capture I/O +through stdlib mechanisms (or, for stdin, the unpythonic gap-filler — +see next subsection): + +- stdout: `redirect_stdout(io.StringIO())` (from `contextlib`) around + the call. +- stdin: `redirect_stdin(io.StringIO("..."))` (from `unpythonic`) around + the call. + +Rationale: `seed` is irreducible (no stdlib equivalent), but +`stdin`/`stdout` redirection is a solved problem at the stream level. +Adding kwargs solely for I/O capture would expand the API surface +without giving the caller anything they can't already do with a +context manager. `bf` already follows this approach; `befunge` matches. + +### Prerequisite — separate prior commit: `unpythonic.misc.redirect_stdin` + +`contextlib` ships `redirect_stdout` (3.4) and `redirect_stderr` (3.5), +but not `redirect_stdin`. This is the textbook "stdlib almost gets it +right, then punts" pattern that unpythonic's gap-filling charter targets. +`tests/test_bf.py` already has a local `_redirect_stdin` helper for +this; the Befunge tests are about to need the same, and an obvious +recurring use case elsewhere puts this past the bar for promotion. + +Add it to `unpythonic.misc`, alongside the other small stdlib +gap-fillers (`maybe_open`, `UnionFilter`, `si_prefix`, `timer`, +`safeissubclass`): + +Subclass `contextlib._RedirectStream` directly. Yes, the underscore +makes it private API — but `_RedirectStream` was extracted in 3.5 +specifically so that `redirect_stdout` and `redirect_stderr` could +share machinery, and its shape has been stable across every release +since. unpythonic's floor is 3.10, so we're well downstream of any +shake-out. We're explicitly the third sibling; using the same +machinery is the most honest expression of that. + +```python +from contextlib import _RedirectStream + +class redirect_stdin(_RedirectStream): + """Context manager that redirects ``sys.stdin`` to *target*. + + The third sibling: `contextlib` ships `redirect_stdout` (3.4) and + `redirect_stderr` (3.5), but punted on `redirect_stdin`. This + fills the gap, sharing `contextlib._RedirectStream` machinery so + behavior matches its stdlib siblings exactly — including the + per-instance stack that supports nested re-entry on the same + instance. + + Like its stdlib siblings, this redirects the global ``sys.stdin`` + and is **not** safe under concurrent use from multiple threads — + parallel redirects from different threads will stomp on each other. + For tests (the primary use case), single-threaded use is the norm. + """ + _stream = "stdin" +``` + +Thread safety note: matches stdlib's choice deliberately. A truly +thread-aware variant would need `sys.stdin` replaced by a proxy that +dispatches per-thread (similar in spirit to `unpythonic.dynassign.dyn` +but at the file-like-object level), and that's a different abstraction +worth its own design discussion — not a refinement of this gap-filler. + +Same commit: + +- Add to `unpythonic/misc.py`; update `__all__`. +- Re-export from top-level `unpythonic/__init__.py` (already happens + via `from .misc import *`). +- Add `redirect_stdin` unit tests in `unpythonic/tests/test_misc.py` + (basic redirect, exception path restores `sys.stdin`, nested redirects + on the same instance unwind correctly). +- Add a documentation entry under the **Other** section of + `doc/features.md`, alongside `maybe_open` and `environ_override` — + fellow stdlib gap-fillers in the same file/stream/process-state + category. Both the navigation TOC link near the top of the file and + the per-feature subsection later in the file. +- Replace the local `_redirect_stdin` helper in + `unpythonic/dialects/tests/test_bf.py` with the public function. +- CHANGELOG entry under 2.2.0 "Added". + +### Documentation in the prologue — prefer a module docstring + +Befunge-93 has no comment syntax; `#` is a real command (skip-next-cell). +Trying to recognize Python comments inside the body would necessarily +involve guessing — any rule like "lines starting with `# ` are Python +comments" is a heuristic that picks a convention rather than a clean +parse. We don't do this. + +The unambiguous answer is: **commentary goes above the dialect-import +line.** `split_at_dialectimport` preserves the prologue verbatim as +Python text, so anything before the dialect-import is plain Python. + +The *recommended* form is a **module docstring**, not stand-alone `#` +comments. A docstring shows up in `help(module)`, so a Befunge program +imported as a Python module documents itself the same way any other +Python module does: + +```python +"""Hello from Befunge! + +Demonstrates string-mode push, the ":#,_@" print loop, and the v/^ +vertical IP-redirect cells. +""" + +from unpythonic.dialects.befunge import dialects, Befunge + + +``` + +`# noqa` / Python-tooling directives go in the prologue too, as +ordinary Python comments — same boundary, same machinery. + +The canonical "how to use" example in this module's docstring will +be the docstring-headed form; bare-comment files still work but are +the less idiomatic choice. + +Nothing supports comments inside the body or after the program — +Befunge has no end-of-program textual marker (`@` is a *runtime* +halt), so trailing comments are equally ambiguous. + +### `transform_source` body + +Following `bf`'s shape, with `split_at_dialectimport`: + +```python +class Befunge(Dialect): + def transform_source(self, text): + r = split_at_dialectimport(text, type(self).__name__, self.lineno) + if r is None: + return text + prologue, other, body = r + body = _strip_leading_blank_lines(body) + shim = ( + "from unpythonic.dialects.befunge import run\n" + f"run({body!r})\n" + ) + return prologue + "".join(other) + shim + + +def _strip_leading_blank_lines(text: str) -> str: + lines = text.splitlines(keepends=True) + while lines and not lines[0].strip(): + lines.pop(0) + return "".join(lines) +``` + +**Leading blank lines must be stripped.** A typical dialect-activated +file looks like: + +```python +"""Hello, World.""" + +from unpythonic.dialects.befunge import dialects, Befunge + +>25*"!dlrow ,olleH":v +... +``` + +`split_at_dialectimport` returns `body` starting at the line *after* the +dialect-import — including the blank line that separates it from the +program. If we don't strip, that blank line becomes row 0 of the +playfield (all spaces); the IP starts at `(0, 0)` going east, walks +80 no-op cells, wraps toroidally back to `(0, 0)`, and loops forever. + +The strip is *line-level*: lines whose stripped form is empty get +removed from the start of `body`. **Leading spaces inside a non-blank +line are preserved** — those are meaningful no-op cells in the +playfield and column alignment matters. Trailing blank lines are not +stripped (harmless; `Playfield(src)` pads to 25 rows anyway). + +The playfield text is then embedded as a string literal via `repr()`, +which handles escaping and preserves every remaining character verbatim, +including in-line whitespace. + +### `Playfield` class + +```python +class Playfield: + """Strict Befunge-93 playfield: 80×25 byte cells. + + Reads and writes outside the grid raise `IndexError`. Used by the + interpreter, and exposed publicly so that grid layout, padding, + truncation, and OOB policy can be unit-tested in isolation from + the interpreter loop. + """ + WIDTH = 80 + HEIGHT = 25 + + def __init__(self, src: str = ""): ... + def __getitem__(self, xy: tuple[int, int]) -> int: ... + def __setitem__(self, xy: tuple[int, int], value: int) -> None: ... +``` + +The constructor parses `src`: + +- Split on newlines. +- Strip leading and trailing entirely-blank lines (in-line leading + spaces on a non-blank line are preserved — those are no-op cells). +- If the remaining line count exceeds 25, raise + `SyntaxError("befunge: program exceeds 25-row grid (got N rows)")`. +- If any line exceeds 80 characters, raise + `SyntaxError("befunge: line K exceeds 80-column grid (got M cols)")`. +- Otherwise: pad with blank rows up to 25, right-pad each line with + spaces to 80. +- Cells store `ord(ch) & 0xFF` (academic for ASCII source, keeps the + contract uniform with `__setitem__`). + +**Why `SyntaxError`, not silent truncation.** Strict Befunge-93's +grid is fixed at 80×25; an oversize program *is not* Befunge-93. +Silent truncation could drop the `@` halt and turn a finite program +into an infinite loop — the worst possible failure mode (a program +that compiles, runs, and never returns). Failing loudly at compile/ +load time is symmetric with the runtime `IndexError` policy for +out-of-grid `g`/`p`: out-of-grid is an error, period. + +### `run(src, *, seed=None)` + +The interpreter loop. Pseudo-code: + +```python +def run(src: str, *, seed: int | None = None) -> None: + import sys + rng = random.Random(seed) + pf = Playfield(src) + stack: list[int] = [] + def push(v): stack.append(v) + def pop(): return stack.pop() if stack else 0 + + x, y = 0, 0 + dx, dy = 1, 0 + string_mode = False + + while True: + cell = pf[(x, y)] + ch = chr(cell) + if string_mode: + if ch == '"': + string_mode = False + else: + push(cell) + else: + # dispatch on ch: digits, + - * / % ! `, < > ^ v ?, _ |, " :, \\, $, ., ,, &, ~, #, p, g, @, space + ... + x = (x + dx) % Playfield.WIDTH + y = (y + dy) % Playfield.HEIGHT + if ch == '@': # halt — handled inside dispatch by `return` + break +``` + +(Halt actually short-circuits inside the dispatch with `return`; the +sketch is illustrative.) + +Operator coverage — the full Befunge-93 set: + +| Char | Meaning | +|-----------------|--------------------------------------------------------| +| `0`–`9` | push digit | +| `+ - * / %` | arithmetic; `/` and `%` by zero push 0 (per spec) | +| `!` | logical not | +| `` ` `` | greater-than | +| `> < ^ v` | set IP direction | +| `?` | random direction (uses `rng`) | +| `_` | horizontal if: pop; if 0 go right, else left | +| `\|` | vertical if: pop; if 0 go down, else up | +| `"` | toggle string mode | +| `:` | duplicate top | +| `\\` | swap top two | +| `$` | discard top | +| `.` | pop, print int + space | +| `,` | pop, print char (`chr(value & 0xFF)`) | +| `&` | read int, push | +| `~` | read char, push `ord` | +| `#` | trampoline: skip next cell | +| `p` | put: pop y, x, v; `pf[(x, y)] = v` | +| `g` | get: pop y, x; push `pf[(x, y)]` | +| `@` | halt | +| (space) | no-op | + +Anything else: raise `UnknownOpcodeError(f"unknown command at ({x}, {y}): {ch!r}")`. +Strict mode — Befunge-93 has a fixed command set, and silently treating +unknowns as no-ops would hide source-corruption bugs. + +`UnknownOpcodeError` is a custom exception subclassing `RuntimeError`. The +case for inventing a type rather than reusing a stdlib one: this is a +VM hitting an unknown opcode, which doesn't fit any stdlib category +cleanly — `SyntaxError` is reserved by convention for parse-time use, +`RuntimeError` is too generic, `ValueError` doesn't quite match +("inappropriate value" is a stretch for "byte at this cell isn't a +command"). The pattern is "stdlib has no clean fit for the domain +concept, so define a domain exception": `pickle.UnpicklingError` and +`struct.error` follow the same principle. + +Subclassing `RuntimeError` keeps blanket runtime-error catchers +working; the specific class enables targeted `except UnknownOpcodeError`. + +This error fires at *runtime*, not at `Playfield(src)` load time. A +Befunge source can be entirely valid (every cell is printable ASCII) +yet contain a cell that's never a recognized command. Self-modifying +code (`p`) can also write arbitrary bytes into cells. Detection +necessarily happens when the IP actually visits the cell as an +instruction. + +Three distinct error categories, three distinct conditions: + +- `SyntaxError` — source-level malformation at `Playfield(src)` + construction (oversize grid). Pre-execution. +- `IndexError` — out-of-grid runtime access via `g`/`p`. Sequence-subscript + convention. +- `UnknownOpcodeError` — in-grid byte at the IP isn't a recognized command. + Runtime, custom domain exception. + +Each picks the most precise type available; we only invent where +stdlib has no clean answer. + +## Testing + +`unpythonic/dialects/tests/test_befunge.py`, with the usual `runtests()` +entry point. Coverage: + +- **`Playfield` unit tests**: + - 80×25 dimensions; default-blank cells read as `ord(' ')`. + - Source shorter than 25 lines pads with blank rows. + - Source with >25 lines raises `SyntaxError`. + - Lines shorter than 80 cols right-pad with spaces. + - Lines longer than 80 cols raise `SyntaxError`. + - Trailing blank lines are stripped before the dimension check + (so a 25-line program with a trailing blank line still loads). + - OOB read raises `IndexError`. + - OOB write raises `IndexError`. + - In-bounds write masks to byte (`pf[(0, 0)] = 0x1FF; pf[(0, 0)] == 0xFF`). + +- **Interpreter unit tests** via direct `run` calls with captured stdout: + - Arithmetic: `9 5 - .` style. + - String mode: `"!dlroW">:#,_@`-style print loop. + - Stack ops: `:`, `\\`, `$`. + - `#` trampoline. + - `_` and `|` conditional direction. + - `?` with seeded rng — assert deterministic output for fixed seed. + - `p` / `g` round-trip on in-bounds coordinates. + - `g` / `p` raise `IndexError` for OOB coordinates. + - Toroidal IP wrap (program at `x = 79` moving east lands at `x = 0`). + - Empty stack underflow returns 0. + - `@` halts. + - Unknown command raises `UnknownOpcodeError`. + +- **`Hello from Befunge!`**: a custom Hello World matching the + Lispython / Pytkell / bf family tradition. Rewards the curious reader + of CI logs, and exercises the full machinery — string mode, the + `:#,_@` print loop, vertical IP-redirect cells, halt. + +- **Dialect activation test**: a minimal Befunge-in-`.py` snippet + loaded through the dialect machinery (`mcpyrate.compiler.create_module` + + `run`, same idiom as `test_bf.py`) actually executes and produces + the expected stdout. Cover the realistic case with **a blank line + between the dialect-import and the program** — verifies the + leading-blank-line strip; without it the program would loop + forever on a blank row 0. + +- **`&` / `~` smoke test** using `redirect_stdin` (public, from + `unpythonic`, added in commit 1); EOF returns 0. + +## Non-goals + +- Befunge-98 features: unbounded grid, multiple IPs, stack-of-stacks, + fingerprints, `k` (iterate), etc. Strict-93 only. +- Optimization. The interpreter is a clean dispatch loop. No JIT, no + basic-block caching, no peephole optimizations, no static analysis + of common Befunge idioms. (And in any case, full Befunge static + analysis is defeated by `p` — self-modifying code can change a cell + between any two visits, so a pre-execution analysis can never be + sound for general programs.) Pedagogic transparency trumps cleverness, + same call as `bf`. +- Compile-time tracing or partial evaluation. Befunge-93 is Turing + complete; symbolic execution at compile time would over-promise and + under-deliver, and would conflate compilation with interpretation + in a way `bf` deliberately avoided. +- A `befunge_compile` (or `compile`) function that returns a Python + source string. The shim's only content is `run(src!r)`, which has no + pedagogic value to expose as a separate API. + +## Delivery style + +Conventional `unpythonic.dialects` module style — matter-of-fact +docstrings, commit messages, and test names, matching `bf` and the +other sibling dialect modules. The `transform_source` contrast with +`bf` is presented as a design observation in the module docstring. + +## Milestone + +2.2.0. Three commits: + +1. `unpythonic.misc.redirect_stdin` + tests + CHANGELOG; replace the + local helper in `test_bf.py`. +2. `bf` rename: `bf_compile` → `compile` (small, isolated). +3. Befunge dialect: module + tests + CHANGELOG entry. + +Commits 1 and 2 are mutually independent; both must land before +commit 3. All three on `master` for the in-progress 2.2.0 release. +Memory's "Queued for the 2.2.0 release" list grows accordingly. diff --git a/briefs/bf-dialect.md b/briefs/bf-dialect.md new file mode 100644 index 00000000..98cb4f52 --- /dev/null +++ b/briefs/bf-dialect.md @@ -0,0 +1,215 @@ +# CC Brief: `bf` — a source-level dialect for brainfuck + +## Goal + +Add a new dialect `unpythonic.dialects.bf` that compiles brainfuck source into +macro-enabled Python and runs it. Fills the one remaining gap in unpythonic's +dialect-example collection: the existing dialects (Lispython, Lispy, Listhell, +Pytkell) all demonstrate `transform_ast`. None demonstrate `transform_source`, +the mcpyrate hook for full-module source-text transformers — the modern +equivalent of what old Lisp folks called a *reader macro*. + +mcpyrate's own `Dialect.transform_source` docstring uses brainfuck as its +illustrative example and ends with the line *"Implementing the actual +BF->Python transpiler is left as an exercise"*. This brief picks up that +gauntlet. + +Two simultaneous goals on a single code path: + +1. **Practical joke**: canonical-brainfuck-compatible dialect that lets you + put `++++++[>++++++++<-]>.` in a `.py` file and run it. +2. **Pedagogic tool**: `bf_compile(src)` returns human-readable Python source, + useful for understanding what a given brainfuck program does by rewriting + it in a language a human can actually read. + +The dialect activation just runs what `bf_compile` produces. The pedagogic +value is a consequence of insisting the compiled output be legible Python; +we do not maintain two compilation modes. + +## Layout + +``` +unpythonic/dialects/bf.py # Tape, bf_compile, BF dialect class +unpythonic/dialects/tests/test_bf.py # runtests() per existing convention +``` + +Matches the existing dialect-examples layout (one module per dialect, tests in +`dialects/tests/`). + +## Public API + +```python +from unpythonic.dialects.bf import dialects, BF # dialect activation +from unpythonic.dialects.bf import bf_compile # pedagogic / pure function +from unpythonic.dialects.bf import Tape # exposed for testability +``` + +`__all__ = ["BF", "bf_compile", "Tape"]`. + +## Design decisions (all confirmed in pre-build discussion) + +### Dialect is source-level only + +`BF` overrides `transform_source` and leaves `transform_ast` at its default +(returns `NotImplemented`). This is the whole point of the exercise — the +other dialects in the package already demonstrate `transform_ast` extensively. + +A previous iteration of the design put `reset` behind a `@namemacro`, but +that conflicted with the pedagogic goal: the output of `bf_compile` must be +self-contained runnable Python, not a bare identifier that only resolves +during AST expansion. So `reset` is handled entirely in the source +transformer. This also keeps the dialect purely single-layer, which sharpens +the example. + +### Cell semantics: 8-bit wrapping via a `Tape` class + +```python +class Tape(defaultdict): + def __init__(self): + super().__init__(int) + def __setitem__(self, key, value): + super().__setitem__(key, value & 0xFF) +``` + +Because `tape[i] += n` desugars to `tape[i] = tape[i] + n`, overriding +`__setitem__` alone covers `=`, `+=`, `-=` uniformly. Masking at the setter +means the compiled body stays free of `& 0xFF` noise, which preserves +legibility. + +Rationale for wrapping rather than using unbounded ints: compatibility with +canonical brainfuck programs that rely on `255 + 1 == 0`. The "Hello, World!" +superposition demands it. + +### Tape structure: `defaultdict(int)` + +`tape[ptr]` auto-extends in both directions (negative and positive indices) +and reads zero for untouched cells. No explicit size, no circular wrapping of +the pointer. + +### EOF on `,` returns 0 + +Classical brainfuck has three conventions (`0`, `-1`, cell-unchanged). We +pick `0`. Documented in the module docstring. + +### Folding + +Consecutive *identical* command chars fold into one statement: + +- `+++++++` → `tape[ptr] += 7` +- `>>>` → `ptr += 3` + +No cancellation of opposites (`+-`, `><` do not annihilate). Compilation is +honest: what you wrote is what you get, just collapsed where collapse is +lossless. + +### Loop structure + +`[` → `while tape[ptr]:` plus indent; `]` → dedent. No label arithmetic. +Indentation carries the structure — the goto-hell of brainfuck becomes +ordinary Python `while` loops, which is already a pedagogic payoff on its +own. + +### I/O + +- `.` → `stdout.write(chr(tape[ptr])); stdout.flush()` +- `,` → reads one byte; EOF → 0 + +`stdin` and `stdout` are imported in the emitted prelude from `sys`. + +### Comments — the "everything-not-a-command is a comment" rule + +Classical brainfuck treats any non-command character as a no-op. We preserve +that semantics *and* the comment text: consecutive runs of non-command +characters compile into Python comments, positioned where they appeared in +the source. + +Per-line handling: + +- Line (stripped) equals `reset`: emit the reset block (see below). +- Otherwise, walk the line alternating between command runs and non-command + runs: + - Command runs emit folded statements. + - Non-command runs, if they contain non-whitespace text, emit as + `# ` on their own output line, in position. + - `[` and `]` govern indentation as described above. +- Fully-blank lines pass through; consecutive blanks collapse to one. + +If a non-command run already begins with `# ` or `#`, one leading `#` (and +its trailing space) is stripped before we re-prepend `# `. This makes the +bf-author-style `# real comment` and the bare `real comment` both come out as +`# real comment` in the output. Uniform. + +Example: `+++ move right >>>` compiles to + +```python +tape[ptr] += 3 +# move right +ptr += 3 +``` + +### `reset` — source-level keyword + +Triggered *only* when a line, stripped, equals exactly `reset`. Substrings +in longer comments (`"we reset here"`) do not trigger. Compiles to: + +```python +# reset +tape.clear() +ptr = 0 +``` + +Enables multiple brainfuck programs in the same file. + +### Emitted prelude + +Every compiled module starts with: + +```python +from collections import defaultdict +from sys import stdin, stdout +from unpythonic.dialects.bf import Tape +tape = Tape() +ptr = 0 +``` + +`bf_compile(src)` output is thus self-contained and runnable (given +`unpythonic` installed) without going through the dialect machinery. + +## Testing + +`unpythonic/dialects/tests/test_bf.py` with the usual `runtests()` entry +point. Coverage: + +- **`bf_compile` snapshot tests**: fixed bf input → expected Python output, + for a handful of inputs exercising folding, loops, comments, `reset`, and + `#`-style comments. +- **`Tape` unit tests**: wrap at 256, negative indices, default zero. +- **End-to-end execution tests**: compile → exec → capture stdout. Programs: + - `"Hello from bf!"` printer (parallels `"Hello from Lispython!"` etc. + in sibling dialect tests — rewards the curious reader of CI logs). + - A trivial multi-program file using `reset` between programs. + - A small arithmetic-loop smoke test. +- **Dialect activation test**: a minimal bf-in-`.py` snippet loaded through + the dialect machinery actually runs and produces the expected stdout. + +No `cat`-style `,`-using test in the initial batch — stdin redirection for +the dialect-activated path is awkward and not worth the machinery for a +smoke test. A direct unit test of the compiled `,` behavior at the Python +level covers the semantics. + +## Non-goals + +- Optimization beyond run-folding. No `[-]` → `tape[ptr] = 0`, no balanced + `[->+<]` → copy-loop recognition, no loop-invariant motion. Pedagogic + transparency trumps cleverness. +- Arbitrary-precision cells. Canonical 8-bit behavior. +- Multiple tapes, variable cell widths, or any of the brainfuck-dialect + extensions (`brainfuck++`, etc.). +- Error reporting beyond a single exception on unbalanced brackets during + compilation. + +## Delivery style + +Conventional `unpythonic.dialects` module style — matter-of-fact +docstrings, commit messages, and test names. Module docstring treats +brainfuck as a target language; no commentary on the choice. diff --git a/briefs/done/python-3.15-support.md b/briefs/done/python-3.15-support.md new file mode 100644 index 00000000..e3218248 --- /dev/null +++ b/briefs/done/python-3.15-support.md @@ -0,0 +1,127 @@ +# CC Brief: Python 3.15 support (unpythonic side) + +Companion to `mcpyrate/briefs/done/python-3.15-support.md`, which carries the full AST survey and the expander-side work. This one is the unpythonic half, written to stand on its own for anyone looking only at this repo. + +Three fleet projects read the Python AST directly and so are the ones a CPython minor version can break: `mcpyrate`, `unpythonic`, and `pyan` (which has its own brief at `pyan/briefs/done/python-3.15-support.md`, and is the only one with a confirmed 3.15 crash). + +## Context + +CPython 3.15 reached rc1 in August 2026. `unpythonic`'s `requires-python = ">=3.10,<3.15"` cap is deliberate and stays until this work lands: a macro layer running against an AST grammar it does not know invites a crash, or worse, a silent misexpansion. Raising the cap is the last step. + +`mcpyrate` goes first, being the dependency. Nothing here can be finished before the expander understands the new grammar. + +Prior art for the analysis shape: issue #93 (closed), which tracked the 3.10–3.12 AST changes by asking, per new form, which macro-layer components must learn to detect it. + +## What changed in the AST + +Three field changes, no new node types, from two PEPs. Verified 2026-08-16 by diffing `Parser/Python.asdl` between the 3.14 and 3.15 tags. + +``` +- | Import(alias* names) +- | ImportFrom(identifier? module, alias* names, int? level) ++ | Import(alias* names, int? is_lazy) ++ | ImportFrom(identifier? module, alias* names, int? level, int? is_lazy) +- | DictComp(expr key, expr value, comprehension* generators) ++ | DictComp(expr key, expr? value, comprehension* generators) +``` + +- **PEP 810, lazy imports** — `lazy import json`, `lazy from pathlib import Path`. Module scope only. + - `lazy` is a soft keyword admitted only before `import` / `from`, so it does **not** collide with this library's own `lazy` macro. `lazy[...]`, `from unpythonic.syntax import macros, lazy` and `lazy = 5` all parse exactly as before. This was checked specifically, in the grammar, because the name clash looks alarming. +- **PEP 798, unpacking in comprehensions** — two different AST consequences: + - `{**d for d in dicts}` builds `DictComp(key=d, value=None)`. The mapping lands in `key`; `value` being `None` *is* the marker. Previously `value` was always a node, so this is the one that can fail silently. + - `[*L for L in lists]`, `{*s for s in sets}`, `(*L for L in lists)` put a `Starred` in `elt`. No grammar change was needed for these, and CPython's unparser needed no new code — existing `Starred` handling covers them. + +Also, AST node constructors now raise `TypeError` for a missing required field or an unknown kwarg, promoted from the `DeprecationWarning` in force since 3.13. + +## What is already verified about this repo + +- **No AST-constructor deprecations remain.** Full suite on 3.14.6 under `-W error::DeprecationWarning`, with bytecode caches cleared first so every macro genuinely re-expands: 3830 pass, 0 fail, 2 errors, neither AST-related. The 3.13-era `arguments(posonlyargs=[])` cleanup holds. + - Clearing the caches is not optional for this check. A warm cache skips expansion entirely, and the run then proves nothing while looking identical. See the `runtests.py` item in `TODO_DEFERRED.md`. +- **A value-less `DictComp` traverses safely.** `mcpyrate`'s `ASTVisitor` / `ASTTransformer` inherit CPython's `generic_visit`, which leaves a field alone when its value is neither a list nor an `AST` — so `value=None` passes through every walker in this library without special handling. +- **`scopeanalyzer` needs no change.** Its comprehension branch (`scopeanalyzer.py:242`) reads only `generators`; its import branch (`:337`) reads only `names`. Neither `is_lazy` nor a value-less `DictComp` reaches it. + +## How to start (2026-08-17) + +**mcpyrate's side is done**, so this is unblocked: its import hook, unparser and lazy +macro-import rejection all landed, and its suite is green on 3.15.0rc1. Two practical notes +before touching anything. + +**Run the full suite on 3.15 first, before working from the list below.** The static survey in +this brief is triage, not evidence of health. `pyan`'s equivalent brief predicted exactly one +bug and there were two — the second being a `symtable` rename that broke every module +containing a lambda, invisible to the ASDL diff (not a grammar change) and invisible to an +import check (the import succeeds). Only running the suite found it. Clear the bytecode caches +first (`macropython -C .`), or the expander does not re-run and the pass proves nothing. + +**Getting a 3.15 interpreter here needs a workaround until the cap moves.** `requires-python` +is still `>=3.10,<3.15`, so `pdm venv create 3.15` will refuse. Either raise the cap first, or +run against a standalone 3.15 venv with `PYTHONPATH` pointed at the repo — the latter needs +`mcpyrate` importable and `colorama` installed, since `unpythonic` pulls in mcpyrate's +colorizer path. + +**Do not tag a release for this alone.** `mcpyrate` and `unpythonic` ship together, once +verified against each other. mcpyrate went first because the dependency forces it: see the release +ordering below. Its side shipped as 4.3.0 on 2026-08-17. See also the `release` skill. + +## Measured on 3.15.0rc1 (2026-08-17) — the open questions are answered + +**The suite is green on 3.15**: 3862 pass, 0 fail, 0 error, with bytecode caches cleared first +so every macro genuinely re-expanded. For comparison 3.14.6 gives 3883 — the 21-test gap is +SymPy and mpmath missing from the ad-hoc 3.15 venv, not a 3.15 difference. Note the earlier +figure of 3830 in this brief came from a run under `-W error::DeprecationWarning`, where two +errors cut their testsets short; it is not comparable. + +**That result is weaker than it looks, and does not close the work.** No existing test contains +PEP 798 or PEP 810 syntax, so a green suite shows only that nothing *broke* — it never exercises +the new forms at all. Probing them directly is what settles it, and that probe is now run: + +| construct | `lazify` | `autocurry` | `tco` | +|---|---|---|---| +| `[*items(k) for k in ks]` | ok | ok | ok | +| `(*items(k) for k in ks)` | ok | — | — | +| `{**mapping(k) for k in ks}` | ok | ok | ok | + +All produce correct values. **So items 1-3 below need no code change.** The specific hazard +anticipated for `lazify` — wrapping the `Starred`'s value in a promise, so that `*promise` fails +at unpacking — does not occur. + +What remains is therefore small: turn that probe into version-gated tests so the invariant is +kept rather than rediscovered, and move the version metadata. + +## Work items + +Items 1-3 are settled as above and need tests rather than fixes. Python 3.15.0rc1 is installed on both machines. + +**Release ordering is forced, and the CI matrix has to wait for it.** `unpythonic` declares +`mcpyrate>=4.2.0`, and **mcpyrate 4.2.0 as published cannot import any module under 3.15** — +verified by installing it into a clean 3.15 venv, where importing an ordinary module dies with +`TypeError: source_to_xcode() takes 3 positional arguments but 4 were given`. So a 3.15 job in +`unpythonic`'s CI would resolve `mcpyrate` from PyPI, get 4.2.0, and fail for reasons that have +nothing to do with the code under test. The sequence is therefore: + +1. ~~Land unpythonic's code changes — tests, `requires-python` cap, classifier. No CI matrix entry.~~ Done. +2. ~~Release mcpyrate.~~ Done: **4.3.0** *"Weigh anchor"*, on PyPI 2026-08-17. Minor rather than + patch, since a newly supported language version is a capability. +3. ~~Bump the pin to `mcpyrate>=4.3.0` **and** add `"3.15"` to the CI matrix with + `allow-prereleases: true`.~~ Done, in one commit — that CI job is the first check that resolves + mcpyrate from PyPI rather than from a working tree, so it is what actually verifies the two + released packages against each other. +4. Release **unpythonic 2.3.1** — or a minor, on the same reasoning as mcpyrate's 4.3.0. + +"Released together" therefore means same sitting, verified against each other — not simultaneous. +The verification itself is already done: unpythonic's suite was run against the working-tree +mcpyrate and passed 3862/3862. + +1. **`lazify` with a `Starred` comprehension element.** `lazify.py` has no comprehension-specific handling at all, and its `Starred` handling is scoped to call arguments (line 537) and container literals (line 770). A `Starred` in `elt` position is a new shape reaching the generic path. The hazard is wrapping the starred value in a promise, since `*promise` fails at unpacking. Test `with lazify:` over all four new comprehension forms. +2. **`autocurry` and `tailtools` over the same forms.** Same question, same reason; `tailtools.py:1011,1026` already reasons about `Starred` in a different context. +3. **Any macro that dereferences a `DictComp` field directly.** The walkers are safe, but a macro reading the fields is not, and there are two distinct ways to be wrong: + - assuming `value` is a node — it is `None` for the unpacking form; + - assuming `key` is a key — in the unpacking form it holds the whole mapping expression, so the field name lies. + + The second is the easier one to miss, because nothing raises: the code runs and quietly treats a mapping as a key. Note also that the convention is mirrored from the dict *literal* encoding, where `{**a}` is `Dict(keys=[None], values=[Name('a')])` — `None` in `keys`, mapping in `values`, i.e. the opposite halves from the comprehension. Reasoning from the literal to the comprehension gives the wrong answer. Audit both fields, and re-grep once 3.15 can parse the new forms into test fixtures. +4. **Test modules for the new syntax**, version-suffix gated so they skip on older interpreters — the same mechanism `mcpyrate` uses for `test_020_unparser_3_13.py` / `_3_14.py`. +5. **Raise the cap, last.** `pyproject.toml`: `>=3.10,<3.15` → `>=3.10,<3.16`, plus the `Programming Language :: Python :: 3.15` classifier, plus the CI matrix. Keep the upper bound rather than removing it — an unbounded floor makes the resolver seek a version valid for every future Python, and it will silently fall back to an ancient release rather than fail. + +## Adjacent finding + +`unpythonic/tests/test_typecheck.py:205` errors under `-W error::DeprecationWarning` because `isinstance` against `typing.ByteString` reaches `collections.abc.ByteString`, deprecated and slated for removal in 3.17. Python 3.15 widens the warning to mere import or attribute access. Not a blocker for 3.15, but it needs version gating before 3.17 regardless. diff --git a/briefs/fleet-sweep-coverage-and-deps.md b/briefs/fleet-sweep-coverage-and-deps.md new file mode 100644 index 00000000..254204ba --- /dev/null +++ b/briefs/fleet-sweep-coverage-and-deps.md @@ -0,0 +1,231 @@ +# Fleet sweep: coverage hygiene + dev-deps-via-pdm + +Two related but distinct cleanups, queued from the unpythonic 2026-05-06 session +(see `briefs/2.2.0-remaining-issues.md` "Fleet-wide follow-ups"). Read this whole +spec before starting; the two sweeps interact in places and the verification +recipe is shared. + +## Scope + +The fleet projects targeted are PDM-managed Python projects with CI / coverage +workflows. Inferring from `~/.claude/CLAUDE.md` "Active projects": + +- **pylu, pydgq, wlsqm** — Cython projects. +- **pyan3** — pure Python; uses `pytest-cov`. +- **mcpyrate** — pure Python; macro-using. +- **raven** — DPG app, pure Python. +- **unpythonic** — already updated this session; reference implementation. +- **arxiv-api-search** — minimal pure-Python reference; check if active. +- **substrate-independent** — *writing project, not Python*; out of scope unless + its embedded Python tool has its own pyproject.toml. + +For each project: verify it has a `pyproject.toml`, then check whether CI +actually runs coverage (look for `.github/workflows/coverage.yml`, or `--cov` +flags / `coverage run` invocations in `ci.yml`). **Both sweeps are about +coverage tooling**, so projects with no coverage in CI fall out of scope +entirely — there's nothing to align. The broader cleanup of non-coverage +`pip install` lines (sphinx, etc.) is explicitly deferred (see "Out of +scope" below). + +Survey result for the current fleet (2026-05-06): `pylu`, `pydgq`, `wlsqm` +have no coverage in CI and are out of scope — these are Cython projects, and +coverage on Cython modules requires `linetrace=True` plus a separate tracing +build, which none of them set up (the Python-level wrapping is thin enough +that line coverage of the `.py` glue would mostly measure the test scaffolding +anyway). `pyan3`, `mcpyrate`, `raven` are in scope. + +## Sweep A: `[tool.coverage.run]` configuration + +### Why + +Two reasons, in priority order: + +1. **Coverage signal is about which lines of *production code* run.** Tests + are excluded from analysis because their pass/fail/error/total is already + reported by the test runner. Coverage of test files adds rows to the report + without insight. +2. **For projects whose test files use macros that produce invalid surface + Python, this is also a correctness fix.** Coverage.py's report step + (`coverage xml` / `coverage html`) parses each file as standard Python to + map line numbers. If a test uses a macro that rewrites the AST in a way + that yields invalid surface Python (e.g. `nonlocal x` after `x = None`, + which is legal post-`continuations`-macro because the body is split into + separate functions, but rejected by Python's parser as written), + `coverage xml` fails with `Couldn't parse '...' as Python source`. + Excluding tests sidesteps the parse step entirely. **Reason 2 only applies + to projects whose tests use such macros — primarily unpythonic itself, and + any downstream consumer of `unpythonic.syntax.continuations` *in tests*.** + For most projects, Reason 1 is the operative one. + +Canonical pattern documented at `~/.claude/CI-SETUP-NOTES.md` §4a. + +### What + +Add to `pyproject.toml`: + +```toml +[tool.coverage.run] +source = [""] # e.g. "pylu", "raven", "pyan" +omit = [ + "*/tests/*", # OR "*/test/*" — match the project's actual layout +] +``` + +**Path varies by project — there is no fleet convention.** The naming drifted +organically: unpythonic uses `tests/` (plural) because `unpythonic.test` is +reserved for the test framework module (`unpythonic.test.fixtures`); the other +projects' current testsuites were written without an explicit convention being +set, so each picked a locally sensible name. Survey result for the current +in-scope set: pyan uses `tests/` (plural, top-level), raven uses `tests/` +(plural, scattered under `raven//tests/`), mcpyrate uses `test/` +(singular, at `mcpyrate/test/`). Pick the glob that matches the project's +actual layout, and don't add the *other* one as a precaution: in unpythonic, +`*/test/*` would mistakenly omit the framework, which *is* production code. + +The `omit` config applies even when the CI workflow uses `--source=.` from the +command line — config-level omit is composed with whatever source is active. + +### Verify + +Locally, after applying: + +```bash +python -m coverage erase +python -m coverage run --source=. -m +python -m coverage xml +``` + +`coverage xml` should write `coverage.xml` without parse errors. Open the XML +and confirm the test-file paths are absent from `` entries. + +## Sweep B: declare `coverage` / `pytest-cov` in `[dependency-groups].dev` + +### Why + +The fleet's current practice is inconsistent: most projects do `pip install +` in `coverage.yml` ad hoc, which bypasses the `pyproject.toml` +declaration. This: + +- Leaves the local dev env inconsistent with CI (a fresh `pdm install` doesn't + give you the tools CI uses). +- Hides the dependency from `pdm.lock` reproducibility (for app-class projects + that commit `pdm.lock`). +- Violates the rule recently codified in `~/.claude/CLAUDE.md` ("Dev deps go + in `pyproject.toml`, installed via the project's package manager"). + +### What + +For each project that uses `coverage` (or `pytest-cov`) in CI: + +1. Add the tool to `[dependency-groups].dev` in `pyproject.toml`. Use the same + version-pin style as the other dev deps in that project (often unpinned for + leaf tools, sometimes minimum-pinned). +2. Update `.github/workflows/coverage.yml` (and `ci.yml` if relevant) to + install via `pdm install` (which picks up dev deps from + `[dependency-groups].dev`) instead of `pip install `. + +The shared baseline in `~/.claude/PROJECT-SETUP-NOTES.md` was updated this +session to include `coverage` — so when you `pdm install` on a project whose +dev deps match the baseline, you'll get coverage automatically. + +**Projects using pytest-cov** (currently `pyan3`, `raven`, possibly others) +need `pytest-cov` declared in addition to / instead of `coverage` — pick what +the workflow actually uses. + +**Exception — raven:** raven's CI deliberately uses ad-hoc `pip install` lines +*because* a full `pdm install` would pull in the project's torch/torchvision +ML stack (multi-gigabyte, and the workflow already takes pains to use the +CPU-only PyTorch wheel index). For raven, do **only the declaration half** of +Sweep B: add `pytest-cov` to `[dependency-groups].dev` so a fresh local +`pdm install` includes it, but **leave the coverage workflow's ad-hoc install +in place** — just keep `pytest-cov` in the explicit `pip install` list. The +per-project `CLAUDE.md` (or a comment in `coverage.yml`) should note why the +workflow doesn't use `pdm install`. + +### Verify + +Locally: + +```bash +pdm install # installs dev deps from pyproject.toml +which coverage # should resolve into .venv/ +coverage --version # should match what's in pdm.lock (if committed) +``` + +In CI: the `coverage.yml` step that previously did `pip install coverage` +should be removed; only `pdm install` should remain on the dependency-install +side. After pushing, verify the coverage workflow still passes. + +## Per-project survey checklist + +Run this from each project root before changing anything, to surface the +project's current state: + +```bash +echo "=== $(basename $PWD) ===" +echo "--- pyproject.toml: dependency-groups ---" +grep -A 20 '\[dependency-groups\]' pyproject.toml | sed -n '/dev = \[/,/^]/p' +echo "--- pyproject.toml: coverage config ---" +grep -A 10 '\[tool.coverage' pyproject.toml || echo "(none)" +echo "--- coverage.yml: install lines ---" +grep -E 'pip install|pdm install|coverage' .github/workflows/coverage.yml 2>/dev/null || echo "(no coverage.yml)" +echo "--- ci.yml: install lines ---" +grep -E 'pip install|pdm install|coverage' .github/workflows/ci.yml 2>/dev/null | head -10 || echo "(no ci.yml)" +echo "--- test directory name (depth ≤ 3) ---" +find . -maxdepth 3 -type d \( -name tests -o -name test \) \ + -not -path '*/.venv/*' -not -path '*/__pycache__/*' 2>/dev/null | head -10 +``` + +The `find` form (vs the simpler `ls -d */tests */test`) is needed because some +projects scatter test directories deeper — raven has them at +`raven//tests/`, three levels down. A depth-1 glob would miss those +entirely and you'd write the wrong `omit` glob. + +Use the output to decide: + +- If `[tool.coverage.run]` already exists, sweep A may be partial — verify it + has the `omit` clause. +- If `coverage` / `pytest-cov` is already in dev deps, sweep B may be partial — + verify the workflow no longer pip-installs it. +- If both look right, the project is already done; skip it. + +## Order of operations + +For each project, in order: + +1. Run the survey checklist; capture output. +2. Apply sweep A (coverage config in pyproject.toml). +3. Verify coverage runs locally. +4. Apply sweep B (dev deps + workflow update). +5. Verify `pdm install` brings the tool in. +6. Commit each project's changes as a single commit + (`pyproject, .github/workflows: align with fleet coverage convention` or similar). +7. Push, verify CI green. +8. Move to the next project. + +Bundling A+B per project keeps each commit atomic and bisectable. Avoid sweeping +all projects in one mega-commit — if one breaks, the bisect needs per-project +granularity. + +## References + +- `~/.claude/CI-SETUP-NOTES.md` §4a — `[tool.coverage.run]` canonical pattern. +- `~/.claude/PROJECT-SETUP-NOTES.md` — Shared dev dependency baseline (now + includes `coverage`). +- `~/.claude/CLAUDE.md` — "Dev deps go in `pyproject.toml`" rule. +- `unpythonic/pyproject.toml` (commit `5715d88`) — reference implementation + for sweep A. The detailed comment in the `[tool.coverage.run]` section + explains both reasons. +- `unpythonic/briefs/2.2.0-remaining-issues.md` — where this sweep was queued. + +## Out of scope (explicitly) + +- **Migrating from `pip install` to `pdm install` for non-coverage tools in + CI.** Many projects also `pip install` other things (sphinx, etc.). That's + a broader cleanup; only do it if it falls naturally out of touching the + workflow file anyway. If it expands the scope, defer to a separate session. +- **Pinning coverage versions.** The baseline doesn't pin and neither should + we, unless a project has a specific version constraint. Lockfile-committing + app-class projects will pin via `pdm.lock` automatically. +- **substrate-independent and arxiv-api-search.** Verify they're in scope + (have pyproject.toml, have CI) before touching. diff --git a/briefs/modernization-phase1-audit-brief.md b/briefs/modernization-phase1-audit-brief.md new file mode 100644 index 00000000..5b6b581c --- /dev/null +++ b/briefs/modernization-phase1-audit-brief.md @@ -0,0 +1,124 @@ +# CC Brief: unpythonic Modernization — Phase 1 (Audit) + +## Context + +unpythonic is being updated from Python 3.8–3.12 to 3.10–3.14. This follows the mcpyrate 4.0.0 update — unpythonic is mcpyrate's primary downstream consumer. Version will be 2.0.0 (floor bump + mcpyrate 4.0.0 dependency is breaking). + +unpythonic has three tiers: pure Python layer (`unpythonic/`), macro layer (`unpythonic/syntax/`), and dialect layer (`unpythonic/dialects/`). The macro and dialect layers depend on mcpyrate. The pure Python layer has no mcpyrate dependency at runtime. + +No code changes in this phase, only a report. + +## Reference + +- unpythonic CLAUDE.md (in repo root) — architecture, conventions. +- unpythonic issue #93: consolidated AST change notes (covers mcpyrate, unpythonic, Pyan3). +- mcpyrate 4.0.0 changelog: removed `getconstant()`, `Num`, `Str`, `Bytes`, `NameConstant`, `Ellipsis`, `Index`, `ExtSlice` from `astcompat` public API. +- mcpyrate 4.0.0 source tree: `~/Documents/koodit/mcpyrate/` — consult when you need to check what `astcompat` exports, how the unparser handles new nodes, or any other mcpyrate 4.0.0 API details. + +## What to audit + +### 1. Imports from mcpyrate.astcompat (mcpyrate 4.0.0 breakage) + +mcpyrate 4.0.0 removed these from `astcompat`: `getconstant`, `Num`, `Str`, `Bytes`, `NameConstant`, `Ellipsis`, `Index`, `ExtSlice`. + +**Find all imports from `mcpyrate.astcompat`** and flag any that reference removed names. + +**Known instances** (verify — there may be more): +- `syntax/lambdatools.py`: imports `getconstant`, `Str`, `NamedExpr` +- `syntax/letdoutil.py`: imports `getconstant`, `Str`, `NamedExpr` +- `syntax/tailtools.py`: imports `getconstant`, `NameConstant`, `TryStar` +- `syntax/autoref.py`: imports `getconstant` +- `syntax/util.py`: imports `getconstant` +- `syntax/autocurry.py`: imports `TypeAlias` (still valid) +- `syntax/lazify.py`: imports `TypeAlias` (still valid) +- `syntax/scopeanalyzer.py`: imports `TryStar`, `MatchStar`, `MatchMapping`, `MatchClass`, `MatchAs` (still valid) +- `syntax/tests/test_letdoutil.py`: imports `getconstant`, `Num` +- `syntax/tests/test_util.py`: imports `getconstant`, `Num`, `Str` + +For each removed import, find all usage sites in that file. Most will be type checks like `type(k) in (Constant, Str)` that collapse to `type(k) is Constant`, and `getconstant(node)` calls that become `node.value`. + +### 2. `hasattr` checks on AST node fields (3.13) + +In Python 3.13, omitted optional fields on AST nodes are set to `None` instead of being absent. Code that uses `hasattr(node, "field")` to detect absence will now always return `True`, breaking guards that relied on absence to detect "not set". + +**Known instances** (scan for more): + +**Dialect files:** +- `dialects/listhell.py` (~line 29): `if hasattr(self, "lineno"):` +- `dialects/lispython.py` (~lines 45, 82): `if hasattr(self, "lineno"):` +- `dialects/pytkell.py` (~line 45): `if hasattr(self, "lineno"):` + +Note: these check `hasattr(self, "lineno")` on dialect classes, not directly on AST nodes. The comment says "mcpyrate 3.6.0+". Check whether `self` here is an AST node or a dialect instance — if it's a dialect instance, the 3.13 AST field change doesn't apply. + +**Macro layer:** +- `syntax/testingtools.py` (~lines 803, 904, 941, 1013): `hasattr(tree, "lineno")` / `hasattr(first_stmt, "lineno")` +- `syntax/dbg.py` (~lines 229, 240): `hasattr(tree, "lineno")` +- `syntax/lambdatools.py` (~line 403): `if hasattr(tree, "lineno"):` +- `syntax/lambdatools.py` (~line 539): `tree.ctx if hasattr(tree, "ctx") else None` +- `syntax/scopeanalyzer.py` (~lines 389, 411): `hasattr(tree, "ctx") and type(tree.ctx) is Store/Del` +- `syntax/letdoutil.py` (~line 763): `hasattr(oldb, "lineno") and hasattr(oldb, "col_offset")` +- `syntax/letdo.py` (~line 478): `hasattr(tree, "ctx")` + +**Tests:** +- `syntax/tests/test_conts_multishot.py` (~line 68): `hasattr(tree, "ctx")` + +For `ctx` checks: these are likely checking whether a macro-generated node has had `ctx` set. In 3.13, `ctx` defaults to `Load()` on omission, so `hasattr` will always be `True` — but the node *does* have a meaningful `ctx` now (`Load()`). Determine whether this is a behavior change or harmless. + +### 3. `sys.version_info` guards (floor bump cleanup) + +With floor at 3.10, all `>= (3, 8)` and `>= (3, 9)` checks are always true. Find all and list. + +**Known instances** (~14+ sites, mostly `ast.Index` wrapper removal): +- `syntax/testingtools.py` (~line 883): `>= (3, 9, 0)` — `ast.Index` wrapper +- `syntax/letsyntax.py` (~line 372): `>= (3, 9, 0)` — `ast.Index` wrapper +- `syntax/prefix.py` (~line 197): `>= (3, 9, 0)` — `ast.Index` wrapper +- `syntax/letdoutil.py` (~lines 25, 30): `>= (3, 9, 0)` — `ast.Index` wrapper +- `syntax/nameutil.py` (~line 127): `>= (3, 9, 0)` — `ast.Index` wrapper +- `syntax/letdo.py` (~line 595): `>= (3, 8, 0)` — positional-only args +- `syntax/tailtools.py` (~line 1118): `>= (3, 8, 0)` — positional-only args +- `syntax/tests/test_conts_multishot.py` (~line 194): `>= (3, 9, 0)` — `ast.Index` +- `syntax/tests/test_letdoutil.py` (~lines 611, 624, 631, 650, 663, 670): `>= (3, 9, 0)` — `ast.Index` +- `typecheck.py` (~line 187): `>= (3, 10, 0)` — `types.UnionType`. Always true with floor at 3.10. +- `tests/test_fun.py` (~line 259): `< (3, 11, 0)` — check what this guards + +### 4. Direct references to deprecated/removed AST node types + +Outside of `mcpyrate.astcompat` imports, check for any direct `ast.Num`, `ast.Str`, etc. references. + +**Known instance:** +- `syntax/letdoutil.py` (~line 732): error message mentions `ast.Str` — just a string literal, but should be updated for accuracy. + +### 5. `autoreturn` and `match`/`case` (feature gap from issue #93) + +`autoreturn` in `syntax/tailtools.py` doesn't handle `match`/`case` statements. This is a known gap — it's a feature addition, not strictly a compat fix, but it's the most significant modernization issue identified in issue #93. + +**Scope the work:** check how `autoreturn` handles other compound statements (`if`/`elif`/`else`, `try`/`except`, `with`). The `match`/`case` handler should follow the same pattern — autoreturn the last expression in each `case` body. + +**Decision:** Include in 2.0.0. The 3.10 floor means `match`/`case` is always available, and the version bump is happening anyway. This is self-contained relative to the rest of the `autoreturn` machinery — the scary parts of unpythonic (TCO, lazify, autocurry, continuations) are not involved. + +### 6. AST constructor calls (3.13 strictness) + +In 3.13, omitting required fields or passing unknown kwargs on `ast.*` node constructors emits `DeprecationWarning` (becomes an error in 3.15). Scan for AST node constructor calls that omit required fields or pass unknown kwargs. + +Focus on the macro layer (`syntax/*.py`) which constructs AST nodes extensively. The pure Python layer doesn't touch AST. + +**Exception — `ctx` fields**: Many AST node constructors intentionally omit `ctx` — mcpyrate's `astfixers.fix_ctx()` auto-injects the correct `ctx` after macro expansion. In 3.13, omitted `ctx` defaults to `Load()`, which is harmless since `astfixers` overwrites it. Don't flag missing `ctx`. + +### 7. Version metadata + +- `pyproject.toml`: `python_requires`, classifiers, mcpyrate dependency version +- CI workflow: matrix versions, PyPy versions +- `CLAUDE.md`: version range mentions +- `README.md`: any version range mentions +- `CHANGELOG.md`: will need a 2.0.0 entry (not part of audit, just note) +- Module docstrings mentioning version ranges + +## Deliverable + +A report (markdown) listing all sites that need attention, grouped by file. For each site, note: +- File and line number +- What the issue is +- Category: mcpyrate 4.0.0 breakage / floor bump cleanup / 3.13 compat / 3.14 compat / feature gap +- Severity (will break / will warn / cleanup only) + +No code changes. diff --git a/briefs/modernization-phase1-audit-report.md b/briefs/modernization-phase1-audit-report.md new file mode 100644 index 00000000..60bcba4c --- /dev/null +++ b/briefs/modernization-phase1-audit-report.md @@ -0,0 +1,277 @@ +# unpythonic Phase 1 Audit Report — Python 3.10–3.14 Modernization + +**Date:** 2026-03-11 +**Scope:** Audit for Python 3.10–3.14 support (version 2.0.0), mcpyrate 4.0.0 dependency + +## Summary + +| Category | Count | Will break | Will warn | Cleanup only | +|----------|------:|:----------:|:---------:|:------------:| +| mcpyrate 4.0.0 breakage | 21 | 21 | — | — | +| 3.13 compat (`hasattr`) | 8 | 2 | — | 6 | +| 3.13 compat (AST constructors) | 2 | — | 2 | — | +| Floor bump cleanup (`sys.version_info`) | 18 | — | — | 18 | +| Feature gap (`autoreturn` + match/case) | 1 | — | — | 1 | +| Direct AST ref in string | 1 | — | — | 1 | +| Version metadata | 15+ | — | — | 15+ | +| TODOs now actionable | 20+ | — | — | 20+ | + +--- + +## 1. `unpythonic/syntax/lambdatools.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 17 | Imports `getconstant`, `Str` from `mcpyrate.astcompat` — both removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 371 | `type(k) in (Constant, Str)` — collapse to `type(k) is Constant` | mcpyrate 4.0.0 breakage | **will break** | +| 372 | `getconstant(k)` — replace with `k.value` | mcpyrate 4.0.0 breakage | **will break** | +| 403 | `if hasattr(tree, "lineno"):` on Lambda AST node — always True in 3.13; creates name like `""` for macro-generated lambdas | 3.13 compat | cleanup (cosmetic) | +| 454 | `if hasattr(a, "posonlyargs"):` — version check for 3.8+ `arguments` field; always True at floor 3.10 but harmless | floor bump cleanup | cleanup only | +| 539 | `tree.ctx if hasattr(tree, "ctx") else None` — safe by accident in 3.13 (macro nodes get `ctx=None`, result is the same) | 3.13 compat | cleanup (safe) | + +## 2. `unpythonic/syntax/letdoutil.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 14 | Imports `getconstant`, `Str` from `mcpyrate.astcompat` — both removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 25 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper removal; always true | floor bump cleanup | cleanup only | +| 30 | `if sys.version_info >= (3, 9, 0):` — same, setter variant | floor bump cleanup | cleanup only | +| 186 | `type(mode[0]) in (Constant, Str)` — collapse to `type(mode[0]) is Constant` | mcpyrate 4.0.0 breakage | **will break** | +| 187 | `getconstant(mode[0])` — replace with `mode[0].value` | mcpyrate 4.0.0 breakage | **will break** | +| 731 | `type(newk) not in (Constant, Str)` — collapse to `type(newk) is not Constant` | mcpyrate 4.0.0 breakage | **will break** | +| 732 | Error message string mentions `ast.Str` — update for accuracy | direct AST ref | cleanup only | +| 744 | `getconstant(newk)` — replace with `newk.value` | mcpyrate 4.0.0 breakage | **will break** | +| 763 | `if hasattr(oldb, "lineno") and hasattr(oldb, "col_offset"):` — always True in 3.13; passes `None` values to `Tuple()` constructor instead of letting mcpyrate fix them | 3.13 compat | **will break** | + +## 3. `unpythonic/syntax/tailtools.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 25 | Imports `getconstant`, `NameConstant` from `mcpyrate.astcompat` — both removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 685–716 | `autoreturn`'s `TailStatementTransformer` does not handle `ast.Match` (match/case). With floor at 3.10, match/case is always available. | feature gap | cleanup only | +| 1038 | `type(theexpr) in (Constant, NameConstant) and getconstant(theexpr) is None` — collapse to `type(theexpr) is Constant and theexpr.value is None` | mcpyrate 4.0.0 breakage | **will break** | +| 1043 | Same pattern as 1038 but on `tree` | mcpyrate 4.0.0 breakage | **will break** | +| 1112–1119 | `arguments()` constructor omits `posonlyargs`; adds it conditionally after. In 3.13, omitting required fields emits DeprecationWarning (error in 3.15) | 3.13 compat (constructors) | **will warn** | +| 1118 | `if sys.version_info >= (3, 8, 0):` — always true at floor 3.10 | floor bump cleanup | cleanup only | + +## 4. `unpythonic/syntax/autoref.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 12 | Imports `getconstant` from `mcpyrate.astcompat` — removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 237 | `getconstant(get_resolver_list(tree)[-1])` — replace with `.value` | mcpyrate 4.0.0 breakage | **will break** | + +## 5. `unpythonic/syntax/util.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 21 | Imports `getconstant` from `mcpyrate.astcompat` — removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 358 | `getconstant(tree.test)` inside try/except — replace with `tree.test.value` (guard `type(tree.test) is Constant` first) | mcpyrate 4.0.0 breakage | **will break** | + +## 6. `unpythonic/syntax/letdo.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 478 | `hasctx = hasattr(tree, "ctx")` — in 3.13, always True; macro-created nodes get `ctx=None`, so `type(None) is not Load` → True → incorrect early return. **Breaks let-binding envify.** | 3.13 compat | **will break** | +| 593–596 | `arguments()` constructor omits `posonlyargs`; adds it conditionally. DeprecationWarning in 3.13, error in 3.15 | 3.13 compat (constructors) | **will warn** | +| 595 | `if sys.version_info >= (3, 8, 0):` — always true at floor 3.10 | floor bump cleanup | cleanup only | + +## 7. `unpythonic/syntax/scopeanalyzer.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 389 | `hasattr(tree, "ctx") and type(tree.ctx) is Store` — hasattr always True in 3.13; works by accident (`type(None) is Store` → False) but fragile | 3.13 compat | cleanup (fragile) | +| 411 | `hasattr(tree, "ctx") and type(tree.ctx) is Del` — same as above | 3.13 compat | cleanup (fragile) | +| 428 | `if hasattr(a, "posonlyargs"):` — always True at floor 3.10 | floor bump cleanup | cleanup only | + +## 8. `unpythonic/syntax/testingtools.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 803 | `q[u[tree.lineno]] if hasattr(tree, "lineno") else q[None]` — safe in 3.13 (gets `None` either way) | 3.13 compat | cleanup (safe) | +| 883 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | +| 904 | Same safe `hasattr` lineno pattern as 803 | 3.13 compat | cleanup (safe) | +| 941 | Same safe `hasattr` lineno pattern on `first_stmt` | 3.13 compat | cleanup (safe) | +| 1013 | Same safe `hasattr` lineno pattern on `first_stmt` | 3.13 compat | cleanup (safe) | + +## 9. `unpythonic/syntax/dbg.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 229 | `tree.lineno if hasattr(tree, "lineno") else None` — safe in 3.13 | 3.13 compat | cleanup (safe) | +| 240 | `q[u[tree.lineno]] if hasattr(tree, "lineno") else q[None]` — safe in 3.13 | 3.13 compat | cleanup (safe) | + +## 10. `unpythonic/syntax/letsyntax.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 372 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | + +## 11. `unpythonic/syntax/prefix.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 197 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | + +## 12. `unpythonic/syntax/nameutil.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 127 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | + +## 13. `unpythonic/syntax/tests/test_letdoutil.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 7 | Imports `getconstant`, `Num` from `mcpyrate.astcompat` — both removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 253 | `type(the[view.value]) in (Constant, Num) and getconstant(view.value) == 42` — collapse type check, use `.value` | mcpyrate 4.0.0 breakage | **will break** | +| 259 | Same pattern, value 23 | mcpyrate 4.0.0 breakage | **will break** | +| 277 | Same pattern, value 42 | mcpyrate 4.0.0 breakage | **will break** | +| 283 | Same pattern, value 23 | mcpyrate 4.0.0 breakage | **will break** | +| 528 | Same pattern, variable value | mcpyrate 4.0.0 breakage | **will break** | +| 611, 624, 631, 650, 663, 670 | Six `if sys.version_info >= (3, 9, 0):` guards — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | + +## 14. `unpythonic/syntax/tests/test_util.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 7 | Imports `getconstant`, `Num`, `Str` from `mcpyrate.astcompat` — all removed in 4.0.0 | mcpyrate 4.0.0 breakage | **will break** | +| 159 | `type(lam.body) in (Constant, Num)` — collapse to `type(lam.body) is Constant` | mcpyrate 4.0.0 breakage | **will break** | +| 160 | `getconstant(lam.body) == 42` — replace with `lam.body.value == 42` | mcpyrate 4.0.0 breakage | **will break** | +| 188 | `type(tree.value) in (Constant, Str)` — collapse | mcpyrate 4.0.0 breakage | **will break** | +| 189 | `getconstant(tree.value)` — replace with `tree.value.value` | mcpyrate 4.0.0 breakage | **will break** | +| 196 | `type(tree.value) in (Constant, Str) and getconstant(tree.value) == "hello"` — collapse and use `.value` | mcpyrate 4.0.0 breakage | **will break** | + +## 15. `unpythonic/syntax/tests/test_conts_multishot.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 68 | `if hasattr(tree, "ctx") and type(tree.ctx) is not ast.Load:` — always True in 3.13; macro-created nodes get `ctx=None`, `type(None) is not Load` → True → incorrect early return | 3.13 compat | **will break** | +| 194 | `if sys.version_info >= (3, 9, 0):` — `ast.Index` wrapper; always true | floor bump cleanup | cleanup only | + +## 16. `unpythonic/misc.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 109 | `if version_info >= (3, 8, 0):` — always true at floor 3.10; else branch uses fragile `CodeType()` positional construction | floor bump cleanup | cleanup only | + +## 17. `unpythonic/typecheck.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 187 | `if sys.version_info >= (3, 10, 0):` — always true at floor 3.10; `isinstance(T, typing.NewType)` is the only path needed | floor bump cleanup | cleanup only | + +## 18. `unpythonic/tests/test_fun.py` + +| Line | Issue | Category | Severity | +|-----:|-------|----------|----------| +| 259 | `if sys.version_info < (3, 11, 0):` — always false at floor 3.10; entire block is dead code (uninspectable builtins test) | floor bump cleanup | cleanup only | + +## 19. Dialect files (NOT affected by 3.13 AST change) + +`lispython.py:45,82`, `listhell.py:29`, `pytkell.py:45` — all check `hasattr(self, "lineno")` on **Dialect instances**, not AST nodes. The mcpyrate 3.6.0+ compat check is unrelated to the 3.13 AST field change. **No action needed.** + +--- + +## 20. `autoreturn` and `match`/`case` — Feature Gap Detail + +`TailStatementTransformer` in `tailtools.py:685–716` handles: +- `If` → recurse into both branches +- `With`/`AsyncWith` → recurse into body +- `Try`/`TryStar` → recurse into else (or body if no else) + each except handler; skip finally +- `FunctionDef`/`AsyncFunctionDef`/`ClassDef` → append `return ` +- `Expr` → convert to `return expr` + +**Missing:** `Match` (Python 3.10+ structural pattern matching). The handler would follow the same pattern — recurse into `case.body[-1]` for each `match_case`. Approximately 3–5 lines of code plus an `ast.Match` import. `scopeanalyzer.py` already handles match/case for scope analysis, so the infrastructure is in place. + +There is also a TODO for `For`/`AsyncFor`/`While` at line 687, which is explicitly documented as intentionally unhandled (loops don't have a natural tail-position value). + +--- + +## 21. Version Metadata + +### Packaging + +| File | Line | Current | Target | +|------|-----:|---------|--------| +| `pyproject.toml` | 7 | `requires-python = ">=3.8,<3.13"` | `">=3.10,<3.15"` | +| `pyproject.toml` | 22 | `"mcpyrate>=3.6.4"` | `"mcpyrate>=4.0.0"` | +| `pyproject.toml` | 29–40 | Classifiers for 3.8–3.12 | Remove 3.8, 3.9; add 3.13, 3.14 | +| `.pdm-build/pyproject.toml` | 7 | Same as above | Sync with main | +| `.pdm-build/pyproject.toml` | 38–42 | Same classifiers | Sync with main | + +### CI + +| File | Line | Current | Target | +|------|-----:|---------|--------| +| `.github/workflows/python-package.yml` | 20 | `["3.8", "3.9", "3.10", "3.11", "3.12", pypy-3.8, pypy-3.9, pypy-3.10]` | `["3.10", "3.11", "3.12", "3.13", "3.14", pypy-3.10]` | +| `.github/workflows/coverage.yml` | 18 | `["3.10"]` | Consider updating to `["3.12"]` or `["3.13"]` | + +### Documentation + +| File | Line(s) | What to update | +|------|---------|----------------| +| `CLAUDE.md` | 15, 19 | Version range mentions ("3.8–3.12", "1.1.0" plan → actual 2.0.0 state) | +| `README.md` | 20 | "CPython 3.8, 3.9 and 3.10, 3.11, 3.12, and PyPy3 (language versions 3.8, 3.9, 3.10)" | +| `CONTRIBUTING.md` | 121 | "main target platforms are CPython 3.8 and PyPy3 3.7" (very outdated) | +| `CHANGELOG.md` | — | Will need a 2.0.0 entry | + +### Newly actionable TODOs (selected high-value items) + +| File | Line | TODO | +|------|-----:|------| +| `syntax/letdoutil.py` | 200, 217, 500, 529 | "Python 3.9+: remove once we bump minimum Python to 3.9" — remove parens syntax for macro args | +| `syntax/letsyntax.py` | 331, 392, 448 | Same — remove parens syntax support | +| `syntax/letdo.py` | 931 | "Remove the parens when we bump minimum Python to 3.10" — walrus in subscripts | +| `syntax/tests/test_letdo.py` | 5, 31 | Switch macro args to brackets; remove parens | +| `syntax/tests/test_letdoutil.py` | 42 | Remove the parens | +| `syntax/tests/test_scopeanalyzer.py` | 17, 18 | "Add tests for match/case once we bump to 3.10" / "Add tests for try/except* once we bump to 3.11" — **both now actionable** | +| `syntax/__init__.py` | 84 | "Change decorator macro invocations to use [] instead of ()" — now actionable at floor 3.10 | + +--- + +## Migration patterns + +### `getconstant(node)` → `node.value` + +```python +# Before +from mcpyrate.astcompat import getconstant +value = getconstant(tree.test) + +# After +# Just access .value on the Constant node directly +value = tree.test.value +``` + +Where `getconstant` was called inside a try/except (e.g., `util.py:358`), guard with `type(node) is Constant` first. + +### `type(x) in (Constant, Str/Num/NameConstant)` → `type(x) is Constant` + +All legacy node types (`Str`, `Num`, `Bytes`, `NameConstant`, `Ellipsis`) have been unified into `ast.Constant` since Python 3.8. With floor at 3.10, only `Constant` exists. + +### `hasattr(node, "field")` → `node.field is not None` + +For optional AST fields that may be `None` on macro-generated nodes in 3.13: + +```python +# Before +if hasattr(tree, "ctx"): + ...use tree.ctx... + +# After +if tree.ctx is not None: + ...use tree.ctx... +``` + +### `arguments()` constructor — add `posonlyargs=[]` + +```python +# Before +noargs = arguments(args=[], kwonlyargs=[], vararg=None, kwarg=None, + defaults=[], kw_defaults=[]) + +# After +noargs = arguments(args=[], kwonlyargs=[], vararg=None, kwarg=None, + defaults=[], kw_defaults=[], posonlyargs=[]) +``` diff --git a/briefs/modernization-phase2-5-implementation.md b/briefs/modernization-phase2-5-implementation.md new file mode 100644 index 00000000..33f7f7cd --- /dev/null +++ b/briefs/modernization-phase2-5-implementation.md @@ -0,0 +1,352 @@ +# CC Brief: unpythonic Modernization — Phases 2–5 (Implementation) + +**Prerequisite**: Phase 1 audit report reviewed and approved. This brief incorporates its findings. + +## Goal + +Update unpythonic from Python 3.8–3.12 to 3.10–3.14. This is a **major version bump to 2.0.0** — floor bump + mcpyrate 4.0.0 dependency is breaking. + +Four phases, each a separate commit (or small group of commits). Don't mix cleanup with compat work — we need clean bisect boundaries. + +unpythonic has three tiers: pure Python layer, macro layer (`syntax/`), and dialect layer (`dialects/`). The macro layer is where almost all the work is. The pure Python layer has a few version guards to clean up. The dialect layer is clean (confirmed by audit). + +## Reference + +- Phase 1 audit report (attached/in context). +- unpythonic CLAUDE.md (in repo root) — architecture, conventions. +- unpythonic issue #93: consolidated AST change notes. +- mcpyrate 4.0.0 source tree: `~/Documents/koodit/mcpyrate/` — consult for current `astcompat` exports, API details. + +--- + +## Phase 2: Floor bump to 3.10 + +Drop support for Python 3.8 and 3.9. Remove dead code paths and version guards. This is mechanical cleanup. + +### `sys.version_info` guards — remove dead branches (18 sites) + +All `>= (3, 8)` and `>= (3, 9)` guards are always true at floor 3.10. Delete the `else` branches and the conditionals, keeping only the true branch. + +**`ast.Index` wrapper guards** (always true, `>= (3, 9)`): +- `syntax/letdoutil.py` lines ~25, ~30 +- `syntax/letsyntax.py` line ~372 +- `syntax/prefix.py` line ~197 +- `syntax/testingtools.py` line ~883 +- `syntax/nameutil.py` line ~127 +- `syntax/tests/test_letdoutil.py` lines ~611, ~624, ~631, ~650, ~663, ~670 +- `syntax/tests/test_conts_multishot.py` line ~194 + +**Positional-only args guards** (always true, `>= (3, 8)`): +- `syntax/tailtools.py` line ~1118 +- `syntax/letdo.py` line ~595 + +**Other version guards:** +- `typecheck.py` line ~187: `>= (3, 10)` — `types.UnionType`. Always true, remove guard. +- `misc.py` line ~109: `>= (3, 8)` — `CodeType()` construction. Remove else branch with fragile positional construction. +- `tests/test_fun.py` line ~259: `< (3, 11)` — still needed (runs on 3.10 only). Leave as-is. + +**`hasattr` version guards** (always true with floor 3.10, not 3.13-related): +- `syntax/lambdatools.py` line ~454: `hasattr(a, "posonlyargs")` — always true since 3.8. Remove guard. +- `syntax/scopeanalyzer.py` line ~428: `hasattr(a, "posonlyargs")` — same. + +### Newly actionable TODOs — macro arg brackets + +With the floor at 3.10+, the bracket syntax (`macro[args]`) for macro arguments is always available. The codebase has TODOs saying "remove parens syntax once we bump minimum Python." However, the parens branches are small (2–4 lines each) and keeping backward compat doesn't hurt. **Don't remove parens support.** Instead: + +- **Verify** that the bracket syntax alternative works everywhere parens syntax is accepted. If it doesn't, that's a bug — fix it now. +- **Update tests** to prefer bracket syntax as the modern idiom. Keep at least one test per macro that exercises the parens syntax code path, with a comment like `# Test deprecated parens syntax (backward compat)`. +- **Update TODO comments** to say: "Parens syntax deprecated; kept for backward compatibility." + +Sites with these TODOs: +- `syntax/letdoutil.py` lines ~200, ~217, ~500, ~529 +- `syntax/letsyntax.py` lines ~331, ~392, ~448 +- `syntax/letdo.py` line ~931: walrus in subscripts (requires 3.10) +- `syntax/__init__.py` line ~84: decorator macro invocations +- `syntax/tests/test_letdo.py` lines ~5, ~31 +- `syntax/tests/test_letdoutil.py` line ~42 + +### Version metadata + +**pyproject.toml:** +- Bump version to `2.0.0` +- `requires-python`: `">=3.8,<3.13"` → `">=3.10,<3.15"` +- `mcpyrate` dependency: `"mcpyrate>=3.6.4"` → `"mcpyrate>=4.0.0"` +- Classifiers: remove 3.8, 3.9; add 3.13, 3.14 + +**.pdm-build/pyproject.toml:** Sync all changes from main `pyproject.toml`. + +**CI (.github/workflows/python-package.yml):** +- Matrix: `["3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.11"]` (remove 3.8, 3.9, pypy-3.8, pypy-3.9, pypy-3.10; add 3.13, 3.14, pypy-3.11) + +**CI (.github/workflows/coverage.yml):** +- Update from `["3.10"]` to `["3.12"]`. + +**Documentation:** +- `CLAUDE.md` lines ~15, ~19: update version range and release plan +- `README.md` line ~20: update supported versions +- `CONTRIBUTING.md` line ~121: "main target platforms are CPython 3.8 and PyPy3 3.7" — very outdated, update + +--- + +## Phase 3: mcpyrate 4.0.0 adaptation + +Replace all usage of APIs removed in mcpyrate 4.0.0. This is mechanical — the patterns are uniform. + +### `getconstant()` → `.value` (10+ call sites) + +`getconstant(node)` becomes `node.value`. Where `getconstant` was called inside a try/except or guard, use `type(node) is Constant` first. + +**Sites:** +- `syntax/lambdatools.py` line ~372: `getconstant(k)` → `k.value` +- `syntax/letdoutil.py` lines ~187, ~744: `getconstant(mode[0])` → `mode[0].value`, `getconstant(newk)` → `newk.value` +- `syntax/tailtools.py` lines ~1038, ~1043: `getconstant(theexpr)` → `theexpr.value` +- `syntax/autoref.py` line ~237: `getconstant(...)` → `....value` +- `syntax/util.py` line ~358: `getconstant(tree.test)` → `tree.test.value` (guard with `type(tree.test) is Constant` first — currently in a try/except) +- `syntax/tests/test_letdoutil.py` lines ~253, ~259, ~277, ~283, ~528: `getconstant(view.value)` → use intermediate variable, **never `.value.value`**: +```python +node = view.value # the AST Constant node +test[node.value == 42] # the Python value inside it +``` +- `syntax/tests/test_util.py` lines ~160, ~189, ~196: same pattern — use intermediate variable, **never `.value.value`** + +### Removed type imports — collapse type checks (11 sites) + +`Str`, `Num`, `NameConstant` are removed from `mcpyrate.astcompat`. All type checks like `type(x) in (Constant, Str)` collapse to `type(x) is Constant`. + +**Update imports** — remove `Str`, `Num`, `NameConstant`, `getconstant` from all `from mcpyrate.astcompat import ...` lines: +- `syntax/lambdatools.py` line ~17: remove `getconstant`, `Str` (keep `NamedExpr`) +- `syntax/letdoutil.py` line ~14: remove `getconstant`, `Str` (keep `NamedExpr`) +- `syntax/tailtools.py` line ~25: remove `getconstant`, `NameConstant` (keep `TryStar`) +- `syntax/autoref.py` line ~12: remove entire `from mcpyrate.astcompat` import +- `syntax/util.py` line ~21: remove entire `from mcpyrate.astcompat` import +- `syntax/tests/test_letdoutil.py` line ~7: remove `getconstant`, `Num` +- `syntax/tests/test_util.py` line ~7: remove `getconstant`, `Num`, `Str` + +**Collapse type checks:** +- `syntax/lambdatools.py` line ~371: `type(k) in (Constant, Str)` → `type(k) is Constant` +- `syntax/letdoutil.py` line ~186: `type(mode[0]) in (Constant, Str)` → `type(mode[0]) is Constant` +- `syntax/letdoutil.py` line ~731: `type(newk) not in (Constant, Str)` → `type(newk) is not Constant` +- `syntax/tailtools.py` line ~1038: `type(theexpr) in (Constant, NameConstant)` → `type(theexpr) is Constant` +- `syntax/tailtools.py` line ~1043: same pattern +- `syntax/tests/test_letdoutil.py` lines ~253, ~259, ~277, ~283, ~528: `type(...) in (Constant, Num)` → `type(...) is Constant` +- `syntax/tests/test_util.py` lines ~159, ~188, ~196: collapse `Num`/`Str` branches + +**Update error message:** +- `syntax/letdoutil.py` line ~732: error string mentions `ast.Str` — update to reference only `ast.Constant` + +--- + +## Phase 4: Python 3.13 compatibility + +### `hasattr` fixes — CRITICAL (2 sites that will break + 1 in tests) + +**`syntax/letdo.py` line ~478** — **Breaks let-binding envify:** +```python +# Before (broken on 3.13): +hasctx = hasattr(tree, "ctx") +# ...later... +if hasctx and type(tree.ctx) is not Load: + return tree # early return + +# In 3.13: hasctx=True, ctx=Load() (default), type(Load()) is not Load → False. +# This happens to be correct by accident, BUT the intent is to skip Store/Del. +# Make the intent explicit: + +# After — check for what we actually care about: +if type(getattr(tree, "ctx", None)) in (Store, Del): + return tree # skip assignments and deletes +``` +Also update the `ctx` copy at line ~482 (`if hasctx: attr_node.ctx = tree.ctx`). Since the `hasctx` variable is gone, just copy unconditionally: `attr_node.ctx = getattr(tree, "ctx", None)`. On 3.13+ this is always `Load()` at this point (Store/Del already returned); on pre-3.13 it may be `None` (mcpyrate's astfixers will fix it later). + +**`syntax/letdoutil.py` line ~763** — passes `None` location values to constructor: +```python +# Before: +if hasattr(oldb, "lineno") and hasattr(oldb, "col_offset"): + +# In 3.13: always True, but lineno/col_offset may be None +``` +Fix: `if getattr(oldb, "lineno", None) is not None and getattr(oldb, "col_offset", None) is not None:` + +**`syntax/tests/test_conts_multishot.py` line ~68** — same `ctx` pattern: +```python +# Before: +if hasattr(tree, "ctx") and type(tree.ctx) is not ast.Load: + +# After — check for what we actually care about: +if type(getattr(tree, "ctx", None)) in (ast.Store, ast.Del): +``` + +### `hasattr` fixes — cleanup (6 sites, accidentally correct) + +These are safe in 3.13 by coincidence but should be fixed for consistency and clarity: + +**`syntax/scopeanalyzer.py`** lines ~389, ~411: `hasattr(tree, "ctx") and type(tree.ctx) is Store/Del` — works accidentally on 3.13 (`type(Load()) is Store` → False). Simplify to check directly: `type(getattr(tree, "ctx", None)) is Store` / `type(getattr(tree, "ctx", None)) is Del`. + +**`syntax/lambdatools.py`** line ~539: `tree.ctx if hasattr(tree, "ctx") else None` — safe. Simplify to `getattr(tree, "ctx", None)`. + +**`syntax/testingtools.py`** lines ~803, ~904, ~941, ~1013: `hasattr(tree, "lineno")` — safe (gets `None` either way). Simplify to `getattr(tree, "lineno", None)`. + +**`syntax/dbg.py`** lines ~229, ~240: same safe lineno pattern. Simplify. + +### `arguments()` constructor — include `posonlyargs` directly (2 sites) + +In 3.13, omitting required fields emits DeprecationWarning (error in 3.15). Two sites construct `arguments()` and add `posonlyargs` conditionally afterward. Since the floor is 3.10, just include it in the constructor: + +- `syntax/tailtools.py` lines ~1112–1119: add `posonlyargs=[]` to `arguments()` call, remove conditional. +- `syntax/letdo.py` lines ~593–596: same. + +### Document `ctx` design constraint + +In Python 3.13, AST nodes that omit `ctx` get `Load()` by default (previously the field was absent). This changes the failure mode for macros that create `Name` nodes without setting `ctx`: +- Pre-3.13: no `ctx` → invisible to code that checks `hasattr(tree, "ctx")` +- 3.13+: `ctx=Load()` → silently treated as a Load context node + +The existing contract is: if you want `Store` or `Del` semantics, you **must** set `ctx` explicitly. `astfixers.fix_ctx()` handles this in the postprocessing pass, but any macro code that inspects `ctx` *during* expansion (before astfixers runs) relies on this contract. + +**Document this in both projects:** + +**mcpyrate** — in `doc/main.md` or wherever macro authoring best practices are documented: +- When constructing `Name`, `Starred`, `Subscript`, or `Attribute` nodes in a macro for an AST slot that expects `Store` or `Del` context (e.g. assignment targets, `del` targets, `for` loop variables, `with ... as` targets), you **must** set `ctx` explicitly. If you don't, Python 3.13+ will populate it with `Load()`, which is *incorrect* for that position. `astfixers.fix_ctx()` will unconditionally overwrite `ctx` based on tree position in the postprocessing pass, but it runs *after* all macros have expanded — so any macro code that inspects `ctx` during expansion will see the wrong value. +- On Python 3.12 and earlier, omitted `ctx` resulted in an absent attribute. On 3.13+, it results in `Load()`. Neither is correct for a Store/Del slot, but the failure mode is different: old behavior was "invisible to ctx checks", new behavior is "silently classified as Load, and the resulting AST will likely fail to compile." + +**unpythonic** — in `doc/macros.md` or the macro authoring section: +- Same guidance, with specific reference to `scopeanalyzer` and `letdo` envify as code that inspects `ctx` during expansion. +- Note that unpythonic's own macros follow this contract: they do not create `Name` nodes in `Store` or `Del` context without explicitly setting `ctx`. + +--- + +## Phase 5: Feature additions + +### `autoreturn` + `match`/`case` + +Add `ast.Match` handling to `TailStatementTransformer` in `syntax/tailtools.py` (lines ~685–716). + +The handler follows the existing pattern — recurse into the tail statement of each case body: +```python +elif type(tree) is Match: + for case in tree.cases: + if case.body: + case.body[-1] = self.visit(case.body[-1]) +``` + +This is approximately 3–5 lines. Import `Match` directly from `ast` (exists since 3.10, which is the floor). + +`scopeanalyzer.py` already handles match/case for scope analysis, so the infrastructure is in place. + +### New tests + +**`syntax/tests/test_scopeanalyzer.py`** line ~17: TODO says "Add tests for match/case once we bump to 3.10" — now actionable. Add scope analysis tests for match/case patterns. + +**Verify `MatchCapturesCollector` correctness**: The collector walks `.patterns` and `.kwd_patterns` of `MatchMapping`/`MatchClass` looking for `Name` nodes. But in practice, captures appear as `MatchAs(name='x')` and `MatchStar(name='rest')` with bare strings — not `Name` nodes. The only `Name` nodes in match patterns are class references like `Point` in `MatchClass.cls` and dotted names in `MatchValue`. The comment at line ~358 says match/case "uses names in `Load` context to denote captures" — verify whether this is accurate, or whether `MatchCapturesCollector` is dead code (or worse, incorrectly collecting class references as captures). The new tests should cover nested patterns like `case {'key': Point(x, y)}:` to exercise this. + +**`syntax/tests/test_scopeanalyzer.py`** line ~18: TODO says "Add tests for try/except* once we bump to 3.11" — now actionable. These tests must go in a **separate version-suffixed module** (e.g. `test_scopeanalyzer_3_11.py`) since `except*` syntax won't parse on 3.10. Add a TODO comment in the new file: "Merge into test_scopeanalyzer.py when floor bumps to Python 3.11+." + +**Test runner**: Add version-suffix gating to `runtests.py`. The convention is: `test_*_3_NN.py` means "requires Python 3.NN+". Port the `_version_suffix` parsing function from mcpyrate's `runtests.py` (see `~/Documents/koodit/mcpyrate/runtests.py`), but integrate it differently — keep skipped modules in the test list and check inside the per-module `testset()` block: + +```python +with testset(m): + ver = _version_suffix(m) + if ver is not None and sys.version_info < ver: + # Log skip using framework idioms (maybe_colorize, TestConfig.printer) + continue + mod = import_module(m) + mod.runtests() +``` + +Use `maybe_colorize` with the framework's `ColorScheme` (probably `GREYED_OUT` or `WARNING`) for the skip message — don't use mcpyrate's `colorize()` directly. This keeps the skip message visually consistent with the testset nesting structure. This is a new testing capability for unpythonic — up to now, all version-specific tests used AST-based approaches that didn't require the parser to handle newer syntax. + +Note: `_version_suffix` parses module names (dotted), not filenames. Adjust the regex to match on the final component, e.g. `test_scopeanalyzer_3_11` at the end of `unpythonic.syntax.tests.test_scopeanalyzer_3_11`. + +**`autoreturn` test**: Add a test in `test_autoret.py` verifying that `autoreturn` correctly returns from the tail of each `match`/`case` branch. + +Use `unpythonic.test.fixtures` (`test[]`, `test_raises[]` macros, `testset()` context managers). Follow existing test examples in the codebase. + +The `match`/`case` and `autoreturn` tests go in regular test modules (not version-suffixed) since the floor is 3.10 and `match`/`case` exists since 3.10. Only `except*` tests (3.11+) need a version-suffixed module. + +### Changelog + +After all phases are complete, update `CHANGELOG.md` with a 2.0.0 entry covering: +- Python version support: 3.10–3.14 (dropped 3.8, 3.9; added 3.13, 3.14) +- Requires mcpyrate >= 4.0.0 +- Deprecated: parens syntax for macro arguments; use bracket syntax instead +- `autoreturn` now handles `match`/`case` statements +- Updated `hasattr` checks for Python 3.13 AST field defaults +- Updated `arguments()` constructors for Python 3.13 strictness +- New scopeanalyzer tests for match/case and try/except* + +### Issue tracker + +**Close with 2.0.0:** +- #92 — "Remove Python 3.8 support once EOL" +- #93 — "Support Python 3.10+ changes to the AST" + +**Re-milestone from 1.1.0 to 2.1.0** (1.1.0 is not happening — it became 2.0.0; these are non-breaking and don't need to be in the major bump): +- #80, #82, #83, #97 + +If any of these turn out to introduce breaking changes upon closer inspection, move them to 2.0.0 and implement before release. + +**#83** ("Support new source location fields in Python 3.8+") — the `hasattr` fixes in Phase 4 are partial progress, but the broader goal of propagating `end_lineno`/`end_col_offset` everywhere remains open. Keep the ticket open, note the partial progress. + +**Post-release:** A full triage of all open tickets is overdue. Do this after 2.0.0 ships, not as part of the modernization work. + +--- + +## Testing + +Run the full test suite and all demos **after each phase**, on all supported versions: +- Python 3.10 (floor, known working) +- Python 3.11 (supported, not explicitly tested before) +- Python 3.12 (known working) +- Python 3.13 +- Python 3.14 + +```bash +python runtests.py +``` + +Additionally, on 3.13, catch AST constructor warnings: + +```bash +python -W error::DeprecationWarning runtests.py +``` + +--- + +## Files affected (summary) + +| File | Phase 2 (floor bump) | Phase 3 (mcpyrate 4.0.0) | Phase 4 (3.13) | Phase 5 (features) | +|------|---------------------|--------------------------|----------------|-------------------| +| `syntax/lambdatools.py` | remove version guards | `getconstant`→`.value`, remove `Str` | `hasattr` cleanup | — | +| `syntax/letdoutil.py` | remove version guards, TODOs | `getconstant`→`.value`, remove `Str` | `hasattr` fix (**critical**) | — | +| `syntax/letdo.py` | remove version guard, TODOs | — | `hasattr` fix (**critical**), `arguments()` | — | +| `syntax/tailtools.py` | remove version guard | `getconstant`→`.value`, remove `NameConstant` | `arguments()` | `autoreturn` match/case | +| `syntax/autoref.py` | — | `getconstant`→`.value` | — | — | +| `syntax/util.py` | — | `getconstant`→`.value` | — | — | +| `syntax/scopeanalyzer.py` | remove version guard | — | `hasattr` cleanup | — | +| `syntax/testingtools.py` | remove version guard | — | `hasattr` cleanup | — | +| `syntax/dbg.py` | — | — | `hasattr` cleanup | — | +| `syntax/letsyntax.py` | remove version guard, TODOs | — | — | — | +| `syntax/prefix.py` | remove version guard | — | — | — | +| `syntax/nameutil.py` | remove version guard | — | — | — | +| `syntax/__init__.py` | macro arg brackets TODO | — | — | — | +| `syntax/tests/test_letdoutil.py` | remove version guards, TODO | `getconstant`→`.value`, remove `Num` | — | — | +| `syntax/tests/test_util.py` | — | `getconstant`→`.value`, remove `Num`/`Str` | — | — | +| `syntax/tests/test_conts_multishot.py` | remove version guard | — | `hasattr` fix (**critical**) | — | +| `syntax/tests/test_letdo.py` | TODO brackets | — | — | — | +| `syntax/tests/test_scopeanalyzer.py` | — | — | — | match/case + except* tests | +| `typecheck.py` | remove version guard | — | — | — | +| `misc.py` | remove version guard | — | — | — | +| `tests/test_fun.py` | leave as-is (3.11 guard still needed) | — | — | — | +| `runtests.py` | — | — | — | port version-suffix gating from mcpyrate | +| `syntax/tests/test_scopeanalyzer_3_11.py` | — | — | — | new: except* scope tests | +| `pyproject.toml` | version 2.0.0, deps, classifiers | — | — | — | +| `.pdm-build/pyproject.toml` | sync | — | — | — | +| CI workflows | update matrices | — | — | — | +| `README.md`, `CLAUDE.md`, `CONTRIBUTING.md` | update version ranges | — | — | — | +| `CHANGELOG.md` | — | — | — | add 2.0.0 entry | + +## Style notes + +Follow existing unpythonic conventions: `from ... import ...` style, ~110 char line width, reStructuredText docstrings. See CLAUDE.md in repo root for full conventions. + +Don't rename unpythonic features with `as` — macro code depends on original bare names. The testing framework uses `test[]` and `test_raises[]` macros, not `assert`. diff --git a/briefs/monads-implementation.md b/briefs/monads-implementation.md new file mode 100644 index 00000000..437ac826 --- /dev/null +++ b/briefs/monads-implementation.md @@ -0,0 +1,255 @@ +# CC Brief: Monad subpackage for unpythonic + +## Goal + +Port the teaching-code monads from `~/Documents/python-opetus-2017/examples/monads.py` into unpythonic as a new subpackage `unpythonic.monads`, plus a `with monadic_do(M):` macro in `unpythonic.syntax`. Adapt to fit unpythonic idioms; do not import to top-level (subpackage-only access, similar to how `from unpythonic.env import env` is the standard import for `env`). + +Resolves deferred item D13. + +## Reference + +- Source: `~/Documents/python-opetus-2017/examples/monads.py` (1521 lines, 6 monads + helpers) +- Existing precedent: `unpythonic/amb.py` (`MonadicList`, `forall`, `choice`, `insist`, `deny`) +- Existing macro precedent: `unpythonic/syntax/forall.py` +- mcpyrate `ASTMarker` for tracking processed AST nodes (see mcpyrate's source/docs and `unpythonic.syntax.tailtools` for usage examples) +- `unpythonic.syntax.letdoutil` — binding parser/destructurer (already understands modern `[x := mx, y := my(x)]` and discordian-deprecated `[x << mx, y << my(x)]`; we reuse it for **parsing only**, not for runtime expansion) +- `unpythonic.llist.nil` — singleton used in place of a new `Empty` sentinel (see List monad below) +- `unpythonic.slicing.Sliced` — model for the ABC-with-inheritance pattern we'll use for `Monad` + +## Design decisions (all confirmed in pre-build discussion) + +### Subpackage layout + +``` +unpythonic/monads/ +├── __init__.py # re-exports public API of subpackage; NOT re-exported at top level +├── abc.py # Monad Protocol (runtime-checkable, structural) +├── core.py # liftm, liftm2, liftm3 (no function-form do-notation; punted) +├── identity.py +├── maybe.py +├── either.py # NEW (parallel to Maybe, carrying error value) +├── list.py # the new home of MonadicList → renamed List +├── writer.py +├── state.py +├── reader.py +└── tests/ + └── test_*.py + +unpythonic/syntax/monadic_do.py # `with monadic_do(M) as result:` macro +unpythonic/syntax/tests/test_monadic_do.py +``` + +The top-level `unpythonic/__init__.py` does **not** star-import from `monads`. Users write `from unpythonic.monads import Maybe, Either, ...` explicitly. This is an exception to the usual top-level re-export convention; it mirrors how `from unpythonic.env import env` is the standard import path for `env`. Composition with the rest of unpythonic is fine — see "Integration tests" below. + +### Seven monads + +`Identity`, `Maybe`, `Either`, `List`, `Writer`, `State`, `Reader`. `Either` is added beyond the teaching code (natural complement to `Maybe`, carrying an error value). Faithful ports otherwise — minor adaptations to unpythonic style, no behavior changes intended. + +### Bind / sequence spelling + +- Bind: `>>` (Python's `>>=` is `__irshift__`, in-place, can't chain) +- Sequence: `.then(other_monad)` +- Unit: the class constructor itself (so `Identity(x)`, `Maybe(x)`, `List(x)` are units) + +These match the teaching code; keep. + +### `Monad` ABC, `LiftableMonad(Monad)` ABC + +Real ABCs with inheritance, modeled on `unpythonic.slicing.Sliced`. Two-level split because `lift` (`f: a -> b → a -> M b`) doesn't make sense for every monad — `State.lift` and `Reader.lift` are not implementable in the obvious way (the teaching code's `State.lift` raises `NotImplementedError`, `Reader.lift` is missing). + +**`Monad` (base ABC)** — required for all monads: + +- `__init__` (unit) — `@abstractmethod` +- `fmap(self, f)` — `@abstractmethod` +- `join(self)` — `@abstractmethod` + +Default (non-abstract) implementations: +- `__rshift__(self, f)` (bind) = `self.fmap(f).join()` — override only for efficiency (e.g., `Writer` overrides to avoid double-logging) +- `then(self, f)` = `self >> (lambda _: f)` — override usually unnecessary + +**`LiftableMonad(Monad)`** — adds `lift` for monads where it makes sense: + +- `lift(cls, f)` = `lambda x: cls(f(x))` — default classmethod; subclasses may override + +Membership: +- `LiftableMonad`: `Identity`, `Maybe`, `Either`, `List`, `Writer` +- `Monad` directly (no lift): `State`, `Reader` + +**Docstrings** make clear for each method whether it's `@abstractmethod` (must override), has a default implementation (override is optional, usually for efficiency), or is a final concrete method. + +Lives in `unpythonic/monads/abc.py`. Whether these also dispatch into the new D4/D5 typecheck layer is a follow-up — flag in TODO_DEFERRED if deeper integration looks valuable after the basic port lands. + +### State.join — fill in during port + +The teaching code's `State.join` is a TODO-punt (raises `NotImplementedError`), not a fundamental obstacle. The standard Haskell definition (`join mm = State $ \s -> let (m, s') = runState mm s in runState m s'`) ports cleanly to Python. Implement it during the port. State is a proper monad with a well-defined join; no `JoinableMonad` split is needed. + +**Docstring / code comment** for `State.join` should explain the operation in plain words, since the abstract definition is dense: + +> Given `mm : State(s -> (State(s -> (a, s)), s))`, run the outer state function to get `(inner_m, s')`, then run the inner with `s'` — standard "thread the state" pattern. + +Reader's `join` already works in the teaching code — only `lift` is missing, which `LiftableMonad` already handles. + +### `MonadicList` migration + +- Move the implementation to `unpythonic/monads/list.py`, renamed `List`. +- Add back the **varargs constructor** (`List(1, 2, 3)`) we reviewed-and-removed from `MonadicList` recently — turns out monadic-list ergonomics specifically need it, because monadic `unit` is then literally the class (`List(x)` = singleton list containing `x`). +- Use `nil` from `unpythonic.llist` in place of a fresh `Empty` sentinel (avoid proliferating singletons). +- Keep all the richer protocol from `MonadicList`: full `Sequence` ABC interface (`__len__`, `__eq__`, `__contains__`, `__reversed__`, `index`, `count`, ABC registration), type annotations. +- `unpythonic.amb.MonadicList` becomes a silent alias of `unpythonic.monads.List` (no `DeprecationWarning` on import — gentle path). +- Add a `# TODO(3.0.0): remove MonadicList alias` comment at the alias site. +- Add a `TODO_DEFERRED.md` entry tracking the alias removal for 3.0.0. + +### Do-notation: macro only + +No function-form `monadic_do`. The eval-based codegen approach in `amb.forall` is the cautionary tale we shouldn't repeat. Macro can do the same job cleanly via AST rewriting. Users who don't want do-notation just use `>>` chains directly with any monad. + +### Macro syntax + +```python +with monadic_do(Maybe) as result: + [x := mx, + y := my(x)] in + result << M.unit(x + y) +``` + +Cultural note: `let-in` is arguably the only correct syntax for monadic do, since this whole tradition is Haskell. + +**Body shape**: a single `Expr` statement whose `.value` is a `Compare` with `In` op. LHS is a `List` of `NamedExpr` bindings. RHS is `BinOp(LShift, Name('result'), final_monadic_expr)`. + +**Bindings**: `:=` is the modern operator; `<<` is supported as a discordian-deprecated alternative (`letdoutil` already understands both — we reuse). Sequencing-only lines (Haskell `do { mx; ... }` using `>>` not `>>=`) are spelled `_ := mexpr`. The throwaway `_` makes intent visible in teaching contexts. + +**Empty bindings**: `[] in result << M.unit(x)` is supported. Reduces beautifully as `len(bindings) → 0`. + +**Strict RHS**: must be `result << expr` (the `as result` name on the LHS, `<<` on the RHS). Anything else is a macro-time error. Strict because `<<` makes data flow visible and matches the `with ... as result` declaration. + +**Why `<<` on the RHS instead of `return`**: `return` at the top level of a `with` breaks reader expectations (usually exits the surrounding function). The "send to box" idiom (`result << expr`) is what unpythonic uses elsewhere (e.g., conditions/restarts subsystem) for this same problem. + +**Xmas-tree placement**: `monadic_do` is always the innermost `with`. + +```python +# xmas-tree macros (any combination thereof) +with prefix, autoreturn, quicklambda, multilambda, envify, lazify, namedlambda, autoref, autocurry, tco: + with monadic_do(M) as result: + [x := mx, y := my(x)] in result << M.unit(x + y) +``` + +**Rationale — forced and correct:** + +The "innermost" position is both *forced* (by body shape) and *correct* (edit order works out): + +- **Forced**: `monadic_do`'s body must be a single `Expr` statement of the form `[bindings] in result << expr`. It syntactically *cannot* contain `with X:` statements, so lexically wrapping anything else inside a `monadic_do` block is impossible by construction. + +- **Correct**: two-pass macros (`lazify`, `tco`, `continuations`, `autocurry`, `envify`, `namedlambda`, `autoref`) do their first pass, then explicitly expand inner macros via `dyn._macro_expander.visit_recursively(body)`, then their second pass. `monadic_do` (one-pass outside-in) fires during the outer macro's `visit_recursively`, producing the bind chain. The outer macro's second pass then edits the expanded chain — exactly the order we want (autocurry curries the calls, lazify force-wraps references, tco optimizes tails, CPS transforms for continuations). One-pass outside-in surface-syntax macros (`prefix`, `autoreturn`, `quicklambda`, `multilambda`) normalize their body before descending, so `monadic_do` sees normal Python when it fires. + +Usage is just the ordinary nested-`with` style that all block macros typically use — no special "must be own `with`" rule needed. + +**Dialects**: the same analysis applies transparently. Dialects (e.g., Lispython) wrap a module in block macros at parse-assembly time; `monadic_do` sits innermost within whatever the dialect adds. No dialect-specific integration testing is prioritized — the generic integration tests cover the same underlying macros. + +**Expansion**: the whole `with` is rewritten away into nested lambda binds: + +```python +result = mx >> (lambda x: my(x) >> (lambda y: M.unit(x + y))) +``` + +The `with monadic_do(M) as result:` is purely syntactic — `monadic_do(M)` is never called at runtime. Same pattern as `with continuations:`, `with autocurry:`, `with lazify:`. + +**Implementation pattern**: +- mcpyrate visitor that processes `with monadic_do(...)` statements +- `letdoutil` (`UnexpandedLetView` and friends) to parse/destructure the bindings list — **parsing only**; no env-based runtime is involved +- Expansion target is plain nested lambdas. Each `x := mx` becomes `mx >> (lambda x: )`. Python's lexical scoping handles name shadowing in nested lambdas correctly — no `env` runtime needed. +- `ASTMarker` to mark rewritten nodes during expansion +- Require single-statement body; helpful error message if violated + +### Lazify interaction — analysis (no special handling expected) + +Verified by reading `lazify.py` source and tests: + +1. Lazify computes `userlambdas = detect_lambda(body)` in its **first** (outside-in) pass, then expands inner macros via `dyn._macro_expander.visit_recursively(body)`. The lambdas `monadic_do` produces during that expansion have node ids *not* in `userlambdas` → lazify treats them as macro-introduced and skips the `passthrough_lazy_args` wrapping. It still recurses into the lambda body, applying normal force/lazyrec on references and call args. +2. The bind-chain expansion `mx >> (lambda x: my(x) >> (lambda y: ...))` provides natural deferral via lambda boundaries — `my(x)` is inside the lambda body, only invoked when the first bind fires. No extra `lazy[]` wrapping adds anything for nested bindings. +3. The first binding's RHS has to be evaluated to produce the receiver of `>>` anyway — wrapping it in `lazy[]` and immediately needing to call `__rshift__` on it (which `Lazy` doesn't have) would just need a `force()` to undo, net no-op. +4. References to outer-scope names get auto-`force()`'d in Load context (existing lazify behavior on `Name` nodes), so a `lazy_var` from surrounding scope gets unwrapped before being used as a `>>` receiver. + +**Short-circuit preservation (the real concern)**: for monads like `Maybe` and `Either` that short-circuit on the failure path, bindings after the short-circuit point must *not* be forced. This holds automatically because: + +- The macro puts later bindings *inside* lambda bodies (`leftM >> (lambda x: rightM >> (lambda y: ...))`) +- `rightM` becomes a Load-context Name, so lazify wraps it as `force(rightM)` — but that wrapping is itself inside the lambda body +- When `leftM` is `Left(err)` (or `Maybe(Empty)`), `__rshift__` returns `self` without invoking the passed lambda → the `force(rightM)` is never reached + +This is the guarantee that would break if we got the macro expansion wrong — e.g., by hoisting binding RHSs to an outer scope for "efficiency." Don't. **A dedicated integration test pins this down**: a `Maybe`/`Either` do-block inside `with lazify`, where a later binding RHS contains an observable side effect (e.g., `nonlocal counter; counter += 1; return M.unit(42)`) or would raise (`1/0`). Trigger the short-circuit path. Assert the side effect didn't happen / no exception raised. + +**Conclusion**: `monadic_do` should require no macro-side intervention to compose with `with lazify`. The integration test and the short-circuit test verify the contract. If either fails, revisit and consider explicit lazy-marking via `letdoutil` or directly in the macro. + +## Test plan + +- `monads/tests/test_*.py` — one file per monad, exercising unit, bind, sequence, fmap, join, guard, lift; classical examples (sqrt chain for Maybe, multivalued sqrt for List, Pythagorean triples for List, log accumulation for Writer, state-passing counter for State, env-reading for Reader, Either left/right paths) +- `monads/tests/test_core.py` — `liftm`, `liftm2`, `liftm3`; `Monad` Protocol structural check +- `monads/tests/test_abc.py` — `isinstance(x, Monad)` works for all seven monads (via inheritance), fails for non-monads; default `then`/`__rshift__` from the ABC actually fire +- `syntax/tests/test_monadic_do.py` — macro tests: each monad through do-notation; `:=` and `<<` both accepted; `_ := mexpr` sequencing; empty bindings; strict-RHS error case; nested do-blocks; the Pythagorean-triples canonical test +- `tests/test_amb.py` — confirm the `MonadicList` alias still works (existing tests should pass unchanged) + +All tests use the `unpythonic.test.fixtures` framework (`test[]`, `test_raises[]`). Use `the[]` **only when the default auto-capture (LHS of a comparison) is not what we want** — e.g., to capture a container instead of a leaf, or to capture multiple subexpressions, or in non-comparison assertions. + +### Integration tests (separate module, e.g. `syntax/tests/test_monadic_do_integration.py`) + +`monadic_do` shouldn't fall apart when nested inside other unpythonic block macros. Use one nested `with` per outer macro (no chaining in the same `with` per `doc/macros.md`). Xmas-tree ordering applies — `monadic_do` is the inner block in all combinations. + +**Must test**: +- `with continuations:` — bind chain inside a continuations block +- `with autocurry:` — autocurry shouldn't munge the bind chain (`__rshift__` is a method call, but `>>` operator uses dunder dispatch — should be transparent) +- `with lazify:` — **especially important** (Haskellism; see "Lazify interaction" above for the analysis) +- `with tco:` — deep do-blocks produce a lambda tower that ends in tail calls to `>>`; verify no stack issues, or document that TCO doesn't reach into bind chains + +**Smoke test** (verify "doesn't crash", may interact in interesting ways): +- `with multilambda:` — `lambda: [a, b, c]` semantics could appear inside the body +- `with quicklambda:` — `f[...]` shorthand might appear in user expressions +- `with namedlambda:` — automatic naming of macro-introduced lambdas +- `with autoreturn:` — interaction with the `result << expr` exit pattern (`autoreturn` may try to inject `return` somewhere awkward) + +**Likely orthogonal but smoke test for safety**: +- `with envify:` +- `with autoref:` + +**Low priority (decide after smoke test)**: +- `with prefix:` — Listhell-specific. If the monads happen to work with it, nice; if not, not a priority. Smoke-test, see what happens, then decide whether to spend effort on interop. + +If any combination needs real interaction work beyond "doesn't crash," flag in `TODO_DEFERRED.md` rather than expanding scope here. + +## Out of scope + +- Free monads, monad transformers, applicative-only structures +- IO monad (Python is impure already; no value-add) +- Continuation monad (we already have `with continuations:` — strictly more powerful) +- Deeper `@generic` / D4/D5 typecheck integration for `Monad` Protocol — flag for follow-up if the basic port suggests it would pay off +- Performance: faithful port; no optimization pass + +## Order of work + +1. `monads/abc.py` — `Monad` ABC (model on `unpythonic.slicing.Sliced`) +2. `monads/core.py` — `liftm`, `liftm2`, `liftm3` +3. `monads/identity.py` — simplest, sets the per-monad pattern +4. `monads/maybe.py`, `monads/either.py` — error handling pair +5. `monads/list.py` — port `MonadicList`, rename, varargs constructor, `nil` sentinel +6. `amb.py` — alias `MonadicList = unpythonic.monads.List`, TODO comment, TODO_DEFERRED entry +7. `monads/writer.py`, `monads/state.py`, `monads/reader.py` +8. `monads/__init__.py` — re-exports +9. Tests for the pure-Python layer (one per monad, plus core/abc) +10. `syntax/monadic_do.py` — the macro +11. Tests for the macro +12. **Pytkell dialect example update** — add monad usage examples to `unpythonic/dialects/tests/test_pytkell.py` (Pytkell is the Haskell-flavored joke dialect; no kell is complete without its monads). Faithful Haskell-do feel: at minimum show `Maybe`-chained sqrt, `List`-based Pythagorean triples, and `Writer`-based logging — all via `with monadic_do(M):`. +13. **Documentation**: + - `doc/features.md` — document the new pure-Python API surface: `Monad` and `LiftableMonad` ABCs, the seven monads (`Identity`, `Maybe`, `Either`, `List`, `Writer`, `State`, `Reader`), `liftm`/`liftm2`/`liftm3`. Match the existing section style (brief intro, usage example, method reference where relevant). + - `doc/macros.md` — document the `with monadic_do(M) as result:` macro: the binding syntax (`:=` and `<<`), the `result << expr` exit pattern, the always-own-`with` convention, the xmas-tree placement between `multilambda` and `envify`. Match the existing macro documentation style. + - `README.md` — a short usage example in the style of the existing ones (likely a `Maybe`-chained computation or `List`-based Pythagorean triples, with enough prose to show the flavor without bloating the README). + - `CHANGELOG.md`, `AUTHORS.md` as applicable. + +Each significant step a separate commit. Don't mix the pure-Python port with the macro work — clean bisect boundaries. + +## Conventions reminder + +- `__all__` per module, ordered to mirror file order +- Type annotations on all new code +- reStructuredText docstrings, ~110 char line width +- `from ... import ...` style, no renaming with `as` +- Bind = `>>`, sequence = `.then`, unit = constructor +- `nil` (from `unpythonic.llist`) for empty-list sentinel in the List monad +- `:=` is primary bind syntax in macro; `<<` accepted as discordian-deprecated alternative diff --git a/briefs/multishot-implementation.md b/briefs/multishot-implementation.md new file mode 100644 index 00000000..974c62a9 --- /dev/null +++ b/briefs/multishot-implementation.md @@ -0,0 +1,133 @@ +# CC Brief: Multi-shot generator macros for unpythonic + +## Goal + +Promote the working proof-of-concept in `unpythonic/syntax/tests/test_conts_multishot.py` to a real public-API macro module. Resolves issue #80 (milestone 2.2.0). + +Ships `@multishot` (decorator macro), `myield` (name/expr macro, four forms), and `MultishotIterator` (pure-Python adapter conforming to a subset of the generator protocol). Multi-shot continuations: a `@multishot` function can be resumed *from any earlier `myield`, arbitrarily many times*, branching execution into independent timelines. + +Sits alongside raw `call_cc[]` (which stays as the alien-grade primitive) and `get_cc()` (the human-friendly Racket-`let/cc`-styled name binding). `@multishot`/`myield` is the third tier — ergonomic, generator-shaped, for the pattern that motivates `get_cc` in the first place. + +## Reference + +- Source: `unpythonic/syntax/tests/test_conts_multishot.py` (the working demo, including `@multishot`, `myield`, and `MultishotIterator`) +- Earlier didactic version: `unpythonic/syntax/tests/test_conts_gen.py` (single-shot `dlet`-based generators, kept as teaching cross-reference) +- Existing precedent for syntax-module + adapter pairing: `unpythonic.syntax.continuations` + `unpythonic.fun` helpers +- Continuations primitives used: `call_cc`, `get_cc`, `iscontinuation` (already public) +- Documentation home: `doc/macros.md` (chapter on continuations) — new subsection parallel to "Topology of continuations" and "Scoping of locals in continuations" added during #82 + +## Design decisions (confirmed in pre-build discussion 2026-05-06) + +### Module layout + +``` +unpythonic/syntax/multishot.py # @multishot, myield, MultishotIterator together +unpythonic/syntax/tests/test_multishot.py +``` + +Adapter (`MultishotIterator`) lives **alongside the macro**, not in a separate pure-Python module. Rationale: the adapter is unusable without `@multishot`, and grouping the public surface in one file keeps the contract obvious. Re-export both macros and the adapter via `unpythonic/syntax/__init__.py` (and from there into the top-level `__init__.py`'s star-import set, per project convention). + +### Naming + +`@multishot`, `myield`, `MultishotIterator`. Keep the demo's names — `myield` is consistent with unpythonic's `m`-prefixed style, and grep-friendly enough with `\bmyield\b`. + +### Scope: separate API, not a replacement + +`@multishot` is an additional ergonomic layer on top of `call_cc[]`/`get_cc()`. It does **not** supersede them. Users who want the low-level form keep it. unpythonic is partly a pedagogic project; raw `call_cc` stays for readers studying continuations. + +### `with continuations` is required and explicit + +`@multishot` only works inside an enclosing `with continuations:` block. The macro does not auto-wrap. Zen of Python: explicit is better than implicit. Document the requirement at the top of the docstring; raise a clear macro-expansion-time error if the user forgets (detectable when `call_cc` isn't macro-imported in scope, or — failing that — fall through to whatever error the unconverted `call_cc[]` produces and document the symptom). + +### `myield` placement constraint + +Statement-only, top-level-only inside the `@multishot` body. This is a real limitation of `call_cc[]`, not negotiable. Document loudly. Macro raises `SyntaxError` at expansion time if `myield` is found outside the top level of a `@multishot` `def` (the demo already does this). + +### `MultishotIterator` API surface (v1) + +Standard generator protocol subset: +- `__iter__` +- `__next__` +- `send(value)` +- `throw(typ_or_exc)` — quirky semantics documented (no paused frame to throw into; re-entering the continuation makes it raise) +- `close()` — quirky semantics documented (closing rejects further `next`/`send` unless `self.k` is overwritten) + +Generator introspection attributes: +- `gi_frame` — **always `None`**. A multi-shot generator has no paused frame: every `myield` terminated its frame and returned a continuation closure; state lives in closure cells, not a frame. The real-generator idiom `gen.gi_frame is None ↔ exhausted` does **not** apply here — there's never a paused frame, by construction. Document loudly under "Differences from standard Python generators". +- `gi_code` — `self.k.__code__` while live; `None` after `close()`. This is what consumers should use as the liveness signal. Gives debuggers the code object that the next advance will run. +- `gi_running` — **always `False`**. Nothing is ever paused: every continuation is a separately-activated closure. +- `gi_yieldfrom` — `None` in v1 (no `myield_from` yet). Becomes meaningful in the post-v1 follow-up. + +Beyond the standard surface: +- `__copy__` — fork the iterator. Both copies share the current continuation; subsequent advances diverge into independent timelines. **This is the entire point of multi-shot, exposed through the stdlib `copy` protocol.** Docstring should mention the technical distinction — standard Python generators don't support `copy.copy()` at all — since that's what makes the protocol meaningful here. Suggested docstring: *"Forks this multi-shot iterator at the current continuation; subsequent advances of the two iterators are independent. Unlike standard Python generators, multi-shot generators are copyable."* +- `__deepcopy__` — raises `TypeError("multi-shot iterators cannot be deep-copied; use copy.copy() to fork")`. The continuation closes over caller state we can't meaningfully deep-copy, and the stdlib's default deep-copy fallback (recurse into `__dict__`) would either error obscurely or produce a nonsensical clone. Fail loudly and point the user at the right tool. +- `__del__` calls `close()`. Mostly cosmetic for multishot (no paused frame to clean up), but mirrors generator GC semantics. + +Not in v1, with documented rationale: +- `myield_from` — planned as the immediate follow-up changeset (see "Post-v1, while CI engines are still warm" below). v1 ships without. +- Async multi-shot generators (`__aiter__`, `asend`, `athrow`, `aclose`) — out of scope. unpythonic has no async support yet across the library; deferred until that's a project. Note in docs as future work. +- Pickling / `__reduce__` — continuations are closures; not picklable. Note and skip. +- `yield from` *across* a real generator and a multishot — wontfix. Real generators have paused state, multishots don't; the semantic mismatch can't be papered over. Documented as a known limitation. + +### PEP 479 boundary + +`return value` inside `@multishot` is rewritten to `raise StopIteration(value)` (the demo already does this). PEP 479's "StopIteration leaking out of a generator becomes RuntimeError" applies inside *real* generator frames; the multishot body is a regular function under the hood, so the rewrite produces a `StopIteration` that the `MultishotIterator` wrapper catches and re-raises cleanly to the caller. Add a test confirming `return 42` surfaces as `StopIteration(42)` to the iterator consumer, not `RuntimeError`. + +### Documentation + +New subsection in `doc/macros.md` under the existing continuations chapter, parallel to "Topology of continuations" and "Scoping of locals in continuations" added during #82. Sections: + +1. **Why multi-shot.** One-paragraph framing: classical Python generator + can resume from any earlier `myield` arbitrarily many times. +2. **Usage.** `@multishot` + `myield` four-form table (the one already in the demo's docstring), plus the `MultishotIterator` wrapper for generator-protocol-shaped consumption. +3. **Differences from standard Python generators.** `copy()` is the headline: *"Unlike standard generators, multi-shot generators support `copy.copy()`. The fork shares the current continuation; subsequent advances of the two iterators diverge into independent timelines."* Then the limitations: no `yield from` across real/multishot, exception/`finally` semantics differ across `myield` boundaries, no async form, no pickling, statement-only top-level `myield`. +4. **Cross-references.** Pointer to `test_conts_gen.py` (single-shot didactic version, raw `call_cc`) and `test_conts_multishot.py` — wait, that one *is* the implementation, will be replaced. Cross-link to whatever didactic example survives, plus the new `test_multishot.py` for the canonical usage. + +TOC updated. + +### Tests + +New file `unpythonic/syntax/tests/test_multishot.py`. Promote/rewrite from `test_conts_multishot.py`, dropping the multi-phase compilation scaffolding (the real module won't need it — users just import). Keep `test_conts_gen.py` as the didactic single-shot raw-`call_cc` example, cross-referenced from the new docs. Retire `test_conts_multishot.py` once the new tests cover its content (it's superseded by the real module + tests). + +Test coverage targets: +- Each of the four `myield` forms. +- Basic linear consumption via `MultishotIterator` (matches the demo's `[x for x in mi] == [1, 2, 3]`). +- `send` round-trip into `var = myield`. +- `throw` re-entry into a continuation. +- `close` rejecting subsequent `next`. +- **Multi-shot fork via `copy.copy`** — the headline test. Two iterators from the same continuation, advance independently, assert the timelines differ. +- `copy.deepcopy(mi)` raises `TypeError`. +- `gi_running is False` always (including mid-iteration, sampled between `next` calls). +- `gi_frame is None` always (including mid-iteration and after `close()`). +- `gi_code` matches `self.k.__code__` while live, becomes `None` after `close()`. +- `gi_yieldfrom is None` (becomes meaningful after `myield_from` lands). +- `return value` inside `@multishot` raises `StopIteration(value)` to the consumer, not `RuntimeError`. +- `myield` outside a `@multishot` raises `SyntaxError` at macro-expansion time. +- `myield` inside a nested scope (lambda, comprehension, nested `def`) raises `SyntaxError` at expansion time. + +## Post-v1, while CI engines are still warm + +Add `myield_from` as an immediate follow-up changeset: +- New name/expr macro `myield_from(other_multishot)` that delegates to another `@multishot`'s continuation. +- Restricted to multishot-to-multishot; cross-talk with real generators stays wontfix. +- Update `gi_yieldfrom` to point to the inner iterator's current continuation while delegating. +- Tests + doc subsection. + +Treat `myield_from` as a separate PR/commit on top of the v1 module landing — keeps the v1 review surface manageable and gives a clean point to bail if `myield_from` turns out harder than it looks. + +## CHANGELOG + +Under 2.2.0 in-progress section, **Added**: +- `@multishot` and `myield` macros + `MultishotIterator` adapter (`unpythonic.syntax.multishot`). Multi-shot generators that can resume from any earlier `myield` arbitrarily many times. See `doc/macros.md`. + +`myield_from` follow-up gets its own line under **Added** when it lands. + +## Out of scope + +- Async multishot generators. +- Pickling support. +- `yield from` across real and multishot generators. +- Removing `call_cc[]` or `get_cc()` from the public API. They stay. + +## Open questions to resolve during implementation + +- `__copy__` is shallow: `MultishotIterator(self.k)`. Both forks legitimately share the same closure cells — that's the multi-shot semantics. Document this in the `__copy__` docstring. diff --git a/doc/callcc_topology.png b/doc/callcc_topology.png new file mode 100644 index 00000000..a9695c47 Binary files /dev/null and b/doc/callcc_topology.png differ diff --git a/doc/callcc_topology.svg b/doc/callcc_topology.svg index 4156905e..16c399c1 100644 --- a/doc/callcc_topology.svg +++ b/doc/callcc_topology.svg @@ -2,20 +2,23 @@ + inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)" + sodipodi:docname="callcc_topology.svg" + inkscape:export-filename="/home/jje/Documents/koodit/unpythonic/doc/callcc_topology.png" + inkscape:export-xdpi="149.99475" + inkscape:export-ydpi="149.99475" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:dc="http://purl.org/dc/elements/1.1/"> + fit-margin-bottom="0" + inkscape:pagecheckerboard="0" + inkscape:snap-page="true"> image/svg+xml - + + + call_cc: running [code] with scissors + style="font-size:20px;line-height:1.25">call_cc: running [code] with scissors f + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f f_cont + y="143.79164" + style="font-size:15px;line-height:1.25">f_cont call_cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">call_cc g + sodipodi:role="line" + style="font-size:15px;line-height:1.25">g cc=f_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=f_cont cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc cc=... + y="16.545696" + style="font-size:15px;line-height:1.25">cc=... pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc pcc=... + y="193.07268" + style="font-size:15px;line-height:1.25">pcc=... @@ -1082,26 +1098,26 @@ y="427.36221" /> f + y="469.22406" + style="font-size:15px;line-height:1.25">f f_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f_cont call_cc + y="452.94775" + style="font-size:15px;line-height:1.25">call_cc h + y="542.50256" + style="font-size:15px;line-height:1.25">h cc=g_cont + y="487.75632" + style="font-size:15px;line-height:1.25">cc=g_cont pcc + y="598.45447" + style="font-size:15px;line-height:1.25">pcc g + sodipodi:role="line" + style="font-size:15px;line-height:1.25">g g_cont + y="581.29163" + style="font-size:15px;line-height:1.25">g_cont call_cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">call_cc cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc cc=f_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=f_cont cc=... + y="420.1134" + style="font-size:15px;line-height:1.25">cc=... pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc pcc=f_cont + y="634.86218" + style="font-size:15px;line-height:1.25">pcc=f_cont pcc=... + y="594.77802" + style="font-size:15px;line-height:1.25">pcc=... @@ -1487,26 +1503,26 @@ y="227.36221" /> f + y="258.24609" + style="font-size:15px;line-height:1.25">f f_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f_cont call_cc + y="227.94778" + style="font-size:15px;line-height:1.25">call_cc g + y="267.50259" + style="font-size:15px;line-height:1.25">g cc=f_cont + y="267.87674" + style="font-size:15px;line-height:1.25">cc=f_cont cc + y="284.55215" + style="font-size:15px;line-height:1.25">cc f_cont1 + y="357.02982" + style="font-size:15px;line-height:1.25">f_cont1 h + sodipodi:role="line" + style="font-size:15px;line-height:1.25">h cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc call_cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">call_cc cc=f_cont1 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=f_cont1 cc=... + y="219.823" + style="font-size:15px;line-height:1.25">cc=... pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc pcc=... + y="391.34497" + style="font-size:15px;line-height:1.25">pcc=... Base case: One continuation + style="font-weight:bold;font-size:15px;line-height:1.25">Base case: One continuation Sequence of continuations + style="font-weight:bold;font-size:15px;line-height:1.25">Sequence of continuations Nested continuations + style="font-weight:bold;font-size:15px;line-height:1.25">Nested continuations To see where thisTo see where thisusage patternusage patternis useful, thinkis useful, thinkg=h=ambg=h=amb⇒ can behave like⇒ can behave like nested loops + y="223.79616" + style="font-size:15px;line-height:1.25"> nested loops f_cont is a closuref_cont is a closurethat is instantiatedthat is instantiatedwhen f is called;when f is called;it lives in theit lives in thelexical scope of f. + id="tspan11801" + style="font-size:15px;line-height:1.25">lexical scope of f. g can stash cc forg can stash cc foruse later (that'suse later (that'sthe whole point) + style="font-size:15px;line-height:1.25;text-align:start;text-anchor:start;fill:#808080;fill-opacity:1">the whole point) The f_cont1 closureThe f_cont1 closurelives in the lexicallives in the lexicalscope of f_cont. + id="tspan14644" + style="font-size:15px;line-height:1.25">scope of f_cont. @@ -2040,27 +2063,27 @@ id="rect19851" style="color:#000000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000000;solid-opacity:1;fill:none;fill-opacity:1;fill-rule:nonzero;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate" /> v + sodipodi:role="line" + style="font-size:15px;line-height:1.25">v v_cont + y="743.79175" + style="font-size:15px;line-height:1.25">v_cont call_cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">call_cc g_cont + y="706.72418" + style="font-size:15px;line-height:1.25">g_cont cc=v_cont, but... + y="649.62738" + style="font-size:15px;line-height:1.25">cc=v_cont, but... cc=... + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=... pcc + y="738.58588" + style="font-size:15px;line-height:1.25">pcc f_cont + y="794.22412" + style="font-size:15px;line-height:1.25">f_cont ...we should first...we should firstjump to f_cont,jump to f_cont,which is the cc of gwhich is the cc of gfrom the time whenfrom the time whenthe g_cont closurethe g_cont closureinstance was created.instance was created. Let's call it Let's call it pcc forparent cc, and whencalling it, pass to itcalling it, pass to itthe value of (g_cont's)the value of (g_cont's)cc so that they will chain. + id="tspan25866" + style="font-size:15px;line-height:1.25">cc so that they will chain. Stashed earlierStashed earlierby h (above) pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc The Confetti Scenarios:pretending the whole tail of acomputation is just one entity cc + y="861.41534" + style="font-size:15px;line-height:1.25">cc @@ -2398,30 +2430,31 @@ style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1, 1;stroke-dashoffset:0;stroke-opacity:1;marker-end:url(#marker6624-2)" /> cc=f2cc=f2pcc=f_cont + id="tspan26003" + style="font-size:15px;line-height:1.25">pcc=f_cont tail call + y="898.89661" + style="font-size:15px;line-height:1.25">tail call f1 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f1 f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f2 g_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">g_cont f_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f_cont cc=f2 + y="854.00293" + style="font-size:15px;line-height:1.25">cc=f2 pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc cc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc f1 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f1 f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f2 q_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">q_cont p_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">p_cont cc=f2cc=f2pcc=None + id="tspan25935" + style="font-size:15px;line-height:1.25">pcc=None s_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">s_cont r_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">r_cont tail call + sodipodi:role="line" + style="font-size:15px;line-height:1.25">tail call tail call + sodipodi:role="line" + style="font-size:15px;line-height:1.25">tail call cc + y="1348.3112" + style="font-size:15px;line-height:1.25">cc pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc cc=f2cc=f2pcc=p_cont + id="tspan25931" + style="font-size:15px;line-height:1.25">pcc=p_cont cc=f2cc=f2pcc=r_cont + id="tspan25933" + style="font-size:15px;line-height:1.25">pcc=r_cont A tail callmust propagatethe value cchas at the siteof the tail call. f1 + y="1481.7241" + style="font-size:15px;line-height:1.25">f1 f2 + y="1569.224" + style="font-size:15px;line-height:1.25">f2 cc=f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=f2 tail call + y="1441.0825" + style="font-size:15px;line-height:1.25">tail call v + y="1494.2242" + style="font-size:15px;line-height:1.25">v v_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">v_cont w + sodipodi:role="line" + style="font-size:15px;line-height:1.25">w cc=v_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=v_cont call_cc + y="1477.9479" + style="font-size:15px;line-height:1.25">call_cc cc + y="1571.1123" + style="font-size:15px;line-height:1.25">cc pcc + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc cc=f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">cc=f2 pcc=f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">pcc=f2 f1 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f1 f2 + sodipodi:role="line" + style="font-size:15px;line-height:1.25">f2 y_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">y_cont x_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">x_cont cc=f2cc=f2pcc=None + id="tspan26005" + style="font-size:15px;line-height:1.25">pcc=None s_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">s_cont r_cont + sodipodi:role="line" + style="font-size:15px;line-height:1.25">r_cont tail call + sodipodi:role="line" + style="font-size:15px;line-height:1.25">tail call tail call + sodipodi:role="line" + style="font-size:15px;line-height:1.25">tail call cc=f2cc=f2pcc=x_cont + id="tspan25743" + style="font-size:15px;line-height:1.25">pcc=x_cont pcc + y="1072.8546" + style="font-size:15px;line-height:1.25">pcc cc=<composed>cc=<composed>pcc=r_cont + id="tspan26007" + style="font-size:15px;line-height:1.25">pcc=r_cont <composed><composed>tail-calls x_conttail-calls x_contwith cc=f2 + id="tspan25765" + style="font-size:15px;line-height:1.25">with cc=f2 cc + y="1147.2625" + style="font-size:15px;line-height:1.25">cc Nearly equivalent? + style="font-weight:bold;font-size:15px;line-height:1.25">Nearly equivalent? Must compose and pass along a ccthat chains pcc and cc, in that order. Finally, we need one moreFinally, we need one moremechanism to treat the casemechanism to treat the casewhere both cc and pcc are set,where both cc and pcc are set,and a tail call is encountered: + id="tspan5126" + style="font-size:15px;line-height:1.25">and a tail call is encountered: When a function ends, check forWhen a function ends, check forpcc first; if it's set, tail-call it (andpcc first; if it's set, tail-call it (andset its cc); if it isn't, then tail-callset its cc); if it isn't, then tail-callthe cc.the cc. A separate arg is needed for the pcc,A separate arg is needed for the pcc,because the because the cc arg is a public API,for setting by call_cc and tail calls.for setting by call_cc and tail calls. The The cc arg stores nothing persistently,so we may use the so we may use the cc arg of pcc topass in the desired continuation.pass in the desired continuation. This also chains correctly if the tail consists ofThis also chains correctly if the tail consists ofthree or more parts (e.g. h_cont, g_cont, f_cont):three or more parts (e.g. h_cont, g_cont, f_cont):the cc that was passed in will be invoked the cc that was passed in will be invoked last,after the pcc chain itself completes. + id="tspan25929" + style="font-size:15px;line-height:1.25">after the pcc chain itself completes. So the general solution for a tail callis to check for pcc; if set, make acomposed cc; if not, just pass alongthe existing cc. The only place that setsThe only place that setspcc is the call_cc mechanismthat creates the definitionthat creates the definitionof the continuation function.of the continuation function. + y="1030.959" + id="tspan5100" + style="font-size:15px;line-height:1.25">  This chains correctly also in the presence ofThis chains correctly also in the presence ofmore nested tail calls. + id="tspan5132" + style="font-size:15px;line-height:1.25">more nested tail calls. x_cont can internally do whatever it wants,x_cont can internally do whatever it wants,including calling more including calling more pcc continuations(passing along the cc). + id="tspan5166" + style="font-size:15px;line-height:1.25">(passing along the cc). diff --git a/doc/design-notes.md b/doc/design-notes.md index 77272724..50a97d81 100644 --- a/doc/design-notes.md +++ b/doc/design-notes.md @@ -7,6 +7,7 @@ - [REPL server](repl.md) - [Troubleshooting](troubleshooting.md) - **Design notes** +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) @@ -16,15 +17,13 @@ - [Design Philosophy](#design-philosophy) - [Macros do not Compose](#macros-do-not-compose) - [Language Discontinuities](#language-discontinuities) - - [What Belongs in Python?](#what-belongs-in-python) - - [Killer features of Common Lisp](#killer-features-of-common-lisp) - - [Common Lisp, Python, and productivity](#common-lisp-python-and-productivity) + - [`unpythonic` and the Killer Features of Common Lisp](#unpythonic-and-the-killer-features-of-common-lisp) - [Python is not a Lisp](#python-is-not-a-lisp) - - [On ``let`` and Python](#on-let-and-python) + - [On `let` and Python](#on-let-and-python) - [Assignment syntax](#assignment-syntax) - [TCO syntax and speed](#tco-syntax-and-speed) - - [No Monads?](#no-monads) - - [No Types?](#no-types) + - [Monads](#monads) + - [Types](#types) - [Detailed Notes on Macros](#detailed-notes-on-macros) - [Miscellaneous notes](#miscellaneous-notes) @@ -48,17 +47,18 @@ The library is split into **three layers**, providing **four kinds of features** We believe syntactic macros are [*the nuclear option of software engineering*](https://www.factual.com/blog/thinking-in-clojure-for-java-programmers-part-2/). Accordingly, we aim to [minimize macro magic](https://macropy3.readthedocs.io/en/latest/discussion.html#minimize-macro-magic). If a feature can be implemented - *with a level of usability on par with pythonic standards* - without resorting to macros, then it belongs in the pure-Python layer. (The one exception is when building the feature as a macro is the *simpler* solution. Consider `unpythonic.amb.forall` (overly complicated, to avoid macros) vs. `unpythonic.syntax.forall` (a clean macro-based design of the same feature) as an example. Keep in mind [ZoP](https://www.python.org/dev/peps/pep-0020/) §17 and §18.) -When that is not possible, we implement the actual feature as a pure-Python core, not meant for direct use, and provide a macro layer on top. The purpose of the macro layer is then to improve usability, by eliminating the [accidental complexity](https://en.wikipedia.org/wiki/No_Silver_Bullet) from the user interface of the pure-Python core. Examples are *automatic* currying, *automatic* tail-call optimization, and (beside a much leaner syntax) lexical scoping for the ``let`` and ``do`` constructs. We believe a well-designed macro layer can bring a difference in user experience similar to that between programming in [Brainfuck](https://en.wikipedia.org/wiki/Brainfuck) (or to be fair, in Fortran or in Java) versus in Python. +When that is not possible, we implement the actual feature as a pure-Python core, not meant for direct use, and provide a macro layer on top. The purpose of the macro layer is then to improve usability, by eliminating the [accidental complexity](https://en.wikipedia.org/wiki/No_Silver_Bullet) from the user interface of the pure-Python core. Examples are *automatic* currying, *automatic* tail-call optimization, and (beside a much leaner syntax) lexical scoping for the `let` and `do` constructs. We believe a well-designed macro layer can bring a difference in user experience similar to that between programming in [Brainfuck](https://en.wikipedia.org/wiki/Brainfuck) (or to be fair, in Fortran or in Java) versus in Python. Finally, when the whole purpose of the feature is to automatically transform a piece of code into a particular style (`continuations`, `lazify`, `autoreturn`), or when run-time access to the original [AST](https://en.wikipedia.org/wiki/Abstract_syntax_tree) is essential to the purpose (`dbg`), then the feature belongs squarely in the macro layer, with no pure-Python core underneath. When to implement your own feature as a syntactic macro, see the discussion in Chapter 8 of [Paul Graham: On Lisp](http://paulgraham.com/onlisp.html). MacroPy's documentation also provides [some advice on the topic](https://macropy3.readthedocs.io/en/latest/discussion.html). + ## Macros do not Compose Making macros work together is nontrivial, essentially because *macros don't compose*. [As pointed out by John Shutt](https://fexpr.blogspot.com/2013/12/abstractive-power.html), in a multilayered language extension implemented with macros, the second layer of macros needs to understand all of the first layer. The issue is that the macro abstraction leaks the details of its expansion. Contrast with functions, which operate on values: the process that was used to arrive at a value doesn't matter. It's always possible for a function to take this value and transform it into another value, which can then be used as input for the next layer of functions. That's composability at its finest. -The need for interaction between macros may arise already in what *feels* like a single layer of abstraction; for example, it's not only that the block macros must understand ``let[]``, but some of them must understand other block macros. This is because what feels like one layer of abstraction is actually implemented as a number of separate macros, which run in a specific order. Thus, from the viewpoint of actually applying the macros, if the resulting software is to work correctly, the mere act of allowing combos between the block macros already makes them into a multilayer system. The compartmentalization of conceptually separate features into separate macros facilitates understanding and maintainability, but fails to reach the ideal of modularity. +The need for interaction between macros may arise already in what *feels* like a single layer of abstraction; for example, it's not only that the block macros must understand `let[]`, but some of them must understand other block macros. This is because what feels like one layer of abstraction is actually implemented as a number of separate macros, which run in a specific order. Thus, from the viewpoint of actually applying the macros, if the resulting software is to work correctly, the mere act of allowing combos between the block macros already makes them into a multilayer system. The compartmentalization of conceptually separate features into separate macros facilitates understanding and maintainability, but fails to reach the ideal of modularity. Therefore, any particular combination of macros that has not been specifically tested might not work. That said, if some particular combo doesn't work and *is not at least documented as such*, that's an error; please raise an issue. The unit tests should cover the combos that on the surface seem the most useful, but there's no guarantee that they cover everything that actually is useful somewhere. @@ -66,44 +66,19 @@ Some aspects in the design of `unpythonic` could be simplified by expanding macr The lack of composability is a problem mainly when using macros to create a language extension, because the features of the extended language often interact. Macros can also be used in a much more everyday way, where composability is mostly a non-issue - to abstract and name common patterns that just happen to be of a nature that cannot be extracted as a regular function. See [Peter Seibel: Practical Common Lisp, chapter 3](http://www.gigamonkeys.com/book/practical-a-simple-database.html) for an example. + ## Language Discontinuities The very act of extending a language creates points of discontinuity between the extended language and the original. This can become a particularly bad source of extra complexity, if the extension can be enabled locally for a piece of code - as is the case with block macros. Then the design of the extended language must consider how to treat interactions between pieces of code that use the extension and those that don't. Then exponentiate those design considerations by the number of extensions that can be enabled independently. This issue is simply absent when designing a new language from scratch. For an example, look at what the rest of `unpythonic` has to do to make `lazify` behave as the user expects! Grep the codebase for `lazyutil`; especially the `passthrough_lazy_args` decorator, and its sister, the utility `maybe_force_args`. The decorator is essentially just an annotation for the `lazify` transformer, that marks a function as *not necessarily needing* evaluation of its arguments. Such functions often represent language-level constructs, such as `let` or `curry`, that essentially just *pass through* user data to other user-provided code, without *accessing* that data. The annotation is honored by the compiler when programming in the lazy (call-by-need) extended language, and otherwise it does nothing. Another pain point is the need of a second trampoline implementation (that only differs in one minor detail) just to make `lazify` interact correctly with TCO (while not losing an order of magnitude of performance in the trampoline used with standard Python). -For another example, it's likely that e.g. `continuations` still doesn't integrate completely seamlessly - and I'm not sure if that is possible even in principle. Calling a traditional function from a [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style) function is no problem; the traditional function uses no continuations, and (barring exceptions) will always return normally. The other way around can be a problem. Also, having TCO implemented as a trampoline system on top of the base language (instead of being already provided under the hood, like in Scheme) makes the `continuations` transformer more complex than absolutely necessary. +For another example, it is likely that e.g. `continuations` still does not integrate completely seamlessly - and I am not sure if that is possible even in principle. Calling a traditional function from a [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style) function is no problem; the traditional function uses no continuations, and (barring exceptions) will always return normally. The other way around can be a problem. Also, having TCO implemented as a trampoline system on top of the base language (instead of being already provided under the hood, like in Scheme) makes the `continuations` transformer more complex than absolutely necessary. For a third example, consider *decorated lambdas*. This is an `unpythonic` extension - essentially, a compiler feature implemented (by calling some common utility code) by each of the transformers of the pure-macro features - that understands a lambda enclosed in a nested sequence of single-argument function calls *as a decorated function definition*. This is painful, because the Python AST has no place to store the decorator list for a lambda; Python sees it just as a nested sequence of function calls, terminating in a lambda. This has to be papered over by the transformers. We also introduce a related complication, the decorator registry (see `regutil`), so that we can automatically sort decorator invocations - so that pure-macro features know at which index to inject a particular decorator (so it works properly) when they need to do that. Needing such a registry is already a complication, but the *decorated lambda* machinery feels the pain more acutely. -## What Belongs in Python? - -If you feel [my hovercraft is full of eels](http://stupidpythonideas.blogspot.com/2015/05/spam-spam-spam-gouda-spam-and-tulips.html), it is because they come with the territory. - -Some have expressed the opinion [the statement-vs-expression dichotomy is a feature](http://stupidpythonideas.blogspot.com/2015/01/statements-and-expressions.html). The BDFL himself has famously stated that TCO has no place in Python [[1]](http://neopythonic.blogspot.com/2009/04/tail-recursion-elimination.html) [[2]](http://neopythonic.blogspot.fi/2009/04/final-words-on-tail-calls.html), and less famously that multi-expression lambdas or continuations have no place in Python [[3]](https://www.artima.com/weblogs/viewpost.jsp?thread=147358). Several potentially interesting PEPs have been deferred [[1]](https://www.python.org/dev/peps/pep-3150/) [[2]](https://www.python.org/dev/peps/pep-0403/) or rejected [[3]](https://www.python.org/dev/peps/pep-0511/) [[4]](https://www.python.org/dev/peps/pep-0463/) [[5]](https://www.python.org/dev/peps/pep-0472/). - -In general, I like Python, and my hat's off to the devs. It's no mean feat to create a high-level language that focuses on readability and approachability, keep it alive for 30 years and counting, and have a large part of the programming community adopt it. But regarding the particular points above, if I agreed, I wouldn't be doing this, or [`mcpyrate`](https://github.com/Technologicat/mcpyrate) either. - -I think that with macros, Python can be so much more than just a beginner's language, and that language-level extensibility is the logical endpoint of that. I don't get the sentiment against metaprogramming, or toward some language-level features. For me, macros (and full-module transforms a.k.a. dialects) are just another tool for creating abstractions, at yet another level. We can already extract procedures, methods, and classes. Why limit that ability - namely, the ability to create abstractions - to what an [eager](https://en.wikipedia.org/wiki/Evaluation_strategy#Strict_evaluation) language can express at run time? If the point is to keep code understandable, then it's a matter of education. It's perfectly possible to write unreadable code without macros, and in Python, no less. And it's perfectly possible to write readable code with macros. I'm willing to admit the technical objection that *macros don't compose*; but that doesn't make them useless. - -Of the particular points above, in my opinion TCO should at least be an option. I like that *by default*, Python will complain about a call stack overflow rather than hang, when entering an accidentally infinite mutual recursion. I do occasionally make such mistakes when developing complex algorithms. But sometimes, I'd like to enable TCO selectively. If you ask for it, you know what to expect. This is precisely why `unpythonic.syntax` has `with tco`. I'm not very happy with having a custom TCO layer on top of a language core that doesn't like the idea, because TCO support in the core (like Scheme and Racket have) would simplify the implementation of certain other language extensions; but then again, [this is exactly what Clojure did](https://clojuredocs.org/clojure.core/trampoline), too. - -I think a multi-expression `lambda` is, on the surface, a good idea, but really the issue is that Python's `lambda` construct itself is broken. It's essentially a duplicate of `def`, but lacking some features. We would be much better off if `def` was an expression. Much of the time, anonymous functions aren't such a great idea, but defining closures inline is - and sometimes, the most readily understandable presentation order for an algorithm requires to do that in an expression position. The convenience is similar to being able to nest `def` statements, an ability Python already has. (Also, why are lambdas strictly anonymous? In cases where it is useful to be able to omit a name (because sometimes there are many small helpers and [naming is hard](https://martinfowler.com/bliki/TwoHardThings.html)), why not include the source location information in the auto-generated name, instead of just `""`?) - -The macros in `unpythonic.syntax` inject lots of lambdas, because that makes them much simpler to implement than if we had to always lift a `def` statement into the nearest enclosing statement context. Another case in point is [`pampy`](https://github.com/santinic/pampy). The code to perform a pattern match would read a lot nicer if you could define also slightly more complex actions inline (see [Racket's pattern matcher](https://docs.racket-lang.org/reference/match.html) for a comparison). It's unlikely you'll need the action functions elsewhere, and it's just silly to define a bunch of functions *before* the call to `match`. If this isn't a job for either something like `let-where` (to invert the presentation order locally) or multi-expression lambdas (to define the actions inline), I don't know what is. - -On a point raised [here](https://www.artima.com/weblogs/viewpost.jsp?thread=147358) with respect to indentation-sensitive vs. indentation-insensitive parser modes, having seen [SRFI-110: Sweet-expressions (t-expressions)](https://srfi.schemers.org/srfi-110/srfi-110.html), I think Python is confusing matters by linking the parser mode to statements vs. expressions. A workable solution is to make *everything* support both modes (or even preprocess the source code text to use only one of the modes), which *uniformly* makes parentheses an alternative syntax for grouping. -It would be nice to be able to use indentation to structure expressions to improve their readability, like one can do in Racket with [sweet](https://docs.racket-lang.org/sweet/), but I suppose ``lambda x: [expr0, expr1, ...]`` will have to do for a multi-expression lambda. Unless I decide at some point to make a source filter for [`mcpyrate`](https://github.com/Technologicat/mcpyrate) to auto-convert between indentation and parentheses; but for Python this is somewhat difficult to do, because statements **must** use indentation whereas expressions **must** use parentheses, and this must be done before we can invoke the standard parser to produce an AST. (And I don't want to maintain a [Pyparsing](https://github.com/pyparsing/pyparsing) grammar to parse a modified version of Python.) - -As for true multi-shot continuations... `unpythonic.syntax` has `with continuations` for that, but I'm not sure if I'll ever use it in production code. Most of the time, it seems to me full continuations are a solution looking for a problem. However, the feature is great to have for teaching the concept of continuations in a programming course, when teaching in Python. For everyday use, one-shot continuations (a.k.a. resumable functions, a.k.a. Python's generators) are often all that's needed to simplify certain patterns, especially those involving backtracking. I'm a big fan of the idea that, for example, you can make your anagram-making algorithm only yield valid anagrams, with the backtracking state (to eliminate dead-ends) implicitly stored in the paused generator! - -Finally, how about subtly incompatible Python-like languages (see the rejected [PEP 511](https://www.python.org/dev/peps/pep-0511/))? It is pretty much the point of language-level extensibility, to allow users to do that if they want. I wouldn't worry about it. Racket is *designed* for extensibility, and its community seems to be doing just fine - they even *encourage* the creation of new languages to solve problems. On the other hand, Racket demands some sophistication on the part of its users, and it is not very popular; it's hard to say what the programming community at large would do with an extensible language. - -What I can say is, `unpythonic` is not meant for the average Python project, either. But if used intelligently, it can make your code shorter, yet readable. Obviously, in a large project with a high developer turnover, the optimal solution looks different. - - -## Killer features of Common Lisp +## `unpythonic` and the Killer Features of Common Lisp In my opinion, Common Lisp has three legendary killer features: @@ -133,45 +108,35 @@ But for those of us that [don't like parentheses](https://srfi.schemers.org/srfi - PyPy (the JIT-enabled Python interpreter) itself is not the full story; the [RPython](https://rpython.readthedocs.io/en/latest/) toolchain from the PyPy project can *automatically produce a JIT for an interpreter for any new dynamic language implemented in the RPython language* (which is essentially a restricted dialect of Python 2.7). Now **that's** higher-order magic if anything is. - For the use case of numerics specifically, instead of Python, [Julia](https://docs.julialang.org/en/v1/manual/methods/) may be a better fit for writing high-level, yet performant code. It's a spiritual heir of Common Lisp, Fortran, *and Python*. Compilation to efficient machine code, with the help of gradual typing and automatic type inference, is a design goal. -## Common Lisp, Python, and productivity - -The various essays by Paul Graham, especially [Revenge of the Nerds (2002)](http://paulgraham.com/icad.html), have given the initial impulse to many programmers for studying Lisp. The essays are well written and have provided a lot of exposure for Lisp. So how does the programming world look in that light now, 20 years later? - -The base abstraction level of programming languages, even those in popular use, has increased. The trend was visible already then, and was indeed noted in the essays. The focus on low-level languages such as C++ has decreased. Java is still popular, but high-level FP languages that compile to JVM bytecode (Kotlin, Scala, Clojure) are rising. - -Python has become highly popular, and is now also closer to Lisp than it was 20 years ago, especially after `MacroPy` introduced syntactic macros to Python (in 2013, [according to the git log](https://github.com/lihaoyi/macropy/commits/python2/macropy/__init__.py)). Python wasn't bad as a Lisp replacement even back in 2000 - see Peter Norvig's essay [Python for Lisp Programmers](https://norvig.com/python-lisp.html). Some more historical background, specifically on lexically scoped closures (and the initial lack thereof), can be found in [PEP 3104](https://www.python.org/dev/peps/pep-3104/), [PEP 227](https://www.python.org/dev/peps/pep-0227/), and [Historical problems with closures in JavaScript and Python](http://giocc.com/problems-with-closures-in-javascript-and-python.html). - -In 2020, does it still make sense to learn [the legendary](https://xkcd.com/297/) Common Lisp? - -To know exactly what it has to offer, yes. As baroque as some parts are, there are a lot of great ideas there. [Conditions](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html) are one. [CLOS](http://www.gigamonkeys.com/book/object-reorientation-generic-functions.html) is another. (Nowadays [Julia](https://docs.julialang.org/en/v1/manual/methods/) has CLOS-style [multiple-dispatch generic functions](https://docs.julialang.org/en/v1/manual/methods/).) More widely, in the ecosystem, Swank is one. Having more perspectives at one's disposal makes one a better programmer. - -But as a practical tool? Is CL hands-down better than Python? Maybe no. Python has already delivered on 90% of the productivity promise of Lisp. Both languages cut down significantly on [accidental complexity](https://en.wikipedia.org/wiki/No_Silver_Bullet). Python has a huge library ecosystem. [`mcpyrate`](https://github.com/Technologicat/mcpyrate) and `unpythonic` are trying to push the language-level features a further 5%. (A full 100% is likely impossible when extending an existing language; if nothing else, there will be seams.) - -As for productivity, [it may be](https://medium.com/smalltalk-talk/lisp-smalltalk-and-the-power-of-symmetry-8bd96aaa0c0c) that a form of code-data equivalence (symmetry!), not macros specifically, is what makes Lisp powerful. If so, there may be other ways to reach that equivalence. For example Smalltalk, like Lisp, *runs in the same context it's written in*. All Smalltalk data are programs. Smalltalk [may be making a comeback](https://hackernoon.com/how-to-evangelize-a-programming-language-0p7p3y02), in the form of [Pharo](https://pharo.org/). - -Haskell aims at code-data equivalence from a third angle (memoized pure functions are in essence infinite lookup tables), but I haven't used it in practice, so I don't have the experience to say whether this is enough to make it feel powerful in the same way. - -Image-based programming (live programming) is a common factor between Pharo and Common Lisp + Swank. This is another productivity booster that much of the programming world isn't that familiar with. It eliminates not only the edit/compile/restart cycle, but the edit/restart cycle as well, making the workflow a concurrent *edit/run* instead (without restarting the whole app at each change). Julia has [Revise.jl](https://github.com/timholy/Revise.jl) for something similar. ## Python is not a Lisp -The point behind providing `let` and `begin` (and the ``let[]`` and ``do[]`` [macros](macros.md)) is to make Python lambdas slightly more useful - which was really the starting point for the whole `unpythonic` experiment. - -The oft-quoted single-expression limitation of the Python ``lambda`` is ultimately a herring, as this library demonstrates. The real problem is the statement/expression dichotomy. In Python, the looping constructs (`for`, `while`), the full power of `if`, and `return` are statements, so they cannot be used in lambdas. (This observation has been earlier made by others, too; see e.g. the [Wikipedia page on anonymous functions](https://en.wikipedia.org/wiki/Anonymous_function#Python).) We can work around some of this: - - - The expr macro ``cond[]`` gives us a general ``if``/``elif``/``else`` expression. - - Without it, the expression form of `if` (that Python already has) could be used, but readability suffers if nested, since it has no ``elif``. Actually, [`and` and `or` are sufficient for full generality](https://www.ibm.com/developerworks/library/l-prog/), but readability suffers even more. - - So we use macros to define a ``cond`` expression, essentially duplicating a feature the language already almost has. See [our macros](macros.md). - - Functional looping (with TCO, to boot) is possible. See the constructs in ``unpythonic.fploop``. - - ``unpythonic.ec.call_ec`` gives us ``return`` (the ec). - - ``unpythonic.misc.raisef`` gives us ``raise``, and ``unpythonic.misc.tryf`` gives us ``try``/``except``/``else``/``finally``. - - A lambda can be named (``unpythonic.misc.namelambda``, with some practical limitations on the fully qualified name of nested lambdas). - - Even an anonymous function can recurse with some help (``unpythonic.fun.withself``). - - Context management (``with``) is currently **not** available for lambdas, even in ``unpythonic``. +The point behind providing `let` and `begin` (and the `let[]` and `do[]` [macros](macros.md)) is to make Python lambdas slightly more useful - which was really the starting point for the whole `unpythonic` experiment. + +The oft-quoted single-expression limitation of the Python `lambda` is ultimately a herring, as this library demonstrates. The real problem is the statement/expression dichotomy. In Python, the looping constructs (`for`, `while`), the full power of `if`, and `return` are statements, so they cannot be used in lambdas. (This observation has been earlier made by others, too; see e.g. the [Wikipedia page on anonymous functions](https://en.wikipedia.org/wiki/Anonymous_function#Python).) We can work around some of this: + + - The expr macro `do[]` gives us sequencing, i.e. allows to use, in any expression position, multiple expressions that run in the specified order. + - The expr macro `cond[]` gives us a general `if`/`elif`/`else` expression. + - Without it, the expression form of `if` (that Python already has) could be used, but readability suffers if nested, since it has no `elif`. Actually, [`and` and `or` are sufficient for full generality](https://www.ibm.com/developerworks/library/l-prog/), but readability suffers even more. + - So we use macros to define a `cond` expression, essentially duplicating a feature the language already almost has. See [our macros](macros.md). + - Functional looping (with TCO) gives us equivalents of `for` and `while`. See the constructs in `unpythonic.fploop`, particularly `looped` and `breakably_looped`. + - `unpythonic.ec.call_ec` gives us `return` (the ec). + - `unpythonic.excutil.raisef` gives us `raise`, `unpythonic.excutil.tryf` gives us `try`/`except`/`else`/`finally`, and `unpythonic.excutil.withf` gives us `with`. + - A lambda can be named, see `unpythonic.misc.namelambda`. + - There are some practical limitations on the fully qualified name of nested lambdas. + - Note this does not bind the name to an identifier at the use site, so the name cannot be used to recurse. The point is that the name is available for inspection, and it will show in tracebacks. + - A lambda can recurse using `unpythonic.fun.withself`. You will get a `self` argument that points to the lambda itself, and is passed implicitly, like `self` usually in Python. + - A lambda can define a class using the three-argument form of the builtin `type` function. For an example, see [Peter Corbett (2005): Statementless Python](https://gist.github.com/brool/1679908), a complete minimal Lisp interpreter implemented as a single Python expression. + - A lambda can import a module using the builtin `__import__`, or better, `importlib.import_module`. + - A lambda can assert by using an if-expression and then `raisef` to actually raise the `AssertionError`. + - Or use the `test[]` macro, which also shows the source code for the asserted expression if the assertion fails. + - Technically, `test[]` will `signal` the `TestFailure` (part of the public API of `unpythonic.test.fixtures`), not raise it, but essentially, `test[]` is a more convenient assert that optionally hooks into a testing framework. The error signal, if unhandled, will automatically chain into raising a `ControlError` exception, which is often just fine. + - Context management (`with`) is available in expression position via `unpythonic.excutil.withf`. Aside from the `async` stuff, this was the last hold-out preventing full generality of lambdas. Still, ultimately one must keep in mind that Python is not a Lisp. Not all of Python's standard library is expression-friendly; some standard functions and methods lack return values - even though a call is an expression! For example, `set.add(x)` returns `None`, whereas in an expression context, returning `x` would be much more useful, even though it does have a side effect. -## On ``let`` and Python + +## On `let` and Python Why no `let*`, as a function? In Python, name lookup always occurs at runtime. Python gives us no compile-time guarantees that no binding refers to a later one - in [Racket](http://racket-lang.org/), this guarantee is the main difference between `let*` and `letrec`. @@ -181,17 +146,18 @@ In contrast, in a `let*` form, attempting such a definition is *a compile-time e Our `letrec` behaves like `let*` in that if `valexpr` is not a function, it may only refer to bindings above it. But this is only enforced at run time, and we allow mutually recursive function definitions, hence `letrec`. -Note the function versions of our `let` constructs, in the pure-Python API, are **not** properly lexically scoped; in case of nested ``let`` expressions, one must be explicit about which environment the names come from. +Note the function versions of our `let` constructs, in the pure-Python API, are **not** properly lexically scoped; in case of nested `let` expressions, one must be explicit about which environment the names come from. -The [macro versions](macros.md) of the `let` constructs **are** lexically scoped. The macros also provide a ``letseq[]`` that, similarly to Racket's ``let*``, gives a compile-time guarantee that no binding refers to a later one. +The [macro versions](macros.md) of the `let` constructs **are** lexically scoped. The macros also provide a `letseq[]` that, similarly to Racket's `let*`, gives a compile-time guarantee that no binding refers to a later one. Inspiration: [[1]](https://nvbn.github.io/2014/09/25/let-statement-in-python/) [[2]](https://stackoverflow.com/questions/12219465/is-there-a-python-equivalent-of-the-haskell-let) [[3]](http://sigusr2.net/more-about-let-in-python.html). + ## Assignment syntax Why the clunky `e.set("foo", newval)` or `e << ("foo", newval)`, which do not directly mention `e.foo`? This is mainly because in Python, the language itself is not customizable. If we could define a new operator `e.foo newval` to transform to `e.set("foo", newval)`, this would be easily solved. -Our [macros](macros.md) essentially do exactly this, but by borrowing the ``<<`` operator to provide the syntax ``foo << newval``, because even with macros, it is not possible to define new [BinOp](https://greentreesnakes.readthedocs.io/en/latest/nodes.html#BinOp)s in Python. That **is** possible essentially as a *reader macro* (as it's known in the Lisp world), to transform custom BinOps into some syntactically valid Python code before proceeding with the rest of the import machinery, but it seems as of this writing, no one has done this. +Our [macros](macros.md) essentially do exactly this, but by borrowing the `<<` operator to provide the syntax `foo << newval`, because even with macros, it is not possible to define new [BinOp](https://greentreesnakes.readthedocs.io/en/latest/nodes.html#BinOp)s in Python. That **is** possible essentially as a *reader macro* (as it's known in the Lisp world), to transform custom BinOps into some syntactically valid Python code before proceeding with the rest of the import machinery, but it seems as of this writing, no one has done this. If you want a framework to play around with reader macros in Python, see [`mcpyrate`](https://github.com/Technologicat/mcpyrate). You'll still have to write a parser, where [Pyparsing](https://github.com/pyparsing/pyparsing) may help; but supporting something as complex as a customized version of the surface syntax of Python is still a lot of work, and may quickly go out of date. (You'll want to look at the official [full grammar specification](https://docs.python.org/3/reference/grammar.html), as well as the source code linked therein.) @@ -206,41 +172,50 @@ If we later choose go this route nevertheless, `<<` is a better choice for the s The current solution for the assignment syntax issue is to use macros, to have both clean syntax at the use site and a relatively hackfree implementation. + ## TCO syntax and speed -Benefits and costs of ``return jump(...)``: +Benefits and costs of `return jump(...)`: - - Explicitly a tail call due to ``return``. - - The trampoline can be very simple and (relatively speaking) fast. Just a dumb ``jump`` record, a ``while`` loop, and regular function calls and returns. - - The cost is that ``jump`` cannot detect whether the user forgot the ``return``, leaving a possibility for bugs in the client code (causing an FP loop to immediately exit, returning ``None``). Unit tests of client code become very important. + - Explicitly a tail call due to `return`. + - The trampoline can be very simple and (relatively speaking) fast. Just a dumb `jump` record, a `while` loop, and regular function calls and returns. + - The cost is that `jump` cannot detect whether the user forgot the `return`, leaving a possibility for bugs in the client code (causing an FP loop to immediately exit, returning `None`). Unit tests of client code become very important. - This is somewhat mitigated by the check in `__del__`, but it can only print a warning, not stop the incorrect program from proceeding. - - We could mandate that trampolined functions must not return ``None``, but: - - Uniformity is lost between regular and trampolined functions, if only one kind may return ``None``. + - We could mandate that trampolined functions must not return `None`, but: + - Uniformity is lost between regular and trampolined functions, if only one kind may return `None`. - This breaks the *don't care about return value* use case, which is rather common when using side effects. - - Failing to terminate at the intended point may well fall through into what was intended as another branch of the client code, which may correctly have a ``return``. So this would not even solve the problem. + - Failing to terminate at the intended point may well fall through into what was intended as another branch of the client code, which may correctly have a `return`. So this would not even solve the problem. -The other simple-ish solution is to use exceptions, making the jump wrest control from the caller. Then ``jump(...)`` becomes a verb, but this approach is 2-5x slower, when measured with a do-nothing loop. (See the old default TCO implementation in v0.9.2.) +The other simple-ish solution is to use exceptions, making the jump wrest control from the caller. Then `jump(...)` becomes a verb, but this approach is 2-5x slower, when measured with a do-nothing loop. (See the old default TCO implementation in v0.9.2.) -Our [macros](macros.md) provide an easy-to use solution. Just wrap the relevant section of code in a ``with tco:``, to automatically apply TCO to code that looks exactly like standard Python. With the macro, function definitions (also lambdas) and returns are automatically converted. It also knows enough not to add a ``@trampolined`` if you have already declared a ``def`` as ``@looped`` (or any of the other TCO-enabling decorators in ``unpythonic.fploop``, or ``unpythonic.fix.fixtco``). +Our [macros](macros.md) provide an easy-to use solution. Just wrap the relevant section of code in a `with tco:`, to automatically apply TCO to code that looks exactly like standard Python. With the macro, function definitions (also lambdas) and returns are automatically converted. It also knows enough not to add a `@trampolined` if you have already declared a `def` as `@looped` (or any of the other TCO-enabling decorators in `unpythonic.fploop`, or `unpythonic.fix.fixtco`). For other libraries bringing TCO to Python, see: - [tco](https://github.com/baruchel/tco) by Thomas Baruchel, based on exceptions. - - [ActiveState recipe 474088](https://github.com/ActiveState/code/tree/master/recipes/Python/474088_Tail_Call_Optimization_Decorator), based on ``inspect``. - - ``recur.tco`` in [fn.py](https://github.com/fnpy/fn.py), the original source of the approach used here. - - [MacroPy](https://github.com/azazel75/macropy) uses an approach similar to ``fn.py``. + - [ActiveState recipe 474088](https://github.com/ActiveState/code/tree/master/recipes/Python/474088_Tail_Call_Optimization_Decorator), based on `inspect`. + - `recur.tco` in [fn.py](https://github.com/fnpy/fn.py), the original source of the approach used here. + - [MacroPy](https://github.com/azazel75/macropy) uses an approach similar to `fn.py`. + + +## Monads + +*Added in v2.1.0.* + +We provide have [`unpythonic.monads`](../unpythonic/monads/) and [`unpythonic.syntax.monadic_do`](../unpythonic/syntax/monadic_do.py). -## No Monads? +For understanding monads, read Stephan Boyer's quick introduction [[part 1]](https://www.stephanboyer.com/post/9/monads-part-1-a-design-pattern) [[part 2]](https://www.stephanboyer.com/post/10/monads-part-2-impure-computations) [[super quick intro]](https://www.stephanboyer.com/post/83/super-quick-intro-to-monads) and figure it out, it's easy. (Until you get to `State` and `Reader`, where [this](http://brandon.si/code/the-state-monad-a-tutorial-for-the-confused/) and maybe [this](https://gaiustech.wordpress.com/2010/09/06/on-monads/) can be helpful.) -(Beside List inside ``forall``.) +If you don't need the language kitchen sink that is `unpythonic`, there are also specialized monad libraries for Python, such as [OSlash](https://github.com/dbrattli/OSlash). -Admittedly unpythonic, but Haskell feature, not Lisp. Besides, already done elsewhere, see [OSlash](https://github.com/dbrattli/OSlash) if you need them. -If you want to roll your own monads for whatever reason, there's [this silly hack](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/monads.py) that wasn't packaged into this; or just read Stephan Boyer's quick introduction [[part 1]](https://www.stephanboyer.com/post/9/monads-part-1-a-design-pattern) [[part 2]](https://www.stephanboyer.com/post/10/monads-part-2-impure-computations) [[super quick intro]](https://www.stephanboyer.com/post/83/super-quick-intro-to-monads) and figure it out, it's easy. (Until you get to `State` and `Reader`, where [this](http://brandon.si/code/the-state-monad-a-tutorial-for-the-confused/) and maybe [this](https://gaiustech.wordpress.com/2010/09/06/on-monads/) can be helpful.) +## Types -## No Types? +*Changed in v2.1.0.* -The `unpythonic` project will likely remain untyped indefinitely, since I don't want to enter that particular marshland with things like `curry` and `with continuations`. It may be possible to gradually type some carefully selected parts - but that's currently not on [the roadmap](https://github.com/Technologicat/unpythonic/milestones). I'm not against it, if someone wants to contribute. +We now provide type annotations for most of `unpythonic`'s pure-Python layer. + +The remaining modules will likely remain untyped indefinitely, since I don't want to enter that particular marshland with things like `curry` and `with continuations`. That said, I'm not against the idea, if someone wants to contribute. In general, on type systems, [this three-part discussion on LtU](http://lambda-the-ultimate.org/node/220) was interesting: @@ -263,58 +238,65 @@ More on type systems: - Serious about types? [Bartosz Milewski: Category Theory for Programmers](https://bartoszmilewski.com/2014/10/28/category-theory-for-programmers-the-preface/) (online book) - [Chris Smith: What To Know Before Debating Type Systems](http://blogs.perl.org/users/ovid/2010/08/what-to-know-before-debating-type-systems.html) - [Martin Fowler on dynamic typing](https://www.martinfowler.com/bliki/DynamicTyping.html) -- Do we need types? At least John Shutt (the author of the [Kernel](https://web.cs.wpi.edu/~jshutt/kernel.html) programming language) seems to think we don't: [Where do types come from?](http://fexpr.blogspot.com/2011/11/where-do-types-come-from.html) +- Do we need types? At least John Shutt (the author of the [Kernel](https://web.cs.wpi.edu/~jshutt/kernel.html) programming language) thought that we don't: [Where do types come from?](http://fexpr.blogspot.com/2011/11/where-do-types-come-from.html) - In physics, units as used for dimension analysis are essentially a form of static typing. - This has been discussed on LtU, see e.g. [[1]](http://lambda-the-ultimate.org/node/33) [[2]](http://lambda-the-ultimate.org/classic/message11877.html). + ## Detailed Notes on Macros - - ``continuations`` and ``tco`` are mutually exclusive, since ``continuations`` already implies TCO. - - However, the ``tco`` macro skips any ``with continuations`` blocks inside it, **for the specific reason** of allowing modules written in the [Lispython dialect](https://github.com/Technologicat/pydialect) (which implies TCO for the whole module) to use ``with continuations``. + - `continuations` and `tco` are mutually exclusive, since `continuations` already implies TCO. + - However, the `tco` macro skips any `with continuations` blocks inside it, **for the specific reason** of allowing modules written in the [Lispython dialect](https://github.com/Technologicat/pydialect) (which implies TCO for the whole module) to use `with continuations`. - - ``prefix``, ``autoreturn``, ``quicklambda`` and ``multilambda`` expand outside-in, because they change the semantics: - - ``prefix`` transforms things-that-look-like-tuples into function calls, - - ``autoreturn`` adds ``return`` statements where there weren't any, - - ``quicklambda`` transforms things-that-look-like-list-lookups into ``lambda`` function definitions, - - ``multilambda`` transforms things-that-look-like-lists (in the body of a ``lambda``) into sequences of multiple expressions, using ``do[]``. + - `prefix`, `autoreturn`, `quicklambda` and `multilambda` expand outside-in, because they change the semantics: + - `prefix` transforms things-that-look-like-tuples into function calls, + - `autoreturn` adds `return` statements where there weren't any, + - `quicklambda` transforms things-that-look-like-list-lookups into `lambda` function definitions, + - `multilambda` transforms things-that-look-like-lists (in the body of a `lambda`) into sequences of multiple expressions, using `do[]`. - Hence, a lexically outer block of one of these types *will expand first*, before any macros inside it are expanded. - This yields clean, standard-ish Python for the rest of the macros, which then don't need to worry about their input meaning something completely different from what it looks like. - - An already expanded ``do[]`` (including that inserted by `multilambda`) is accounted for by all ``unpythonic.syntax`` macros when handling expressions. + - An already expanded `do[]` (including that inserted by `multilambda`) is accounted for by all `unpythonic.syntax` macros when handling expressions. - For simplicity, this is **the only** type of sequencing understood by the macros. - - E.g. the more rudimentary ``unpythonic.seq.begin`` is not treated as a sequencing operation. This matters especially in ``tco``, where it is critically important to correctly detect a tail position in a return-value expression or (multi-)lambda body. + - E.g. the more rudimentary `unpythonic.seq.begin` is not treated as a sequencing operation. This matters especially in `tco`, where it is critically important to correctly detect a tail position in a return-value expression or (multi-)lambda body. - *Sequencing* is here meant in the Racket/Haskell sense of *running sub-operations in a specified order*, unrelated to Python's *sequences*. - - The TCO transformation knows about TCO-enabling decorators provided by ``unpythonic``, and adds the ``@trampolined`` decorator to a function definition only when it is not already TCO'd. - - This applies also to lambdas; they are decorated by directly wrapping them with a call: ``trampolined(lambda ...: ...)``. - - This allows ``with tco`` to work together with the functions in ``unpythonic.fploop``, which imply TCO. + - The TCO transformation knows about TCO-enabling decorators provided by `unpythonic`, and adds the `@trampolined` decorator to a function definition only when it is not already TCO'd. + - This applies also to lambdas; they are decorated by directly wrapping them with a call: `trampolined(lambda ...: ...)`. + - This allows `with tco` to work together with the functions in `unpythonic.fploop`, which imply TCO. - - Macros that transform lambdas (notably ``continuations`` and ``tco``): + - Macros that transform lambdas (notably `continuations` and `tco`): - Perform an outside-in pass to take note of all lambdas that appear in the code *before the expansion of any inner macros*. Then in an inside-out pass, *after the expansion of all inner macros*, only the recorded lambdas are transformed. - This mechanism distinguishes between explicit lambdas in the client code, and internal implicit lambdas automatically inserted by a macro. The latter are a technical detail that should not undergo the same transformations as user-written explicit lambdas. - - The identification is based on the ``id`` of the AST node instance. Hence, if you plan to write your own macros that work together with those in ``unpythonic.syntax``, avoid going overboard with FP. Modifying the tree in-place, preserving the original AST node instances as far as sensible, is just fine. - - For the interested reader, grep the source code for ``userlambdas``. - - Support a limited form of *decorated lambdas*, i.e. trees of the form ``f(g(h(lambda ...: ...)))``. + - The identification is based on the `id` of the AST node instance. Hence, if you plan to write your own macros that work together with those in `unpythonic.syntax`, avoid going overboard with FP. Modifying the tree in-place, preserving the original AST node instances as far as sensible, is just fine. + - For the interested reader, grep the source code for `userlambdas`. + - Support a limited form of *decorated lambdas*, i.e. trees of the form `f(g(h(lambda ...: ...)))`. - The macros will reorder a chain of lambda decorators (i.e. nested calls) to use the correct ordering, when only known decorators are used on a literal lambda. - - This allows some combos such as ``tco``, ``unpythonic.fploop.looped``, ``autocurry``. - - Only decorators provided by ``unpythonic`` are recognized, and only some of them are supported. For details, see ``unpythonic.regutil``. - - If you need to combo ``unpythonic.fploop.looped`` and ``unpythonic.ec.call_ec``, use ``unpythonic.fploop.breakably_looped``, which does exactly that. - - The problem with a direct combo is that the required ordering is the trampoline (inside ``looped``) outermost, then ``call_ec``, and then the actual loop, but because an escape continuation is only valid for the dynamic extent of the ``call_ec``, the whole loop must be run inside the dynamic extent of the ``call_ec``. - - ``unpythonic.fploop.breakably_looped`` internally inserts the ``call_ec`` at the right step, and gives you the ec as ``brk``. - - For the interested reader, look at ``unpythonic.syntax.util``. + - This allows some combos such as `tco`, `unpythonic.fploop.looped`, `autocurry`. + - Only decorators provided by `unpythonic` are recognized, and only some of them are supported. For details, see `unpythonic.regutil`. + - If you need to combo `unpythonic.fploop.looped` and `unpythonic.ec.call_ec`, use `unpythonic.fploop.breakably_looped`, which does exactly that. + - The problem with a direct combo is that the required ordering is the trampoline (inside `looped`) outermost, then `call_ec`, and then the actual loop, but because an escape continuation is only valid for the dynamic extent of the `call_ec`, the whole loop must be run inside the dynamic extent of the `call_ec`. + - `unpythonic.fploop.breakably_looped` internally inserts the `call_ec` at the right step, and gives you the ec as `brk`. + - For the interested reader, look at `unpythonic.syntax.util`. - - ``namedlambda`` is a two-pass macro. In the outside-in pass, it names lambdas inside ``let[]`` expressions before they are expanded away. The inside-out pass of ``namedlambda`` must run after ``autocurry`` to analyze and transform the auto-curried code produced by ``with autocurry``. + - `namedlambda` is a two-pass macro. In the outside-in pass, it names lambdas inside `let[]` expressions before they are expanded away. The inside-out pass of `namedlambda` must run after `autocurry` to analyze and transform the auto-curried code produced by `with autocurry`. - - ``autoref`` does not need in its output to be curried (hence after ``autocurry`` to gain some performance), but needs to run before ``lazify``, so that both branches of each transformed reference get the implicit forcing. Its transformation is orthogonal to what ``namedlambda`` does, so it does not matter in which exact order these two run. + - `autoref` does not need in its output to be curried (hence after `autocurry` to gain some performance), but needs to run before `lazify`, so that both branches of each transformed reference get the implicit forcing. Its transformation is orthogonal to what `namedlambda` does, so it does not matter in which exact order these two run. - - ``lazify`` is a rather invasive rewrite that needs to see the output from most of the other macros. + - `lazify` is a rather invasive rewrite that needs to see the output from most of the other macros. - - ``envify`` needs to see the output of ``lazify`` in order to shunt function args into an unpythonic ``env`` without triggering the implicit forcing. + - `envify` needs to see the output of `lazify` in order to shunt function args into an unpythonic `env` without triggering the implicit forcing. - - With MacroPy, it used to be so that some of the block macros could be comboed as multiple context managers in the same ``with`` statement (expansion order is then *left-to-right*), whereas some (notably ``autocurry`` and ``namedlambda``) required their own ``with`` statement. In `mcpyrate`, block macros can be comboed in the same ``with`` statement (and expansion order is *left-to-right*). + - `nb` needs to determine whether an expression should be printed. + - It needs to see invocations of testing macros, because those are akin to asserts - while they are technically implemented as expr macros, they expand into function calls into test asserter functions that have no meaningful return value. Thus, just in case the user has requested testing macros to expand first, `nb` needs to expand before anything that may edit function calls, such as `tco` and `autocurry`. + - It needs to see bare expressions (technically, in the AST, an *expression statements* `ast.Expr`). Thus `nb` should expand before `autoreturn`, to treat also expressions that appear in tail position. + - `nb` performs the printing using a passthrough helper function, so that the value that was printed is available as the return value of the print helper, so that `return theprint(value)` works, for co-operation with `autoreturn`. + + - With MacroPy, it used to be so that some of the block macros could be comboed as multiple context managers in the same `with` statement (expansion order is then *left-to-right*), whereas some (notably `autocurry` and `namedlambda`) required their own `with` statement. In `mcpyrate`, block macros can be comboed in the same `with` statement (and expansion order is *left-to-right*). - See the relevant [issue report](https://github.com/azazel75/macropy/issues/21) and [PR](https://github.com/azazel75/macropy/pull/22). - - When in doubt, you can use a separate ``with`` statement for each block macro that applies to the same section of code, and nest the blocks. In ``mcpyrate``, this is almost equivalent to having the macros invoked in a single ``with`` statement, in the same order. - - Load the macro expansion debug utility `from mcpyrate.debug import macros, step_expansion`, and put a ``with step_expansion:`` around your use site. Then add your macro invocations one by one, and make sure the expansion looks like what you intended. (And of course, while testing, try to keep the input as simple as possible.) + - When in doubt, you can use a separate `with` statement for each block macro that applies to the same section of code, and nest the blocks. In `mcpyrate`, this is almost equivalent to having the macros invoked in a single `with` statement, in the same order. + - Load the macro expansion debug utility `from mcpyrate.debug import macros, step_expansion`, and put a `with step_expansion:` around your use site. Then add your macro invocations one by one, and make sure the expansion looks like what you intended. (And of course, while testing, try to keep the input as simple as possible.) + ## Miscellaneous notes diff --git a/doc/dialects.md b/doc/dialects.md index 90349ec2..478d249d 100644 --- a/doc/dialects.md +++ b/doc/dialects.md @@ -7,9 +7,12 @@ - [Lispython](dialects/lispython.md) - [Listhell](dialects/listhell.md) - [Pytkell](dialects/pytkell.md) + - [BF](dialects/bf.md) + - [Befunge](dialects/befunge.md) - [REPL server](repl.md) - [Troubleshooting](troubleshooting.md) - [Design notes](design-notes.md) +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) @@ -32,9 +35,11 @@ Hence *dialects*. As examples of what can be done with a dialects system together with a kitchen-sink language extension macro package such as `unpythonic`, we currently provide the following dialects: - [**Lispython**: The love child of Python and Scheme](dialects/lispython.md) - - [**Pytkell**: Because it's good to have a kell](dialects/pytkell.md) - [**Listhell**: It's not Lisp, it's not Python, it's not Haskell](dialects/listhell.md) + - [**Pytkell**: Because it's good to have a kell](dialects/pytkell.md) + - [**BF**: The classical human-incomprehensible automaton](dialects/bf.md) + - [**Befunge**: Two-dimensional, self-modifying, deeply confused](dialects/befunge.md) -All three dialects support `unpythonic`'s ``continuations`` block macro, to add ``call/cc`` to the language; but it is not enabled automatically. +Lispython, Listhell, and Pytkell are AST-transforming dialects, built on top of `unpythonic`'s macro layer. All three support `unpythonic`'s `continuations` block macro, to add `call/cc` to the language; but it is not enabled automatically. BF and Befunge are source-to-source compilers — the body of a BF or Befunge file is not parseable as Python — and demonstrate the other half of `mcpyrate`'s dialect system. The two source-level dialects make complementary points: BF compiles a 1-D character stream into legible structured Python (the dialect-as-transpiler model), while Befunge wraps a 2-D playfield in a runtime interpreter call (the dialect-as-reader model). Mostly, these dialects are intended as a cross between teaching material and a (fully functional!) practical joke, but Lispython may occasionally come in handy. diff --git a/doc/dialects/befunge.md b/doc/dialects/befunge.md new file mode 100644 index 00000000..fe5362b2 --- /dev/null +++ b/doc/dialects/befunge.md @@ -0,0 +1,124 @@ +**Navigation** + +- [README](../../README.md) +- [Pure-Python feature set](../features.md) +- [Syntactic macro feature set](../macros.md) +- [Examples of creating dialects using `mcpyrate`](../dialects.md) + - [Lispython](lispython.md) + - [Listhell](listhell.md) + - [Pytkell](pytkell.md) + - [BF](bf.md) + - **Befunge** +- [REPL server](../repl.md) +- [Troubleshooting](../troubleshooting.md) +- [Design notes](../design-notes.md) +- [Essays](../essays.md) +- [Additional reading](../readings.md) +- [Contribution guidelines](../../CONTRIBUTING.md) + + +**Table of Contents** + +- [Befunge: two-dimensional, self-modifying, deeply confused](#befunge-two-dimensional-self-modifying-deeply-confused) + - [Features](#features) + - [Errors](#errors) + - [What Befunge is](#what-befunge-is) + - [Contrast with BF](#contrast-with-bf) + - [Comboability](#comboability) + - [CAUTION](#caution) + - [Etymology?](#etymology) + + + +# Befunge: two-dimensional, self-modifying, deeply confused + +A [Befunge-93](https://en.wikipedia.org/wiki/Befunge) interpreter wrapped as a whole-module source dialect. + +Powered by [`mcpyrate`](https://github.com/Technologicat/mcpyrate/). + +```python +"""Hello from Befunge!""" + +from unpythonic.dialects.befunge import dialects, Befunge # noqa: F401 + +"!egnufeB morf olleH">:#,_@ +``` + +The body of the file (everything after the dialect-import) is parsed as a Befunge-93 playfield and run by the runtime interpreter shipped in this module. + +The recommended form for commentary is a **module docstring** above the dialect-import line, as shown — `help(some_befunge_module)` then displays it the same way it would for any documented Python module. Stand-alone `# …` comments above the dialect-import line work too. Below the dialect-import everything is the playfield; comments inside the body are not supported, because Befunge has no comment syntax (`#` is a real command — *trampoline / skip-next-cell*). + +## Features + + - **Strict Befunge-93**: 80 × 25 toroidal playfield, byte-valued cells, unbounded-int stack. + - The IP wraps toroidally on motion: off the right edge → column 0 of the same row, off the bottom → row 0 of the same column. + - Stack underflow on `pop` returns 0 (per spec). + - **All 93 commands supported**: + - Arithmetic: `+`, `-`, `*`, `/`, `%` (division and modulo by zero push 0, per spec). + - Comparison and logic: `!` (not), `` ` `` (greater-than). + - Direction: `>`, `<`, `^`, `v`, `?` (random). + - Conditional direction: `_` (pop; 0 → east, else west), `|` (pop; 0 → south, else north). + - String mode: `"` toggles. While on, every cell pushes `ord(ch)` instead of executing. + - Stack: `:` duplicate, `\` swap, `$` discard. + - I/O: `.` print int (followed by a space), `,` print char, `&` read int, `~` read char. + - Trampoline: `#` skip the next cell. + - Self-modifying: `g` (get cell value), `p` (put cell value); both pop `(x, y)` first. + - Halt: `@`. + - Space is a no-op. + - **`?` (random direction) is seedable for tests**: `run(src, *, seed=int)` uses a `random.Random` instance, so the seed reaches the dispatch and doesn't perturb the user's process-wide RNG. `seed=None` (the default) picks via OS entropy. + +The same interpreter is available as a plain function: + +```python +from unpythonic.dialects.befunge import run + +run(playfield_source) +``` + +I/O goes through `sys.stdin` / `sys.stdout`. To capture output in tests, wrap the call with `contextlib.redirect_stdout`; for input, use `unpythonic.redirect_stdin` (the third sibling — `contextlib` ships only the output redirectors). + +## Errors + +Three distinct conditions, three distinct exception types: + + - **`SyntaxError`** — the source is malformed at `Playfield(src)` construction time: more than 25 rows, or any line longer than 80 columns. Pre-execution. + - **`IndexError`** — runtime out-of-grid access via `g` or `p`. The IP itself never goes out of grid (its motion wraps toroidally); `IndexError` only fires on programs that compute their own coordinates and address outside the 80 × 25 bounds. + - **`UnknownOpcodeError`** (a `RuntimeError` subclass exported from `unpythonic.dialects.befunge`) — the IP visited a cell whose byte value isn't a recognized command. Includes both source-level typos and `p`-modified cells that ended up holding a non-command byte. + +## What Befunge is + +Befunge is a dialect ~of Python~ implemented as a whole-module *source-to-source* transform — at the text level, before `mcpyrate`'s AST stage. The dialect definition and runtime interpreter both live in [`unpythonic.dialects.befunge`](../../unpythonic/dialects/befunge.py). Usage examples can be found in [the unit tests](../../unpythonic/dialects/tests/test_befunge.py). + +The `transform_source` hook for the `Befunge` dialect class wraps the playfield text in a single `run()` call, so a Befunge-dialect file ultimately compiles to two lines of Python: + +```python +from unpythonic.dialects.befunge import run +run() +``` + +Leading entirely-blank lines in the body are stripped before the playfield is built. Without that strip, the blank line that typically follows the dialect-import would become row 0 (all spaces); the IP would walk the full no-op row, wrap toroidally back to column 0 of row 0, and loop forever before reaching the actual program. + +## Contrast with BF + +`unpythonic.dialects.bf` and `unpythonic.dialects.befunge` are the package's two source-level dialects, but they make complementary teaching points about `transform_source`: + + - **BF is a transpiler**. `bf.compile(src)` produces structured, legible Python that mirrors the input program. The pedagogic value is in the output text — you can read a BF program by reading the Python it compiles to. + - **Befunge is a reader**. `transform_source` wraps the playfield in a runtime interpreter call; the compiled output is a thin shim, and the language's semantics live in the interpreter. This isn't a design failure: BF is structurally close to Python (linear stream, lexical loops), but Befunge is fundamentally IP-driven on a 2-D, self-modifying grid, and a legible static translation is impossible. (The `p` command rewrites the playfield at runtime, so no static analysis can be sound.) + +The same `transform_source` hook serves both shapes; what differs is how much of the language's behavior the hook can statically lower into Python. + +I/O operator names also differ between the two: BF uses `.` for character output, while Befunge uses `.` for *integer* output and `,` for character output. Worth keeping in mind when switching contexts. + +## Comboability + +Source-transforming dialects consume the whole module body, so combining Befunge with another source-transforming dialect (BF, or itself) on the same file doesn't really make sense. Composition with **AST-transforming** dialects is supported: `from … import dialects, Befunge, SomeOptimizer` (or on separate `from` lines) places `SomeOptimizer` after Befunge in the transform chain, running its AST pass on the output of the Befunge compiler — which, since Befunge's output is a one-liner `run(...)` call, mostly amounts to processing that one statement. + +The mechanism is the `mcpyrate.dialects.split_at_dialectimport` helper (new in `mcpyrate` 4.1.0): the dialect's `transform_source` uses it to peel off its own dialect-import line while preserving any others for the next round of dialect processing. + +## CAUTION + +Not intended for ~serious~ use. + +## Etymology? + +See [Wikipedia](https://en.wikipedia.org/wiki/Befunge). diff --git a/doc/dialects/bf.md b/doc/dialects/bf.md new file mode 100644 index 00000000..0cec1596 --- /dev/null +++ b/doc/dialects/bf.md @@ -0,0 +1,126 @@ +**Navigation** + +- [README](../../README.md) +- [Pure-Python feature set](../features.md) +- [Syntactic macro feature set](../macros.md) +- [Examples of creating dialects using `mcpyrate`](../dialects.md) + - [Lispython](lispython.md) + - [Listhell](listhell.md) + - [Pytkell](pytkell.md) + - **BF** + - [Befunge](befunge.md) +- [REPL server](../repl.md) +- [Troubleshooting](../troubleshooting.md) +- [Design notes](../design-notes.md) +- [Essays](../essays.md) +- [Additional reading](../readings.md) +- [Contribution guidelines](../../CONTRIBUTING.md) + + +**Table of Contents** + +- [BF: the classical human-incomprehensible automaton](#bf-the-classical-human-incomprehensible-automaton) + - [Features](#features) + - [Reading the compiled Python](#reading-the-compiled-python) + - [What BF is](#what-bf-is) + - [Comboability](#comboability) + - [CAUTION](#caution) + - [Etymology?](#etymology) + + + +# BF: the classical human-incomprehensible automaton + +A [BF](https://en.wikipedia.org/wiki/Brainfuck) to Python source-to-source compiler. + +Powered by [`mcpyrate`](https://github.com/Technologicat/mcpyrate/). + +```python +from unpythonic.dialects.bf import dialects, BF # noqa: F401 + +# 'A' via a 5 × 13 multiplication loop ++++++++++++++[>+++++<-]>. +``` + +## Features + + - **Cell semantics**: 8-bit wrapping cells. + - The tape is a `defaultdict[int, int]` subclass (`unpythonic.dialects.bf.Tape`) whose `__setitem__` masks assigned values to the range `0..255`. + - The pointer is unbounded in either direction (infinite Turing tape); untouched cells read as zero. + - **Folding**: consecutive identical commands collapse (`+++` → `tape[ptr] += 3`). + - No cancellation of opposites — `+-` and `><` emit both operations, what you wrote is what you get. + - **Loops**: `[` compiles to `while tape[ptr]:` plus an indent; `]` dedents. + - An empty loop body gets a `pass`. + - **I/O**: `.` writes `chr(tape[ptr])` to `stdout`; `,` reads one character from `stdin`. On EOF, `,` stores `0` in the current cell. + - **Comments**: classical BF treats any non-command character as a no-op. + - The dialect preserves the text — consecutive runs of non-command characters compile into Python `# ...` comments, positioned where they appeared in the source. + - A leading `# ` in the BF source is passed through cleanly, so both `# real comment` and bare `real comment` come out as `# real comment` in the compiled Python. + - **`reset`**: a line whose stripped content is exactly `reset` compiles to `tape.clear(); ptr = 0`. This lets several BF programs share one file. + +## Reading the compiled Python + +BF is a *transpiler*: it takes a BF program and emits human-readable Python that does the same thing. The compiled output is intentionally legible — `+++` becomes `tape[ptr] += 3`, `[…]` becomes `while tape[ptr]: …`, comments are preserved as Python comments — so reading the Python is a perfectly good way to understand a non-trivial BF program. The pedagogic value of the dialect lives in the output text, not in any stored knowledge of the input. + +Two ways to actually see the compiled Python: + +**1. Programmatically, via `bf.compile`.** Useful for offline inspection, ad-hoc experiments, and printing the compiled form into a notebook or a paper: + +```python +from unpythonic.dialects import bf + +src = "+++++++++++++[>+++++<-]>." # 'A' via a 5×13 multiplication loop +print(bf.compile(src)) +``` + +For the program above, this prints: + +```python +from sys import stdin, stdout +from unpythonic.dialects.bf import Tape +tape = Tape() +ptr = 0 + +tape[ptr] += 13 +while tape[ptr]: + ptr += 1 + tape[ptr] += 5 + ptr -= 1 + tape[ptr] -= 1 +ptr += 1 +stdout.write(chr(tape[ptr])); stdout.flush() +``` + +The qualified `bf.compile` form is recommended over `from … import compile` to avoid shadowing `builtins.compile` in the importer's namespace. + +**2. Live, while running the dialect file, via `mcpyrate.debug.StepExpansion`.** When `StepExpansion` is the *first* dialect in the import chain, the dialect expander prints the source after each transformer pass — so for a BF file it shows the BF body before transformation and the generated Python afterward, then runs the result: + +```python +from mcpyrate.debug import dialects, StepExpansion +from unpythonic.dialects.bf import dialects, BF + ++++++++++++++[>+++++<-]>. +``` + +Run via `macropython` (or by `import`-ing the file) and the BF→Python translation is printed to stderr alongside the program's normal execution. Useful when the BF source already lives inside a `.py` file and you'd rather not retype it as a string for `bf.compile`. + +`StepExpansion` is documented in [`mcpyrate`'s troubleshooting guide](https://github.com/Technologicat/mcpyrate/blob/master/doc/troubleshooting.md). It works for any dialect, not just BF. + +## What BF is + +BF is a dialect ~of Python~ implemented as a whole-module *source-to-source* transform. The dialect definition lives in [`unpythonic.dialects.bf`](../../unpythonic/dialects/bf.py). Usage examples can be found in [the unit tests](../../unpythonic/dialects/tests/test_bf.py). + +It's also a minimal example of how to make a **source-transforming** dialect, the modern equivalent of what old Lisp folks used to call a *reader macro*. All other dialects in this collection — Lispython, Listhell, Pytkell — are AST-transforming, built on top of `unpythonic`'s macro layer. BF shares none of that machinery: the body of a BF file is not parseable as Python at all, so the compiler runs at the text level, before `mcpyrate`'s AST-level dialect stage. + +## Comboability + +Source-transforming dialects consume the whole module body, so combining BF with another source-transforming dialect on the same file doesn't really make sense. Composition with **AST-transforming** dialects is supported: `from X import dialects, BF, SomeOptimizer` (or on separate `from` lines) places `SomeOptimizer` after BF in the transform chain, running its AST pass on the output of the BF compiler. + +The mechanism is the `mcpyrate.dialects.split_at_dialectimport` helper (new in `mcpyrate` 4.1.0): BF's `transform_source` uses it to peel off its own dialect-import line while preserving any others for the next round of dialect processing. + +## CAUTION + +Not intended for ~serious~ use. + +## Etymology? + +See [Wikipedia](https://en.wikipedia.org/wiki/Brainfuck). diff --git a/doc/dialects/lispython.md b/doc/dialects/lispython.md index 3874914a..c837ee98 100644 --- a/doc/dialects/lispython.md +++ b/doc/dialects/lispython.md @@ -7,9 +7,12 @@ - **Lispython** - [Listhell](listhell.md) - [Pytkell](pytkell.md) + - [BF](bf.md) + - [Befunge](befunge.md) - [REPL server](../repl.md) - [Troubleshooting](../troubleshooting.md) - [Design notes](../design-notes.md) +- [Essays](../essays.md) - [Additional reading](../readings.md) - [Contribution guidelines](../../CONTRIBUTING.md) @@ -18,6 +21,8 @@ - [Lispython: The love child of Python and Scheme](#lispython-the-love-child-of-python-and-scheme) - [Features](#features) + - [The `Lispy` variant](#the-lispy-variant) + - [The `Lispython` variant](#the-lispython-variant) - [What Lispython is](#what-lispython-is) - [Comboability](#comboability) - [Lispython and continuations (call/cc)](#lispython-and-continuations-callcc) @@ -55,9 +60,6 @@ square = lambda x: x**2 assert square(3) == 9 assert square.__name__ == "square" -# - brackets denote a multiple-expression lambda body -# (if you want to have one expression that is a literal list, -# double the brackets: `lambda x: [[5 * x]]`) # - local[name << value] makes an expression-local variable g = lambda x: [local[y << 2 * x], y + 1] @@ -72,35 +74,68 @@ assert ll(1, 2, 3) == llist((1, 2, 3)) ## Features -In terms of ``unpythonic.syntax``, we implicitly enable ``tco``, ``autoreturn``, ``multilambda``, ``namedlambda``, and ``quicklambda`` for the whole module: +In terms of `unpythonic.syntax`, we implicitly enable `autoreturn`, `tco`, `multilambda`, `namedlambda`, and `quicklambda` for the whole module: + + - In tail position, the `return` keyword can be omitted, like in Lisps. + - In a `def`, the last statement at the top level of the `def` is in tail position. + - If the tail position contains an expression, a `return` will be automatically injected, with that expression as the return value. + - It is still legal to use `return` whenever you would in Python; this just makes the `return` keyword non-mandatory in places where a Lisp would not require it. + - To be technically correct, Schemers and Racketeers should read this as, *"in places where a Lisp would not require explicitly invoking an escape continuation"*. + - Automatic tail-call optimization (TCO) for both `def` and `lambda`. + - In a `def`, the last statement at the top level of the `def` is in tail position. + - Tail positions *inside an expression* that itself appears in tail position are: + - Both the `body` and `orelse` branches of an if-expression. (Exactly one of them runs, hence both are in tail position.) + - The lexically last item of an `and`/`or` chain. + - Note the analysis is performed at compile time, whence it does **not** care about the short-circuit behavior that occurs at run time. + - The last item of a `do[]`. + - The last item of an implicit `do[]` in a `let[]` where the body uses the extra bracket syntax. (All `let` constructs provided by `unpythonic.syntax` are supported.) + - For the gritty details, see the syntax transformer `_transform_retexpr` in [`unpythonic.syntax.tailtools`](../../unpythonic/syntax/tailtools.py). + - Multiple-expression lambdas, using bracket syntax, for example `lambda x: [expr0, ...]`. + - Brackets denote a multiple-expression lambda body. Technically, the brackets create a `do[]` environment. + - If you want your lambda to have one expression that is a literal list, double the brackets: `lambda x: [[5 * x]]`. + - Lambdas are automatically named whenever the machinery can figure out a name from the surrounding context. + - When not, source location is auto-injected into the name. + +The multi-expression lambda syntax uses `do[]`, so it also allows lambdas to manage local variables using `local[name << value]` and `delete[name]`. See the documentation of `do[]` for details. - - TCO in both ``def`` and ``lambda``, fully automatic - - Omit ``return`` in any tail position, like in Lisps - - Multiple-expression lambdas, ``lambda x: [expr0, ...]`` - - Named lambdas (whenever the machinery can figure out a name) - - The underscore: ``f[_*3] --> lambda x: x*3`` (name ``f`` is **reserved**) +If you need more stuff, `unpythonic` is effectively the standard library of Lispython, on top of what Python itself already provides. -We also import some macros and functions to serve as dialect builtins: +There are **two variants** of the dialect, `Lispython` and `Lispy`. - - All ``let[]`` and ``do[]`` constructs from ``unpythonic.syntax`` - - ``cons``, ``car``, ``cdr``, ``ll``, ``llist``, ``nil``, ``prod`` - - ``dyn``, for dynamic assignment - - ``Values``, for returning multiple values and/or named return values. (This ties in to `unpythonic`'s function composition subsystem, e.g. `curry`, the `pipe` family, the `compose` family, and the `with continuations` macro.) -For detailed documentation of the language features, see [``unpythonic.syntax``](https://github.com/Technologicat/unpythonic/tree/master/doc/macros.md), especially the macros ``tco``, ``autoreturn``, ``multilambda``, ``namedlambda``, ``quicklambda``, ``let`` and ``do``. +### The `Lispy` variant -The multi-expression lambda syntax uses ``do[]``, so it also allows lambdas to manage local variables using ``local[name << value]`` and ``delete[name]``. See the documentation of ``do[]`` for details. +In the `Lispy` variant, that's it - the dialect changes the semantics only. Nothing is imported implicitly, except the macros injected by the dialect template (to perform the whole-module semantic changes at macro expansion time). -The builtin ``let[]`` constructs are ``let``, ``letseq``, ``letrec``, the decorator versions ``dlet``, ``dletseq``, ``dletrec``, the block versions (decorator, call immediately, replace def'd name with result) ``blet``, ``bletseq``, ``bletrec``, and the code-splicing variants ``let_syntax`` and ``abbrev``. Bindings may be made using any syntax variant supported by ``unpythonic.syntax``. +This is the pythonic variant of Lispython, keeping in line with *explicit is better than implicit*. The rule is: *if a name appears in user code, it must be defined explicitly*, as is usual in Python. -The builtin ``do[]`` constructs are ``do`` and ``do0``. +Note this implies that you must **explicitly import** the `local[]` macro if you want to declare local variables in a multiple-expression lambda, and the `fn[]` macro if you want to take advantage of the implicit `quicklambda`. Both are available in `unpythonic.syntax`, as usual. (Note that you can rename the `fn[]` macro with an as-import, and the implicit `quicklambda` will still work.) -If you need more stuff, `unpythonic` is effectively the standard library of Lispython, on top of what Python itself already provides. +The point of the implicit `quicklambda` is that all invocations of `fn[]`, if there are any, will expand early, so that other macros that expect lambdas to be in standard Python notation will get exactly that. This includes other macros invoked by the dialect definition, namely `multilambda`, `namedlambda`, and `tco`. + +The main point of `Lispy`, compared to plain Python, is automatic TCO. The ability to omit `return` is a minor convenience, and the other three features only improve the usability of lambdas. + + +### The `Lispython` variant + +In the `Lispython` variant, we implicitly import some macros and functions to serve as dialect builtins, keeping in line with expectations for a ~language in the~ *somewhat distant relative of the* Lisp family: + + - `cons`, `car`, `cdr`, `ll`, `llist`, `nil`, `prod`. + - All `let[]` and `do[]` constructs from `unpythonic.syntax`. + - The underscore: e.g. `fn[_ * 3]` becomes `lambda x: x * 3`, and `fn[_ * _]` becomes `lambda x, y: x * y`. + - `dyn`, for dynamic assignment. + - `Values`, for returning multiple values and/or named return values. (This ties in to `unpythonic`'s function composition subsystem, e.g. `curry`, `unfold`, `iterate`, the `pipe` family, the `compose` family, and the `with continuations` macro.) + +For detailed documentation of the language features, see [`unpythonic.syntax`](../macros.md), especially the macros `tco`, `autoreturn`, `multilambda`, `namedlambda`, `quicklambda`, `let` and `do`. + +The dialect builtin `let[]` constructs are `let`, `letseq`, `letrec`, the decorator versions `dlet`, `dletseq`, `dletrec`, the block versions (decorator, call immediately, replace def'd name with result) `blet`, `bletseq`, `bletrec`, and the code-splicing variants `let_syntax` and `abbrev`. Bindings may be made using any syntax variant supported by `unpythonic.syntax`. + +The dialect builtin `do[]` constructs are `do` and `do0`. ## What Lispython is -Lispython is a dialect of Python implemented via macros and a thin whole-module AST transformation. The dialect definition lives in [`unpythonic.dialects.lispython`](../../unpythonic/dialects/lispython.py). Usage examples can be found in [the unit tests](../../unpythonic/dialects/tests/test_lispython.py). +Lispython is a dialect of Python implemented via macros and a thin whole-module AST transformation. The dialect definition lives in [`unpythonic.dialects.lispython`](../../unpythonic/dialects/lispython.py). Usage examples can be found in the unit tests, [for `Lispy`](../../unpythonic/dialects/tests/test_lispy.py) and [for `Lispython`](../../unpythonic/dialects/tests/test_lispython.py). Lispython essentially makes Python feel slightly more lispy, in parts where that makes sense. @@ -115,43 +150,43 @@ Performance is only a secondary concern; performance-critical parts fare better The aforementioned block macros are enabled implicitly for the whole module; this is the essence of the Lispython dialect. Other block macros can still be invoked manually in the user code. -Of the other block macros in ``unpythonic.syntax``, code written in Lispython supports only ``continuations``. ``autoref`` should also be harmless enough (will expand too early, but shouldn't matter). +Of the other block macros in `unpythonic.syntax`, code written in Lispython supports only `continuations`. `autoref` should also be harmless enough (will expand too early, but shouldn't matter). -``prefix``, ``curry``, ``lazify`` and ``envify`` are **not compatible** with the ordering of block macros implicit in the Lispython dialect. +`prefix`, `autocurry`, `lazify` and `envify` are **not compatible** with the ordering of block macros implicit in the Lispython dialect. -``prefix`` is an outside-in macro that should expand first, so it should be placed in a lexically outer position with respect to the ones Lispython invokes implicitly; but nothing can be more outer than the dialect template. +`prefix` is an outside-in macro that should expand first, so it should be placed in a lexically outer position with respect to the ones Lispython invokes implicitly; but nothing can be more outer than the dialect template. The other three are inside-out macros that should expand later, so similarly, also they should be placed in a lexically outer position. -Basically, any block macro that can be invoked *lexically inside* a ``with tco`` block will work, the rest will not. +Basically, any block macro that can be invoked *lexically inside* a `with tco` block will work, the rest will not. -If you need e.g. a lazy Lispython, the way to do that is to make a copy of the dialect module, change the dialect template to import the ``lazify`` macro, and then include a ``with lazify`` in the appropriate position, outside the ``with namedlambda`` block. Other customizations can be made similarly. +If you need e.g. a lazy Lispython, the way to do that is to make a copy of the dialect module, change the dialect template to import the `lazify` macro, and then include a `with lazify` in the appropriate position, outside the `with namedlambda` block. Other customizations can be made similarly. ## Lispython and continuations (call/cc) -Just use ``with continuations`` from ``unpythonic.syntax`` where needed. See its documentation for usage. +Just use `with continuations` from `unpythonic.syntax` where needed. See its documentation for usage. -Lispython works with ``with continuations``, because: +Lispython works with `with continuations`, because: - - Nesting ``with continuations`` within a ``with tco`` block is allowed, for the specific reason of supporting continuations in Lispython. + - Nesting `with continuations` within a `with tco` block is allowed, for the specific reason of supporting continuations in Lispython. - The dialect's implicit ``with tco`` will just skip the ``with continuations`` block (``continuations`` implies TCO). + The dialect's implicit `with tco` will just skip the `with continuations` block (`continuations` implies TCO). - - ``autoreturn``, ``quicklambda`` and ``multilambda`` are outside-in macros, so although they will be in a lexically outer position with respect to the manually invoked ``with continuations`` in the user code, this is correct (because being on the outside, they run before ``continuations``, as they should). + - `autoreturn`, `quicklambda` and `multilambda` are outside-in macros, so although they will be in a lexically outer position with respect to the manually invoked `with continuations` in the user code, this is correct (because being on the outside, they run before `continuations`, as they should). - - The same applies to the outside-in pass of ``namedlambda``. Its inside-out pass, on the other hand, must come after ``continuations``, which it does, since the dialect's implicit ``with namedlambda`` is in a lexically outer position with respect to the ``with continuations``. + - The same applies to the outside-in pass of `namedlambda`. Its inside-out pass, on the other hand, must come after `continuations`, which it does, since the dialect's implicit `with namedlambda` is in a lexically outer position with respect to the `with continuations`. -Be aware, though, that the combination of the ``autoreturn`` implicit in the dialect and ``with continuations`` might have usability issues, because ``continuations`` handles tail calls specially (the target of a tail-call in a ``continuations`` block must be continuation-enabled; see the documentation of ``continuations``), and ``autoreturn`` makes it visually slightly less clear which positions are in factorial tail calls (since no explicit ``return``). Also, the top level of a ``with continuations`` block may not use ``return`` - while Lispython happily auto-injects a ``return`` to whatever is the last statement in any particular function. +Be aware, though, that the combination of the `autoreturn` implicit in the dialect and `with continuations` might have usability issues, because `continuations` handles tail calls specially (the target of a tail-call in a `continuations` block must be continuation-enabled; see the documentation of `continuations`), and `autoreturn` makes it visually slightly less clear which positions are in fact tail calls (since no explicit `return`). Also, the top level of a `with continuations` block may not use `return` - while Lispython's implicit `autoreturn` happily auto-injects a `return` to whatever is the last statement in any particular function. ## Why extend Python? [Racket](https://racket-lang.org/) is an excellent Lisp, especially with [sweet](https://docs.racket-lang.org/sweet/), sweet expressions [[1]](https://sourceforge.net/projects/readable/) [[2]](https://srfi.schemers.org/srfi-110/srfi-110.html) [[3]](https://srfi.schemers.org/srfi-105/srfi-105.html), not to mention extremely pythonic. The word is *rackety*; the syntax of the language comes with an air of Zen minimalism (as perhaps expected of a descendant of Scheme), but the focus on *batteries included* and understandability are remarkably similar to the pythonic ideal. Racket even has an IDE (DrRacket) and an equivalent of PyPI, and the documentation is simply stellar. -Python, on the other hand, has a slight edge in usability to the end-user programmer, and importantly, a huge ecosystem of libraries, second to ``None``. Python is where science happens (unless you're in CS). Python is an almost-Lisp that has delivered on [the productivity promise](http://paulgraham.com/icad.html) of Lisp. Python also gets many things right, such as well developed support for lazy sequences, and decorators. +Python, on the other hand, has a slight edge in usability to the end-user programmer, and importantly, a huge ecosystem of libraries, second to `None`. Python is where science happens (unless you're in CS). Python is an almost-Lisp that has delivered on [the productivity promise](http://paulgraham.com/icad.html) of Lisp. Python also gets many things right, such as well developed support for lazy sequences, and decorators. -In certain other respects, Python the base language leaves something to be desired, if you have been exposed to Racket (or Haskell, but that's a different story). Writing macros is harder due to the irregular syntax, but thankfully MacroPy already exists, and any set of macros only needs to be created once. +In certain other respects, Python the base language leaves something to be desired, if you have been exposed to Racket (or Haskell, but that's a different story). Writing macros is harder due to the irregular syntax, but thankfully macro expanders already exist, and any set of macros only needs to be created once. Practicality beats purity ([ZoP §9](https://www.python.org/dev/peps/pep-0020/)): hence, fix the minor annoyances that would otherwise quickly add up, and reap the benefits of both worlds. If Python is software glue, Lispython is an additive that makes it flow better. @@ -176,7 +211,7 @@ foo = lambda n0: let[[n << n0] in (lambda i: n << n + i)] ``` -This still sets up a separate place for the accumulator (that is, separate from the argument of the outer function). The modern pure Python solution avoids that, but needs many lines: +This still sets up a separate place for the accumulator (that is, separate from the argument of the outer function). The pure Python 3 solution avoids that, but needs many lines: ```python def foo(n): @@ -187,11 +222,21 @@ def foo(n): return accumulate ``` -The problem is that assignment to a lexical variable (including formal parameters) is a statement in Python. Python 3.8's walrus operator does not solve this, because `n := n + i` by itself is a syntax error. +The Python 3.8+ solution, using the new walrus operator, is one line shorter: -If we abbreviate ``accumulate`` as a lambda, it needs a ``let`` environment to write in, to use `unpythonic`'s expression-assignment (`name << value`). +```python +def foo(n): + def accumulate(i): + nonlocal n + return (n := n + i) + return accumulate +``` + +This is rather clean, but still needs the `nonlocal` declaration, which is a statement. -But see ``envify`` in ``unpythonic.syntax``, which shallow-copies function arguments into an `env` implicitly: +If we abbreviate `accumulate` as a lambda, it needs a `let` environment to write in, to use `unpythonic`'s expression-assignment (`name << value`). + +But see `envify` in `unpythonic.syntax`, which shallow-copies function arguments into an `env` implicitly: ```python from unpythonic.syntax import macros, envify @@ -208,7 +253,9 @@ with envify: foo = lambda n: lambda i: n << n + i ``` -``envify`` is not part of the Lispython dialect definition, because this particular, perhaps rarely used, feature is not really worth a global performance hit whenever a function is entered. +`envify` is not part of the Lispython dialect definition, because this particular, perhaps rarely used, feature is not really worth a global performance hit whenever a function is entered. + +Note that `envify` is **not** compatible with Lispython, because it would need to appear in a lexically outer position compared to macros already invoked by the dialect template. If you need an envified Lispython, copy `unpythonic/dialects/lispython.py` and modify the template therein. [The xmas tree combo](../macros.md#the-xmas-tree-combo) says `envify` should come lexically after `multilambda`, but before `namedlambda`. ## CAUTION diff --git a/doc/dialects/listhell.md b/doc/dialects/listhell.md index f171320b..65c4ef66 100644 --- a/doc/dialects/listhell.md +++ b/doc/dialects/listhell.md @@ -7,9 +7,12 @@ - [Lispython](lispython.md) - **Listhell** - [Pytkell](pytkell.md) + - [BF](bf.md) + - [Befunge](befunge.md) - [REPL server](../repl.md) - [Troubleshooting](../troubleshooting.md) - [Design notes](../design-notes.md) +- [Essays](../essays.md) - [Additional reading](../readings.md) - [Contribution guidelines](../../CONTRIBUTING.md) @@ -46,16 +49,16 @@ assert (my_map, double, (q, 1, 2, 3)) == (ll, 2, 4, 6) ## Features -In terms of ``unpythonic.syntax``, we implicitly enable ``prefix`` and ``curry`` for the whole module. +In terms of `unpythonic.syntax`, we implicitly enable `prefix` and `autocurry` for the whole module. The following are dialect builtins: - - ``apply``, aliased to ``unpythonic.fun.apply`` - - ``compose``, aliased to unpythonic's currying right-compose ``composerc`` - - ``q``, ``u``, ``kw`` for the prefix syntax (note these are not `mcpyrate`'s - ``q`` and ``u``, but those from `unpythonic.syntax`, specifically for ``prefix``) + - `apply`, aliased to `unpythonic.fun.apply` + - `compose`, aliased to unpythonic's currying right-compose `composerc` + - `q`, `u`, `kw` for the prefix syntax (note these are not `mcpyrate`'s + `q` and `u`, but those from `unpythonic.syntax`, specifically for `prefix`) -For detailed documentation of the language features, see [``unpythonic.syntax``](https://github.com/Technologicat/unpythonic/tree/master/doc/macros.md). +For detailed documentation of the language features, see [`unpythonic.syntax`](https://github.com/Technologicat/unpythonic/tree/master/doc/macros.md). If you need more stuff, `unpythonic` is effectively the standard library of Listhell, on top of what Python itself already provides. @@ -71,7 +74,7 @@ It's also a minimal example of how to make an AST-transforming dialect. ## Comboability -Only outside-in macros that should expand after ``curry`` (currently, `unpythonic` provides no such macros) and inside-out macros that should expand before ``curry`` (there are two, namely ``tco`` and ``continuations``) can be used in programs written in the Listhell dialect. +Only outside-in macros that should expand after `autocurry` (currently, `unpythonic` provides no such macros) and inside-out macros that should expand before `autocurry` (there are two, namely `tco` and `continuations`) can be used in programs written in the Listhell dialect. ## Notes diff --git a/doc/dialects/pytkell.md b/doc/dialects/pytkell.md index b91ad174..caef7139 100644 --- a/doc/dialects/pytkell.md +++ b/doc/dialects/pytkell.md @@ -7,9 +7,12 @@ - [Lispython](lispython.md) - [Listhell](listhell.md) - **Pytkell** + - [BF](bf.md) + - [Befunge](befunge.md) - [REPL server](../repl.md) - [Troubleshooting](../troubleshooting.md) - [Design notes](../design-notes.md) +- [Essays](../essays.md) - [Additional reading](../readings.md) - [Contribution guidelines](../../CONTRIBUTING.md) @@ -69,36 +72,36 @@ assert x == 42 ## Features -In terms of ``unpythonic.syntax``, we implicitly enable ``curry`` and ``lazify`` for the whole module. +In terms of `unpythonic.syntax`, we implicitly enable `autocurry` and `lazify` for the whole module. We also import some macros and functions to serve as dialect builtins: - - All ``let[]`` and ``do[]`` constructs from ``unpythonic.syntax`` - - ``lazy[]`` and ``lazyrec[]`` for manual lazification of atoms and data structure literals, respectively - - If-elseif-else expression ``cond[]`` - - Nondeterministic evaluation ``forall[]`` (do-notation in the List monad) - - Function composition, ``compose`` (like Haskell's ``.`` operator), aliased to `unpythonic`'s currying right-compose ``composerc`` - - Linked list utilities ``cons``, ``car``, ``cdr``, ``ll``, ``llist``, ``nil`` - - Folds and scans ``foldl``, ``foldr``, ``scanl``, ``scanr`` - - Memoization ``memoize``, ``gmemoize``, ``imemoize``, ``fimemoize`` - - Functional updates ``fup`` and ``fupdate`` - - Immutable dict ``frozendict`` - - Mathematical sequences ``s``, ``imathify``, ``gmathify`` - - Iterable utilities ``islice`` (`unpythonic`'s version), ``take``, ``drop``, ``split_at``, ``first``, ``second``, ``nth``, ``last`` - - Function arglist reordering utilities ``flip``, ``rotate`` + - All `let[]` and `do[]` constructs from `unpythonic.syntax` + - `lazy[]` and `lazyrec[]` for manual lazification of atoms and data structure literals, respectively + - If-elseif-else expression `cond[]` + - Nondeterministic evaluation `forall[]` (do-notation in the List monad) + - Function composition, `compose` (like Haskell's `.` operator), aliased to `unpythonic`'s currying right-compose `composerc` + - Linked list utilities `cons`, `car`, `cdr`, `ll`, `llist`, `nil` + - Folds and scans `foldl`, `foldr`, `scanl`, `scanr` + - Memoization `memoize`, `gmemoize`, `imemoize`, `fimemoize` + - Functional updates `fup` and `fupdate` + - Immutable dict `frozendict` + - Mathematical sequences `s`, `imathify`, `gmathify` + - Iterable utilities `islice` (`unpythonic`'s version), `take`, `drop`, `split_at`, `first`, `second`, `nth`, `last` + - Function arglist reordering utilities `flip`, `rotate` -For detailed documentation of the language features, see [``unpythonic.syntax``](https://github.com/Technologicat/unpythonic/tree/master/doc/macros.md). +For detailed documentation of the language features, see [`unpythonic.syntax`](https://github.com/Technologicat/unpythonic/tree/master/doc/macros.md). -The builtin ``let[]`` constructs are ``let``, ``letseq``, ``letrec``, the decorator versions ``dlet``, ``dletseq``, ``dletrec``, the block versions (decorator, call immediately, replace `def`'d name with result) ``blet``, ``bletseq``, ``bletrec``. Bindings may be made using any syntax variant supported by ``unpythonic.syntax``. +The builtin `let[]` constructs are `let`, `letseq`, `letrec`, the decorator versions `dlet`, `dletseq`, `dletrec`, the block versions (decorator, call immediately, replace `def`'d name with result) `blet`, `bletseq`, `bletrec`. Bindings may be made using any syntax variant supported by `unpythonic.syntax`. -The builtin ``do[]`` constructs are ``do`` and ``do0``. +The builtin `do[]` constructs are `do` and `do0`. If you need more stuff, `unpythonic` is effectively the standard library of Pytkell, on top of what Python itself already provides. ## What Pytkell is -Pytkell is a dialect of Python implemented via macros and a thin whole-module AST transformation. The dialect definition lives in [`unpythonic.dialects.pytkell`](../../unpythonic/dialects/lispython.py). Usage examples can be found in [the unit tests](../../unpythonic/dialects/tests/test_pytkell.py). +Pytkell is a dialect of Python implemented via macros and a thin whole-module AST transformation. The dialect definition lives in [`unpythonic.dialects.pytkell`](../../unpythonic/dialects/pytkell.py). Usage examples can be found in [the unit tests](../../unpythonic/dialects/tests/test_pytkell.py). Pytkell essentially makes Python feel slightly more haskelly. @@ -107,9 +110,9 @@ It's also a minimal example of how to make an AST-transforming dialect. ## Comboability -**Not** comboable with most of the block macros in ``unpythonic.syntax``, because ``curry`` and ``lazify`` appear in the dialect template, hence at the lexically outermost position. +**Not** comboable with most of the block macros in `unpythonic.syntax`, because `autocurry` and `lazify` appear in the dialect template, hence at the lexically outermost position. -Only outside-in macros that should expand after ``lazify`` has recorded its userlambdas (currently, `unpythonic` provides no such macros) and inside-out macros that should expand before ``curry`` (there are two, namely ``tco`` and ``continuations``) can be used in programs written in the Pytkell dialect. +Only outside-in macros that should expand after `lazify` has recorded its userlambdas (currently, `unpythonic` provides no such macros) and inside-out macros that should expand before `autocurry` (there are two, namely `tco` and `continuations`) can be used in programs written in the Pytkell dialect. ## CAUTION diff --git a/doc/essays.md b/doc/essays.md new file mode 100644 index 00000000..245519ca --- /dev/null +++ b/doc/essays.md @@ -0,0 +1,181 @@ +**Navigation** + +- [README](../README.md) +- [Pure-Python feature set](features.md) +- [Syntactic macro feature set](macros.md) +- [Examples of creating dialects using `mcpyrate`](dialects.md) +- [REPL server](repl.md) +- [Troubleshooting](troubleshooting.md) +- [Design notes](design-notes.md) +- **Essays** +- [Additional reading](readings.md) +- [Contribution guidelines](../CONTRIBUTING.md) + +For now, essays are listed in chronological order, most recent last. + + +**Table of Contents** + +- [What Belongs in Python?](#what-belongs-in-python) +- [Common Lisp, Python, and productivity](#common-lisp-python-and-productivity) +- [`hoon`: The C of Functional Programming](#hoon-the-c-of-functional-programming) + + + + +# What Belongs in Python? + +*Originally written in 2020; updated 9 June 2021; small update 16 November 2022.* + +You may feel that [my hovercraft is full of eels](http://stupidpythonideas.blogspot.com/2015/05/spam-spam-spam-gouda-spam-and-tulips.html). It is because they come with the territory. + +Some have expressed the opinion [the statement-vs-expression dichotomy is a feature](http://stupidpythonideas.blogspot.com/2015/01/statements-and-expressions.html). The BDFL himself has famously stated that TCO has no place in Python [[1]](http://neopythonic.blogspot.com/2009/04/tail-recursion-elimination.html) [[2]](http://neopythonic.blogspot.fi/2009/04/final-words-on-tail-calls.html), and less famously that multi-expression lambdas or continuations have no place in Python [[3]](https://www.artima.com/weblogs/viewpost.jsp?thread=147358). Several potentially interesting PEPs have been deferred [[1]](https://www.python.org/dev/peps/pep-3150/) [[2]](https://www.python.org/dev/peps/pep-0403/) or rejected [[3]](https://www.python.org/dev/peps/pep-0511/) [[4]](https://www.python.org/dev/peps/pep-0463/) [[5]](https://www.python.org/dev/peps/pep-0472/). + +In general, I like Python. My hat is off to the devs. It is no mean feat to create a high-level language that focuses on readability and approachability, keep it alive for 30 years and counting, and have a large part of the programming community adopt it. But regarding the particular points above, if I agreed, I would not have built `unpythonic`, or [`mcpyrate`](https://github.com/Technologicat/mcpyrate) either. + +I think that with macros, Python can be so much more than just a beginner's language. Language-level extensibility is just the logical endpoint of that. I do not share the sentiment of the Python community against metaprogramming, or toward some language-level features. For me, macros (and full-module transforms a.k.a. dialects) are just another tool for creating abstractions, at yet another level. We can already extract procedures, methods, and classes. Why limit that ability - namely, the ability to create abstractions - to what an [eager](https://en.wikipedia.org/wiki/Evaluation_strategy#Strict_evaluation) language can express at run time? + +If the point is to keep code understandable, I respect the goal; but that is a matter of education. It is perfectly possible to write unreadable code without macros, and in Python, no less. Just use a complex class hierarchy so that the programmer reading the code must hunt through everything to find each method definition; write big functions without abstracting the steps of the overall algorithm; keep lots of mutable state, and store it in top-level variables; and maybe top that off with an overuse of dependency injection. No one will be able to figure out how the program works, at least not in any reasonable amount of time. + +It is also perfectly possible to write readable code with macros. Just keep in mind that macros are a different kind of abstraction, and use them where that kind of abstraction lends itself to building a clean solution. I am willing to admit the technical objection that *macros do not compose*; but that does not make them useless. + +Of the particular points above, in my opinion TCO should at least be an option. I like that *by default*, Python will complain about a call stack overflow rather than hang, when entering an accidentally infinite mutual recursion. I do occasionally make such mistakes when developing complex algorithms - especially when quickly sketching out new ideas. But sometimes, it would be nice to enable TCO selectively. If you ask for it, you know what to expect. This is precisely why `unpythonic.syntax` has `with tco`. I am not very happy with a custom TCO layer on top of a language core that eschews the whole idea, because TCO support in the core (like Scheme and Racket have) would simplify the implementation of certain other language extensions; but then again, [this is exactly what Clojure did](https://clojuredocs.org/clojure.core/trampoline), in similar technical circumstances. + +As for a multi-expression `lambda`, on the surface it sounds like a good idea. But really the issue is that in Python, the `lambda` construct itself is broken. It is essentially a duplicate of `def`, but lacking some features. As of Python 3.8, the latest addition of insult to injury is the lack of support for type annotations. A more uniform solution would be to make `def` into an expression. Much of the time, anonymous functions are not a good idea, because names help understanding and debugging - especially when all you have is a traceback. But defining closures inline **is** a great idea - and sometimes, the most readily understandable presentation order for an algorithm requires to do that in an expression position. The convenience is similar to being able to nest `def` statements, an ability Python already has. + +The macros in `unpythonic.syntax` inject many lambdas, because that makes them much simpler to implement than if we had to always lift a `def` statement into the nearest enclosing statement context. Another case in point is [`pampy`](https://github.com/santinic/pampy). The code to perform a pattern match would read a lot nicer if one could define also slightly more complex actions inline (see [Racket's pattern matcher](https://docs.racket-lang.org/reference/match.html) for a comparison). It is unlikely that the action functions will be needed elsewhere, and it is just silly to define a bunch of functions *before* the call to `match`. If this is not a job for either something like `let-where` (to invert the presentation order locally) or a multi-expression lambda (to define the actions inline), I do not know what is. + +While on the topic of usability, why are lambdas strictly anonymous? In cases where it is useful to be able to omit a name, because sometimes many small helper functions may be needed and [naming is hard](https://martinfowler.com/bliki/TwoHardThings.html), why not include the source location information in the auto-generated name, instead of just `""`? (As of v0.15.0, the `with namedlambda` macro does this.) + +On a point raised [here by the BDFL](https://www.artima.com/weblogs/viewpost.jsp?thread=147358), with respect to indentation-sensitive vs. indentation-insensitive parser modes; having seen [SRFI-110: Sweet-expressions (t-expressions)](https://srfi.schemers.org/srfi-110/srfi-110.html), I think Python is confusing matters by linking the parser mode to statements vs. expressions. A workable solution is to make *everything* support both modes (or even preprocess the source code text to use only one of the modes), which *uniformly* makes parentheses an alternative syntax for grouping. + +It would be nice to be able to use indentation to structure expressions to improve their readability, like one can do in Racket with [sweet](https://docs.racket-lang.org/sweet/), but I suppose `lambda x: [expr0, expr1, ...]` will have to do for a multi-expression lambda. Unless I decide at some point to make a source filter for [`mcpyrate`](https://github.com/Technologicat/mcpyrate) to auto-convert between indentation and parentheses; but for Python this is somewhat difficult to do, because statements **must** use indentation whereas expressions **must** use parentheses, and this must be done before we can invoke the standard parser to produce an AST. (And I do not want to maintain a [Pyparsing](https://github.com/pyparsing/pyparsing) grammar to parse a modified version of Python.) + +As for true multi-shot continuations, `unpythonic.syntax` has `with continuations` for that, but I am not sure if I will ever use it in production code. Most of the time, it seems to me full continuations are a solution looking for a problem. (A very elegant solution, even if the usability of the `call/cc` interface leaves much to be desired. The solution to *that* issue is `let/cc`, which in `unpythonic`, becomes `k = call_cc[get_cc()]`.) For everyday use, one-shot continuations (a.k.a. resumable functions, a.k.a. generators in Python) are often all that is needed to simplify certain patterns, especially those involving backtracking. I am a big fan of the idea that, for example, you can make your [anagram-making algorithm](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/anagram.py) only yield valid anagrams, with the backtracking state (to eliminate dead-ends) implicitly stored in the paused generator! However, having multi-shot continuations is great for teaching the concept of continuations in a programming course, when teaching in Python. + +Finally, there is the issue of implicitly encouraging subtly incompatible Python-like languages (see the rejected [PEP 511](https://www.python.org/dev/peps/pep-0511/)). It is pretty much the point of language-level extensibility, to allow users to do that if they want. I would not worry about it. Racket is *designed* for extensibility, and its community seems to be doing just fine - they even *encourage* the creation of new languages to solve problems. On the other hand, Racket demands some sophistication on the part of its user, and it is not very popular in the programming community at large. + +What I can say is, `unpythonic` is not meant for the average Python project, either. If used intelligently, it can make code shorter, yet readable. For a lone developer who needs to achieve as much as possible in the fewest lines reasonably possible, it seems to me that language extension - and in general, as Alexis King put it, [climbing the infinite ladder of abstraction](https://lexi-lambda.github.io/blog/2016/08/11/climbing-the-infinite-ladder-of-abstraction/) - is the way to go. In a large project with a high developer turnover, the situation is different. + +For general programming in the early 2020s, Python still has the ecosystem advantage, so it does not make sense to move to anything else, at least yet. So, let us empower what we have. Even if we have to build something that could be considered *unpythonic*. + + +# Common Lisp, Python, and productivity + +*Originally written in 2020; updated 9 June 2021; small update 16 November 2022.* + +The various essays Paul Graham wrote near the turn of the millennium, especially [Revenge of the Nerds (2002)](http://paulgraham.com/icad.html), have given the initial impulse to many programmers for studying Lisp. The essays are well written and have provided a lot of exposure for the Lisp family of languages. So how does the programming world look in that light now, 20 years later? + +The base abstraction level of programming languages, even those in popular use, has increased. The trend was visible already then, and was indeed noted in the essays. The focus on low-level languages such as C++ has decreased. Java is still popular, but high-level FP languages that compile to JVM bytecode (Kotlin, Scala, Clojure) are rising. + +Python has become highly popular, and is now also closer to Lisp than it was 20 years ago, especially after `MacroPy` introduced syntactic macros to Python (in 2013, [according to the git log](https://github.com/lihaoyi/macropy/commits/python2/macropy/__init__.py)). Python was not bad as a Lisp replacement even back in 2000 - see Peter Norvig's essay [Python for Lisp Programmers](https://norvig.com/python-lisp.html). Some more historical background, specifically on lexically scoped closures (and the initial lack thereof), can be found in [PEP 3104](https://www.python.org/dev/peps/pep-3104/), [PEP 227](https://www.python.org/dev/peps/pep-0227/), and [Historical problems with closures in JavaScript and Python](http://giocc.com/problems-with-closures-in-javascript-and-python.html). + +In 2020, does it still make sense to learn [the legendary](https://xkcd.com/297/) Common Lisp? + +As a practical tool? Is CL hands-down better than Python? Maybe no. Python has already delivered on 90% of the productivity promise of Lisp. Both languages cut down significantly on [accidental complexity](https://en.wikipedia.org/wiki/No_Silver_Bullet). Python has a huge library ecosystem. [`mcpyrate`](https://github.com/Technologicat/mcpyrate) and `unpythonic` are trying to push the language-level features a further 5%. (A full 100% is likely impossible when extending an existing language; if nothing else, there will be seams.) + +As for productivity, [it may be](https://medium.com/smalltalk-talk/lisp-smalltalk-and-the-power-of-symmetry-8bd96aaa0c0c) that a form of code-data equivalence (symmetry!), not macros specifically, is what makes Lisp powerful. If so, there may be other ways to reach that equivalence. For example Smalltalk, like Lisp, *runs in the same context it's written in*. All Smalltalk data are programs. Smalltalk [may be making a comeback](https://hackernoon.com/how-to-evangelize-a-programming-language-0p7p3y02), in the form of [Pharo](https://pharo.org/). + +Haskell aims at code-data equivalence from a third angle (memoized pure functions are in essence infinite lookup tables), but I have not used it in practice, so I do not have the experience to say whether this is enough to make it feel powerful in a similar way. + +Image-based programming (live programming) is a common factor between Pharo and Common Lisp + Swank. This is another productivity booster that much of the programming world is not that familiar with. It eliminates not only the edit/compile/restart cycle, but the edit/restart cycle as well, making the workflow a concurrent *edit/run* instead - without restarting the whole app at each change. Julia has [Revise.jl](https://github.com/timholy/Revise.jl) for something similar. In web applications, [REST](https://en.wikipedia.org/wiki/Representational_state_transfer) is a small step in a somewhat similar direction (as long as one can restart the server app easily, to make it use the latest definitions). Notebooks (such as [Jupyter](https://jupyter.org/)) provide the edit/run paradigm for scientific scripts. + +But to know exactly what Common Lisp has to offer, **yes**, it does make sense to learn it. As baroque as some parts are, there are a lot of great ideas there. [Conditions](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html) are one. [CLOS](http://www.gigamonkeys.com/book/object-reorientation-generic-functions.html) is another. (Nowadays [Julia](https://docs.julialang.org/en/v1/manual/methods/) has CLOS-style [multiple-dispatch generic functions](https://docs.julialang.org/en/v1/manual/methods/).) More widely, in the ecosystem, Swank is one. + +Having more perspectives at one's disposal makes one a better programmer - and that is what ultimately counts. As [Alan Perlis said in 1982](https://en.wikiquote.org/wiki/Alan_Perlis): + +*A language that doesn't affect the way you think about programming, is not worth knowing.* + +In this sense, Common Lisp is very much worth knowing. Although, if you want a beautiful, advanced Lisp, maybe go for [Racket](https://racket-lang.org/) first; but that is an essay for another day. + + +# `hoon`: The C of Functional Programming + +*9 June 2021* + +Some days I wonder if this whole `unpythonic` endeavor even makes any sense. Then, turning the pages of [the book of sand](https://en.wikipedia.org/wiki/The_Book_of_Sand) that is the web, I [happen to run into something](http://axisofeval.blogspot.com/2015/07/what-i-learned-about-urbit-so-far.html) like `hoon`. + +Its philosophy is best described by this gem from an [early version of its documentation](https://github.com/cgyarvin/urbit/blob/master/doc/book/0-intro.markdown#hoon): + +*So we could describe Hoon as a pure, strict, higher-order typed functional language. But don't do this in front of a Haskell purist, unless you put quotes around "typed," "functional," and possibly even "language." We could also say "object-oriented," with the same scare quotes for the cult of Eiffel.* + +While I am not sure if I will ever *use* `hoon`, it is hard not to like a language that puts quotes around "language". Few languages go that far in shaking up preconceptions. Critically examining what we believe, and why, often leads to useful insights. + +The claim that `hoon` is not a language, but a "language", fully makes sense after reading some of the documentation. `hoon` is essentially an *ab initio* language with an axiomatic approach to defining its operational semantics, similarly to how *Arc* approaches defining Lisp. Furthermore, `hoon` is the *functional equivalent of C* to the underlying virtual assembly language, `nock`. From a certain viewpoint, the "language" essentially consists of *glorified Nock macros*. Glorified assembly macros are pretty much all a *low-level* [HLL](https://en.wikipedia.org/wiki/High-level_programming_language) essentially is, so the claim seems about right. + +Nock is a peculiar assembly language. According to the comments in [`hoon.hoon`](https://github.com/cgyarvin/urbit/blob/master/urb/zod/arvo/hoon.hoon), it is a *Turing-complete non-lambda automaton*. The instruction set is permanently frozen, as if it was a physical CPU chip. Opcodes are just natural numbers, 0 through 11, and it is very minimalistic. For example, there is not even a decrement opcode. This is because from an axiomatic viewpoint, decrement can be defined recursively via increment. At which point, every systems programmer objects, rightfully, that no one sane actually does so, because that costs `O(n)`. Indeed, the `hoon` standard library uses C FFI to take advantage of the physical processor's instruction set to perform arithmetic operations. Each piece of C code used for such acceleration purposes is termed a *jet*. + +Since - by the fact that the programmer called a particular standard library function - the system knows we want to compute a decrement (or a multiplication, a power, maybe some floating point operation, etc.), it can *accelerate* that particular operation by using the available hardware. + +The important point is, you *could* write out a `nock` macro that does the same thing, only it would be unbearably slow. In the axiomatic perspective - which is about proving programs correct - speed does not matter. At the same time, FFI gives speed for the real world. + +To summarize; as someone already put it, `hoon` offers a glimpse into an alternative universe of systems programming, where the functional camp won. It may also be a useful tool, or a source for further unconventional ideas - but to know for sure, I will have to read more about it. + +I think the perfect place to end this piece is to quote a few lines from the language definition [`hoon.hoon`](https://github.com/cgyarvin/urbit/blob/master/urb/zod/arvo/hoon.hoon), to give a flavor: + +``` +++ doos :: sleep until + |= hap=path ^- (unit ,@da) + (doze:(wink:(vent bud (dink (dint hap))) now 0 (beck ~)) now [hap ~]) +:: +++ hurl :: start loop no id + |= ovo=ovum + ^- [p=(list ovum) q=(list ,[p=@tas q=vase])] + (kick [[~ [[(dint p.ovo) ~] p.ovo ~] q.ovo] ~]) +:: +++ hymn :: start loop with id + |= [who=ship ovo=ovum] + ^- [p=(list ovum) q=(list ,[p=@tas q=vase])] + (kick [[[~ %iron who] [[(dint p.ovo) ~] p.ovo ~] q.ovo] ~]) +:: +++ kick :: complete loop + |= mor=(list move) + =| ova=(list ovum) + |- ^- [p=(list ovum) q=(list ,[p=@tas q=vase])] + ?~ mor + [(flop ova) fan] + :: ~& [%kick-move q.i.mor -.r.i.mor] + ?> ?=(^ q.i.mor) + ?~ t.q.i.mor + $(mor t.mor, ova [[i.q.i.mor r.i.mor] ova]) + ?> ?=(^ i.q.i.mor) + =- $(mor (weld p.nyx t.mor), fan q.nyx) + ^= nyx + =+ naf=fan + |- ^- [p=(list move) q=_fan] + ?~ naf [~ ~] + ?. =(i.i.q.i.mor p.i.naf) + =+ tuh=$(naf t.naf) + [p.tuh [i.naf q.tuh]] + =+ ven=(vent bud q.i.naf) + =+ win=(wink:ven now (shax now) (beck p.i.mor)) + =+ ^= yub + %- beat:win + [p.i.mor t.i.q.i.mor t.q.i.mor r.i.mor] + [p.yub [[p.i.naf ves:q.yub] t.naf]] +-- +``` + +The Lisp family (particularly the Common Lisp branch) has a reputation for silly terminology, but I think `hoon` deserves the crown. All control structures are punctuation-only ASCII digraphs, and almost every name is a monosyllabic nonsense word. Still, this Lewis-Carroll-esque naming convention of making words mean what you define them to mean makes at least as much sense as the standard naming convention in mathematics, naming theorems after their discoverers! (Or at least, [after someone else](https://en.wikipedia.org/wiki/Stigler's_law_of_eponymy).) + +I actually like the phonemic base, making numbers sound like [*sorreg-namtyv*](https://urbit.org/docs/hoon/hoon-school/nouns/); that is 5 702 400 for the rest of us. And I think I will, quite seriously, adopt the verb *bunt*, meaning *to take the default value of*. That is such a common operation in programming that I find it hard to believe there is no standard abbreviation. I wonder what other discoveries await. + +Finally, in some way I cannot quite put a finger on, to me the style has echoes of [Jorge Luis Borges](https://en.wikipedia.org/wiki/Jorge_Luis_Borges). Maybe it is that the `hoon` source code sounds like something out of [The Library of Babel](https://en.wikipedia.org/wiki/The_Library_of_Babel). The Borgesian flavor seems intentional, too; the company building the Urbit stack, which `hoon` is part of, is itself named *[Tlon](https://en.wikipedia.org/wiki/Tl%C3%B6n%2C_Uqbar%2C_Orbis_Tertius)*. Remaking the world by re-imagining it, indeed. + +Maybe there is a place for `unpythonic`, too. + + +**Links** + +- [Latest documentation for `hoon`](https://urbit.org/docs/hoon/) +- There is a [whole operating system](https://github.com/urbit/urbit) built on `hoon` and `nock`. +- [Wikipedia has an entry on it](https://en.wikipedia.org/wiki/Urbit). Deconstructing the client-server model sounds very [postmodern](https://en.wikipedia.org/wiki/Deconstructivism). + + +**Note on natural-number opcodes** + +Using natural numbers for the opcodes at first glance sounds like a [Gödel numbering](https://en.wikipedia.org/wiki/G%C3%B6del_numbering) for the program space; but actually, the input to [the VM](https://urbit.org/docs/nock/definition/) contains some linked-list structure, which is not represented that way. Also, **any** programming language imposes its own Gödel numbering on the program space. Just take, for example, the UTF-8 representation of the source code text (which, in Python terms, is a `bytes` object), and interpret those bytes as one single bignum. + +Obviously, any interesting programs correspond to very large numbers, and are few and far between, so decoding random numbers via a Gödel numbering is not a practical way to generate interesting programs. [Genetic programming](https://en.wikipedia.org/wiki/Genetic_programming) works much better, because unlike Gödel numbering, it was actually designed specifically to do that. GP takes advantage of the semantic structure present in the source code (or AST) representation. + +The purpose of the original Gödel numbering was to prove Gödel's incompleteness theorem. In the case of `nock`, my impression is that the opcodes are natural numbers just for flavoring purposes. If you are building an ab initio software stack, what better way to announce that than to use natural numbers as your virtual machine's opcodes? diff --git a/doc/features.md b/doc/features.md index 470d71ae..4c3008ff 100644 --- a/doc/features.md +++ b/doc/features.md @@ -7,6 +7,7 @@ - [REPL server](repl.md) - [Troubleshooting](troubleshooting.md) - [Design notes](design-notes.md) +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) @@ -19,70 +20,123 @@ The exception are the features marked **[M]**, which are primarily intended as a ### Features [**Bindings**](#bindings) -- [``let``, ``letrec``: local bindings in an expression](#let-letrec-local-bindings-in-an-expression) **[M]** +- [`let`, `letrec`: local bindings in an expression](#let-letrec-local-bindings-in-an-expression) **[M]** + - [`let`](#let) + - [`dlet`, `blet`](#dlet-blet): *let-over-def*, like the classic let-over-lambda. + - [`letrec`](#letrec) - [Lispylet: alternative syntax](#lispylet-alternative-syntax) **[M]** -- [``env``: the environment](#env-the-environment) -- [``assignonce``](#assignonce), a relative of ``env``. -- [``dyn``: dynamic assignment](#dyn-dynamic-assignment) a.k.a. parameterize, special variables, fluid variables, "dynamic scoping". +- [`env`: the environment](#env-the-environment) +- [`assignonce`](#assignonce), a relative of `env`. +- [`dyn`: dynamic assignment](#dyn-dynamic-assignment) a.k.a. parameterize, special variables, fluid variables, "dynamic scoping". [**Containers**](#containers) -- [``frozendict``: an immutable dictionary](#frozendict-an-immutable-dictionary) +- [`frozendict`: an immutable dictionary](#frozendict-an-immutable-dictionary) - [`cons` and friends: pythonic lispy linked lists](#cons-and-friends-pythonic-lispy-linked-lists) -- [``box``: a mutable single-item container](#box-a-mutable-single-item-container) -- [``Shim``: redirect attribute accesses](#shim-redirect-attribute-accesses) -- [Container utilities](#container-utilities): ``get_abcs``, ``in_slice``, ``index_in_slice`` - -[**Sequencing**](#sequencing), run multiple expressions in any expression position (incl. inside a ``lambda``). -- [``begin``: sequence side effects](#begin-sequence-side-effects) -- [``do``: stuff imperative code into an expression](#do-stuff-imperative-code-into-an-expression) **[M]** -- [``pipe``, ``piped``, ``lazy_piped``: sequence functions](#pipe-piped-lazy_piped-sequence-functions) +- [`box`: a mutable single-item container](#box-a-mutable-single-item-container) + - [`box`](#box) + - [`Some`](#some): immutable box, to explicitly indicate the presence of a value. + - [`ThreadLocalBox`](#threadlocalbox) +- [`Shim`: redirect attribute accesses](#shim-redirect-attribute-accesses) +- [Container utilities](#container-utilities): `get_abcs`, `in_slice`, `index_in_slice` + +[**Sequencing**](#sequencing), run multiple expressions in any expression position (incl. inside a `lambda`). +- [`begin`: sequence side effects](#begin-sequence-side-effects) +- [`do`: stuff imperative code into an expression](#do-stuff-imperative-code-into-an-expression) **[M]** + - [`do`](#do) + - [`do0`](#do0) +- [`pipe`, `piped`, `lazy_piped`: sequence functions](#pipe-piped-lazy_piped-sequence-functions) + - [`pipe`](#pipe) + - [`piped`](#piped) + - [`lazy_piped`](#lazy_piped) [**Batteries**](#batteries) missing from the standard library. -- [**Batteries for functools**](#batteries-for-functools): `memoize`, `curry`, `compose`, `withself`, `fix` and more. - - [``curry`` and reduction rules](#curry-and-reduction-rules): we provide some extra features for bonus Haskellness. - - [``fix``: break infinite recursion cycles](#fix-break-infinite-recursion-cycles) +- [**Batteries for functools**](#batteries-for-functools): `curry`, `compose`, `withself`, and more. + - [`memoize`](#memoize): a detailed explanation of the memoizer. + - [`curry`](#curry): a detailed explanation of the curry utility and its haskelly extra features. + - [`fix`: break infinite recursion cycles](#fix-break-infinite-recursion-cycles) - [**Batteries for itertools**](#batteries-for-itertools): multi-input folds, scans (lazy partial folds); unfold; lazy partial unpacking of iterables, etc. - [**Batteries for network programming**](#batteries-for-network-programming): message protocol, PTY/socket proxy, etc. -- [``islice``: slice syntax support for ``itertools.islice``](#islice-slice-syntax-support-for-itertoolsislice) + - [`unpythonic.net.msg`](#unpythonic-net-msg): message protocol. +- [`islice`: slice syntax support for `itertools.islice`](#islice-slice-syntax-support-for-itertoolsislice) - [`gmemoize`, `imemoize`, `fimemoize`: memoize generators](#gmemoize-imemoize-fimemoize-memoize-generators), iterables and iterator factories. -- [``fup``: functional update; ``ShadowedSequence``](#fup-functional-update-shadowedsequence): like ``collections.ChainMap``, but for sequences. -- [``view``: writable, sliceable view into a sequence](#view-writable-sliceable-view-into-a-sequence) with scalar broadcast on assignment. -- [``mogrify``: update a mutable container in-place](#mogrify-update-a-mutable-container-in-place) -- [``s``, ``imathify``, ``gmathify``: lazy mathematical sequences with infix arithmetic](#s-imathify-gmathify-lazy-mathematical-sequences-with-infix-arithmetic) -- [``sym``, ``gensym``, ``Singleton``: symbols and singletons](#sym-gensym-Singleton-symbols-and-singletons) +- [`fup`: functional update; `ShadowedSequence`](#fup-functional-update-shadowedsequence): like `collections.ChainMap`, but for sequences. + - [`fup`](#fup): the high-level syntactic sugar to update a sequence functionally. + - [`fupdate`](#fupdate): the low-level workhorse. +- [`view`: writable, sliceable view into a sequence](#view-writable-sliceable-view-into-a-sequence) with scalar broadcast on assignment. +- [`mogrify`: update a mutable container in-place](#mogrify-update-a-mutable-container-in-place) +- [`s`, `imathify`, `gmathify`, `slift1`, `slift2`: lazy mathematical sequences with infix arithmetic](#s-imathify-gmathify-slift1-slift2-lazy-mathematical-sequences-with-infix-arithmetic) +- [`sym`, `gensym`, `Singleton`: symbols and singletons](#sym-gensym-Singleton-symbols-and-singletons) [**Control flow tools**](#control-flow-tools) -- [``trampolined``, ``jump``: tail call optimization (TCO) / explicit continuations](#trampolined-jump-tail-call-optimization-tco--explicit-continuations) -- [``looped``, ``looped_over``: loops in FP style (with TCO)](#looped-looped_over-loops-in-fp-style-with-tco) -- [``gtrampolined``: generators with TCO](#gtrampolined-generators-with-tco): tail-chaining; like ``itertools.chain``, but from inside a generator. -- [``catch``, ``throw``: escape continuations (ec)](#catch-throw-escape-continuations-ec) (as in [Lisp's `catch`/`throw`](http://www.gigamonkeys.com/book/the-special-operators.html), unlike C++ or Java) - - [``call_ec``: first-class escape continuations](#call_ec-first-class-escape-continuations), like Racket's ``call/ec``. -- [``forall``: nondeterministic evaluation](#forall-nondeterministic-evaluation), a tuple comprehension with multiple body expressions. -- [``handlers``, ``restarts``: conditions and restarts](#handlers-restarts-conditions-and-restarts), a.k.a. **resumable exceptions**. -- [``generic``, ``typed``, ``isoftype``: multiple dispatch](#generic-typed-isoftype-multiple-dispatch): create generic functions with type annotation syntax; also some friendly utilities. +- [`trampolined`, `jump`: tail call optimization (TCO) / explicit continuations](#trampolined-jump-tail-call-optimization-tco--explicit-continuations) + - [Tail recursion in a `lambda`](#tail-recursion-in-a-lambda) + - [Mutual recursion with TCO](#mutual-recursion-with-tco) + - [Mutual recursion in `letrec` with TCO](#mutual-recursion-in-letrec-with-tco) + - [Reinterpreting TCO as explicit continuations](#reinterpreting-tco-as-explicit-continuations) +- [`looped`, `looped_over`: loops in FP style (with TCO)](#looped-looped_over-loops-in-fp-style-with-tco) + - [Relation to the TCO system](#relation-to-the-tco-system) + - [FP loop over an iterable](#fp-loop-over-an-iterable): the `looped_over` parametric decorator + - [Accumulator type and runtime cost](#accumulator-type-and-runtime-cost) + - [`break`](#break) + - [`continue`](#continue) + - [Prepackaged `break` and `continue`](#prepackaged-break-and-continue) + - [FP loops using a lambda as body](#fp-loops-using-a-lambda-as-body) +- [`gtrampolined`: generators with TCO](#gtrampolined-generators-with-tco): tail-chaining; like `itertools.chain`, but from inside a generator. +- [`catch`, `throw`: escape continuations (ec)](#catch-throw-escape-continuations-ec) (as in [Lisp's `catch`/`throw`](http://www.gigamonkeys.com/book/the-special-operators.html), unlike C++ or Java) + - [`call_ec`: first-class escape continuations](#call_ec-first-class-escape-continuations), like Racket's `call/ec`. +- [`forall`: nondeterministic evaluation](#forall-nondeterministic-evaluation), a tuple comprehension with multiple body expressions. +- [Monads](#monads): Identity, Maybe, Either, List, Writer, State, Reader — plus `liftm`/`liftm2`/`liftm3`. For do-notation syntax, see the [`monadic_do` macro](macros.md#monadic_do-do-notation-for-any-monad). +- [`handlers`, `restarts`: conditions and restarts](#handlers-restarts-conditions-and-restarts), a.k.a. **resumable exceptions**. + - [Fundamental signaling protocol](#fundamental-signaling-protocol) + - [API summary](#api-summary) + - [High-level signaling protocols](#high-level-signaling-protocols) + - [Conditions vs. exceptions](#conditions-vs-exceptions) +- [`generic`, `typed`, `isoftype`: multiple dispatch](#generic-typed-isoftype-multiple-dispatch): create generic functions with type annotation syntax; also some friendly utilities. + - [`generic`: multiple dispatch with type annotation syntax](#generic-multiple-dispatch-with-type-annotation-syntax) + - [`augment`: add a new multimethod to an existing generic function](#augment-add-a-new-multimethod-to-an-existing-generic-function) + - [`typed`: add run-time type checks with type annotation syntax](#typed-add-run-time-type-checks-with-type-annotation-syntax) + - [`isoftype`: the big sister of `isinstance`](#isoftype-the-big-sister-of-isinstance) [**Exception tools**](#exception-tools) -- [``raisef``, ``tryf``: ``raise`` and ``try`` as functions](#raisef-tryf-raise-and-try-as-functions), useful inside a lambda. -- [``equip_with_traceback``](#equip-with-traceback), equip a manually created exception instance with a traceback. -- [``async_raise``: inject an exception to another thread](#async_raise-inject-an-exception-to-another-thread) *(CPython only)* +- [`raisef`, `tryf`: `raise` and `try` as functions](#raisef-tryf-raise-and-try-as-functions), useful inside a lambda. +- [`withf`: `with` as a function](#withf-with-as-a-function), useful inside a lambda. +- [`equip_with_traceback`](#equip-with-traceback), equip a manually created exception instance with a traceback. +- [`async_raise`: inject an exception to another thread](#async_raise-inject-an-exception-to-another-thread) *(CPython only)* - [`reraise_in`, `reraise`: automatically convert exception types](#reraise_in-reraise-automatically-convert-exception-types) +[**Function call and return value tools**](#function-call-and-return-value-tools) +- [`def` as a code block: `@call`](#def-as-a-code-block-call): run a block of code immediately, in a new lexical scope. +- [`@callwith`: freeze arguments, choose function later](#callwith-freeze-arguments-choose-function-later) +- [Spreading a `Values` into `call` / `callwith`](#spreading-a-values-into-call--callwith) +- [`Values`: multiple and named return values](#values-multiple-and-named-return-values) + - [`valuify`](#valuify): convert pythonic multiple-return-values idiom of `tuple` into `Values`. + +[**Numerical tools**](#numerical-tools) + - [`almosteq`: floating-point almost-equality](#almosteq-floating-point-almost-equality) + - [`fixpoint`: arithmetic fixed-point finder](#fixpoint-arithmetic-fixed-point-finder) + - [`partition_int`: partition integers](#partition_int-partition-integers) + - [`ulp`: unit in last place](#ulp-unit-in-last-place) + - [`si_prefix`: format numbers with SI or IEC prefixes](#si_prefix-format-numbers-with-si-or-iec-prefixes) + [**Other**](#other) -- [``def`` as a code block: ``@call``](#def-as-a-code-block-call): run a block of code immediately, in a new lexical scope. -- [``@callwith``: freeze arguments, choose function later](#callwith-freeze-arguments-choose-function-later) -- [``callsite_filename``](#callsite-filename) -- [``safeissubclass``](#safeissubclass), convenience function. -- [``pack``: multi-arg constructor for tuple](#pack-multi-arg-constructor-for-tuple) -- [``namelambda``: rename a function](#namelambda-rename-a-function) -- [``timer``: a context manager for performance testing](#timer-a-context-manager-for-performance-testing) -- [``getattrrec``, ``setattrrec``: access underlying data in an onion of wrappers](#getattrrec-setattrrec-access-underlying-data-in-an-onion-of-wrappers) -- [``arities``, ``kwargs``, ``resolve_bindings``: Function signature inspection utilities](#arities-kwargs-resolve_bindings-function-signature-inspection-utilities) -- [``Popper``: a pop-while iterator](#popper-a-pop-while-iterator) -- [``ulp``: unit in last place](#ulp-unit-in-last-place) +- [`callsite_filename`](#callsite-filename) +- [`safeissubclass`](#safeissubclass), convenience function. +- [`environ_override`: temporarily override environment variables](#environ_override-temporarily-override-environment-variables) +- [`maybe_open`: open a file or use a fallback stream](#maybe_open-open-a-file-or-use-a-fallback-stream) +- [`redirect_stdin`: feed `sys.stdin` from a stream](#redirect_stdin-feed-sysstdin-from-a-stream) +- [`UnionFilter`: OR-combine logging filters](#unionfilter-or-combine-logging-filters) +- [`pack`: multi-arg constructor for tuple](#pack-multi-arg-constructor-for-tuple) +- [`namelambda`: rename a function](#namelambda-rename-a-function) +- [`timer`: a context manager for performance testing](#timer-a-context-manager-for-performance-testing) +- [`format_human_time`: seconds to days, hours, minutes, seconds](#format_human_time-seconds-to-days-hours-minutes-seconds) +- [`ETAEstimator`: estimate the time of completion of a long-running task](#etaestimator-estimate-the-time-of-completion-of-a-long-running-task) +- [`getattrrec`, `setattrrec`: access underlying data in an onion of wrappers](#getattrrec-setattrrec-access-underlying-data-in-an-onion-of-wrappers) +- [`arities`, `kwargs`, `resolve_bindings`: Function signature inspection utilities](#arities-kwargs-resolve_bindings-function-signature-inspection-utilities) +- [`Popper`: a pop-while iterator](#popper-a-pop-while-iterator) For many examples, see [the unit tests](unpythonic/tests/), the docstrings of the individual features, and this guide. -*This document doubles as the API reference, but despite maintenance on a best-effort basis, may occasionally be out-of-date at places. In case of conflicts in documentation, believe the unit tests first; specifically the code, not necessarily the comments. Everything else (comments, docstrings and this guide) should agree with the unit tests. So if something fails to work as advertised, check what the tests say - and optionally file an issue on GitHub so that the documentation can be fixed.* +*This document doubles as the API reference, but despite maintenance on a best-effort basis, may occasionally be out-of-date at places. In case of conflicts in documentation, believe the unit tests first; specifically the code, not necessarily the comments. Everything else (comments, docstrings and this guide) should agree with the unit tests. So if something fails to work as advertised, check what the tests do - and optionally file an issue on GitHub so that the documentation can be fixed.* **This document is up-to-date for v0.15.0.** @@ -90,17 +144,20 @@ For many examples, see [the unit tests](unpythonic/tests/), the docstrings of th Tools to bind identifiers in ways not ordinarily supported by Python. -### ``let``, ``letrec``: local bindings in an expression +### `let`, `letrec`: local bindings in an expression + +**NOTE**: *This is primarily a code generation target API for the `let[]` family of [macros](macros.md), which make the constructs easier to use, and make the code look almost like normal Python. Below is the documentation for the raw API.* -**NOTE**: This is primarily a code generation target API for the ``let[]`` family of [macros](macros.md), which make the constructs easier to use. Below is the documentation for the raw API. +The `let` constructs introduce bindings local to an expression, like Scheme's `let` and `letrec`. -Introduces bindings local to an expression, like Scheme's ``let`` and ``letrec``. For easy-to-use versions of these constructs that look almost like normal Python, see [our macros](macros.md). +#### `let` -In ``let``, the bindings are independent (do not see each other). A binding is of the form ``name=value``, where ``name`` is a Python identifier, and ``value`` is any expression. +In `let`, the bindings are independent (do not see each other). A binding is of the form `name=value`, where `name` is a Python identifier, and `value` is any expression. Use a `lambda e: ...` to supply the environment to the body: ```python +# These six are the constructs covered in this section of documentation. from unpythonic import let, letrec, dlet, dletrec, blet, bletrec u = lambda lst: let(seen=set(), @@ -112,9 +169,11 @@ u(L) # --> [1, 3, 2, 4] Generally speaking, `body` is a one-argument function, which takes in the environment instance as the first positional parameter (by convention, named `e` or `env`). In typical inline usage, `body` is `lambda e: expr`. -*Let over lambda*. Here the inner ``lambda`` is the definition of the function ``counter``: +*Let over lambda*. Here the inner `lambda` is the definition of the function `counter`: ```python +from unpythonic import let, begin + counter = let(x=0, body=lambda e: lambda: @@ -124,6 +183,21 @@ counter() # --> 1 counter() # --> 2 ``` +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, let, do + +counter = let[[x << 0] in + (lambda: + do[x << x + 1, + x])] +counter() # --> 1 +counter() # --> 2 +``` + +(*The parentheses around the lambda are just to make the expression into syntactically valid Python. You can also use brackets instead, denoting a multiple-expression `let` body - which is also valid even if there is just one expression. The `do` makes a multiple-expression `lambda` body. For more, see the [macro documentation](macros.md).*) + Compare the sweet-exp [Racket](http://racket-lang.org/) (see [SRFI-110](https://srfi.schemers.org/srfi-110/srfi-110.html) and [sweet](https://docs.racket-lang.org/sweet/)): ```racket @@ -136,9 +210,13 @@ counter() ; --> 1 counter() ; --> 2 ``` -*Let over def* decorator ``@dlet``, to *let over lambda* more pythonically: +#### `dlet`, `blet` + +*Let over def* decorator `@dlet`, to *let over lambda* more pythonically: ```python +from unpythonic import dlet + @dlet(x=0) def counter(*, env=None): # named argument "env" filled in by decorator env.x += 1 @@ -147,9 +225,30 @@ counter() # --> 1 counter() # --> 2 ``` -In `letrec`, bindings may depend on ones above them in the same `letrec`, by using `lambda e: ...` (**Python 3.6+**): +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, dlet + +@dlet(x << 0) +def counter(): + x << x + 1 + return x +counter() # --> 1 +counter() # --> 2 +``` + +The `@blet` decorator is otherwise the same as `@dlet`, but instead of decorating a function definition in the usual manner, it runs the `def` block immediately, and upon exit, replaces the function definition with the return value. The name `blet` is an abbreviation of *block let*, since the role of the `def` is just a code block to be run immediately. + +#### `letrec` + +The name of this construct comes from the Scheme family of Lisps, and stands for *let (mutually) recursive*. The "[mutually recursive](https://en.wikipedia.org/wiki/Mutual_recursion)" refers to the kind of scoping between the bindings in the same `letrec`. + +In plain English, in `letrec`, the value of a binding may depend on other bindings in the same `letrec`. The raw API in `unpythonic` uses a `lambda e: ...` to provide the environment: ```python +from unpythonic import letrec + x = letrec(a=1, b=lambda e: e.a + 1, @@ -157,13 +256,27 @@ x = letrec(a=1, e.b) # --> 2 ``` -In `letrec`, the ``value`` of each binding is either a simple value (non-callable, and doesn't use the environment), or an expression of the form ``lambda e: valexpr``, providing access to the environment as ``e``. If ``valexpr`` itself is callable, the binding **must** have the ``lambda e: ...`` wrapper to prevent any misunderstandings in the environment initialization procedure. +The ordering of the definitions is respected, because Python 3.6 and later preserve the ordering of named arguments passed in a function call. See [PEP 468](https://www.python.org/dev/peps/pep-0468/). + +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, letrec + +x = letrec[[a << 1, + b << a + 1] in + b] +``` + +In the non-macro `letrec`, the `value` of each binding is either a simple value (non-callable, and doesn't use the environment), or an expression of the form `lambda e: valexpr`, providing access to the environment as `e`. If `valexpr` itself is callable, the binding **must** have the `lambda e: ...` wrapper to prevent misinterpretation by the machinery when the environment initialization procedure runs. -In a non-callable ``valexpr``, trying to depend on a binding below it raises ``AttributeError``. +In a non-callable `valexpr`, trying to depend on a binding below it raises `AttributeError`. -A callable ``valexpr`` may depend on any bindings (also later ones) in the same `letrec`. Mutually recursive functions: +A callable `valexpr` may depend on any bindings (**also later ones**) in the same `letrec`. For example, here is a pair of [mutually recursive](https://en.wikipedia.org/wiki/Mutual_recursion) functions: ```python +from unpythonic import letrec + letrec(evenp=lambda e: lambda x: (x == 0) or e.oddp(x - 1), @@ -174,9 +287,24 @@ letrec(evenp=lambda e: e.evenp(42)) # --> True ``` +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, letrec + +letrec[[evenp << (lambda x: + (x == 0) or oddp(x - 1)), + oddp << (lambda x: + (x != 0) and evenp(x - 1))] in + evenp(42)] # --> True +``` + + Order-preserving list uniqifier: ```python +from unpythonic import letrec, begin + u = lambda lst: letrec(seen=set(), see=lambda e: lambda x: @@ -186,19 +314,30 @@ u = lambda lst: letrec(seen=set(), [e.see(x) for x in lst if x not in e.seen]) ``` -**CAUTION**: in Pythons older than 3.6, bindings are **initialized in an arbitrary order**, also in `letrec`. This is a limitation of the kwargs abuse. Hence mutually recursive functions are possible, but a non-callable `valexpr` cannot depend on other bindings in the same `letrec`. +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, letrec, do + +u = lambda lst: letrec[[seen << set(), + see << (lambda x: + do[seen.add(x), + x])] in + [[see(x) for x in lst if x not in seen]]] +``` + +(*The double brackets around the `letrec` body are needed because brackets denote a multiple-expression `letrec` body. So it is a multiple-expression body that contains just one expression, which is a list comprehension.*) -Trying to access `e.foo` from `e.bar` arbitrarily produces either the intended value of `e.foo`, or the uninitialized `lambda e: ...`, depending on whether `e.foo` has been initialized or not at the point of time when `e.bar` is being initialized. +The decorators `@dletrec` and `@bletrec` work otherwise exactly like `@dlet` and `@blet`, respectively, but the bindings are scoped like in `letrec` (mutually recursive scope). -This has been fixed in Python 3.6, see [PEP 468](https://www.python.org/dev/peps/pep-0468/). #### Lispylet: alternative syntax -**NOTE**: This is primarily a code generation target API for the ``let[]`` family of [macros](macros.md), which make the constructs easier to use. Below is the documentation for the raw API. +**NOTE**: *This is primarily a code generation target API for the `let[]` family of [macros](macros.md), which make the constructs easier to use. Below is the documentation for the raw API.* The `lispylet` module was originally created to allow guaranteed left-to-right initialization of `letrec` bindings in Pythons older than 3.6, hence the positional syntax and more parentheses. The only difference is the syntax; the behavior is identical with the other implementation. As of 0.15, the main role of `lispylet` is to act as the run-time backend for the `let` family of macros. -These constructs are available in the top-level `unpythonic` namespace, with the ``ordered_`` prefix: ``ordered_let``, ``ordered_letrec``, ``ordered_dlet``, ``ordered_dletrec``, ``ordered_blet``, ``ordered_bletrec``. +These constructs are available in the top-level `unpythonic` namespace, with the `ordered_` prefix: `ordered_let`, `ordered_letrec`, `ordered_dlet`, `ordered_dletrec`, `ordered_blet`, `ordered_bletrec`. It is also possible to override the default `let` constructs by the `ordered_` variants, like this: @@ -223,12 +362,37 @@ letrec((("evenp", lambda e: The syntax is `let(bindings, body)` (respectively `letrec(bindings, body)`), where `bindings` is `((name, value), ...)`, and `body` is like in the default variants. The same rules concerning `name` and `value` apply. +For comparison, with the macro API, the above becomes: + +```python +from unpythonic.syntax import macros, letrec + +letrec[[a << 1, + b << a + 1] in + b] + +letrec[[evenp << (lambda x: + (x == 0) or oddp(x - 1)), + oddp << (lambda x: + (x != 0) and evenp(x - 1))] in + evenp(42)] # --> True +``` + +(*The transformations made by the macros may be the most apparent when comparing these examples. Note that the macros scope the let-bindings lexically, automatically figuring out which `let` environment, if any, to refer to.*) -### ``env``: the environment -The environment used by all the ``let`` constructs and ``assignonce`` (but **not** by `dyn`) is essentially a bunch with iteration, subscripting and context manager support. It is somewhat similar to [`types.SimpleNamespace`](https://docs.python.org/3/library/types.html#types.SimpleNamespace), but with many extra features. For details, see `unpythonic.env`. +### `env`: the environment -Our ``env`` allows things like: +**Changed in v0.15.2.** *`env` objects are now pickleable.* + +The environment used by all the `let` constructs and `assignonce` (but **not** by `dyn`) is essentially a bunch with iteration, subscripting and context manager support. It is somewhat similar to [`types.SimpleNamespace`](https://docs.python.org/3/library/types.html#types.SimpleNamespace), but with many extra features. For details, see `unpythonic.env.env` (and note the unfortunate module name). + +**Import it as `from unpythonic.env import env`.** The module shares its name with the class, and the +top-level package does not re-export the class, so `from unpythonic import env` hands you the +*module* — after which `env(x=1)` fails with `TypeError: 'module' object is not callable`. The error +does not hint at the cause, so it is worth knowing in advance. + +Our `env` allows things like: ```python let(x=1, y=2, z=3, @@ -264,10 +428,12 @@ When the `with` block exits, the environment clears itself. The environment inst (This allows using `with env(...) as e:` as a poor man's `let`, if you have a block of statements you want to locally scope some names to, but don't want to introduce a `def`.) -``env`` provides the ``collections.abc.Mapping`` and ``collections.abc.MutableMapping`` APIs. +`env` provides the `collections.abc.Mapping` and `collections.abc.MutableMapping` APIs. + +### `assignonce` -### ``assignonce`` +*As of v0.15.0, `assignonce` is mostly a standalone curiosity that has never been integrated with the rest of `unpythonic`. But anything that works with arbitrary subclasses of `env`, for example `mogrify`, works with it, too.* In Scheme terms, make `define` and `set!` look different: @@ -281,18 +447,22 @@ with assignonce() as e: e.foo = "quux" # AttributeError, e.foo already defined. ``` -It's a subclass of ``env``, so it shares most of the same [features](#env-the-environment) and allows similar usage. +The `assignonce` construct is a subclass of `env`, so it shares most of the same [features](#env-the-environment) and allows similar usage. #### Historical note The fact that in Python creating bindings and updating (rebinding) them look the same was already noted in 2000, in [PEP 227](https://www.python.org/dev/peps/pep-0227/#discussion), which introduced true closures to Python 2.1. For related history concerning the `nonlocal` keyword, see [PEP 3104](https://www.python.org/dev/peps/pep-3104/). -### ``dyn``: dynamic assignment +### `dyn`: dynamic assignment -([As termed by Felleisen.](https://groups.google.com/forum/#!topic/racket-users/2Baxa2DxDKQ) Other names seen in the wild for variants of this feature include *parameters* (not to be confused with function parameters), *special variables*, *fluid variables*, *fluid let*, and even the misnomer *"dynamic scoping"*.) +**Changed in v0.14.2.** *To bring this in line with [SRFI-39](https://srfi.schemers.org/srfi-39/srfi-39.html), `dyn` now supports rebinding, using assignment syntax such as `dyn.x = 42`, and the function `dyn.update(x=42, y=17, ...)`.* + +([As termed by Felleisen.](https://groups.google.com/forum/#!topic/racket-users/2Baxa2DxDKQ) Other names seen in the wild for variants of this feature include *parameters* ([Scheme](https://srfi.schemers.org/srfi-39/srfi-39.html) and [Racket](https://docs.racket-lang.org/reference/parameters.html); not to be confused with function parameters), *special variables* (Common Lisp), *fluid variables*, *fluid let* (e.g. Emacs Lisp), and even the misnomer *"dynamic scoping"*.) -Like global variables, but better-behaved. Useful for sending some configuration parameters through several layers of function calls without changing their API. Best used sparingly. +The feature itself is *dynamic assignment*; the things it creates are *dynamic variables* (a.k.a. *dynvars*). + +Dynvars are like global variables, but better-behaved. Useful for sending some configuration parameters through several layers of function calls without changing their API. Best used sparingly. There's a singleton, `dyn`: @@ -322,46 +492,44 @@ def g(): g() ``` -Dynamic variables (a.k.a. *dynvars*) are created using `with dyn.let(k0=v0, ...)`. The syntax is in line with the nature of the assignment, which is in effect *for the dynamic extent* of the `with`. Exiting the `with` block pops the dynamic environment stack. Inner dynamic environments shadow outer ones. +Dynvars are created using `with dyn.let(k0=v0, ...)`. The syntax is in line with the nature of the assignment, which is in effect *for the dynamic extent* of the `with`. Exiting the `with` block pops the dynamic environment stack. Inner dynamic environments shadow outer ones. -The point of dynamic assignment is that dynvars are seen also by code that is outside the lexical scope where the `with dyn.let` resides. The use case is to avoid a function parameter definition cascade, when you need to pass some information through several layers that don't care about it. This is especially useful for passing "background" information, such as plotter settings in scientific visualization, or the macro expander instance in metaprogramming. +The point of dynamic assignment is that dynvars are seen also by code that is *outside the lexical scope* where the `with dyn.let` resides. The use case is to avoid a function parameter definition cascade, when you need to pass some information through several layers that do not care about it. This is especially useful for passing "background" information, such as plotter settings in scientific visualization, or the macro expander instance in metaprogramming. -To give a dynvar a top-level default value, use ``make_dynvar(k0=v0, ...)``. Usually this is done at the top-level scope of the module for which that dynvar is meaningful. Each dynvar, of the same name, should only have one default set; the (dynamically) latest definition always overwrites. However, we do not prevent overwrites, because in some codebases the same module may run its top-level initialization code multiple times (e.g. if a module has a ``main()`` for tests, and the file gets loaded both as a module and as the main program). +To give a dynvar a top-level default value, use `make_dynvar(k0=v0, ...)`. Usually this is done at the top-level scope of the module for which that dynvar is meaningful. Each dynvar, of the same name, should only have one default set; the (dynamically) latest definition always overwrites. However, we do not prevent overwrites, because in some codebases the same module may run its top-level initialization code multiple times (e.g. if a module has a `main()` for tests, and the file gets loaded both as a module and as the main program). -To rebind existing dynvars, use `dyn.k = v`, or `dyn.update(k0=v0, ...)`. Rebinding occurs in the closest enclosing dynamic environment that has the target name bound. If the name is not bound in any dynamic environment (including the top-level one), ``AttributeError`` is raised. +To rebind existing dynvars, use `dyn.k = v`, or `dyn.update(k0=v0, ...)`. Rebinding occurs in the closest enclosing dynamic environment that has the target name bound. If the name is not bound in any dynamic environment (including the top-level one), `AttributeError` is raised. -**CAUTION**: Use rebinding of dynvars carefully, if at all. Stealth updates of dynvars defined in an enclosing dynamic extent can destroy any chance of statically reasoning about the code. +**CAUTION**: Use rebinding of dynvars carefully, if at all. Stealth updates of dynvars defined in an enclosing dynamic extent can destroy any chance of statically reasoning about your code. There is no `set` function or `<<` operator, unlike in the other `unpythonic` environments. -**Changed in v0.14.2.** *To bring this in line with [SRFI-39](https://srfi.schemers.org/srfi-39/srfi-39.html), `dyn` now supports rebinding, using assignment syntax such as `dyn.x = 42`, and the function `dyn.update(x=42, y=17, ...)`.* +
Each thread has its own dynamic scope stack. There is also a global dynamic scope for default values, shared between threads. -
-Each thread has its own dynamic scope stack. There is also a global dynamic scope for default values, shared between threads. A newly spawned thread automatically copies the then-current state of the dynamic scope stack **from the main thread** (not the parent thread!). Any copied bindings will remain on the stack for the full dynamic extent of the new thread. Because these bindings are not associated with any `with` block running in that thread, and because aside from the initial copying, the dynamic scope stacks are thread-local, any copied bindings will never be popped, even if the main thread pops its own instances of them. -The source of the copy is always the main thread mainly because Python's `threading` module gives no tools to detect which thread spawned the current one. (If someone knows a simple solution, PRs welcome!) +The source of the copy is always the main thread mainly because Python's `threading` module gives no tools to detect which thread spawned the current one. (If someone knows a simple solution, a PR is welcome!) -Finally, there is one global dynamic scope shared between all threads, where the default values of dynvars live. The default value is used when ``dyn`` is queried for the value outside the dynamic extent of any ``with dyn.let()`` blocks. Having a default value is convenient for eliminating the need for ``if "x" in dyn`` checks, since the variable will always exist (after the global definition has been executed). +Finally, there is one global dynamic scope shared between all threads, where the default values of dynvars live. The default value is used when `dyn` is queried for the value outside the dynamic extent of any `with dyn.let()` blocks. Having a default value is convenient for eliminating the need for `if "x" in dyn` checks, since the variable will always exist (at any time after the global definition has been executed).
-For more details, see the methods of ``dyn``; particularly noteworthy are ``asdict`` and ``items``, which give access to a *live view* to dyn's contents in a dictionary format (intended for reading only!). The ``asdict`` method essentially creates a ``collections.ChainMap`` instance, while ``items`` is an abbreviation for ``asdict().items()``. The ``dyn`` object itself can also be iterated over; this creates a ``ChainMap`` instance and redirects to iterate over it. ``dyn`` also provides the ``collections.abc.Mapping`` API. +For more details, see the methods of `dyn`; particularly noteworthy are `asdict` and `items`, which give access to a *live view* to dyn's contents in a dictionary format (intended for reading only!). The `asdict` method essentially creates a `collections.ChainMap` instance, while `items` is an abbreviation for `asdict().items()`. The `dyn` object itself can also be iterated over; this creates a `ChainMap` instance and redirects to iterate over it. `dyn` also provides the `collections.abc.Mapping` API. -To support dictionary-like idioms in iteration, dynvars can alternatively be accessed by subscripting; ``dyn["x"]`` has the same meaning as ``dyn.x``, so you can do things like: +To support dictionary-like idioms in iteration, dynvars can alternatively be accessed by subscripting; `dyn["x"]` has the same meaning as `dyn.x`, to allow things like: ```python print(tuple((k, dyn[k]) for k in dyn)) ``` -Finally, ``dyn`` supports membership testing as ``"x" in dyn``, ``"y" not in dyn``, where the string is the name of the dynvar whose presence is being tested. +Finally, `dyn` supports membership testing as `"x" in dyn`, `"y" not in dyn`, where the string is the name of the dynvar whose presence is being tested. For some more details, see [the unit tests](../unpythonic/tests/test_dynassign.py). ### Relation to similar features in Lisps -This is essentially [SRFI-39: Parameter objects](https://srfi.schemers.org/srfi-39/), using the MzScheme approach in the presence of multiple threads. +This is essentially [SRFI-39: Parameter objects](https://srfi.schemers.org/srfi-39/) for Python, using the MzScheme approach in the presence of multiple threads. -[Racket](http://racket-lang.org/)'s [`parameterize`](https://docs.racket-lang.org/guide/parameterize.html) behaves similarly. However, Racket seems to be the state of the art in many lispy language design related things, so its take on the feature may have some finer points I haven't thought of. +[Racket](http://racket-lang.org/)'s [`parameterize`](https://docs.racket-lang.org/guide/parameterize.html) behaves similarly. However, Racket seems to be the state of the art in many lispy language design related things, so its take on the feature may have some finer points I have not thought of. On Common Lisp's special variables, see [Practical Common Lisp by Peter Seibel](http://www.gigamonkeys.com/book/variables.html), especially footnote 10 in the linked chapter, for a definition of terms. Similarly, dynamic variables in our `dyn` have *indefinite scope* (because `dyn` is implemented as a module-level global, accessible from anywhere), but *dynamic extent*. @@ -370,13 +538,15 @@ So what we have in `dyn` is almost exactly like Common Lisp's special variables, ## Containers -We provide some additional containers. +We provide some additional low-level containers beyond those provided by Python itself. The class names are lowercase, because these are intended as low-level utility classes in principle on par with the builtins. The immutable containers are hashable. All containers are pickleable (if their contents are). -### ``frozendict``: an immutable dictionary +### `frozendict`: an immutable dictionary -Given the existence of ``dict`` and ``frozenset``, this one is oddly missing from the standard library. +**Changed in 0.14.2**. *[A bug in `frozendict` pickling](https://github.com/Technologicat/unpythonic/issues/55) has been fixed. Now also the empty `frozendict` pickles and unpickles correctly.* + +Given the existence of `dict` and `frozenset`, this one is oddly missing from the language. ```python from unpythonic import frozendict @@ -402,7 +572,7 @@ assert d4['a'] == 23 and d4['b'] == 2 assert d3['a'] == 42 and d3['b'] == 2 # ...of course without touching the original ``` -Any mappings used when creating an instance are shallow-copied, so that the bindings of the ``frozendict`` do not change even if the original input is later mutated: +Any mappings used when creating an instance are shallow-copied, so that the bindings of the `frozendict` do not change even if the original input is later mutated: ```python d = {1:2, 3:4} @@ -414,7 +584,7 @@ assert fd == {1: 2, 3: 4} **The usual caution** concerning immutable containers in Python applies: the container protects only the bindings against changes. If the values themselves are mutable, the container cannot protect from mutations inside them. -All the usual read-access stuff works: +All the usual read-access features work: ```python d7 = frozendict({1:2, 3:4}) @@ -433,7 +603,7 @@ assert d7.get(5, 0) == 0 assert d7.get(5) is None ``` -In terms of ``collections.abc``, a ``frozendict`` is a hashable immutable mapping: +In terms of `collections.abc`, a `frozendict` is a hashable immutable mapping: ```python assert issubclass(frozendict, Mapping) @@ -444,21 +614,21 @@ assert hash(d7) == hash(frozendict({1:2, 3:4})) assert hash(d7) != hash(frozendict({1:2})) ``` -The abstract superclasses are virtual, just like for ``dict`` (i.e. they do not appear in the MRO). +The abstract superclasses are virtual, just like for `dict`. We mean *virtual* in the sense of [`abc.ABCMeta`](https://docs.python.org/3/library/abc.html#abc.ABCMeta), i.e. a virtual superclass does not appear in the MRO. -Finally, ``frozendict`` obeys the empty-immutable-container singleton invariant: +Finally, `frozendict` obeys the empty-immutable-container singleton invariant: ```python assert frozendict() is frozendict() ``` -**Changed in 0.14.2**. *[A bug in `frozendict` pickling](https://github.com/Technologicat/unpythonic/issues/55) has been fixed. Now also the empty `frozendict` pickles and unpickles correctly.* - ### `cons` and friends: pythonic lispy linked lists *Laugh, it's funny.* +**Changed in v0.14.2.** *`nil` is now a `Singleton`, so it is treated correctly by `pickle`. The `nil` instance refresh code inside the `cons` class has been removed, so the previous caveat about pickling a standalone `nil` value no longer applies.* + ```python from unpythonic import (cons, nil, ll, llist, car, cdr, caar, cdar, cadr, cddr, @@ -494,13 +664,13 @@ assert lzip(ll(1, 2, 3), ll(4, 5, 6)) == ll(ll(1, 4), ll(2, 5), ll(3, 6)) Cons cells are immutable à la Racket (no `set-car!`/`rplaca`, `set-cdr!`/`rplacd`). Accessors are provided up to `caaaar`, ..., `cddddr`. -Although linked lists are created with ``ll`` or ``llist``, the data type (for e.g. ``isinstance``) is ``cons``. +Although linked lists are created with the functions `ll` or `llist`, the data type (for e.g. `isinstance`) is `cons`. -Iterators are supported to walk over linked lists (this also gives sequence unpacking support). When ``next()`` is called, we return the car of the current cell the iterator points to, and the iterator moves to point to the cons cell in the cdr, if any. When the cdr is not a cons cell, it is the next (and last) item returned; except if it `is nil`, then iteration ends without returning the `nil`. +Iterators are supported, to walk over linked lists. This also gives sequence unpacking support. When `next()` is called, we return the `car` of the current cell the iterator points to, and the iterator moves to point to the cons cell in the `cdr`, if any. When the `cdr` is not a cons cell, it is the next (and last) item returned; except if it `is nil`, then iteration ends without returning the `nil`. -Python's builtin ``reversed`` can be applied to linked lists; it will internally ``lreverse`` the list (which is O(n)), then return an iterator to that. The ``llist`` constructor is special-cased so that if the input is ``reversed(some_ll)``, it just returns the internal already reversed list. (This is safe because cons cells are immutable.) +Python's builtin `reversed` can be applied to linked lists; it will internally `lreverse` the list (which is O(n)), then return an iterator to that. The `llist` constructor is special-cased so that if the input is `reversed(some_ll)`, it just returns the internal already reversed list. (This is safe because cons cells are immutable.) -Cons structures, by default, print in a pythonic format suitable for ``eval`` (if all elements are): +Cons structures, by default, print in a pythonic format suitable for `eval` (if all elements are): ```python print(cons(1, 2)) # --> cons(1, 2) @@ -516,26 +686,24 @@ print(ll(1, 2, 3).lispyrepr()) # --> (1 2 3) print(cons(cons(1, 2), cons(3, 4)).lispyrepr()) # --> ((1 . 2) . (3 . 4)) ``` -For more, see the ``llist`` submodule. +For more, see the `llist` submodule. #### Notes -There is no ``copy`` method or ``lcopy`` function, because cons cells are immutable; which makes cons structures immutable. +There is no `copy` method or `lcopy` function, because cons cells are immutable; which makes cons structures immutable. -(However, for example, it is possible to ``cons`` a new item onto an existing linked list; that's fine because it produces a new cons structure - which shares data with the original, just like in Racket.) +However, for example, it is possible to `cons` a new item onto an existing linked list; that is fine, because it produces a new cons structure - which shares data with the original, just like in Racket. In general, copying cons structures can be error-prone. Given just a starting cell it is impossible to tell if a given instance of a cons structure represents a linked list, or something more general (such as a binary tree) that just happens to locally look like one, along the path that would be traversed if it was indeed a linked list. -The linked list iteration strategy does not recurse in the ``car`` half, which could lead to incomplete copying. The tree strategy that recurses on both halves, on the other hand, will flatten nested linked lists and produce also the final ``nil``. - -We provide a ``JackOfAllTradesIterator`` as a compromise that understands both trees and linked lists. Nested lists will be flattened, and in a tree any ``nil`` in a ``cdr`` position will be omitted from the output. ``BinaryTreeIterator`` and ``JackOfAllTradesIterator`` use an explicit data stack instead of implicitly using the call stack for keeping track of the recursion. All ``cons`` iterators work for arbitrarily deep cons structures without causing Python's call stack to overflow, and without the need for TCO. +The linked list iteration strategy does not recurse in the `car` half, which could lead to incomplete copying. The tree strategy that recurses on both halves, on the other hand, will flatten nested linked lists and produce also the final `nil`. -``cons`` has no ``collections.abc`` virtual superclasses (except the implicit ``Hashable`` since ``cons`` provides ``__hash__`` and ``__eq__``), because general cons structures do not fit into the contracts represented by membership in those classes. For example, size cannot be known without iterating, and depends on which iteration scheme is used (e.g. ``nil`` dropping, flattening); which scheme is appropriate depends on the content. +We provide a `JackOfAllTradesIterator` as a compromise that understands both trees and linked lists. Nested lists will be flattened, and in a tree any `nil` in a `cdr` position will be omitted from the output. `BinaryTreeIterator` and `JackOfAllTradesIterator` use an explicit data stack instead of implicitly using the call stack for keeping track of the recursion. All `cons` iterators work for arbitrarily deep cons structures without causing Python's call stack to overflow, and without the need for TCO. -**Caution**: the ``nil`` singleton is freshly created in each session; newnil is not oldnil, so don't pickle a standalone ``nil``. The unpickler of ``cons`` automatically refreshes any ``nil`` instances inside a pickled cons structure, so that **cons structures** support the illusion that ``nil`` is a special value like ``None`` or ``...``. After unpickling, ``car(c) is nil`` and ``cdr(c) is nil`` still work as expected, even though ``id(nil)`` has changed between sessions. +`cons` has no `collections.abc` virtual superclasses (except the implicit `Hashable` since `cons` provides `__hash__` and `__eq__`), because general cons structures do not fit into the contracts represented by membership in those classes. For example, size cannot be known without iterating, and depends on which iteration scheme is used (e.g. `nil` dropping, flattening); which scheme is appropriate depends on the content. -### ``box``: a mutable single-item container +### `box`: a mutable single-item container **Changed in v0.14.2**. *The `box` container API is now `b.set(newvalue)` to rebind, returning the new value as a convenience. The equivalent syntactic sugar is `b << newvalue`. The item inside the box can be extracted with `b.get()`. The equivalent syntactic sugar is `unbox(b)`.* @@ -545,7 +713,9 @@ We provide a ``JackOfAllTradesIterator`` as a compromise that understands both t **Changed in v0.14.2**. *Accessing the `.x` attribute of a `box` directly is now deprecated. It will continue to work with `box` at least until 0.15, but it does not and cannot work with `ThreadLocalBox`, which must handle things differently due to implementation reasons. Use the API mentioned above; it supports both kinds of boxes with the same syntax.* -No doubt anyone programming in an imperative language has run into the situation caricatured by this highly artificial example: +#### `box` + +Consider this highly artificial example: ```python animal = "dog" @@ -557,9 +727,9 @@ f(animal) assert animal == "dog" ``` -Many solutions exist. Common pythonic ones are abusing a ``list`` to represent a box (and then trying to manually remember that it is supposed to hold only a single item), or (if the lexical structure of the particular piece of code allows it) using the ``global`` or ``nonlocal`` keywords to tell Python, on assignment, to overwrite a name that already exists in a surrounding scope. +Many solutions exist. Common pythonic ones are abusing a `list` to represent a box (and then trying to remember that it is supposed to hold only a single item), or (if the lexical structure of the particular piece of code allows it) using the `global` or `nonlocal` keywords to tell Python, on assignment, to overwrite a name that already exists in a surrounding scope. -As an alternative to the rampant abuse of lists, we provide a rackety ``box``, which is a minimalistic mutable container that holds exactly one item. Any code that has a reference to the box can update the data in it: +As an alternative to the rampant abuse of lists, we provide a rackety `box`, which is a minimalistic mutable container that holds exactly one item. Any code that has a reference to the box can update the data in it: ```python from unpythonic import box, unbox @@ -591,7 +761,7 @@ f("dog") Here `g` *effectively rebinds a local variable of `f`* - whether that is a good idea is a separate question, but technically speaking, this would not be possible without a container. As mentioned, abusing a `list` is the standard Python (but not very pythonic!) solution. Using specifically a `box` makes the intent explicit. -The ``box`` API is summarized by: +The `box` API is summarized by: ```python from unpythonic import box, unbox @@ -624,13 +794,23 @@ box3.set("fox") # same without syntactic sugar assert "fox" in box3 ``` -The expression ``item in b`` has the same meaning as ``unbox(b) == item``. Note ``box`` is a **mutable container**, so it is **not hashable**. +The expression `item in b` has the same meaning as `unbox(b) == item`. Note `box` is a **mutable container**, so it is **not hashable**. -The expression `unbox(b)` has the same meaning as `b.get()`, but because it is a function (instead of a method), it additionally sanity checks that `b` is a box, and if not, raises `TypeError`. +The expression `unbox(b)` has the same meaning as `b.get()`, but because it is a function (instead of a method), it additionally sanity-checks that `b` is a box, and if not, raises `TypeError`. The expression `b << newitem` has the same meaning as `b.set(newitem)`. In both cases, the new value is returned as a convenience. -`ThreadLocalBox` is otherwise exactly like `box`, but it's magic: its contents are thread-local. It also holds a default object, which is set initially when the `ThreadLocalBox` is instantiated. The default object is seen by threads that have not placed any object into the box. +#### `Some` + +We also provide an **immutable** box, `Some`. This can be useful to represent optional data. + +The idea is that the value, when present, is placed into a `Some`, such as `Some(42)`, `Some("cat")`, `Some(myobject)`. Then, the situation where the value is absent can be represented as a bare `None`. So specifically, `Some(None)` means that a value is present and this value is `None`, whereas a bare `None` means that there is no value. + +It is like the `Some` constructor of a `Maybe` monad, but with no monadic magic. In this interpretation, the bare constant `None` plays the role of `Nothing`. + +#### `ThreadLocalBox` + +`ThreadLocalBox` is otherwise exactly like `box`, but magical: its contents are thread-local. It also holds a default object, which is set initially when the `ThreadLocalBox` is instantiated. The default object is seen by threads that have not placed any object into the box. ```python from unpythonic import ThreadLocalBox, unbox @@ -687,18 +867,16 @@ tlb.clear() # When we clear the box in this thread... assert unbox(tlb) == "cat" # ...this thread sees the current default object again. ``` -We also provide an **immutable** box, `Some`. This can be useful for optional data. The idea is that the value, when present, is placed into a `Some`, such as `Some(42)`, `Some("cat")`, `Some(myobject)`. Then, the situation where the value is absent can be represented as a bare `None`. So specifically, `Some(None)` means that a value is present and this value is `None`, whereas a bare `None` means that there is no value. - -### ``Shim``: redirect attribute accesses +### `Shim`: redirect attribute accesses **Added in v0.14.2**. -A `Shim` is an attribute access proxy. The shim holds a `box` (or a `ThreadLocalBox`), and redirects attribute accesses on the shim to whatever object happens to currently be in the box. The point is that the object in the box can be replaced with a different one later (by sending another object into the box), and the code accessing the proxied object through the shim doesn't need to be aware that anything has changed. +A `Shim` is an *attribute access proxy*. The shim holds a `box` (or a `ThreadLocalBox`; your choice), and redirects attribute accesses on the shim to whatever object happens to currently be in the box. The point is that the object in the box can be replaced with a different one later (by sending another object into the box), and the code accessing the proxied object through the shim does not need to be aware that anything has changed. -For example, this can combo with `ThreadLocalBox` to redirect standard output only in particular threads. Place the stream object in a `ThreadLocalBox`, shim that box, then replace `sys.stdout` with the shim. See the source code of `unpythonic.net.server` for an example that actually does (and cleanly undoes) this. +For example, `Shim` can combo with `ThreadLocalBox` to redirect standard output only in particular threads. Place the stream object in a `ThreadLocalBox`, shim that box, then replace `sys.stdout` with the shim. See the source code of `unpythonic.net.server` for an example that actually does (and cleanly undoes) this. -Since deep down, attribute access is the whole point of objects, `Shim` is essentially a transparent object proxy. (For example, a method call is an attribute read (via a descriptor), followed by a function call.) +Since deep down, attribute access is the whole point of objects, `Shim` is essentially a transparent object proxy. (For example, a method call is an attribute read (via a [descriptor](https://docs.python.org/3/howto/descriptor.html)), followed by a function call.) ```python from unpythonic import Shim, box, unbox @@ -729,9 +907,9 @@ assert s.getme() == 42 assert not hasattr(s, "y") # The new TestTarget instance doesn't have "y". ``` -A shim can have an optional fallback object. It can be either any object, or a box if you want to replace the fallback later. **For attribute reads** (i.e. `__getattr__`), if the object in the primary box does not have the requested attribute, `Shim` will try to get it from the fallback. If `fallback` is boxed, the attribute read takes place on the object in the box. If it is not boxed, the attribute read takes place directly on `fallback`. +A shim can have an optional fallback object. It can be either any object, or a `box` (or `ThreadLocalBox`) if you want to replace the fallback later. **For attribute reads** (i.e. `__getattr__`), if the object in the primary box does not have the requested attribute, `Shim` will try to get it from the fallback. If `fallback` is boxed, the attribute read takes place on the object in the box. If it is not boxed, the attribute read takes place directly on `fallback`. -Any **attribute writes** (i.e. `__setattr__`, binding or rebinding an attribute) always take place on the object in the primary box. +Any **attribute writes** (i.e. `__setattr__`, binding or rebinding an attribute) always take place on the object in the **primary** box. That is, binding or rebinding of attributes is never performed on the fallback object. ```python from unpythonic import Shim, box, unbox @@ -774,9 +952,34 @@ assert s.y == "hi from Wai" assert s.z == "hi from Zee" ``` +Or, since the operation takes just one `elt` and an `acc`, we can also use `reducer` instead of `foldr`, shortening this by one line: + +```python +from unpythonic import Shim, box, unbox, reducer + +class Ex: + x = "hi from Ex" +class Wai: + x = "hi from Wai" + y = "hi from Wai" +class Zee: + x = "hi from Zee" + y = "hi from Zee" + z = "hi from Zee" + + # There will be tried from left to right. +boxes = [box(obj) for obj in (Ex(), Wai(), Zee())] +s = reducer(Shim, boxes) # Shim(box, fallback) <-> op(elt, acc) +assert s.x == "hi from Ex" +assert s.y == "hi from Wai" +assert s.z == "hi from Zee" +``` + ### Container utilities +**Changed in v0.15.0.** *The sequence length argument in `in_slice`, `index_in_slice` is now named `length`, not `l` (ell). This avoids an E741 warning in `flake8`, and is more descriptive.* + **Inspect the superclasses** that a particular container type has: ```python @@ -784,7 +987,7 @@ from unpythonic import get_abcs print(get_abcs(list)) ``` -This includes virtual superclasses, i.e. those that are not part of the MRO. This works by ``issubclass(cls, v)`` on all classes defined in ``collections.abc``. +This includes virtual superclasses, i.e. those that are not part of the MRO. This works by `issubclass(cls, v)` on all classes defined in `collections.abc`. **Reflection on slices**: @@ -805,41 +1008,53 @@ An optional length argument can be given to interpret negative indices. See the Sequencing refers to running multiple expressions, in sequence, in place of one expression. -Keep in mind the only reason to ever need multiple expressions: *side effects.* (Assignment is a side effect, too; it modifies the environment. In functional style, intermediate named definitions to increase readability are perhaps the most useful kind of side effect.) +Keep in mind the only reason to ever need multiple expressions: *side effects.* Assignment is a side effect, too; it modifies the environment. In functional style, intermediate named definitions to increase readability are perhaps the most useful kind of side effect. + +See also `multilambda` in [macros](macros.md). -See also ``multilambda`` in [macros](macros.md). +### `begin`: sequence side effects -### ``begin``: sequence side effects +**CAUTION**: the `begin` family of forms are provided **for use in pure-Python projects only**, and are a permanent part of the `unpythonic` API for that purpose. They are somewhat simpler and less flexible than the `do` family, described further below. -**CAUTION**: the `begin` family of forms are provided **for use in pure-Python projects only** (and are a permanent part of the `unpythonic` API for that purpose). If your project uses macros, prefer the `do[]` and `do0[]` macros; these are the only sequencing constructs understood by other macros in `unpythonic.syntax` that need to perform tail-position analysis (e.g. `tco`, `autoreturn`, `continuations`). The `do[]` and `do0[]` macros also provide some convenience features, such as expression-local variables. +*If your project uses macros, prefer the `do[]` and `do0[]` macros; those are the only sequencing constructs understood by other macros in `unpythonic.syntax` that need to perform tail-position analysis (e.g. `tco`, `autoreturn`, `continuations`). The `do[]` and `do0[]` macros also provide some convenience features, such as expression-local variables.* ```python from unpythonic import begin, begin0 f1 = lambda x: begin(print("cheeky side effect"), - 42*x) + 42 * x) f1(2) # --> 84 -f2 = lambda x: begin0(42*x, +f2 = lambda x: begin0(42 * x, print("cheeky side effect")) f2(2) # --> 84 ``` -Actually a tuple in disguise. If worried about memory consumption, use `lazy_begin` and `lazy_begin0` instead, which indeed use loops. The price is the need for a lambda wrapper for each expression to delay evaluation, see [`unpythonic.seq`](../unpythonic/seq.py) for details. +The `begin` and `begin0` forms are actually tuples in disguise; evaluation of **all** items occurs before the `begin` or `begin0` form gets control. Items are evaluated left-to-right due to Python's argument passing rules. +We provide also `lazy_begin` and `lazy_begin0`, which use loops. The price is the need for a lambda wrapper for each expression to delay evaluation. See the module [`unpythonic.seq`](../unpythonic/seq.py) for details. -### ``do``: stuff imperative code into an expression -**NOTE**: This is primarily a code generation target API for the ``do[]`` [macro](macros.md), which makes the construct easier to use. Below is the documentation for the raw API. +### `do`: stuff imperative code into an expression -No monadic magic. Basically, ``do`` is: +**NOTE**: *This is primarily a code generation target API for the `do[]` and `do0[]` [macros](macros.md), which make the constructs easier to use, and make the code look almost like normal Python. Below is the documentation for the raw API.* - - An improved ``begin`` that can bind names to intermediate results and then use them in later items. +Basically, the `do` family is a more advanced and flexible variant of the `begin` family. - - A ``let*`` (technically, ``letrec``) where making a binding is optional, so that some items can have only side effects if so desired. No semantically distinct ``body``; all items play the same role. + - `do` can bind names to intermediate results and then use them in later items. -Like in ``letrec`` (see below), use ``lambda e: ...`` to access the environment, and to wrap callable values (to prevent misunderstandings). + - `do` is effectively a `let*` (technically, `letrec`) where making a binding is optional, so that some items can have only side effects if so desired. There is no semantically distinct `body`; all items play the same role. + + - Despite the name, there is no monadic magic. + +Like in `letrec`, use `lambda e: ...` to access the environment, and to wrap callable values (to prevent misinterpretation by the machinery). + +Unlike `begin` (and `begin0`), there is no separate `lazy_do` (`lazy_do0`), because using a `lambda e: ...` wrapper for an item will already delay its evaluation; and the main point of `do`/`do0` is that there is an environment that holds local definitions. If you want a lazy variant, just wrap each item with a `lambda e: ...`, also those that don't otherwise need it. + +#### `do` + +Like `begin` and `lazy_begin`, the `do` form evaluates all items in order, and then returns the value of the **last** item. ```python from unpythonic import do, assign @@ -852,7 +1067,7 @@ y = do(assign(x=17), # create and set e.x assert y == 42 y = do(assign(x=17), - assign(z=lambda e: 2*e.x), + assign(z=lambda e: 2 * e.x), lambda e: e.z) assert y == 34 @@ -863,16 +1078,89 @@ y = do(assign(x=5), assert y == 25 ``` -If you need to return the first value instead of the last one, use this trick: +For comparison, with the macro API, this becomes: + +```python +from unpythonic.syntax import macros, do, local + +y = do[local[x << 17], # create and set an x local to the environment + print(x), + x << 23, # overwrite x + print(x), + 42] # return value +assert y == 42 + +y = do[local[x << 17], + local[z << 2 * x], + z] +assert y == 34 + +y = do[local[x << 5], + local[f << (lambda x: x**2)], + print("hello from 'do'"), + f(x)] +assert y == 25 +``` + +*In the macro version, all items are delayed automatically; that is, **every** item has an implicit `lambda e: ...`. Note that instead of the `assign` function, the macro version uses the syntax `local[name << value]` to **create** an expression-local variable. Updating an existing variable in the `do` environment is just `name << value`. Finally, there is also `delete[name]`.* + +When using the raw API, beware of this pitfall: + +```python +from unpythonic import do + +do(lambda e: print("hello 2 from 'do'"), # delayed because lambda e: ... + print("hello 1 from 'do'"), # Python prints immediately before do() + "foo") # gets control, because technically, it is + # **the return value** that is an argument + # for do(). +``` + +The above pitfall also applies to using escape continuations inside a `do`. To do that, wrap the ec call into a `lambda e: ...` to delay its evaluation until the `do` actually runs: + +```python +from unpythonic import call_ec, do, assign + +call_ec( + lambda ec: + do(assign(x=42), + lambda e: ec(e.x), # IMPORTANT: must delay this! + lambda e: print("never reached"))) # and this (as above) +``` + +This way, any assignments made in the `do` (which occur only after `do` gets control), performed above the line with the `ec` call, will have been performed when the `ec` is called. + +For comparison, with the macro API, the last example becomes: + +```python +from unpythonic.syntax import macros, do, local +from unpythonic import call_ec + +call_ec( + lambda ec: + do[local[x << 42], + ec(x), + print("never reached")]) +``` + +*In the macro version, all items are delayed automatically, so there `do`/`do0` gets control before any items are evaluated. The `ec` fires when the `do` evaluates that item, and the `print` is indeed never reached.* + +#### `do0` + +Like `begin0` and `lazy_begin0`, the `do0` form evaluates all items in order, and then returns the value of the **first** item. + +It effectively does this internally: ```python +from unpythonic import do, assign + y = do(assign(result=17), print("assigned 'result' in env"), lambda e: e.result) # return value assert y == 17 ``` -Or use ``do0``, which does it for you: +So we can write: ```python from unpythonic import do0, assign @@ -888,39 +1176,54 @@ y = do0(assign(x=17), # the first item of do0 can be an assignment, too assert y == 17 ``` -Beware of this pitfall: +For comparison, with the macro API, this becomes: ```python -do(lambda e: print("hello 2 from 'do'"), # delayed because lambda e: ... - print("hello 1 from 'do'"), # Python prints immediately before do() - "foo") # gets control, because technically, it is - # **the return value** that is an argument - # for do(). -``` +from unpythonic.syntax import macros, do, local -Unlike ``begin`` (and ``begin0``), there is no separate ``lazy_do`` (``lazy_do0``), because using a ``lambda e: ...`` wrapper will already delay evaluation of an item. If you want a lazy variant, just wrap each item (also those which don't otherwise need it). +y = do[local[result << 17], + print("assigned 'result' in env"), + result] +assert y == 17 -The above pitfall also applies to using escape continuations inside a ``do``. To do that, wrap the ec call into a ``lambda e: ...`` to delay its evaluation until the ``do`` actually runs: +y = do0[17, + local[x << 42], + print(x), + print("hello from 'do0'")] +assert y == 17 -```python -call_ec( - lambda ec: - do(assign(x=42), - lambda e: ec(e.x), # IMPORTANT: must delay this! - lambda e: print("never reached"))) # and this (as above) +y = do0[local[x << 17], + print(x)] +assert y == 17 ``` -This way, any assignments made in the ``do`` (which occur only after ``do`` gets control), performed above the line with the ``ec`` call, will have been performed when the ``ec`` is called. +### `pipe`, `piped`, `lazy_piped`: sequence functions + +**Changed in v0.15.0.** *Multiple return values and named return values, for unpacking to the args and kwargs of the next function in the pipe, as well as in the final return value from the pipe, are now represented as a `Values`.* + +*The variants `pipe` and `pipec` now expect a `Values` initial value if you want to unpack it into the args and kwargs of the first function in the pipe. Otherwise, the initial value is sent as a single positional argument (notably tuples too).* + +*The variants `piped` and `lazy_piped` automatically pack the initial arguments into a `Values`.* + +*The deprecated names `getvalue` and `runpipe` have been removed.* + +**Changed in v0.14.2**. *Both `getvalue` and `runpipe`, used in the shell-like syntax, are now known by the single unified name `exitpipe`. This is just a rename, with no functionality changes. The old names are now deprecated.* + +Similar to Racket's [threading macros](https://docs.racket-lang.org/threading/), but no macros. A pipe performs a sequence of operations, starting from an initial value, and then returns the final value. It is just function composition, but with an emphasis on data flow, which helps improve readability. -### ``pipe``, ``piped``, ``lazy_piped``: sequence functions +Both one-in-one-out (*1-to-1*) and n-in-m-out (*n-to-m*) pipes are provided. The 1-to-1 versions have names suffixed with `1`, and they are slightly faster than the general versions. The use case is one-argument functions that return one value. -**Changed in v0.15.0.** Multiple return values and named return values, for passing on to the next function in the pipe, as well as in the final return value from the pipe, are now represented as a `Values`. +In the n-to-m versions, when a function returns a `Values`, it is unpacked to the args and kwargs of the next function in the pipeline. When a pipe exits, the `Values` wrapper (if any) around the final result is discarded if it contains only one positional value. The main use case is computations that deal with multiple values, the number of which may also change during the computation (as long as the args/kwargs of each output `Values` can be accepted as input by the next function in the pipe). -Similar to Racket's [threading macros](https://docs.racket-lang.org/threading/). A pipe performs a sequence of operations, starting from an initial value, and then returns the final value. It's just function composition, but with an emphasis on data flow, which helps improve readability: +Additional examples can be found in [the unit tests](../unpythonic/tests/test_seq.py). + +#### `pipe` + +The function `pipe` represents a self-contained pipeline that starts from a given value (or values), applies some operations in sequence, and then exits: ```python -from unpythonic import pipe +from unpythonic import pipe, Values double = lambda x: 2 * x inc = lambda x: x + 1 @@ -929,11 +1232,43 @@ x = pipe(42, double, inc) assert x == 85 ``` -We also provide ``pipec``, which curries the functions before applying them. Useful with passthrough (see below on ``curry``). +To pass several positional values and/or named values, use a `Values` object: + +```python +from unpythonic import pipe, Values + +a, b = pipe(Values(2, 3), + lambda x, y: Values(x=(x + 1), y=(2 * y)), + lambda x, y: Values(x * 2, y + 1)) +assert (a, b) == (6, 7) +``` + +In this example, we pass the initial values positionally into the first function in the pipeline; that function passes its return values by name; and the second function in the pipeline passes the final results positionally. Because there are only positional values in the final `Values` object, it can be unpacked like a tuple. + +#### `pipec` + +The function `pipec` is otherwise exactly like `pipe`, but it curries the functions before applying them. This is useful with the passthrough feature of `curry`. + +With `pipec` you can do things like: + +```python +from unpythonic import pipec, Values + +a, b = pipec(Values(1, 2), + lambda x: x + 1, # extra values passed through by curry (positionals on the right) + lambda x, y: Values(x * 2, y + 1)) +assert (a, b) == (4, 3) +``` + +For more on passthrough, see the section on `curry`. + +#### `piped` -Optional **shell-like syntax**, with purely functional updates. +We also provide a **shell-like syntax**, with purely functional updates. -**Changed in v0.14.2**. *Both `getvalue` and `runpipe` are now known by the single unified name `exitpipe`. This is just a rename, with no functionality changes. The old names are now deprecated, and will be removed in 0.15.0.* +To set up a pipeline for use with the shell-like syntax, call `piped` to load the initial value(s). It is possible to provide both positional and named values. Each use of the pipe operator applies the given function, but keeps the result inside the pipeline, ready to accept another function. + +When done, pipe into the sentinel `exitpipe` to exit the pipeline and return the current value(s): ```python from unpythonic import piped, exitpipe @@ -946,9 +1281,33 @@ assert p | inc | exitpipe == 85 assert p | exitpipe == 84 # p itself is never modified by the pipe system ``` -Set up a pipe by calling ``piped`` for the initial value. Pipe into the sentinel ``exitpipe`` to exit the pipe and return the current value. +Multiple values work like in `pipe`, except the initial value(s) passed to `piped` are automatically packed into a `Values`. The pipe system then automatically unpacks a `Values` object into the args/kwargs of the next function in the pipeline. + +To return multiple positional values and/or named values, return a `Values` object from your function. + +When `exitpipe` is applied, if the last function returned anything other than one positional value, you will get a `Values` object. + +```python +from unpythonic import piped, exitpipe, Values + +f = lambda x, y: Values(2 * x, y + 1) +g = lambda x, y: Values(x + 1, 2 * y) +x = piped(2, 3) | f | g | exitpipe # --> (5, 8) +assert x == Values(5, 8) +``` + +Unpacking works also here, because in the final result, there are only positional values: -**Lazy pipes**, useful for mutable initial values. To perform the planned computation, pipe into the sentinel ``exitpipe``: +```python +from unpythonic import piped, exitpipe + +a, b = piped(2, 3) | f | g | exitpipe # --> (5, 8) +assert (a, b) == (5, 8) +``` + +#### `lazy_piped` + +Lazy pipes are useful when you have mutable initial values. To perform the planned computation, pipe into the sentinel `exitpipe`: ```python from unpythonic import lazy_piped1, exitpipe @@ -971,22 +1330,17 @@ from unpythonic import lazy_piped, exitpipe fibos = [] def nextfibo(a, b): # multiple arguments allowed fibos.append(a) # store result by side effect - # New state, handed to next function in the pipe. + # New state, handed to the next function in the pipe. # As of v0.15.0, use `Values(...)` to represent multiple return values. # Positional args will be passed positionally, named ones by name. - return Values(a=b, b=a + b) + return Values(a=b, b=(a + b)) p = lazy_piped(1, 1) # load initial state for _ in range(10): # set up pipeline p = p | nextfibo -p | exitpipe -assert (p | exitpipe) == Values(a=89, b=144) # final state +assert (p | exitpipe) == Values(a=89, b=144) # run; check final state assert fibos == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55] ``` -Both one-in-one-out (*1-to-1*) and n-in-m-out (*n-to-m*) pipes are provided. The 1-to-1 versions have names suffixed with ``1``. The use case is one-argument functions that return one value (which may also be a tuple). - -In the n-to-m versions, when a function returns a tuple, it is unpacked to the argument list of the next function in the pipe. At ``exitpipe`` time, the tuple wrapper (if any) around the final result is discarded if it contains only one item. (This allows the n-to-m versions to work also with a single value, as long as it is not a tuple.) The main use case is computations that deal with multiple values, the number of which may also change during the computation (as long as there are as many "slots" on both sides of each individual connection). - ## Batteries @@ -994,73 +1348,35 @@ Things missing from the standard library. ### Batteries for functools - - `memoize`: - - Caches also exceptions à la Racket. If the memoized function is called again with arguments with which it raised an exception the first time, the same exception instance is raised again. - - Works also on instance methods, with results cached separately for each instance. - - This is essentially because ``self`` is an argument, and custom classes have a default ``__hash__``. - - Hence it doesn't matter that the memo lives in the ``memoized`` closure on the class object (type), where the method is, and not directly on the instances. The memo itself is shared between instances, but calls with a different value of ``self`` will create unique entries in it. - - For a solution that performs memoization at the instance level, see [this ActiveState recipe](https://github.com/ActiveState/code/tree/master/recipes/Python/577452_memoize_decorator_instance) (and to demystify the magic contained therein, be sure you understand [descriptors](https://docs.python.org/3/howto/descriptor.html)). - - `curry`, with some extra features: - - **Changed in v0.15.0.** `curry` supports both positional and named arguments, and binds arguments to function parameters like Python itself does. The call triggers when all parameters are bound, regardless of whether they were passed by position or by name, and at which step of the currying process they were passed. - - **Changed in v0.15.0.** `unpythonic`'s multiple-dispatch system (`@generic`, `@typed`) is supported. `curry` looks for an exact match first, then a match with extra args/kwargs, and finally a partial match. If there is still no match, this implies that at least one parameter would get a binding that fails the type check. In such a case `TypeError` regarding failed multiple dispatch is raised. - - **Changed in v0.15.0.** If the function being curried is `@generic` or `@typed`, or has type annotations on its parameters, the parameters being passed in are type-checked. A type mismatch immediately raises `TypeError`. This helps support [fail-fast](https://en.wikipedia.org/wiki/Fail-fast) in code using `curry`. - - Passthrough for args/kwargs that are incompatible with the target function's call signature (à la Haskell; or [spicy](https://github.com/Technologicat/spicy) for Racket). - - Here *incompatible* means too many positional args, or named args that have no corresponding parameter. (Note that if the function has a `**kwargs` parameter, then all named args are considered compatible, because it absorbs anything.) - - Multiple return values (both positional and named) are denoted using `Values` (which see). A standard return value is considered to consist of one positional return value only. - - Positional args are passed through **on the right**. Any positional return values of the curried function are prepended, on the left. - - If the first positional return value of an intermediate result of a passthrough is callable, it is (curried and) invoked on the remaining args and kwargs, after merging the rest of the return values into the args and kwargs. This helps with some instances of [point-free style](https://en.wikipedia.org/wiki/Tacit_programming). - - If more args/kwargs are still remaining when the top-level curry context exits, by default ``TypeError`` is raised. - - To override, set the dynvar ``curry_context``. It is a list representing the stack of currently active curry contexts. A context is any object, a human-readable label is fine. See below for an example. - - To set the dynvar, `from unpythonic import dyn`, and then `with dyn.let(curry_context=...):`. - - Can be used both as a decorator and as a regular function. - - As a regular function, `curry` itself is curried à la Racket. If it gets extra arguments (beside the function ``f``), they are the first step. This helps eliminate many parentheses. - - **Caution**: If the signature of ``f`` cannot be inspected, currying fails, raising ``ValueError``, like ``inspect.signature`` does. This may happen with builtins such as ``list.append``, ``operator.add``, ``print``, or ``range``, depending on which version of Python you have (and whether CPython or PyPy3). - - **Added in v0.15.0.** `partial` with run-time type checking, which helps a lot with fail-fast in code that uses partial application. This function type-checks arguments against type annotations, then delegates to `functools.partial`. Supports `unpythonic`'s `@generic` and `@typed` functions, too. + - `memoize`, with exception caching. + - `curry`, with passthrough like in Haskell. + - `fix`: detect and break infinite recursion cycles. **Added in v0.14.2.** + - `partial` with run-time type checking, which helps a lot with fail-fast in code that uses partial application. This function type-checks arguments against type annotations, then delegates to `functools.partial`. Supports `unpythonic`'s `@generic` and `@typed` functions, too. **Added in v0.15.0.** - `composel`, `composer`: both left-to-right and right-to-left function composition, to help readability. - - Any number of positional arguments is supported, with the same rules as in the pipe system. Multiple return values packed into a tuple are unpacked to the argument list of the next function in the chain. - - `composelc`, `composerc`: curry each function before composing them. Useful with passthrough. - - An implicit top-level curry context is inserted around all the functions except the one that is applied last. - - `composel1`, `composer1`: 1-in-1-out chains (faster; also useful for a single value that is a tuple). + - **Changed in v0.15.0.** *For the benefit of code using the `with lazify` macro, the compose functions are now marked lazy. Arguments will be forced only when a lazy function in the chain actually uses them, or when an eager (not lazy) function is encountered in the chain.* + - Any number of positional and keyword arguments are supported, with the same rules as in the pipe system. Multiple return values, or named return values, represented as a `Values`, are automatically unpacked to the args and kwargs of the next function in the chain. + - `composelc`, `composerc`: curry each function before composing them. This comboes well with the passthrough of extra args/kwargs in `curry`. + - An implicit top-level curry context is inserted around all the functions except the one that is applied last, to allow passthrough to the top level while applying the composed function. + - `composel1`, `composer1`: 1-in-1-out chains (faster). - suffix `i` to use with an iterable that contains the functions (`composeli`, `composeri`, `composelci`, `composerci`, `composel1i`, `composer1i`) - `withself`: essentially, the Y combinator trick as a decorator. Allows a lambda to refer to itself. - - The ``self`` argument is declared explicitly, but passed implicitly (as the first positional argument), just like the ``self`` argument of a method. - - `apply`: the lispy approach to starargs. Mainly useful with the ``prefix`` [macro](macros.md). + - The `self` argument is declared explicitly, but passed implicitly (as the first positional argument), just like the `self` argument of a method. + - `apply`: the lispy approach to starargs. Mainly useful with the `prefix` [macro](macros.md). - `andf`, `orf`, `notf`: compose predicates (like Racket's `conjoin`, `disjoin`, `negate`). + - **Changed in v0.15.0.** *For the benefit of code using the `with lazify` macro, `andf` and `orf` are now marked lazy. Arguments will be forced only when a lazy predicate in the chain actually uses them, or when an eager (not lazy) predicate is encountered in the chain.* - `flip`: reverse the order of positional arguments. - `rotate`: a cousin of `flip`. Permute the order of positional arguments in a cycle. - `to1st`, `to2nd`, `tokth`, `tolast`, `to` to help inserting 1-in-1-out functions into m-in-n-out compose chains. (Currying can eliminate the need for these.) - `identity`, `const` which sometimes come in handy when programming with higher-order functions. - - `fix`: detect and break infinite recursion cycles. **Added in v0.14.2.** -Examples (see also the next section): +We will discuss `memoize`, `curry` and `fix` in more detail shortly; but first, we will give some examples of the other utilities. Note that as always, more examples can be found in [the unit tests](../unpythonic/tests/test_fun.py). ```python -from operator import add, mul from typing import NoReturn -from unpythonic import (memoize, fix, andf, orf, flatmap, rotate, curry, dyn, - zipr, rzip, foldl, foldr, composer, to1st, cons, nil, ll, - withself) - -# memoize: cache the results of pure functions (arguments must be hashable) -ncalls = 0 -@memoize # <-- important part -def square(x): - global ncalls - ncalls += 1 - return x**2 -assert square(2) == 4 -assert ncalls == 1 -assert square(3) == 9 -assert ncalls == 2 -assert square(3) == 9 -assert ncalls == 2 # called only once for each unique set of arguments -assert square(x=3) == 9 -assert ncalls == 2 # only the resulting bindings matter, not how you pass the args - - # "memoize lambda": classic evaluate-at-most-once thunk -thunk = memoize(lambda: print("hi from thunk")) -thunk() # the message is printed only the first time -thunk() +from unpythonic import (fix, andf, orf, rotate, + foldl, foldr, + withself, + composel) # detect and break infinite recursion cycles: # a(0) -> b(1) -> a(2) -> b(0) -> a(1) -> b(2) -> a(0) -> ... @@ -1072,6 +1388,7 @@ def b(k): return a((k + 1) % 3) assert a(0) is NoReturn # the call does return, saying the original function wouldn't. +# andf, orf: short-circuiting predicate combinators isint = lambda x: isinstance(x, int) iseven = lambda x: x % 2 == 0 isstr = lambda s: isinstance(s, str) @@ -1094,93 +1411,254 @@ myzipr = curry(foldr, zipper, ()) assert myzipl((1, 2, 3), (4, 5, 6), (7, 8)) == ((1, 4, 7), (2, 5, 8)) assert myzipr((1, 2, 3), (4, 5, 6), (7, 8)) == ((2, 5, 8), (1, 4, 7)) -# zip and reverse don't commute for inputs with different lengths -assert tuple(zipr((1, 2, 3), (4, 5, 6), (7, 8))) == ((2, 5, 8), (1, 4, 7)) # zip first -assert tuple(rzip((1, 2, 3), (4, 5, 6), (7, 8))) == ((3, 6, 8), (2, 5, 7)) # reverse first +# composel: compose functions, applying the leftmost first +with_n = lambda *args: (partial(f, n) for n, f in args) +clip = lambda n1, n2: composel(*with_n((n1, drop), (n2, take))) +assert tuple(clip(5, 10)(range(20))) == tuple(range(5, 15)) +``` -# curry with passthrough (positionals passed through on the right) -# final result is a tuple of the result(s) and the leftover args -double = lambda x: 2 * x -with dyn.let(curry_context=["whatever"]): # set a context to allow passthrough to the top level - assert curry(double, 2, "foo") == (4, "foo") # arity of double is 1 +In the last example, essentially we just want to `clip 5 10 (range 20)`, the grouping of the parentheses being pretty much an implementation detail. Using the passthrough in `curry` (more on which in the section on `curry`, below), we can rewrite the last line as: -mysum = curry(foldl, add, 0) -myprod = curry(foldl, mul, 1) -a = ll(1, 2) -b = ll(3, 4) -c = ll(5, 6) -append_two = lambda a, b: foldr(cons, b, a) -append_many = lambda *lsts: foldr(append_two, nil, lsts) # see unpythonic.lappend -assert mysum(append_many(a, b, c)) == 21 -assert myprod(b) == 12 +```python +assert tuple(curry(clip, 5, 10, range(20)) == tuple(range(5, 15)) +``` -map_one = lambda f: curry(foldr, composer(cons, to1st(f)), nil) -doubler = map_one(double) -assert doubler((1, 2, 3)) == ll(2, 4, 6) -assert curry(map_one, double, ll(1, 2, 3)) == ll(2, 4, 6) -``` +#### `memoize` -*Minor detail*: We could also write the last example as: +**Changed in v0.15.0.** *Fix bug: `memoize` is now thread-safe. Even when the same memoized function instance is called concurrently from multiple threads, exactly one thread will compute the result. If `f` is recursive, the thread that acquired the lock is the one that is allowed to recurse into the memoized `f`.* -```python -double = lambda x: 2 * x -rmap_one = lambda f: curry(foldl, composer(cons, to1st(f)), nil) # essentially reversed(map(...)) -map_one = lambda f: composer(rmap_one(f), lreverse) -assert curry(map_one, double, ll(1, 2, 3)) == ll(2, 4, 6) -``` +[*Memoization*](https://en.wikipedia.org/wiki/Memoization) is a functional programming technique, meant to be used with [pure functions](https://en.wikipedia.org/wiki/Pure_function). It caches the return value, so that *for each unique set of arguments*, the original function will be evaluated only once. All arguments must be hashable. -which may be a useful pattern for lengthy iterables that could overflow the call stack (although not in ``foldr``, since our implementation uses a linear process). +Our `memoize` caches also exceptions, à la the [Mischief package in Racket](https://docs.racket-lang.org/mischief/memoize.html). If the memoized function is called again with arguments with which it raised an exception the first time, **that same exception instance** is raised again. -In ``rmap_one``, we can use either ``curry`` or ``functools.partial``. In this case it doesn't matter which, since we want just one partial application anyway. We provide two arguments, and the minimum arity of ``foldl`` is 3, so ``curry`` will trigger the call as soon as (and only as soon as) it gets at least one more argument. +The decorator **works also on instance methods**, with results cached separately for each instance. This is essentially because `self` is an argument, and custom classes have a default `__hash__`. Hence it doesn't matter that the memo lives in the `memoized` closure on the class object (type), where the method is, and not directly on the instances. The memo itself is shared between instances, but calls with a different value of `self` will create unique entries in it. (This approach does have the expected problem: if lots of instances are created and destroyed, and a memoized method is called for each, the memo will grow without bound.) -The final ``curry`` uses both of the extra features. It invokes passthrough, since ``map_one`` has arity 1. It also invokes a call to the callable returned from ``map_one``, with the remaining arguments (in this case just one, the ``ll(1, 2, 3)``). +*For a solution that performs memoization at the instance level, see [this ActiveState recipe](https://github.com/ActiveState/code/tree/master/recipes/Python/577452_memoize_decorator_instance) (and to demystify the magic contained therein, be sure you understand [descriptors](https://docs.python.org/3/howto/descriptor.html)).* -Yet another way to write ``map_one`` is: +There are some **important differences** to the nearest equivalents in the standard library, [`functools.cache`](https://docs.python.org/3/library/functools.html#functools.cache) (Python 3.9+) and [`functools.lru_cache`](https://docs.python.org/3/library/functools.html#functools.lru_cache): -```python -mymap = lambda f: curry(foldr, composer(cons, curry(f)), nil) -``` + - `memoize` **binds arguments** like Python itself does, so given this definition: -The curried ``f`` uses up one argument (provided it is a one-argument function!), and the second argument is passed through on the right; these two values then end up as the arguments to ``cons``. + ```python + from unpythonic import memoize -Using a currying compose function (name suffixed with ``c``), the inner curry can be dropped: + @memoize + def f(a, b): + return a + b + ``` + + the calls `f(1, 2)`, `f(1, b=2)`, `f(a=1, b=2)`, and `f(b=2, a=1)` all hit **the same cache key**. + + As of Python 3.9, in `functools.lru_cache` this is not so; see the internal function `functools._make_key` in [`functools.py`](https://github.com/python/cpython/blob/main/Lib/functools.py), where the comments explicitly say so. + + - `memoize` **caches exceptions**, too. A pure function that crashed for some combination of arguments, if given the same inputs again, will just crash again with the same error, so there is no reason to run it again. + + - `memoize` has **no** maximum cache size or hit/miss statistics counting. + + - `memoize` does **not** have a `typed` mode to treat `42` and `42.0` as different keys to the memo. The function arguments are hashed, and both an `int` and an equal `float` happen to hash to the same value. + + The `typed` mode of the standard library functions is actually a form of dispatch. Hence, you can use `@generic` (which see), and `@memoize` each individual multimethod: + + ```python + from unpythonic import generic, memoize + + @generic + @memoize + def thrice(x: int): + return 3 * x + + @generic + @memoize + def thrice(x: float): + return 3.0 * x + ``` + + Without using `@generic`, the essential idea is: + + ```python + from unpythonic import memoize + + def thrice(x): # the dispatcher + if isinstance(x, int): + return thrice_int(x) + elif isinstance(x, float): + return thrice_float(x) + raise TypeError(f"unsupported argument: {type(x)} with value {repr(x)}") + + @memoize + def thrice_int(x): + return 3 * x + + @memoize + def thrice_float(x): + return 3.0 * x + ``` + + Observe that we memoize **each implementation**, not the dispatcher. + + This solution keeps dispatching and memoization orthogonal. + +Examples: + +```python +from unpythonic import memoize + +ncalls = 0 +@memoize # <-- important part +def square(x): + global ncalls + ncalls += 1 + return x**2 +assert square(2) == 4 +assert ncalls == 1 +assert square(3) == 9 +assert ncalls == 2 +assert square(3) == 9 +assert ncalls == 2 # called only once for each unique set of arguments +assert square(x=3) == 9 +assert ncalls == 2 # only the resulting bindings matter, not how you pass the args + +# "memoize lambda": classic evaluate-at-most-once thunk +# See also the `lazy[]` macro. +thunk = memoize(lambda: print("hi from thunk")) +thunk() # the message is printed only the first time +thunk() +``` + + +#### `curry` + +**Changed in v0.15.0.** *`curry` supports both positional and named arguments, and binds arguments to function parameters like Python itself does. The call triggers when all parameters are bound, regardless of whether they were passed by position or by name, and at which step of the currying process they were passed.* + +*`unpythonic`'s multiple-dispatch system (`@generic`, `@typed`) is supported. `curry` looks for an exact match first, then a match with extra args/kwargs, and finally a partial match. If there is still no match, this implies that at least one parameter would get a binding that fails the type check. In such a case `TypeError` regarding failed multiple dispatch is raised.* + +*If the function being curried is `@generic` or `@typed`, or has type annotations on its parameters, the parameters being passed in are type-checked. A type mismatch immediately raises `TypeError`. This helps support [fail-fast](https://en.wikipedia.org/wiki/Fail-fast) in code using `curry`.* + +[*Currying*](https://en.wikipedia.org/wiki/Currying) is a technique in functional programming, where a function that takes multiple arguments is converted to a sequence of nested one-argument functions, each one *specializing* (fixing the value of) the leftmost remaining positional parameter. Each such function returns another function that takes the next parameter. The last function, when no more parameters remain, then performs the actual computation and returns the result. + +Some languages, such as Haskell, curry all functions natively. In languages that do not, like Python or [Racket](https://docs.racket-lang.org/reference/procedures.html#%28def._%28%28lib._racket%2Ffunction..rkt%29._curry%29%29), when currying is implemented as a library function, this is often done as a form of [partial application](https://en.wikipedia.org/wiki/Partial_application), which is a subtly different concept, but encompasses the curried behavior as a special case. In practice this means that you can pass several arguments in a single step, and the original function will be called when all parameters have been bound. + +Our `curry` can be used both as a decorator and as a regular function. As a decorator, `curry` takes no decorator arguments. As a regular function, `curry` itself is curried à la Racket. If any args or kwargs are given (beside the function to be curried), they are the first step. This helps eliminate many parentheses. + +**CAUTION**: If the signature of `f` cannot be inspected, currying fails, raising `ValueError`, like `inspect.signature` does. This may happen with builtins such as `list.append`, `operator.add`, `print`, or `range`, depending on which version of Python is used (and whether it is CPython or PyPy3). + +Like Haskell, and [`spicy` for Racket](https://github.com/Technologicat/spicy), our `curry` supports *passthrough*; but we pass through **both positional and named arguments**. + +Any args and/or kwargs that are incompatible with the target function's call signature, are *passed through* in the sense that the function is called with the args and kwargs compatible with its call signature, and then its return value is merged with the remaining args and kwargs. + +If the *first positional return value* of the result of passthrough is callable, it is (curried and) invoked on the remaining args and kwargs, after the merging. This helps with some instances of [point-free style](https://en.wikipedia.org/wiki/Tacit_programming). + +Some finer points concerning the passthrough feature: + + - *Incompatible* means too many positional args, or named args that have no corresponding parameter. Note that if the function has a `**kwargs` parameter, then all named args are considered compatible, because it absorbs anything. + + - Multiple return values (both positional and named) are denoted using `Values` (which see). A standard return value is considered to consist of *one positional return value* only (even if it is a `tuple`). + + - Extra positional args are passed through **on the right**. Any positional return values of the curried function are prepended, on the left. + + - Extra named args are passed through by name. They may be overridden by named return values (with the same name) from the curried function. + + - If more args/kwargs are still remaining when the top-level curry context exits, by default `TypeError` is raised. + - To override this behavior, set the dynvar `curry_context`. It is a list representing the stack of currently active curry contexts. A context is any object, a human-readable label is fine. See below for an example. + - To set the dynvar, `from unpythonic import dyn`, and then `with dyn.let(curry_context=["whatever"]):`. + +Examples: + +```python +from operator import add, mul +from unpythonic import curry, foldl, foldr, composer, to1st, cons, nil, ll, dyn, Values + +mysum = curry(foldl, add, 0) +myprod = curry(foldl, mul, 1) +a = ll(1, 2) +b = ll(3, 4) +c = ll(5, 6) +append_two = lambda a, b: foldr(cons, b, a) +append_many = lambda *lsts: foldr(append_two, nil, lsts) # see unpythonic.lappend +assert mysum(append_many(a, b, c)) == 21 +assert myprod(b) == 12 + +# curry with passthrough +double = lambda x: 2 * x +with dyn.let(curry_context=["whatever"]): # set a context to allow passthrough to the top level + # positionals are passed through on the right + assert curry(double, 2, "foo") == Values(4, "foo") # arity of double is 1 + # named args are passed through by name + assert curry(double, 2, nosucharg="foo") == Values(4, nosucharg="foo") + +# actual use case for passthrough +map_one = lambda f: curry(foldr, composer(cons, to1st(f)), nil) +doubler = map_one(double) +assert doubler((1, 2, 3)) == ll(2, 4, 6) + +assert curry(map_one, double, ll(1, 2, 3)) == ll(2, 4, 6) +``` + +We could also write the last example as: + +```python +from unpythonic import curry, foldl, composer, const, to1st, nil, lreverse + +double = lambda x: 2 * x +rmap_one = lambda f: curry(foldl, composer(cons, to1st(f)), nil) # essentially reversed(map(...)) +map_one = lambda f: composer(rmap_one(f), lreverse) +assert curry(map_one, double, ll(1, 2, 3)) == ll(2, 4, 6) +``` + +which may be a useful pattern for lengthy iterables that could overflow the call stack (although not in `foldr`, since our implementation uses a linear process). + +In the example, in `rmap_one`, we can use either `curry` or `partial`. In this case it does not matter which, since we want just one partial application anyway. We provide two arguments, and the minimum arity of `foldl` is 3, so `curry` will trigger the call as soon as (and only as soon as) it gets at least one more argument. + +The final `curry` in the example uses the passthrough features. The function `map_one` has arity 1, but two positional arguments are given. It also invokes a call to the callable returned by `map_one`, with the remaining arguments (in this case just one, the `ll(1, 2, 3)`). + +Yet another way to write `map_one` is: ```python +from unpythonic import curry, foldr, composer, cons, nil + +mymap = lambda f: curry(foldr, composer(cons, curry(f)), nil) +``` + +The curried `f` uses up one argument (provided it is a one-argument function!), and the second argument is passed through on the right; these two values then end up as the arguments to `cons`. + +Using a **currying compose function** (name suffixed with `c`), we can drop the inner curry: + +```python +from unpythonic import curry, foldr, composerc, cons, nil + mymap = lambda f: curry(foldr, composerc(cons, f), nil) myadd = lambda a, b: a + b assert curry(mymap, myadd, ll(1, 2, 3), ll(2, 4, 6)) == ll(3, 6, 9) ``` -This is as close to ```(define (map f) (foldr (compose cons f) empty)``` (in ``#lang`` [``spicy``](https://github.com/Technologicat/spicy)) as we're gonna get in Python. +This is as close to ```(define (map f) (foldr (compose cons f) empty)``` (in `#lang` [`spicy`](https://github.com/Technologicat/spicy)) as we're gonna get in pure Python. -Notice how the last two versions accept multiple input iterables; this is thanks to currying ``f`` inside the composition. An element from each of the iterables is taken by the processing function ``f``. Being the last argument, ``acc`` is passed through on the right. The output from the processing function - one new item - and ``acc`` then become two arguments, passed into cons. +Notice how the last two versions accept multiple input iterables; this is thanks to currying `f` inside the composition. An element from each of the iterables is taken by the processing function `f`. Being the last argument, `acc` is passed through on the right. The output from the processing function - one new item - and `acc` then become two arguments, passed into cons. -Finally, keep in mind this exercise is intended as a feature demonstration. In production code, the builtin ``map`` is much better. It produces a lazy iterable, and does not care which kind of actual data structure the items will be stored in (once computed). +Finally, keep in mind the `mymap` example is intended as a feature demonstration. In production code, the builtin `map` is much better. It produces a lazy iterable, so it does not care which kind of actual data structure the items will be stored in (once they are computed). In other words, a lazy iterable is a much better model for a process that produces a sequence of values; how, and whether, to store that sequence is an orthogonal concern. -The example we have here evaluates all items immediately, and specifically produces a linked list. It's just a nice example of function composition involving incompatible arities, thus demonstrating the kind of situation where the passthrough feature of `curry` is useful. It is taken from a paper by [John Hughes (1984)](https://www.cse.chalmers.se/~rjmh/Papers/whyfp.html). +The example we have here evaluates all items immediately, and specifically produces a linked list. It is just a nice example of function composition involving incompatible positional arities, thus demonstrating the kind of situation where the passthrough feature of `curry` is useful. It is taken from a paper by [John Hughes (1984)](https://www.cse.chalmers.se/~rjmh/Papers/whyfp.html). -#### ``curry`` and reduction rules +##### `curry` and reduction rules -**Changed in v0.15.0.** *`curry` now supports kwargs, too, and binds parameters like Python itself does. Also, `@generic` and `@typed` functions are supported.* - -Our ``curry``, beside what it says on the tin, is effectively an explicit local modifier to Python's reduction rules, which allows some Haskell-like idioms. Let's consider a simple example with positional arguments only. When we say: +Our `curry`, beside what it says on the tin, is effectively an explicit local modifier to Python's reduction rules, which allows some Haskell-like idioms. Let's consider a simple example with positional arguments only. When we say: ```python curry(f, a0, a1, ..., a[n-1]) ``` -it means the following. Let ``m1`` and ``m2`` be the minimum and maximum positional arity of the callable ``f``, respectively. +it means the following. Let `m1` and `m2` be the minimum and maximum positional arity of the callable `f`, respectively. - - If ``n > m2``, call ``f`` with the first ``m2`` arguments. + - If `n > m2`, call `f` with the first `m2` arguments. - If the result is a callable, curry it, and recurse. - - Else form a tuple, where first item is the result, and the rest are the remaining arguments ``a[m2]``, ``a[m2+1]``, ..., ``a[n-1]``. Return it. - - If more positional args are still remaining when the top-level curry context exits, by default ``TypeError`` is raised. Use the dynvar ``curry_context`` to override; see above for an example. - - If ``m1 <= n <= m2``, call ``f`` and return its result (like a normal function call). - - **Any** positional arity accepted by ``f`` triggers the call; beware when working with [variadic](https://en.wikipedia.org/wiki/Variadic_function) functions. - - If ``n < m1``, partially apply ``f`` to the given arguments, yielding a new function with smaller ``m1``, ``m2``. Then curry the result and return it. - - Internally we stack ``functools.partial`` applications, but there will be only one ``curried`` wrapper no matter how many invocations are used to build up arguments before ``f`` eventually gets called. + - Else form a tuple, where first item is the result, and the rest are the remaining arguments `a[m2]`, `a[m2+1]`, ..., `a[n-1]`. Return it. + - If more positional args are still remaining when the top-level curry context exits, by default `TypeError` is raised. Use the dynvar `curry_context` to override; see above for an example. + - If `m1 <= n <= m2`, call `f` and return its result (like a normal function call). + - **Any** positional arity accepted by `f` triggers the call; beware when working with [variadic](https://en.wikipedia.org/wiki/Variadic_function) functions. + - If `n < m1`, partially apply `f` to the given arguments, yielding a new function with smaller `m1`, `m2`. Then curry the result and return it. + - Internally we stack `functools.partial` applications, but there will be only one `curried` wrapper no matter how many invocations are used to build up arguments before `f` eventually gets called. As of v0.15.0, the actual algorithm by which `curry` decides what to do, in the presence of kwargs, `@generic` functions, and `Values` multiple-return-values (and named return values), is: @@ -1189,7 +1667,7 @@ As of v0.15.0, the actual algorithm by which `curry` decides what to do, in the - Note we keep track of which arguments were passed positionally and which by name. To avoid subtle errors, they are eventually passed to `f` the same way they were passed to `curry`. (Positional args are passed positionally, and kwargs are passed by name.) - If there are no unbound parameters, and no args/kwargs are left over, we have an exact match. Call `f` and return its result, like a normal function call. - Any sequence of curried calls that ends up binding all parameters of `f` triggers the call. - - As before, beware when working with variadic functions. Particularly, keep in mind that `*args` matches **zero or more** positional arguments (as the [Kleene star](https://en.wikipedia.org/wiki/Kleene_star)-ish notation indeed suggests). + - Beware when working with variadic functions. Particularly, keep in mind that `*args` matches **zero or more** positional arguments (as the [Kleene star](https://en.wikipedia.org/wiki/Kleene_star)-ish notation indeed suggests). - If there are no unbound parameters, but there are args/kwargs left over, arrange passthrough for the leftover args/kwargs (that were rejected by the call signature of `f`), and call `f`. Any leftover positional arguments are passed through **on the right**. - Merge the return value of `f` with the leftover args/kwargs, thus forming updated leftover args/kwargs. - If the return value of `f` is a `Values`: prepend positional return values into the leftover args (i.e. insert them **on the left**), and update the leftover kwargs with the named return values. (I.e. a key name conflict causes an overwrite in the leftover kwargs.) @@ -1202,9 +1680,9 @@ As of v0.15.0, the actual algorithm by which `curry` decides what to do, in the - First, try for an exact match that passes the type check. **If any such match is found**, pick that multimethod. Call it and return its result (as above). - Then, try for a match that passes the type check, but has extra args/kwargs. **If any such match is found**, pick that multimethod. Arrange passthrough... (as above). - Then, try for a partial match that passes the type check. **If any such match is found**, keep currying. - - If none of the above match, it implies that no matter which multimethod we pick, at least one parameter would get a binding that fails the type check. Raise `TypeError`. + - If none of the above match, it implies that no matter which multimethod we pick, at least one parameter will get a binding that fails the type check. Raise `TypeError`. -(If *really* interested in the gritty details, look at the source code of `unpythonic.fun.curry`. It calls some functions from `unpythonic.dispatch` for its `@generic` support, but otherwise it's pretty much self-contained.) +If interested in the gritty details, see [the source code](../unpythonic/fun.py) of `unpythonic.curry`, in the module `unpythonic.fun`. It calls some functions from the module `unpythonic.dispatch` for its `@generic` support, but otherwise it is pretty much self-contained. Getting back to the simple case, in the above example: @@ -1212,13 +1690,13 @@ Getting back to the simple case, in the above example: curry(mapl_one, double, ll(1, 2, 3)) ``` -the callable ``mapl_one`` takes one argument, which is a function. It yields another function, let us call it ``g``. We are left with: +the callable `mapl_one` takes one argument, which is a function. It returns another function, let us call it `g`. We are left with: ```python curry(g, ll(1, 2, 3)) ``` -The argument is then passed into ``g``; we obtain a result, and reduction is complete. +The remaining argument is then passed into `g`; we obtain a result, and reduction is complete. A curried function is also a curry context: @@ -1228,19 +1706,19 @@ a2 = curry(add2) a2(a, b, c) # same as curry(add2, a, b, c); reduces to (a + b, c) ``` -so on the last line, we don't need to say +so on the last line, we do not need to say ```python curry(a2, a, b, c) ``` -because ``a2`` is already curried. Doing so does no harm, though; ``curry`` automatically prevents stacking ``curried`` wrappers: +because `a2` is already curried. Doing so does no harm, though; `curry` automatically prevents stacking `curried` wrappers: ```python curry(a2) is a2 # --> True ``` -If we wish to modify precedence, parentheses are needed, which takes us out of the curry context, unless we explicitly ``curry`` the subexpression. This works: +If we wish to modify precedence, parentheses are needed, which takes us out of the curry context, unless we explicitly `curry` the subexpression. This works: ```python curry(f, a, curry(g, x, y), b, c) @@ -1252,17 +1730,31 @@ but this **does not**: curry(f, a, (g, x, y), b, c) ``` -because ``(g, x, y)`` is just a tuple of ``g``, ``x`` and ``y``. This is by design; as with all things Python, *explicit is better than implicit*. +because `(g, x, y)` is just a tuple of `g`, `x` and `y`. This is by design; as with all things Python, *explicit is better than implicit*. + +**Note**: to code in curried style, a [contract system](https://en.wikipedia.org/wiki/Design_by_contract) or a type checker can be useful. Also, be careful with variadic functions, because any allowable arity will trigger the call. + +(The `map` function in the standard library is a particular offender here, since it requires at least one iterable to actually do anything but raise `TypeError`, but its call signature suggests it can be called without any iterables. Hence, for curry-friendliness we provide a wrapper `unpythonic.map` that *requires* at least one iterable.) + +- Contract systems for Python include [icontract](https://github.com/Parquery/icontract) and [PyContracts](https://github.com/AndreaCensi/contracts). -**Note**: to code in curried style, a [contract system](https://en.wikipedia.org/wiki/Design_by_contract) (such as [icontract](https://github.com/Parquery/icontract) or [PyContracts](https://github.com/AndreaCensi/contracts)) or the [mypy static type checker](http://mypy-lang.org/) can be useful; also, be careful with variadic functions. +- For static type checking, consider [mypy](http://mypy-lang.org/). +- For run-time type checking, consider `@typed` or `@generic` right here in `unpythonic`. -#### ``fix``: break infinite recursion cycles +- You can also just use Python's type annotations; `unpythonic`'s `curry` type-checks the arguments before accepting the curried function. The annotations work if the stdlib function [`typing.get_type_hints`](https://docs.python.org/3/library/typing.html#typing.get_type_hints) can find them. -The name `fix` comes from the *least fixed point* with respect to the definedness relation, which is related to Haskell's `fix` function. However, this `fix` is not that function. Our `fix` breaks recursion cycles in strict functions - thus causing some non-terminating strict functions to return. (Here *strict* means that the arguments are evaluated eagerly.) + +#### `fix`: break infinite recursion cycles + +The name `fix` comes from the *least fixed point* with respect to the definedness relation, which is related to Haskell's `fix` function. However, this `fix` is **not** that function. Our `fix` breaks recursion cycles in strict functions - thus causing some non-terminating strict functions to return. (Here [*strict*](https://en.wikipedia.org/wiki/Evaluation_strategy#Strict_evaluation) means that the arguments are evaluated eagerly.) **CAUTION**: Worded differently, this function solves a small subset of the halting problem. This should be hint enough that it will only work for the advertised class of special cases - i.e., a specific kind of recursion cycles. +If you need `fix` for code that uses TCO, use `fixtco`. The implementations of recursion cycle breaking and TCO must interact in a very particular way to work properly; this is done by `fixtco`. + +For examples, see [the unit tests](../unpythonic/tests/test_fix.py). + Usage: ```python @@ -1285,11 +1777,11 @@ If no recursion cycle occurs, `f` returns normally. If a cycle occurs, the call - In the latter example, the name `"f"` and the offending args are returned. -**A cycle is detected when** `f` is called again with a set of args that have already been previously seen in the current call chain. Infinite mutual recursion is detected too, at the point where any `@fix`-instrumented function is entered again with a set of args already seen during the current call chain. +**A cycle is detected when** `f` is called again with a set of args that have already been previously seen in the current call chain. Infinite *mutual recursion* is detected too, at the point where any `@fix`-instrumented function is entered again with a set of args already seen during the current call chain. -**CAUTION**: The infinitely recursive call sequence `f(0) → f(1) → ... → f(k+1) → ...` contains no cycles in the sense detected by `fix`. The `fix` function will not catch all cases of infinite recursion, but only those where a previously seen set of arguments is seen again. (If `f` is pure, the same arguments appearing again implies the call will not return, so we can terminate it.) +**CAUTION**: The infinitely recursive call sequence `f(0) → f(1) → ... → f(k+1) → ...` contains no cycles in the sense detected by `fix`. The `fix` function will **not** catch all cases of infinite recursion, but only those where a previously seen set of arguments is seen again. If `f` is [pure](https://en.wikipedia.org/wiki/Pure_function), the same arguments appearing again during recursion implies the call will not return, so we can terminate it. -**CAUTION**: If we have a function `g(a, b)`, the argument lists of the invocations `g(1, 2)` and `g(a=1, b=2)` are in principle different. This is a Python gotcha that was originally noticed by the author of the `wrapt` library, and mentioned in [its documentation](https://wrapt.readthedocs.io/en/latest/decorators.html#processing-function-arguments). However, once arguments are bound to the formal parameters of `g`, the result is the same. We consider the *resulting bindings*, not the exact way the arguments were passed. +**CAUTION**: If we have a function `g(a, b)`, the argument lists of the invocations `g(1, 2)` and `g(a=1, b=2)` are in principle different. However, we bind arguments like Python itself does, and consider the *resulting bindings* only. It does not matter how the arguments were passed. We can use `fix` to find the (arithmetic) fixed point of `cos`: @@ -1334,7 +1826,7 @@ c = fixpoint(cos, x0=1) assert c == cos(c) ``` -**NOTE**: But see `unpythonic.fixpoint`, which is meant specifically for finding *arithmetic* fixed points, and `unpythonic.iterate1`, which produces a generator that iterates `f` without needing recursion. +**NOTE**: *See `unpythonic.fixpoint`, which is meant specifically for finding arithmetic fixed points, and `unpythonic.iterate1`, which produces a generator that iterates `f` without needing recursion.* **Notes**: @@ -1354,15 +1846,15 @@ assert c == cos(c) - `bottom` can be a callable, in which case the function name and args at the point where the cycle was detected are passed to it, and its return value becomes the final return value. This is useful e.g. for debug logging. - - The `memo` flag controls whether to memoize also intermediate results. It adds some additional function call layers between function entries from recursive calls; if that is a problem (due to causing Python's call stack to blow up faster), use `memo=False`. You can still memoize the final result if you want; just put `@memoize` on the outside. + The function name is provided, because we catch also infinite *mutual recursion*; so it can be a useful piece of information *which function* it was that was first called with already-seen arguments. -**NOTE**: If you need `fix` for code that uses TCO, use `fixtco` instead. The implementations of recursion cycle breaking and TCO must interact in a very particular way to work properly; this is done by `fixtco`. + - The `memo` flag controls whether to memoize intermediate results. It adds some additional function call layers between function entries from recursive calls; if that is a problem (due to causing Python's call stack to blow up faster), use `memo=False`. You can still memoize the final result if you want; just put `@memoize` on the outside. ##### Real-world use and historical note This kind of `fix` is sometimes helpful in recursive pattern-matching definitions for parsers. When the pattern matcher gets stuck in an infinite left-recursion, it can return a customizable special value instead of not terminating. Being able to not care about non-termination may simplify definitions. -This `fix` can also be used to find fixed points of functions, as in the above examples. +This `fix` can also be used to find arithmetic fixed points of functions, as in the above examples. The idea comes from Matthew Might's article on [parsing with (Brzozowski's) derivatives](http://matt.might.net/articles/parsing-with-derivatives/), where it was a utility implemented in Racket as the `define/fix` form. It was originally ported to Python [by Per Vognsen](https://gist.github.com/pervognsen/8dafe21038f3b513693e) (linked from the article). The `fix` in `unpythonic` is a redesign with kwargs support, thread safety, and TCO support. @@ -1378,7 +1870,7 @@ A simple way to explain Haskell's `fix` is: fix f = let x = f x in x ``` -so anywhere the argument is referred to in the definition of `f`, it is replaced by another application of `f`, recursively. This obviously yields a notation useful for corecursively defining infinite lazy lists. +so anywhere the argument is referred to in the definition of `f`, it is replaced by another application of `f`, recursively. This obviously yields a notation useful for [corecursively](https://en.wikipedia.org/wiki/Corecursion) defining infinite lazy lists. For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[2]](https://www.vex.net/~trebla/haskell/fix.xhtml) [[3]](https://stackoverflow.com/questions/4787421/how-do-i-use-fix-and-how-does-it-work) [[4]](https://medium.com/@cdsmithus/fixpoints-in-haskell-294096a9fc10) [[5]](https://en.wikibooks.org/wiki/Haskell/Fix_and_recursion). @@ -1386,15 +1878,15 @@ For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[ ### Batteries for itertools - `unpack`: lazily unpack an iterable. Suitable for infinite inputs. - - Return the first ``n`` items and the ``k``th tail, in a tuple. Default is ``k = n``. - - Use ``k > n`` to fast-forward, consuming the skipped items. Works by `drop`. - - Use ``k < n`` to peek without permanently extracting an item. Works by [tee](https://docs.python.org/3/library/itertools.html#itertools.tee)ing; plan accordingly. - - *folds, scans, unfold*: + - Return the first `n` items and the `k`th tail, in a tuple. Default is `k = n`. + - Use `k > n` to fast-forward, consuming the skipped items. Works by `drop`. + - Use `k < n` to peek without permanently extracting an item. Works by [tee](https://docs.python.org/3/library/itertools.html#itertools.tee)ing; plan accordingly. + - *fold, scan, unfold*: - `foldl`, `foldr` with support for multiple input iterables, like in Racket. - Like in Racket, `op(elt, acc)`; general case `op(e1, e2, ..., en, acc)`. Note Python's own `functools.reduce` uses the ordering `op(acc, elt)` instead. - No sane default for multi-input case, so the initial value for `acc` must be given. - One-input versions with optional init are provided as `reducel`, `reducer`, with semantics similar to Python's `functools.reduce`, but with the rackety ordering `op(elt, acc)`. - - By default, multi-input folds terminate on the shortest input. To instead terminate on the longest input, use the ``longest`` and ``fillvalue`` kwargs. + - By default, multi-input folds terminate on the shortest input. To instead terminate on the longest input, use the `longest` and `fillvalue` kwargs. - For multiple inputs with different lengths, `foldr` syncs the **left** ends. - `rfoldl`, `rreducel` reverse each input and then left-fold. This syncs the **right** ends. - `scanl`, `scanr`: scan (a.k.a. accumulate, partial fold); a lazy fold that returns a generator yielding intermediate results. @@ -1407,11 +1899,12 @@ For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[ - `rscanl`, `rscanl1` reverse each input and then left-scan. This syncs the **right** ends. - `unfold1`, `unfold`: generate a sequence [corecursively](https://en.wikipedia.org/wiki/Corecursion). The counterpart of `foldl`. - `unfold1` is for 1-in-2-out functions. The input is `state`, the return value must be `(value, newstate)` or `None`. - - `unfold` is for n-in-(1+n)-out functions. The input is `*states`, the return value must be `(value, *newstates)` or `None`. - - Unfold returns a generator yielding the collected values. The output can be finite or infinite; to signify that a finite sequence ends, the user function must return `None`. + - `unfold` is for n-in-(1+n)-out functions. + - **Changed in v0.15.0.** *The initial args/kwargs are unpacked to the args/kwargs of the user function. The function must return a `Values` object, where the first positional return value is the value to yield, and anything else is unpacked to the args/kwargs of the user function at the next iteration.* + - Unfold returns a generator yielding the collected values. The output can be finite or infinite; to signify that a finite sequence ends, the user function must return `None`. (Beside a `Values` object, a bare `None` is the only other allowed return value from the user function.) - *mapping and zipping*: - `map_longest`: the final missing battery for `map`. - - Essentially `starmap(func, zip_longest(*iterables))`, so it's [spanned](https://en.wikipedia.org/wiki/Linear_span) by ``itertools``. + - Essentially `starmap(func, zip_longest(*iterables))`, so it's [spanned](https://en.wikipedia.org/wiki/Linear_span) by `itertools`, but it's convenient to have a named shorthand to do that. - `rmap`, `rzip`, `rmap_longest`, `rzip_longest`: reverse each input, then map/zip. For multiple inputs, syncs the **right** ends. - `mapr`, `zipr`, `mapr_longest`, `zipr_longest`: map/zip, then reverse the result. For multiple inputs, syncs the **left** ends. - `map`: curry-friendly wrapper for the builtin, making it mandatory to specify at least one iterable. **Added in v0.14.2.** @@ -1423,7 +1916,7 @@ For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[ - This differs from `zip` in that the output is flattened, and the termination condition is checked after each item. So e.g. `interleave(['a', 'b', 'c'], ['+', '*'])` → `['a', '+', 'b', '*', 'c']` (the actual return value is a generator, not a list). - *flattening*: - `flatmap`: map a function, that returns a list or tuple, over an iterable and then flatten by one level, concatenating the results into a single tuple. - - Essentially, ``composel(map(...), flatten1)``; the same thing the bind operator of the List monad does. + - Essentially, `composel(map(...), flatten1)`; the same thing the bind operator of the List monad does. - `flatten1`, `flatten`, `flatten_in`: remove nested list structure. - `flatten1`: outermost level only. - `flatten`: recursive, with an optional predicate that controls whether to flatten a given sublist. @@ -1431,7 +1924,7 @@ For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[ - *extracting items, subsequences*: - `take`, `drop`, `split_at`: based on `itertools` [recipes](https://docs.python.org/3/library/itertools.html#itertools-recipes). - Especially useful for testing generators. - - `islice` is maybe more pythonic than `take` and `drop`. We provide a utility that supports the slice syntax. + - `islice` is maybe more pythonic than `take` and `drop`; it enables slice syntax for any iterable. - `tail`: return the tail of an iterable. Same as `drop(1, iterable)`; common use case. - `butlast`, `butlastn`: return a generator that yields from iterable, dropping the last `n` items if the iterable is finite. Inspired by a similar utility in PG's [On Lisp](http://paulgraham.com/onlisp.html). - Works by using intermediate storage. **Do not** use the original iterator after a call to `butlast` or `butlastn`. @@ -1442,40 +1935,46 @@ For more, see [[1]](https://www.parsonsmatt.org/2016/10/26/grokking_fix.html) [[ - Can be useful for the occasional abuse of `collections.deque` as an *alist* [[1]](https://en.wikipedia.org/wiki/Association_list) [[2]](http://www.gigamonkeys.com/book/beyond-lists-other-uses-for-cons-cells.html). Use `.appendleft(...)` to add new items, and then this `find` to get the currently active association. - `running_minmax`, `minmax`: Extract both min and max in one pass over an iterable. The `running_` variant is a scan and returns a generator; the just-give-me-the-final-result variant is a fold. **Added in v0.14.2.** - *math-related*: - - `fixpoint`: arithmetic fixed-point finder (not to be confused with `fix`). **Added in v0.14.2.** - `within`: yield items from iterable until successive iterates are close enough. Useful with [Cauchy sequences](https://en.wikipedia.org/wiki/Cauchy_sequence). **Added in v0.14.2.** - `prod`: like the builtin `sum`, but compute the product. Oddly missing from the standard library. - `iterate1`, `iterate`: return an infinite generator that yields `x`, `f(x)`, `f(f(x))`, ... - - `iterate1` is for 1-to-1 functions; `iterate` for n-to-n, unpacking the return value to the argument list of the next call. + - `iterate1` is for 1-to-1 functions. + - `iterate` is for n-to-n, unpacking the return value to the args/kwargs of the next call. + - **Changed in v0.15.0.** *In the n-to-n version, now the user function must return a `Values` object in the same shape as it accepts args and kwargs. This `Values` object is the `x` that is yielded at each iteration.* - *miscellaneous*: - `uniqify`, `uniq`: remove duplicates (either all or consecutive only, respectively), preserving the original ordering of the items. - `rev` is a convenience function that tries `reversed`, and if the input was not a sequence, converts it to a tuple and reverses that. The return value is a `reversed` object. - - `scons`: prepend one element to the start of an iterable, return new iterable. ``scons(x, iterable)`` is lispy shorthand for ``itertools.chain((x,), iterable)``, allowing to omit the one-item tuple wrapper. - - `inn`: contains-check (``x in iterable``) with automatic termination for monotonic divergent infinite iterables. - - Only applicable to monotonic divergent inputs (such as ``primes``). Increasing/decreasing is auto-detected from the first non-zero diff, but the function may fail to terminate if the input is actually not monotonic, or has an upper/lower bound. - - `iindex`: like ``list.index``, but for a general iterable. Consumes the iterable, so only makes sense for memoized inputs. - - `CountingIterator`: count how many items have been yielded, as a side effect. The count is stored in the `.count` attribute. **Added in v0.14.2.** + - `scons`: prepend one element to the start of an iterable, return new iterable. `scons(x, iterable)` is lispy shorthand for `itertools.chain((x,), iterable)`, allowing to omit the one-item tuple wrapper. The name is an abbreviation of [`stream-cons`](https://docs.racket-lang.org/reference/streams.html). + - `inn`: contains-check (`x in iterable`) with automatic termination for monotonic divergent infinite iterables. + - Only applicable to monotonic divergent inputs (such as `primes`). Increasing/decreasing is auto-detected from the first non-zero diff, but the function may fail to terminate if the input is actually not monotonic, or has an upper/lower bound. + - `iindex`: like `list.index`, but for a general iterable. Consumes the iterable, so only makes sense for memoized inputs. + - `CountingIterator`: use `CountingIterator(iterable)` instead of `iter(iterable)` to produce an iterator that, as a side effect, counts how many items have been yielded. The count is stored in the `.count` attribute. **Added in v0.14.2.** - `slurp`: extract all items from a `queue.Queue` (until it is empty) to a list, returning that list. **Added in v0.14.2.** - `subset`: test whether an iterable is a subset of another. **Added in v0.14.3.** - `powerset`: yield the power set (set of all subsets) of an iterable. Works also for potentially infinite iterables, if only a finite prefix is ever requested. (But beware, both runtime and memory usage are exponential in the input size.) **Added in v0.14.2.** - - `partition_int`: split a small positive integer, in all possible ways, into smaller integers that sum to it. Useful e.g. for determining how many letters the components of an anagram may have. **Added in v0.14.2.** - `allsame`: test whether all elements of an iterable are the same. Sometimes useful in writing testing code. **Added in v0.14.3.** Examples: ```python from functools import partial +from itertools import count, takewhile +from operator import add, mul from unpythonic import (scanl, scanr, foldl, foldr, - mapr, zipr, + mapr, zipr, rmap, rzip, identity, uniqify, uniq, flatten1, flatten, flatten_in, flatmap, take, drop, unfold, unfold1, + unpack, cons, nil, ll, curry, - s, inn, iindex, + imemoize, gmemoize, + s, inn, iindex, find, + partition, partition_int, window, subset, powerset, - allsame) + allsame, + Values) assert tuple(scanl(add, 0, range(1, 5))) == (0, 1, 3, 6, 10) assert tuple(scanr(add, 0, range(1, 5))) == (0, 4, 7, 9, 10) @@ -1490,7 +1989,10 @@ def step2(k): # x0, x0 + 2, x0 + 4, ... assert tuple(take(10, unfold1(step2, 10))) == (10, 12, 14, 16, 18, 20, 22, 24, 26, 28) def nextfibo(a, b): - return (a, b, a + b) # value, *newstates + # First positional return value is the value to yield. + # Everything else is newstate, to be unpacked to `nextfibo`'s + # args/kwargs at the next iteration. + return Values(a, a=b, b=a + b) assert tuple(take(10, unfold(nextfibo, 1, 1))) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55) def fibos(): @@ -1508,7 +2010,7 @@ assert inn(42, evens()) assert not inn(41, evens()) @gmemoize -def primes(): +def primes(): # FP sieve of Eratosthenes yield 2 for n in count(start=3, step=2): if not any(n % p == 0 for p in takewhile(lambda x: x*x <= n, primes())): @@ -1520,8 +2022,9 @@ assert not inn(1337, primes()) iseven = lambda x: x % 2 == 0 assert [tuple(it) for it in partition(iseven, range(10))] == [(1, 3, 5, 7, 9), (0, 2, 4, 6, 8)] +# CAUTION: not to be confused with: # partition_int: split a small positive integer, in all possible ways, into smaller integers that sum to it -assert tuple(partition_int(4)) == ((1, 1, 1, 1), (1, 1, 2), (1, 2, 1), (1, 3), (2, 1, 1), (2, 2), (3, 1), (4,)) +assert tuple(partition_int(4)) == ((4,), (3, 1), (2, 2), (2, 1, 1), (1, 3), (1, 2, 1), (1, 1, 2), (1, 1, 1, 1)) assert all(sum(terms) == 10 for terms in partition_int(10)) # iindex: find index of item in iterable (mostly only makes sense for memoized input) @@ -1564,16 +2067,31 @@ def msqrt(x): # multivalued sqrt return (s, -s) assert tuple(flatmap(msqrt, (0, 1, 4, 9))) == (0., 1., -1., 2., -2., 3., -3.) -# zipr reverses, then iterates. -assert tuple(zipr((1, 2, 3), (4, 5, 6), (7, 8))) == ((3, 6, 8), (2, 5, 7)) +# **CAUTION**: zip and reverse do NOT commute for inputs with different lengths: +assert tuple(zipr((1, 2, 3), (4, 5, 6), (7, 8))) == ((2, 5, 8), (1, 4, 7)) # zip first +assert tuple(rzip((1, 2, 3), (4, 5, 6), (7, 8))) == ((3, 6, 8), (2, 5, 7)) # reverse first + +# zipr syncs *left* ends, then iterates *from the right*. +assert tuple(zipr((1, 2, 3), (4, 5, 6), (7, 8))) == ((2, 5, 8), (1, 4, 7)) + +# so does mapr. +zipr2 = partial(mapr, identity) +assert tuple(zipr2((1, 2, 3), (4, 5, 6), (7, 8))) == (Values(2, 5, 8), Values(1, 4, 7)) + +# rzip syncs *right* ends, then iterates from the right. +assert tuple(rzip((1, 2, 3), (4, 5, 6), (7, 8))) == ((3, 6, 8), (2, 5, 7)) -zipr2 = partial(mapr, identity) # mapr works the same way. -assert tuple(zipr2((1, 2, 3), (4, 5, 6), (7, 8))) == ((3, 6, 8), (2, 5, 7)) +# so does rmap. +rzip2 = partial(rmap, identity) +assert tuple(rzip2((1, 2, 3), (4, 5, 6), (7, 8))) == (Values(3, 6, 8), Values(2, 5, 7)) -# foldr doesn't; it walks from the left, but collects results from the right: +# foldr syncs *left* ends, then collects results from the right: +def zipper(*args): + *rest, acc = args + return acc + (tuple(rest),) zipr1 = curry(foldr, zipper, ()) assert zipr1((1, 2, 3), (4, 5, 6), (7, 8)) == ((2, 5, 8), (1, 4, 7)) -# so the result is reversed(zip(...)), whereas zipr gives zip(*(reversed(s) for s in ...)) +# so the result is tuple(rev(zip(...))), whereas rzip gives tuple(zip(*(rev(s) for s in ...))) assert tuple(uniqify((1, 1, 2, 2, 2, 1, 2, 2, 4, 3, 4, 3, 3))) == (1, 2, 4, 3) # all assert tuple(uniq((1, 1, 2, 2, 2, 1, 2, 2, 4, 3, 4, 3, 3))) == (1, 2, 1, 2, 4, 3, 4, 3) # consecutive @@ -1587,16 +2105,6 @@ assert tuple(flatten((((1, 2), (3, 4)), (5, 6)), is_nested)) == ((1, 2), (3, 4), data = (((1, 2), ((3, 4), (5, 6)), 7), ((8, 9), (10, 11))) assert tuple(flatten(data, is_nested)) == (((1, 2), ((3, 4), (5, 6)), 7), (8, 9), (10, 11)) assert tuple(flatten_in(data, is_nested)) == (((1, 2), (3, 4), (5, 6), 7), (8, 9), (10, 11)) - -with_n = lambda *args: (partial(f, n) for n, f in args) -clip = lambda n1, n2: composel(*with_n((n1, drop), (n2, take))) -assert tuple(clip(5, 10)(range(20))) == tuple(range(5, 15)) -``` - -In the last example, essentially we just want to `clip 5 10 (range 20)`, the grouping of the parentheses being pretty much an implementation detail. With ``curry``, we can rewrite the last line as: - -```python -assert tuple(curry(clip, 5, 10, range(20)) == tuple(range(5, 15)) ``` ### Batteries for network programming @@ -1606,10 +2114,16 @@ assert tuple(curry(clip, 5, 10, range(20)) == tuple(range(5, 15)) While all other pure-Python features of `unpythonic` live in the main `unpythonic` package, the network-related features are placed in the subpackage `unpythonic.net`. This subpackage also contains the [REPL server and client](repl.md) for hot-patching live processes. - `unpythonic.net.msg`: A simplistic message protocol for sending message data over a stream-based transport, such as TCP. -- `unpythonic.net.ptyproxy`: Proxy between a Linux [PTY](https://en.wikipedia.org/wiki/Pseudoterminal) and a network socket. Useful for serving terminal utilities over the network. The selling point is this doesn't use `pty.spawn`, so it can be used for proxying also Python libraries that expect to run in a terminal. +- `unpythonic.net.ptyproxy`: Proxy between a Linux [PTY](https://en.wikipedia.org/wiki/Pseudoterminal) and a network socket. Useful for serving terminal utilities over the network. The selling point is this does **not** use `pty.spawn`, so it can be used for proxying also Python libraries that expect to run in a terminal. - `unpythonic.net.util`: Miscellaneous small utilities. -The thing about stream-based transports is that they have no concept of a message boundary [[1]](http://stupidpythonideas.blogspot.com/2013/05/sockets-are-byte-streams-not-message.html) [[2]](https://eli.thegreenplace.net/2011/08/02/length-prefix-framing-for-protocol-buffers) [[3]](https://docs.python.org/3/howto/sockets.html). This is where a message protocol comes in. We provide a [sans-io](https://sans-io.readthedocs.io/) implementation of a minimalistic custom protocol that adds rudimentary [message framing](https://blog.stephencleary.com/2009/04/message-framing.html) and [stream re-synchronization](https://en.wikipedia.org/wiki/Frame_synchronization). Example: +For a usage example of `unpythonic.net.ptyproxy`, see the source code of `unpythonic.net.server`. + +More details can be found in the docstrings. + +#### `unpythonic.net.msg` + +The problem with stream-based transports, such as network sockets, is that they have no concept of a message boundary [[1]](http://stupidpythonideas.blogspot.com/2013/05/sockets-are-byte-streams-not-message.html) [[2]](https://eli.thegreenplace.net/2011/08/02/length-prefix-framing-for-protocol-buffers) [[3]](https://docs.python.org/3/howto/sockets.html). This is where a message protocol comes in. We provide a [sans-io](https://sans-io.readthedocs.io/) implementation of a minimalistic message protocol that adds rudimentary [message framing](https://blog.stephencleary.com/2009/04/message-framing.html) and [stream re-synchronization](https://en.wikipedia.org/wiki/Frame_synchronization). Example: ```python from io import BytesIO, SEEK_SET @@ -1649,14 +2163,12 @@ assert decoder.decode() == b"mew" assert decoder.decode() is None ``` -For a usage example of `unpythonic.net.PTYProxy`, see the source code of `unpythonic.net.server`. - -### ``islice``: slice syntax support for ``itertools.islice` +### `islice`: slice syntax support for `itertools.islice` **Changed in v0.14.2.** *Added support for negative `start` and `stop`.* -Slice an iterable, using the regular slicing syntax: +Slice any iterable, using the regular slicing syntax: ```python from unpythonic import islice, primes, s @@ -1674,38 +2186,40 @@ assert tuple(islice(odds)[:5]) == (1, 3, 5, 7, 9) assert tuple(islice(odds)[:5]) == (11, 13, 15, 17, 19) # five more ``` -As a convenience feature: a single index is interpreted as a length-1 islice starting at that index. The slice is then immediately evaluated and the item is returned. +As a convenience feature: a single index is interpreted as a length-1 `islice` starting at that index. The slice is then immediately evaluated and the item is returned. -The slicing variant calls ``itertools.islice`` with the corresponding slicing parameters, after possibly converting negative `start` and `stop` to the appropriate positive values. +The slicing variant calls `itertools.islice` with the corresponding slicing parameters, after possibly converting negative `start` and `stop` to the appropriate positive values. -**CAUTION**: When using negative `start` and/or `stop`, we must consume the whole iterable to determine where it ends, if at all. Obviously, this will not terminate for infinite iterables. +**CAUTION**: When using negative `start` and/or `stop`, the whole iterable is consumed to determine where it ends, if at all. Obviously, this will not terminate for infinite iterables. The desired elements are then held in an internal buffer until they are yielded by iterating over the `islice`. **CAUTION**: Keep in mind that negative `step` is not supported, and that the slicing process consumes elements from the iterable. -Like ``fup``, our ``islice`` is essentially a manually curried function with unusual syntax; the initial call to ``islice`` passes in the iterable to be sliced. The object returned by the call accepts a subscript to specify the slice or index. Once the slice or index is provided, the call to ``itertools.islice`` triggers. +Like `fup`, our `islice` is essentially a manually curried function with unusual syntax; the initial call to `islice` passes in the iterable to be sliced. The object returned by the call accepts a subscript to specify the slice or index. Once the slice or index is provided, the call to `itertools.islice` triggers. Inspired by Python itself. ### `gmemoize`, `imemoize`, `fimemoize`: memoize generators +**Changed in v0.15.0.** *The generator instances created by the gfuncs returned by `gmemoize`, `imemoize`, and `fimemoize`, now support the `__len__` and `__getitem__` methods to access the already-yielded, memoized part. Asking for the `len` returns the current length of the memo. For subscripting, both a single `int` index and a slice are accepted. Note that memoized generators do **not** support all of the [`collections.abc.Sequence`](https://docs.python.org/3/library/collections.abc.html) API, because e.g. `__contains__` and `__reversed__` are missing, on purpose.* + Make generator functions (gfunc, i.e. a generator definition) which create memoized generators, similar to how streams behave in Racket. Memoize iterables; like `itertools.tee`, but no need to know in advance how many copies of the iterator will be made. Provided for both iterables and for factory functions that make iterables. - `gmemoize` is a decorator for a gfunc, which makes it memoize the instantiated generators. - If the gfunc takes arguments, they must be hashable. A separate memoized sequence is created for each unique set of argument values seen. - - For simplicity, the generator itself may use ``yield`` for output only; ``send`` is not supported. - - Any exceptions raised by the generator (except StopIteration) are also memoized, like in ``memoize``. - - Thread-safe. Calls to ``next`` on the memoized generator from different threads are serialized via a lock. Each memoized sequence has its own lock. This uses ``threading.RLock``, so re-entering from the same thread (e.g. in recursively defined sequences) is fine. + - For simplicity, the generator itself may use `yield` for output only; `send` is **not** supported. + - Any exceptions raised by the generator (except StopIteration) are also memoized, like in `memoize`. + - Thread-safe. Calls to `next` on the memoized generator from different threads are serialized via a lock. Each memoized sequence has its own lock. This uses `threading.RLock`, so re-entering from the same thread (e.g. in recursively defined mathematical sequences) is fine. - The whole history is kept indefinitely. For infinite iterables, use this only if you can guarantee that only a reasonable number of terms will ever be evaluated (w.r.t. available RAM). - - Typically, this should be the outermost decorator if several are used on the same gfunc. + - Typically, `gmemoize` should be the outermost decorator if several are used on the same gfunc. - `imemoize`: memoize an iterable. Like `itertools.tee`, but keeps the whole history, so more copies can be teed off later. - Same limitation: **do not** use the original iterator after it is memoized. The danger is that if anything other than the memoization mechanism advances the original iterator, some values will be lost before they can reach the memo. - Returns a gfunc with no parameters which, when called, returns a generator that yields items from the memoized iterable. The original iterable is used to retrieve more terms when needed. - Calling the gfunc essentially tees off a new instance, which begins from the first memoized item. - `fimemoize`: convert a factory function, that returns an iterable, into the corresponding gfunc, and `gmemoize` that. Return the memoized gfunc. - - Especially convenient with short lambdas, where `(yield from ...)` instead of `...` is just too much text. + - Especially convenient with short lambdas, where `(yield from ...)` instead of `...` is just too much text. See example below. ```python from itertools import count, takewhile @@ -1745,21 +2259,21 @@ def some_evens(n): # we want to memoize the result without the n first terms assert last(some_evens(25)) == last(some_evens(25)) # iterating twice! ``` -Using a lambda, we can also write ``some_evens`` as: +Using a lambda, we can also write `some_evens` as: ```python se = gmemoize(lambda n: (yield from drop(n, evens()))) assert last(se(25)) == last(se(25)) ``` -Using `fimemoize`, we can omit the ``yield from``, shortening this to: +Using `fimemoize`, we can omit the `yield from`, shortening this to: ```python se = fimemoize(lambda n: drop(n, evens())) assert last(se(25)) == last(se(25)) ``` -If we don't need to take an argument, we can memoize the iterable directly, using ``imemoize``: +If we don't need to take an argument, we can memoize the iterable directly, using `imemoize`: ```python se = imemoize(drop(25, evens())) @@ -1778,33 +2292,46 @@ def some_evens(n): yield from drop(n, evens()) ``` -The only differences are the name of the decorator and ``return`` vs. ``yield from``. The point of `fimemoize` is that in simple cases like this, it allows us to use a regular factory function that makes an iterable, instead of a gfunc. Of course, the gfunc could have several `yield` expressions before it finishes, whereas the factory function terminates at the `return`. +The only differences are the name of the decorator and `return` vs. `yield from`. The point of `fimemoize` is that in simple cases like this, it allows us to use a regular factory function that makes an iterable, instead of a gfunc. Of course, the gfunc could have several `yield` expressions before it finishes, whereas the factory function terminates at the `return`. + + +### `fup`: Functional update; `ShadowedSequence` +**Changed in v0.15.0.** *Bug fixed: Now an infinite replacement sequence to pull items from is actually ok, as the documentation has always claimed.* -### ``fup``: Functional update; ``ShadowedSequence`` +We provide three layers, in increasing order of the level of abstraction: `ShadowedSequence`, `fupdate`, and `fup`. -We provide ``ShadowedSequence``, which is a bit like ``collections.ChainMap``, but for sequences, and only two levels (but it's a sequence; instances can be chained). It supports slicing (read-only), equality comparison, ``str`` and ``repr``. Out-of-range read access to a single item emits a meaningful error, like in ``list``. See the docstring of ``ShadowedSequence`` for details. +The class `ShadowedSequence` is a bit like `collections.ChainMap`, but for sequences, and only two levels (but it's a sequence; instances can be chained). It supports slicing (read-only), equality comparison, `str` and `repr`. Out-of-range read access to a single item emits a meaningful error, like in `list`. We will not discuss `ShadowedSequence` in more detail here, as it is a low-level tool; see its docstring for details. -The function ``fupdate`` functionally updates sequences and mappings. Whereas ``ShadowedSequence`` reads directly from the original sequences at access time, ``fupdate`` makes a shallow copy, of the same type as the given input sequence, when it finalizes its output. +The function `fupdate` functionally updates sequences and mappings. Whereas `ShadowedSequence` reads directly from the original sequences at access time, `fupdate` makes a shallow copy, of the same type as the given input sequence, when it finalizes its output. -**The preferred way** to use ``fupdate`` on sequences is through the ``fup`` utility function, which specializes ``fupdate`` to sequences, and adds support for Python's standard slicing syntax: +Finally, the function `fup` provides a high-level API to functionally update a sequence, with nice syntax. + +#### `fup` + +**The preferred way** to use `fupdate` on sequences is through the `fup` utility function, which specializes `fupdate` to sequences, and adds support for Python's standard **slicing syntax**: ```python from unpythonic import fup from itertools import repeat -lst = (1, 2, 3, 4, 5) -assert fup(lst)[3] << 42 == (1, 2, 3, 42, 5) -assert fup(lst)[0::2] << tuple(repeat(10, 3)) == (10, 2, 10, 4, 10) +tup = (1, 2, 3, 4, 5) +assert fup(tup)[3] << 42 == (1, 2, 3, 42, 5) +assert fup(tup)[0::2] << tuple(repeat(10, 3)) == (10, 2, 10, 4, 10) +assert fup(tup)[0::2] << repeat(10) == (10, 2, 10, 4, 10) # infinite replacement ``` -Currently only one update specification is supported in a single ``fup()``. (The ``fupdate`` function supports more; see below.) +Currently only one *update specification* is supported in a single `fup()`. The low-level `fupdate` function supports more; see below. + +An *update specification* is a combination of **where** to update, and **what** to put there. The *where* part can be a single index or a slice. When it is a single index, the *what* is a single item; and when a slice, the *what* is a sequence or an iterable, which must contain at least as many items as are required to perform the update. For details, see `fupdate` below. + +The `fup` function is essentially curried. It takes in the sequence to be functionally updated. The object returned by the call accepts a subscript to specify the index or indices. This then returns another object that accepts a left-shift to specify the values. Once the values are provided, the underlying call to `fupdate` triggers, and the result is returned. -The notation follows the ``unpythonic`` convention that ``<<`` denotes an assignment of some sort. Here it denotes a functional update, which returns a modified copy, leaving the original untouched. +The notation follows the `unpythonic` convention that `<<` denotes an assignment of some sort. Here it denotes a functional update, which returns a modified copy, leaving the original untouched. -The ``fup`` call is essentially curried. It takes in the sequence to be functionally updated. The object returned by the call accepts a subscript to specify the index or indices. This then returns another object that accepts a left-shift to specify the values. Once the values are provided, the underlying call to ``fupdate`` triggers, and the result is returned. +#### `fupdate` -The ``fupdate`` function itself works as follows: +The `fupdate` function itself, which is the next lower abstraction level, works as follows: ```python from unpythonic import fupdate @@ -1815,58 +2342,97 @@ assert lst == [1, 2, 3] # the original remains untouched assert out == [1, 42, 3] lst = [1, 2, 3] -out = fupdate(lst, -1, 42) # negative indices also supported +out = fupdate(lst, -1, 42) # negative indices are also supported assert lst == [1, 2, 3] assert out == [1, 2, 42] ``` -Immutable input sequences are allowed. Replacing a slice of a tuple by a sequence: +Because the update is functional - i.e. the result is a new object, without mutating the original - immutable update target sequences are allowed. For example, we can replace a slice of a tuple by a sequence: ```python from itertools import repeat -lst = (1, 2, 3, 4, 5) -assert fupdate(lst, slice(0, None, 2), tuple(repeat(10, 3))) == (10, 2, 10, 4, 10) -assert fupdate(lst, slice(1, None, 2), tuple(repeat(10, 2))) == (1, 10, 3, 10, 5) -assert fupdate(lst, slice(None, None, 2), tuple(repeat(10, 3))) == (10, 2, 10, 4, 10) -assert fupdate(lst, slice(None, None, -1), tuple(range(5))) == (4, 3, 2, 1, 0) +tup = (1, 2, 3, 4, 5) +assert fupdate(tup, slice(0, None, 2), tuple(repeat(10, 3))) == (10, 2, 10, 4, 10) +assert fupdate(tup, slice(1, None, 2), tuple(repeat(10, 2))) == (1, 10, 3, 10, 5) +assert fupdate(tup, slice(None, None, 2), tuple(repeat(10, 3))) == (10, 2, 10, 4, 10) +assert fupdate(tup, slice(None, None, -1), range(5)) == (4, 3, 2, 1, 0) +``` + +Slicing supports negative indices and steps, and default starts, stops and steps, as usual in Python. Just remember `a[start:stop:step]` actually means `a[slice(start, stop, step)]` (with `None` replacing omitted `start`, `stop` and `step`), and everything should follow. Multidimensional arrays are **not** supported. + +When `fupdate` constructs its output, the replacement occurs by walking *the input sequence* left-to-right, and pulling an item from the replacement sequence when the given replacement specification so requires. Hence the replacement sequence is not necessarily accessed left-to-right. In the last example above, the `range(5)` was read in the order `4, 3, 2, 1, 0`. This is because when `slice(None, None, -1)` is applied to the input sequence, the first item of the input sequence is index `4` in the slice. So when replacing the first item, `fupdate` looked up index `4` in the replacement sequence. Because the replacement was just `range(5)`, the value at index `4` was also `4`. + +The replacement sequence must have at least as many items as the slice requires, when the slice is applied to the original input sequence. Any extra items in the replacement sequence are simply ignored, but if the replacement is too short, `IndexError` is raised. + +The replacement must have `__len__` and `__getitem__` methods if the slice (when treated as explained above) requires reading the replacement backwards, and/or if you plan to iterate over the `ShadowedSequence` multiple times. If the replacement only needs to be read forwards, **AND** you only plan to iterate over the `ShadowedSequence` just once (e.g., as part of a `fup`/`fupdate` operation), then it is sufficient for the replacement to implement the `collections.abc.Iterator` API only (i.e. just `__iter__` and `__next__`). + +##### Infinite replacements + +An infinite replacement causes `fupdate` (and `fup`) to pull as many items as are needed: + +```python +from itertools import repeat, count +from unpythonic import fup + +tup = (1, 2, 3, 4, 5) +assert fup(tup)[::] << repeat(42) == (42, 42, 42, 42, 42) +assert fup(tup)[::] << count(start=10) == (10, 11, 12, 13, 14) +``` + +The rest of the infinite replacement is considered as extra items, and is ignored. + +**CAUTION**: If converting existing code, **be careful** not to accidentally `tuple(...)` an infinite replacement. Python will happily fill all available RAM and essentially crash your machine trying to exhaust the infinite generator. + +If you need to reverse-walk the start of an infinite replacement: use `imemoize(...)` on the original iterable, instantiate the generator, and use that generator instance as the replacement: + +```python +from itertools import count +from unpythonic import fup, imemoize + +tup = (1, 2, 3, 4, 5) +assert fup(tup)[::-1] << imemoize(count(start=10))() == (14, 13, 12, 11, 10) ``` -Slicing supports negative indices and steps, and default starts, stops and steps, as usual in Python. Just remember ``a[start:stop:step]`` actually means ``a[slice(start, stop, step)]`` (with ``None`` replacing omitted ``start``, ``stop`` and ``step``), and everything should follow. Multidimensional arrays are **not** supported. +Just like above, due to the slice `[::-1]`, `fup` calculates that - when walking *the input sequence* left-to-right - it first needs to take the item at index `4` of the replacement. The `fup` succeeds, because when it retrieves this fifth item, all of the first five items are stored in the memo (which is internally a sequence). So `fup` can retrieve the fifth item, then the fourth, and so on - even though from the viewpoint of the original underlying iterable, the earlier items have already been consumed when the fifth item is accessed. -When ``fupdate`` constructs its output, the replacement occurs by walking *the input sequence* left-to-right, and pulling an item from the replacement sequence when the given replacement specification so requires. Hence the replacement sequence is not necessarily accessed left-to-right. (In the last example above, ``tuple(range(5))`` was read in the order ``(4, 3, 2, 1, 0)``.) +`ShadowedSequence` (and thus also `fupdate` and `fup`) internally uses `__getitem__` to retrieve the actual previous items from the memo, so even the memoized generator is only iterated over once. This functionality supports any generator instance created by the gfuncs returned by `imemoize`, `fimemoize`, or `gmemoize`. -The replacement sequence must have at least as many items as the slice requires (when applied to the original input). Any extra items in the replacement sequence are simply ignored (so e.g. an infinite ``repeat`` is fine), but if the replacement is too short, ``IndexError`` is raised. +##### Multiple update specifications -It is also possible to replace multiple individual items. These are treated as separate specifications, applied left to right (so later updates shadow earlier ones, if updating at the same index): +In `fupdate`, it is also possible to replace multiple individual items: ```python -lst = (1, 2, 3, 4, 5) -out = fupdate(lst, (1, 2, 3), (17, 23, 42)) -assert lst == (1, 2, 3, 4, 5) +tup = (1, 2, 3, 4, 5) +out = fupdate(tup, (1, 2, 3), (17, 23, 42)) # target, (*where), (*what) +assert tup == (1, 2, 3, 4, 5) assert out == (1, 17, 23, 42, 5) ``` +These are treated as separate specifications, applied left to right. This means later updates shadow earlier ones, if updating at the same index: + Multiple specifications can be used with slices and sequences as well: ```python -lst = tuple(range(10)) -out = fupdate(lst, (slice(0, 10, 2), slice(1, 10, 2)), +tup = tuple(range(10)) +out = fupdate(tup, (slice(0, 10, 2), slice(1, 10, 2)), (tuple(repeat(2, 5)), tuple(repeat(3, 5)))) -assert lst == tuple(range(10)) +assert tup == tuple(range(10)) assert out == (2, 3, 2, 3, 2, 3, 2, 3, 2, 3) ``` Strictly speaking, each specification can be either a slice/sequence pair or an index/item pair: ```python -lst = tuple(range(10)) -out = fupdate(lst, (slice(0, 10, 2), slice(1, 10, 2), 6), +tup = tuple(range(10)) +out = fupdate(tup, (slice(0, 10, 2), slice(1, 10, 2), 6), (tuple(repeat(2, 5)), tuple(repeat(3, 5)), 42)) -assert lst == tuple(range(10)) +assert tup == tuple(range(10)) assert out == (2, 3, 2, 3, 2, 3, 42, 3, 2, 3) ``` -Also mappings can be functionally updated: +##### `fupdate` and mappings + +Mappings can be functionally updated, too: ```python d1 = {'foo': 'bar', 'fruit': 'apple'} @@ -1875,9 +2441,11 @@ assert sorted(d1.items()) == [('foo', 'bar'), ('fruit', 'apple')] assert sorted(d2.items()) == [('foo', 'tavern'), ('fruit', 'apple')] ``` -For immutable mappings, ``fupdate`` supports ``frozendict`` (see below). Any other mapping is assumed mutable, and ``fupdate`` essentially just performs ``copy.copy()`` and then ``.update()``. +For immutable mappings, `fupdate` supports `frozendict` (see below). Any other mapping is assumed mutable, and `fupdate` essentially just performs `copy.copy()` and then `.update()`. -We can also functionally update a namedtuple: +##### `fupdate` and named tuples + +Named tuples can be functionally updated, too: ```python from collections import namedtuple @@ -1888,11 +2456,12 @@ assert a == A(17, 23) assert out == A(42, 23) ``` -Namedtuples export only a sequence interface, so they cannot be treated as mappings. +Named tuples export only a sequence interface, so they **cannot** be treated as mappings, even though their elements have names. + +Support for `namedtuple` uses an extra feature of `fupdate`, which is available for custom classes, too. When constructing the output sequence, `fupdate` first checks whether the type of the input sequence has a `._make()` method, and if so, hands the iterable containing the final data to that to construct the output. Otherwise the regular constructor is called (and it must accept a single iterable). -Support for ``namedtuple`` requires an extra feature, which is available for custom classes, too. When constructing the output sequence, ``fupdate`` first checks whether the input type has a ``._make()`` method, and if so, hands the iterable containing the final data to that to construct the output. Otherwise the regular constructor is called (and it must accept a single iterable). -### ``view``: writable, sliceable view into a sequence +### `view`: writable, sliceable view into a sequence A writable view into a sequence, with slicing, so you can take a slice of a slice (of a slice ...), and it reflects the original both ways: @@ -1917,30 +2486,36 @@ v[:] = 42 # scalar broadcast assert lst == [0, 1, 42, 42, 4] ``` -While ``fupdate`` lets you be more functional than Python otherwise allows, ``view`` lets you be more imperative than Python otherwise allows. +While `fupdate` lets you be more functional than Python otherwise allows, `view` lets you be more imperative than Python otherwise allows. We store slice specs, not actual indices, so this works also if the underlying sequence undergoes length changes. -Slicing a view returns a new view. Slicing anything else will usually copy, because the object being sliced does, before we get control. To slice lazily, first view the sequence itself and then slice that. The initial no-op view is optimized away, so it won't slow down accesses. Alternatively, pass a ``slice`` object into the ``view`` constructor. +Slicing a view returns a new view. Slicing anything else will usually shallow-copy, because the object being sliced does, before we get control. To slice lazily, first view the sequence itself and then slice that. The initial no-op view is optimized away, so it won't slow down accesses. Alternatively, pass a `slice` object into the `view` constructor. The view can be efficiently iterated over. As usual, iteration assumes that no inserts/deletes in the underlying sequence occur during the iteration. Getting/setting an item (subscripting) checks whether the index cache needs updating during each access, so it can be a bit slow. Setting a slice checks just once, and then updates the underlying iterable directly. Setting a slice to a scalar value broadcasts the scalar à la NumPy. -The ``unpythonic.collections`` module also provides the ``SequenceView`` and ``MutableSequenceView`` abstract base classes; ``view`` is a ``MutableSequenceView``. +Beside `view` itself, the `unpythonic.collections` module provides also some other related abstractions. + +There is the read-only sister of view, `roview`, which is like `view`, except it has no `__setitem__` or `reverse`. This can be useful for providing explicit read-only access to a sequence, when it is undesirable to have clients write into it. + +The constructor of the writable `view` checks that the input is not read-only (`roview`, or a `Sequence` that is not also a `MutableSequence`) before allowing creation of the writable view. -There is the read-only cousin ``roview``, which behaves the same except it has no ``__setitem__`` or ``reverse``. This can be useful for giving read-only access to an internal sequence. The constructor of the writable ``view`` checks that the input is not read-only (``roview``, or a ``Sequence`` that is not also a ``MutableSequence``) before allowing creation of the writable view. +Finally, there are the `SequenceView` and `MutableSequenceView` abstract base classes. The concrete `view` and `roview` are instances of them. +**NOTE**: A writable view supports also the read-only API, so `isinstance(MutableSequenceView, SequenceView) is True`; as well as `isinstance(view, roview) is True`. Keep in mind the [Liskov substitution principle](https://en.wikipedia.org/wiki/Liskov_substitution_principle). -### ``mogrify``: update a mutable container in-place + +### `mogrify`: update a mutable container in-place **Changed in v0.14.3.** *`mogrify` now skips `nil`, actually making it useful for processing `ll` linked lists.* -Recurse on given container, apply a function to each atom. If the container is mutable, then update in-place; if not, then construct a new copy like ``map`` does. +Recurse on a given container, apply a function to each atom. If the container is mutable, then update in-place; if not, then construct a new copy like `map` does. If the container is a mapping, the function is applied to the values; keys are left untouched. -Unlike ``map`` and its cousins, only a single input container is supported. (Supporting multiple containers as input would require enforcing some compatibility constraints on their type and shape, since ``mogrify`` is not limited to sequences.) +Unlike `map` and its cousins, **`mogrify` only supports a single input container**. Supporting multiple containers as input would require enforcing some compatibility constraints on their type and shape, because `mogrify` is not limited to sequences. ```python from unpythonic import mogrify @@ -1951,47 +2526,62 @@ assert lst2 == [2, 4, 6] assert lst2 is lst1 ``` -Containers are detected by checking for instances of ``collections.abc`` superclasses (also virtuals are ok). Supported abcs are ``MutableMapping``, ``MutableSequence``, ``MutableSet``, ``Mapping``, ``Sequence`` and ``Set``. Any value that does not match any of these is treated as an atom. Containers can be nested, with an arbitrary combination of the types supported. +Containers are detected by checking for instances of `collections.abc` superclasses (also virtuals are ok). Supported abcs are `MutableMapping`, `MutableSequence`, `MutableSet`, `Mapping`, `Sequence` and `Set`. Any value that does not match any of these is treated as an atom. Containers can be nested, with an arbitrary combination of the types supported. + +For convenience, we support some special cases: + + - Any classes created by `collections.namedtuple`; they do not conform to the standard constructor API for a `Sequence`. + + Thus, to support also named tuples: for any immutable `Sequence`, we first check for the presence of a `._make()` method, and if found, use it as the constructor. Otherwise we use the regular constructor. -For convenience, we introduce some special cases: + - `str` is treated as an atom, although technically a `Sequence`. - - Any classes created by ``collections.namedtuple``, because they do not conform to the standard constructor API for a ``Sequence``. + It does not conform to the exact same API (its constructor does not take an iterable), and often one does not want to treat strings as containers anyway. - Thus, for (an immutable) ``Sequence``, we first check for the presence of a ``._make()`` method, and if found, use it as the constructor. Otherwise we use the regular constructor. + If you want to process strings, implement it in your function that is called by `mogrify`. You can e.g. `tuple(thestring)` and then call `mogrify` on that. - - ``str`` is treated as an atom, although technically a ``Sequence``. + - The `box`, `ThreadLocalBox` and `Some` containers from the module `unpythonic.collections`. Although the first two are mutable, their update is not conveniently expressible by the `collections.abc` APIs. - It doesn't conform to the exact same API (its constructor does not take an iterable), and often we don't want to treat strings as containers anyway. + - The `cons` container from the module `unpythonic.llist`, including linked lists created using `ll` or `llist`. This is treated with the general tree strategy, so nested linked lists will be flattened, and the final `nil` is also processed. - If you want to process strings, implement it in your function that is called by ``mogrify``. + Note that since `cons` is immutable, anyway, if you know you have a long linked list where you need to update the values, just iterate over it and produce a new copy - that will work as intended. - - The ``box``, `ThreadLocalBox` and `Some` containers from ``unpythonic.collections``. Although the first two are mutable, their update is not conveniently expressible by the ``collections.abc`` APIs. - - The ``cons`` container from ``unpythonic.llist`` (including the ``ll``, ``llist`` linked lists). This is treated with the general tree strategy, so nested linked lists will be flattened, and the final ``nil`` is also processed. +### `s`, `imathify`, `gmathify`, `slift1`, `slift2`: lazy mathematical sequences with infix arithmetic - Note that since ``cons`` is immutable, anyway, if you know you have a long linked list where you need to update the values, just iterate over it and produce a new copy - that will work as intended. +**Changed in v0.15.0.** *The deprecated names have been removed.* +**Changed in v0.14.3.** *To improve descriptiveness, and for consistency with names of other abstractions in `unpythonic`, `m` has been renamed `imathify` and `mg` has been renamed `gmathify`. This is a one-time change; it is not likely that these names will be changed ever again. The old names are now deprecated.* -### ``s``, ``imathify``, ``gmathify``: lazy mathematical sequences with infix arithmetic +**Changed in v0.14.3.** *Added convenience mode to generate cyclic infinite sequences.* -**Changed in v0.14.3.** Added convenience mode to generate cyclic infinite sequences. +We provide a compact syntax to create lazy constant, cyclic, arithmetic, geometric and power sequences: `s(...)`. Numeric (`int`, `float`, `mpmath`) and symbolic (SymPy) formats are supported. We avoid accumulating roundoff error when used with floating-point formats. -**Changed in v0.14.3.** To improve descriptiveness, and for consistency with names of other abstractions in `unpythonic`, `m` has been renamed `imathify` and `mg` has been renamed `gmathify`. The old names will continue working in v0.14.x, and will be removed in v0.15.0. This is a one-time change; it is not likely that these names will be changed ever again. +We also provide arithmetic operation support for iterables (termwise). To make any iterable infix math aware, use `imathify(iterable)`. The arithmetic is lazy; it just plans computations, returning a new lazy mathematical sequence. To extract values, iterate over the result. (Note this implies that expressions consisting of thousands of operations will overflow Python's call stack. In practice this shouldn't be a problem.) -We provide a compact syntax to create lazy constant, cyclic, arithmetic, geometric and power sequences: ``s(...)``. Numeric (``int``, ``float``, ``mpmath``) and symbolic (SymPy) formats are supported. We avoid accumulating roundoff error when used with floating-point formats. +The function versions of the arithmetic operations (also provided, à la the `operator` module) have an **s** prefix (short for mathematical **sequence**), because in Python the **i** prefix (which could stand for *iterable*) is already used to denote the in-place operators. -We also provide arithmetic operation support for iterables (termwise). To make any iterable infix math aware, use ``imathify(iterable)``. The arithmetic is lazy; it just plans computations, returning a new lazy mathematical sequence. To extract values, iterate over the result. (Note this implies that expressions consisting of thousands of operations will overflow Python's call stack. In practice this shouldn't be a problem.) +We provide the [Cauchy product](https://en.wikipedia.org/wiki/Cauchy_product), and its generalization, the diagonal combination-reduction, for two (possibly infinite) iterables. Note `cauchyprod` **does not sum the series**; given the input sequences `a` and `b`, the call `cauchyprod(a, b)` computes the elements of the output sequence `c`. -The function versions of the arithmetic operations (also provided, à la the ``operator`` module) have an **s** prefix (short for mathematical **sequence**), because in Python the **i** prefix (which could stand for *iterable*) is already used to denote the in-place operators. +We also provide `gmathify`, a decorator to mathify a gfunc, so that it will `imathify()` the generator instances it makes. Combo with `imemoize` for great justice, e.g. `a = gmathify(imemoize(myiterable))`, and then `a()` to instantiate a memoized-and-mathified copy. -We provide the [Cauchy product](https://en.wikipedia.org/wiki/Cauchy_product), and its generalization, the diagonal combination-reduction, for two (possibly infinite) iterables. Note ``cauchyprod`` **does not sum the series**; given the input sequences ``a`` and ``b``, the call ``cauchyprod(a, b)`` computes the elements of the output sequence ``c``. +To apply a custom function termwise to an iterable, use `slift1` (unary) or `slift2` (binary). These lift a scalar operation into one that works on iterables, returning a lazy imathified generator. All the built-in `s`-prefixed operators (`sadd`, `sabs`, ...) are defined using this mechanism. Extra arguments are baked into each call: e.g. `slift1(round, 2)` gives termwise `round(x, 2)`. -We also provide ``gmathify``, a decorator to mathify a gfunc, so that it will ``imathify()`` the generator instances it makes. Combo with ``imemoize`` for great justice, e.g. ``a = gmathify(imemoize(myiterable))``, and then ``a()`` to instantiate a memoized-and-mathified copy. +```python +from unpythonic import slift1, slift2, s, take +from math import sin, atan2 + +ssin = slift1(sin) +assert tuple(take(3, ssin(s(1, 2, 3)))) == (sin(1), sin(2), sin(3)) + +satan2 = slift2(atan2) +assert tuple(take(3, satan2(s(1, 2, 3), s(4, 5, 6)))) == (atan2(1, 4), atan2(2, 5), atan2(3, 6)) +``` -Finally, we provide ready-made generators that yield some common sequences (currently, the Fibonacci numbers and the prime numbers). The prime generator is an FP-ized sieve of Eratosthenes. +Finally, we provide ready-made generators that yield some common sequences (currently, the Fibonacci numbers, the triangular numbers, and the prime numbers). The prime generator is an FP-ized sieve of Eratosthenes. ```python -from unpythonic import s, imathify, cauchyprod, take, last, fibonacci, primes +from unpythonic import s, imathify, cauchyprod, take, last, fibonacci, triangular, primes assert tuple(take(10, s(1, ...))) == (1,)*10 assert tuple(take(10, s(1, 2, ...))) == tuple(range(1, 11)) @@ -2019,9 +2609,10 @@ assert tuple(take(3, cauchyprod(s(1, 3, 5, ...), s(2, 4, 6, ...)))) == (2, 10, 2 assert tuple(take(10, primes())) == (2, 3, 5, 7, 11, 13, 17, 19, 23, 29) assert tuple(take(10, fibonacci())) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55) +assert tuple(take(10, triangular())) == (1, 3, 6, 10, 15, 21, 28, 36, 45, 55) ``` -A math iterable (i.e. one that has infix math support) is an instance of the class ``imathify``: +A math iterable (i.e. one that has infix math support) is an instance of the class `imathify`: ```python a = s(1, 3, ...) @@ -2072,16 +2663,16 @@ s2 = px(s(2, 4, 6, ...)) # 2, 4*x, 6*x**2, ... assert tuple(take(3, cauchyprod(s1, s2))) == (2, 10*x, 28*x**2) ``` -**CAUTION**: Symbolic sequence detection is sensitive to the assumptions on the symbols, because very pythonically, ``SymPy`` only simplifies when the result is guaranteed to hold in the most general case under the given assumptions. +**CAUTION**: Symbolic sequence detection is sensitive to the assumptions on the symbols, because very pythonically, `SymPy` only simplifies when the result is guaranteed to hold in the most general case under the given assumptions. Inspired by Haskell. -### ``sym``, ``gensym``, ``Singleton``: symbols and singletons +### `sym`, `gensym`, `Singleton`: symbols and singletons **Added in v0.14.2**. -We provide **lispy symbols**, an **uninterned symbol generator**, and a **pythonic singleton abstraction**. These are all pickle-aware, and instantiation is thread-safe. +We provide **lispy symbols**, an **uninterned symbol generator**, and a **pythonic singleton abstraction**. These are all pickle-aware and thread-safe. #### Symbol @@ -2095,17 +2686,17 @@ assert cat is sym("cat") assert cat is not sym("dog") ``` -The constructor `sym` produces an ***interned symbol***. Whenever (in the same process) **the same name** is passed to the `sym` constructor, it gives **the same object instance**. Even unpickling a symbol that has the same name produces the same `sym` object instance as any other `sym` with that name. +The constructor `sym` produces an ***interned symbol***. Whenever, in the same process, **the same name** is passed to the `sym` constructor, it gives **the same object instance**. Even unpickling a symbol that has the same name produces the same `sym` object instance as any other `sym` with that name. Thus a `sym` behaves like a Lisp symbol. Technically speaking, it's like a zen-minimalistic [Scheme/Racket symbol](https://stackoverflow.com/questions/8846628/what-exactly-is-a-symbol-in-lisp-scheme), since Common Lisp [stuffs all sorts of additional cruft in symbols](https://www.cs.cmu.edu/Groups/AI/html/cltl/clm/node27.html). If you insist on emulating that, note a `sym` is just a Python object you could customize in the usual ways, even though its instantiation logic plays by somewhat unusual rules. #### Gensym -The function `gensym` creates an ***uninterned symbol***, also known as *a gensym*. The label given in the call to `gensym` is a short human-readable description, like the name of a named symbol, but it has no relation to object identity. Object identity is tracked by an [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier), which is automatically assigned when `gensym` creates the value. Even if `gensym` is called with the same label, the return value is a new unique symbol each time. +The function `gensym`, which is an abbreviation for *generate symbol*, creates an ***uninterned symbol***, also known as *a gensym*. The label given in the call to `gensym` is a short human-readable description, like the name of a named symbol, but it has no relation to object identity. Object identity is tracked by an [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier), which is automatically assigned when `gensym` creates the value. Even if `gensym` is called with the same label, the return value is a new unique symbol each time. A gensym never conflicts with any named symbol; not even if one takes the UUID from a gensym and creates a named symbol using that as the name. -*The return value is the only time you'll see that symbol object; take good care of it!* +*The return value of `gensym` is the only time you will see that particular uninterned symbol object; take good care of it!* For example: @@ -2121,7 +2712,7 @@ print(scottishfold) # gensym:cat:94287f75-02b5-4138-9174-1e422e618d59 Uninterned symbols are useful as guaranteed-unique sentinel or [nonce (sense 2, adapted to programming)](https://en.wiktionary.org/wiki/nonce#Noun) values, like the pythonic idiom `nonce = object()`, but they come with a human-readable label. -They also have a superpower: with the help of the UUID automatically assigned by `gensym`, they survive a pickle roundtrip with object identity intact. Unpickling the *same* gensym value multiple times in the same process will produce just one object instance. (If the original return value from gensym is still alive, it is that same object instance.) +They also have a superpower: with the help of the UUID automatically assigned by `gensym`, they survive a pickle roundtrip with object identity intact. Unpickling the *same* gensym value multiple times in the same process will produce just one object instance. If the original return value from gensym is still alive, it is that same object instance. The UUID is generated with the pseudo-random algorithm [`uuid.uuid4`](https://docs.python.org/3/library/uuid.html). Due to rollover of the time field, it is possible for collisions with current UUIDs (as of the early 21st century) to occur with those generated after (approximately) the year 3400. See [RFC 4122](https://tools.ietf.org/html/rfc4122). @@ -2131,9 +2722,9 @@ Our `sym` is like a Lisp/Scheme/Racket symbol, which is essentially an [interned Our `gensym` is like the [Lisp `gensym`](http://clhs.lisp.se/Body/f_gensym.htm), and the [JavaScript `Symbol`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol). -If you're familiar with `mcpyrate`'s `gensym` or MacroPy's `gen_sym`, those mean something different. Their purpose is to create, in a macro, a lexical identifier that is not already in use in the source code being compiled, whereas our `gensym` creates an uninterned symbol object for run-time use. Lisp macros use symbols to represent identifiers, hence the potential for confusion in Python, where that is not the case. (The symbols of `unpythonic` are a purely run-time abstraction.) +If you're familiar with `mcpyrate`'s `gensym` or MacroPy's `gen_sym`, those mean something different. Their purpose is to create, in a macro, a lexical identifier that is not already in use in the source code being compiled, whereas our `gensym` creates an uninterned symbol object for run-time use. Lisp macros use symbols to represent identifiers, hence the potential for confusion in Python, where that is not the case. The symbols of `unpythonic` are a purely run-time abstraction. -If your background is in C++ or Java, you may notice the symbol abstraction is a kind of a parametric [singleton](https://en.wikipedia.org/wiki/Singleton_pattern); each symbol with the same name is a singleton (as is any gensym with the same UUID). +If your background is in C++ or Java, you may notice the symbol abstraction is a kind of a parametric [singleton](https://en.wikipedia.org/wiki/Singleton_pattern); each symbol with the same name is a singleton, as is any gensym with the same UUID. #### Singleton @@ -2150,7 +2741,7 @@ class SingleXHolder(Singleton): h = SingleXHolder(17) s = pickle.dumps(h) h2 = pickle.loads(s) -assert h2 is h # it's the same instance +assert h2 is h # the same instance! ``` Often the [singleton pattern](https://en.wikipedia.org/wiki/Singleton_pattern) is discussed in the context of classic relatively low-level, static languages such as C++ or Java. [In Python](https://stackoverflow.com/questions/6760685/creating-a-singleton-in-python), some of the classical issues, such as singletons being forced to use a clunky, nonstandard object construction syntax, are moot, because the language itself offers customization hooks that can be used to smooth away such irregularities. @@ -2161,11 +2752,11 @@ As the result of answering these questions, `unpythonic`'s idea of a singleton s However, Python can easily retrieve a singleton instance with syntax that looks like regular object construction, by customizing [`__new__`](https://docs.python.org/3/reference/datamodel.html#object.__new__). Hence no static accessor method is needed. This in turn raises the question, what should we do with constructor arguments, as we surely would like to (in general) to allow those, and they can obviously differ between call sites. Since there is only one object instance to load state into, we could either silently update the state, or silently ignore the new proposed arguments. Good luck tracking down bugs either way. But upon closer inspection, that question depends on an unfounded assumption. What we should be asking instead is, *what should happen* if the constructor of a singleton is called again, while an instance already exists? -We believe in the principles of [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns) and [fail-fast](https://en.wikipedia.org/wiki/Fail-fast). The textbook singleton pattern conflates two concerns, possibly due to language limitations: the *management of object instances*, and the *enforcement of the at-most-one-instance-only guarantee*. If we wish to uncouple these responsibilities, then the obvious pythonic answer is that attempting to construct the singleton again while it already exists **should be considered a run-time error**. Since a singleton **type** does not support that operation, this situation should raise a `TypeError`. This makes the error explicit as early as possible, thus adhering to the fail-fast principle, hence making it difficult for bugs to hide (constructor arguments will either take effect, or the constructor call will explicitly fail). +We believe in the principles of [separation of concerns](https://en.wikipedia.org/wiki/Separation_of_concerns) and [fail-fast](https://en.wikipedia.org/wiki/Fail-fast). The textbook singleton pattern conflates two concerns, possibly due to language limitations: the *management of object instances*, and the *enforcement of the at-most-one-instance-only guarantee*. If we wish to uncouple these responsibilities, then the obvious pythonic answer is that attempting to construct the singleton again while it already exists **should be considered a run-time error**. Since a singleton **type** does not support that operation, this situation should raise a `TypeError`. This makes the error explicit as early as possible, thus adhering to the fail-fast principle, hence making it difficult for bugs to hide. Constructor arguments will either take effect, or the constructor call will explicitly fail. Another question arises due to Python having builtin support for object persistence, namely `pickle`. What *should* happen when a singleton is unpickled, while an instance of that singleton already exists? Arguably, by default, it should load the state from the pickle file into the existing instance, overwriting its current state. -(Scenario: during second and later runs, a program first initializes, which causes the singleton instance to be created, just like during the first run of that program. Then the program loads state from a pickle file, containing (among other data) the state the singleton instance was in when the program previously shut down. In this scenario, considering the singleton, the data in the file is more relevant than the defaults the program initialization feeds in. Hence the default should be to replace the state of the existing singleton instance with the data from the pickle file.) +This design is based on considering the following scenario. Consider a program that uses the singleton abstraction. During its second and later runs, the program first initializes, which causes the singleton instance to be created, just like during the first run of the program. Then the program loads state from a pickle file, containing (among other data) the state the singleton instance was in when the program previously shut down. Considering the singleton, the data in the file is more relevant than the defaults the program initialization step feeds in. Hence, the default should be to *replace the state of the existing singleton instance with the data from the pickle file*. Our `Singleton` abstraction is the result of these pythonifications applied to the classic pattern. For more documentation and examples, see the unit tests in [`unpythonic/tests/test_singleton.py`](../unpythonic/tests/test_singleton.py). @@ -2175,32 +2766,34 @@ Our `Singleton` abstraction is the result of these pythonifications applied to t #### When to use a singleton? -Most often, **don't**. ``Singleton`` is provided for the very rare occasion where it's the appropriate abstraction. There exist **at least** three categories of use cases where singleton-like instantiation semantics are desirable: +Most often, **don't**. `Singleton` is provided for the very rare occasion where it's the appropriate abstraction. There exist **at least** three categories of use cases where singleton-like instantiation semantics are desirable: 1. **A process-wide unique marker value**, which has no functionality other than being quickly and uniquely identifiable as that marker. - - `sym` and `gensym` are the specific tools that cover this use case, depending on whether the intent is to allow that value to be independently "constructed" in several places yet always obtaining the same instance (`sym`), or if the implementation just happens to internally need a guaranteed-unique value that no value passed in from the outside could possibly clash with (`gensym`). For the latter case, sometimes a simple (and much faster) `nonce = object()` will do just as well, if you don't need the human-readable label and `pickle` support. + - `sym` and `gensym` are the specific tools that cover this use case, depending on whether the intent is to allow that value to be independently "constructed" in several places yet always obtaining the same instance (`sym`), or if the implementation just happens to internally need a guaranteed-unique value that no value passed in from the outside could possibly clash with (`gensym`). For the latter case, sometimes the simple (and much faster) pythonic idiom `nonce = object()` will do just as well, if you don't need a human-readable label, and `pickle` support. - If you need the singleton object to have extra functionality (e.g. our `nil` supports the iterator protocol), it's possible to subclass `sym` or `gsym`, but subclassing `Singleton` is also a possible solution. 2. **An empty immutable collection**. - - It can't have elements added to it after construction, so there's no point in creating more than one instance of an empty *immutable* collection of any particular type. - - Unfortunately, a class can't easily be partly `Singleton` (i.e., only when the instance is empty). So this use case is better coded manually, like `frozendict` does. Also, for this use case silently returning the existing instance is the right thing to do. + - An immutable collection instance cannot have elements added to it after construction, so there is no point in creating more than one instance of an *empty* immutable collection of any particular type. + - Unfortunately, a class cannot easily be partly `Singleton` (i.e., only when the instance is empty). So this use case is better coded manually, like `frozendict` does. Also, for this use case silently returning the existing instance is the right thing to do. 3. **A service that may have at most one instance** per process. - *But only if it is certain* that there can't arise a situation where multiple simultaneous instances of the service are needed. - The dynamic assignment controller `dyn` is an example, and it is indeed a `Singleton`. Cases 1 and 2 have no meaningful instance data. Case 3 may or may not, depending on the specifics. If your object does, and if you want it to support `pickle`, you may want to customize [`__getnewargs__`](https://docs.python.org/3/library/pickle.html#object.__getnewargs__) (called *at pickling time*), [`__setstate__`](https://docs.python.org/3/library/pickle.html#object.__setstate__), and sometimes maybe also [`__getstate__`](https://docs.python.org/3/library/pickle.html#object.__getstate__). Note that unpickling skips `__init__`, and calls just `__new__` (with the "newargs") and then `__setstate__`. -I'm not completely sure if it's meaningful to provide a generic `Singleton` abstraction for Python, except for teaching purposes. Practical use cases may differ so much, and some of the implementation details of the specific singleton object (esp. related to pickling) may depend so closely on the implementation details of the singleton abstraction, that it may be easier to just roll your own singleton code when needed. If you're new to customizing this part of Python, the code we have here should at least demonstrate an approach for how to do this. +I am not completely sure if it is meaningful to provide a generic `Singleton` abstraction for Python, except for teaching purposes. Practical use cases may differ so much, and some of the implementation details of the specific singleton object (especially related to pickling) may depend so closely on the implementation details of the singleton abstraction, that it may be easier to just roll your own singleton code when needed. If you are new to customizing this part of Python, the code we have here should at least demonstrate how to do that. ## Control flow tools -Tools related to control flow. +Tools related to [control flow](https://en.wikipedia.org/wiki/Control_flow). -### ``trampolined``, ``jump``: tail call optimization (TCO) / explicit continuations +### `trampolined`, `jump`: tail call optimization (TCO) / explicit continuations -Express algorithms elegantly without blowing the call stack - with explicit, clear syntax. +*See also the `with tco` [macro](macros.md), which applies tail call optimization **automatically**.* -*Tail recursion*: +*Tail call optimization* is a technique to treat [tail calls](https://en.wikipedia.org/wiki/Tail_call) in such a way that they do not grow the call stack. It sometimes allows expressing algorithms very elegantly. Some functional programming patterns such as functional loops are based on tail calls. + +The factorial function is a classic example of *tail recursion*: ```python from unpythonic import trampolined, jump @@ -2209,62 +2802,94 @@ from unpythonic import trampolined, jump def fact(n, acc=1): if n == 0: return acc - else: - return jump(fact, n - 1, n * acc) + return jump(fact, n - 1, n * acc) print(fact(4)) # 24 +fact(5000) # no crash ``` -Functions that use TCO **must** be `@trampolined`. Calling a trampolined function normally starts the trampoline. +Functions that use TCO **must** be `@trampolined`. The decorator wraps the original function with a [trampoline](https://en.wikipedia.org/wiki/Trampoline_(computing)#High-level_programming). Calling a trampolined function normally starts the trampoline. Inside a trampolined function, a normal call `f(a, ..., kw=v, ...)` remains a normal call. -A tail call with target `f` is denoted `return jump(f, a, ..., kw=v, ...)`. This explicitly marks that it is indeed a tail call (due to the explicit ``return``). Note that `jump` is **a noun, not a verb**. The `jump(f, ...)` part just evaluates to a `jump` instance, which on its own does nothing. Returning it to the trampoline actually performs the tail call. +A tail call with target `f` is denoted `return jump(f, a, ..., kw=v, ...)`. This explicitly marks that it is indeed a tail call, due to the explicit `return`. Note that `jump` is **a noun, not a verb**. The `jump(f, ...)` part just evaluates to a `jump` instance, which on its own does nothing. Returning the `jump` instance to the trampoline actually performs the tail call. + +If the jump target has a trampoline, the trampoline implementation will automatically strip it and jump into the actual entry point. + +To return a final result, just `return` it normally. Returning anything but a `jump` shuts down the trampoline, and returns the given value from the initial call (to the `@trampolined` function) that originally started that trampoline. -If the jump target has a trampoline, don't worry; the trampoline implementation will automatically strip it and jump into the actual entrypoint. +**CAUTION**: Trying to `jump(...)` without the `return` does nothing useful, and will **usually** print an *unclaimed jump* warning. It does this by checking a flag in the `__del__` method of `jump`; any correctly used jump instance should have been claimed by a trampoline before it gets garbage-collected. It can only print a warning, not raise an exception or halt the program, due to the limitations of `__del__`. -Trying to ``jump(...)`` without the ``return`` does nothing useful, and will **usually** print an *unclaimed jump* warning. It does this by checking a flag in the ``__del__`` method of ``jump``; any correctly used jump instance should have been claimed by a trampoline before it gets garbage-collected. +Some *unclaimed jump* warnings may appear also if the process is terminated by Ctrl+C (`KeyboardInterrupt`). This is normal; it just means that the termination occurred after a jump object was instantiated but before it was claimed by a trampoline. -(Some *unclaimed jump* warnings may appear also if the process is terminated by Ctrl+C (``KeyboardInterrupt``). This is normal; it just means that the termination occurred after a jump object was instantiated but before it was claimed by the trampoline.) +For comparison, with the macro API, the example becomes: -The final result is just returned normally. This shuts down the trampoline, and returns the given value from the initial call (to a ``@trampolined`` function) that originally started that trampoline. +```python +from unpythonic.syntax import macros, tco + +with tco: + def fact(n, acc=1): + if n == 0: + return acc + return fact(n - 1, n * acc) +print(fact(4)) # 24 +fact(5000) # no crash +``` + +*The `with tco` macro implicitly inserts the `@trampolined` decorator, and converts any regular call that appears in tail position into a `jump`. It also transforms lambdas in a similar way.* +#### Tail recursion in a `lambda` -*Tail recursion in a lambda*: +To make a tail-recursive anonymous function, use `trampolined` together with `withself`. The `self` argument is declared explicitly, but passed implicitly, just like the `self` argument of a method: ```python +from unpythonic import trampolined, jump, withself + t = trampolined(withself(lambda self, n, acc=1: acc if n == 0 else jump(self, n - 1, n * acc))) print(t(4)) # 24 ``` -Here the jump is just `jump` instead of `return jump`, since lambda does not use the `return` syntax. +Here the jump is just `jump` instead of `return jump`, because `lambda` does not use the `return` syntax. + +For comparison, with the macro API, this becomes: -To denote tail recursion in an anonymous function, use ``unpythonic.fun.withself``. The ``self`` argument is declared explicitly, but passed implicitly, just like the ``self`` argument of a method. +```python +from unpythonic.syntax import macros, tco +from unpythonic import withself + +with tco: + t = withself(lambda self, n, acc=1: + acc if n == 0 else self(n - 1, n * acc)) +print(t(4)) # 24 +``` +#### Mutual recursion with TCO -*Mutual recursion with TCO*: +[Mutual recursion](https://en.wikipedia.org/wiki/Mutual_recursion) is also supported. Just ask the trampoline to `jump` into the desired function: ```python +from unpythonic import trampolines,jump + @trampolined def even(n): if n == 0: return True - else: - return jump(odd, n - 1) + return jump(odd, n - 1) @trampolined def odd(n): if n == 0: return False - else: - return jump(even, n - 1) + return jump(even, n - 1) assert even(42) is True assert odd(4) is False assert even(10000) is True # no crash ``` -*Mutual recursion in `letrec` with TCO*: +#### Mutual recursion in `letrec` with TCO ```python +from unpythonic import letrec, trampolined, jump + letrec(evenp=lambda e: trampolined(lambda x: (x == 0) or jump(e.oddp, x - 1)), @@ -2275,6 +2900,18 @@ letrec(evenp=lambda e: e.evenp(10000)) ``` +For comparison, with the macro API of `letrec`, this becomes: + +```python +from unpythonic.syntax import macros, letrec +from unpythonic import trampolined, jump + +letrec[[evenp << trampolined(lambda x: + (x == 0) or jump(oddp, x - 1)), + oddp << trampolined(lambda x: + (x != 0) and jump(evenp, x - 1))] in + evenp(10000)] +``` #### Reinterpreting TCO as explicit continuations @@ -2317,22 +2954,25 @@ Clojure has [`(trampoline ...)`](https://clojuredocs.org/clojure.core/trampoline The `return jump(...)` solution is essentially the same there (the syntax is `#(...)`), but in Clojure, the trampoline must be explicitly enabled at the call site, instead of baking it into the function definition, as our decorator does. -Clojure's trampoline system is thus more explicit and simple than ours (the trampoline doesn't need to detect and strip the tail-call target's trampoline, if it has one - because with Clojure's solution, it never does), at some cost to convenience at each use site. We have chosen to emphasize use-site convenience. +Clojure's trampoline system is thus more explicit and simple than ours (the trampoline does not need to detect and strip the tail-call target's trampoline, if it has one - because with Clojure's solution, it never does), at some cost to convenience at each use site. We have chosen to emphasize use-site convenience. + +### `looped`, `looped_over`: loops in FP style (with TCO) -### ``looped``, ``looped_over``: loops in FP style (with TCO) +In functional programming, looping can be represented as recursion. The loop body is written as a recursive function. To loop, the function tail-calls itself, possibly with new argument values. Both `for` and `while` loops can be expressed in this way. -*Functional loop with automatic tail call optimization* (for calls re-invoking the loop body): +As a practical detail, tail-call optimization is important, to avoid growing the call stack at each iteration of the loop. + +Here is a functional loop using `unpythonic`, with automatic tail call optimization - no macros needed: ```python -from unpythonic import looped, looped_over +from unpythonic import looped @looped def s(loop, acc=0, i=0): if i == 10: return acc - else: - return loop(acc + i, i + 1) + return loop(acc + i, i + 1) print(s) # 45 ``` @@ -2349,32 +2989,39 @@ define s displayln s ; 45 ``` -The `@looped` decorator is essentially sugar. Behaviorally equivalent code: +In `@looped`, the function name of the loop body is the name of the final result, like in `@call`. To terminate the loop, just `return` the final result normally. This shuts down the loop and replaces the loop body definition (in the example, `s`) with the final result value. -```python -@trampolined -def s(acc=0, i=0): - if i == 10: - return acc - else: - return jump(s, acc + i, i + 1) -s = s() -print(s) # 45 -``` +The first parameter of the loop body is the magic parameter `loop`. It is *self-ish*, representing a jump back to the loop body itself, starting a new iteration. Just like Python's `self`, `loop` can have any name; it is passed positionally. + +Note that `loop` is **a noun, not a verb.** This is because the expression `loop(...)` is essentially the same as `jump(...)` to the loop body itself. However, it also arranges things so that the trampolined call inserts the magic parameter `loop`, which can only be set up via this mechanism. + +Additional arguments can be given to `loop(...)`. When the loop body is called, any additional positional arguments are appended to the implicit ones, and can be anything. Additional arguments can also be passed by name. The initial values of any additional arguments **must** be declared as defaults in the formal parameter list of the loop body. The loop is automatically started by `@looped`, by calling the body with the magic `loop` as the only argument. -In `@looped`, the function name of the loop body is the name of the final result, like in `@call`. The final result of the loop is just returned normally. +Any loop variables such as `i` in the above example are **in scope only in the loop body**; there is no `i` in the surrounding scope. Moreover, it is a fresh `i` at each iteration; nothing is mutated by the looping mechanism. -The first parameter of the loop body is the magic parameter ``loop``. It is *self-ish*, representing a jump back to the loop body itself, starting a new iteration. Just like Python's ``self``, ``loop`` can have any name; it is passed positionally. +**Be careful** if you use a mutable object instance as a loop variable: the loop body is just a function call like any other, so the usual rules apply. -Note that ``loop`` is **a noun, not a verb.** This is because the expression ``loop(...)`` is essentially the same as ``jump(...)`` to the loop body itself. However, it also inserts the magic parameter ``loop``, which can only be set up via this mechanism. +For another example of functional looping, here is a typical `while True` loop in FP style: -Additional arguments can be given to ``loop(...)``. When the loop body is called, any additional positional arguments are appended to the implicit ones, and can be anything. Additional arguments can also be passed by name. The initial values of any additional arguments **must** be declared as defaults in the formal parameter list of the loop body. The loop is automatically started by `@looped`, by calling the body with the magic ``loop`` as the only argument. +```python +from unpythonic import looped -Any loop variables such as ``i`` in the above example are **in scope only in the loop body**; there is no ``i`` in the surrounding scope. Moreover, it's a fresh ``i`` at each iteration; nothing is mutated by the looping mechanism. (But be careful if you use a mutable object instance as a loop variable. The loop body is just a function call like any other, so the usual rules apply.) +@looped +def _(loop): + print("Enter your name (or 'q' to quit): ", end='') + s = input() + if s.lower() == 'q': + return # ...the implicit None. In a "while True:", "break" here. + else: + print(f"Hello, {s}!") + return loop() +``` -FP loops don't have to be pure: +Functional loops do not have to be pure. Here is a functional loop with a side effect: ```python +from unpythonic import looped + out = [] @looped def _(loop, i=0): @@ -2385,59 +3032,73 @@ def _(loop, i=0): assert out == [0, 1, 2, 3] ``` -Keep in mind, though, that this pure-Python FP looping mechanism is slow, so it may make sense to use it only when "the FP-ness" (no mutation, scoping) is important. +**CAUTION**: This pure-Python FP looping mechanism is slow, so it may make sense to use it only when "the FP-ness" (no mutation, scoping) is important. -Also be aware that `@looped` is specifically neither a ``for`` loop nor a ``while`` loop; instead, it is a general looping mechanism that can express both kinds of loops. +#### Relation to the TCO system -*Typical `while True` loop in FP style*: +The `@looped` decorator is essentially sugar. If you read the section further above on TCO, you may have guessed how it is implemented: the `loop` function is actually a jump record in disguise, and `@looped` installs a trampoline. + +Indeed, the following code is behaviorally equivalent to the first example: ```python -@looped -def _(loop): - print("Enter your name (or 'q' to quit): ", end='') - s = input() - if s.lower() == 'q': - return # ...the implicit None. In a "while True:", "break" here. - else: - print(f"Hello, {s}!") - return loop() +from unpythonic import trampolined, jump + +@trampolined +def s(acc=0, i=0): + if i == 10: + return acc + return jump(s, acc + i, i + 1) +s = s() +print(s) # 45 ``` +However, the actual implementation of `@looped` slightly differs from what would be implied by this straightforward translation, because the feature uses no macros. + #### FP loop over an iterable -In Python, loops often run directly over the elements of an iterable, which markedly improves readability compared to dealing with indices. Enter ``@looped_over``: +In Python, loops often run directly over the elements of an iterable, which markedly improves readability compared to dealing with indices. + +For this use case, we provide `@looped_over`: ```python +from unpythonic import looped_over + @looped_over(range(10), acc=0) def s(loop, x, acc): return loop(acc + x) assert s == 45 ``` -The ``@looped_over`` decorator is essentially sugar. Behaviorally equivalent code: +The `@looped_over` decorator is essentially sugar. Behaviorally equivalent code: ```python +from unpythonic import call, looped + @call def s(iterable=range(10)): it = iter(iterable) @looped - def _tmp(loop, acc=0): + def tmp(loop, acc=0): try: x = next(it) - return loop(acc + x) + return loop(acc + x) # <-- the loop body except StopIteration: return acc - return _tmp + return tmp assert s == 45 ``` -In ``@looped_over``, the loop body takes three magic positional parameters. The first parameter ``loop`` works like in ``@looped``. The second parameter ``x`` is the current element. The third parameter ``acc`` is initialized to the ``acc`` value given to ``@looped_over``, and then (functionally) updated at each iteration, taking as the new value the first positional argument given to ``loop(...)``, if any positional arguments were given. Otherwise ``acc`` retains its last value. +In `@looped_over`, the loop body takes **three** magic positional parameters. The first parameter `loop` is similar to that in `@looped`. The second parameter `x` is the current element. The third parameter `acc` is initialized to the `acc` value given to `@looped_over`, and then (functionally) updated at each iteration. -If ``acc`` is a mutable object, mutating it is allowed. For example, if ``acc`` is a list, it is perfectly fine to ``acc.append(...)`` and then just ``loop()`` with no arguments, allowing ``acc`` to retain its last value. To be exact, keeping the last value means *the binding of the name ``acc`` does not change*, so when the next iteration starts, the name ``acc`` still points to the same object that was mutated. This strategy can be used to pythonically construct a list in an FP loop. +The new value of `acc` is the first positional argument given to `loop(...)`, if any positional arguments were given. Otherwise `acc` retains its last value. -Additional arguments can be given to ``loop(...)``. The same notes as above apply. For example, here we have the additional parameters ``fruit`` and ``number``. The first one is passed positionally, and the second one by name: +If `acc` is a mutable object, mutating it **is allowed**. For example, if `acc` is a list, it is perfectly fine to `acc.append(...)` and then just `loop()` with no arguments, allowing `acc` to retain its last value. To be exact, keeping the last value means *the binding of the name `acc` does not change*, so when the next iteration starts, the name `acc` still points to the same object that was mutated. This strategy can be used to pythonically construct a list in an FP loop. + +Additional arguments can be given to `loop(...)`. The same notes as above apply. For example, here we have the additional parameters `fruit` and `number`. The first one is passed positionally, and the second one by name: ```python +from unpythonic import looped_over + @looped_over(range(10), acc=0) def s(loop, x, acc, fruit="pear", number=23): print(fruit, number) @@ -2447,13 +3108,15 @@ def s(loop, x, acc, fruit="pear", number=23): assert s == 45 ``` -The loop body is called once for each element in the iterable. When the iterable runs out of elements, the last ``acc`` value that was given to ``loop(...)`` becomes the return value of the loop. If the iterable is empty, the body never runs; then the return value of the loop is the initial value of ``acc``. +The loop body is called once for each element in the iterable. When the iterable runs out of elements, the final value of `acc` becomes the return value of the loop. If the iterable is empty, the body never runs; then the return value of the loop is the initial value of `acc`. -To terminate the loop early, just ``return`` your final result normally, like in ``@looped``. (It can be anything, does not need to be ``acc``.) +To terminate the loop early, just `return` your final result normally, like in `@looped`. It can be anything, it does not need to be `acc`. -Multiple input iterables work somewhat like in Python's ``for``, except any sequence unpacking must be performed inside the body: +Multiple input iterables work somewhat like in Python's `for`, except any sequence unpacking must be performed inside the body: ```python +from unpythonic import looped_over + @looped_over(zip((1, 2, 3), ('a', 'b', 'c')), acc=()) def p(loop, item, acc): numb, lett = item @@ -2472,6 +3135,8 @@ This is because while *tuple parameter unpacking* was supported in Python 2.x, i FP loops can be nested (also those over iterables): ```python +from unpythonic import looped_over + @looped_over(range(1, 4), acc=()) def outer_result(outer_loop, y, outer_acc): @looped_over(range(1, 3), acc=()) @@ -2481,18 +3146,20 @@ def outer_result(outer_loop, y, outer_acc): assert outer_result == ((1, 2), (2, 4), (3, 6)) ``` -If you feel the trailing commas ruin the aesthetics, see ``unpythonic.misc.pack``. +If you feel the trailing commas ruin the aesthetics, see `unpythonic.pack`. #### Accumulator type and runtime cost As [the reference warns (note 6)](https://docs.python.org/3/library/stdtypes.html#common-sequence-operations), repeated concatenation of tuples has an O(n²) runtime cost, because each concatenation creates a new tuple, which needs to copy all of the already existing elements. To keep the runtime O(n), there are two options: - - *Pythonic solution*: Destructively modify a mutable sequence. Particularly, ``list`` is a dynamic array that has a low amortized cost for concatenation (most often O(1), with the occasional O(n) when the allocated storage grows). - - *Unpythonic solution*: ``cons`` a linked list, and reverse it at the end. Cons cells are immutable; consing a new element to the front costs O(1). Reversing the list costs O(n). + - *Pythonic solution*: Destructively modify a mutable sequence. Particularly, `list` is a dynamic array that has a low amortized cost for concatenation (most often O(1), with the occasional O(n) when the allocated storage grows). + - *Unpythonic solution*: `cons` a linked list, and reverse it at the end. Cons cells are immutable; consing a new element to the front costs O(1). Reversing the list costs O(n). -Mutable sequence (Python ``list``): +Mutable sequence (Python `list`): ```python +from unpythonic import looped_over + @looped_over(zip((1, 2, 3), ('a', 'b', 'c')), acc=[]) def p(loop, item, acc): numb, lett = item @@ -2505,7 +3172,7 @@ assert p == ['1a', '2b', '3c'] Linked list: ```python -from unpythonic import cons, nil, ll +from unpythonic import looped_over, cons, nil, ll, lreverse @lreverse @looped_over(zip((1, 2, 3), ('a', 'b', 'c')), acc=nil) @@ -2516,11 +3183,13 @@ def p(loop, item, acc): assert p == ll('1a', '2b', '3c') ``` -Note the unpythonic use of the ``lreverse`` function as a decorator. ``@looped_over`` overwrites the def'd name by the return value of the loop; then ``lreverse`` takes that as input, and overwrites once more. Thus ``p`` becomes the final list. +Note the unpythonic use of the `lreverse` function as a decorator. `@looped_over` overwrites the def'd name by the return value of the loop; then `lreverse` takes that as input, and overwrites once more. Thus `p` becomes the final list. -To get the output as a tuple, we can add ``tuple`` to the decorator chain: +To get the output as a tuple, we can add `tuple` to the decorator chain: ```python +from unpythonic import looped_over, cons, nil, ll, lreverse + @tuple @lreverse @looped_over(zip((1, 2, 3), ('a', 'b', 'c')), acc=nil) @@ -2533,17 +3202,19 @@ assert p == ('1a', '2b', '3c') This works in both solutions. The cost is an additional O(n) step. -#### ``break`` +#### `break` -The main way to exit an FP loop (also early) is, at any time, to just ``return`` the final result normally. +The main way to exit an FP loop (also early) is, at any time, to just `return` the final result normally. If you want to exit the function *containing* the loop from inside the loop, see **escape continuations** below. -#### ``continue`` +#### `continue` -The main way to *continue* an FP loop is, at any time, to ``loop(...)`` with the appropriate arguments that will make it proceed to the next iteration. Or package the appropriate `loop(...)` expression into your own function ``cont``, and then use ``cont(...)``: +The main way to *continue* an FP loop is, at any time, to `loop(...)` with the appropriate arguments that will make the loop proceed to the next iteration. Or package the appropriate `loop(...)` expression into your own function `cont`, and then use `cont(...)`: ```python +from unpythonic import looped + @looped def s(loop, acc=0, i=0): cont = lambda newacc=acc: loop(newacc, i + 1) # always increase i; by default keep current value of acc @@ -2558,31 +3229,35 @@ print(s) # 35 This approach separates the computations of the new values for the iteration counter and the accumulator. -#### Prepackaged ``break`` and ``continue`` +#### Prepackaged `break` and `continue` -See ``@breakably_looped`` (offering `brk`) and ``@breakably_looped_over`` (offering `brk` and `cnt`). +See `@breakably_looped` (offering `brk`) and `@breakably_looped_over` (offering `brk` and `cnt`). -The point of `brk(value)` over just `return value` is that `brk` is first-class, so it can be passed on to functions called by the loop body (so that those functions then have the power to directly terminate the loop). +The point of `brk(value)` over just `return value` is that `brk` is first-class, so it can be passed on to functions called by the loop body - so that those functions then have the power to directly terminate the loop. -In ``@looped``, a library-provided ``cnt`` wouldn't make sense, since all parameters except ``loop`` are user-defined. *The client code itself defines what it means to proceed to the "next" iteration*. Really the only way in a construct with this degree of flexibility is for the client code to fill in all the arguments itself. +In `@looped`, a library-provided `cnt` would not make sense, since all parameters except `loop` are user-defined. *The client code itself defines what it means to proceed to the "next" iteration*. Really the only way in a construct with this degree of flexibility is for the client code to fill in all the arguments itself. -Because ``@looped_over`` is a more specific abstraction, there the concept of *continue* is much more clear-cut. We define `cnt` to mean *proceed to take the next element from the iterable, keeping the current value of `acc`*. Essentially `cnt` is a partially applied `loop(...)` with the first positional argument set to the current value of `acc`. +Because `@looped_over` is a more specific abstraction, there the concept of *continue* is much more clear-cut. We define `cnt` to mean *proceed to take the next element from the iterable, keeping the current value of `acc`*. Essentially `cnt` is a partially applied `loop(...)` with the first positional argument set to the current value of `acc`. #### FP loops using a lambda as body Just call the `looped()` decorator manually: ```python +from unpythonic import looped + s = looped(lambda loop, acc=0, i=0: loop(acc + i, i + 1) if i < 10 else acc) print(s) ``` -It's not just a decorator; in Lisps, a construct like this would likely be named ``call/looped``. +It's not just a decorator; in the Scheme family of Lisps, a construct like this would likely be named `call/looped`. -We can also use ``let`` to make local definitions: +We can also use `let` to make local definitions: ```python +from unpythonic import looped, let + s = looped(lambda loop, acc=0, i=0: let(cont=lambda newacc=acc: loop(newacc, i + 1), @@ -2594,28 +3269,55 @@ print(s) The `looped_over()` decorator also works, if we just keep in mind that parameterized decorators in Python are actually decorator factories: ```python +from unpythonic import looped_over + r10 = looped_over(range(10), acc=0) s = r10(lambda loop, x, acc: loop(acc + x)) assert s == 45 ``` -If you **really** need to make that into an expression, bind ``r10`` using ``let`` (if you use ``letrec``, keeping in mind it is a callable), or to make your code unreadable, just inline it. +If you **really** need to make that into an expression, bind `r10` using `let` (if you use `letrec`, keeping in mind it is a callable), or to make your code unreadable, just inline it. -With ``curry``, this is also a possible solution: +With `curry`, using its passthrough feature, this is also a possible solution: ```python +from unpythonic import curry, looped_over + s = curry(looped_over, range(10), 0, lambda loop, x, acc: loop(acc + x)) assert s == 45 ``` -### ``gtrampolined``: generators with TCO +As of v0.15.0, `curry` handles also named arguments, so we can make explicit what the `0` means: + +```python +from unpythonic import curry, looped_over + +s = curry(looped_over, range(10), acc=0, + body=(lambda loop, x, acc: + loop(acc + x))) +assert s == 45 +``` + +but because, due to syntactic limitations of Python, no positional arguments can be given *after* a named argument, you then have to know - in order to be able to provide the loop body - that the decorator returned by the factory `looped_over` calls it `body`. + +You can of course obtain such information by inspection (here shown in IPython running Python 3.8): + +```python +In [2]: looped_over(range(10), acc=0) +Out[2]: .run(body)> +``` + +or by looking at [the source code](../unpythonic/fploop.py). + + +### `gtrampolined`: generators with TCO -In ``unpythonic``, a generator can tail-chain into another generator. This is like invoking ``itertools.chain``, but as a tail call from inside the generator - so the generator itself can choose the next iterable in the chain. If the next iterable is a generator, it can again tail-chain into something else. If it is not a generator, it becomes the last iterable in the TCO chain. +In `unpythonic`, a generator can tail-chain into another generator. This is like invoking `itertools.chain`, but as a tail call from inside the generator - so that the generator itself can choose the next iterable in the chain. If the next iterable is a generator, it can again tail-chain into something else. If it is not a generator, it becomes the last iterable in the TCO chain. -Python provides a convenient hook to build things like this, in the guise of ``return``: +Python provides a convenient hook to build things like this, in the guise of `return`: ```python from unpythonic import gtco, take, last @@ -2628,7 +3330,7 @@ assert tuple(take(6, gtco(march()))) == (1, 2, 1, 2, 1, 2) last(take(10000, gtco(march()))) # no crash ``` -Note the calls to ``gtco`` at the use sites. For convenience, we provide ``@gtrampolined``, which automates that: +Note the calls to `gtco` at the use sites. For convenience, we provide `@gtrampolined`, which automates that: ```python from unpythonic import gtrampolined, take, last @@ -2641,7 +3343,7 @@ assert tuple(take(10, ones())) == (1,) * 10 last(take(10000, ones())) # no crash ``` -It is safe to tail-chain into a ``@gtrampolined`` generator; the system strips the TCO target's trampoline if it has one. +It is safe to tail-chain into a `@gtrampolined` generator; the system strips the TCO target's trampoline if it has one. Like all tail calls, this works for any *iterative* process. In contrast, this **does not work**: @@ -2656,26 +3358,30 @@ def fibos(): # see numerics.py print(tuple(take(10, fibos()))) # --> (1, 1, 2), only 3 terms?! ``` -This sequence (technically iterable, but in the mathematical sense) is recursively defined, and the ``return`` shuts down the generator before it can yield more terms into ``scanl``. With ``yield from`` instead of ``return`` the second example works (but since it is recursive, it eventually blows the call stack). +This sequence (technically iterable, but in the mathematical sense) is recursively defined, and the `return` shuts down the generator before it can yield more terms into `scanl`. With `yield from` instead of `return` the second example works - but since it is recursive, it eventually blows the call stack. This particular example can be converted into a linear process with a different higher-order function, no TCO needed: ```python -from unpythonic import unfold, take, last +from unpythonic import unfold, take, last, Values def fibos(): def nextfibo(a, b): - return a, b, a + b # value, *newstates + return Values(a, a=b, b=a + b) return unfold(nextfibo, 1, 1) assert tuple(take(10, fibos())) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55) last(take(10000, fibos())) # no crash ``` -### ``catch``, ``throw``: escape continuations (ec) +### `catch`, `throw`: escape continuations (ec) -**Changed in v0.14.2.** *These constructs were previously named `setescape`, `escape`. The names have been changed to match the standard naming for this feature in several Lisps. Starting in 0.14.2, using the old names emits a `FutureWarning`, and the old names will be removed in 0.15.0.* +**Changed in v0.15.0.** *The deprecated names have been removed.* -Escape continuations can be used as a *multi-return*: +**Changed in v0.14.2.** *These constructs were previously named `setescape`, `escape`. The names have been changed to match the standard naming for this feature in several Lisps. The old names are now deprecated.* + +In a nutshell, an *escape continuation*, often abbreviated *ec*, transfers control outward on the call stack. Escape continuations are a generalization of `continue`, `break` and `return`. Those three constructs are essentially second-class ecs with a hard-coded escape point (respectively: end of iteration of loop; end of loop; end of function). A general escape continuation mechanism allows setting an escape point explicitly. + +For example, escape continuations can be used as a *multi-return*: ```python from unpythonic import catch, throw @@ -2690,13 +3396,11 @@ def f(): assert f() == "hello from g" ``` -**CAUTION**: The implementation is based on exceptions, so catch-all ``except:`` statements will intercept also throws, breaking the escape mechanism. As you already know, be specific in which exception types you catch in an `except` clause! +In Lisp terms, `@catch` essentially captures the escape continuation (ec) of the function decorated with it. The nearest (dynamically) surrounding ec can then be invoked by `throw(value)`. When the `throw` is performed, the function decorated with `@catch` immediately terminates, returning `value`. -In Lisp terms, `@catch` essentially captures the escape continuation (ec) of the function decorated with it. The nearest (dynamically) surrounding ec can then be invoked by `throw(value)`. When the `throw` is performed, the function decorated with `@catch` immediately terminates, returning ``value``. +In Python terms, a throw (in the escape continuation sense) means just raising a specific type of exception; the usual rules concerning `try/except/else/finally` and `with` blocks apply. The `throw` is a function call, so it works also in lambdas. -In Python terms, a throw means just raising a specific type of exception; the usual rules concerning ``try/except/else/finally`` and ``with`` blocks apply. It is a function call, so it works also in lambdas. - -Escaping the function surrounding an FP loop, from inside the loop: +For another example, here we return from the function surrounding an FP loop, from inside the loop: ```python @catch() @@ -2710,7 +3414,7 @@ def f(): f() # --> 15 ``` -For more control, both ``@catch`` points and ``throw`` instances can be tagged: +For more control, both `@catch` points and `throw` instances can be tagged: ```python @catch(tags="foo") # catch point tags can be single value or tuple (tuples OR'd, like isinstance()) @@ -2730,24 +3434,28 @@ def foo(): assert foo() == 15 ``` -For details on tagging, especially how untagged and tagged throw and catch points interact, and how to make one-to-one connections, see the docstring for ``@catch``. +For details on tagging, especially how untagged and tagged throw and catch points interact, and how to make one-to-one connections, see the docstring for `@catch`. See also `call_ec` (below), which is a compact syntax to make a one-to-one connection. + +**CAUTION**: The implementation is based on exceptions, so catch-all `except:` statements will intercept also throws, breaking the escape mechanism. As you already know, be specific in which exception types you catch in an `except` clause! **Etymology** -This feature is known as `catch`/`throw` in several Lisps, e.g. in Emacs Lisp and in Common Lisp (as well as some of its ancestors). This terminology is independent of the use of `throw`/`catch` in C++/Java for the exception handling mechanism. Common Lisp also provides a lexically scoped variant (`BLOCK`/`RETURN-FROM`) that is more idiomatic [according to Seibel](http://www.gigamonkeys.com/book/the-special-operators.html). +This feature is known as `catch`/`throw` in several Lisps, e.g. in Emacs Lisp and in Common Lisp (as well as some of its ancestors). This terminology is independent of the use of `throw`/`catch` in C++/Java for the exception handling mechanism. +Common Lisp also provides a lexically scoped variant (`BLOCK`/`RETURN-FROM`) that is more idiomatic ([according to Seibel](http://www.gigamonkeys.com/book/the-special-operators.html)), but we currently provide only this dynamic variant. -#### ``call_ec``: first-class escape continuations -We provide ``call/ec`` (a.k.a. ``call-with-escape-continuation``), in Python spelled as ``call_ec``. It's a decorator that, like ``@call``, immediately runs the function and replaces the def'd name with the return value. The twist is that it internally sets up a catch point, and hands a **first-class escape continuation** to the callee. +#### `call_ec`: first-class escape continuations -The function to be decorated **must** take one positional argument, the ec instance. +We provide the function `call/ec` (a.k.a. [`call-with-escape-continuation`](https://docs.racket-lang.org/reference/cont.html#(def._((quote._~23~25kernel)._call-with-escape-continuation)))), in Python spelled as `call_ec`. It's a decorator that, like `@call`, immediately runs the function and replaces the def'd name with the return value. The twist is that it internally sets up a catch point, and hands a **first-class escape continuation** to the callee. -The ec instance itself is another function, which takes one positional argument: the value to send to the catch point. The ec instance and the catch point are connected one-to-one. No other ``@catch`` point will catch the ec instance, and the catch point catches only this particular ec instance and nothing else. +The function to be decorated **must** take one positional argument, the ec instance. The parameter is conventionally named `ec`. -Any particular ec instance is only valid inside the dynamic extent of the ``call_ec`` invocation that created it. Attempting to call the ec later raises ``RuntimeError``. +The ec instance itself is another function, which takes one positional argument: the value to send to the catch point. That value can also be a `Values` object if you want to escape with multiple-return-values or named return values; the ec will send any argument given to it. -This builds on ``@catch`` and ``throw``, so the caution about catch-all ``except:`` statements applies here, too. +The ec instance and the catch point are connected one-to-one. No other `@catch` point will catch the ec instance, and the catch point catches only the ec instances created by this invocation of `call_ec`, and nothing else. + +Any particular ec instance is only valid inside the dynamic extent of the `call_ec` invocation that created it. Attempting to call the ec later raises `RuntimeError`. ```python from unpythonic import call_ec @@ -2774,7 +3482,7 @@ def result(ec): assert result == 42 ``` -The ec doesn't have to be called from the lexical scope of the call_ec'd function, as long as the call occurs within the dynamic extent of the ``call_ec``. It's essentially a *return from me* for the original function: +The ec does not have to be called from the lexical scope of the `call_ec`'d function, as long as the call occurs *within the dynamic extent* of the `call_ec`. It's essentially a *return from me* for the original function: ```python def f(ec): @@ -2788,7 +3496,7 @@ def result(ec): assert result == 42 ``` -This also works with lambdas, by using ``call_ec()`` directly. No need for a trampoline: +This also works with lambdas, by using `call_ec()` directly. No need for a trampoline: ```python result = call_ec(lambda ec: @@ -2798,11 +3506,11 @@ result = call_ec(lambda ec: assert result == 42 ``` -Normally ``begin()`` would return the last value, but the ec overrides that; it is effectively a ``return`` for multi-expression lambdas! +Normally `begin()` would return the last value, but the ec overrides that; it is effectively a `return` for multi-expression lambdas! But wait, doesn't Python evaluate all the arguments of `begin(...)` before the `begin` itself has a chance to run? Why doesn't the example print also *never reached*? This is because escapes are implemented using exceptions. Evaluating the ec call raises an exception, preventing any further elements from being evaluated. -This usage is valid with named functions, too - ``call_ec`` is not only a decorator: +This usage is valid with named functions, too, so strictly speaking, `call_ec` is not only a decorator: ```python def f(ec): @@ -2816,31 +3524,35 @@ result = call_ec(f) assert result == 42 ``` +*If you use the macro API of `unpythonic`, be aware that the macros cannot analyze this last example properly, because there is no lexical clue that `f` will actually be called using `call_ec`. To be safe in situations like this, name your ec parameter `ec`; then it will be recognized as an escape continuation. Also `brk` (defined by `@looped_over`) and `throw` are recognized by name.* + +**CAUTION**: The `call_ec` mechanism builds on `@catch` and `throw`, so the caution about catch-all `except:` statements applies here, too. + -### ``forall``: nondeterministic evaluation +### `forall`: nondeterministic evaluation We provide a simple variant of nondeterministic evaluation. This is essentially a toy that has no more power than list comprehensions or nested for loops. See also the easy-to-use [macro](macros.md) version with natural syntax and a clean implementation. -An important feature of McCarthy's [`amb` operator](https://rosettacode.org/wiki/Amb) is its nonlocality - being able to jump back to a choice point, even after the dynamic extent of the function where that choice point resides. If that sounds a lot like ``call/cc``, that's because that's how ``amb`` is usually implemented. See examples [in Ruby](http://www.randomhacks.net/2005/10/11/amb-operator/) and [in Racket](http://www.cs.toronto.edu/~david/courses/csc324_w15/extra/choice.html). +An important feature of McCarthy's [`amb` operator](https://rosettacode.org/wiki/Amb) is its nonlocality - being able to jump back to a choice point, even after the dynamic extent of the function where that choice point resides. If that sounds a lot like `call/cc`, that is because that's how `amb` is usually implemented. See examples [in Ruby](http://www.randomhacks.net/2005/10/11/amb-operator/) and [in Racket](http://www.cs.toronto.edu/~david/courses/csc324_w15/extra/choice.html). -Python can't do that, short of transforming the whole program into [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style), while applying TCO everywhere to prevent stack overflow. **If that's what you want**, see ``continuations`` in [the macros](macros.md). +Python cannot do that, short of transforming the whole program into [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style), while applying TCO everywhere to prevent stack overflow. **If that is what you want**, see `continuations` in [the macros](macros.md). -This ``forall`` is essentially a tuple comprehension that: +This `forall` is essentially a tuple comprehension that: - Can have multiple body expressions (side effects also welcome!), by simply listing them in sequence. - Allows filters to be placed at any level of the nested looping. - Presents the source code in the same order as it actually runs. -The ``unpythonic.amb`` module defines four operators: +The module `unpythonic.amb` defines four operators: - - ``forall`` is the control structure, which marks a section with nondeterministic evaluation. - - ``choice`` binds a name: ``choice(x=range(3))`` essentially means ``for e.x in range(3):``. - - ``insist`` is a filter, which allows the remaining lines to run if the condition evaluates to truthy. - - ``deny`` is ``insist not``; it allows the remaining lines to run if the condition evaluates to falsey. + - `forall` is the control structure, which marks a section that uses nondeterministic evaluation. + - `choice` binds a name: `choice(x=range(3))` essentially means `for e.x in range(3):`. + - `insist` is a filter, which allows the remaining lines to run if the condition evaluates to truthy. + - `deny` is `insist not`; it allows the remaining lines to run if the condition evaluates to falsey. -Choice variables live in the environment, which is accessed via a ``lambda e: ...``, just like in ``letrec``. Lexical scoping is emulated. In the environment, each line only sees variables defined above it; trying to access a variable defined later raises ``AttributeError``. +Choice variables live in the environment, which is accessed via a `lambda e: ...`, just like in `letrec`. Lexical scoping is emulated. In the environment, each line only sees variables defined above it; trying to access a variable defined later raises `AttributeError`. -The last line in a ``forall`` describes one item of the output. The output items are collected into a tuple, which becomes the return value of the ``forall`` expression. +The last line in a `forall` describes one item of the output. The output items are collected into a tuple, which becomes the return value of the `forall` expression. ```python out = forall(choice(y=range(3)), @@ -2872,36 +3584,181 @@ assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), Beware: ```python -out = forall(range(2), # do the rest twice! +out = forall(range(2), # evaluate remaining items twice! choice(x=range(1, 4)), lambda e: e.x) assert out == (1, 2, 3, 1, 2, 3) ``` -The initial ``range(2)`` causes the remaining lines to run twice - because it yields two output values - regardless of whether we bind the result to a variable or not. In effect, each line, if it returns more than one output, introduces a new nested loop at that point. +The initial `range(2)` causes the remaining items to run twice - because it yields two output values - regardless of whether we bind the result to a variable or not. In effect, each line, if it returns more than one output, introduces a new nested loop at that point. -For more, see the docstring of ``forall``. +For more, see the docstring of `forall`. #### For haskellers The implementation is based on the List monad, and a bastardized variant of do-notation. Quick vocabulary: - - ``forall(...)`` = ``do ...`` (for a List monad) - - ``choice(x=foo)`` = ``x <- foo``, where ``foo`` is an iterable - - ``insist x`` = ``guard x`` - - ``deny x`` = ``guard (not x)`` - - Last line = implicit ``return ...`` + - `forall(...)` = `do ...` (for a List monad) + - `choice(x=foo)` = `x <- foo`, where `foo` is an iterable + - `insist x` = `guard x` + - `deny x` = `guard (not x)` + - Last line = implicit `return ...` -### ``handlers``, ``restarts``: conditions and restarts +### Monads -**Added in v0.14.2**. +**Added in v2.1.0.** -**Changed in v0.14.3**. *Conditions can now inherit from `BaseException`, not only from `Exception.` `with handlers` catches also derived types, e.g. a handler for `Exception` now catches a signaled `ValueError`.* +A small zoo of classical monads, living in the `unpythonic.monads` subpackage. For do-notation syntax over any of these, see the [`monadic_do` macro](macros.md#monadic_do-do-notation-for-any-monad). -*When an unhandled `error` or `cerror` occurs, the original unhandled error is now available in the `__cause__` attribute of the `ControlError` exception that is raised in this situation.* +The subpackage is **not** re-exported at the top level — import directly as `from unpythonic.monads import Maybe, Left, Right, ...`. Same style as `from unpythonic.env import env`. -*Signaling a class, as in `signal(SomeExceptionClass)`, now implicitly creates an instance with no arguments, just like the `raise` statement does. On Python 3.7+, `signal` now automatically equips the condition instance with a traceback, just like the `raise` statement does for an exception.* +Bind uses `>>` (Python's `>>=` is `__irshift__`, in-place, doesn't chain). Sequence uses `.then(other_monad)`. The class itself is the `unit` constructor, so `Identity(x)`, `Maybe(x)`, `List(x)` are the monadic unit forms. + +#### The base classes + +- `Monad` — the base class all monads inherit from. Provides default `__rshift__` (bind, via `fmap . join`) and `then` (sequence). Abstract methods: `__init__` (unit), `fmap`, `join`. + +- `LiftableMonad(Monad)` — adds `lift`, i.e. `(a -> b) -> (a -> M b)`. Inherited by monads where `lift` is well-defined (`Identity`, `Maybe`, `Either`, `List`, `Writer`). `State` and `Reader` inherit from `Monad` directly. + +Modeled on `unpythonic.slicing.Sliced`: duck-first, `@abstractmethod` as documentation marker rather than strict enforcement. + +#### `Identity` + +Pedagogical no-op — ordinary function composition dressed as a monad. Useful as a reference when building or debugging other monads. + +```python +from unpythonic.monads import Identity + +result = Identity(2) >> (lambda x: Identity(x + 1)) +assert result == Identity(3) +``` + +#### `Maybe` + +Short-circuiting on "nothing." The unpythonic convention uses `nil` (from `unpythonic.llist`) as the "nothing" sentinel, avoiding proliferation of null singletons. `Maybe(x)` for `x is not nil` is "Just x"; `Maybe(nil)` is "Nothing." + +```python +from unpythonic.llist import nil +from unpythonic.monads import Maybe + +# happy path +assert Maybe(10) >> (lambda x: Maybe(x + 1)) == Maybe(11) + +# short-circuit: Nothing propagates; the lambda is never called +assert Maybe(nil) >> (lambda x: Maybe(x + 1)) == Maybe(nil) +``` + +Trade-off: This encoding cannot wrap ``nil`` itself as a present value (Haskell: `Just nil`). In all other cases this yields better UX vs. demanding a ``Some(...)`` wrapper per value. + +#### `Either`, `Left`, `Right` + +Maybe's richer sibling — carries an error value down the short-circuit path. `Right` is success, `Left` is failure (by Haskell convention). `Either` itself is abstract; use `Left` and `Right` directly. + +```python +from unpythonic.monads import Left, Right + +assert Right(10) >> (lambda x: Right(x + 1)) == Right(11) +assert Left("boom") >> (lambda x: Right(x + 1)) == Left("boom") +``` + +#### `List` + +Multivalued / nondeterministic computation. Binding through a `List` is `flatMap`: each value in the list becomes a sub-computation that produces its own list of results, and all sub-results are concatenated. + +Replaces `MonadicList` from `unpythonic.amb`, which is kept as a deprecated alias — see `unpythonic.amb.MonadicList` for the back-compat note. + +Varargs constructor — the class itself is the monadic unit. `List(1, 2, 3)` for literals; `List.from_iterable(iter)` to build from an existing iterable. + +```python +from unpythonic.monads import List + +# bind = flatMap +assert (List(1, 2, 3) >> (lambda x: List(x, x * 10))) == List(1, 10, 2, 20, 3, 30) + +# Pythagorean triples — the canonical List-monad example +def r(lo, hi): + return List.from_iterable(range(lo, hi)) +pt = r(1, 21) >> (lambda z: + r(1, z + 1) >> (lambda x: + r(x, z + 1) >> (lambda y: + List.guard(x*x + y*y == z*z).then( + List((x, y, z)))))) +assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20)) +``` + +Full `Sequence` ABC (`__len__`, `__getitem__`, `__contains__`, etc.); ABC registration so `isinstance(List(...), Sequence)` is `True`. Concatenation via `+`. + +#### `Writer` + +Pure-functional audit log. `Writer(value, log)` wraps a pair; binding threads the value through while concatenating logs. The log can be any type supporting `+` (default: empty `""`). + +```python +from unpythonic.monads import Writer + +result = (Writer(10) + >> (lambda x: Writer(x + 1, "added 1; ")) + >> (lambda y: Writer(y * 2, "doubled; "))) +assert result.data == (22, "added 1; doubled; ") +``` + +`Writer.tell(msg)` interleaves a log entry without touching the value — useful as `computation.then(Writer.tell("done; "))`. + +#### `State` + +Threading a state value through a pure computation. Wraps a function `s -> (a, s)`: takes an input state, produces a data value, returns a new state. The state only becomes bound when the composed chain is `.run(s0)`; until then, it's a recipe. + +```python +from unpythonic.monads import State + +bump = State(lambda s: (s, s + 1)) +chain = (bump + >> (lambda a: bump + >> (lambda b: bump + >> (lambda c: State.unit((a, b, c)))))) +values, final = chain.run(10) +assert values == (10, 11, 12) and final == 13 +``` + +Helper classmethods: `State.unit`, `State.get`, `State.put`, `State.modify`, `State.gets`. Accessors on a `State` instance: `.run(s)`, `.eval(s)` (data only), `.exec(s)` (state only). + +Does not inherit from `LiftableMonad` — `lift` doesn't have a canonical shape for State. + +#### `Reader` + +Read-only shared environment. Wraps a function `e -> a`. The environment threads through the chain; each step can `.ask()` for it. + +```python +from unpythonic.monads import Reader + +config = {"multiplier": 3, "offset": 10} +chain = (Reader.asks(lambda e: e["multiplier"]) + >> (lambda m: Reader.asks(lambda e: e["offset"]) + >> (lambda o: Reader.unit(m * 5 + o)))) +assert chain.run(config) == 25 +``` + +Helper classmethods: `Reader.unit`, `Reader.ask`, `Reader.asks`. Instance methods: `.run(env)`, `.local(f)` (run in an `f`-modified environment). + +Does not inherit from `LiftableMonad` for the same reason as `State`. + +#### `liftm`, `liftm2`, `liftm3` + +Lift regular 1-, 2-, 3-argument functions into monadic ones. Distinct from `LiftableMonad.lift`: `lift: (a -> b) -> (a -> M b)` expects the caller to bind; `liftm: (a -> r) -> (M a -> M r)` binds internally. + +```python +from unpythonic.monads import Maybe, liftm2 + +add = lambda x, y: x + y +add_m = liftm2(Maybe, add) +assert add_m(Maybe(3), Maybe(4)) == Maybe(7) +``` + +The `M` parameter is curry-friendly (changes least often) — `functools.partial(liftm2, Maybe)` gives you a Maybe-specific lifter. + + +### `handlers`, `restarts`: conditions and restarts **Changed in v0.15.0.** *Functions `resignal_in` and `resignal` added; these perform the same job for conditions as `reraise_in` and `reraise` do for exceptions, that is, they allow you to map library exception types to semantically appropriate application exception types, with minimum boilerplate.* @@ -2909,15 +3766,23 @@ The implementation is based on the List monad, and a bastardized variant of do-n *The error-handling protocol that was used to send a signal is now available for inspection in the `__protocol__` attribute of the condition instance. It is the callable that sent the signal, such as `signal`, `error`, `cerror` or `warn`. It is the responsibility of each error-handling protocol (except the fundamental `signal` itself) to pass its own function to `signal` as the `protocol` argument; if not given, `protocol` defaults to `signal`. The protocol information is used by the `resignal` mechanism.* +**Changed in v0.14.3**. *Conditions can now inherit from `BaseException`, not only from `Exception.` Just like the `except` statement, `with handlers` catches also derived types, e.g. a handler for `Exception` now catches a signaled `ValueError`.* + +*When an unhandled `error` or `cerror` occurs, the original unhandled error is now available in the `__cause__` attribute of the `ControlError` exception that is raised in this situation.* + +*Signaling a class, as in `signal(SomeExceptionClass)`, now implicitly creates an instance with no arguments, just like the `raise` statement does. On Python 3.7+, `signal` now automatically equips the condition instance with a traceback, just like the `raise` statement does for an exception.* + +**Added in v0.14.2**. + One of the killer features of Common Lisp are *conditions*, which are essentially **resumable exceptions**. -Following Peter Seibel ([Practical Common Lisp, chapter 19](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html)), we define *errors* as the consequences of [Murphy's Law](https://en.wikipedia.org/wiki/Murphy%27s_law), i.e. situations where circumstances cause interaction between the program and the outside world to fail. An error is no bug, but failing to handle an error certainly is. +Following Peter Seibel ([Practical Common Lisp, chapter 19](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html)), we define *errors* as the consequences of [Murphy's Law](https://en.wikipedia.org/wiki/Murphy%27s_law), i.e. situations where circumstances cause interaction between the program and the outside world to fail. An error is not a bug, but failing to handle an error certainly is. An exception system splits error-recovery responsibilities into two parts. In Python terms, we speak of *raising* and then *handling* an exception. In comparison, a condition system splits error-recovery responsibilities into **three parts**: *signaling*, *handling* and *restarting*. -The result is improved modularity. Consider [separation of mechanism and policy](https://en.wikipedia.org/wiki/Separation_of_mechanism_and_policy). We place the actual error-recovery code (the mechanism) in *restarts*, at the inner level (of the call stack) - which has access to all the low-level technical details that are needed to actually perform the recovery. We can provide *several different* canned recovery strategies, which implement any appropriate ways to recover, in the context of each low- or middle-level function. We defer the decision of which one to use (the policy), *to an outer level*. The outer level knows about the big picture - *why* the inner levels are running in this particular case, i.e. what we are trying to accomplish and how. Hence, it is in the ideal position to choose which error-recovery strategy should be used *in its high-level context*. +The result is improved modularity and better [separation of mechanism and policy](https://en.wikipedia.org/wiki/Separation_of_mechanism_and_policy). The actual error-recovery code (the **mechanism**) lives in *restarts*, at the inner level (of the call stack) - which has access to all the low-level technical details that are needed to actually perform an error recovery. It is possible to provide *several different* canned recovery strategies, which implement any appropriate ways to recover, in the context of each low- or middle-level function. The decision of which strategy to use (the **policy**) in any particular situation is deferred *to an outer level* (of the call stack). The outer level knows the big picture - *why* the inner levels are running in this particular case, i.e., what we are trying to accomplish and how. Hence, it is the appropriate place to choose which error-recovery strategy should be used *in its high-level context*. -Practical Common Lisp explains conditions in the context of a log file parser. In contrast, let us explain them with some Theoretical Python: +Seibel's *Practical Common Lisp* explains conditions in the context of a log file parser. In contrast, let us explain them with some *Theoretical Python*: ```python from unpythonic import restarts, handlers, signal, invoke, unbox @@ -2960,19 +3825,21 @@ high3() #### Fundamental signaling protocol -Generally a condition system operates as follows. A *signal* is sent (outward on the call stack) from the actual location where the error was detected. A *handler* at any outer level may then respond to it, and execution resumes from the *restart* that is *invoked* by the handler. +Generally a conditions-and-restarts system operates as follows. A *signal* is sent, outward on the call stack, from the actual location where an error was detected. A *handler* at any outer level (of the call stack) may then respond to it, and execution resumes from the *restart* that is *invoked* by the handler. -The sequence of catching a signal and invoking a restart is termed *handling* the signal. Handlers are searched in order from innermost to outermost on the call stack. (Strictly speaking, the handlers live on a separate stack; we consider those handlers whose dynamic extent the point of execution is in, at the point of time when the signal is sent.) +The sequence of catching a signal and invoking a restart is termed *handling* the signal. Handlers are searched in order from innermost to outermost on the call stack. Strictly speaking, though, the handlers live on a separate stack; we consider those handler bindings whose dynamic extent the point of execution is in, at the point of time when the signal is sent. In general, it is allowed for a handler to fall through (return normally); then the next outer handler for the same signal type gets control. This allows the programmer to chain handlers to obtain their side effects, such as logging. This is referred to as *canceling*, since as a result, the signal remains unhandled. -Viewed with respect to the call stack, the restarts live between the (outer) level of the handler, and the (inner) level where the signal was sent from. The main difference to the exception model is that unlike raising an exception, **sending a signal does not unwind the call stack**. Although the handlers live further out on the call stack, the stack does not unwind that far. The handlers are just consulted for what to do. The call stack unwinds only when a restart is being invoked. Then, only the part of the call stack between the location that sent the signal, and the invoked restart, is unwound. +Viewed with respect to the call stack, the restarts live between the (outer) level of the handler, and the (inner) level where the signal was sent from. The main difference to the exception model is that unlike raising an exception, **sending a signal does not unwind the call stack**. (Let that sink in for a moment.) -Restarts, despite the name, are a mildly behaved, structured control construct. The block of code that encountered the error is actually not arbitrarily resumed; instead, the restart code runs instead of the rest of the block, and the return value of the restart replaces the normal return value. (But see `cerror`.) +Although the handlers live further out on the call stack, the stack does not unwind that far. The handlers are just consulted for what to do. **The call stack unwinds only when a restart is invoked.** Then, only the part of the call stack between the location that sent the signal, and the invoked restart, is unwound. + +Restarts, despite the name, are a mildly behaved, structured control construct. The block of code that encountered the error is actually not arbitrarily resumed; instead, the code of the invoked restart runs instead of the rest of the block, and the return value of the restart replaces the normal return value. (But see `cerror`.) #### API summary -Restarts are set up using the `with restarts` context manager (Common Lisp: `RESTART-CASE`). Restarts are defined by giving named arguments to the `restarts` form; the argument name sets the restart name. The restart name is distinct from the name (if any) of the function that is used as the restart. A restart can only be invoked from within the dynamic extent of its `with restarts` (the same rule is effect also in Common Lisp). A restart may take any args and kwargs; any that it expects must be provided when it is invoked. +Restarts are set up using the `with restarts` context manager (Common Lisp: `RESTART-CASE`). Restarts are defined by passing named arguments to the `restarts` form; the argument name sets the *restart name*. The restart name is distinct from the name (if any) of the function that is used as the restart. A restart can only be invoked from within the dynamic extent of its `with restarts` (the same rule is effect also in Common Lisp). A restart may take any args and kwargs; any that it expects must be provided when it is invoked. *Note difference to the API of [python-cl-conditions](https://github.com/svetlyak40wt/python-cl-conditions/), which requires functions used as restarts to be named, and uses the function name as the restart name.* @@ -2982,21 +3849,21 @@ Signals are sent using `signal` (Common Lisp: `SIGNAL`). Any exception or warnin Handlers are established using the `with handlers` context manager (Common Lisp: `HANDLER-BIND`). Handlers are bound to exception types, or tuples of types, just like regular exception handlers in Python. The `handlers` form takes as its arguments any number of `(exc_spec, handler)` pairs. Here `exc_spec` specifies the exception types to catch (when sent via `signal`), and `handler` is a callable. When catching a signal, in case of multiple matches in the same `with handlers` form, the handler that appears earlier in the argument list wins. -A handler catches signals of the types it is bound to. The code in the handler may invoke a restart by calling `invoke` (Common Lisp: `INVOKE-RESTART`), with the desired restart name as a string. In case of duplicate names, the most recently established restart (that is still in scope) with the given name wins. Any extra args and kwargs are passed through to the restart. The `invoke` function always transfers control, never returns normally. +A handler catches signals of the types it is bound to, and their subtypes. The code in the handler may invoke a restart by calling `invoke` (Common Lisp: `INVOKE-RESTART`), with the desired restart name as a string. In case of duplicate names, the most recently established restart (that is still in scope) with the given name wins. Any extra args and kwargs are passed through to the restart. The `invoke` function always transfers control, it never returns normally. -A handler **may** take one optional positional argument, the exception instance being signaled. Roughly, API-wise signal handlers are similar to exception handlers (`except` clauses). A handler that accepts an argument is like an `except ... as ...`, whereas one that does not is like `except ...`. **The main difference** to an exception handler is that a **signal handler should not try to recover from the error itself**; instead, **it should just choose** which strategy the lower-level code should use to recover from the error. Usually, the only thing a signal handler needs to do, is to invoke a particular restart. +A handler **may** take one optional positional argument, the exception instance being signaled. Roughly, API-wise signal handlers are similar to exception handlers (`except` clauses). A handler that accepts an argument is like an `except ... as ...`, whereas one that does not is like `except ...`. **The main difference** to an exception handler is that a **signal handler should not try to recover from the error by itself**; instead, **it should just choose** which strategy the lower-level code should use to recover from the error. Usually, the only thing a signal handler needs to do is to invoke a particular restart. To create a simple handler that does not take an argument, and just invokes a pre-specified restart, see `invoker`. If you instead want to create a function that you can call from a handler, in order to invoke a particular restart immediately (so to define a shorthand notation similar to `use_value`), use `functools.partial(invoke, "my_restart_name")`. -Following Common Lisp terminology, *a named function that invokes a specific restart* - whether it is intended to act as a handler or to be called from one - is termed a *restart function*. (This is somewhat confusing, as a *restart function* is not a function that implements a restart, but a function that *invokes* a specific one.) The `use_value` function mentioned above is an example. +Following Common Lisp terminology, *a named function that invokes a specific restart* - whether it is intended to act as a handler or to be called from one - is termed a *restart function*. This is somewhat confusing, as a *restart function* is not a function that implements a restart, but a function that *invokes* a specific one. The `use_value` function mentioned above is an example. -For a detailed API reference, see the module ``unpythonic.conditions``. +For a detailed API reference, see the module `unpythonic.conditions`. #### High-level signaling protocols We actually provide four signaling protocols: `signal` (i.e. the fundamental protocol), and three that build additional behavior on top of it: `error`, `cerror` and `warn`. Each of the three is modeled after its Common Lisp equivalent. -If no handler *handles* the signal, the `signal(...)` protocol just returns normally. In effect, with respect to control flow, unhandled signals are ignored by this protocol. (But any side effects of handlers that caught the signal but did not invoke a restart, still take place.) +If no handler *handles* the signal, the `signal(...)` protocol just returns normally. In effect, with respect to control flow, unhandled signals are ignored by this protocol. However, any side effects of handlers that caught the signal but did not invoke a restart, still take place. The `error(...)` protocol first delegates to `signal`, and if the signal was not handled by any handler, then **raises** `ControlError` as a regular exception. (Note the Common Lisp `ERROR` function would at this point drop you into the debugger.) The implementation of `error` itself is the only place in the condition system that *raises* an exception for the end user; everything else (including any error situations) uses the signaling mechanism. @@ -3006,17 +3873,19 @@ Finally, there is the `warn(...)` protocol, which is just a lispy interface to P The combination of `warn` and `muffle` (as well as `cerror` when a handler invokes its `proceed` restart) behaves somewhat like [`contextlib.suppress`](https://docs.python.org/3/library/contextlib.html#contextlib.suppress), except that execution continues normally from the next statement in the caller of `warn` (respectively `cerror`) instead of unwinding to the handler. -If the standard protocols don't cover what you need, you can also build your own high-level protocols on top of `signal`. See the source code of `error`, `cerror` and `warn` for examples (it's just a few lines in each case). +If the standard protocols do not cover what you need, you can also build your own high-level protocols on top of `signal`. See the source code of `error`, `cerror` and `warn` for examples (it's just a few lines in each case). ##### Notes The name `cerror` stands for *correctable error*, see e.g. [CERROR in the CL HyperSpec](http://clhs.lisp.se/Body/f_cerror.htm). What we call `proceed`, Common Lisp calls `CONTINUE`; the name is different because in Python the function naming convention is lowercase, and `continue` is a reserved word. -If you really want to emulate `ON ERROR RESUME NEXT`, just use `Exception` as the condition type for your handler, and all `cerror` calls within the block will return normally, provided that no other handler handles those conditions first. +If you really want to emulate `ON ERROR RESUME NEXT`, just use `Exception` as the condition type for your handler, and all `cerror` calls within the block will return normally, provided that no other handler (that appears in an inner position on the call stack) handles those conditions first. #### Conditions vs. exceptions -Using the condition system essentially requires eschewing exceptions, using only restarts and handlers instead. A regular `raise` will fly past a `with handlers` form uncaught. The form just maintains a stack of functions; it does not establish an *exception* handler. Similarly, a `try`/`except` cannot catch a signal, because no exception is raised yet at handler lookup time. Delaying the stack unwind, to achieve the three-way split of responsibilities, is the whole point of the condition system. Which of the two systems to use is a design decision that must be made consistently on a per-project basis. +Using the condition system essentially requires eschewing exceptions, using only restarts and handlers instead. A regular `raise` will fly past a `with handlers` form uncaught. The form just maintains a stack of functions; it does not establish an *exception* handler. Similarly, a `try`/`except` cannot catch a signal, because no exception is raised yet at handler lookup time. Delaying the stack unwind, to achieve the three-way split of responsibilities, is the whole point of the condition system. + +Which of the two systems to use is a design decision that must be made consistently on a per-project basis. Even better would be to make it globally on a per-language basis. Python's standard library, as well as all existing libraries, use exceptions instead of conditions, so to obtain a truly seamless conditions-and-restarts user experience, one would have to wrap (or rewrite) at least all of the standard library, plus any other libraries a project needs, to be protected from sudden, unexpected unwinds of the call stack. (The nature of both conditions and exceptions is that, in principle, they may be triggered anywhere.) Be aware that error-recovery code in a Lisp-style signal handler is of a very different nature compared to error-recovery code in an exception handler. A signal handler usually only chooses a restart and invokes it; as was explained above, the code that actually performs the error recovery (i.e. the *restart*) lives further in on the call stack, and still has available (in its local variables) the state that is needed to perform the recovery. An exception handler, on the other hand, must respond by directly performing error recovery right where it is, without any help from inner levels - because the stack has already unwound when the exception handler gets control. @@ -3030,54 +3899,59 @@ If this `ControlError` signal is not handled, a `ControlError` will then be **ra #### Historical note -Conditions are one of the killer features of Common Lisp, so if you're new to conditions, [Peter Seibel: Practical Common Lisp, chapter 19](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html) is a good place to learn about them. There's also a relevant [discussion on Lambda the Ultimate](http://lambda-the-ultimate.org/node/1544). +Conditions are one of the killer features of Common Lisp, so if you are new to conditions, [Peter Seibel: Practical Common Lisp, chapter 19](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html) is a good place to learn about them. There is also a relevant [discussion on Lambda the Ultimate](http://lambda-the-ultimate.org/node/1544). For Python, conditions were first implemented in [python-cl-conditions](https://github.com/svetlyak40wt/python-cl-conditions/) by Alexander Artemenko (2016). -What we provide here is essentially a rewrite, based on studying that implementation. The main reasons for the rewrite are to give the condition system an API consistent with the style of `unpythonic`, to drop any and all historical baggage without needing to consider backward compatibility, and to allow interaction with (and customization taking into account) the other parts of `unpythonic`. If you specifically need a condition system, not a kitchen-sink language extension, then by all means go for `python-cl-conditions`! +What we provide here is essentially a rewrite, based on studying that implementation. The main reasons for the rewrite are to give the condition system an API consistent with the style of `unpythonic`, to drop any and all historical baggage without needing to consider backward compatibility, and to allow interaction with (and customization taking into account) the other parts of `unpythonic`. -The core idea can be expressed in fewer than 100 lines of Python; ours is (as of v0.14.2) 151 lines, not counting docstrings, comments, or blank lines. The main reason our module is over 700 lines are the docstrings. +The core idea can be expressed in fewer than 100 lines of Python; ours is (as of v0.15.0) 199 lines, not counting docstrings, comments, or blank lines. The main reason our module is over 900 lines are the docstrings. -### ``generic``, ``typed``, ``isoftype``: multiple dispatch +### `generic`, `typed`, `isoftype`: multiple dispatch -**Added in v0.14.2**. - -**Changed in v0.14.3**. *The multiple-dispatch decorator `@generic` no longer takes a master definition. Multimethods are registered directly with `@generic`; the first method definition implicitly creates the generic function.* - -**Changed in v0.14.3**. *The `@generic` and `@typed` decorators can now decorate also instance methods, class methods and static methods (beside regular functions, as previously in 0.14.2).* +**Changed in v2.0.0.** *`isoftype` now supports many more `typing` features: `NoReturn`, `Never`, `Literal`, `Type`, `ClassVar`, `Final`, `DefaultDict`, `OrderedDict`, `Counter`, `ChainMap`, `IO`/`TextIO`/`BinaryIO`, `Pattern`/`Match`, `ContextManager`/`AsyncContextManager`, `Awaitable`/`Coroutine`, `AsyncIterable`/`AsyncIterator`, `Generator`/`AsyncGenerator`, `TypedDict`, `@runtime_checkable` `Protocol`, and parametric forms of abstract ABCs (`Iterable[T]`, `Collection[T]`, `Reversible[T]` with best-effort element checking; `Iterator[T]`, `Container[T]`). See the [`isoftype` section](#isoftype-the-big-sister-of-isinstance) for the full list.* **Changed in v0.15.0**. *The `dispatch` and `typecheck` modules providing this functionality are now considered stable (no longer experimental). Starting with this release, they receive the same semantic-versioning guarantees as the rest of `unpythonic`.* -*Added the `@augment` parametric decorator that can register a new multimethod on an existing generic function originally defined in another lexical scope. Be careful of [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy) when you use it.* +*Added the `@augment` parametric decorator that can register a new multimethod on an existing generic function originally defined in another lexical scope.* -*Added the function `methods`, which displays a list of multimethods of a generic function.* +*Added the function `methods`, which displays a list of multimethods of a generic function. This is especially useful in the REPL.* *Docstrings of the multimethods are now automatically concatenated to make up the docstring of the generic function, so you can document each multimethod separately.* -*`curry` now supports `@generic`. In the case where the **number** of positional arguments supplied so far matches at least one multimethod, but there is no match for the given combination of argument **types**, `curry` waits for more arguments (returning the curried function).* +*`curry` now supports `@generic`. In the case where the **number** of positional arguments supplied so far matches at least one multimethod, but there is no match for the given combination of argument **types**, `curry` waits for more arguments (returning the curried function). See the manual section on `curry` for details.* *It is now possible to dispatch also on a homogeneous type of contents collected by a `**kwargs` parameter. In the type signature, use `typing.Dict[str, mytype]`. Note that in this use, the key type is always `str`.* -The ``generic`` decorator allows creating multiple-dispatch generic functions with type annotation syntax. We also provide some friendly utilities: ``augment`` adds a new multimethod to an existing generic function, ``typed`` creates a single-method generic with the same syntax (i.e. provides a compact notation for writing dynamic type checking code), and ``isoftype`` (which powers the first three) is the big sister of ``isinstance``, with support for many (but unfortunately not all) features of the ``typing`` standard library module. +**Changed in v0.14.3**. *The multiple-dispatch decorator `@generic` no longer takes a master definition. Multimethods are registered directly with `@generic`; the first multimethod definition implicitly creates the generic function.* + +*The `@generic` and `@typed` decorators can now decorate also instance methods, class methods and static methods (beside regular functions, as previously in 0.14.2).* -For what kind of things can be done with this, see particularly the [*holy traits*](https://ahsmart.com/pub/holy-traits-design-patterns-and-best-practice-book/) example in [`unpythonic.tests.test_dispatch`](../unpythonic/tests/test_dispatch.py). +**Added in v0.14.2**. + +The `generic` decorator allows creating [multiple-dispatch](https://en.wikipedia.org/wiki/Multiple_dispatch) generic functions with type annotation syntax. We also provide some friendly utilities: `augment` adds a new multimethod to an existing generic function, `typed` creates a single-method generic with the same syntax (i.e. provides a compact notation for writing dynamic type-checking code), and `isoftype` (which powers the first three) is the big sister of `isinstance`, with support for many (but not all) features of the `typing` standard library module. -**NOTE**: This was inspired by the [multi-methods of CLOS](http://www.gigamonkeys.com/book/object-reorientation-generic-functions.html) (the Common Lisp Object System), and the [generic functions of Julia](https://docs.julialang.org/en/v1/manual/methods/). +This is a purely run-time implementation, so it does **not** give performance benefits, but it can make code more readable, and makes it modular to add support for new input types (or different call signatures) to an existing function later. -In `unpythonic`, the terminology is as follows: +The terminology is: - The function that supports multiple call signatures is a *generic function*. - Each of its individual implementations is a *multimethod*. The term *multimethod* distinguishes them from the OOP sense of *method*, already established in Python, as well as reminds that multiple arguments participate in dispatching. -#### ``generic``: multiple dispatch with type annotation syntax +**CAUTION**: Code using the `with lazify` macro cannot usefully use `@generic` or `@typed`, because all arguments of each function call will be wrapped in a promise (`unpythonic.Lazy`) that carries no type information on its contents. + -The ``generic`` decorator essentially allows replacing the `if`/`elif` dynamic type checking boilerplate of polymorphic functions with type annotations on the function parameters, with support for features from the `typing` stdlib module. This not only kills boilerplate, but makes the dispatch extensible, since the dispatcher lives outside the original function definition. There is no need to monkey-patch the original to add a new case. +#### `generic`: multiple dispatch with type annotation syntax + +The `generic` decorator essentially allows replacing the `if`/`elif` dynamic type checking boilerplate of polymorphic functions with type annotations on the function parameters, with support for features from the `typing` stdlib module. This not only kills boilerplate, but makes the dispatch extensible, since the dispatcher is separate from the actual function definition, and has a mechanism to register new multimethods. If several multimethods of the same generic function match the arguments given, the most recently registered multimethod wins. +To see what multimethods are registered on a given generic function `f`, call `methods(f)`. It will print a human-readable description to stdout. + **CAUTION**: The winning multimethod is chosen differently from Julia, where the most specific multimethod wins. Doing that requires a more careful type analysis than what we have here. The details are best explained by example: @@ -3151,38 +4025,177 @@ assert kittify(x=1, y=2) == "int" assert kittify(x=1.0, y=2.0) == "float" ``` -See [the unit tests](../unpythonic/tests/test_dispatch.py) for more. For which features of the ``typing`` stdlib module are supported, see ``isoftype`` below. +See [the unit tests](../unpythonic/tests/test_dispatch.py) for more. For which features of the `typing` stdlib module are supported, see [`isoftype`](#isoftype-the-big-sister-of-isinstance) below. + + +##### `@generic` and OOP -##### ``@generic`` and OOP +Beginning with v0.14.3, `@generic` and `@typed` can decorate instance methods, class methods and static methods (beside regular functions as in v0.14.2). -As of version 0.14.3, `@generic` and `@typed` can decorate instance methods, class methods and static methods (beside regular functions as in 0.14.2). +When using both `@generic` or `@typed` and OOP, important things to know are: -When using both `@generic` or `@typed` and OOP: + - In case of `@generic`, consider first if that is what you really want. + - The method access syntax already hides a single-dispatch mechanism behind the dot-access syntax: the syntax `x.op(...)` picks the definition of `op` based on the type of `x`. This behaves exactly like a single-dispatch function where the first argument is `x`, i.e., we could as well write `op(x, ...)`. + - So the question to ask is, is the use case best served by two overlapping dispatch mechanisms? + - If not, what are the alternative strategies? Would it be better, for example, to represent the operations as top-level `@generic` *functions*, and perform the dispatch there, dispatching to OOP methods as appropriate? + - `@typed` is fine to use with OOP, because semantically, it is not really a dispatch mechanism, but a run-time type-checking mechanism, even though it is implemented in terms of the multiple-dispatch machinery. - **`self` and `cls` parameters**. - The `self` and `cls` parameters do not participate in dispatching, and need no type annotation. - - Beside appearing as the first positional-or-keyword parameter, the self-like parameter **must be named** one of `self`, `this`, `cls`, or `klass` to be detected by the ignore mechanism. This limitation is due to implementation reasons; while a class body is being evaluated, the context needed to distinguish a method (OOP sense) from a regular function is not yet present. + - Beside appearing as the first positional-or-keyword parameter, the self-like parameter **must be named** one of `self`, `this`, `cls`, or `klass` to be detected by the ignore mechanism. + + This limitation is due to implementation reasons; while a class body is being evaluated, the context needed to distinguish a method (in the OOP sense) from a regular function is not yet present. In Python, OOP method binding is performed by the [descriptor](https://docs.python.org/3/howto/descriptor.html) that triggers when the method attribute is read on an instance. + + If curious, try this (tested in Python 3.8): + + ```python + class Thing: + def f(self): + pass + + print(type(Thing.f)) # --> "function", i.e. the same type as a bare function + assert Thing.f is Thing.f # it's always the same function object + + thing = Thing() + print(type(thing.f)) # --> "method", i.e. a bound method of Thing instance at 0x... + assert thing.f is not thing.f # each read produces a **new** bound method object + + lst = [1, 2, 3] + print(type(lst.append)) # --> "builtin_function_or_method" + assert lst.append is not lst.append # this happens even for builtins + ``` - **OOP inheritance**. - When `@generic` is installed on a method (instance method, or `@classmethod`), then at call time, classes are tried in [MRO](https://en.wikipedia.org/wiki/C3_linearization) order. All multimethods of the method defined in the class currently being looked up are tested for matches first, before moving on to the next class in the MRO. This has subtle consequences, related to in which class in the hierarchy the various multimethods for a particular method are defined. - To work with OOP inheritance, `@generic` must be the outermost decorator (except `@classmethod` or `@staticmethod`, which are essentially compiler annotations). - - However, when installed on a `@staticmethod`, the `@generic` decorator does not support MRO lookup, because that would make no sense. See discussions on interaction between `@staticmethod` and `super` in Python: [[1]](https://bugs.python.org/issue31118) [[2]](https://stackoverflow.com/questions/26788214/super-and-staticmethod-interaction/26807879). + - However, when installed on a `@staticmethod`, the `@generic` decorator does not support MRO lookup, because that would make no sense. A static method is just a bare function that happens to be stored in a class namespace. See discussions on the interaction between `@staticmethod` and `super` in Python: [[1]](https://bugs.python.org/issue31118) [[2]](https://stackoverflow.com/questions/26788214/super-and-staticmethod-interaction/26807879). + - When inspecting an **instance method** that is `@generic`, be sure to call the `methods` function **on an instance**: -##### Notes + ```python + class Thing: + @generic + def f(self, x: int): + pass -In both CLOS and in Julia, *function* is the generic entity, while *method* refers to its specialization to a particular combination of argument types. Note that *no object instance or class is needed*. Contrast with the classical OOP sense of *method*, i.e. a function that is associated with an object instance or class, with single dispatch based on the class (or in exotic cases, such as monkey-patched instances, on the instance). + @classmethod + @generic + def g(cls, x: int): + pass + + thing = Thing() + methods(thing.f) + + methods(Thing.g) + ``` -Based on my own initial experiments with this feature, the machinery itself works well enough, but to really shine - just like resumable exceptions - multiple dispatch needs to be used everywhere, throughout the language's ecosystem. Python obviously doesn't do that. + This allows seeing registered multimethods also from linked dispatchers in the MRO. -The machinery itself is also missing some advanced features, such as matching the most specific multimethod candidate instead of the most recently defined one; an `issubclass` equivalent that understands `typing` type specifications; and a mechanism to remove previously declared multimethods. + If we instead call it as `methods(Thing.f)`, the `self` argument is not bound yet (because `Thing.f` is just a bare function), so the dispatch machinery cannot get a reference to the MRO. This is obviously not an issue when actually *using* `f`, since an instance method is pretty much always invoked on an instance. -**CAUTION**: Multiple dispatch can be dangerous. Particularly, `@augment` can be dangerous to the readability of your codebase. If a new multimethod is added for a generic function defined elsewhere, for types defined elsewhere, this may lead to [*spooky action at a distance*](https://lexi-lambda.github.io/blog/2016/02/18/simple-safe-multimethods-in-racket/) (as in [action at a distance](https://en.wikipedia.org/wiki/Action_at_a_distance_(computer_programming))). In the Julia community, this is known as [*type piracy*](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy). Keep in mind that the multiple-dispatch table is global state! + For class methods, `methods(Thing.g)` sees the MRO, because `cls` is already bound. +For usage examples of `@generic` with OOP, see [the unit tests](../unpythonic/tests/test_dispatch.py). -#### ``typed``: add run-time type checks with type annotation syntax -The ``typed`` decorator creates a one-multimethod pony, which automatically enforces its argument types. Just like with ``generic``, the type specification may use features from the `typing` stdlib module. +#### `augment`: add a new multimethod to an existing generic function + +The `@augment` decorator adds a new multimethod to an existing generic function. With this system, it is possible to implement [*holy traits*](https://ahsmart.com/pub/holy-traits-design-patterns-and-best-practice-book/): + +```python +import typing +from unpythonic import generic, augment + +class FunninessTrait: + pass +class IsFunny(FunninessTrait): + pass +class IsNotFunny(FunninessTrait): + pass + +@generic +def funny(x: typing.Any): # default + raise NotImplementedError(f"`funny` trait not registered for anything matching {type(x)}") + +@augment(funny) +def funny(x: str): # noqa: F811 + return IsFunny() +@augment(funny) +def funny(x: int): # noqa: F811 + return IsNotFunny() + +@generic +def laugh(x: typing.Any): + return laugh(funny(x), x) + +@augment(laugh) +def laugh(traitvalue: IsFunny, x: typing.Any): + return f"Ha ha ha, {x} is funny!" +@augment(laugh) +def laugh(traitvalue: IsNotFunny, x: typing.Any): + return f"{x} is not funny." + +assert laugh("that") == "Ha ha ha, that is funny!" +assert laugh(42) == "42 is not funny." +``` + +**CAUTION**: `@augment` can be dangerous to the readability of your codebase. Keep in mind that the multiple-dispatch table is global state. If you add a new multimethod for a generic function defined elsewhere, for types defined elsewhere, this may lead to [*spooky action at a distance*](https://lexi-lambda.github.io/blog/2016/02/18/simple-safe-multimethods-in-racket/) (as in [action at a distance](https://en.wikipedia.org/wiki/Action_at_a_distance_(computer_programming))), because it may change the meaning of existing code. In the Julia community, this is known as [*type piracy*](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy). + +As Alexis King points out, no type piracy occurs if **at least one** of the following conditions holds: + + 1. At least one of the types in the call signature of the new multimethod is defined by you. + + 2. The generic function you are augmenting is defined by you. + + +##### How to augment a function that is not already `@generic` + +Given this: + +```python +# thirdparty.py +def op(x): + if isinstance(x, int): + return 2 * x + elif isinstance(x, float): + return 2.0 * x + raise TypeError(f"unsupported argument: {type(x)} with value {repr(x)}") +``` + +you do not have to change that code, but you will have to know which argument types the existing function supports (because that information is not available in an inspectable form at its interface), and then overwrite the original binding, with something like this: + +```python +# ours.py +import thirdparty + +original_op = thirdparty.op + +# Multimethod implementations for the types supported by the original `op`. +# We just re-dispatch here. +@generic +def op(x: int): + return original_op(x) +@generic +def op(x: float): + return original_op(x) + +thirdparty.op = op # unavoidable bit of monkey-patching +``` + +Then it can be augmented as usual: + +```python +@augment(op) +def op(x: str): # "ha" -> "ha, ha" + return ", ".join(x for _ in range(2)) +``` + +while preserving the meaning of all existing code that uses `thirdparty.op`. + + +#### `typed`: add run-time type checks with type annotation syntax + +The `typed` decorator creates a one-multimethod pony, which automatically enforces its argument types. Just like with `generic`, the type specification may use features from the `typing` stdlib module. ```python import typing @@ -3205,14 +4218,45 @@ assert jack("foo") == "foo" jack(3.14) # TypeError ``` -For which features of the ``typing`` stdlib module are supported, see ``isoftype`` below. +For which features of the `typing` stdlib module are supported, see [`isoftype`](#isoftype-the-big-sister-of-isinstance) below. + + +#### `isoftype`: the big sister of `isinstance` +Type check object instances against type specifications at run time. This is the machinery that powers `generic` and `typed`. This goes beyond `isinstance` in that many (but not all) features of the `typing` standard library module are supported. -#### ``isoftype``: the big sister of ``isinstance`` +`isoftype` is a **non-destructive** runtime type checker. It never consumes iterators, calls functions, or enters context managers to inspect their types. This limits what it can check — for example, element types of iterators and argument/return types of callables cannot be verified — but it means `isoftype` is always safe to call, even in hot loops or dispatch logic. -Type check object instances against type specifications at run time. This is the machinery that powers ``generic`` and ``typed``. This goes beyond ``isinstance`` in that many (but unfortunately not all) features of the ``typing`` standard library module are supported. +Any checks on the type arguments of the meta-utilities defined in the `typing` stdlib module are performed recursively using `isoftype` itself, in order to allow compound specifications. -Any checks on the type arguments of the meta-utilities defined in the ``typing`` stdlib module are performed recursively using `isoftype` itself, in order to allow compound abstract specifications. +**Supported `typing` features:** + +| Category | Supported types | +|----------|----------------| +| Basics | `Any`, `TypeVar`, `NewType`, `Union`, `Optional` | +| Bottom | `NoReturn`, `Never` (3.11+) | +| Values | `Literal[v1, v2, ...]` | +| Classes | `Type[X]` | +| Wrappers | `ClassVar[T]`, `Final[T]` (stripped, inner type checked) | +| Tuples | `Tuple`, `Tuple[T, ...]`, `Tuple[T1, T2, ..., TN]` | +| Sequences | `List[T]`, `Sequence[T]`, `MutableSequence[T]`, `Deque[T]` | +| Sets | `Set[T]`, `FrozenSet[T]`, `AbstractSet[T]`, `MutableSet[T]` | +| Mappings | `Dict[K, V]`, `DefaultDict[K, V]`, `OrderedDict[K, V]`, `Counter[T]`, `ChainMap[K, V]`, `Mapping[K, V]`, `MutableMapping[K, V]` | +| Views | `KeysView[K]`, `ValuesView[V]`, `ItemsView[K, V]` | +| IO | `IO`, `IO[str]`, `IO[bytes]`, `TextIO`, `BinaryIO` | +| Regex | `Pattern[T]`, `Match[T]` (string type checked) | +| Callables | `Callable` (arg/return types **not** checked) | +| Generators | `Generator`, `AsyncGenerator` (yield/send/return types **not** checked) | +| Async | `Awaitable`, `Coroutine`, `AsyncIterable`, `AsyncIterator` | +| Context managers | `ContextManager`, `AsyncContextManager` | +| Protocols | `SupportsInt`, `SupportsFloat`, `SupportsComplex`, `SupportsBytes`, `SupportsIndex`, `SupportsAbs`, `SupportsRound` | +| Protocol (user) | `@runtime_checkable` Protocol subclasses (structural subtyping via `isinstance`) | +| TypedDict | Structural check: required/optional keys, value types recursively checked | +| ABCs (best-effort) | `Iterable[T]`, `Collection[T]`, `Reversible[T]` (elements checked when value is `Sized`; ABC-only when not) | +| ABCs (type arg ignored) | `Iterator[T]`, `Container[T]` (parametric form accepted, type arg silently ignored) | +| ABCs (non-generic) | `Hashable`, `Sized` | + +**Not supported:** `Generic`, `ForwardRef`. Specific `NamedTuple` subclasses work via the `isinstance` fallback. Non-`@runtime_checkable` Protocols raise `TypeError` with an actionable message. Some examples: @@ -3220,11 +4264,11 @@ Some examples: import typing from unpythonic import isoftype -# concrete types - uninteresting, we just delegate to `isinstance` +# concrete types — just delegates to isinstance assert isoftype(17, int) assert isoftype(lambda: ..., typing.Callable) -# typing.newType +# typing.NewType UserId = typing.NewType("UserId", int) assert isoftype(UserId(42), UserId) # Note limitation: since NewType types discard their type information at @@ -3268,31 +4312,79 @@ assert isoftype({1: "foo", 2: "bar"}, typing.MutableMapping[int, str]) assert isoftype((1, 2, 3), typing.Sequence[int]) assert isoftype({1, 2, 3}, typing.AbstractSet[int]) +# new in 2.0.0 +assert isoftype(200, typing.Literal[200, 404, 500]) +assert isoftype(int, typing.Type[int]) +assert isoftype(bool, typing.Type[int]) # bool is a subclass of int +import collections +assert isoftype(collections.Counter("hello"), typing.Counter[str]) +import re +assert isoftype(re.compile(r"\d+"), typing.Pattern[str]) +import io +assert isoftype(io.StringIO("hi"), typing.TextIO) +assert isoftype(io.BytesIO(b"hi"), typing.BinaryIO) + # one-trick ponies assert isoftype(3.14, typing.SupportsRound) assert isoftype([1, 2, 3], typing.Sized) + +# best-effort element checking for abstract iterables +assert isoftype([1, 2, 3], typing.Iterable[int]) # concrete → elements checked +assert not isoftype([1, 2, 3], typing.Iterable[str]) # wrong element type +assert isoftype(iter([1, 2, 3]), typing.Iterable[int]) # opaque iterator → ABC only + +# TypedDict — structural checking of keys and value types +class Point(typing.TypedDict): + x: float + y: float +assert isoftype({"x": 1.0, "y": 2.0}, Point) +assert not isoftype({"x": 1.0}, Point) # missing required key + +# Protocol (must be @runtime_checkable) +@typing.runtime_checkable +class Drawable(typing.Protocol): + def draw(self) -> None: ... +class Circle: + def draw(self): + pass +assert isoftype(Circle(), Drawable) ``` -See [the unit tests](../unpythonic/tests/test_typecheck.py) for more. +See [the unit tests](../unpythonic/tests/test_typecheck.py) for the full set of supported features. -**CAUTION**: Callables are just checked for being callable; no further analysis is done. Type-checking callables properly requires a much more complex type checker. +**CAUTION**: For types where the type parameters describe behavior rather than stored data — `Callable`, `Generator`, `AsyncGenerator`, `ContextManager`, `AsyncContextManager`, `Awaitable`, `Coroutine`, `AsyncIterable`, `AsyncIterator`, `Iterator`, `Container` — only the ABC is checked. The type parameters are silently ignored, because checking them would require consuming or invoking the value. -**CAUTION**: The `isoftype` function is one big hack. In Python 3.6 through 3.9, there is no consistent way to handle a type specification at run time. We must access some private attributes of the ``typing`` meta-utilities, because that seems to be the only way to get what we need to do this. +For `Iterable[T]`, `Collection[T]`, and `Reversible[T]`, element types are checked **best-effort**: if the value is `Sized` (a concrete collection like `list`, `set`, etc.), elements are checked; if it's an opaque iterator, only the ABC is checked. Empty concrete collections reject parametric specs (consistent with `List[T]`, `Sequence[T]`, etc.). -For a similar tool for run-time type-checking, see also the [`typeguard`](https://github.com/agronholm/typeguard) library. + +#### Notes + +The multiple-dispatch subsystem of `unpythonic` was inspired by the [multi-methods of CLOS](http://www.gigamonkeys.com/book/object-reorientation-generic-functions.html) (the Common Lisp Object System), and the [generic functions of Julia](https://docs.julialang.org/en/v1/manual/methods/). + +In both CLOS and in Julia, *function* is the generic entity, while *method* refers to its specialization to a particular combination of argument types. Note that *no object instance or class is needed*. Contrast with the classical OOP sense of *method*, i.e. a function that is associated with an object instance or class, with single dispatch based on the class (or in exotic cases, such as monkey-patched instances, on the instance). + +Based on my own initial experiments with this feature in Python, the machinery itself works well enough, but to really shine - just like conditions and restarts - multiple dispatch needs to be used everywhere, throughout the language's ecosystem. Julia is impressive here. Python obviously does not do that. + +Our machinery is missing some advanced features, such as matching the most specific multimethod candidate instead of the most recently defined one; an `issubclass` equivalent that understands `typing` type specifications; and a mechanism to remove previously declared multimethods. + +*If you need multiple dispatch, but not the other features of `unpythonic`, see the [multipledispatch](https://github.com/mrocklin/multipledispatch) library, which likely runs faster.* + +*If you need a run-time type checker, but not the other features of `unpythonic`, see the [`typeguard`](https://github.com/agronholm/typeguard) library. If you are fine with a separate static type checker (which is the step where type checking arguably belongs), just use [`Mypy`](http://mypy-lang.org/).* ## Exception tools Utilities for dealing with exceptions. -### ``raisef``, ``tryf``: ``raise`` and ``try`` as functions +### `raisef`, `tryf`: `raise` and `try` as functions + +**Changed in v0.15.0.** *Deprecated parameters for `raisef` removed.* **Changed in v0.14.3**. *Now we have also `tryf`.* -**Changed in v0.14.2**. *The parameters of `raisef` now more closely match what would be passed to `raise`. See examples below. Old-style parameters are now deprecated, and support for them will be dropped in v0.15.0.* +**Changed in v0.14.2**. *The parameters of `raisef` now more closely match what would be passed to `raise`. See examples below. Old-style parameters are now deprecated.* -Raise an exception from an expression position: +The `raisef` function allows to raise an exception from an expression position: ```python from unpythonic import raisef @@ -3305,7 +4397,7 @@ exc = TypeError("oof") g = lambda x: raisef(RuntimeError("I'm in ur lambda raising exceptions"), cause=exc) ``` -Catch an exception in an expression position: +The `tryf` function is a `try`/`except`/`else`/`finally` construct for an expression position: ```python from unpythonic import raisef, tryf @@ -3315,16 +4407,46 @@ test[tryf(lambda: raise_instance(), (ValueError, lambda err: f"got a ValueError: '{err.args[0]}'")) == "got a ValueError: 'all ok'"] ``` -The exception handler is a function. It may optionally accept one argument, the exception instance. +The exception handler is a function. It may optionally accept one argument, the exception instance. Just like in an `except` clause, the exception specification can be either an exception type, or a `tuple` of exception types. + +Functions can also be specified to represent the `else` and `finally` blocks; the keyword parameters to do this are `elsef` and `finallyf`. Each of them is a thunk (a 0-argument function). See the docstring of `unpythonic.tryf` for details. + +Examples can be found in [the unit tests](../unpythonic/tests/test_excutil.py). -Functions can also be specified for the `else` and `finally` behavior; see the docstring of `unpythonic.misc.tryf` for details. +### `withf`: `with` as a function -### ``equip_with_traceback`` +**Added in v2.2.0**. + +The `withf` function is a `with` block for an expression position. This rounds out the set with `raisef` and `tryf`, completing the suite of statement-as-expression utilities the language otherwise omits. + +```python +from unpythonic import withf + +# Single context manager; body receives the as-value. +contents = withf(open("README.md"), lambda f: f.read()) + +# Multiple context managers, entered left-to-right and exited in reverse, +# analogously to `with cm1, cm2, ...:`. Pass them as a tuple. +combined = withf((open("a.txt"), open("b.txt")), + lambda fa, fb: fa.read() + fb.read()) + +# Body may be a thunk if the as-values aren't needed (the `with lock:` style). +result = withf(lock, lambda: critical_section()) +``` + +The body's arity is auto-detected: if it accepts as many positional arguments as there are context managers, it receives the as-values in order; if it is a thunk, the as-values are discarded. This mirrors the optional-argument convention of `tryf`'s exception handlers. + +The return value of `withf` is whatever the body returns. (Lispily, `with` is an expression here, even though Python's statement form is value-less. *Value of everything, cost of nothing.*) + +Examples can be found in [the unit tests](../unpythonic/tests/test_excutil.py). + + +### `equip_with_traceback` **Added in v0.14.3**. -In Python 3.7 and later, equip a manually created exception instance with a traceback. This is useful mainly in special cases, where `raise` cannot be used for some reason. (The `signal` function in the conditions-and-restarts system uses this.) +In Python 3.7 and later, the `equip_with_traceback` function equips a manually created exception instance with a traceback. This is useful mainly in special cases, where `raise` cannot be used for some reason. (The `signal` function in the conditions-and-restarts system uses this.) ```python e = SomeException(...) @@ -3333,22 +4455,24 @@ e = equip_with_traceback(e) The traceback is automatically extracted from the call stack of the calling thread. -Optionally, you can cull a number of the topmost frames by passing the optional argument `stacklevel=...`. Typically, for direct use of this function `stacklevel` should be the default `1` (so it excludes `equip_with_traceback` itself, but shows all stack levels from your code), and for use in a utility function that itself is called from your code, it should be `2` (so it excludes the utility function, too). +Optionally, you can cull a number of the topmost frames by passing the optional argument `stacklevel=...`. Typically, for direct use of this function `stacklevel` should be the default `1` (so it excludes `equip_with_traceback` itself, but shows all stack levels from your code), and for use in a utility function that itself is called from your code, it should be `2` (so it excludes the utility function, too). If the utility function itself calls a separate low-level utility, `3` can be useful (see [the source code](../unpythonic/conditions.py) of the conditions-and-restarts system for an example). -### ``async_raise``: inject an exception to another thread +### `async_raise`: inject an exception to another thread **Added in v0.14.2**. -*Currently CPython only, because as of this writing (March 2020) PyPy3 does not expose the required functionality to the Python level, nor there seem to be any plans to do so.* +**CAUTION**: *Currently this is supported by CPython only, because as of June 2021, PyPy3 does not expose the required functionality to the Python level, nor there seem to be any plans to do so.* + +Usually injecting an exception into an unsuspecting thread makes absolutely no sense. But there are special cases, notably `KeyboardInterrupt`. Especially, a REPL server may need to send a `KeyboardInterrupt` into a REPL session thread that is happily stuck waiting for input inside [`InteractiveConsole.interact`](https://docs.python.org/3/library/code.html#code.InteractiveConsole.interact) - while the client that receives the actual `Ctrl+C` is running in a separate process, possibly even on a different machine. This and similar awkward situations in network programming are pretty much the only use case for this feature. -Usually injecting an exception into an unsuspecting thread makes absolutely no sense. But there are special cases, such as a REPL server which needs to send a `KeyboardInterrupt` into a REPL session thread that's happily stuck waiting for input at [`InteractiveConsole.interact()`](https://docs.python.org/3/library/code.html#code.InteractiveConsole.interact) - while the client that receives the actual `Ctrl+C` is running in a separate process. This and similar awkward situations in network programming are pretty much the only legitimate use case for this feature. +The function is named `async_raise`, because it injects an *asynchronous exception*. This has nothing to do with `async`/`await`. Synchronous vs. asynchronous exceptions [mean something different](https://en.wikipedia.org/wiki/Exception_handling#Exception_synchronicity). -The name is `async_raise`, because it injects an *asynchronous exception*. This has nothing to do with `async`/`await`. Synchronous vs. asynchronous exceptions [mean something different](https://en.wikipedia.org/wiki/Exception_handling#Exception_synchronicity). +In a nutshell, a *synchronous* exception (which is the usual kind of exception) has an explicit `raise` somewhere in the code that the thread that encountered the exception is running. In contrast, an *asynchronous* exception **does not**, it just suddenly magically materializes from the outside. As such, it can in principle happen *anywhere*, with absolutely no hint about it in any obvious place in the code. -In a nutshell, a *synchronous* exception (which is the usual kind of exception) has an explicit `raise` somewhere in the code that the thread that encountered the exception is running. In contrast, an *asynchronous* exception **doesn't**, it just suddenly magically materializes from the outside. As such, it can in principle happen *anywhere*, with absolutely no hint about it in any obvious place in the code. +Obviously, this can be very confusing, so this feature should be used sparingly, if at all. **We only provide it because the REPL server needs it**, and it would be silly to have such a feature but not make it public. -Needless to say this can be very confusing, so this feature should be used sparingly, if at all. **We only have it because the REPL server needs it.** +Here is an example: ```python from unpythonic import async_raise, box @@ -3364,16 +4488,16 @@ def worker(): t = threading.Thread(target=worker) t.start() sleep(0.1) # make sure the worker has entered the loop -async_raise(t, KeyboardInterrupt) +async_raise(t, KeyboardInterrupt) # CPython only! This will gracefully error out on PyPy. t.join() assert unbox(out) < 9 # thread terminated early due to the injected KeyboardInterrupt ``` -#### So this is how KeyboardInterrupt works under the hood? +#### Is this how KeyboardInterrupt works under the hood? -No, this is **not** how `KeyboardInterrupt` usually works. Rather, the OS sends a [SIGINT](https://en.wikipedia.org/wiki/Signal_(IPC)#SIGINT), which is then trapped by an [OS signal handler](https://docs.python.org/3/library/signal.html) that runs in the main thread. +**No, it is not.** The way `KeyboardInterrupt` usually works is, the OS sends a [SIGINT](https://en.wikipedia.org/wiki/Signal_(IPC)#SIGINT), which is then trapped by an [OS signal handler](https://docs.python.org/3/library/signal.html) that runs in the main thread. -(Note OS signal, in the *nix sense; this is unrelated to the Lisp sense, as in conditions-and-restarts.) +Note that it is an OS signal, in the *nix sense; which is unrelated to the Lisp/`unpythonic` sense, as in conditions-and-restarts. At that point the magic has already happened: the control of the main thread is now inside the signal handler, as if the signal handler was called from the otherwise currently innermost point on the call stack. All the handler needs to do is to perform a regular `raise`, and the exception will propagate correctly. @@ -3381,9 +4505,11 @@ At that point the magic has already happened: the control of the main thread is Original detective work by [Federico Ficarelli](https://gist.github.com/nazavode/84d1371e023bccd2301e) and [LIU Wei](https://gist.github.com/liuw/2407154). -Raising async exceptions is a [documented feature of Python's public C API](https://docs.python.org/3/c-api/init.html#c.PyThreadState_SetAsyncExc), but it was never meant to be invoked from within pure Python code. But then the CPython devs gave us [ctypes.pythonapi](https://docs.python.org/3/library/ctypes.html#accessing-values-exported-from-dlls), which allows access to Python's C API from within Python. (If you think ctypes.pythonapi is too quirky, the [pycapi](https://pypi.org/project/pycapi/) PyPI package smooths over the rough edges.) Combining the two gives `async_raise` without the need to compile a C extension. +Raising async exceptions is a [documented feature of Python's public C API](https://docs.python.org/3/c-api/init.html#c.PyThreadState_SetAsyncExc), but it was never meant to be invoked from within pure Python code. But then the CPython devs gave us [ctypes.pythonapi](https://docs.python.org/3/library/ctypes.html#accessing-values-exported-from-dlls), which allows access to CPython's C API from within Python. Combining the two gives `async_raise` without the need to compile a C extension. -Unfortunately PyPy doesn't currently (March 2020) implement this function in its CPython C API emulation layer, `cpyext`. See `unpythonic` issue [#58](https://github.com/Technologicat/unpythonic/issues/58). +(If you think `ctypes.pythonapi` is too quirky, the [pycapi](https://pypi.org/project/pycapi/) PyPI package smooths over the rough edges.) + +Unfortunately PyPy does **not** currently (June 2021) implement this function in its CPython C API emulation layer, `cpyext`. See `unpythonic` issue [#58](https://github.com/Technologicat/unpythonic/issues/58). ### `reraise_in`, `reraise`: automatically convert exception types @@ -3453,26 +4579,30 @@ except ApplicationException: ``` -If that's not much shorter than the hand-written `try`/`except`/`raise from`, consider that you can create the mapping once and then use it from a variable - this shortens it to just `with reraise(my_mapping)`. +If that does not seem much shorter than a hand-written `try`/`except`/`raise from`, consider that you can create the mapping once and then use it from a variable - this shortens it to just `with reraise(my_mapping)`. -Any exceptions that don't match anything in the mapping are passed through. When no exception occurs, `reraise_in` passes the return value of `thunk` through, and `reraise` does nothing. +Any exceptions that do not match anything in the mapping are passed through. When no exception occurs, `reraise_in` passes the return value of `thunk` through, and `reraise` does nothing. Full details in docstrings. If you use the conditions-and-restarts system, see also `resignal_in`, `resignal`, which perform the same job for conditions. The new signal is sent using the same error handling protocol as the original signal, so e.g. an `error` will remain an `error` even if re-signaling changes its type. +Examples can be found in [the unit tests](../unpythonic/tests/test_excutil.py). -## Other -Stuff that didn't fit elsewhere. +## Function call and return value tools -### ``def`` as a code block: ``@call`` +### `def` as a code block: `@call` -Fuel for different thinking. Compare `call-with-something` in Lisps - but without parameters, so just `call`. A `def` is really just a new lexical scope to hold code to run later... or right now! +Fuel for different thinking. Compare `call-with-something` in Lisps - but without parameters, so just `call`. A `def` is really just a new lexical scope to hold code to run later... or as `@call` does, right now! -At the top level of a module, this is seldom useful, but keep in mind that Python allows nested function definitions. Used with an inner ``def``, this becomes a versatile tool. +At the top level of a module, this is seldom useful, but keep in mind that Python allows nested function definitions. Used with an inner `def`, this becomes a versatile tool. -*Make temporaries fall out of scope as soon as no longer needed*: +Note that beside use as a decorator, `call` can also be used as a normal function: `call(f, *a, **kw)` is the same as `f(*a, **kw)`. This is occasionally useful. + +Let us consider some example use cases of `@call`. + +#### Make temporaries fall out of scope as soon as no longer needed ```python from unpythonic import call @@ -3486,9 +4616,13 @@ def x(): print(x) # 30 ``` -*Multi-break out of nested loops* - `continue`, `break` and `return` are really just second-class [ec](https://docs.racket-lang.org/reference/cont.html#%28def._%28%28lib._racket%2Fprivate%2Fletstx-scheme..rkt%29._call%2Fec%29%29)s. So `def` to make `return` escape to exactly where you want: +#### Multi-break out of nested loops + +As was noted in the section on escape continuations, `continue`, `break` and `return` are really just second-class [ec](https://docs.racket-lang.org/reference/cont.html#%28def._%28%28lib._racket%2Fprivate%2Fletstx-scheme..rkt%29._call%2Fec%29%29)s. So use a `def` to make `return` escape to exactly where you want: ```python +from unpythonic import call + @call def result(): for x in range(10): @@ -3498,7 +4632,7 @@ def result(): print(result) # (6, 7) ``` -(But see ``@catch``, ``throw``, and ``call_ec``.) +But if you need a *multi-return*, see `@catch`, `throw`, and `call_ec`. Compare the sweet-exp Racket: @@ -3513,9 +4647,11 @@ define result displayln result ; (6 7) ``` -Noting [what ``let/ec`` does](https://docs.racket-lang.org/reference/cont.html#%28form._%28%28lib._racket%2Fprivate%2Fletstx-scheme..rkt%29._let%2Fec%29%29), using ``call_ec`` we can make the Python even closer to the Racket: +Noting [what `let/ec` does](https://docs.racket-lang.org/reference/cont.html#%28form._%28%28lib._racket%2Fprivate%2Fletstx-scheme..rkt%29._let%2Fec%29%29), using `call_ec` we can make the Python even closer to the Racket: ```python +from unpythonic import call_ec + @call_ec def result(rtn): for x in range(10): @@ -3525,20 +4661,24 @@ def result(rtn): print(result) # (6, 7) ``` -*Twist the meaning of `def` into a "let statement"*: +#### Twist the meaning of `def` into a "let statement" ```python +from unpythonic import call + @call def result(x=1, y=2, z=3): return x * y * z print(result) # 6 ``` -(But see `blet`, `bletrec` if you want an `env` instance.) +If you want an `env` instance, see `blet` and `bletrec`. -*Letrec without `letrec`*, when it doesn't have to be an expression: +#### Letrec without `letrec`, when a statement is acceptable ```python +from unpythonic import call + @call def t(): def evenp(x): return x == 0 or oddp(x - 1) @@ -3547,22 +4687,22 @@ def t(): print(t) # True ``` -Essentially the implementation is just `def call(thunk): return thunk()`. The point is to: +#### Notes - - Make it explicit right at the definition site that this block is *going to be called now* (in contrast to an explicit call and assignment *after* the definition). Centralize the related information. Align the presentation order with the thought process. +Essentially the implementation is just `def call(thunk): return thunk()`. The point of this seemingly trivial construct is to: - - Help eliminate errors, in the same way as the habit of typing parentheses only in pairs. No risk of forgetting to call the block after writing the definition. + - Make it explicit right at the definition site that this block is *going to be called now*, in contrast to an explicit call and assignment *after* the definition. This centralizes the related information, and aligns the presentation order with the thought process. - - Document that the block is going to be used only once. Tell the reader there's no need to remember this definition. + - Help eliminate errors, in the same way as the habit of typing parentheses only in pairs (or using a tool like Emacs's `smartparens-mode` to enforce that). With `@call`, there is no risk of forgetting to call the block after writing the definition. -Note [the grammar](https://docs.python.org/3/reference/grammar.html) requires a newline after a decorator. + - Document that the block is going to be used only once. Tell your readers there is no need to remember this definition. -**NOTE**: ``call`` can also be used as a normal function: ``call(f, *a, **kw)`` is the same as ``f(*a, **kw)``. This is occasionally useful. +Note [the grammar](https://docs.python.org/3/reference/grammar.html) requires a newline after a decorator. -### ``@callwith``: freeze arguments, choose function later +### `@callwith`: freeze arguments, choose function later -If you need to pass arguments when using ``@call`` as a decorator, use its cousin ``@callwith``: +If you need to pass arguments when using `@call` as a decorator, use its sister `@callwith`: ```python from unpythonic import callwith @@ -3573,9 +4713,11 @@ def result(x): assert result == 9 ``` -Like ``call``, it can also be called normally. It's essentially an argument freezer: +Like `call`, beside use as a decorator, `callwith` can also be called normally. It is essentially an argument freezer: ```python +from unpythonic import callwith + def myadd(a, b): return a + b def mymul(a, b): @@ -3585,16 +4727,17 @@ assert apply23(myadd) == 5 assert apply23(mymul) == 6 ``` -When called normally, the two-step application is mandatory. The first step stores the given arguments. It returns a function ``f(callable)``. When ``f`` is called, it calls its ``callable`` argument, passing in the arguments stored in the first step. +When `callwith` is called normally, the two-step application is mandatory. The first step stores the given arguments. It then returns a function `f(callable)`. When `f` is called, it calls its `callable` argument, passing in the arguments stored in the first step. -In other words, ``callwith`` is similar to ``functools.partial``, but without specializing to any particular function. The function to be called is given later, in the second step. +In other words, `callwith` is similar to `functools.partial`, but without specializing to any particular function. The function to be called is given later, in the second step. -Hence, ``callwith(2, 3)(myadd)`` means "make a function that passes in two positional arguments, with values ``2`` and ``3``. Then call this function for the callable ``myadd``". But if we instead write``callwith(2, 3, myadd)``, it means "make a function that passes in three positional arguments, with values ``2``, ``3`` and ``myadd`` - not what we want in the above example. +Hence, `callwith(2, 3)(myadd)` means *make a function that passes in two positional arguments, with values `2` and `3`. Then call this function for the callable `myadd`*. But if we instead write `callwith(2, 3, myadd)`, it means *make a function that passes in three positional arguments, with values `2`, `3` and `myadd`* - not what we want in the above example. -If you want to specialize some arguments now and some later, combine with ``partial``: +If you want to specialize some arguments now and some later, combine `callwith` with `partial`: ```python from functools import partial +from unpythonic import callwith p1 = partial(callwith, 2) p2 = partial(p1, 3) @@ -3610,20 +4753,22 @@ assert apply234(mul3) == 24 If the code above feels weird, it should. Arguments are gathered first, and the function to which they will be passed is chosen in the last step. -Another use case of ``callwith`` is ``map``, if we want to vary the function instead of the data: +Another use case of `callwith` is `map`, if we want to vary the function instead of the data: ```python +from unpythonic import callwith + m = map(callwith(3), [lambda x: 2*x, lambda x: x**2, lambda x: x**(1/2)]) assert tuple(m) == (6, 9, 3**(1/2)) ``` -If you use the quick lambda macro `f[]` (underscore notation for Python), this combines nicely: +If you use the quick lambda macro `fn[]` (underscore notation for Python), these features combine nicely: ```python -from unpythonic.syntax import macros, f +from unpythonic.syntax import macros, fn from unpythonic import callwith -m = map(callwith(3), [f[2 * _], f[_**2], f[_**(1/2)]]) +m = map(callwith(3), [fn[2 * _], fn[_**2], fn[_**(1/2)]]) assert tuple(m) == (6, 9, 3**(1/2)) ``` @@ -3655,23 +4800,327 @@ assert tuple(m) == (6, 9, 3**(1/2)) Inspired by *Function application with $* in [LYAH: Higher Order Functions](http://learnyouahaskell.com/higher-order-functions). -### ``callsite_filename`` +### Spreading a `Values` into `call` / `callwith` + +**Added in v2.2.0.** + +Both `call` and `callwith` recognize [`Values`](#values-multiple-and-named-return-values) in their positional arguments and spread it into the call. Each `Values` expands in place, left-to-right: its `rets` splice into the positional arguments, its `kwrets` merge into the keyword arguments. Across multiple `Values` and the caller's explicit keyword arguments, rightmost wins per unique keyword name (explicit `kwargs` are syntactically last, so they always override). + +```python +from unpythonic import call, callwith, Values + +# Apply a Values bundle as the arguments to a function. +v = Values(1, 2, x=3) +assert call(lambda a, b, x: (a, b, x), v) == (1, 2, 3) + +# Spread anywhere, mixed with regular arguments. +def f(a, b, c): + return (a, b, c) +assert call(f, 1, Values(2, 3)) == (1, 2, 3) +assert call(f, Values(1, 2), 3) == (1, 2, 3) + +# Multiple Values expand in left-to-right order. +def f4(a, b, c, d): + return (a, b, c, d) +assert call(f4, Values(1, 2), Values(3, 4)) == (1, 2, 3, 4) + +# Spread-and-override: kwrets carry defaults, explicit kwargs win. +defaults = Values(timeout=30, retries=3) +def api(*, timeout, retries): + return (timeout, retries) +assert call(api, defaults, retries=5) == (30, 5) +``` + +Mirrors Python's familiar spread/merge — `[*a, *b, c]` for the positional side, `{**a, **b}` for the keyword side. The motivating use is taking a `Values` produced by one function and applying it as the arguments to another, including spread-and-override patterns where a `Values` carries defaults and explicit kwargs override individual keys. + +For `callwith`, the spread happens at the moment `callwith` is invoked, when the arguments are frozen — so the inner closure already sees the expanded positional and keyword arguments. + + +### `Values`: multiple and named return values + +**Added in v0.15.0.** + +`Values` is a structured multiple-return-values type. + +With `Values`, you can return multiple values positionally, and **return values by name**. This completes the symmetry between passing function arguments and returning values from a function. Python itself allows passing arguments by name, but has no concept of returning values by name. This class adds that concept. + +Having a `Values` type separate from `tuple` helps with semantic accuracy. In `unpythonic` 0.15.0 and later, a `tuple` return value means just that - one value that is a `tuple`. It is distinct from a `Values` that contains several positional return values (that are meant to be treated separately e.g. by a function composition utility). + +Inspired by the [`values`](https://docs.racket-lang.org/reference/values.html) form of Racket. + +#### When to use `Values` + +Most of the time, returning a tuple to denote multiple-return-values and unpacking it is just fine, and that is exactly what `unpythonic` does internally in many places. + +But the distinction is critically important in function composition, so that positional return values can be automatically mapped into positional arguments to the next function in the chain, and named return values into named arguments. + +Accordingly, various parts of `unpythonic` that deal with function composition use the `Values` abstraction; particularly `curry`, `unfold`, `iterate`, the `compose` and `pipe` families, and the `with continuations` macro. + +#### Behavior + +`Values` is a duck-type with some features of both sequences and mappings, but not the full [`collections.abc`](https://docs.python.org/3/library/collections.abc.html) API of either. + +If there are no named return values in a `Values` object, it can be unpacked like a tuple. This covers the common use case of multiple positional return values with a minimum of fuss. + +Each operation that obviously and without ambiguity makes sense only for the positional or named part, accesses that part. + +The only exception is `__getitem__` (subscripting), which makes sense for both parts, unambiguously, because the key types differ. If the index expression is an `int` or a `slice`, it is an index/slice for the positional part. If it is an `str`, it is a key for the named part. + +If you need to explicitly access either part (and its full API), use the `rets` and `kwrets` attributes. The names are in analogy with `args` and `kwargs`. + +`rets` is a `tuple`, and `kwrets` is an `unpythonic.frozendict`. + +`Values` objects can be compared for equality. Two `Values` objects are equal if both their `rets` and `kwrets` (respectively) are. + +See the docstrings, [the source code](../unpythonic/funutil.py), and [the unit tests](../unpythonic/tests/test_funutil.py) for full details. + +Examples: + +```python +from unpythonic import Values + +def f(): + return Values(1, 2, 3) +result = f() +assert isinstance(result, Values) +assert result.rets == (1, 2, 3) +assert not result.kwrets +assert result[0] == 1 +assert result[:-1] == (1, 2) +a, b, c = result # if no kwrets, can be unpacked like a tuple +a, b, c = f() + +def g(): + return Values(x=3) # named return value +result = g() +assert isinstance(result, Values) +assert not result.rets +assert result.kwrets == {"x": 3} # actually a `frozendict` +assert "x" in result # `in` looks in the named part +assert result["x"] == 3 +assert result.get("x", None) == 3 +assert result.get("y", None) is None +assert tuple(result.keys()) == ("x",) # also `values()`, `items()` + +def h(): + return Values(1, 2, x=3) +result = h() +assert isinstance(result, Values) +assert result.rets == (1, 2) +assert result.kwrets == {"x": 3} +a, b = result.rets # positionals can always be unpacked explicitly +assert result[0] == 1 +assert "x" in result +assert result["x"] == 3 + +def silly_but_legal(): + return Values(42) +result = silly_but_legal() +assert result.rets[0] == 42 +assert result.ret == 42 # shorthand for single-value case +``` + +The last example is silly, but legal, because it is preferable to just omit the `Values` if it is known that there is only one return value. This also applies when that value is a `tuple`, when the intent is to return it as a single `tuple`, in contexts where this distinction matters. + + +### `valuify` + +The `valuify` decorator converts the pythonic tuple-as-multiple-return-values idiom into `Values`, to easily use existing code with our function composition utilities. + +It converts a `tuple` return value, exactly; no subclasses. + +Demonstrating only the conversion: + +```python +from unpythonic import valuify, Values + +@valuify +def f(x, y, z): + return x, y, z + +assert isinstance(f(1, 2, 3), Values) +assert f(1, 2, 3) == Values(1, 2, 3) +``` + + +## Numerical tools + +We briefly introduce the functions below. More details and examples can be found in the docstrings and in [the unit tests](../unpythonic/tests/test_numutil.py). + +**CAUTION** for anyone new to numerics: + +When working with floating-point numbers, keep in mind that they are, very roughly speaking, a finite-precision logarithmic representation of [ℝ](https://en.wikipedia.org/wiki/Real_line). They are, necessarily, actually a subset of [ℚ](https://en.wikipedia.org/wiki/Rational_number), that is not even [dense](https://en.wikipedia.org/wiki/Dense_set). The spacing between adjacent floats depends on where you are on the real line; see `ulp` below. + +For finer points concerning the behavior of floating-point numbers, see [David Goldberg (1991): What every computer scientist should know about floating-point arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html), or for a [tl;dr](http://catplanet.org/tldr-cat-meme/) version, [the floating point guide](https://floating-point-gui.de/). + +Or you could look at [my lecture slides from 2018](https://github.com/Technologicat/python-3-scicomp-intro/tree/master/lecture_slides); particularly, [lecture 7](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/lecture_slides/lectures_tut_2018_7.pdf) covers the floating-point representation. It collects the most important details in a few slides, and contains some more links to further reading. + + +### `almosteq`: floating-point almost-equality + +Test floating-point numbers for near-equality. Beside the built-in `float`, we support also the arbitrary-precision software-implemented floating-point type `mpf` from `SymPy`'s `mpmath` package. + +Anything else, for example `SymPy` expressions, strings, and containers (regardless of content), is tested for exact equality. + +For `mpmath.mpf`, we just delegate to `mpmath.almosteq`, with the given tolerance. + +For `float`, we use the strategy suggested in [the floating point guide](https://floating-point-gui.de/errors/comparison/), because naive absolute and relative comparisons against a tolerance fail in commonly encountered situations. + + +### `fixpoint`: arithmetic fixed-point finder + +**Added in v0.14.2.** + +*Not to be confused with the logical fixed point with respect to the definedness ordering, which is what Haskell's `fix` function relates to.* + +Compute the (arithmetic) fixed point of a function, starting from a given initial guess. The fixed point must be attractive for this to work. See the [Banach fixed point theorem](https://en.wikipedia.org/wiki/Banach_fixed-point_theorem). + +If the fixed point is attractive, and the values are represented in floating point (hence finite precision), the computation should eventually converge down to the last bit (barring roundoff or catastrophic cancellation in the final few steps). Hence the default tolerance is zero; but any desired tolerance can be passed as an argument. + +**CAUTION**: an arbitrary function from ℝ to ℝ **does not** necessarily have a fixed point. Limit cycles and chaotic behavior of the function will cause non-termination. Keep in mind the classic example, [the logistic map](https://en.wikipedia.org/wiki/Logistic_map). + +Examples: + +```python +from math import cos, sqrt +from unpythonic import fixpoint, ulp + +c = fixpoint(cos, x0=1) + +# Actually "Newton's" algorithm for the square root was already known to the +# ancient Babylonians, ca. 2000 BCE. (Carl Boyer: History of mathematics) +# Concerning naming, see also https://en.wikipedia.org/wiki/Stigler's_law_of_eponymy +def sqrt_newton(n): + def sqrt_iter(x): # has an attractive fixed point at sqrt(n) + return (x + n / x) / 2 + return fixpoint(sqrt_iter, x0=n / 2) +assert abs(sqrt_newton(2) - sqrt(2)) <= ulp(1.414) +``` + + +### `partition_int`: partition integers + +**Changed in v0.15.0.** *Added `partition_int_triangular` and `partition_int_custom`.* + +**Added in v0.14.2.** + +*Not to be confused with `unpythonic.partition`, which partitions an iterable based on a predicate.* + +The `partition_int` function [partitions](https://en.wikipedia.org/wiki/Partition_(number_theory)) a small positive integer, i.e., splits it in all possible ways, into smaller integers that sum to it. This is useful e.g. to determine the number of letters to allocate for each component of an anagram that may consist of several words. + +The `partition_int_triangular` function is like `partition_int`, but accepts only triangular numbers (1, 3, 6, 10, ...) as components of the partition. This function answers a timeless question: if I have `n` stackable plushies, what are the possible stack configurations? + +The `partition_int_custom` function is like `partition_int`, but lets you specify which numbers are acceptable as components of the partition. + +Examples: + +```python +from itertools import count, takewhile +from unpythonic import partition_int, partition_int_triangular, rev + +assert tuple(partition_int(4)) == ((4,), (3, 1), (2, 2), (2, 1, 1), (1, 3), (1, 2, 1), (1, 1, 2), (1, 1, 1, 1)) +assert tuple(partition_int(5, lower=2)) == ((5,), (3, 2), (2, 3)) +assert tuple(partition_int(5, lower=2, upper=3)) == ((3, 2), (2, 3)) + +assert (frozenset(tuple(sorted(c)) for c in partition_int_triangular(78, lower=10)) == + frozenset({(10, 10, 10, 10, 10, 28), + (10, 10, 15, 15, 28), + (15, 21, 21, 21), + (21, 21, 36), + (78,)})) + +evens_upto_n = lambda n: takewhile(lambda m: m <= n, count(start=2, step=2)) +assert tuple(partition_int_custom(6, rev(evens_upto_n(6)))) == ((6,), (4, 2), (2, 4), (2, 2, 2)) +``` + +As the first example demonstrates, most of the splits are a ravioli consisting mostly of ones. It is much faster to not generate such splits than to filter them out from the result. Use the `lower` parameter to set the smallest acceptable value for one component of the split; the default value `lower=1` generates all splits. Similarly, the `upper` parameter sets the largest acceptable value for one component of the split. The default `upper=None` sets no upper limit, so in effect the upper limit becomes `n`. + +In `partition_int_triangular`, the `lower` and `upper` parameters work exactly the same. The only difference to `partition_int` is that each component of the split must be a triangular number. + +In `partition_int_custom`, the components are given as an iterable, which is immediately forced (so if it is consumable, it will be completely consumed; and if it is infinite, the function will use up all available RAM and not terminate). Each component `x` must be an integer that satisfies `1 <= x <= n`. + +**CAUTION**: The number of possible partitions grows very quickly with `n`, so in practice these functions are only useful for small numbers, or when the smallest allowed component is not too much smaller than `n / 2`. + + +### `ulp`: unit in last place + +**Added in v0.14.2.** + +Given a floating point number `x`, return the value of the *unit in the last place* (the "least significant bit"). This is the local size of a "tick", i.e. the difference between `x` and the *next larger* float. At `x = 1.0`, this is the [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon), by definition of the machine epsilon. + +The float format is [IEEE-754](https://en.wikipedia.org/wiki/IEEE_754), i.e. standard Python `float`. + +This is just a small convenience function that is for some reason missing from the `math` standard library. + +```python +from unpythonic import ulp + +# in IEEE-754, exponent changes at integer powers of two +print([ulp(x) for x in (0.25, 0.5, 1.0, 2.0, 4.0)]) +# --> [5.551115123125783e-17, +# 1.1102230246251565e-16, +# 2.220446049250313e-16, # x = 1.0, so this is sys.float_info.epsilon +# 4.440892098500626e-16, +# 8.881784197001252e-16] +print(ulp(1e10)) +# --> 1.9073486328125e-06 +print(ulp(1e100)) +# --> 1.942668892225729e+84 +print(ulp(2**52)) +# --> 1.0 # yes, exactly 1 +``` + +When `x` is a round number in base-10, the ULP is not, because the usual kind of floats use base-2. + + +### `si_prefix`: format numbers with SI or IEC prefixes + +**Added in v2.1.0.** + +Format a number with an [SI decimal prefix](https://en.wikipedia.org/wiki/Metric_prefix) (powers of 1000) or an [IEC binary prefix](https://en.wikipedia.org/wiki/Binary_prefix) (powers of 1024). Both positive and negative (sub-unity) prefixes are supported in either mode. The micro prefix is `µ` (U+00B5 MICRO SIGN). + +```python +from unpythonic import si_prefix + +si_prefix(1500) # "1.50 k" +si_prefix(2_500_000) # "2.50 M" +si_prefix(0.0015) # "1.50 m" +si_prefix(0.000001) # "1.00 µ" +si_prefix(-1500) # "-1.50 k" +si_prefix(42) # "42.00" +si_prefix(42, precision=0) # "42" + +# IEC binary mode (base 1024) +si_prefix(1536, binary=True) # "1.50 Ki" +si_prefix(2_621_440, binary=True) # "2.50 Mi" +``` + + +## Other + +Stuff that didn't fit elsewhere. + +### `callsite_filename` + +**Changed in v0.15.0.** *This utility now ignores `unpythonic`'s call helpers, and gives the filename from the deepest stack frame that does not match one of our helpers. This allows the testing framework report the source code filename correctly when testing code using macros that make use of these helpers (e.g. `autocurry`, `lazify`).* **Added in v0.14.3**. Return the filename from which this function is being called. Useful as a building block for debug utilities and similar. -### ``safeissubclass`` +### `safeissubclass` **Added in v0.14.3**. Convenience function. Like `issubclass(cls)`, but if `cls` is not a class, swallow the `TypeError` and return `False`. -### ``pack``: multi-arg constructor for tuple +### `pack`: multi-arg constructor for tuple + +The default `tuple` constructor accepts a single iterable. But sometimes one needs to pass in the elements separately. Most often a literal tuple such as `(1, 2, 3)` is then the right solution, but there are situations that do not admit a literal tuple. -The default ``tuple`` constructor accepts a single iterable. But sometimes one needs to pass in the elements separately. Most often a literal tuple such as ``(1, 2, 3)`` is then the right solution, but there are situations that do not admit a literal tuple. Enter ``pack``: +In such cases it is possible to use `pack`: ```python from unpythonic import pack @@ -3682,13 +5131,13 @@ assert tuple(myzip(lol)) == ((1, 3, 5), (2, 4, 6)) ``` -### ``namelambda``: rename a function +### `namelambda`: rename a function -Rename any function object (including lambdas). The return value of ``namelambda`` is a modified copy; the original function object is not mutated. The input can be any function object (``isinstance(f, (types.LambdaType, types.FunctionType))``). It will be renamed even if it already has a name. +Rename any function object, even a lambda. The return value of `namelambda` is a modified copy; the original function object is not mutated. The input can be any function object (`isinstance(f, (types.LambdaType, types.FunctionType))`). It will be renamed even if it already has a name. This is mainly useful in those situations where you return a lambda as a closure, call it much later, and it happens to crash - so you can tell from the stack trace *which* of the *N* lambdas in your codebase it is. -For technical reasons, ``namelambda`` conforms to the parametric decorator API. Usage: +`namelambda` conforms to the parametric decorator API. Usage: ```python from unpythonic import namelambda @@ -3702,7 +5151,7 @@ kaboom() # --> stack trace, showing the function name "kaboom" The first call returns a *foo-renamer*, which takes a function object and returns a copy that has its name changed to *foo*. -Technically, this updates ``__name__`` (the obvious place), ``__qualname__`` (used by ``repr()``), and ``__code__.co_name`` (used by stack traces). +Technically, this updates `__name__` (the obvious place), `__qualname__` (used by `repr()`), and `__code__.co_name` (used by stack traces). **CAUTION**: There is one pitfall: @@ -3714,10 +5163,12 @@ print(nested.__qualname__) # "outer" print(nested().__qualname__) # "..inner" ``` -The inner lambda does not see the outer's new name; the parent scope names are baked into a function's ``__qualname__`` too early for the outer rename to be in effect at that time. +The inner lambda does not see the outer's new name; the parent scope names are baked into a function's `__qualname__` too early for the outer rename to be in effect at that time. + +### `timer`: a context manager for performance testing -### ``timer``: a context manager for performance testing +This is a small convenience utility, used as follows: ```python from unpythonic import timer @@ -3732,10 +5183,43 @@ with timer(p=True): # if p, auto-print result pass ``` -The auto-print mode is a convenience feature to minimize bureaucracy if you just want to see the *Δt*. To instead access the *Δt* programmatically, name the timer instance using the ``with ... as ...`` syntax. After the context exits, the *Δt* is available in its ``dt`` attribute. +The auto-print mode is a convenience feature to minimize bureaucracy if you just want to see the *Δt*. To instead access the *Δt* programmatically, name the timer instance using the `with ... as ...` syntax. After the context exits, the *Δt* is available in its `dt` attribute. The timer instance itself stays alive due to Python's scoping rules. -### ``getattrrec``, ``setattrrec``: access underlying data in an onion of wrappers +### `format_human_time`: seconds to days, hours, minutes, seconds + +**Added in v0.15.1.** + +Convert a duration from seconds (`float` or `int`) to a human-readable string of days, hours, minutes and seconds. + +```python +assert format_human_time(30) == "30 seconds" +assert format_human_time(90) == "01:30" # mm:ss +assert format_human_time(3690) == "01:01:30" # hh:mm:ss +assert format_human_time(86400 + 3690) == "1 day 01:01:30" +assert format_human_time(2 * 86400 + 3690) == "2 days 01:01:30" +``` + + +### `ETAEstimator`: estimate the time of completion of a long-running task + +**Added in v0.15.1.** + +Simple but useful: + +```python +n = 1000 +est = ETAEstimator(total=n, keep_last=10) +for k in range(n): + print(f"Processing item {k + 1} out of {n}, {est.formatted_eta}") + ... # do something + est.tick() +``` + +The ETA estimate is automatically formatted using `format_human_time` (see above) to maximize readability. + + +### `getattrrec`, `setattrrec`: access underlying data in an onion of wrappers ```python from unpythonic import getattrrec, setattrrec @@ -3756,17 +5240,17 @@ assert getattrrec(w, "x") == 23 ``` -### ``arities``, ``kwargs``, ``resolve_bindings``: Function signature inspection utilities - -**Added in v0.14.2**: `resolve_bindings`. *Get the parameter bindings a given callable would establish if it was called with the given args and kwargs. This is mainly of interest for implementing memoizers, since this allows them to see (e.g.) `f(1)` and `f(a=1)` as the same thing for `def f(a): pass`.* +### `arities`, `kwargs`, `resolve_bindings`: Function signature inspection utilities **Changed in v0.15.0.** *Now `resolve_bindings` is a thin wrapper on top of `inspect.Signature.bind`, which was added in Python 3.5. In `unpythonic` 0.14.2 and 0.14.3, we used to have our own implementation of the parameter binding algorithm (that ran also on Python 3.4), but it is no longer needed, since now we support only Python 3.6 and later. Now `resolve_bindings` returns an `inspect.BoundArguments` object.* *Now `tuplify_bindings` accepts an `inspect.BoundArguments` object instead of its previous input format. The function is only ever intended to be used to postprocess the output of `resolve_bindings`, so this change shouldn't affect your own code.* -Convenience functions providing an easy-to-use API for inspecting a function's signature. The heavy lifting is done by ``inspect``. +**Added in v0.14.2**: `resolve_bindings`. *Get the parameter bindings a given callable would establish if it was called with the given args and kwargs. This is mainly of interest for implementing memoizers, since this allows them to see (e.g.) `f(1)` and `f(a=1)` as the same thing for `def f(a): pass`. Thanks to Graham Dumpleton, the author of the [`wrapt`](https://pypi.org/project/wrapt/) library, for [noticing and documenting this gotcha](https://wrapt.readthedocs.io/en/latest/decorators.html#processing-function-arguments).* + +These are convenience functions providing an easy-to-use API for inspecting a function's signature. The heavy lifting is done by `inspect`. -Methods on objects and classes are treated specially, so that the reported arity matches what the programmer actually needs to supply when calling the method (i.e., implicit ``self`` and ``cls`` are ignored). +Methods on objects and classes are treated specially, so that the reported arity matches what the programmer actually needs to supply when calling the method (i.e., implicit `self` and `cls` are ignored). ```python from unpythonic import (arities, arity_includes, UnknownArity, @@ -3821,16 +5305,16 @@ assert tuple(resolve_bindings(f, 1, c=3, b=2).items()) == (("a", 1), ("b", 2), ( assert tuple(resolve_bindings(f, c=3, b=2, a=1).items()) == (("a", 1), ("b", 2), ("c", 3)) ``` -We special-case the builtin functions that either fail to return any arity (are uninspectable) or report incorrect arity information, so that also their arities are reported correctly. Note we **do not** special-case the *methods* of any builtin classes, so e.g. ``list.append`` remains uninspectable. This limitation might or might not be lifted in a future version. +We special-case the builtin functions that either fail to return any arity (are uninspectable) or report incorrect arity information, so that also their arities are reported correctly. Note we **do not** special-case the *methods* of any builtin classes, so e.g. `list.append` remains uninspectable. This limitation might or might not be lifted in a future version. -If the arity cannot be inspected, and the function is not one of the special-cased builtins, the ``UnknownArity`` exception is raised. +If the arity cannot be inspected, and the function is not one of the special-cased builtins, the `UnknownArity` exception is raised. -These functions are internally used in various places in unpythonic, particularly ``curry``, ``fix``, and ``@generic``. The ``let`` and FP looping constructs also use these to emit a meaningful error message if the signature of user-provided function does not match what is expected. +Up to v0.14.3, various places in `unpythonic` used to internally use `arities`; particularly `curry`, `fix`, and `@generic`. As of v0.15.0, we have started to prefer `resolve_bindings`, because often what matters are the parameter bindings established, and performing the binding covers all possible ways to pass arguments. The `let` and FP looping constructs still use `arities` to emit a meaningful error message if the signature of user-provided function does not match what is expected. -Inspired by various Racket functions such as ``(arity-includes?)`` and ``(procedure-keywords)``. +Inspired by various Racket functions such as `(arity-includes?)` and `(procedure-keywords)`. -### ``Popper``: a pop-while iterator +### `Popper`: a pop-while iterator Consider this highly artificial example: @@ -3846,7 +5330,7 @@ assert inp == deque([]) assert out == list(range(5)) ``` -``Popper`` condenses the ``while`` and ``pop`` into a ``for``, while allowing the loop body to mutate the input iterable in arbitrary ways (we never actually ``iter()`` it): +`Popper` condenses the `while` and `pop` into a `for`, while allowing the loop body to mutate the input iterable in arbitrary ways (we never actually `iter()` it): ```python from collections import deque @@ -3869,7 +5353,7 @@ assert inp == deque([]) assert out == [0, 10, 1, 11, 2, 12] ``` -``Popper`` comboes with other iterable utilities, such as ``window``: +`Popper` comboes with other iterable utilities, such as `window`: ```python from collections import deque @@ -3885,47 +5369,90 @@ assert inp == deque([]) assert out == [(0, 1), (1, 2), (2, 10), (10, 11), (11, 12)] ``` -(Although ``window`` invokes ``iter()`` on the ``Popper``, this works because the ``Popper`` never invokes ``iter()`` on the underlying container. Any mutations to the input container performed by the loop body will be understood by ``Popper`` and thus also seen by the ``window``. The first ``n`` elements, though, are read before the loop body gets control, because the window needs them to initialize itself.) +Although `window` invokes `iter()` on the `Popper` instance, this works because the `Popper` never invokes `iter()` on the underlying container. Any mutations to the input container performed by the loop body will be understood by `Popper` and thus also seen by the `window`. The first `n` elements, though, are read before the loop body gets control, because the window needs them to initialize itself. -One possible real use case for ``Popper`` is to split sequences of items, stored as lists in a deque, into shorter sequences where some condition is contiguously ``True`` or ``False``. When the condition changes state, just commit the current subsequence, and push the rest of that input sequence (still requiring analysis) back to the input deque, to be dealt with later. +One possible real use case for `Popper` is to split sequences of items, stored as lists in a deque, into shorter sequences where some condition is contiguously `True` or `False`. When the condition changes state, just commit the current subsequence, and push the rest of that input sequence (still requiring analysis) back to the input deque, to be dealt with later. -The argument to ``Popper`` (here ``lst``) contains the **remaining** items. Each iteration pops an element **from the left**. The loop terminates when ``lst`` is empty. +The argument to `Popper` contains the **remaining** items. Each iteration pops an element **from the left**. The loop terminates when, at the start of an iteration, there are no more items remaining. -The input container must support either ``popleft()`` or ``pop(0)``. This is fully duck-typed. At least ``collections.deque`` and any ``collections.abc.MutableSequence`` (including ``list``) are fine. +The input container must support either `popleft()` or `pop(0)`. This is fully duck-typed. At least `collections.deque` and any [`collections.abc.MutableSequence`](https://docs.python.org/3/library/collections.abc.html) (including `list`) are fine. -Per-iteration efficiency is O(1) for ``collections.deque``, and O(n) for a ``list``. +Per-iteration efficiency is O(1) for `collections.deque`, and O(n) for a `list`. Named after [Karl Popper](https://en.wikipedia.org/wiki/Karl_Popper). -### ``ulp``: unit in last place +### `environ_override`: temporarily override environment variables -**Added in v0.14.2.** +**Added in v2.1.0.** -Given a floating point number `x`, return the value of the *unit in the last place* (the "least significant bit"). This is the local size of a "tick", i.e. the difference between `x` and the next larger float. At `x = 1.0`, this is the [machine epsilon](https://en.wikipedia.org/wiki/Machine_epsilon), by definition of the machine epsilon. +Context manager to temporarily override OS environment variables within a `with` block, restoring the previous state on exit. If a variable was unset before entry, it is removed again on exit. -The float format is [IEEE-754](https://en.wikipedia.org/wiki/IEEE_754), i.e. standard Python `float`. +Thread-safe: concurrent overrides from different threads are serialised by a module-level `RLock`, so only one set of overrides is active at a time. Same-thread nesting is supported (the lock is reentrant). -This is just a small convenience function that is for some reason missing from the `math` standard library. +```python +import os +from unpythonic import environ_override + +os.environ["MY_VAR"] = "original" +with environ_override(MY_VAR="temporary", OTHER="added"): + print(os.environ["MY_VAR"]) # "temporary" + print(os.environ["OTHER"]) # "added" +print(os.environ["MY_VAR"]) # "original" +print("OTHER" in os.environ) # False +``` + +The function lives in `unpythonic.environ` as `override`; at the top level it is re-exported as `environ_override`. + + +### `maybe_open`: open a file or use a fallback stream + +**Added in v2.1.0.** + +Context manager that opens a file when given a path, or yields a fallback stream when given `None`. This lets callers always use `with` syntax regardless of whether the target is a file or a standard stream. ```python -from unpythonic import ulp +import sys +from unpythonic import maybe_open -# in IEEE-754, exponent changes at integer powers of two -print([ulp(x) for x in (0.25, 0.5, 1.0, 2.0, 4.0)]) -# --> [5.551115123125783e-17, -# 1.1102230246251565e-16, -# 2.220446049250313e-16, # x = 1.0, so this is sys.float_info.epsilon -# 4.440892098500626e-16, -# 8.881784197001252e-16] -print(ulp(1e10)) -# --> 1.9073486328125e-06 -print(ulp(1e100)) -# --> 1.942668892225729e+84 -print(ulp(2**52)) -# --> 1.0 # yes, exactly 1 +def process(filename=None): + with maybe_open(filename, "r", sys.stdin) as f: + for line in f: + print(line, end="") + +process("data.txt") # reads from file +process() # reads from stdin ``` -When `x` is a round number in base-10, the ULP is not, because the usual kind of floats use base-2. -For more reading, see [David Goldberg (1991): What every computer scientist should know about floating-point arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html), or for a [tl;dr](http://catplanet.org/tldr-cat-meme/) version, [the floating point guide](https://floating-point-gui.de/). +### `redirect_stdin`: feed `sys.stdin` from a stream + +**Added in v2.2.0.** + +Context manager that feeds `sys.stdin` from a given stream — the third sibling of [`contextlib.redirect_stdout`](https://docs.python.org/3/library/contextlib.html#contextlib.redirect_stdout) (Python 3.4) and [`contextlib.redirect_stderr`](https://docs.python.org/3/library/contextlib.html#contextlib.redirect_stderr) (Python 3.5). The standard library ships those two but not this one; this fills the gap. Subclasses `contextlib._RedirectStream` so behavior matches the stdlib siblings exactly, including the per-instance stack that supports nested re-entry on the same instance. + +```python +from io import StringIO +from unpythonic import redirect_stdin + +with redirect_stdin(StringIO("42\n")): + value = input() # reads "42" +``` + +Like its stdlib siblings, this redirects the global `sys.stdin` and is **not** safe under concurrent use from multiple threads — parallel redirects from different threads will stomp on each other. For test code (the typical use case), single-threaded use is the norm. + + +### `UnionFilter`: OR-combine logging filters + +**Added in v2.1.0.** + +A `logging.Filter` that matches a log record if *any* of its sub-filters match. The standard library provides `logging.Filter` for a single logger-name prefix, but no OR combinator. `UnionFilter` fills the gap. + +```python +import logging +from unpythonic import UnionFilter + +for handler in logging.root.handlers: + handler.addFilter(UnionFilter(logging.Filter("myapp.core"), + logging.Filter("myapp.io"))) +``` diff --git a/doc/macros.md b/doc/macros.md index 61f7a347..9f41d357 100644 --- a/doc/macros.md +++ b/doc/macros.md @@ -7,18 +7,19 @@ - [REPL server](repl.md) - [Troubleshooting](troubleshooting.md) - [Design notes](design-notes.md) +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) -# Language extensions using ``unpythonic.syntax`` +# Language extensions using `unpythonic.syntax` -Our extensions to the Python language are built on [``mcpyrate``](https://github.com/Technologicat/mcpyrate), from the PyPI package [``mcpyrate``](https://pypi.org/project/mcpyrate/). +Our extensions to the Python language are built on [`mcpyrate`](https://github.com/Technologicat/mcpyrate), from the PyPI package [`mcpyrate`](https://pypi.org/project/mcpyrate/). -Because in Python macro expansion occurs *at import time*, Python programs whose main module uses macros, such as [our unit tests that contain usage examples](../unpythonic/syntax/test/), cannot be run directly. Instead, run them via `macropython`, included in `mcpyrate`. +Because in Python macro expansion occurs *at import time*, Python programs whose main module uses macros, such as [our unit tests that contain usage examples](../unpythonic/syntax/tests/), cannot be run directly by `python3`. Instead, run them via the `macropython` bootstrapper, included in `mcpyrate`. **Our macros expect a from-import style** for detecting uses of `unpythonic` constructs, *even when those constructs are regular functions*. For example, the function `curry` is detected from its bare name. So if you intend to use these macros, then, for regular imports from `unpythonic`, use `from unpythonic import ...` and avoid renaming (`as`). -*This document doubles as the API reference, but despite maintenance on a best-effort basis, may occasionally be out of date at places. In case of conflicts in documentation, believe the unit tests first; specifically the code, not necessarily the comments. Everything else (comments, docstrings and this guide) should agree with the unit tests. So if something fails to work as advertised, check what the tests say - and optionally file an issue on GitHub so that the documentation can be fixed.* +*This document doubles as the API reference, but despite maintenance on a best-effort basis, may occasionally be out of date at places. In case of conflicts in documentation, believe the unit tests first; specifically the code, not necessarily the comments. Everything else (comments, docstrings and this guide) should agree with the unit tests. So if something fails to work as advertised, check what the tests do - and optionally file an issue on GitHub so that the documentation can be fixed.* **Changed in v0.15.0.** *To run macro-enabled programs, use the [`macropython`](https://github.com/Technologicat/mcpyrate/blob/master/doc/repl.md#macropython-the-universal-bootstrapper) bootstrapper from [`mcpyrate`](https://github.com/Technologicat/mcpyrate).* @@ -28,50 +29,55 @@ Because in Python macro expansion occurs *at import time*, Python programs whose ### Features [**Bindings**](#bindings) -- [``let``, ``letseq``, ``letrec`` as macros](#let-letseq-letrec-as-macros); proper lexical scoping, no boilerplate. -- [``dlet``, ``dletseq``, ``dletrec``, ``blet``, ``bletseq``, ``bletrec``: decorator versions](#dlet-dletseq-dletrec-blet-bletseq-bletrec-decorator-versions) -- [``let_syntax``, ``abbrev``: syntactic local bindings](#let_syntax-abbrev-syntactic-local-bindings); splice code at macro expansion time. -- [Bonus: barebones ``let``](#bonus-barebones-let): pure AST transformation of ``let`` into a ``lambda``. +- [`let`, `letseq`, `letrec` as macros](#let-letseq-letrec-as-macros); proper lexical scoping, no boilerplate. +- [`dlet`, `dletseq`, `dletrec`, `blet`, `bletseq`, `bletrec`: decorator versions](#dlet-dletseq-dletrec-blet-bletseq-bletrec-decorator-versions) +- [Caution on name resolution and scoping](#caution-on-name-resolution-and-scoping) +- [`let_syntax`, `abbrev`: syntactic local bindings](#let_syntax-abbrev-syntactic-local-bindings); splice code at macro expansion time. +- [Bonus: barebones `let`](#bonus-barebones-let): pure AST transformation of `let` into a `lambda`. [**Sequencing**](#sequencing) -- [``do`` as a macro: stuff imperative code into an expression, *with style*](#do-as-a-macro-stuff-imperative-code-into-an-expression-with-style) +- [`do` as a macro: stuff imperative code into an expression, *with style*](#do-as-a-macro-stuff-imperative-code-into-an-expression-with-style) [**Tools for lambdas**](#tools-for-lambdas) -- [``multilambda``: supercharge your lambdas](#multilambda-supercharge-your-lambdas); multiple expressions, local variables. -- [``namedlambda``: auto-name your lambdas](#namedlambda-auto-name-your-lambdas) by assignment. -- [``f``: underscore notation (quick lambdas) for Python](#f-underscore-notation-quick-lambdas-for-python) -- [``quicklambda``: expand quick lambdas first](#quicklambda-expand-quick-lambdas-first) -- [``envify``: make formal parameters live in an unpythonic ``env``](#envify-make-formal-parameters-live-in-an-unpythonic-env) +- [`multilambda`: supercharge your lambdas](#multilambda-supercharge-your-lambdas); multiple expressions, local variables. +- [`namedlambda`: auto-name your lambdas](#namedlambda-auto-name-your-lambdas) by assignment. +- [`fn`: underscore notation (quick lambdas) for Python](#f-underscore-notation-quick-lambdas-for-python) +- [`quicklambda`: expand quick lambdas first](#quicklambda-expand-quick-lambdas-first) +- [`envify`: make formal parameters live in an unpythonic `env`](#envify-make-formal-parameters-live-in-an-unpythonic-env) [**Language features**](#language-features) -- [``autocurry``: automatic currying for Python](#autocurry-automatic-currying-for-python) -- [``lazify``: call-by-need for Python](#lazify-call-by-need-for-python) - - [``lazy[]`` and ``lazyrec[]`` macros](#lazy-and-lazyrec-macros) +- [`autocurry`: automatic currying for Python](#autocurry-automatic-currying-for-python) +- [`lazify`: call-by-need for Python](#lazify-call-by-need-for-python) + - [`lazy[]` and `lazyrec[]` macros](#lazy-and-lazyrec-macros) - [Forcing promises manually](#forcing-promises-manually) - [Binding constructs and auto-lazification](#binding-constructs-and-auto-lazification) - [Note about TCO](#note-about-tco) -- [``tco``: automatic tail call optimization for Python](#tco-automatic-tail-call-optimization-for-python) +- [`tco`: automatic tail call optimization for Python](#tco-automatic-tail-call-optimization-for-python) - [TCO and continuations](#tco-and-continuations) -- [``continuations``: call/cc for Python](#continuations-callcc-for-python) +- [`continuations`: call/cc for Python](#continuations-callcc-for-python) - [General remarks on continuations](#general-remarks-on-continuations) - - [Differences between ``call/cc`` and certain other language features](#differences-between-callcc-and-certain-other-language-features) (generators, exceptions) - - [``call_cc`` API reference](#call_cc-api-reference) + - [Topology of continuations: how the wiring works](#topology-of-continuations-how-the-wiring-works) + - [Scoping of locals in continuations](#scoping-of-locals-in-continuations) + - [Differences between `call/cc` and certain other language features](#differences-between-callcc-and-certain-other-language-features) (generators, exceptions) + - [`call_cc` API reference](#call_cc-api-reference) - [Combo notes](#combo-notes) - [Continuations as an escape mechanism](#continuations-as-an-escape-mechanism) + - [Multi-shot generators with `@multishot` and `myield`](#multi-shot-generators-with-multishot-and-myield) - [What can be used as a continuation?](#what-can-be-used-as-a-continuation) - - [This isn't ``call/cc``!](#this-isnt-callcc) + - [This isn't `call/cc`!](#this-isnt-callcc) - [Why this syntax?](#why-this-syntax) -- [``prefix``: prefix function call syntax for Python](#prefix-prefix-function-call-syntax-for-python) -- [``autoreturn``: implicit ``return`` in tail position](#autoreturn-implicit-return-in-tail-position), like in Lisps. -- [``forall``: nondeterministic evaluation](#forall-nondeterministic-evaluation) with monadic do-notation for Python. +- [`prefix`: prefix function call syntax for Python](#prefix-prefix-function-call-syntax-for-python) +- [`autoreturn`: implicit `return` in tail position](#autoreturn-implicit-return-in-tail-position), like in Lisps. +- [`monadic_do`: do-notation for any monad](#monadic_do-do-notation-for-any-monad), over [unpythonic's classical monad zoo](features.md#monads). +- [`forall`: nondeterministic evaluation](#forall-nondeterministic-evaluation) with monadic do-notation for Python. [**Convenience features**](#convenience-features) -- [``cond``: the missing ``elif`` for ``a if p else b``](#cond-the-missing-elif-for-a-if-p-else-b) -- [``aif``: anaphoric if](#aif-anaphoric-if), the test result is ``it``. -- [``autoref``: implicitly reference attributes of an object](#autoref-implicitly-reference-attributes-of-an-object) +- [`cond`: the missing `elif` for `a if p else b`](#cond-the-missing-elif-for-a-if-p-else-b) +- [`aif`: anaphoric if](#aif-anaphoric-if), the test result is `it`. +- [`autoref`: implicitly reference attributes of an object](#autoref-implicitly-reference-attributes-of-an-object) [**Testing and debugging**](#testing-and-debugging) -- [``unpythonic.test.fixtures``: a test framework for macro-enabled Python](#unpythonic-test-fixtures-a-test-framework-for-macro-enabled-python) +- [`unpythonic.test.fixtures`: a test framework for macro-enabled Python](#unpythonic-test-fixtures-a-test-framework-for-macro-enabled-python) - [Overview](#overview) - [Testing syntax quick reference](#testing-syntax-quick-reference) - [Expansion order](#expansion-order) @@ -82,10 +88,10 @@ Because in Python macro expansion occurs *at import time*, Python programs whose - [Advanced: building a custom test framework](#advanced-building-a-custom-test-framework) - [Why another test framework?](#why-another-test-framework) - [Etymology and roots](#etymology-and-roots) -- [``dbg``: debug-print expressions with source code](#dbg-debug-print-expressions-with-source-code) +- [`dbg`: debug-print expressions with source code](#dbg-debug-print-expressions-with-source-code) [**Other**](#other) -- [``nb``: silly ultralight math notebook](#nb-silly-ultralight-math-notebook) +- [`nb`: silly ultralight math notebook](#nb-silly-ultralight-math-notebook) [**Meta**](#meta) - [The xmas tree combo](#the-xmas-tree-combo): notes on the macros working together. @@ -96,65 +102,82 @@ Because in Python macro expansion occurs *at import time*, Python programs whose Macros that introduce new ways to bind identifiers. -### ``let``, ``letseq``, ``letrec`` as macros +### `let`, `letseq`, `letrec` as macros -**Changed in v0.15.0.** *Added support for env-assignment syntax in the bindings subform. For consistency with other env-assignments, this is now the preferred syntax to establish let bindings. Additionally, the old lispy syntax now accepts also brackets, for consistency with the use of brackets for macro invocations.* +**Changed in v0.15.3.** *Added support for the walrus operator `:=` for env-assignment. This is the new preferred syntax to establish let-bindings. All old syntaxes are still supported for backward compatibility.* -Properly lexically scoped ``let`` constructs, no boilerplate: +**Changed in v0.15.0.** *Added support for env-assignment syntax in the bindings subform. For consistency with other env-assignments, this is now the preferred syntax to establish let-bindings. Additionally, the old lispy syntax now accepts also brackets, for consistency with the use of brackets for macro invocations.* + +These macros provide properly lexically scoped `let` constructs, no boilerplate: ```python from unpythonic.syntax import macros, let, letseq, letrec -let[x << 17, # parallel binding, i.e. bindings don't see each other - y << 23][ +let[x := 17, # parallel binding, i.e. bindings don't see each other + y := 23][ print(x, y)] -letseq[x << 1, # sequential binding, i.e. Scheme/Racket let* - y << x+1][ +letseq[x := 1, # sequential binding, i.e. Scheme/Racket let* + y := x + 1][ print(x, y)] -letrec[evenp << (lambda x: (x == 0) or oddp(x - 1)), # mutually recursive binding, sequentially evaluated - oddp << (lambda x: (x != 0) and evenp(x - 1))][ +letrec[evenp := (lambda x: (x == 0) or oddp(x - 1)), # mutually recursive binding, sequentially evaluated + oddp := (lambda x: (x != 0) and evenp(x - 1))][ print(evenp(42))] ``` Even with just one binding, the syntax remains the same: ```python -let[x << 21][2 * x] +let[x := 21][2 * x] ``` There must be at least one binding; `let[][...]` is a syntax error, since Python's parser rejects an empty subscript slice. -Bindings are established using the `unpythonic` *env-assignment* syntax, ``name << value``. The let bindings can be rebound in the body with the same env-assignment syntax, e.g. ``x << 42``. +Bindings are established using standard assignment expression syntax, `name := value`. The let-bindings can be rebound in the body with the same syntax, e.g. `x := 42`. + +The old `unpythonic` env-assignment syntax, `name << value`, is also supported for backward compatibility. This was the preferred syntax in v0.15.0 to v0.15.2. + +**NOTE**: All let-bindings must be established in the bindings subform. If you absolutely need to do establish more bindings in the body, see the sequencing construct `do[]` and its syntax `local[x := 42]`. + +**NOTE**: Language support for using an assignment expression inside a subscript *without parenthesizing it* was [added in Python 3.10](https://docs.python.org/3/whatsnew/3.10.html#other-language-changes). The syntax accepted when running on Python 3.8 or 3.9 is: + +```python +let[(x := 17), + (y := 23)][ + print(x, y)] +``` + +That is, Python 3.8 and 3.9 require parentheses around each let binding if you use the new `:=` syntax, because syntactically, the bindings subform looks like a subscript. The unit tests use this syntax so that they work on 3.8 and 3.9. But for new code using Python 3.10 or later, it is preferable to omit the parentheses to improve readability. The same syntax for the bindings subform is used by: -- ``let``, ``letseq``, ``letrec`` (expressions) -- ``dlet``, ``dletseq``, ``dletrec``, ``blet``, ``bletseq``, ``bletrec`` (decorators) -- ``let_syntax``, ``abbrev`` (expression mode) +- `let`, `letseq`, `letrec` (expressions) +- `dlet`, `dletseq`, `dletrec`, `blet`, `bletseq`, `bletrec` (decorators) + - As of v0.15.0, it is possible to use `@dlet(...)` instead of `@dlet[...]` in Python 3.8 and earlier. +- `let_syntax`, `abbrev` (expression mode) #### Haskelly let-in, let-where -The following Haskell-inspired, perhaps more pythonic alternate syntaxes are also available: +The following Haskell-inspired, perhaps more pythonic alternative syntaxes are also available: ```python -let[[x << 21, - y << 17, - z << 4] in +let[[x := 21, + y := 17, + z := 4] in x + y + z] let[x + y + z, - where[x << 21, - y << 17, - z << 4]] + where[x := 21, + y := 17, + z := 4]] -let[[x << 21] in 2 * x] -let[2 * x, where[x << 21]] +let[[x := 21] in 2 * x] +let[2 * x, where[x := 21]] ``` -These syntaxes take no macro arguments; both the let-body and the bindings are placed inside the ``...`` in `let[...]`. +These syntaxes take no macro arguments; both the let-body and the bindings are placed inside the `...` in `let[...]`. Note the bindings subform is always enclosed by brackets. @@ -165,20 +188,20 @@ The `where` operator, if used, must be macro-imported. It may only appear at the >The bindings are evaluated first, and then the body is evaluated with the bindings in place. The purpose of the second variant (the *let-where*) is just readability; sometimes it looks clearer to place the body expression first, and only then explain what the symbols in it mean. > ->These syntaxes are valid for all **expression forms** of ``let``, namely: ``let[]``, ``letseq[]``, ``letrec[]``, ``let_syntax[]`` and ``abbrev[]``. The decorator variants (``dlet`` et al., ``blet`` et al.) and the block variants (``with let_syntax``, ``with abbrev``) support only the formats where the bindings subform is given in the macro arguments part, because there the body is in any case placed differently (it's the body of the function being decorated). +>These syntaxes are valid for all **expression forms** of `let`, namely: `let[]`, `letseq[]`, `letrec[]`, `let_syntax[]` and `abbrev[]`. The decorator variants (`dlet` et al., `blet` et al.) and the block variants (`with let_syntax`, `with abbrev`) support only the formats where the bindings subform is given in the macro arguments part, because there the body is in any case placed differently (it's the body of the function being decorated). > ->In the first variant above (the *let-in*), note that even there, the bindings block needs the brackets. This is due to Python's precedence rules; ``in`` binds more strongly than the comma (which makes sense almost everywhere else), so to make the ``in`` refer to all of the bindings, the bindings block must be bracketed. If the ``let`` expander complains your code does not look like a ``let`` form and you have used *let-in*, check your brackets. +>In the first variant above (the *let-in*), note that even there, the bindings block needs the brackets. This is due to Python's precedence rules; `in` binds more strongly than the comma (which makes sense almost everywhere else), so to make the `in` refer to all of the bindings, the bindings block must be bracketed. If the `let` expander complains your code does not look like a `let` form and you have used *let-in*, check your brackets. > ->In the second variant (the *let-where*), note the comma between the body and ``where``; it is compulsory to make the expression into syntactically valid Python. (It's however semi-easyish to remember, since also English requires the comma for a where-expression. It's not only syntactically valid Python, it's also syntactically valid English (at least for mathematicians).) +>In the second variant (the *let-where*), note the comma between the body and `where`; it is compulsory to make the expression into syntactically valid Python. (It's however semi-easyish to remember, since also English requires the comma for a where-expression. It's not only syntactically valid Python, it is also syntactically valid English, at least for mathematicians.)
-#### Alternate syntaxes for the bindings subform +#### Alternative syntaxes for the bindings subform **Changed in v0.15.0.** -Beginning with v0.15.0, the env-assignment syntax presented above is the preferred syntax to establish let bindings, for consistency with other env-assignments. (Let variables live in an `env`, which is created by the `let`.) +Beginning with v0.15.0, the env-assignment syntax presented above is the preferred syntax to establish let-bindings, for consistency with other env-assignments. This reminds that let variables live in an `env`, which is created by the `let` form. -There is also an alternate, lispy notation for the bindings subform, where each name-value pair is given using brackets: +There is also an alternative, lispy notation for the bindings subform, where each name-value pair is given using brackets: ```python let[[x, 42], [y, 9001]][...] @@ -206,7 +229,7 @@ let[(x, 42) in ...] let[..., where(x, 42)] ``` -Even though an expr macro invocation itself is always denoted using brackets, as of `unpythonic` v0.15.0 parentheses can still be used *to pass macro arguments*, hence ``let(...)[...]`` is still accepted. The code that interprets the AST for the let bindings accepts both lists and tuples for each key-value pair, and the top-level container for the bindings subform in a let-in or let-where can be either list or tuple, so whether brackets or parentheses are used does not matter there, either. +Even though an expr macro invocation itself is always denoted using brackets, as of `unpythonic` v0.15.0 parentheses can still be used *to pass macro arguments*, hence `let(...)[...]` is still accepted. The code that interprets the AST for the let-bindings accepts both lists and tuples for each key-value pair, and the top-level container for the bindings subform in a let-in or let-where can be either list or tuple, so whether brackets or parentheses are used does not matter there, either. Still, brackets are now the preferred delimiter, for consistency between the bindings and body subforms. @@ -217,82 +240,84 @@ The issue has been fixed in Python 3.9. If you already only use 3.9 and later, p #### Multiple expressions in body -The `let` constructs can now use a multiple-expression body. The syntax to activate multiple expression mode is an extra set of brackets around the body ([like in `multilambda`](#multilambda-supercharge-your-lambdas)): +The `let` constructs can use a multiple-expression body. The syntax to activate multiple expression mode is an extra set of brackets around the body ([like in `multilambda`](#multilambda-supercharge-your-lambdas)): ```python -let[x << 1, - y << 2][[ # note extra [ - y << x + y, +let[x := 1, + y := 2][[ # note extra [ + y := x + y, print(y)]] -let[[x << 1, - y << 2] in - [y << x + y, # body starts here +let[[x := 1, + y := 2] in + [y := x + y, # body starts here print(y)]] -let[[y << x + y, +let[[y := x + y, print(y)], # body ends here - where[x << 1, - y << 2]] + where[x := 1, + y := 2]] ``` -The let macros implement this by inserting a ``do[...]`` (see below). In a multiple-expression body, also an internal definition context exists for local variables that are not part of the ``let``; see [``do`` for details](#do-as-a-macro-stuff-imperative-code-into-an-expression-with-style). +The let macros implement this by inserting a `do[...]` (see below). In a multiple-expression body, a separate internal definition context exists for local variables that are not part of the `let`; see [the `do` macro for details](#do-as-a-macro-stuff-imperative-code-into-an-expression-with-style). -Only the outermost set of extra brackets is interpreted as a multiple-expression body. The rest are interpreted as usual, as lists. If you need to return a literal list from a ``let`` form with only one body expression, use three sets of brackets: +Only the outermost set of extra brackets is interpreted as a multiple-expression body. The rest are interpreted as usual, as lists. If you need to return a literal list from a `let` form with only one body expression, double the brackets on the *body* part: ```python -let[x << 1, - y << 2][[ +let[x := 1, + y := 2][[ [x, y]]] -let[[x << 1, - y << 2] in +let[[x := 1, + y := 2] in [[x, y]]] let[[[x, y]], - where[x << 1, - y << 2]] + where[x := 1, + y := 2]] ``` -The outermost brackets delimit the ``let`` form, the middle ones activate multiple-expression mode, and the innermost ones denote a list. +The outermost brackets delimit the `let` form itself, the middle ones activate multiple-expression mode, and the innermost ones denote a list. Only brackets are affected; parentheses are interpreted as usual, so returning a literal tuple works as expected: ```python -let[x << 1, - y << 2][ +let[x := 1, + y := 2][ (x, y)] -let[[x << 1, - y << 2] in +let[[x := 1, + y := 2] in (x, y)] let[(x, y), - where[x << 1, - y << 2]] + where[x := 1, + y := 2]] ``` #### Notes -The main difference of the `let` family to Python's own named expressions (a.k.a. walrus operator, added in Python 3.8) is that `x := 42` does not create a scope, but `let[(x, 42)][...]` does. The walrus operator assigns to the name `x` in the scope it appears in, whereas in the `let` expression, the `x` only exists in that expression. +The main difference of the `let` family to Python's own named expressions (a.k.a. the walrus operator, added in Python 3.8) is that `x := 42` does not create a scope, but `let[x := 42][...]` does. The walrus operator assigns to the name `x` in the scope it appears in, whereas in the `let` expression, the `x` only exists in that expression. + +As of v0.15.3, this is somewhat complicated by the fact that now the syntax `x := 42` can be used to rebind let variables. See the unit test examples for `@dlet` above, at the beginning of the `let` section. -``let`` and ``letrec`` expand into the ``unpythonic.lispylet`` constructs, implicitly inserting the necessary boilerplate: the ``lambda e: ...`` wrappers, quoting variable names in definitions, and transforming ``x`` to ``e.x`` for all ``x`` declared in the bindings. Assignment syntax ``x << 42`` transforms to ``e.set('x', 42)``. The implicit environment parameter ``e`` is actually named using a gensym, so lexically outer environments automatically show through. ``letseq`` expands into a chain of nested ``let`` expressions. +`let` and `letrec` expand into the `unpythonic.lispylet` constructs, implicitly inserting the necessary boilerplate: the `lambda e: ...` wrappers, quoting variable names in definitions, and transforming `x` to `e.x` for all `x` declared in the bindings. Assignment syntax `x := 42` transforms to `e.set('x', 42)`. The implicit environment parameter `e` is actually named using a gensym, so lexically outer environments automatically show through. `letseq` expands into a chain of nested `let` expressions. -Nesting utilizes an inside-out macro expansion order: +All the `let` macros respect lexical scope, so this works as expected: ```python -letrec[z << 1][[ +letrec[z := 1][[ print(z), - letrec[z << 2][ + letrec[z := 2][ print(z)]]] ``` -Hence the ``z`` in the inner scope expands to the inner environment's ``z``, which makes the outer expansion leave it alone. (This works by transforming only ``ast.Name`` nodes, stopping recursion when an ``ast.Attribute`` is encountered.) +The `z` in the inner `letrec` expands to the inner environment's `z`, and the `z` in the outer `letrec` to the outer environment's `z`. -### ``dlet``, ``dletseq``, ``dletrec``, ``blet``, ``bletseq``, ``bletrec``: decorator versions +### `dlet`, `dletseq`, `dletrec`, `blet`, `bletseq`, `bletrec`: decorator versions -Similar to ``let``, ``letseq``, ``letrec``, these sugar the corresponding ``unpythonic.lispylet`` constructs, with the ``dletseq`` and ``bletseq`` constructs existing only as macros (expanding to nested ``dlet`` or ``blet``, respectively). +Similar to `let`, `letseq`, `letrec`, these macros sugar the corresponding `unpythonic.lispylet` constructs, with the `dletseq` and `bletseq` constructs existing only as macros. They expand to nested `dlet` or `blet`, respectively. Lexical scoping is respected; each environment is internally named using a gensym. Nesting is allowed. @@ -301,83 +326,109 @@ Examples: ```python from unpythonic.syntax import macros, dlet, dletseq, dletrec, blet, bletseq, bletrec -@dlet[x << 0] # up to Python 3.8, use `@dlet(x << 0)` instead +@dlet[x := 0] # up to Python 3.8, use `@dlet(x := 0)` instead (decorator subscripting was added in 3.9) def count(): - x << x + 1 + (x := x + 1) # update `x` in let env return x assert count() == 1 assert count() == 2 -@dletrec[evenp << (lambda x: (x == 0) or oddp(x - 1)), - oddp << (lambda x: (x != 0) and evenp(x - 1))] +@dletrec[evenp := (lambda x: (x == 0) or oddp(x - 1)), + oddp := (lambda x: (x != 0) and evenp(x - 1))] def f(x): return evenp(x) assert f(42) is True assert f(23) is False -@dletseq[x << 1, - x << x + 1, - x << x + 2] +@dletseq[x := 1, + x := x + 1, + x := x + 2] def g(a): return a + x assert g(10) == 14 # block versions: the def takes no arguments, runs immediately, and is replaced by the return value. -@blet[x << 21] +@blet[x := 21] def result(): return 2*x assert result == 42 -@bletrec[evenp << (lambda x: (x == 0) or oddp(x - 1)), - oddp << (lambda x: (x != 0) and evenp(x - 1))] +@bletrec[evenp := (lambda x: (x == 0) or oddp(x - 1)), + oddp := (lambda x: (x != 0) and evenp(x - 1))] def result(): return evenp(42) assert result is True -@bletseq[x << 1, - x << x + 1, - x << x + 2] +@bletseq[x := 1, + x := x + 1, + x := x + 2] def result(): return x assert result == 4 ``` -**CAUTION**: assignment to the let environment uses the syntax ``name << value``, as always with ``unpythonic`` environments. The standard Python syntax ``name = value`` creates a local variable, as usual - *shadowing any variable with the same name from the ``let``*. +**CAUTION**: assignment to the let environment uses the assignment expression syntax `name := value`. The assignment statement `name = value` creates a local variable, as usual - *shadowing any variable with the same name from the `let`*. -The write of a ``name << value`` always occurs to the lexically innermost environment (as seen from the write site) that has that ``name``. If no lexically surrounding environment has that ``name``, *then* the expression remains untransformed, and means a left-shift (if ``name`` happens to be otherwise defined). +The write of a `name := value` always occurs to the lexically innermost environment (as seen from the write site) that has that `name`. If no lexically surrounding environment has that `name`, *then* the expression remains untransformed, and means binding a new lexical variable in the nearest enclosing scope, as per Python's standard rules. -**CAUTION**: formal parameters of a function definition, local variables, and any names declared as ``global`` or ``nonlocal`` in a given lexical scope shadow names from the ``let`` environment. Mostly, this applies *to the entirety of that lexical scope*. This is modeled after Python's standard scoping rules. +**CAUTION**: formal parameters of a function definition, local variables, and any names declared as `global` or `nonlocal` in a given lexical scope shadow names from an enclosing `let` environment. Mostly, this applies *to the entirety of that lexical scope*. This is modeled after Python's standard scoping rules. -As an exception to the rule, for the purposes of the scope analysis performed by ``unpythonic.syntax``, creations and deletions *of lexical local variables* take effect from the next statement, and remain in effect for the **lexically** remaining part of the current scope. This allows ``x = ...`` to see the old bindings on the RHS, as well as allows the client code to restore access to a surrounding env's ``x`` (by deleting a local ``x`` shadowing it) when desired. +As an exception to the rule, for the purposes of the scope analysis performed by `unpythonic.syntax`, creations and deletions *of lexical local variables* take effect from the next statement, and remain in effect for the **lexically** remaining part of the current scope. This allows `x = ...` to see the old bindings on the RHS, as well as allows the client code to restore access to a surrounding env's `x` (by deleting a local `x` shadowing it) when desired. -To clarify, here's a sampling from the unit tests: +To clarify, here is a sampling from [the unit tests](../unpythonic/syntax/tests/test_letdo.py): ```python -@dlet[x << "the env x"] +@dlet[x := "the env x"] def f(): - return x + return x # No lexical variable `x` exists; this refers to the env `x`. assert f() == "the env x" -@dlet[x << "the env x"] +@dlet[x := "the env x"] def f(): - x = "the local x" + x = "the local x" # The lexical variable shadows the env `x`. return x assert f() == "the local x" -@dlet[x << "the env x"] +@dlet[x := "the env x"] def f(): return x - x = "the unused local x" + x = "the unused local x" # This appears *lexically after* the read access on the previous line. assert f() == "the env x" +@dlet[x := "the env x"] +def test15(): + def inner(): + (x := "updated env x") # noqa: F841, this writes to the let env since there is no `x` in an intervening scope, according to Python's standard rules. + inner() + return x +assert test15() == "updated env x" + +@dlet[x := "the env x"] +def test16(): + def inner(): + x = "the inner x" # noqa: F841, unused on purpose, for testing. An assignment *statement* does NOT write to the let env. + inner() + return x +assert test16() == "the env x" + +@dlet[x := "the env x"] +def test17(): + x = "the local x" # This lexical variable shadows the env x. + def inner(): + # The env x is shadowed. Since we don't say `nonlocal x`, this creates a new lexical variable scoped to `inner`. + (x := "the inner x") # noqa: F841, unused on purpose, for testing. + inner() + return x +assert test17() == "the local x" + x = "the global x" -@dlet[x << "the env x"] +@dlet[x := "the env x"] def f(): global x return x assert f() == "the global x" -@dlet[x << "the env x"] +@dlet[x := "the env x"] def f(): x = "the local x" del x # deleting a local, ok! @@ -386,7 +437,7 @@ assert f() == "the env x" try: x = "the global x" - @dlet[x << "the env x"] + @dlet[x := "the env x"] def f(): global x del x # ignored by unpythonic's scope analysis, deletion of globals is too dynamic @@ -399,7 +450,50 @@ else: ``` -### ``let_syntax``, ``abbrev``: syntactic local bindings +### Caution on name resolution and scoping + +The name resolution behavior described above **does not fully make sense**, because to define things this way is to conflate static (lexical) and dynamic (run-time) concepts. This feature unfortunately got built before I understood the matter clearly. + +Python itself performs name resolution purely lexically, which is arguably the right thing to do. In any given lexical scope, an identifier such as `x` always refers to the same variable. Whether that variable has been initialized, or has already been deleted, is another matter, which has to wait until run time - but `del x` will **not** cause the identifier `x` to point to a different variable for the remainder of the same scope, like `delete[x]` **does** in the body of an `unpythonic` `let[]` or `do[]`. + +#### Aside: Names and variables + +To be technically correct, in Python, an identifier `x` refers to a *name*, not to a "variable". Python, like Lisp, has [*names and values*](https://nedbatchelder.com/text/names.html). + +Roughly, an *identifier* is a certain kind of token in the source code text - something that everyday English calls a "name". However, in programming, a *name* is technically the *key* component of a key-value pair that is stored in a particular *environment*. + +Very roughly speaking, an *environment* is just a place to store such pairs, for the purposes of "the variables subsystem" of the language. There are important details, such as that each *activation* of a function (think: "a particular call of the function") will create a new environment instance, to hold the local variables of that activation; this detail allows [lexical closures](https://en.wikipedia.org/wiki/Closure_(computer_programming)) to work. The piece of bookkeeping for this is termed an *activation record*. But the important point here is, an environment stores name-value pairs. + +An identifier *refers to* a name. Scoping rules concern themselves with the details of mapping identifiers to names. In *lexical scoping* (like in Python), the position of the identifier in the source code text determines the search order of environments for the target name, when resolving a particular instance of an identifier in the source code text. Python uses the LEGB ordering (local, enclosing, global, builtin). + +Finally, *values* are the run-time things names point to. They are the *value* component of the key-value pair. + +In this simple example: + +```python +def outer(): + x = 17 + def inner(): + x = 23 +``` + + - The piece of source code text `x` is an *identifier*. + - *The outer `x`* and *the inner `x`* are *names*, both of which have the textual representation `x`. + - *Which one of these the identifier `x` refers to depends on where it appears.* + - The integers `17` and `23` are *values*. + +Note that classically, names have no type; values do. + +Nowadays, a name may have a type annotation, which reminds the programmer about the type of *value* that is safe to bind to that particular name. In other words, the code that defines that name (e.g. as a function parameter) promises (in the sense of a contract) that the code knows how to behave if a value of that type is bound to that name (e.g. by passing such a value as a function argument that will be bound to that name). + +Here *type* may be a concrete [nominal type](https://en.wikipedia.org/wiki/Nominal_type_system) such as `int`, or for example, it may represent a particular interface (such as the types in [`collections.abc`](https://docs.python.org/3/library/collections.abc.html)), or it may allow multiple mutually exclusive options (a *union*). + +By default, Python treats type annotations as a form of comments; to actually statically type-check Python, [Mypy](http://mypy-lang.org/) can be used. + +Compare the *name*/*value* concept to the concept of a *variable* in the classical sense, such as in C, or `cdef` in Cython. In such *low-level* [HLLs](https://en.wikipedia.org/wiki/High-level_programming_language), a *variable* is a named, fixed memory location, with a static data type determining how to interpret the bits at that memory location. The contents of the memory location can be changed, hence "variable" is an apt description. + + +### `let_syntax`, `abbrev`: syntactic local bindings **Note v0.15.0.** *Now that we use `mcpyrate` as the macro expander, `let_syntax` and `abbrev` are not really needed. We are keeping them mostly for backwards compatibility, and because they exercise a different feature set in the macro expander, making the existence of these constructs particularly useful for system testing.* @@ -407,9 +501,9 @@ else: *To rename existing macros, you can as-import them. As of `unpythonic` v0.15.0, doing so for `unpythonic.syntax` constructs is not recommended, though, because there is still a lot of old analysis code in the macro implementations that may scan for the original name. This may or may not be fixed in a future release.* -These constructs allow to locally splice code at macro expansion time (it's almost like inlining functions): +These constructs allow to locally splice code at macro expansion time. It is almost like inlining functions. -#### ``let_syntax`` +#### `let_syntax` ```python from unpythonic.syntax import macros, let_syntax, block, expr @@ -418,28 +512,28 @@ def verylongfunctionname(x=1): return x # works as an expr macro -y = let_syntax[f << verylongfunctionname][[ # extra brackets: implicit do in body +y = let_syntax[f := verylongfunctionname][[ # extra brackets: implicit do in body print(f()), f(5)]] assert y == 5 -y = let_syntax[f[a] << verylongfunctionname(2*a)][[ # template with formal parameter "a" +y = let_syntax[f[a] := verylongfunctionname(2*a)][[ # template with formal parameter "a" print(f[2]), f[3]]] assert y == 6 -y = let_syntax[[f << verylongfunctionname] in +y = let_syntax[[f := verylongfunctionname] in [print(f()), f(5)]] y = let_syntax[[print(f()), f(5)], - where[f << verylongfunctionname]] -y = let_syntax[[f[a] << verylongfunctionname(2*a)] in + where[f := verylongfunctionname]] +y = let_syntax[[f[a] := verylongfunctionname(2*a)] in [print(f[2]), f[3]]] y = let_syntax[[print(f[2]), f[3]], - where[f[a] << verylongfunctionname(2*a)]] + where[f[a] := verylongfunctionname(2*a)]] # works as a block macro with let_syntax: @@ -466,28 +560,28 @@ with let_syntax: assert lst == [7, 8, 9]*2 ``` -After macro expansion completes, ``let_syntax`` has zero runtime overhead; it completely disappears in macro expansion. +After macro expansion completes, `let_syntax` has zero runtime overhead; it completely disappears in macro expansion. The `expr` and `block` operators, if used, must be macro-imported. They may only appear in `with expr` and `with block` subforms at the top level of a `with let_syntax` or `with abbrev`. In any invalid position, `expr` and `block` are both considered a syntax error at macro expansion time.
There are two kinds of substitutions: ->*Bare name* and *template*. A bare name substitution has no parameters. A template substitution has positional parameters. (Named parameters, ``*args``, ``**kwargs`` and default values are **not** supported.) +>*Bare name* and *template*. A bare name substitution has no parameters. A template substitution has positional parameters. (Named parameters, `*args`, `**kwargs` and default values are **not** supported.) > ->When used as an expr macro, the formal parameter declaration is placed where it belongs; on the name side (LHS) of the binding. In the above example, ``f[a]`` is a template with a formal parameter ``a``. But when used as a block macro, the formal parameters are declared on the ``block`` or ``expr`` "context manager" due to syntactic limitations of Python. To define a bare name substitution, just use ``with block as ...:`` or ``with expr as ...:`` with no macro arguments. +>When used as an expr macro, the formal parameter declaration is placed where it belongs; on the name side (LHS) of the binding. In the above example, `f[a]` is a template with a formal parameter `a`. But when used as a block macro, the formal parameters are declared on the `block` or `expr` "context manager" due to syntactic limitations of Python. To define a bare name substitution, just use `with block as ...:` or `with expr as ...:` with no macro arguments. > ->In the body of ``let_syntax``, a bare name substitution is invoked by name (just like a variable). A template substitution is invoked like an expr macro. Any instances of the formal parameters of the template get replaced by the argument values from the use site, at macro expansion time. +>In the body of `let_syntax`, a bare name substitution is invoked by name (just like a variable). A template substitution is invoked like an expr macro. Any instances of the formal parameters of the template get replaced by the argument values from the use site, at macro expansion time. > ->Note each instance of the same formal parameter (in the definition) gets a fresh copy of the corresponding argument value. In other words, in the example above, each ``a`` in the body of ``twice`` separately expands to a copy of whatever code was given as the macro argument ``a``. +>Note each instance of the same formal parameter (in the definition) gets a fresh copy of the corresponding argument value. In other words, in the example above, each `a` in the body of `twice` separately expands to a copy of whatever code was given as the macro argument `a`. > ->When used as a block macro, there are furthermore two capture modes: *block of statements*, and *single expression*. (The single expression can be an explicit ``do[]`` if multiple expressions are needed.) When invoking substitutions, keep in mind Python's usual rules regarding where statements or expressions may appear. +>When used as a block macro, there are furthermore two capture modes: *block of statements*, and *single expression*. The single expression can be an explicit `do[]`, if multiple expressions are needed. When invoking substitutions, keep in mind Python's usual rules regarding where statements or expressions may appear. > ->(If you know about Python ASTs, don't worry about the ``ast.Expr`` wrapper needed to place an expression in a statement position; this is handled automatically.) +>(If you know about Python ASTs, do not worry about the `ast.Expr` wrapper needed to place an expression in a statement position; this is handled automatically.)

-**HINT**: If you get a compiler error that some sort of statement was encountered where an expression was expected, check your uses of ``let_syntax``. The most likely reason is that a substitution is trying to splice a block of statements into an expression position. +**HINT**: If you get a compiler error that some sort of statement was encountered where an expression was expected, check your uses of `let_syntax`. The most likely reason is that a substitution is trying to splice a block of statements into an expression position.

Expansion of this macro is a two-step process: @@ -499,44 +593,44 @@ The `expr` and `block` operators, if used, must be macro-imported. They may only > >Within each step, the substitutions are applied **in definition order**: > -> - If the bindings are ``[x << y, y << z]``, then an ``x`` at the use site transforms to ``z``. So does a ``y`` at the use site. -> - But if the bindings are ``[y << z, x << y]``, then an ``x`` at the use site transforms to ``y``, and only an explicit ``y`` at the use site transforms to ``z``. +> - If the bindings are `[x := y, y := z]`, then an `x` at the use site transforms to `z`. So does a `y` at the use site. +> - But if the bindings are `[y := z, x := y]`, then an `x` at the use site transforms to `y`, and only an explicit `y` at the use site transforms to `z`. > >Even in block templates, arguments are always expressions, because invoking a template uses the subscript syntax. But names and calls are expressions, so a previously defined substitution (whether bare name or an invocation of a template) can be passed as an argument just fine. Definition order is then important; consult the rules above.

-Nesting ``let_syntax`` is allowed. Lexical scoping is supported (inner definitions of substitutions shadow outer ones). +Nesting `let_syntax` is allowed. Lexical scoping is respected. Inner definitions of substitutions shadow outer ones. -When used as an expr macro, all bindings are registered first, and then the body is evaluated. When used as a block macro, a new binding (substitution declaration) takes effect from the next statement onward, and remains active for the lexically remaining part of the ``with let_syntax:`` block. +When used as an expr macro, all bindings are registered first, and then the body is evaluated. When used as a block macro, a new binding (substitution declaration) takes effect from the next statement onward, and remains active for the lexically remaining part of the `with let_syntax` block. #### `abbrev` -The ``abbrev`` macro is otherwise exactly like ``let_syntax``, but it expands outside-in. Hence, no lexically scoped nesting, but it has the power to locally rename also macros, because the ``abbrev`` itself expands before any macros invoked in its body. This allows things like: +The `abbrev` macro is otherwise exactly like `let_syntax`, but it expands outside-in. Hence, it has no lexically scoped nesting support, but it has the power to locally rename also macros, because the `abbrev` itself expands before any macros invoked in its body. This allows things like: ```python -abbrev[m << macrowithverylongname][ +abbrev[m := macrowithverylongname][ m[tree1] if m[tree2] else m[tree3]] -abbrev[[m << macrowithverylongname] in +abbrev[[m := macrowithverylongname] in m[tree1] if m[tree2] else m[tree3]] abbrev[m[tree1] if m[tree2] else m[tree3], - where[m << macrowithverylongname]] + where[m := macrowithverylongname]] ``` -which can be useful when writing macros. +which is sometimes useful when writing macros. But using `mcpyrate`, note that you can just as-import a macro if you need to rename it. -**CAUTION**: ``let_syntax`` is essentially a toy macro system within the real macro system. The usual caveats of macro systems apply. Especially, ``let_syntax`` and ``abbrev`` support absolutely no form of hygiene. Be very, very careful to avoid name conflicts. +**CAUTION**: `let_syntax` is essentially a toy macro system within the real macro system. The usual caveats of macro systems apply. Especially, `let_syntax` and `abbrev` support absolutely no form of hygiene. Be very, very careful to avoid name conflicts. -The ``let_syntax`` macro is meant for simple local substitutions where the elimination of repetition can shorten the code and improve its readability, in cases where the final "unrolled" code should be written out at compile time. If you need to do something complex (or indeed save a definition and reuse it somewhere else, non-locally), write a real macro directly in `mcpyrate`. +The `let_syntax` macro is meant for simple local substitutions where the elimination of repetition can shorten the code and improve its readability, in cases where the final "unrolled" code should be written out at compile time. If you need to do something complex (or indeed save a definition and reuse it somewhere else, non-locally), write a real macro directly in `mcpyrate`. -This was inspired by Racket's [``let-syntax``](https://docs.racket-lang.org/reference/let.html) and [``with-syntax``](https://docs.racket-lang.org/reference/stx-patterns.html). +This was inspired by Racket's [`let-syntax`](https://docs.racket-lang.org/reference/let.html) and [`with-syntax`](https://docs.racket-lang.org/reference/stx-patterns.html) forms. -### Bonus: barebones ``let`` +### Bonus: barebones `let` -As a bonus, we provide classical simple ``let`` and ``letseq``, wholly implemented as AST transformations, providing true lexical variables but no assignment support (because in Python, assignment is a statement) or multi-expression body support. Just like in Lisps, this version of ``letseq`` (Scheme/Racket ``let*``) expands into a chain of nested ``let`` expressions, which expand to lambdas. +As a bonus, we provide classical simple `let` and `letseq`, wholly implemented as AST transformations, providing true lexical variables, but no multi-expression body support. Just like in some Lisps, this version of `letseq` (Scheme/[Racket `let*`](https://docs.racket-lang.org/reference/let.html#%28form._%28%28lib._racket%2Fprivate%2Fletstx-scheme..rkt%29._let%2A%29%29)) expands into a chain of nested `let` expressions, which expand to lambdas. -These are provided in the separate module ``unpythonic.syntax.simplelet``, and are not part of the `unpythonic.syntax` macro API. For simplicity, they support only the lispy list syntax in the bindings subform (using brackets, specifically!), and no haskelly syntax at all: +These are provided in the separate module `unpythonic.syntax.simplelet`, and are not part of the `unpythonic.syntax` macro API. For simplicity, they support only the lispy list syntax in the bindings subform (using brackets, specifically!), and no haskelly syntax at all: ```python from unpythonic.syntax.simplelet import macros, let, letseq @@ -547,73 +641,91 @@ letseq[[x, 1], [x, x + 1]][...] letseq[[x, 1]][...] ``` +Starting with Python 3.8, assignment (rebinding) is possible also in these barebones `let` constructs via the walrus operator. For example: + +```python +assert let[[x, 42]][x] == 42 +assert let[[x, 42]][(x := 5)] == 5 +``` + +However, this only works for variables created by the innermost `let` (viewed from the point where the assignment happens), because `nonlocal` is a statement and so cannot be used in expressions. + + ## Sequencing Macros that run multiple expressions, in sequence, in place of one expression. -### ``do`` as a macro: stuff imperative code into an expression, *with style* +**In macro-enabled code, sequence with `do[]` / `do0[]`, not with `begin` / `begin0`.** The +pure-Python layer publishes `begin`, `begin0`, `lazy_begin` and `lazy_begin0` (in +[`unpythonic.seq`](../unpythonic/seq.py)), and the macro layer does not recognize any of them as a +sequencing abstraction — to a macro they are ordinary function calls, so none of the transformations +described below apply inside one. Each of those functions carries this caution in its docstring; it +is repeated here because a reader arriving from the macro side has no reason to open them. + +### `do` as a macro: stuff imperative code into an expression, *with style* + +**Changed in v0.15.3.** *Env-assignments now use the walrus syntax `x := 42`. The old syntax `x << 42` is still supported for backward compatibility.* -We provide an ``expr`` macro wrapper for ``unpythonic.seq.do``, with some extra features. +We provide an `expr` macro wrapper for `unpythonic.do` and `unpythonic.do0`, with some extra features. -This essentially allows writing imperative code in any expression position. For an `if-elif-else` conditional, [see `cond`](#cond-the-missing-elif-for-a-if-p-else-b); for loops, see [the functions in `unpythonic.fploop`](../unpythonic/fploop.py) (esp. `looped`). +This essentially allows writing imperative code in any expression position. For an `if-elif-else` conditional, [see `cond`](#cond-the-missing-elif-for-a-if-p-else-b); for loops, see the functions in the module [`unpythonic.fploop`](../unpythonic/fploop.py) (`looped` and `looped_over`). ```python from unpythonic.syntax import macros, do, local, delete -y = do[local[x << 17], +y = do[local[x := 17], print(x), - x << 23, + x := 23, x] print(y) # --> 23 a = 5 -y = do[local[a << 17], +y = do[local[a := 17], print(a), # --> 17 delete[a], print(a), # --> 5 True] ``` -Local variables are declared and initialized with ``local[var << value]``, where ``var`` is a bare name. To explicitly denote "no value", just use ``None``. ``delete[...]`` allows deleting a ``local[...]`` binding. This uses ``env.pop()`` internally, so a ``delete[...]`` returns the value the deleted local variable had at the time of deletion. (So if you manually use the ``do()`` function in some code without macros, feel free to ``env.pop()`` in a do-item if needed.) +Local variables are declared and initialized with `local[var := value]`, where `var` is a bare name. To explicitly denote "no value", just use `None`. The syntax `delete[...]` allows deleting a `local[...]` binding. This uses `env.pop()` internally, so a `delete[...]` returns the value the deleted local variable had at the time of deletion. (This also means that if you manually use the `do()` function in some code without macros, you can `env.pop(...)` in a do-item if needed.) The `local[]` and `delete[]` declarations may only appear at the top level of a `do[]`, `do0[]`, or implicit `do` (extra bracket syntax, e.g. for the body of a `let` form). In any invalid position, `local[]` and `delete[]` are considered a syntax error at macro expansion time. -A ``local`` declaration comes into effect in the expression following the one where it appears, capturing the declared name as a local variable for the **lexically** remaining part of the ``do``. In a ``local``, the RHS still sees the previous bindings, so this is valid (although maybe not readable): +A `local` declaration comes into effect in the expression following the one where it appears, capturing the declared name as a local variable for the **lexically** remaining part of the `do`. In a `local`, the RHS still sees the previous bindings, so this is valid (although maybe not readable): ```python result = [] -let[lst << []][[result.append(lst), # the let "lst" - local[lst << lst + [1]], # LHS: do "lst", RHS: let "lst" +let[lst := []][[result.append(lst), # the let "lst" + local[lst := lst + [1]], # LHS: do "lst", RHS: let "lst" result.append(lst)]] # the do "lst" assert result == [[], [1]] ``` -Already declared local variables are updated with ``var << value``. Updating variables in lexically outer environments (e.g. a ``let`` surrounding a ``do``) uses the same syntax. +Already declared local variables are updated with `var := value`. Updating variables in lexically outer environments (e.g. a `let` surrounding a `do`) uses the same syntax.

The reason we require local variables to be declared is to allow write access to lexically outer environments. ->Assignments are recognized anywhere inside the ``do``; but note that any ``let`` constructs nested *inside* the ``do``, that define variables of the same name, will (inside the ``let``) shadow those of the ``do`` - as expected of lexical scoping. +>Assignments are recognized anywhere inside the `do`; but note that any `let` constructs nested *inside* the `do`, that define variables of the same name, will (inside the `let`) shadow those of the `do` - as expected of lexical scoping. > ->The necessary boilerplate (notably the ``lambda e: ...`` wrappers) is inserted automatically, so the expressions in a ``do[]`` are only evaluated when the underlying ``seq.do`` actually runs. +>The boilerplate needed by the underlying `unpythonic.do` form (notably the `lambda e: ...` wrappers) is inserted automatically. The expressions in a `do[]` are only evaluated when the underlying `unpythonic.do` actually runs. > ->When running, ``do`` behaves like ``letseq``; assignments **above** the current line are in effect (and have been performed in the order presented). Re-assigning to the same name later overwrites (this is afterall an imperative tool). -> ->We also provide a ``do0`` macro, which returns the value of the first expression, instead of the last. +>When running, `do` behaves like `letseq`; assignments **above** the current line are in effect (and have been performed in the order presented). Re-assigning to the same name later overwrites.

-**CAUTION**: ``do[]`` supports local variable deletion, but the ``let[]`` constructs don't, by design. When ``do[]`` is used implicitly with the extra bracket syntax, any ``delete[]`` refers to the scope of the implicit ``do[]``, not any surrounding ``let[]`` scope. +**CAUTION**: `do[]` supports local variable deletion, but the `let[]` constructs do **not**, by design. When `do[]` is used implicitly with the extra bracket syntax, any `delete[]` refers to the scope of the implicit `do[]`, not any surrounding `let[]` scope. + ## Tools for lambdas Macros that introduce additional features for Python's lambdas. -### ``multilambda``: supercharge your lambdas +### `multilambda`: supercharge your lambdas -**Multiple expressions**: use ``[...]`` to denote a multiple-expression body. The macro implements this by inserting a ``do``. +**Multiple expressions**: use `[...]` to denote a multiple-expression body. The macro implements this by inserting a `do`. -**Local variables**: available in a multiple-expression body. For details on usage, see ``do``. +**Local variables**: available in a multiple-expression body. For details on usage, see `do`. ```python from unpythonic.syntax import macros, multilambda, let @@ -622,21 +734,21 @@ with multilambda: echo = lambda x: [print(x), x] assert echo("hi there") == "hi there" - count = let[x << 0][ - lambda: [x << x + 1, # x belongs to the surrounding let + count = let[x := 0][ + lambda: [x := x + 1, # x belongs to the surrounding let x]] assert count() == 1 assert count() == 2 - test = let[x << 0][ - lambda: [x << x + 1, - local[y << 42], # y is local to the implicit do + test = let[x := 0][ + lambda: [x := x + 1, + local[y := 42], # y is local to the implicit do (x, y)]] assert test() == (1, 42) assert test() == (2, 42) myadd = lambda x, y: [print("myadding", x, y), - local[tmp << x + y], + local[tmp := x + y], print("result is", tmp), tmp] assert myadd(2, 3) == 5 @@ -646,10 +758,12 @@ with multilambda: assert t() == [1, 2] ``` -In the second example, returning ``x`` separately is redundant, because the assignment to the let environment already returns the new value, but it demonstrates the usage of multiple expressions in a lambda. +In the second example, returning `x` separately is redundant, because the assignment to the let environment already returns the new value, but it demonstrates the usage of multiple expressions in a lambda. -### ``namedlambda``: auto-name your lambdas +### `namedlambda`: auto-name your lambdas + +**Changed in v0.15.0.** *When `namedlambda` encounters a lambda definition it cannot infer a name for, it instead injects source location info into the name, provided that the AST node for that particular `lambda` has a line number for it. The result looks like ``.* Who said lambdas have to be anonymous? @@ -659,14 +773,14 @@ from unpythonic.syntax import macros, namedlambda with namedlambda: f = lambda x: x**3 # assignment: name as "f" assert f.__name__ == "f" - gn, hn = let[x << 42, g << None, h << None][[ - g << (lambda x: x**2), # env-assignment: name as "g" - h << f, # still "f" (no literal lambda on RHS) + gn, hn = let[x := 42, g := None, h := None][[ + g := (lambda x: x**2), # env-assignment: name as "g" + h := f, # still "f" (no literal lambda on RHS) (g.__name__, h.__name__)]] assert gn == "g" assert hn == "f" - foo = let[[f7 << (lambda x: x)] in f7] # let-binding: name as "f7" + foo = let[[f7 := (lambda x: x)] in f7] # let-binding: name as "f7" def foo(func1, func2): assert func1.__name__ == "func1" @@ -681,84 +795,99 @@ with namedlambda: assert d["g"].__name__ == "g" ``` -Lexically inside a ``with namedlambda`` block, any literal ``lambda`` that is assigned to a name using one of the supported assignment forms is named to have the name of the LHS of the assignment. The name is captured at macro expansion time. +Lexically inside a `with namedlambda` block, any literal `lambda` that is assigned to a name using one of the supported assignment forms is named to have the name of the LHS of the assignment. The name is captured at macro expansion time. -Decorated lambdas are also supported, as is a ``curry`` (manual or auto) where the last argument is a lambda. The latter is a convenience feature, mainly for applying parametric decorators to lambdas. See [the unit tests](../unpythonic/syntax/test/test_lambdatools.py) for detailed examples. +Decorated lambdas are also supported, as is a `curry` (manual or auto) where the last argument is a lambda. The latter is a convenience feature, mainly for applying parametric decorators to lambdas. See [the unit tests](../unpythonic/syntax/tests/test_lambdatools.py) for detailed examples. -The naming is performed using the function ``unpythonic.misc.namelambda``, which will return a modified copy with its ``__name__``, ``__qualname__`` and ``__code__.co_name`` changed. The original function object is not mutated. +The naming is performed using the function `unpythonic.namelambda`, which will return a modified copy with its `__name__`, `__qualname__` and `__code__.co_name` changed. The original function object is not mutated. **Supported assignment forms**: - - Single-item assignment to a local name, ``f = lambda ...: ...`` - - - **Added in v0.15.0**: Named expressions (a.k.a. walrus operator, Python 3.8+), ``f := lambda ...: ...`` + - Single-item assignment to a local name, `f = lambda ...: ...` - - Expression-assignment to an unpythonic environment, ``f << (lambda ...: ...)`` - - Env-assignments are processed lexically, just like regular assignments. + - Named expressions (a.k.a. walrus operator, Python 3.8+), `f := lambda ...: ...`. **Added in v0.15.0.** - - Let bindings, ``let[[f << (lambda ...: ...)] in ...]``, using any let syntax supported by unpythonic (here using the haskelly let-in just as an example). + - Expression-assignment to an unpythonic environment, `f := (lambda ...: ...)`, and the old syntax `f << (lambda ...: ...)`. + - Env-assignments are processed lexically, just like regular assignments. This should not cause problems, because left-shifting by a literal lambda most often makes no sense (whence, that syntax is *almost* guaranteed to mean an env-assignment). - - **Added in v0.14.2**: Named argument in a function call, as in ``foo(f=lambda ...: ...)``. + - Let-bindings, `let[[f := (lambda ...: ...)] in ...]`, using any let syntax supported by unpythonic (here using the haskelly let-in with env-assign style bindings just as an example). - - **Added in v0.14.2**: In a dictionary literal ``{...}``, an item with a literal string key, as in ``{"f": lambda ...: ...}``. + - Named argument in a function call, as in `foo(f=lambda ...: ...)`. **Added in v0.14.2.** -Support for other forms of assignment may or may not be added in a future version. + - In a dictionary literal `{...}`, an item with a literal string key, as in `{"f": lambda ...: ...}`. **Added in v0.14.2.** -### ``f``: underscore notation (quick lambdas) for Python. +Support for other forms of assignment may or may not be added in a future version. We will maintain a list here; but if you want the gritty details, see the `_namedlambda` syntax transformer in [`unpythonic.syntax.lambdatools`](../unpythonic/syntax/lambdatools.py). -**Changed in v0.15.0.** *Up to 0.14.x, the `f[]` macro used to be provided by `macropy`, but now that we use `mcpyrate`, we provide this ourselves. The underscore `_` is no longer a macro on its own. The `f` macro treats the underscore magically, as before, but anywhere else the underscore is available to be used as a regular variable. If you use `f[]`, change your import of this macro to `from unpythonic.syntax import macros, f`.* +### `fn`: underscore notation (quick lambdas) for Python -The syntax ``f[...]`` creates a lambda, where each underscore in the ``...`` part introduces a new parameter. The macro does not descend into any nested ``f[]``. +**Changed in v0.15.0.** *Up to 0.14.x, the `f[]` macro used to be provided by `macropy`, but now that we use `mcpyrate`, we provide this ourselves. Note that the name of the construct is now `fn[]`.* -Example: +The syntax `fn[...]` creates a lambda, where each underscore `_` in the `...` part introduces a new parameter: ```python -func = f[_ * _] # --> func = lambda x, y: x * y +from unpythonic.syntax import macros, fn +from unpythonic.syntax import _ # optional, makes IDEs happy + +double = fn[_ * 2] # --> double = lambda x: x * 2 +mul = fn[_ * _] # --> mul = lambda x, y: x * y ``` -Since in `mcpyrate`, macros can be as-imported, you can rename `f` at import time to have any name you want. The `quicklambda` block macro (see below) respects the as-import. Now you **must** import also the macro `f` when you import the macro `quicklambda`, because `quicklambda` internally queries the expander to determine the name(s) the macro `f` is currently bound to. +The macro does not descend into any nested `fn[]`, to allow the macro expander itself to expand those separately. + +We have named the construct `fn`, because `f` is often used as a function name in code examples, local temporaries, and similar. Also, `fn[]` is a less ambiguous abbreviation for a syntactic construct that means *function*, while remaining shorter than the equivalent `lambda`. + +The underscore `_` itself is not a macro. The `fn` macro treats the underscore magically, just like MacroPy's `f`, but anywhere else the underscore is available to be used as a regular variable. + +The underscore does not need to be imported for `fn[]` to recognize it, but if you want to make your IDE happy, there is a symbol named `_` in `unpythonic.syntax` you can import to silence any "undefined name" errors regarding the use of `_`. It is a regular run-time object, not a macro. It is available in `unpythonic.syntax` (not at the top level of `unpythonic`) because it is basically an auxiliary syntactic construct, with no meaningful run-time functionality of its own. + +(It *could* be made into a `@namemacro` that triggers a syntax error when it appears in an improper context, like starting with v0.15.0, many auxiliary constructs in similar roles already do. But it was decided that in this particular case, it is more valuable to have the name `_` available for other uses in other contexts, because it is a standard dummy name in Python. The lambdas created using `fn[]` are likely short enough that not automatically detecting misplaced underscores does not cause problems in practice.) + +Because in `mcpyrate`, macros can be as-imported, you can rename `fn` at import time to have any name you want. The `quicklambda` block macro (see below) respects the as-import. You **must** import also the macro `fn` if you use `quicklambda`, because `quicklambda` internally queries the expander to determine the name(s) the macro `fn` is currently bound to. If the `fn` macro is not bound to any name, `quicklambda` will do nothing. + +It is sufficient that `fn` has been macro-imported by the time when the `with quicklambda` expands. So it is possible, for example, for a dialect template to macro-import just `quicklambda` and inject an invocation for it, and leave macro-importing `fn` to the user code. The `Lispy` variant of the [Lispython dialect](dialects/lispython.md) does exactly this. -### ``quicklambda``: expand quick lambdas first +### `quicklambda`: expand quick lambdas first -To be able to transform correctly, the block macros in ``unpythonic.syntax`` that transform lambdas (e.g. ``multilambda``, ``tco``) need to see all ``lambda`` definitions written with Python's standard ``lambda``. +To be able to transform correctly, the block macros in `unpythonic.syntax` that transform lambdas (e.g. `multilambda`, `tco`) need to see all `lambda` definitions written with Python's standard `lambda`. -However, the ``f`` macro uses the syntax ``f[...]``, which (to the analyzer) does not look like a lambda definition. This macro changes the expansion order, forcing any ``f[...]`` lexically inside the block to expand before any other macros do. +However, the `fn` macro uses the syntax `fn[...]`, which (to the analyzer) does not look like a lambda definition. The `quicklambda` block macro changes the expansion order, forcing any `fn[...]` lexically inside the block to expand before any other macros do. -Any expression of the form ``f[...]``, where ``f`` is any name bound in the current macro expander to the macro `unpythonic.syntax.f`, is understood as a quick lambda. (In plain English, this respects as-imports of the macro ``f``.) +Any expression of the form `fn[...]`, where `fn` is any name bound in the current macro expander to the macro `unpythonic.syntax.fn`, is understood as a quick lambda. (In plain English, this respects as-imports of the macro `fn`.) Example - a quick multilambda: ```python -from unpythonic.syntax import macros, multilambda, quicklambda, f, local +from unpythonic.syntax import macros, multilambda, quicklambda, fn, local +from unpythonic.syntax import _ # optional, makes IDEs happy with quicklambda, multilambda: - func = f[[local[x << _], - local[y << _], - x + y]] + func = fn[[local[x := _], + local[y := _], + x + y]] assert func(1, 2) == 3 ``` -This is of course rather silly, as an unnamed formal parameter can only be mentioned once. If we're giving names to them, a regular ``lambda`` is shorter to write. A more realistic combo is: +This is of course rather silly, as an unnamed formal parameter can only be mentioned once. If we are giving names to them, a regular `lambda` is shorter to write. A more realistic combo is: ```python with quicklambda, tco: def g(x): - return 2*x - func1 = f[g(3*_)] # tail call + return 2 * x + func1 = fn[g(3 * _)] # tail call assert func1(10) == 60 - func2 = f[3*g(_)] # no tail call + func2 = fn[3 * g(_)] # no tail call assert func2(10) == 60 ``` -### ``envify``: make formal parameters live in an unpythonic ``env`` +### `envify`: make formal parameters live in an unpythonic `env` -When a function whose definition (``def`` or ``lambda``) is lexically inside a ``with envify`` block is entered, it copies references to its arguments into an unpythonic ``env``. At macro expansion time, all references to the formal parameters are redirected to that environment. This allows rebinding, from an expression position, names that were originally the formal parameters. +When a function whose definition (`def` or `lambda`) is lexically inside a `with envify` block is entered, it copies references to its arguments into an unpythonic `env`. At macro expansion time, all references to the formal parameters are redirected to that environment. This allows rebinding, from an expression position, names that were originally the formal parameters. Wherever could *that* be useful? For an illustrative caricature, consider [PG's accumulator puzzle](http://paulgraham.com/icad.html). -The modern pythonic solution: +The Python 3 solution: ```python def foo(n): @@ -769,43 +898,58 @@ def foo(n): return accumulate ``` -This avoids allocating an extra place to store the accumulator ``n``. If you want optimal bytecode, this is the best solution in Python 3. +This avoids allocating an extra place to store the accumulator `n`. The Python 3.8+ solution, using the new walrus operator, is one line shorter: -But what if, instead, we consider the readability of the unexpanded source code? The definition of ``accumulate`` requires many lines for something that simple. What if we wanted to make it a lambda? Because all forms of assignment are statements in Python, the above solution is not admissible for a lambda, even with macros. +```python +def foo(n): + def accumulate(i): + nonlocal n + return (n := n + i) + return accumulate +``` + +This is rather clean, but still needs the `nonlocal` declaration, which is available as a statement only. + +If you want optimal bytecode, these two are the best solutions of the puzzle in Python. + +But what if we want to shorten the source code even more, for readability? We could make `accumulate` a lambda. But then, to rebind the `n` that lives in an enclosing scope - because Python does not support doing that from an expression position - we must make it live in an `unpythonic` `env`. -So if we want to use a lambda, we have to create an ``env``, so that we can write into it. Let's use the let-over-lambda idiom: +Let's use the let-over-lambda idiom: ```python def foo(n0): - return let[[n << n0] in - (lambda i: n << n + i)] + return let[[n := n0] in + (lambda i: (n := n + i))] ``` -Already better, but the ``let`` is used only for (in effect) altering the passed-in value of ``n0``; we don't place any other variables into the ``let`` environment. Considering the source text already introduces an ``n0`` which is just used to initialize ``n``, that's an extra element that could be eliminated. +This is already shorter, but the `let` is used only for (in effect) storing the passed-in value of `n0`; we do not place any other variables into the `let` environment. Considering the source text already introduces a name `n0` which is just used to initialize `n`, that's an extra element that could be eliminated. -Enter the ``envify`` macro, which automates this: +Enter the `envify` macro, which automates this: ```python with envify: def foo(n): - return lambda i: n << n + i + return lambda i: (n := n + i) ``` -Combining with ``autoreturn`` yields the fewest-elements optimal solution to the accumulator puzzle: +Note this does not work without `envify`, because then the assignment expression will create a local variable (local to the lambda) instead of rebinding the outer existing `n`. + +Combining with `autoreturn` yields the fewest-source-code-elements optimal solution to the accumulator puzzle: ```python with autoreturn, envify: def foo(n): - lambda i: n << n + i + lambda i: (n := n + i) ``` -The ``with`` block adds a few elements, but if desired, it can be refactored into the definition of a custom dialect in [Pydialect](https://github.com/Technologicat/pydialect). +The `with` block adds a few elements, but if desired, it can be refactored into the definition of a custom dialect using `mcpyrate`. See [dialect examples](dialects.md). + ## Language features To boldly go where Python without macros just won't. Changing the rules by code-walking and making significant rewrites. -### ``autocurry``: automatic currying for Python +### `autocurry`: automatic currying for Python **Changed in v0.15.0.** *The macro is now named `autocurry`, to avoid shadowing the `curry` function.* @@ -829,25 +973,27 @@ with autocurry: assert add3(1)(2)(3) == 6 ``` -*Lexically* inside a ``with autocurry`` block: +*Lexically* inside a `with autocurry` block: - - All **function calls** and **function definitions** (``def``, ``lambda``) are automatically curried, somewhat like in Haskell, or in ``#lang`` [``spicy``](https://github.com/Technologicat/spicy). + - All **function calls** and **function definitions** (`def`, `lambda`) are automatically curried, somewhat like in Haskell, or in `#lang` [`spicy`](https://github.com/Technologicat/spicy). - - Function calls are autocurried, and run ``unpythonic.fun.curry`` in a special mode that no-ops on uninspectable functions (triggering a standard function call with the given args immediately) instead of raising ``TypeError`` as usual. + - Function calls are autocurried, and run `unpythonic.curry` in a special mode that no-ops on uninspectable functions (triggering a standard function call with the given args immediately) instead of raising `TypeError` as usual. -**CAUTION**: Some built-ins are uninspectable or may report their arities incorrectly; in those cases, ``curry`` may fail, occasionally in mysterious ways. The function ``unpythonic.arity.arities``, which ``unpythonic.fun.curry`` internally uses, has a workaround for the inspectability problems of all built-ins in the top-level namespace (as of Python 3.7), but e.g. methods of built-in types are not handled. +**CAUTION**: Some built-ins are uninspectable or may report their call signature incorrectly; in those cases, `curry` may fail, occasionally in mysterious ways. When inspection fails, `curry` raises ``ValueError``, like `inspect.signature` does. Manual uses of the `curry` decorator (on both `def` and `lambda`) are detected, and in such cases the macro skips adding the decorator. -### ``lazify``: call-by-need for Python +### `lazify`: call-by-need for Python -**Changed in v0.15.0.** *Up to 0.14.x, the `lazy[]` macro, that is used together with `with lazify`, used to be provided by `macropy`, but now that we use `mcpyrate`, we provide it ourselves. If you use `lazy[]`, change your import of that macro to `from unpythonic.syntax import macros, lazy`*. +**Changed in v0.15.0.** *The `lazy[]` macro, that is used together with `with lazify`, used to be provided by `macropy` up to `unpythonic` v0.14.3. But now that we use `mcpyrate`, we provide a `lazy[]` macro and an underlying `Lazy` class ourselves. For details, see the separate section about `lazy[]` and `lazyrec[]` below.* -Also known as *lazy functions*. Like [lazy/racket](https://docs.racket-lang.org/lazy/index.html), but for Python. Note if you want *lazy sequences* instead, Python already provides those; just use the generator facility (and decorate your gfunc with ``unpythonic.gmemoize`` if needed). +Also known as *lazy functions*. Like [lazy/racket](https://docs.racket-lang.org/lazy/index.html), but for Python. Note if you want *lazy sequences* instead, Python already provides those; just use the generator facility (and decorate your gfunc with `unpythonic.gmemoize` if needed). Lazy function example: ```python +from unpythonic.syntax import macros, lazify + with lazify: def my_if(p, a, b): if p: @@ -864,15 +1010,15 @@ with lazify: assert f(21, 1/0) == 42 ``` -In a ``with lazify`` block, function arguments are evaluated only when actually used, at most once each, and in the order in which they are actually used (regardless of the ordering of the formal parameters that receive them). Delayed values (*promises*) are automatically evaluated (*forced*) on access. Automatic lazification applies to arguments in function calls and to let-bindings, since they play a similar role. **No other binding forms are auto-lazified.** +In a `with lazify` block, function arguments are evaluated only when actually used, at most once each, and in the order in which they are actually used (regardless of the ordering of the formal parameters that receive them). Delayed values (*promises*) are automatically evaluated (*forced*) on access. Automatic lazification applies to arguments in function calls and to let-bindings, since they play a similar role. **No other binding forms are auto-lazified.** -Automatic lazification uses the ``lazyrec[]`` macro (see below), which recurses into certain types of container literals, so that the lazification will not interfere with unpacking. +Automatic lazification uses the `lazyrec[]` macro (see below), which recurses into certain types of container literals, so that the lazification will not interfere with unpacking. -Note ``my_if`` in the example is a regular function, not a macro. Only the ``with lazify`` is imbued with any magic. Essentially, the above code expands into: +Note `my_if` in the example is a regular function, not a macro. Only the `with lazify` is imbued with any magic. Essentially, the above code expands into: ```python from unpythonic.syntax import macros, lazy -from unpythonic.syntax import force +from unpythonic import force def my_if(p, a, b): if force(p): @@ -889,99 +1035,109 @@ def f(a, b): assert f(lazy[21], lazy[1/0]) == 42 ``` -plus some clerical details to allow mixing lazy and strict code. This second example relies on the magic of closures to capture f's ``a`` and ``b`` into the ``lazy[]`` promises. +plus some clerical details to allow mixing lazy and strict code. This second example relies on the magic of closures to capture f's `a` and `b` into the `lazy[]` promises. -Like ``with continuations``, no state or context is associated with a ``with lazify`` block, so lazy functions defined in one block may call those defined in another. +Like `with continuations`, no state or context is associated with a `with lazify` block, so lazy functions defined in one block may call those defined in another. Lazy code is allowed to call strict functions and vice versa, without requiring any additional effort. -Comboing with other block macros in ``unpythonic.syntax`` is supported, including ``autocurry`` and ``continuations``. See the [meta](#meta) section of this README for the correct ordering. +Comboing `lazify` with other block macros in `unpythonic.syntax` is supported, including `autocurry` and `continuations`. See the [meta](#meta) section of this README for the correct ordering. + +For more details, see the docstring of `unpythonic.syntax.lazify`. + +Inspired by Haskell, Racket's `(delay)` and `(force)`, and [lazy/racket](https://docs.racket-lang.org/lazy/index.html). -For more details, see the docstring of ``unpythonic.syntax.lazify``. +**CAUTION**: The functions in the module `unpythonic.fun` are lazify-aware (so that e.g. `curry` and `compose` work with lazy functions), as are `call` and `callwith` in the module `unpythonic.funutil`, but a large part of `unpythonic` is not. Keep in mind that any call to a strict (regular Python) function will evaluate all of its arguments. -Inspired by Haskell, Racket's ``(delay)`` and ``(force)``, and [lazy/racket](https://docs.racket-lang.org/lazy/index.html). +#### `lazy[]` and `lazyrec[]` macros -**CAUTION**: The functions in ``unpythonic.fun`` are lazify-aware (so that e.g. ``curry`` and ``compose`` work with lazy functions), as are ``call`` and ``callwith`` in ``unpythonic.misc``, but a large part of ``unpythonic`` is not. Keep in mind that any call to a strict (regular Python) function will evaluate all of its arguments. +**Changed in v0.15.0.** *Previously, the `lazy[]` macro was provided by MacroPy. Now that we use `mcpyrate`, which doesn't provide it, we provide it ourselves, in `unpythonic.syntax`. We now provide also the underlying `Lazy` class ourselves.* -#### ``lazy[]`` and ``lazyrec[]`` macros +*Note that a lazy value (an instance of `Lazy`) now no longer has a `__call__` operator; instead, it has a `force()` method. However, the preferred way is to use the top-level function `force`, which abstracts away this detail.* -**Changed in v0.15.0.** *Previously, the `lazy[]` macro was provided by MacroPy. Now that we use `mcpyrate`, which doesn't provide it, we provide it ourselves, in `unpythonic.syntax`. Note that a lazy value now no longer has a `__call__` operator; instead, it has a `force()` method. The utility `unpythonic.lazyutil.force` (previously exported in `unpythonic.syntax`; now moved to the top-level namespace of `unpythonic`) abstracts away this detail.* +*The `force` function was previously exported in `unpythonic.syntax`; now it is available in the top-level namespace of `unpythonic`. This follows the general convention that regular functions live in the top-level `unpythonic` package, while macros (and in general, syntactic constructs) live in `unpythonic.syntax`.* -We provide the macros ``unpythonic.syntax.lazy``, which explicitly lazifies a single expression, and ``unpythonic.syntax.lazyrec``, which can be used to lazify expressions inside container literals, recursively. +We provide the macros `unpythonic.syntax.lazy`, which explicitly lazifies a single expression, and `unpythonic.syntax.lazyrec`, which can be used to lazify expressions inside container literals, recursively. -Essentially, ``lazy[...]`` achieves the same result as ``memoize(lambda: ...)``, with the practical difference that a ``lazy[]`` promise ``p`` is evaluated by calling ``unpythonic.lazyutil.force(p)`` or ``p.force()``. In ``unpythonic``, the promise datatype (``unpythonic.lazyutil.Lazy``) does not have a ``__call__`` method, because the word ``force`` better conveys the intent. +Essentially, `lazy[...]` achieves the same result as `memoize(lambda: ...)`, with the practical difference that the `lazify` subsystem expects the `lazy[...]` notation in its analyzer, and will not recognize `memoize(lambda: ...)` as a delayed value. -It is preferable to use the ``force`` function instead of the ``.force`` method, because the function will also pass through any non-promise value, whereas (obviously) a non-promise value will not have a ``.force`` method. Using the function, you can ``force`` a value just to be sure, without caring whether that value was a promise. The ``force`` function is available in the top-level namespace of ``unpythonic``. +A `lazy[]` promise `p` is evaluated by calling `force(p)` or `p.force()`. In `unpythonic`, the promise datatype (`Lazy`) does not have a `__call__` method, because the word `force` better conveys the intent. -The ``lazify`` subsystem expects the ``lazy[...]`` notation in its analyzer, and will not recognize ``memoize(lambda: ...)`` as a delayed value. +It is preferable to use the `force` top-level function instead of the `.force` method, because the function will also pass through any non-promise value, whereas (obviously) a non-promise value will not have a `.force` method. Using the function, you can `force` a value just to be sure, without caring whether that value was a promise. The `force` function is available in the top-level namespace of `unpythonic`. -The ``lazyrec[]`` macro allows code like ``tpl = lazyrec[(1*2*3, 4*5*6)]``. Each item becomes wrapped with ``lazy[]``, but the container itself is left alone, to avoid interfering with unpacking. Because ``lazyrec[]`` is a macro and must work by names only, it supports a fixed set of container types: ``list``, ``tuple``, ``set``, ``dict``, ``frozenset``, ``unpythonic.collections.frozendict``, ``unpythonic.collections.box``, and ``unpythonic.llist.cons`` (specifically, the constructors ``cons``, ``ll`` and ``llist``). +The `lazyrec[]` macro allows code like `tpl = lazyrec[(1*2*3, 4*5*6)]`. Each item becomes wrapped with `lazy[]`, but the container itself is left alone, to avoid interfering with its unpacking. Because `lazyrec[]` is a macro and must work by names only, it supports a fixed set of container types: `list`, `tuple`, `set`, `dict`, `frozenset`, `unpythonic.frozendict`, `unpythonic.box`, and `unpythonic.cons` (specifically, the constructors `cons`, `ll` and `llist`). -The `unpythonic` containers **must be from-imported** for ``lazyrec[]`` to recognize them. Either use ``from unpythonic import xxx`` (**recommended**), where ``xxx`` is a container type, or import the ``containers`` subpackage by ``from unpythonic import containers``, and then use ``containers.xxx``. (The analyzer only looks inside at most one level of attributes. This may change in the future.) +The `unpythonic` containers **must be from-imported** for `lazyrec[]` to recognize them. Either use `from unpythonic import xxx` (**recommended**), where `xxx` is a container type, or import the `containers` subpackage by `from unpythonic import containers`, and then use `containers.xxx`. (The analyzer only looks inside at most one level of attributes. This may change in the future.) -(The analysis in ``lazyrec[]`` must work by names only, because in an eager language any lazification must be performed as a syntax transformation before the code actually runs, so the analysis must be performed statically - and locally, because ``lazyrec[]`` is an expr macro. [Fexprs](https://fexpr.blogspot.com/2011/04/fexpr.html) (along with [a new calculus to go with them](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html)) are the clean, elegant solution, but this requires redesigning the whole language from ground up. Of course, if you're fine with a language not particularly designed for extensibility, and lazy evaluation is your top requirement, you could just use Haskell.) +Observe that the analysis in `lazyrec[]` must work by names only, because in an eager language any lazification must be performed as a syntax transformation before the code actually runs. Hence, the analysis must be performed statically - and locally, because `lazyrec[]` is an expr macro. [Fexprs](https://fexpr.blogspot.com/2011/04/fexpr.html) (along with [a new calculus to go with them](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html)) are the clean, elegant solution, but this requires redesigning the whole language from ground up. Of course, if you are fine with a language not particularly designed for extensibility, and lazy evaluation is your top requirement, you could just use Haskell. #### Forcing promises manually **Changed in v0.15.0.** *The functions `force1` and `force` now live in the top-level namespace of `unpythonic`, no longer in `unpythonic.syntax`.* -This is mainly useful if you ``lazy[]`` or ``lazyrec[]`` something explicitly, and want to compute its value outside a ``with lazify`` block. +This is mainly useful if you `lazy[]` or `lazyrec[]` something explicitly, and want to compute its value outside a `with lazify` block. -We provide the functions ``force1`` and ``force``. Using ``force1``, if ``x`` is a ``lazy[]`` promise, it will be forced, and the resulting value is returned. If ``x`` is not a promise, ``x`` itself is returned, à la Racket. The function ``force``, in addition, descends into containers (recursively). When an atom ``x`` (i.e. anything that is not a container) is encountered, it is processed using ``force1``. +We provide the functions `force1` and `force`. Using `force1`, if `x` is a `lazy[]` promise, it will be forced, and the resulting value is returned. If `x` is not a promise, `x` itself is returned, à la Racket. The function `force`, in addition, descends into containers (recursively). When an atom `x` (i.e. anything that is not a container) is encountered, it is processed using `force1`. -Mutable containers are updated in-place; for immutables, a new instance is created, but as a side effect the promise objects **in the input container** will be forced. Any container with a compatible ``collections.abc`` is supported. (See ``unpythonic.collections.mogrify`` for details.) In addition, as special cases ``unpythonic.collections.box`` and ``unpythonic.llist.cons`` are supported. +Mutable containers are updated in-place; for immutables, a new instance is created, but as a side effect the promise objects **in the input container** will be forced. Any container with a compatible `collections.abc` is supported. (See `unpythonic.mogrify` for details.) In addition, as special cases `unpythonic.box` and `unpythonic.cons` are supported. #### Binding constructs and auto-lazification -Why do we auto-lazify in certain kinds of binding constructs, but not in others? Function calls and let-bindings have one feature in common: both are guaranteed to bind only new names (even if that name is already in scope, they are distinct; the new binding will shadow the old one). Auto-lazification of all assignments, on the other hand, in a language that allows mutation is dangerous, because then this superficially innocuous code will fail: +Why do we auto-lazify in certain kinds of binding constructs, but not in others? Function calls and let-bindings have one feature in common: both are guaranteed to bind only new names. Even if a name that uses the same identifier is already in scope, they are distinct; the new binding will shadow the old one. Auto-lazification of all assignments, on the other hand, in a language that allows mutation is dangerous, because then this superficially innocuous code will fail: ```python -a = 10 -a = 2*a -print(a) # 20, right? +from unpythonic.syntax import macros, lazify + +with lazify: + a = 10 + a = 2 * a + print(a) # 20, right? ``` -If we chose to auto-lazify assignments, then assuming a ``with lazify`` around the example, it would expand to: +If we chose to auto-lazify assignments, then the example would expand to: ```python from unpythonic.syntax import macros, lazy -from unpythonic.syntax import force +from unpythonic import force a = lazy[10] -a = lazy[2*force(a)] +a = lazy[2 * force(a)] print(force(a)) ``` -In the second assignment, the ``lazy[]`` sets up a promise, which will force ``a`` *at the time when the containing promise is forced*, but at that time the name ``a`` points to a promise, which will force... +Scan that again: in the second assignment, the `lazy[]` sets up a promise, which will force `a` *at the time when the containing promise is forced*, but at that time the name `a` points to a promise, which will force... -The fundamental issue is that ``a = 2*a`` is an imperative update. Therefore, to avoid this infinite loop trap for the unwary, assignments are not auto-lazified. Note that if we use two different names, this works just fine: +The fundamental issue is that `a = 2 * a` is an imperative update. Therefore, to avoid this infinite loop trap for the unwary, assignments are not auto-lazified. Note that if we use two *different* names, this works just fine: ```python from unpythonic.syntax import macros, lazy -from unpythonic.syntax import force +from unpythonic import force a = lazy[10] -b = lazy[2*force(a)] +b = lazy[2 * force(a)] print(force(b)) ``` -because now at the time when ``b`` is forced, the name ``a`` still points to the value we intended it to. +because now at the time when `b` is forced, the name `a` still points to the value we intended it to. That is, code that is normalized to [static single assignment (SSA) form](https://en.wikipedia.org/wiki/Static_single_assignment_form) could be auto-lazified. -If you're sure you have *new definitions* and not *imperative updates*, just manually use ``lazy[]`` (or ``lazyrec[]``, as appropriate) on the RHS. Or if it's fine to use eager evaluation, just omit the ``lazy[]``, thus allowing Python to evaluate the RHS immediately. +If you are sure you have *new definitions* and not *imperative updates*, you can just manually use `lazy[]` (or `lazyrec[]`, as appropriate) on the RHS. Or if it is fine to use eager evaluation, just omit the `lazy[]`, thus allowing Python to evaluate the RHS immediately. -Beside function calls (which bind the parameters of the callee to the argument values of the call) and assignments, there are many other binding constructs in Python. For a full list, see [here](http://excess.org/article/2014/04/bar-foo/), or locally [here](../unpythonic/syntax/scopeanalyzer.py), in function ``get_names_in_store_context``. Particularly noteworthy in the context of lazification are the ``for`` loop and the ``with`` context manager. +Beside function calls (which bind the parameters of the callee to the argument values of the call) and assignments, there are many other binding constructs in Python. For a full list, see [here](http://excess.org/article/2014/04/bar-foo/), or locally [here](../unpythonic/syntax/scopeanalyzer.py), in function `get_names_in_store_context`. Particularly noteworthy in the context of lazification are the `for` loop and the `with` context manager. -In Python's ``for``, the loop counter is an imperatively updated single name. In many use cases a rapid update is desirable for performance reasons, and in any case, the whole point of the loop is (almost always) to read the counter (and do something with the value) at least once per iteration. So it is much simpler, faster, and equally correct not to lazify there. +In Python's `for`, the loop counter is an imperatively updated single name. In many use cases a rapid update is desirable for performance reasons, and in any case, the whole point of the loop is (almost always) to read the counter (and do something with the value) at least once per iteration. So it is much simpler, faster, and equally correct not to lazify there. -In ``with``, the whole point of a context manager is that it is eagerly initialized when the ``with`` block is entered (and finalized when the block exits). Since our lazy code can transparently use both bare values and promises (due to the semantics of our ``force1``), and the context manager would have to be eagerly initialized anyway, we can choose not to lazify there. +In `with`, the whole point of a context manager is that it is eagerly initialized when the `with` block is entered, and finalized when the block exits. Since our lazy code can transparently use both bare values and promises (due to the semantics of our `force1`), and the context manager would have to be eagerly initialized anyway, we have chosen not to lazify there. #### Note about TCO -To borrow a term from PG's On Lisp, to make ``lazify`` *pay-as-you-go*, a special mode in ``unpythonic.tco.trampolined`` is automatically enabled by ``with lazify`` to build lazify-aware trampolines in order to avoid a drastic performance hit (~10x) in trampolines built for regular strict code. +To borrow a term from PG's On Lisp, to make `lazify` *pay-as-you-go*, a special mode in `unpythonic.trampolined` is automatically enabled by `with lazify` to build lazify-aware trampolines in order to avoid a drastic performance hit (~10x) in trampolines built for regular strict code. -The idea is that the mode is enabled while any function definitions in the ``with lazify`` block run, so they get a lazify-aware trampoline when the ``trampolined`` decorator is applied. This should be determined lexically, but that's complicated to do API-wise, so we currently enable the mode for the dynamic extent of the ``with lazify``. Usually this is close enough; the main case where this can behave unexpectedly is: +The idea is that the mode is enabled while any function definitions in the `with lazify` block run, so they get a lazify-aware trampoline when the `trampolined` decorator is applied. This should be determined lexically, but that is complicated to do, because the decorator is applied at run time; so we currently enable the mode for the dynamic extent of the `with lazify`. Usually this is close enough. The main case where this can behave unexpectedly is: ```python +from unpythonic.syntax import macros, lazify +from unpythonic import trampolined + @trampolined # strict trampoline def g(): ... @@ -1002,20 +1158,32 @@ with lazify: f2 = make_f() # f2 gets the lazify-aware trampoline ``` -TCO chains with an arbitrary mix of lazy and strict functions should work as long as the first function in the chain has a lazify-aware trampoline, because the chain runs under the trampoline of the first function (the trampolines of any tail-called functions are stripped away by the TCO machinery). +TCO chains with an arbitrary mix of lazy and strict functions should work as long as the first function in the chain has a lazify-aware trampoline, because the chain runs under the trampoline of the first function. The trampolines of any tail-called functions are skipped by the TCO machinery. Tail-calling from a strict function into a lazy function should work, because all arguments are evaluated at the strict side before the call is made. -But tail-calling ``strict -> lazy -> strict`` will fail in some cases. The second strict callee may get promises instead of values, because the strict trampoline does not have the ``maybe_force_args`` (the mechanism ``with lazify`` uses to force the args when lazy code calls into strict code). +But tail-calling `strict -> lazy -> strict` will fail in some cases. The second strict callee may get promises instead of values, because the strict trampoline does not have the `maybe_force_args` (the mechanism `with lazify` uses to force the args when lazy code calls into strict code). + +The reason we have this hack is that it allows the performance of strict code using `unpythonic`'s TCO machinery, not even caring that a `lazify` exists, to be unaffected by the additional machinery used to support automatic lazy-strict interaction. -The reason we have this hack is that it allows the performance of strict code using unpythonic's TCO machinery, not even caring that a ``lazify`` exists, to be unaffected by the additional machinery used to support automatic lazy-strict interaction. +### `tco`: automatic tail call optimization for Python -### ``tco``: automatic tail call optimization for Python +*This is the macro that applies tail call optimization (TCO) automatically. See the manual section on [`trampolined` and `jump`](features.md#trampolined-jump-tail-call-optimization-tco--explicit-continuations) on what TCO is and where it is useful.* + +Using `with tco`, there is no need to manually use `trampolined` or `jump`: ```python from unpythonic.syntax import macros, tco +with tco: + def fact(n, acc=1): + if n == 0: + return acc + return fact(n - 1, n * acc) + print(fact(4)) # 24 + fact(5000) # no crash + with tco: evenp = lambda x: (x == 0) or oddp(x - 1) oddp = lambda x: (x != 0) and evenp(x - 1) @@ -1033,77 +1201,109 @@ with tco: assert evenp(10000) is True ``` -All function definitions (``def`` and ``lambda``) lexically inside the block undergo TCO transformation. The functions are automatically ``@trampolined``, and any tail calls in their return values are converted to ``jump(...)`` for the TCO machinery. Here *return value* is defined as: +All function definitions (`def` and `lambda`) lexically inside the `with tco` block undergo TCO transformation. The functions are automatically `@trampolined`, and any tail calls in their return values are converted to `jump(...)` for the TCO machinery. Here *return value* is defined as: - - In a ``def``, the argument expression of ``return``, or of a call to a known escape continuation. + - In a `def`, the argument expression of `return`, or of a call to a known escape continuation. - - In a ``lambda``, the whole body, as well as the argument expression of a call to a known escape continuation. + - In a `lambda`, the whole body, as well as the argument expression of a call to a known escape continuation. -What is a *known escape continuation* is explained below, in the section [TCO and ``call_ec``](#tco-and-call_ec). +What is considered a *known escape continuation* is explained below, in the section [TCO and `call_ec`](#tco-and-call_ec). -To find the tail position inside a compound return value, this recursively handles any combination of ``a if p else b``, ``and``, ``or``; and from ``unpythonic.syntax``, ``do[]``, ``let[]``, ``letseq[]``, ``letrec[]``. Support for ``do[]`` includes also any ``multilambda`` blocks that have already expanded when ``tco`` is processed. The macros ``aif[]`` and ``cond[]`` are also supported, because they expand into a combination of ``let[]``, ``do[]``, and ``a if p else b``. +To find the tail position inside a compound return value, we recursively handle any combination of `a if p else b`, `and`, `or`; and from `unpythonic.syntax`, `do[]`, `let[]`, `letseq[]`, `letrec[]`. Support for `do[]` includes also any `multilambda` blocks that have already expanded when `tco` is processed. The macros `aif[]` and `cond[]` are also supported, because they expand into a combination of `let[]`, `do[]`, and `a if p else b`. -**CAUTION**: In an ``and``/``or`` expression, only the last item of the whole expression is in tail position. This is because in general, it is impossible to know beforehand how many of the items will be evaluated. +**CAUTION**: In an `and`/`or` expression, only the last item of the whole expression is in tail position. This is because in general, it is impossible to know beforehand how many of the items will be evaluated. -**CAUTION**: In a ``def`` you still need the ``return``; it marks a return value. If you want the tail position to imply a ``return``, use the combo ``with autoreturn, tco`` (on ``autoreturn``, see below). +**CAUTION**: In a `def` you still need the `return`; it marks a return value. If you want tail position to imply a `return`, use the combo `with autoreturn, tco` (on `autoreturn`, see below). -TCO is based on a strategy similar to MacroPy's ``tco`` macro, but using unpythonic's TCO machinery, and working together with the macros introduced by ``unpythonic.syntax``. The semantics are slightly different; by design, ``unpythonic`` requires an explicit ``return`` to mark tail calls in a ``def``. A call that is strictly speaking in tail position, but lacks the ``return``, is not TCO'd, and Python's implicit ``return None`` then shuts down the trampoline, returning ``None`` as the result of the TCO chain. +TCO is based on a strategy similar to MacroPy's `tco` macro, but using unpythonic's TCO machinery, and working together with the macros introduced by `unpythonic.syntax`. The semantics are slightly different; by design, `unpythonic` requires an explicit `return` to mark tail calls in a `def`. A call that is strictly speaking in tail position, but lacks the `return`, is not TCO'd, and Python's implicit `return None` then shuts down the trampoline, returning `None` as the result of the TCO chain. #### TCO and continuations -The ``tco`` macro detects and skips any ``with continuations`` blocks inside the ``with tco`` block, because ``continuations`` already implies TCO. This is done **for the specific reason** of allowing the [Lispython dialect](https://github.com/Technologicat/pydialect) to use ``with continuations``, because the dialect itself implies a ``with tco`` for the whole module (so the user code has no way to exit the TCO context). +The `tco` macro detects and skips any `with continuations` blocks inside the `with tco` block, because `continuations` already implies TCO. This is done **for the specific reason** of allowing the [Lispython dialect](https://github.com/Technologicat/pydialect) to use `with continuations`, because the dialect itself implies a `with tco` for the whole module. Hence, in that dialect, the user code has no way to exit the TCO context. -The ``tco`` and ``continuations`` macros actually share a lot of the code that implements TCO; ``continuations`` just hooks into some callbacks to perform additional processing. +The `tco` and `continuations` macros actually share a lot of the code that implements TCO; `continuations`, for its TCO processing, just hooks into some callbacks to make additional AST edits. -#### TCO and ``call_ec`` +#### TCO and `call_ec` -(Mainly of interest for lambdas, which have no ``return``, and for "multi-return" from a nested function.) +This is mainly of interest for lambdas, which have no `return`, and for "multi-return" from a nested function. It is important to recognize a call to an escape continuation as such, because the argument given to an escape continuation is essentially a return value. If this argument is itself a call, it needs the TCO transformation to be applied to it. -For escape continuations in ``tco`` and ``continuations`` blocks, only basic uses of ``call_ec`` are supported, for automatically harvesting names referring to an escape continuation. In addition, the literal function names ``ec``, ``brk`` and ``throw`` are always *understood as referring to* an escape continuation. +For escape continuations in `tco` and `continuations` blocks, only basic uses of `call_ec` are supported, for automatically extracting names referring to an escape continuation. *Basic use* is defined as either of these two cases: + +```python +from unpythonic import call_ec + +# use as decorator +@call_ec +def result(ec): + ... -The name ``ec``, ``brk`` or ``throw`` alone is not sufficient to make a function into an escape continuation, even though ``tco`` (and ``continuations``) will think of it as such. The function also needs to actually implement some kind of an escape mechanism. An easy way to get an escape continuation, where this has already been done for you, is to use ``call_ec``. Another such mechanism is the ``catch``/``throw`` pair. +# use directly on a literal lambda (effectively, as a decorator) +result = call_ec(lambda ec: ...) +``` -See the docstring of ``unpythonic.syntax.tco`` for details. +When macro expansion of the ``with tco`` block starts, names of escape continuations created **anywhere lexically within** the ``with tco`` block are captured, provided that the creation takes place using one of the above *basic use* patterns. +In addition, the literal function names `ec`, `brk` and `throw` are always *understood as referring to* an escape continuation. The name `ec` is the customary name for the parameter of a function passed to `call_ec`. The name `brk` is the customary name for the break continuation created by `@breakably_looped` and `@breakably_looped_over`. The name `throw` is understood as referring to the function `unpythonic.throw`. -### ``continuations``: call/cc for Python +Obviously, having a name of `ec`, `brk` or `throw` is not by itself sufficient to make a function into an escape continuation, even though `tco` (and `continuations`) will think of it as such. The function also needs to actually implement some kind of an escape mechanism. An easy way to get an escape continuation, where this has already been done for you, is to use `call_ec`. Another such mechanism is the `catch`/`throw` pair. + +See the docstring of `unpythonic.syntax.tco` for details. + + +### `continuations`: call/cc for Python *Where control flow is your playground.* -We provide **genuine multi-shot continuations for Python**. Compare generators and coroutines, which are resumable functions, or in other words, single-shot continuations. In single-shot continuations, once execution passes a certain point, it cannot be rewound. Multi-shot continuations [can be emulated](https://gist.github.com/yelouafi/858095244b62c36ec7ebb84d5f3e5b02), but this makes the execution time `O(n**2)`, because when we want to restart again at an already passed point, the execution must start from the beginning, replaying the history. In contrast, **we implement continuations that can natively resume execution multiple times from the same point.** +We provide **genuine multi-shot continuations for Python**. Compare generators and coroutines, which are resumable functions, or in other words, single-shot continuations. In single-shot continuations, once execution passes a certain point, it cannot be rewound. Multi-shot continuations [can be emulated](https://gist.github.com/yelouafi/858095244b62c36ec7ebb84d5f3e5b02) using single-shot continuations, but this makes the execution time `O(n**2)`, because when we want to restart again at an already passed point, the execution must start from the beginning, replaying the whole history. In contrast, **we implement continuations that can natively resume execution multiple times from the same point.** -This feature has some limitations and is mainly intended for teaching continuations in a Python setting. +**CAUTION**: This feature has some limitations, and is mainly intended for experimenting with, and teaching, multi-shot continuations in a Python setting. Particularly: -- Especially, there are seams between continuation-enabled code and regular Python code. (This happens with any feature that changes the semantics of only a part of a program.) + - There are seams between continuation-enabled code and regular Python code. (This happens with any feature that changes the semantics of only a part of a program.) -- There's no [`dynamic-wind`](https://docs.racket-lang.org/reference/cont.html#%28def._%28%28quote._~23~25kernel%29._dynamic-wind%29%29) (the generalization of `try/finally`, when control can jump back in to the block from outside it). + - There is no [`dynamic-wind`](https://docs.racket-lang.org/reference/cont.html#%28def._%28%28quote._~23~25kernel%29._dynamic-wind%29%29): Scheme's generalization of `try/finally`, which beside the `finally` exit hook, has an *entry hook* for when control jumps back into the block from outside it. -- Interaction of continuations with exceptions isn't fully thought out. Interaction with async functions **is currently not even implemented**. This is quite simply because this feature is primarily for teaching, and the implementation is already quite complex. + - Interaction of continuations with exceptions is not fully thought out. -- The implicit `cc` parameter might not be a good idea in the long run, and it might or might not change in a future release. It suffers from the same lack of transparency as the implicit `this` in many languages (e.g. C++ and JavaScript). - - Because it's implicit, it's easy to forget that each function definition implicitly introduces its own `cc`. - - This introduces a bug when one introduces an inner function, and attempts to use the outer `cc` inside the inner function body, forgetting that inside the inner function the name `cc` points to **the inner function's** own `cc`. - - Not introducing its own `this` [was precisely why](http://tc39wiki.calculist.org/es6/arrow-functions/) the arrow function syntax was introduced to JavaScript in ES6. - - Python gets `self` right in that while it's conveniently *passed* implicitly, it must be *declared* explicitly, eliminating the transparency issue. - - On the other hand, a semi-explicit `cc`, like Python's `self`, was tried in a previous release, and it led to a lot of boilerplate. It's especially bad that it effectively needs to be a keyword parameter, necessitating the user to write `def f(x, *, cc)`. + - Interaction with async functions **is not even implemented**. For this reason, an `async def` or `await` appearing inside a `with continuations` block is considered a syntax error. + + - The implicit `cc` parameter might not be a good idea in the long run. + - This design suffers from the same lack of transparency, whence the same potential for bugs, as the implicit `this` in many languages (e.g. C++ and JavaScript). + - Because `cc` is *declared* implicitly, it is easy to forget that *every* function definition *anywhere* inside the `with continuations` block introduces its own `cc` parameter. + - Particularly, also a `lambda` is a function definition. + - This introduces a bug when one introduces an inner function, and attempts to use the outer `cc` inside the inner function body, forgetting that inside the inner function, the name `cc` points to **the inner function's** own `cc`. + - The correct pattern is to `outercc = cc` in the outer function, and then use `outercc` inside the inner function body. + - Not introducing its own `this` [was precisely why](http://tc39wiki.calculist.org/es6/arrow-functions/) the arrow function syntax was introduced to JavaScript in ES6. + - Python gets `self` right in that while it is conveniently *passed* implicitly, it must be *declared* explicitly, eliminating the transparency issue. + - On the other hand, a semi-explicit `cc`, like Python's `self`, was tried in an early version of this continuations subsystem, and it led to a lot of boilerplate. + - It is especially bad that to avoid easily avoidable bugs regarding passing in the wrong arguments, `cc` effectively must be a keyword parameter, necessitating the user to write `def f(x, *, cc)`. Not having to type out the `, *, cc` is much nicer, albeit not as pythonic. #### General remarks on continuations -If you're new to continuations, see the [short and easy Python-based explanation](https://www.ps.uni-saarland.de/~duchier/python/continuations.html) of the basic idea. +If you are new to continuations, see the [short and easy Python-based explanation](https://www.ps.uni-saarland.de/~duchier/python/continuations.html) of the basic idea. + +This continuations system in `unpythonic` began as a very loose pythonification of Paul Graham's continuation-passing macros, chapter 20 in [On Lisp](http://paulgraham.com/onlisp.html). + +The approach differs from native continuation support (such as in Scheme or Racket) in that the continuation is captured only where explicitly requested with `call_cc[]`. This lets most of the code work as usual, while performing the continuation magic where explicitly desired. + +As a consequence of the approach, our continuations are [*delimited*](https://en.wikipedia.org/wiki/Delimited_continuation) in the very crude sense that the captured continuation ends at the end of the body where the *currently dynamically outermost* `call_cc[]` was invoked. Notably, in `unpythonic`, a continuation eventually terminates and returns a value (provided that the code contained in the continuation itself terminates), without hijacking the rest of the whole-program execution. + +Hence, if porting some code that uses `call/cc` from Racket to Python, in the Python version the `call_cc[]` may be need to be placed further out to capture the relevant part of the computation. For example, see `amb` in the demonstration below; a Scheme or Racket equivalent usually has the `call/cc` placed inside the `amb` operator itself, whereas in Python we must place the `call_cc[]` at the call site of `amb`, so that the continuation captures the remainder of the call site. + +Observe that while our outermost `call_cc` already somewhat acts like a prompt (in the sense of delimited continuations), we are currently missing the ability to set a prompt wherever (inside code that already uses `call_cc` somewhere) and make the continuation terminate there. So what we have right now is something between proper delimited continuations and classic whole-computation continuations - not really [co-values](http://okmij.org/ftp/continuations/undelimited.html), but not really delimited continuations, either. -We provide a very loose pythonification of Paul Graham's continuation-passing macros, chapter 20 in [On Lisp](http://paulgraham.com/onlisp.html). +(Mapping to delimited-continuation terminology: the body of the *enclosing* function is the implicit body of `reset` — that's what delimits `cc`. The `call_cc[]` site does the job of `shift` (capture the rest of the enclosing function body as `cc`), and the called function `g` plays the role of `shift`'s body — the code that runs with the captured continuation in hand. The mapping is approximate: in Danvy & Filinski's [`shift`/`reset`](https://en.wikipedia.org/wiki/Delimited_continuation) ([Racket reference](https://docs.racket-lang.org/reference/cont.html#%28form._%28%28lib._racket%2Fcontrol..rkt%29._reset%29%29)), returning from `shift`'s body without invoking the captured continuation makes that value the value of the whole `reset`; in `unpythonic`, returning from `g` without calling `cc` feeds the return value into `cc` automatically, which is more in line with Python's "return continues after the call site" convention.) -The approach differs from native continuation support (such as in Scheme or Racket) in that the continuation is captured only where explicitly requested with ``call_cc[]``. This lets most of the code work as usual, while performing the continuation magic where explicitly desired. +For the closure topology that the `call_cc` machinery actually produces — and what `cc` and `pcc` are doing under the hood — see [Topology of continuations: how the wiring works](#topology-of-continuations-how-the-wiring-works) below. -As a consequence of the approach, our continuations are [*delimited*](https://en.wikipedia.org/wiki/Delimited_continuation) in the very crude sense that the captured continuation ends at the end of the body where the *currently dynamically outermost* ``call_cc[]`` was used (and it returns a value). Hence, if porting some code that uses ``call/cc`` from Racket to Python, in the Python version the ``call_cc[]`` may be need to be placed further out to capture the relevant part of the computation. For example, see ``amb`` in the demonstration below; a Scheme or Racket equivalent usually has the ``call/cc`` placed inside the ``amb`` operator itself, whereas in Python we must place the ``call_cc[]`` at the call site of ``amb``. +For full documentation, see the docstring of `unpythonic.syntax.continuations`. The unit tests [[1]](../unpythonic/syntax/tests/test_conts.py) [[2]](../unpythonic/syntax/tests/test_conts_escape.py) [[3]](../unpythonic/syntax/tests/test_conts_gen.py) [[4]](../unpythonic/syntax/tests/test_conts_topo.py) may also be useful as usage examples. -For various possible program topologies that continuations may introduce, see [these clarifying pictures](callcc_topology.pdf). +**Note on debugging**: If a function containing a `call_cc[]` crashes below a line that has a `call_cc[]` invocation, the stack trace will usually have the continuation function somewhere in it, containing the line number information, so as usual, you can pinpoint the source code line where the error occurred. For a function `f`, continuation definitions created by `call_cc[]` invocations within its body are named `f_cont_`. -For full documentation, see the docstring of ``unpythonic.syntax.continuations``. The unit tests [[1]](../unpythonic/syntax/test/test_conts.py) [[2]](../unpythonic/syntax/test/test_conts_escape.py) [[3]](../unpythonic/syntax/test/test_conts_gen.py) [[4]](../unpythonic/syntax/test/test_conts_topo.py) may also be useful as usage examples. +Be aware that especially in complex block macro combos (e.g. `with lazify, autocurry, continuations`), the other block macros may have spit out many internal function calls that, at run time, get called *after* the relevant stack frame that points to the actual user program. So check the stack trace as usual, but check further up than usual. -**Note on debugging**: If a function containing a ``call_cc[]`` crashes below the ``call_cc[]``, the stack trace will usually have the continuation function somewhere in it, containing the line number information, so you can pinpoint the source code line where the error occurred. (For a function ``f``, it is named ``f_cont_``) But be aware that especially in complex macro combos (e.g. ``continuations, curry, lazify``), the other block macros may spit out many internal function calls *after* the relevant stack frame that points to the actual user program. So check the stack trace as usual, but check further up than usual. +Using the `with step_expansion` macro from `mcpyrate.debug` may help in understanding how the macro-expanded code actually looks like. **Note on exceptions**: Raising an exception, or [signaling and restarting](features.md#handlers-restarts-conditions-and-restarts), will partly unwind the call stack, so the continuation *from the level that raised the exception* will be cancelled. This is arguably exactly the expected behavior. @@ -1114,7 +1314,7 @@ from unpythonic.syntax import macros, continuations, call_cc with continuations: # basic example - how to call a continuation manually: - k = None # kontinuation + k = None # a kontinuation is konventionally kalled k def setk(*args, cc): global k k = cc @@ -1145,7 +1345,7 @@ with continuations: # Pythagorean triples def pt(): z = call_cc[amb(range(1, 21))] - y = call_cc[amb(range(1, z+1)))] + y = call_cc[amb(range(1, z+1))] x = call_cc[amb(range(1, y+1))] if x*x + y*y != z*z: return fail() @@ -1158,94 +1358,168 @@ with continuations: print(fail()) print(fail()) ``` -Code within a ``with continuations`` block is treated specially. + +Code within a `with continuations` block is treated specially.

Roughly: -> - Each function definition (``def`` or ``lambda``) in a ``with continuations`` block has an implicit formal parameter ``cc``, **even if not explicitly declared** in the formal parameter list. -> - The continuation machinery will set the default value of ``cc`` to the default continuation (``identity``), which just returns its arguments. -> - The default value allows these functions to be called also normally without passing a ``cc``. In effect, the function will then return normally. -> - If ``cc`` is not declared explicitly, it is implicitly declared as a by-name-only parameter named ``cc``, and the default value is set automatically. -> - If ``cc`` is declared explicitly, the default value is set automatically if ``cc`` is in a position that can accept a default value, and no default has been set by the user. +> - Each function definition (`def` or `lambda`) in a `with continuations` block has an implicit formal parameter `cc`, **even if not explicitly declared** in the formal parameter list. +> - The continuation machinery will set the default value of `cc` to the default continuation (`identity`), which just returns its argument(s). +> - The default value allows these functions to be called also normally without passing a `cc`. In effect, the function will then return normally. +> - If `cc` is not declared explicitly, it is implicitly declared as a by-name-only parameter named `cc`, and the default value is set automatically. +> - If `cc` is declared explicitly, the default value is set automatically if `cc` is in a position that can accept a default value, and no default has been set by the user. > - Positions that can accept a default value are the last positional parameter that has no default, and a by-name-only parameter in any syntactically allowed position. -> - Having a hidden parameter is somewhat magic, but overall improves readability, as this allows declaring ``cc`` only where actually explicitly needed. -> - **CAUTION**: Usability trap: in nested function definitions, each ``def`` and ``lambda`` comes with **its own** implicit ``cc``. -> - In the above ``amb`` example, the local variable is named ``ourcc``, so that the continuation passed in from outside (into the ``lambda``, by closure) will have a name different from the ``cc`` implicitly introduced by the ``lambda`` itself. +> - Having a hidden parameter is somewhat magic, but overall improves readability, as this allows declaring `cc` only where actually explicitly needed. +> - **CAUTION**: Usability trap: in nested function definitions, each `def` and `lambda` comes with **its own** implicit `cc`. +> - In the above `amb` example, the local variable is named `ourcc`, so that the continuation passed in from outside (into the `lambda`, by closure) will have a name different from the `cc` implicitly introduced by the `lambda` itself. > - This is possibly subject to change in a future version (pending the invention of a better API), but for now just be aware of this gotcha. -> - Beside ``cc``, there's also a mechanism to keep track of the captured tail of a computation, which is important to have edge cases work correctly. See the note on **pcc** (*parent continuation*) in the docstring of ``unpythonic.syntax.continuations``, and [the pictures](callcc_topology.pdf). +> - Beside `cc`, there's also a mechanism to keep track of the captured tail of a computation, which is important to have edge cases work correctly. See the note on **pcc** (*parent continuation*) in the docstring of `unpythonic.syntax.continuations`, and [the pictures](callcc_topology.pdf). > -> - In a function definition inside the ``with continuations`` block: +> - In a function definition inside the `with continuations` block: > - Most of the language works as usual; especially, any non-tail function calls can be made as usual. -> - ``return value`` or ``return v0, ..., vn`` is actually a tail-call into ``cc``, passing the given value(s) as arguments. -> - As in other parts of ``unpythonic``, returning a tuple means returning multiple-values. -> - This is important if the return value is received by the assignment targets of a ``call_cc[]``. If you get a ``TypeError`` concerning the arguments of a function with a name ending in ``_cont``, check your ``call_cc[]`` invocations and the ``return`` in the call_cc'd function. -> - ``return func(...)`` is actually a tail-call into ``func``, passing along (by default) the current value of ``cc`` to become its ``cc``. -> - Hence, the tail call is inserted between the end of the current function body and the start of the continuation ``cc``. -> - To override which continuation to use, you can specify the ``cc=...`` kwarg, as in ``return func(..., cc=mycc)``. -> - The ``cc`` argument, if passed explicitly, **must be passed by name**. -> - **CAUTION**: This is **not** enforced, as the machinery does not analyze positional arguments in any great detail. The machinery will most likely break in unintuitive ways (or at best, raise a mysterious ``TypeError``) if this rule is violated. -> - The function ``func`` must be a defined in a ``with continuations`` block, so that it knows what to do with the named argument ``cc``. -> - Attempting to tail-call a regular function breaks the TCO chain and immediately returns to the original caller (provided the function even accepts a ``cc`` named argument). -> - Be careful: ``xs = list(args); return xs`` and ``return list(args)`` mean different things. -> - TCO is automatically applied to these tail calls. This uses the exact same machinery as the ``tco`` macro. +> - `return value` or `return Values(...)` is actually a tail-call into `cc`, passing the given value(s) as arguments. +> - As in other parts of `unpythonic`, returning a `Values` means returning multiple-return-values and/or named-return-values. +> - This is important if the return value is received by the assignment targets of a `call_cc[]`. If you get a `TypeError` concerning the arguments of a function with a name ending in `_cont_`, check your `call_cc[]` invocations and the `return` in the call_cc'd function. +> - **Changed in v0.15.0.** *Up to v0.14.3, multiple return values used to be represented as a `tuple`. Now returning a `tuple` means returning one value that is a tuple.* +> - `return func(...)` is actually a tail-call into `func`, passing along (by default) the current value of `cc` to become its `cc`. +> - Hence, the tail call is inserted *between* the end of the current function body and the start of the continuation `cc`. +> - To override which continuation to use, you can specify the `cc=...` kwarg, as in `return func(..., cc=mycc)`, as was done in the `amb` example above. +> - The `cc` argument, if passed explicitly, **must be passed by name**. +> - **CAUTION**: This is **not** enforced, as the machinery does not analyze positional arguments in any great detail. The machinery will most likely break in unintuitive ways (or at best, raise a mysterious `TypeError`) if this rule is violated. +> - The function `func` must be a defined in a `with continuations` block, so that it knows what to do with the named argument `cc`. +> - Attempting to tail-call a regular function breaks the TCO chain and immediately returns to the original caller (provided the function even accepts a `cc` named argument; if not, you will get a `TypeError`). +> - Be careful: `xs = list(args); return xs` and `return list(args)` mean different things. +> - Because `list(args)` is a function call, `return list(args)` will attempt to tail-call `list` as a continuation-enabled function (which it is not, you will get a `TypeError`), before passing its result into the current continuation. +> - Using `return xs` instead will pass an inert data value into the current continuation. +> - TCO is automatically applied to these tail calls. The TCO processing of `continuations` uses the exact same machinery as the `tco` macro, performing some additional AST edits via hooks. > -> - The ``call_cc[]`` statement essentially splits its use site into *before* and *after* parts, where the *after* part (the continuation) can be run a second and further times, by later calling the callable that represents the continuation. This makes a computation resumable from a desired point. +> - The `call_cc[]` statement essentially splits its use site into *before* and *after* parts, where the *after* part (the continuation) can be run a second and further times, by later calling the callable that represents the continuation. This makes a computation resumable from a desired point. > - The continuation is essentially a closure. -> - Just like in Scheme/Racket, only the control state is checkpointed by ``call_cc[]``; any modifications to mutable data remain. -> - Assignment targets can be used to get the return value of the function called by ``call_cc[]``. -> - Just like in Scheme/Racket's ``call/cc``, the values that get bound to the ``call_cc[]`` assignment targets on second and further calls (when the continuation runs) are the arguments given to the continuation when it is called (whether implicitly or manually). -> - A first-class reference to the captured continuation is available in the function called by ``call_cc[]``, as its ``cc`` argument. -> - The continuation is a function that takes positional arguments, plus a named argument ``cc``. -> - The call signature for the positional arguments is determined by the assignment targets of the ``call_cc[]``. -> - The ``cc`` parameter is there only so that a continuation behaves just like any continuation-enabled function when tail-called, or when later used as the target of another ``call_cc[]``. -> - Basically everywhere else, ``cc`` points to the identity function - the default continuation just returns its arguments. +> - Just like in Scheme/Racket, only the control state is checkpointed by `call_cc[]`; any modifications to mutable data remain. +> - Assignment targets can be used to get the return value of the function called by `call_cc[]`. +> - Just like in Scheme/Racket's `call/cc`, the values that get bound to the `call_cc[]` assignment targets on second and further calls (when the continuation runs) are the arguments given to the continuation when it is called (whether implicitly or manually). +> - A first-class reference to the captured continuation is available in the function called by `call_cc[]`, as its `cc` argument. +> - The continuation itself is a function that takes positional arguments, plus a named argument `cc`. +> - The call signature for the positional arguments is determined by the assignment targets of the `call_cc[]`. +> - The `cc` parameter is there only so that a continuation behaves just like any continuation-enabled function when tail-called, or when later used as the target of another `call_cc[]`. +> - Basically everywhere else, `cc` points to the identity function - the default continuation just returns its argument(s). > - This is unlike in Scheme or Racket, which implicitly capture the continuation at every expression. -> - Inside a ``def``, ``call_cc[]`` generates a tail call, thus terminating the original (parent) function. (Hence ``call_ec`` does not combo well with this.) -> - At the top level of the ``with continuations`` block, ``call_cc[]`` generates a normal call. In this case there is no return value for the block (for the continuation, either), because the use site of the ``call_cc[]`` is not inside a function. +> - Inside a `def`, `call_cc[]` generates a tail call, thus terminating the original (parent) function. Hence `call_ec` does **not** combo with `with continuations`. +> - At the top level of the `with continuations` block, `call_cc[]` generates a normal call. In this case there is no return value for the block (for the continuation, either), because the use site of the `call_cc[]` is not inside a function.
-#### Differences between ``call/cc`` and certain other language features +#### Topology of continuations: how the wiring works + +If you want to know how the gears actually mesh — what kind of object a continuation *is* at run time, why there are two arguments named `cc` and `pcc`, and how the chain unwinds when a function ends — this is the section. + +Topology of call_cc continuations: closures, cc, and pcc + +The diagram is dense the first time but each panel adds one idea on top of the previous. + +**Base case** (top left). A function `f` does `call_cc[g(...)]`, which captures the rest of `f`'s body as a closure. Let's name that closure `f_cont`. The macro builds it implicitly; the `call_cc` machinery passes it to `g` as the keyword argument `cc`. So `cc=f_cont` is just shorthand for "here is the rest of `f`, wrapped up so you can call it later." `f_cont` lives in `f`'s lexical scope — it sees `f`'s locals as enclosing-scope variables, exactly like any other nested closure. Once `g` has `cc` in hand, it can stash it somewhere for later use; that ability is the whole point of having `call_cc` in the first place. + +**Sequence of continuations** (second panel). If the rest-of-`f` itself contains another `call_cc`, then *the rest of the rest of `f`* gets captured as a second closure, `f_cont1`. By the same mechanism, `f_cont1` lives inside `f_cont`'s lexical scope. Two `call_cc`s in a row produce two nested closures — and the same thing happens for any number. This nesting is also why each `call_cc[]` introduces a scope boundary (see [Scoping of locals in continuations](#scoping-of-locals-in-continuations)): the boundaries are simply the closures' edges. + +**Nested continuations** (third panel). Now consider a `call_cc` that lives *inside* a function `g` that is itself being run as the body of an outer `call_cc`. The outer `call_cc` already arranged for `g`'s `cc` to point at `f_cont`. When `g` does its own `call_cc[h(...)]`, it captures the rest of `g`'s body as `g_cont`, and passes that to `h` as `cc=g_cont`. But there's a wrinkle: when `g_cont` eventually finishes, it needs to continue with `f_cont` (the *original* outer continuation), not just stop. To carry that information forward without disturbing the public `cc` argument, the machinery introduces a second argument: **`pcc`** ("parent cc"). On `g_cont` it is set to `f_cont`. The "Nearly equivalent?" panel on the right makes the same point in the special case where `g` was reached by tail call rather than by `call_cc`: a tail call must propagate whatever value `cc` had at the call site, since `cc` is the public API. + +**The chaining rule** (fourth panel — "The Confetti Scenarios"). Now we can write down the protocol that makes the whole thing work. *When a function ends, check `pcc` first. If `pcc` is set, tail-call it (passing the current `cc` along as its `cc`). If `pcc` is not set, tail-call `cc` directly.* That single rule is what threads a chain like `g_cont → f_cont` together correctly, and it generalises to chains of arbitrary length: the `cc` that was originally passed in always fires *last*, after the entire `pcc` chain has finished. The only place that ever sets `pcc` is the `call_cc[]` mechanism itself — specifically, at the moment it builds the definition of a continuation function. User code never touches `pcc`. + +**Tail-call composition** (bottom panels). One last wrinkle. When a continuation tail-calls another function while its own `pcc` is set, simply forwarding `cc` would skip the link in the chain that `pcc` represents. The fix is to *compose* `pcc`-then-`cc` into a new continuation, and pass that composed value as the callee's `cc`. The composed continuation, when it eventually fires, runs `pcc` first (with the original `cc` set as *its* `cc`), so the chain unwinds in the correct order. This composition rule applies recursively: nested tail calls compose nested chains. The general statement: a tail call checks for `pcc`; if set, makes a composed `cc`; if not, just passes along the existing `cc`. - - Unlike **generators**, ``call_cc[]`` allows resuming also multiple times from an earlier checkpoint, even after execution has already proceeded further. Generators can be easily built on top of ``call/cc``. [Python version](../unpythonic/syntax/test/test_conts_gen.py), [Racket version](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/beyond_python/generator.rkt). +**Putting it together.** `cc` is the public API — it gets set by the `call_cc[]` mechanism at capture time and propagated by tail calls. `pcc` is internal plumbing — set only by `call_cc[]` itself, used only by the chaining rule when a function ends or tail-calls another. From the outside, a user writes `call_cc[func(...)]` and `func` receives `cc`; from the inside, the macro and the chaining rule work together to ensure the captured continuation eventually fires in the right place no matter how deeply the call stack nests. + +#### Scoping of locals in continuations + +Each `call_cc[]` introduces a scope boundary. The continuation captured by `call_cc[]` — the rest of the function body that lexically follows it — is a **new closure**; importantly, it is not part of the surrounding lexical scope. Any name assigned in the continuation is local to the continuation, even if a name with the same spelling existed in the body before the `call_cc[]`. + +This is comparable to how Python's comprehensions and generator expressions also introduce a scope boundary, except here the boundary is invisible in the unexpanded source — the `call_cc[]` does not look like a `def`, but at expansion time it becomes one. + +The mechanism: under the hood, the continuation is a function. The assignment targets of `call_cc[]` become its parameters. Anything assigned later, anywhere in the body that lexically follows the `call_cc[]`, is a fresh local of that closure, by Python's standard scoping rules. For the closure-nesting topology that makes this scoping rule fall out, see [`callcc_topology.pdf`](callcc_topology.pdf) — the "Base case" panel shows a single continuation as a closure living in the lexical scope of its parent function, and the "Sequence of continuations" panel shows how chained `call_cc[]`s nest those closures. + +The practical implication: + +```python +with continuations: + def f(): + x = "before" + k = call_cc[get_cc()] + if iscontinuation(k): + return k + x = "after" # fresh local of the continuation; does not rebind the outer x + return x +``` + +Two distinct names, both spelled `x`. To share state across the boundary, use a [`box`](features.md#box-a-mutable-single-item-container) — replace its contents instead of rebinding the name: + +```python +with continuations: + def f(): + b = box("before") # `b` is now the box - the actual value is inside the box + k = call_cc[get_cc()] + if iscontinuation(k): + return k + b << "after" # send new value into the same box + return unbox(b) # return the value that is currently inside the box +``` + +For the authoritative reference, see the testsets `"scoping, using a box"` and `"scoping, locals only"` in [`unpythonic/syntax/tests/test_conts.py`](../unpythonic/syntax/tests/test_conts.py). + +**Why the macro doesn't auto-`nonlocal` parent locals into the continuation** + +A naïve fix would be to scan the parent body for assignments and emit `nonlocal` declarations for those names at the top of the continuation. An experiment along these lines was tried — the implementation lives as a commented-out `patch_scoping` function inside `unpythonic.syntax.tailtools._continuations` (search for `patch_scoping` in [`tailtools`](unpythonic/syntax/tailtools.py)) — but three load-bearing limits ruled it out: + +1. **Continuation parameters must shadow same-named parent locals.** The assignment targets of `call_cc[]` become the continuation's parameters. If the parent scope already has a name with the same spelling, declaring it `nonlocal` in the continuation would conflict with the parameter. So the auto-`nonlocal` rule has to *exclude* the continuation's parameters — which means at minimum those names cannot be lifted, and the principle of "as if same lexical scope" already breaks for them. + +2. **No upward propagation: a name introduced in the continuation cannot be made visible to code that ran before the `call_cc[]`.** Lifting *parent* locals into the continuation is a one-way street; making it bidirectional would require a second pass to discover continuation-introduced names and retro-declare them in the parent. The macro is one-pass (and already complex enough to make it one of the most intimidating parts of `unpythonic.syntax` to keep working during maintenance work). + +3. **At the top level of `with continuations:`, `nonlocal` vs `global` is undecidable from the AST alone.** Whether the block is at module top level (so parent assignments need `global`) or inside some enclosing function (so they need `nonlocal`) cannot be determined locally — the macro would need whole-module analysis to find out. + +Given limits 1 and 2, even a successful implementation would only *partially* maintain the illusion of "same scope" — and limit 3 would force an additional restriction or an extra analysis pass. The cure was less straightforward than the disease, so the abandoned experiment is preserved as commented-out code with these reasons recorded inline, and the rule "each `call_cc[]` is a scope boundary" is documented instead. The behavior is no worse than how Python itself treats comprehensions and generator expressions. + +For the deepest lurking gotcha: this also applies to *nested* `call_cc[]` invocations within the same function body — each one starts a new closure, so a chain of `call_cc[]`s creates a chain of nested closures, and a name assigned after the *n*th `call_cc[]` is local to the *n*th continuation, distinct from the same-spelled name assigned after the (*n*-1)th. + +#### Differences between `call/cc` and certain other language features + + - Unlike **generators**, `call_cc[]` allows resuming also multiple times from an earlier checkpoint, even after execution has already proceeded further. Generators can be easily built on top of `call/cc`. [Python version](../unpythonic/syntax/tests/test_conts_gen.py), [Racket version](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/beyond_python/generator.rkt). - The Python version is a pattern that could be packaged into a macro with `mcpyrate`; the Racket version has been packaged as a macro. - Both versions are just demonstrations for teaching purposes. In production code, use the language's native functionality. - - Python's built-in generators have no restriction on where ``yield`` can be placed, and provide better performance. + - Python's built-in generators have no restriction on where `yield` can be placed, and provide better performance. - Racket's standard library provides [generators](https://docs.racket-lang.org/reference/Generators.html). - - Unlike **exceptions**, which only perform escapes, ``call_cc[]`` allows to jump back at an arbitrary time later, also after the dynamic extent of the original function where the ``call_cc[]`` appears. Escape continuations are a special case of continuations, so exceptions can be built on top of ``call/cc``. + - Unlike **exceptions**, which only perform escapes, `call_cc[]` allows to jump back at an arbitrary time later, also *after* the dynamic extent of the original function where the `call_cc[]` appears. Escape continuations are a special case of continuations, so exceptions can be built on top of `call/cc`. - [As explained in detail by Matthew Might](http://matt.might.net/articles/implementing-exceptions/), exceptions are fundamentally based on (escape) continuations; the *"unwinding the call stack"* mental image is ["not even wrong"](https://en.wikiquote.org/wiki/Wolfgang_Pauli). -So if all you want is generators or exceptions (or even resumable exceptions a.k.a. [conditions](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html)), then a general ``call/cc`` mechanism is not needed. The point of ``call/cc`` is to provide the ability to *resume more than once* from *the same*, already executed point in the program. In other words, ``call/cc`` is a general mechanism for bookmarking the control state. +So if all you want is generators or exceptions (or even resumable exceptions a.k.a. [conditions](http://www.gigamonkeys.com/book/beyond-exception-handling-conditions-and-restarts.html)), then a general `call/cc` mechanism is not needed. The point of `call/cc` is to provide the ability to *resume more than once* from *the same*, already executed point in the program. In other words, **`call/cc` is a general mechanism for bookmarking the control state**. However, its usability leaves much to be desired. This has been noted e.g. in [Oleg Kiselyov: An argument against call/cc](http://okmij.org/ftp/continuations/against-callcc.html) and [John Shutt: Guarded continuations](http://fexpr.blogspot.com/2012/01/guarded-continuations.html). For example, Shutt writes: *The traditional Scheme device for acquiring a first-class continuation object is **call/cc**, which calls a procedure and passes to that procedure the continuation to which that call would normally return. Frankly, this was always a very clumsy way to work with continuations; one might almost suspect it was devised as an "esoteric programming language" feature, akin to INTERCAL's COME FROM statement.* -#### ``call_cc`` API reference +#### `call_cc` API reference -To keep things relatively straightforward, our ``call_cc[]`` is only allowed to appear **at the top level** of: +To keep things relatively straightforward, our `call_cc[]` is only allowed to appear **at the top level** of: - - the ``with continuations`` block itself - - a ``def`` or ``async def`` + - the `with continuations` block itself + - a `def` inside that block -Nested defs are ok; here *top level* only means the top level of the *currently innermost* ``def``. +Nested defs are ok; here *top level* only means the top level of the *currently innermost* `def`. -If you need to place ``call_cc[]`` inside a loop, use ``@looped`` et al. from ``unpythonic.fploop``; this has the loop body represented as the top level of a ``def``. +If you need to place `call_cc[]` inside a loop, use `@looped` et al. from the module `unpythonic.fploop`; this has the loop body represented as the top level of a `def`. Keep in mind that **only the control state is bookmarked**. -Multiple ``call_cc[]`` statements in the same function body are allowed. These essentially create nested closures. +Multiple `call_cc[]` statements in the same function body are allowed. These essentially create nested closures. In any invalid position, `call_cc[]` is considered a syntax error at macro expansion time. **Syntax**: -In ``unpythonic``, ``call_cc`` is a **statement**, with the following syntaxes: +In `unpythonic`, `call_cc` is a **statement**, with the following syntaxes: ```python -x = call_cc[func(...)] -*xs = call_cc[func(...)] -x0, ... = call_cc[func(...)] -x0, ..., *xs = call_cc[func(...)] -call_cc[func(...)] +x = call_cc[f(...)] +*xs = call_cc[f(...)] +x0, ... = call_cc[f(...)] +x0, ..., *xs = call_cc[f(...)] +call_cc[f(...)] x = call_cc[f(...) if p else g(...)] *xs = call_cc[f(...) if p else g(...)] @@ -1254,23 +1528,25 @@ x0, ..., *xs = call_cc[f(...) if p else g(...)] call_cc[f(...) if p else g(...)] ``` -*NOTE*: ``*xs`` may need to be written as ``*xs,`` in order to explicitly make the LHS into a tuple. The variant without the comma seems to work when run from a ``.py`` file with the `macropython` bootstrapper from [`mcpyrate`](https://pypi.org/project/mcpyrate/), but fails in code run interactively in the `mcpyrate` REPL. +*NOTE*: `*xs` may need to be written as `*xs,` in order to explicitly make the LHS into a tuple. The variant without the comma seems to work when run from a `.py` file with the `macropython` bootstrapper from [`mcpyrate`](https://pypi.org/project/mcpyrate/), but fails in code run interactively in the `mcpyrate` REPL. + +*NOTE*: `f()` and `g()` must be **literal function calls**. Sneaky trickery (such as calling indirectly via `unpythonic.call` or `unpythonic.curry`) is not supported. This limitation is for simplicity; the `call_cc[]` invocation needs to patch the `cc=...` kwarg of the call being made. -*NOTE*: ``f()`` and ``g()`` must be **literal function calls**. Sneaky trickery (such as calling indirectly via ``unpythonic.funutil.call`` or ``unpythonic.fun.curry``) is not supported. (The ``prefix`` and ``curry`` macros, however, **are** supported; just order the block macros as shown in the final section of this README.) This limitation is for simplicity; the ``call_cc[]`` needs to patch the ``cc=...`` kwarg of the call being made. +The `prefix` and `curry` macros, however, **are** supported; just order the block macros as in [The xmas tree combo](#the-xmas-tree-combo). **Assignment targets**: - - To destructure positional multiple-values (from a `Values` return value), use a tuple assignment target (comma-separated names, as usual). Destructuring *named* return values from a `call_cc` is currently not supported. + - To destructure positional multiple-values (from a `Values` return value of the function called by the `call_cc`), use a tuple assignment target (comma-separated names, as usual). Destructuring *named* return values from a `call_cc` is currently not supported due to syntactic limitations. - - The last assignment target may be starred. It is transformed into the vararg (a.k.a. ``*args``, star-args) of the continuation function. (It will capture a whole tuple, or any excess items, as usual.) + - The last assignment target may be starred. It is transformed into the vararg (a.k.a. `*args`, star-args) of the continuation function created by the `call_cc`. It will capture a whole tuple, or any excess items, as usual. - - To ignore the return value, just omit the assignment part. Useful if ``func`` was called only to perform its side-effects (the classic side effect is to stash ``cc`` somewhere for later use). + - To ignore the return value of the `call_cc`'d function, just omit the assignment part. This is useful if `f` was called only to perform its side-effects. The classic side effect is to stash `cc` somewhere for later use. **Conditional variant**: - - ``p`` is any expression. If truthy, ``f(...)`` is called, and if falsey, ``g(...)`` is called. + - `p` is any expression. It is evaluated at run time, as usual. When the result is truthy, `f(...)` is called, and when falsey, `g(...)` is called. - - Each of ``f(...)``, ``g(...)`` may be ``None``. A ``None`` skips the function call, proceeding directly to the continuation. Upon skipping, all assignment targets (if any are present) are set to ``None``. The starred assignment target (if present) gets the empty tuple. + - Each of `f(...)`, `g(...)` may be `None`. A `None` skips the function call, proceeding directly to the continuation. Upon skipping, all assignment targets (if any are present) are set to `None`. The starred assignment target (if present) gets the empty tuple. The main use case of the conditional variant is for things like: @@ -1285,45 +1561,51 @@ with continuations: ... ``` -**Main differences to ``call/cc`` in Scheme and Racket**: +**Main differences to `call/cc` in Scheme and Racket**: -Compared to Scheme/Racket, where ``call/cc`` will capture also expressions occurring further up in the call stack, our ``call_cc`` may be need to be placed differently (further out, depending on what needs to be captured) due to the delimited nature of the continuations implemented here. +Compared to Scheme/Racket, where `call/cc` will capture also expressions occurring further up in the call stack, our `call_cc` may be need to be placed differently (further out, depending on what needs to be captured) due to the delimited nature of the continuations implemented here. -Scheme and Racket implicitly capture the continuation at every position, whereas we do it explicitly, only at the use sites of the ``call_cc[]`` macro. +Scheme and Racket implicitly capture the continuation at every position, whereas we do it explicitly, only at the use sites of the `call_cc[]` macro. -Also, since there are limitations to where a ``call_cc[]`` may appear, some code may need to be structured differently to do some particular thing, if porting code examples originally written in Scheme or Racket. +Also, since there are limitations to where a `call_cc[]` may appear, some code may need to be structured differently to do some particular thing, if porting code examples originally written in Scheme or Racket. -Unlike ``call/cc`` in Scheme/Racket, our ``call_cc`` takes **a function call** as its argument, not just a function reference. Also, there's no need for it to be a one-argument function; any other args can be passed in the call. The ``cc`` argument is filled implicitly and passed by name; any others are passed exactly as written in the client code. +Unlike `call/cc` in Scheme/Racket, our `call_cc` takes **a function call** as its argument, not just a function reference. Also, there is no need for it to be a one-argument function; any other args can be passed in the call. The `cc` argument is filled implicitly and passed by name; any others are passed exactly as you write in the invocation. #### Combo notes -**CAUTION**: Do not use ``with tco`` inside a ``with continuations`` block; ``continuations`` already implies TCO. The ``continuations`` macro **makes no attempt** to skip ``with tco`` blocks inside it. +**CAUTION**: Do not use `with tco` inside a `with continuations` block; `continuations` already implies TCO. The `continuations` macro **makes no attempt** to skip `with tco` blocks inside it. -If you need both ``continuations`` and ``multilambda`` simultaneously, the incantation is: +If you want to use `multilambda` inside a `with continuations` block, it needs to go on the outside: ```python +from unpythonic.syntax import macros, continuations, multilambda + with multilambda, continuations: f = lambda x: [print(x), x**2] assert f(42) == 1764 ``` -This works, because the ``continuations`` macro understands already expanded ``let[]`` and ``do[]``, and ``multilambda`` generates and expands a ``do[]``. (Any explicit use of ``do[]`` in a lambda body or in a ``return`` is also ok; recall that macros expand from inside out.) +This works, because the `continuations` macro understands already expanded `let[]` and `do[]`, and `multilambda` generates and expands a `do[]`. (Any explicit use of `do[]` in a lambda body or in a `return` is also ok; recall that macros expand from inside out.) -Similarly, if you need ``quicklambda``, apply it first: +Similarly, if you want to use `quicklambda` inside a `with continuations` block, place it on the outside: ```python +from unpythonic.syntax import macros, continuations, quicklambda, fn + with quicklambda, continuations: - g = f[_**2] + g = fn[_**2] assert g(42) == 1764 ``` -This ordering makes the ``f[...]`` notation expand into standard ``lambda`` notation before ``continuations`` is expanded. +This ordering makes the `f[...]` notation expand into standard `lambda` notation before `continuations` is expanded. -To enable both of these, use ``with quicklambda, multilambda, continuations`` (although the usefulness of this combo may be questionable). +To enable both of these, use `with quicklambda, multilambda, continuations` (although the usefulness of this combo may be questionable). #### Continuations as an escape mechanism -Pretty much by the definition of a continuation, in a ``with continuations`` block, a trick that *should* at first glance produce an escape is to set ``cc`` to the ``cc`` of the caller, and then return the desired value. There is however a subtle catch, due to the way we implement continuations. +An escape continuation `ec` is a continuation, too. How can we use `cc` to escape? + +Pretty much by the definition of a continuation, in a `with continuations` block, a trick that *should* at first glance produce an escape is to set `cc` to the `cc` of the caller, and then return the desired value. There is however a subtle catch, due to the way we implement continuations. First, consider this basic strategy, without any macros: @@ -1333,7 +1615,7 @@ from unpythonic import call_ec def double_odd(x, ec): if x % 2 == 0: # reject even "x" ec("not odd") - return 2*x + return 2 * x @call_ec def result1(ec): y = double_odd(42, ec) @@ -1348,7 +1630,9 @@ assert result1 == "not odd" assert result2 == "not odd" ``` -Now, can we use the same strategy with the continuation machinery? +Here `ec` is the escape continuation of the `result1`/`result2` block, due to the placement of the `call_ec`. + +Now, can we use the same strategy with the general continuation machinery? ```python from unpythonic.syntax import macros, continuations, call_cc @@ -1358,9 +1642,9 @@ with continuations: if x % 2 == 0: cc = ec return "not odd" - return 2*x + return 2 * x def main1(cc): - # cc actually has a default, so it's ok to not pass anything as cc here. + # cc actually has a default (`identity`), so it's ok to not pass anything as cc here. y = double_odd(42, ec=cc) # y = "not odd" z = double_odd(21, ec=cc) # we could tail-call, but let's keep this similar to the first example. return z @@ -1372,11 +1656,13 @@ with continuations: assert main2() == "not odd" ``` -In the first example, ``ec`` is the escape continuation of the ``result1``/``result2`` block, due to the placement of the ``call_ec``. In the second example, the ``cc`` inside ``double_odd`` is the implicitly passed ``cc``... which, naively, should represent the continuation of the current call into ``double_odd``. So far, so good. +The `cc` inside `double_odd` is the implicitly passed `cc`... which, naively, should represent the continuation of the current call into `double_odd`. So far, so good. -However, because the example code contains no ``call_cc[]`` statements, the actual value of ``cc``, anywhere in this example, is always just ``identity``. *It's not the actual continuation.* Even though we pass the ``cc`` of ``main1``/``main2`` as an explicit argument "``ec``" to use as an escape continuation (like the first example does with ``ec``), it is still ``identity`` - and hence cannot perform an escape. +However, because the example contains no `call_cc[]` statements, the actual value of `cc`, anywhere in this example, is always just `identity`. Scan that again: *in this example, `cc` is not the actual continuation, because no continuation captures were requested.* -We must ``call_cc[]`` to request a capture of the actual continuation: +Even though we pass the `cc` of `main1`/`main2` as an explicit argument "`ec`" to use as an escape continuation (like the first example does with `ec`), it is still `identity` - and hence cannot perform an escape. + +We must `call_cc[]` to request a capture of the continuation, hence populating `cc` with something useful: ```python from unpythonic.syntax import macros, continuations, call_cc @@ -1386,7 +1672,7 @@ with continuations: if x % 2 == 0: cc = ec return "not odd" - return 2*x + return 2 * x def main1(cc): y = call_cc[double_odd(42, ec=cc)] # <-- the only change is adding the call_cc[] z = call_cc[double_odd(21, ec=cc)] # <-- @@ -1401,49 +1687,194 @@ with continuations: This variant performs as expected. -There's also a second, even subtler catch; instead of setting ``cc = ec`` and returning a value, just tail-calling ``ec`` with that value doesn't do what we want. This is because - as explained in the rules of the ``continuations`` macro, above - a tail-call is *inserted* between the end of the function, and whatever ``cc`` currently points to. +There is also a second, even subtler catch; instead of setting `cc = ec` and returning a value, as we did, just tail-calling `ec` with that same value does **not** do what we want. Why? Because - as explained in the rules of the `continuations` macro, above - a tail-call is *inserted* between the end of the function, and whatever continuation `cc` currently points to. + +Most often that is exactly what we want, but in this particular case, it causes *both* continuations to run, in sequence. But if, instead of performing a tail call to the `ec`, we set `cc = ec`, then the function's original `cc` argument (the one supplied by `call_cc[]`) is discarded, hence that continuation never runs - and we get the effect we want, *replacing* the `cc` by the `ec`. + +Such subtleties arise essentially from the difference between a language that natively supports continuations (Scheme, Racket) and one that has continuations hacked on top of it as macros performing a CPS conversion only partially (like Python with `unpythonic.syntax`, or Common Lisp with PG's continuation-passing macros). The macro approach works, but the programmer needs to be careful. + +#### Multi-shot generators with `@multishot` and `myield` + +A `@multishot` function is a generator-shaped construct on top of `call_cc[]`: it looks like a Python generator, but at every `myield` the execution state is captured *as a continuation*, so the function can be resumed from any earlier `myield` arbitrarily many times — branching execution into independent timelines. (Standard generators are single-shot continuations: once execution passes a `yield`, it cannot be rewound. Multi-shot is the more general construct; the one-shot vs. multi-shot distinction goes back at least to [Bruggeman, Waddell & Dybvig 1996](https://legacy.cs.indiana.edu/~dyb/pubs/call1cc.pdf), and Racket's [continuation reference](https://docs.racket-lang.org/reference/cont.html) is the canonical modern source.) + +Only meaningful inside a `with continuations:` block — required, not auto-wrapped (Zen of Python: explicit is better than implicit). The expansion of `myield` produces `call_cc[get_cc()]`, so outside `with continuations` it fails at macro-expansion time with the standard `call_cc[]` error. + +`myield` has four forms: + +| Multi-shot yield | Returns | `k` expects | Single-shot analog | +|----------------------|---------------|---------------|--------------------------| +| `myield` | `k` | no argument | `yield` | +| `myield[expr]` | `(k, value)` | no argument | `yield expr` | +| `var = myield` | `k` | one argument | `var = yield` | +| `var = myield[expr]` | `(k, value)` | one argument | `var = yield expr` | + +To resume, call `k`. In cases where `k` expects an argument, that argument is the value to send into `var`. + +`myield` is a *statement* and may only appear at the top level of a `@multishot` function definition. (This is a real limitation of the underlying `call_cc[]`. Use inside lambdas, comprehensions, or nested `def`s is rejected at macro-expansion time.) + +`return value` inside `@multishot` raises `StopIteration(value)`, mirroring the standard generator protocol. + +Basic usage: + +```python +from unpythonic.syntax import macros, continuations, multishot, myield + +with continuations: + @multishot + def f(): + myield # stop; return continuation `k` + myield[42] # stop; return (k, 42) + k = myield # stop; return k. Upon resume, set local `k` to the value sent in. + k = myield[42] # stop; return (k, 42). Upon resume, set local `k`. + + # Instantiate the multi-shot generator. + # There is always an implicit bare `myield` at the beginning, so f() returns + # the initial continuation rather than running the body immediately. + k0 = f() + k1 = k0() # run up to the explicit bare `myield` + k2, x2 = k1() # to `myield[42]` + k3 = k2() # to `k = myield` + k4, x4 = k3(23) # send 23, run to `k = myield[42]` + # k4(17) → StopIteration + + # Multi-shot: re-invoke an earlier continuation. + k2_alt, x2_alt = k1() +``` + +For ergonomic generator-shaped consumption, wrap the initial continuation in a `MultishotIterator`: + +```python +from unpythonic.syntax import MultishotIterator + +with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + mi = MultishotIterator(g()) + assert [x for x in mi] == [1, 2, 3] +``` + +`MultishotIterator` supports a subset of the generator protocol: `iter`, `next`, `send`, `throw`, `close`, `gi_code`, `gi_frame`, `gi_running`, `gi_yieldfrom`. Plus one method that standard generators don't have: + +```python +import copy + +with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + myield[4] + + mi = MultishotIterator(g()) + next(mi) # → 1 + fork = copy.copy(mi) # snapshot the current continuation + next(mi); next(mi) # original advances independently... → 2, 3 + next(fork) # ...and so does the fork. → 2 +``` + +Unlike standard generators, multi-shot generators support `copy.copy()`. The fork shares the current continuation; each subsequent advance creates a fresh *activation record* (the call frame and local-variable storage for one invocation), so the forks' locals diverge from the next advance onward. The fork is shallow, though. Names that the body reads or writes from an *enclosing* scope — a `nonlocal` in the surrounding function, a captured free variable, a mutable argument — live in shared boxes that CPython calls *closure cells*: each captured name has one cell, and the inner function and its enclosing scope read and write the *same* cell. (Module-level state works similarly via the module's `__dict__`.) Forks share those cells, so a mutation through any of them in one fork's timeline is visible to the others. Forks are independent timelines for *locals* only — not for state reached through closure cells. The same applies to plain re-invocation of an earlier continuation; `copy.copy()` just exposes the multi-shot semantics through the stdlib protocol. + +`copy.deepcopy(mi)` raises `TypeError`. The continuation closes over caller state we can't meaningfully deep-copy; use `copy.copy()` to fork. + +##### How does `copy.copy(mi)` differ from `k = mi.k`? + +It almost doesn't. `copy.copy(mi)` is essentially `MultishotIterator(mi.k)` — plus preservation of the closed-flag, plus the convenience of doing the snapshot in one operation (since `mi.k` is overwritten on each `next`/`send`/`throw`, you'd otherwise have to remember to grab `k` *before* the next advance). The fork is iterator-shaped from day one, so it composes with `for x in fork`, `next(fork)`, and `fork.gi_yieldfrom`; a bare `k` would have to be re-wrapped in a `MultishotIterator` to do the same. + +Use `k = mi.k` (or capture `k` from a destructuring like `k1, x1 = mi.k(...)`) when you want the raw continuation — for instance, to drive it from a custom orchestrator. Use `copy.copy(mi)` when you want a second consumer-shaped iterator and idiomatic stdlib-style code at the call site. + +(With standard generators, neither path is available: `copy.copy(real_gen)` raises `TypeError`, and `gen.gi_frame` isn't a continuation you could wrap and re-invoke. Multi-shot offers both.) + +##### Delegating to another multi-shot: `myield_from` + +The multi-shot analog of `yield from`. Inside an outer `@multishot` body, `myield_from[inner_call()]` drives a second `@multishot` to exhaustion, re-yielding each of its values to outer's caller. On inner's `StopIteration`, execution continues in outer's body. Two forms (subscript syntax — same convention as `myield[expr]`): + +```python +with continuations: + @multishot + def inner(): + myield[1] + myield[2] + return 99 # surfaces as StopIteration(99) at the boundary + + @multishot + def outer(): + myield[0] + result = myield_from[inner()] # binds inner's StopIteration value + myield[result] # → 99 +``` + +The statement form `myield_from[inner_call()]` discards inner's `StopIteration` value; the assignment form `var = myield_from[inner_call()]` binds it. + +`send` and `throw` from the outer's caller are forwarded into the inner. While delegating, `outer_mi.gi_yieldfrom` returns the inner `MultishotIterator` (mirroring the standard generator's `gi_yieldfrom`); it returns `None` again once inner is exhausted. + +`myield_from` is statement-only and may only appear at the top level of a `@multishot` body — same placement constraint as `myield`. Inside lambdas, comprehensions, or nested `def`s it is rejected at macro-expansion time. -Most often that's exactly what we want, but in this particular case, it causes *both* continuations to run, in sequence. But if we overwrite ``cc``, then the function's original ``cc`` argument (the one given by ``call_cc[]``) is discarded, so it never runs - and we get the effect we want, *replacing* the ``cc`` by the ``ec``. +**Architecture note for the curious.** The expansion captures "rest of outer" via `_rest = call_cc[get_cc()]` (the multi-shot analog of Racket's `(let/cc return ...)`), tail-calls a small driver, and uses the *cut-the-tail* trick (`cc = identity`) inside a yield helper to deliver each `(captured_cc, value)` pair straight to the user's `mi._k()` trampoline. When inner exhausts, the driver invokes the captured rest-cc to resume outer's body just past the `myield_from` statement. -Such subtleties arise essentially from the difference between a language that natively supports continuations (Scheme, Racket) and one that has continuations hacked on top of it as macros performing a CPS conversion only partially (like Python with ``unpythonic.syntax``, or Common Lisp with PG's continuation-passing macros). The macro approach works, but the programmer needs to be careful. +**Limitation: cross-form delegation is wontfix.** `myield_from` is multi-shot-to-multi-shot only; you cannot `myield_from` a standard generator (the semantic mismatch is the same as for `yield from` in the other direction). + +##### Differences from standard Python generators + +Beyond what's already mentioned above: + +- **`gi_frame` is always `None`.** A multi-shot generator has no paused frame — every `myield` terminated its frame and returned a continuation closure. State lives in the closure cells of the continuation, not in any frame. The standard-generator idiom `gen.gi_frame is None ↔ exhausted` does **not** apply; use `mi.gi_code is None` as the liveness signal instead. +- **`gi_running` is always `False`.** Nothing is ever paused. +- **`yield from` across a standard generator and a multi-shot generator is not supported.** Standard generators have paused state, multi-shots don't; the semantic mismatch can't be papered over. Multi-shot-to-multi-shot delegation is supported via `myield_from`; see the dedicated subsection above. +- **⚠ `with`, `try`/`finally`, and `with handlers` across `myield` boundaries do not behave as in standard generators — load-bearing gotcha.** When a `myield` is reached, the `@multishot` function technically *returns* (the `myield` macro expansion compiles into a `return` of a continuation). All `with` `__exit__` and `try`/`finally` clauses lexically containing the `myield` therefore fire *at the `myield`*, not at the end of the multi-shot. Resuming the continuation jumps back into mid-body — the `with`/`try` is *not* re-entered (no second `__enter__` call), so a `with open(...) as f: myield[1]; myield[2]` will see `f` already closed at `myield[2]`. Same for unpythonic conditions: a `with handlers(...)` lexically containing a `myield` uninstalls the handler at the `myield` and the handler is *not* re-installed on resume. **Workaround for cleanup:** do it explicitly — call `f.close()` after the consumer is done, or use a `try/except StopIteration` in the caller. **Workaround for handlers:** install `with handlers(...)` *outside* the `@multishot` body, in the calling code that consumes the iterator. Conditions raised at any point during multi-shot consumption then propagate to that outer handler normally. For an example of what the world's serious `call/cc`-having languages do for this kind of thing, see Racket's [`dynamic-wind`](https://docs.racket-lang.org/reference/cont.html#%28def._%28%28quote._~23~25kernel%29._dynamic-wind%29%29). +- **No async form.** No `__aiter__`, `asend`, etc.; multi-shot is sync-only. +- **No pickling.** Continuations are closures. + +##### Cross-references + +- [`unpythonic.syntax.multishot`](../unpythonic/syntax/multishot.py) — implementation: the macros and `MultishotIterator`. +- [`unpythonic/syntax/tests/test_multishot.py`](../unpythonic/syntax/tests/test_multishot.py) — canonical usage of `@multishot`, `myield`, and `MultishotIterator`. +- [`unpythonic/syntax/tests/test_conts_gen.py`](../unpythonic/syntax/tests/test_conts_gen.py) — single-shot generators built directly on `call_cc[]`/`@dlet`, kept as a teaching example for readers studying the underlying mechanics. The toy implementation predates `@multishot`; it shows the manual pattern that `@multishot` automates. #### What can be used as a continuation? -In ``unpythonic`` specifically, a continuation is just a function. ([As John Shutt has pointed out](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html), in general this is not true. The calculus underlying the language becomes much cleaner if continuations are defined as a separate control flow mechanism orthogonal to function application. Continuations are not intrinsically a whole-computation device, either.) +In `unpythonic` specifically, a continuation is just a function. ([As John Shutt has pointed out](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html), in general this is not true. The calculus underlying the language becomes much cleaner if continuations are defined as a separate control flow mechanism orthogonal to function application. Continuations are [not intrinsically a whole-computation device](https://en.wikipedia.org/wiki/Delimited_continuation), either.) The continuation function must be able to take as many positional arguments as the previous function in the TCO chain is trying to pass into it. Keep in mind that: - - In ``unpythonic``, a tuple represents multiple return values. So a ``return a, b``, which is being fed into the continuation, implies that the continuation must be able to take two positional arguments. + - In `unpythonic`, multiple return values (and named return values) are represented as a `Values` object. So if your function does `return Values(a, b)`, and that is being fed into the continuation, this implies that the continuation must be able to take two positional arguments. - - At the end of any function in Python, at least an implicit bare ``return`` always exists. It will try to pass in the value ``None`` to the continuation, so the continuation must be able to accept one positional argument. (This is handled automatically for continuations created by ``call_cc[]``. If no assignment targets are given, ``call_cc[]`` automatically creates one ignored positional argument that defaults to ``None``.) + **Changed in v0.15.0.** *Up to v0.14.3, a `tuple` used to represent multiple-return-values; now it denotes a single return value that is a tuple. The `Values` type allows not only multiple return values, but also **named** return values. Named return values are fed as kwargs.* -If there is an arity mismatch, Python will raise ``TypeError`` as usual. (The actual error message may be unhelpful due to the macro transformations; look for a mismatch in the number of values between a ``return`` and the call signature of a function used as a continuation (most often, the ``f`` in a ``cc=f``).) + - At the end of any function in Python, at least an implicit bare `return` always exists. It will try to pass in the value `None` to the continuation, so a continuation must be able to accept one positional argument. + - This is handled automatically for continuations created by `call_cc[]`. If no assignment targets are given, `call_cc[]` automatically creates one ignored positional argument that defaults to `None`. -Usually, a function to be used as a continuation is defined inside the ``with continuations`` block. This automatically introduces the implicit ``cc`` parameter, and in general makes the source code undergo the transformations needed by the continuation machinery. +If there is an arity mismatch, Python will raise `TypeError` as usual. The actual error message may be unhelpful due to macro transformations. Look for a mismatch between a `return` and the call signature of a function used as a continuation. Most often, this is the `f` in a `cc=f`. -However, as the only exception to this rule, if the continuation is meant to act as the endpoint of the TCO chain - i.e. terminating the chain and returning to the original top-level caller - then it may be defined outside the ``with continuations`` block. Recall that in a ``with continuations`` block, returning an inert data value (i.e. not making a tail call) transforms into a tail-call into the ``cc`` (with the given data becoming its argument(s)); it does not set the ``cc`` argument of the continuation being called, or even require that it has a ``cc`` parameter that could accept one. +Usually, a function to be used as a continuation is defined inside the `with continuations` block. This automatically introduces the implicit `cc` parameter, and in general makes the source code undergo the transformations needed by the continuation machinery. -(Note also that a continuation that has no ``cc`` parameter cannot be used as the target of an explicit tail-call in the client code, since a tail-call in a ``with continuations`` block will attempt to supply a ``cc`` argument to the function being tail-called. Likewise, it cannot be used as the target of a ``call_cc[]``, since this will also attempt to supply a ``cc`` argument.) +However, as the only exception to this rule, if the continuation is meant to act as the endpoint of the TCO chain - i.e. terminating the chain and returning to the original top-level caller - then it may be defined outside the `with continuations` block. Recall that in a `with continuations` block, returning an inert data value (i.e. not making a tail call) transforms into a tail-call into the `cc` (with the given data becoming its argument(s)); it does not set the `cc` argument of the continuation being called, or even require that it has a `cc` parameter that could accept one. -These observations make ``unpythonic.fun.identity`` eligible as a continuation, even though it is defined elsewhere in the library and it has no ``cc`` parameter. +These observations make `unpythonic.identity` eligible as a continuation, even though it is defined elsewhere in the library and it has no `cc` parameter. -#### This isn't ``call/cc``! +Finally, note that a function that has no `cc` parameter cannot be used as the target of an explicit tail-call inside a `with continuations` block, since a tail-call there will attempt to supply a `cc` argument to the function being tail-called. Likewise, it cannot be used as the function called by a `call_cc[]`, since this will also attempt to supply a `cc` argument. -Strictly speaking, ``True``. The implementation is very different (much more than just [exposing a hidden parameter](https://www.ps.uni-saarland.de/~duchier/python/continuations.html)), not to mention it has to be a macro, because it triggers capture - something that would not need to be requested for separately, had we converted the whole program into [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style). +#### This isn't `call/cc`! -The selective capture approach is however more efficient when we implement the continuation system in Python, indeed *on Python* (in the sense of [On Lisp](paulgraham.com/onlisp.html)), since we want to run most of the program the usual way with no magic attached. This way there is no need to sprinkle absolutely every statement and expression with a ``def`` or a ``lambda``. (Not to mention Python's ``lambda`` is underpowered due to the existence of some language features only as statements, so we would need to use a mixture of both, which is already unnecessarily complicated.) Function definitions are not intended as [the only control flow construct](https://dspace.mit.edu/handle/1721.1/5753) in Python, so the compiler likely wouldn't optimize heavily enough (i.e. eliminate **almost all** of the implicitly introduced function definitions), if we attempted to use them as such. +Strictly speaking, `True`. The implementation is very different (much more than just [exposing a hidden parameter](https://www.ps.uni-saarland.de/~duchier/python/continuations.html)), not to mention it has to be a macro, because it triggers capture - something that would not need to be requested for separately, had we converted the whole program into [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style). + +The selective capture approach is however more efficient when we implement the continuation system in Python, indeed *on Python* (in the sense of [On Lisp](paulgraham.com/onlisp.html)), since we want to run most of the program the usual way with no magic attached. This way there is no need to sprinkle absolutely every statement and expression with a `def` or a `lambda`. (Not to mention Python's `lambda` is underpowered due to the existence of some language features only as statements, so we would need to use a mixture of both, which is already unnecessarily complicated.) Function definitions are not intended as [the only control flow construct](https://dspace.mit.edu/handle/1721.1/5753) in Python, so the compiler likely would not optimize heavily enough (i.e. eliminate **almost all** of the implicitly introduced function definitions), if we attempted to use them as such. Continuations only need to come into play when we explicitly request for one ([ZoP §2](https://www.python.org/dev/peps/pep-0020/)); this avoids introducing any more extra function definitions than needed. -The name is nevertheless ``call_cc``, because the resulting behavior is close enough to ``call/cc``. +The name is nevertheless `call_cc`, because the resulting behavior is close enough to `call/cc`. Instead of *call with current continuation*, we could retcon the name to mean *call with **captured** continuation*. -Note our implementation provides a rudimentary form of *delimited* continuations. See [Oleg Kiselyov: Undelimited continuations are co-values rather than functions](http://okmij.org/ftp/continuations/undelimited.html). Delimited continuations return a value and can be composed, so they at least resemble functions (even though are not, strictly speaking, actually functions), whereas undelimited continuations do not even return. (For two different debunkings of the continuations-are-functions myth, approaching the problem from completely different angles, see the above post by Oleg Kiselyov, and [John Shutt: Continuations and term-rewriting calculi](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html).) +Note our implementation provides a rudimentary form of *delimited* continuations. See [Oleg Kiselyov: Undelimited continuations are co-values rather than functions](http://okmij.org/ftp/continuations/undelimited.html). Delimited continuations return a value and can be composed, so they at least resemble functions (even though are not, strictly speaking, actually functions), whereas undelimited continuations do not even return. For two different debunkings of the continuations-are-functions myth, approaching the problem from completely different angles, see the above post by Oleg Kiselyov, and [John Shutt: Continuations and term-rewriting calculi](http://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html). Racket provides a thought-out implementation of delimited continuations and [prompts](https://docs.racket-lang.org/guide/prompt.html) to control them. #### Why this syntax? -As for a function call in ``call_cc[...]`` vs. just a function reference: Typical lispy usage of ``call/cc`` uses an inline lambda, with the closure property passing in everything except ``cc``, but in Python ``def`` is a statement. A technically possible alternative syntax would be: +As for a function call in `call_cc[...]` vs. just a function reference: Typical lispy usage of `call/cc` uses an inline lambda, with the closure property passing in everything except `cc`, but in Python `def` is a statement. A technically possible alternative syntax would be: ```python with call_cc(f): # this syntax not supported! @@ -1453,17 +1884,27 @@ with call_cc(f): # this syntax not supported! but the expr macro variant provides better options for receiving multiple return values, and perhaps remains closer to standard Python. -The ``call_cc[]`` explicitly suggests that these are (almost) the only places where the ``cc`` argument obtains a non-default value. It also visually indicates the exact position of the checkpoint, while keeping to standard Python syntax. +The `call_cc[]` explicitly suggests that these are (almost) the only places where the `cc` argument obtains a non-default value. It also visually indicates the exact position of the checkpoint, while keeping to standard Python syntax. -(*Almost*: As explained above, a tail call passes along the current value of ``cc``, and ``cc`` can be set manually.) +(*Almost*: As explained above, a tail call passes along the current value of `cc`, and `cc` can be set manually.) -### ``prefix``: prefix function call syntax for Python +### `prefix`: prefix function call syntax for Python Write Python almost like Lisp! -Lexically inside a ``with prefix`` block, any literal tuple denotes a function call, unless quoted. The first element is the operator, the rest are arguments. Bindings of the ``let`` macros and the top-level tuple in a ``do[]`` are left alone, but ``prefix`` recurses inside them (in the case of bindings, on each RHS). +Experimental, and not for use in production code — as the module docstring of +[`unpythonic.syntax.prefix`](../unpythonic/syntax/prefix.py) also says. It is one of the components +of the [Listhell](dialects/listhell.md) dialect. + +**Note the name collision on `q` and `u`.** The `q`, `u` and `kw` below are prefix-mode markers, +and are unrelated to `mcpyrate`'s quasiquote operators `q` and `u`, despite sharing both the names +and the words "quote" and "unquote". The two sets can genuinely meet: `prefix.py` itself imports +`mcpyrate`'s pair for its own implementation while exporting its own. A module that needs both must +alias one at the macro-import. + +Lexically inside a `with prefix` block, any literal tuple denotes a function call, unless quoted. The first element is the operator, the rest are arguments. Bindings of the `let` macros and the top-level tuple in a `do[]` are left alone, but `prefix` recurses inside them (in the case of let-bindings, on each RHS). The rest is best explained by example: @@ -1505,7 +1946,7 @@ with prefix: # in case of duplicate name across kws, rightmost wins assert (f, kw(a="hi there"), kw(b="Tom"), kw(b="Jerry")) == (q, "hi there", "Jerry") - # give *args with unpythonic.fun.apply, like in Lisps: + # give *args with unpythonic.apply, like in Lisps: lst = [1, 2, 3] def g(*args): return args @@ -1516,7 +1957,7 @@ with prefix: If you use the `q`, `u` and `kw()` operators, they must be macro-imported. The `q`, `u` and `kw()` operators may only appear in a tuple inside a prefix block. In any invalid position, any of them is considered a syntax error at macro expansion time. -This comboes with ``autocurry`` for an authentic *Listhell* programming experience: +The `prefix` macro comboes with `autocurry` for an authentic *Listhell* programming experience: ```python from unpythonic.syntax import macros, autocurry, prefix, q, u, kw @@ -1529,14 +1970,20 @@ with prefix, autocurry: # important: apply prefix first, then autocurry assert (mymap, double, (q, 1, 2, 3)) == ll(2, 4, 6) ``` -**CAUTION**: The ``prefix`` macro is experimental and not intended for use in production code. +See also [the Listhell dialect](dialects/listhell.md), which pre-packages that combo. + +**CAUTION**: The `prefix` macro is experimental and not intended for use in production code. + +### `autoreturn`: implicit `return` in tail position -### ``autoreturn``: implicit ``return`` in tail position +**Changed in v0.15.0.** *If the item in tail position is a function definition or class definition, return the thing that was defined. This functionality being missing in earlier versions was an oversight.* -In Lisps, a function implicitly returns the value of the expression in tail position (along the code path being executed). Python's ``lambda`` also behaves like this (the whole body is just one return-value expression), but ``def`` doesn't. +In Lisps, a function implicitly returns the value of the expression in tail position along the code path being executed. That is, "the last value" is automatically returned when the function terminates normally. No `return` keyword is needed. -Now ``def`` can, too: +Python's `lambda` also already behaves like this; the whole body is just one expression, whose value will be returned. + +However, `def` requires a `return`, even in tail position. Enter the `autoreturn` macro: ```python from unpythonic.syntax import macros, autoreturn @@ -1560,67 +2007,155 @@ with autoreturn: assert g(42) == "something else" ``` -Each ``def`` function definition lexically within the ``with autoreturn`` block is examined, and if the last item within the body is an expression ``expr``, it is transformed into ``return expr``. Additionally: +Each `def` or `async def` function definition lexically within the `with autoreturn` block is examined. + +Any explicit `return` statements are left alone, so `return` can still be used as usual. This is especially useful if you want to return early (before execution reaches the tail position). + +To find and transform the statement(s) in tail position, we look at the last statement within the function definition. If it is: + + - An expression `expr`, it is transformed into `return expr`. + + - A function or class definition, a return statement is appended to return that function/class. **Added in v0.15.0.** - - If the last item is an ``if``/``elif``/``else`` block, the transformation is applied to the last item in each of its branches. + - An `if`/`elif`/`else` block, the transformation is applied recursively to the last item in each of its branches. + - **CAUTION**: If the final `else` of an `if`/`elif`/`else` is omitted, as often in Python, then only the `else` item is in tail position with respect to the function definition - likely not what you want. So with `autoreturn`, the final `else` should be written out explicitly, to include the `else` branch into the `if`/`elif`/`else` statement. - - If the last item is a ``with`` or ``async with`` block, the transformation is applied to the last item in its body. + - A `with` or `async with` block, the transformation is applied recursively to the last item in its body. - - If the last item is a ``try``/``except``/``else``/``finally`` block: - - **If** an ``else`` clause is present, the transformation is applied to the last item in it; **otherwise**, to the last item in the ``try`` clause. These are the positions that indicate a normal return (no exception was raised). - - In both cases, the transformation is applied to the last item in each of the ``except`` clauses. - - The ``finally`` clause is not transformed; the intention is it is usually a finalizer (e.g. to release resources) that runs after the interesting value is already being returned by ``try``, ``else`` or ``except``. + - A `try`/`except`/`else`/`finally` block: + - **If** an `else` clause is present, the transformation is applied recursively to the last item in it; **otherwise**, to the last item in the `try` clause. These are the positions that indicate a normal return (i.e. no exception was raised). + - In both cases, the transformation is applied recursively to the last item in each of the `except` clauses. + - The `finally` clause is not transformed; it is intended as a finalizer (e.g. to release resources) that runs after the interesting value is already being returned by `try`, `else` or `except`. -If needed, the above rules are applied recursively to locate the tail position(s). +**CAUTION**: `for`, `async for`, `while` are currently not analyzed; effectively, these are defined as always returning `None`. If the last item in your function body is a loop, use an explicit return. -Any explicit ``return`` statements are left alone, so ``return`` can still be used as usual. +**CAUTION**: With `autoreturn` enabled, functions no longer return `None` by default; the whole point of this macro is to change the default return value. The default return value becomes `None` only if the tail position contains a statement other than `if`, `with`, `async with` or `try`. -**CAUTION**: If the final ``else`` of an ``if``/``elif``/``else`` is omitted, as often in Python, then only the ``else`` item is in tail position with respect to the function definition - likely not what you want. So with ``autoreturn``, the final ``else`` should be written out explicitly, to make the ``else`` branch part of the same ``if``/``elif``/``else`` block. +If you wish to omit `return` in tail calls, `autoreturn` comboes with `tco`. For the correct invocation order, see [the xmas tree combo](#the-xmas-tree-combo). -**CAUTION**: ``for``, ``async for``, ``while`` are currently not analyzed; effectively, these are defined as always returning ``None``. If the last item in your function body is a loop, use an explicit return. +For code using **conditions and restarts**: there is no special integration between `autoreturn` and the conditions-and-restarts subsystem of `unpythonic`. However, these should work together, because: -**CAUTION**: With ``autoreturn`` enabled, functions no longer return ``None`` by default; the whole point of this macro is to change the default return value. The default return value is ``None`` only if the tail position contains a statement other than ``if``, ``with``, ``async with`` or ``try``. + - The `with restarts` form is just a `with` block, so it gets the `autoreturn` treatment. + - The handlers in a `with handlers` form are either separately defined functions, or lambdas. + - Lambdas need no `autoreturn`. + - If you `def` the handler functions in a `with autoreturn` block (either the same one or a different one; this does not matter), they will get the `autoreturn` treatment. + - The `with handlers` form itself is just `with` block, so it also gets the `autoreturn` treatment. -If you wish to omit ``return`` in tail calls, this comboes with ``tco``; just apply ``autoreturn`` first (either ``with autoreturn, tco:`` or in nested format, ``with tco:``, ``with autoreturn:``). +### `monadic_do`: do-notation for any monad -### ``forall``: nondeterministic evaluation +**Added in v2.1.0.** -Behaves the same as the multiple-body-expression tuple comprehension ``unpythonic.amb.forall``, but implemented purely by AST transformation, with real lexical variables. This is essentially a macro implementation of Haskell's do-notation for Python, specialized to the List monad (but the code is generic and very short; see ``unpythonic.syntax.forall``). +Monadic do-notation over any of the monads in [`unpythonic.monads`](features.md#monads) (or, for that matter, any object that implements `__rshift__` as monadic bind). + +The body of `with monadic_do[M] as result:` is a single list literal. Each item corresponds to one line of a Haskell do-block: + +- `name := mexpr` — **monadic bind**: unwrap the monadic value and bind it to `name` for subsequent lines. Also accepts the legacy `name << mexpr` form that `letdoutil` understands for `let[]`. +- a bare `mexpr` — **sequencing-only** (Haskell's `do { mx; ... }`): the monadic value is threaded through the chain but its unwrapped value is discarded. Used e.g. for `guard`-style filter lines. + +The **last item** is the final monadic expression — any expression of the right monad type, same semantics as Haskell's last-line-of-do (a constructor call, a call to a monad-producing function, anything of type `M a`). The `as result` on the `with` tells the macro where to land the computed value. ```python -from unpythonic.syntax import macros, forall, insist, deny +from unpythonic.syntax import macros, monadic_do +from unpythonic.monads import Maybe, Left, Right, List, Writer +from unpythonic.llist import nil + +# Maybe — happy path +with monadic_do[Maybe] as result: + [x := Maybe(10), + y := Maybe(x + 1), + Maybe(x + y)] +assert result == Maybe(21) + +# Maybe — short-circuit. The `y := ...` line is never evaluated. +with monadic_do[Maybe] as result: + [x := Maybe(nil), + y := Maybe(x + 1), + Maybe(x + y)] +assert result == Maybe(nil) + +# List — Pythagorean triples. The bare `List.guard(...)` line is a +# sequencing-only bind; its result is discarded. Matches Haskell's +# `guard` in do-notation exactly. +def r(lo, hi): + return List.from_iterable(range(lo, hi)) +with monadic_do[List] as pt: + [z := r(1, 21), + x := r(1, z + 1), + y := r(x, z + 1), + List.guard(x*x + y*y == z*z), + List((x, y, z))] +assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20)) +``` + +The no-binds case is just a single-element list: `[M.unit(x)]` reduces to `result = M.unit(x)`. + +Expands to a nested lambda-bind chain: + +```python +result = mx >> (lambda x: my(x) >> (lambda y: final_expr)) +``` + +Sequencing-only lines (bare `mexpr`) are rewritten to `_ := mexpr` internally and participate in the same chain; their unwrapped value is bound to `_` and ignored. + +**Placement in the xmas tree**: `monadic_do` is always the **innermost** `with`. Its body-shape constraint (a single list-literal statement) forbids lexically wrapping other `with` blocks inside. Outer two-pass macros (`lazify`, `continuations`, `tco`, `autocurry`, etc.) expand inner macros between their passes, so they will see and edit the expanded bind chain in the right order. + +```python +with lazify: + with monadic_do[Maybe] as result: + ... +``` + +For the pure-Python monads themselves and the `liftm` helpers, see [features.md](features.md#monads). -out = forall[y << range(3), - x << range(3), +For the List-monad-specific do-notation that existed first, see [`forall`](#forall-nondeterministic-evaluation) below. + + +### `forall`: nondeterministic evaluation + +**Changed in v0.15.3.** *Env-assignment now uses the assignment expression syntax `x := range(3)`. The old syntax `x << range(3)` is still supported for backward compatibility.* + +This is essentially a macro implementation of Haskell's do-notation for Python, specialized to the List monad. + +The `forall[]` expr macro behaves the same as the multiple-body-expression tuple comprehension `unpythonic.forall`, but the macro is implemented purely by AST transformation, using real lexical variables. + +The implementation is generic and very short; if interested, see the module [`unpythonic.syntax.forall`](../unpythonic/syntax/forall.py). Compare the module [`unpythonic.amb`](../unpythonic/amb.py), which implements the same functionality with a source code generator and `eval`, without macros. The macro implementation is both shorter and more readable; this is effectively a textbook example of a situation where macros are the clean solution. + +```python +from unpythonic.syntax import macros, forall +from unpythonic.syntax import insist, deny # regular functions, not macros + +out = forall[y := range(3), + x := range(3), insist(x % 2 == 0), (x, y)] assert out == ((0, 0), (2, 0), (0, 1), (2, 1), (0, 2), (2, 2)) # pythagorean triples -pt = forall[z << range(1, 21), # hypotenuse - x << range(1, z+1), # shorter leg - y << range(x, z+1), # longer leg +pt = forall[z := range(1, 21), # hypotenuse + x := range(1, z+1), # shorter leg + y := range(x, z+1), # longer leg insist(x*x + y*y == z*z), (x, y, z)] assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), (8, 15, 17), (9, 12, 15), (12, 16, 20)) ``` -Assignment (with List-monadic magic) is ``var << iterable``. It is only valid at the top level of the ``forall`` (e.g. not inside any possibly nested ``let``). +Assignment, **with** List-monadic magic, is `var := iterable`. It is only valid at the top level of the `forall` (e.g. not inside any possibly nested `let`). -``insist`` and ``deny`` are not really macros; they are just the functions from ``unpythonic.amb``, re-exported for convenience. +`insist` and `deny` are not macros; they are just the functions from `unpythonic.amb`, re-exported for convenience. -The error raised by an undefined name in a ``forall`` section is ``NameError``. +The error raised by an undefined name in a `forall[]` section is `NameError`. ## Convenience features Small macros that are not essential but make some things easier or simpler. -### ``cond``: the missing ``elif`` for ``a if p else b`` +### `cond`: the missing `elif` for `a if p else b` -Now lambdas too can have multi-branch conditionals, yet remain human-readable: +With `cond`, lambdas too can have multi-branch conditionals, yet remain human-readable: ```python from unpythonic.syntax import macros, cond @@ -1631,9 +2166,9 @@ answer = lambda x: cond[x == 2, "two", print(answer(42)) ``` -Syntax is ``cond[test1, then1, test2, then2, ..., otherwise]``. Expansion raises an error if the ``otherwise`` branch is missing. +Syntax is `cond[test1, then1, test2, then2, ..., otherwise]`. A missing `otherwise` branch is considered a syntax error at macro expansion time. -Any part of ``cond`` may have multiple expressions by surrounding it with brackets: +Any part of `cond` may have multiple expressions by surrounding it with brackets: ```python cond[[pre1, ..., test1], [post1, ..., then1], @@ -1642,24 +2177,32 @@ cond[[pre1, ..., test1], [post1, ..., then1], [postn, ..., otherwise]] ``` -To denote a single expression that is a literal list, use an extra set of brackets: ``[[1, 2, 3]]``. +This is just the extra bracket syntax that denotes an implicit `do[]`. To denote a single expression that is a literal list, double the brackets: `[[1, 2, 3]]`. Just like in a `let[]` form, the outer brackets enable multiple-expression mode, and then the inner brackets denote a list. The multiple-expression mode is allowed also when there is just one expression. + +Inspired by the `cond` form of many Lisps. There is some variation between Lisp dialects on whether `cond` or `if` is preferable if the dialect provides both. For example, in [Racket](https://racket-lang.org/), `cond` is the [preferred](https://docs.racket-lang.org/style/Choosing_the_Right_Construct.html#%28part._.Conditionals%29) construct for writing conditionals. -### ``aif``: anaphoric if +### `aif`: anaphoric if -This is mainly of interest as a point of [comparison with Racket](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/beyond_python/aif.rkt); ``aif`` is about the simplest macro that relies on either the lack of hygiene or breaking thereof. +**Changed in v0.15.0.** *The `it` helper macro may only appear in the `then` and `otherwise` branches of an `aif[]`. Anywhere else, it is considered a syntax error at macro expansion time.* + +In linguistics, an [*anaphor*](https://en.wikipedia.org/wiki/Anaphora_(linguistics)) is an expression that refers to another, such as the English word *"it"*. [Anaphoric macros](https://en.wikipedia.org/wiki/Anaphoric_macro) are a lispy take on the concept. An anaphoric macro may, for example, implicitly define an `it` that the user code can then use, with the meaning defined by the macro. This is sometimes a useful technique to shorten code, but it can also make code unreadable by hiding definitions, so it should be used sparingly. + +Particularly, the *anaphoric if* is a classic macro, where `it` is automatically bound to the result of the test. We provide that macro as `aif[]`. + +Concerning readability, the anaphoric if is relatively harmless, because it is *almost* obvious from context that the only `it` that makes sense for a human to refer to is the test expression. ```python from unpythonic.syntax import macros, aif, it -aif[2*21, +aif[2 * 21, print(f"it is {it}"), print("it is falsey")] ``` -Syntax is ``aif[test, then, otherwise]``. The magic identifier ``it`` (which **must** be imported as a macro, if used) refers to the test result while (lexically) inside the ``then`` and ``otherwise`` parts of ``aif``, and anywhere else is considered a syntax error at macro expansion time. +Syntax is `aif[test, then, otherwise]`. The magic identifier `it` (which **must** be imported as a macro) refers to the test result while (lexically) inside the `then` and `otherwise` branches of an `aif[]`, and anywhere else is considered a syntax error at macro expansion time. -Any part of ``aif`` may have multiple expressions by surrounding it with brackets (implicit ``do[]``): +Any part of `aif` may have multiple expressions by surrounding it with brackets: ```python aif[[pre, ..., test], @@ -1667,12 +2210,16 @@ aif[[pre, ..., test], [post_false, ..., otherwise]] # "otherwise" branch ``` -To denote a single expression that is a literal list, use an extra set of brackets: ``[[1, 2, 3]]``. +This is just the extra bracket syntax that denotes an implicit `do[]`. To denote a single expression that is a literal list, double the brackets: `[[1, 2, 3]]`. Just like in a `let[]` form, the outer brackets enable multiple-expression mode, and then the inner brackets denote a list. The multiple-expression mode is allowed also when there is just one expression. + +If interested, [compare with a Racket implementation](https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/beyond_python/aif.rkt); `aif` is probably *the* simplest macro that relies on either the lack of [macro hygiene](https://en.wikipedia.org/wiki/Hygienic_macro) or intentional *breaking* thereof. + +### `autoref`: implicitly reference attributes of an object -### ``autoref``: implicitly reference attributes of an object +**CAUTION**: *This is a really, really bad idea that comes with serious readability and security implications. Python does not provide this construct itself, for good reason. Details below. Use with care, if at all.* -Ever wish you could ``with(obj)`` to say ``x`` instead of ``obj.x`` to read attributes of an object? Enter the ``autoref`` block macro: +Ever wish you could `with(obj)` to say `x` instead of `obj.x` to read attributes of an object? Enter the `autoref` block macro: ```python from unpythonic.syntax import macros, autoref @@ -1686,26 +2233,28 @@ with autoref(e): assert c == 3 # no c in e, so just c ``` -The transformation is applied for names in ``Load`` context only, including names found in ``Attribute`` or ``Subscript`` nodes. +The transformation is applied for names in `Load` context only, including names found inside `Attribute` or `Subscript` AST nodes, so things like `a[1]` and `a.x` are also valid (looking up `a` in `e`). -Names in ``Store`` or ``Del`` context are not redirected. To write to or delete attributes of ``o``, explicitly refer to ``o.x``, as usual. +Names in `Store` or `Del` context are not redirected. To write to or delete attributes of `o`, explicitly refer to `o.x`, as usual. Nested autoref blocks are allowed (lookups are lexically scoped). -Reading with ``autoref`` can be convenient e.g. for data returned by [SciPy's ``.mat`` file loader](https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.loadmat.html). +Reading with `autoref` can be convenient e.g. for data returned by [SciPy's `.mat` file loader](https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.loadmat.html). -See the [unit tests](../unpythonic/syntax/test/test_autoref.py) for more usage examples. +See the [unit tests](../unpythonic/syntax/tests/test_autoref.py) for more usage examples. This is similar to the JavaScript [`with` construct](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/with), which is nowadays [deprecated](https://2ality.com/2011/06/with-statement.html). See also [the ES6 reference on `with`](https://www.ecma-international.org/ecma-262/6.0/#sec-with-statement). -**CAUTION**: This construct was deprecated in JavaScript **for security reasons**. Since the autoref'd object **will hijack all name lookups**, use `with autoref` only with an object you trust! +**NOTE**: The JavaScript `with` and the Python `with` have nothing in common except the name. -**CAUTION**: `with autoref` also complicates static code analysis or makes it outright infeasible, for the same reason. It is impossible to statically know whether something that looks like a bare name in the source code is actually a true bare name, or a reference to an attribute of the autoref'd object. That status can also change at any time, since the lookup is dynamic, and attributes can be added and removed dynamically. +**CAUTION**: The `with` construct of JavaScript was deprecated **for security reasons**. Since the autoref'd object **will hijack all name lookups**, use `with autoref` only with an object you trust! In most Python code, this does not matter, as we are all adults here, but this *may* matter if a Python object arrives from an untrusted source in a networked app. + +**CAUTION**: `with autoref` complicates static code analysis or makes it outright infeasible. It is impossible to statically know whether something that looks like a bare name in the source code is actually a true bare name, or a reference to an attribute of the autoref'd object. That status can also change at any time, since the lookup is dynamic, and attributes can be added and removed dynamically. ## Testing and debugging -### ``unpythonic.test.fixtures``: a test framework for macro-enabled Python +### `unpythonic.test.fixtures`: a test framework for macro-enabled Python **Added in v0.14.3.** @@ -1751,38 +2300,86 @@ with session("simple framework demo"): try: import blargly except ImportError: - error["blargly not installed, cannot test integration with it."] + warn["blargly not installed, skipping integration tests."] else: ... # blargly integration tests go here + # Unconditional errors and failures can be emitted with `error[]` and `fail[]`. + # with testset("not implemented"): + # fail["not implemented yet!"] + with testset(postproc=terminate): test[2 * 2 == 5] # fails, terminating the nearest dynamically enclosing `with session` test[2 * 2 == 4] # not reached ``` -By default, running this script through the `macropython` wrapper (from `mcpyrate`) will produce an ANSI-colored test report in the terminal. To actually see how the output looks like, for actual runnable examples, see `unpythonic`'s own automated tests. +By default, running this script through the `macropython` wrapper (from `mcpyrate`) will produce an ANSI-colored test report in the terminal. To actually see how the output looks like, and for actual runnable examples, see `unpythonic`'s own automated tests. -If you want to turn coloring off (e.g. for redirecting stderr to a file), see the `TestConfig` bunch of constants in `unpythonic.test.fixtures`. +If you want to turn coloring off (e.g. for the purposes of redirecting stderr to a file), see the `TestConfig` bunch of constants in `unpythonic.test.fixtures`. -The following is an overview of the framework. For details, look at the docstrings of the various constructs in `unpythonic.test.fixtures` (which provides much of this), those of the test macros, and finally, the automated tests of `unpythonic` itself. +The following is an overview of the framework. For details, look at the docstrings of the various constructs in `unpythonic.test.fixtures` (which provides much of this), those of the testing macros, and finally, the automated tests of `unpythonic` itself. Tests can be found in subfolders named `tests`: [regular code](../unpythonic/tests/), [macros](../unpythonic/syntax/tests/), [dialects](../unpythonic/dialects/tests/). -How to test code using conditions and restarts can be found in [`unpythonic.tests.test_conditions`](../unpythonic/tests/test_conditions.py). +Examples of how to test code using conditions and restarts can be found in [`unpythonic.tests.test_conditions`](../unpythonic/tests/test_conditions.py). -How to test macro utilities (e.g. syntax transformer functions that operate on ASTs) can be found in [`unpythonic.syntax.tests.test_letdoutil`](../unpythonic/syntax/tests/test_letdoutil.py). +Examples of how to test macro utilities (e.g. syntax transformer functions that operate on ASTs) can be found in [`unpythonic.syntax.tests.test_letdoutil`](../unpythonic/syntax/tests/test_letdoutil.py). + +**NOTE**: If you want to compartmentalize macro expansion in your tests (so that an error during macro expansion will not crash your test unit), `mcpyrate` offers more than one way to invoke the macro expander at run time ([*of your test unit*](https://github.com/Technologicat/mcpyrate/blob/master/doc/troubleshooting.md#macro-expansion-time-where-exactly)), depending on what exactly you want to do. One is the `mcpyrate.metatools.expand` family of macros, and another are the functions in the module `mcpyrate.compiler`. See [the `mcpyrate` user manual](https://github.com/Technologicat/mcpyrate/blob/master/doc/main.md): specifically on [`metatools` (and quasiquoting)](https://github.com/Technologicat/mcpyrate/blob/master/doc/quasiquotes.md) and on [`compiler`](https://github.com/Technologicat/mcpyrate/blob/master/doc/compiler.md). The tests of `mcpyrate` itself provide some examples on how to use `compiler`. #### Overview -We provide the low-level syntactic constructs `test[]`, `test_raises[]` and `test_signals[]`, with the usual meanings. The last one is for testing code that uses the `signal` function and its sisters (related to conditions and restarts à la Common Lisp); see [`unpythonic.conditions`](features.md#handlers-restarts-conditions-and-restarts). +All testing *macros* are provided in the module `unpythonic.syntax`. All regular functions related to testing are provided in the module `unpythonic.test.fixtures`. + +We provide the low-level syntactic constructs `test[]`, `test_raises[]` and `test_signals[]`, with the usual meanings. The last one is for testing code that uses `unpythonic.signal` and its sisters (related to conditions and restarts à la Common Lisp); see the module [`unpythonic.conditions`](../unpythonic/conditions.py), and the user manual section on [conditions and restarts](features.md#handlers-restarts-conditions-and-restarts). + +By default, the `test[expr]` macro asserts that the value of `expr` is truthy. If you want to assert only that `expr` runs to completion normally, use `test[returns_normally(expr)]`. Here `returns_normally` is a regular function, which is available in the module `unpythonic.test.fixtures`. + +All three testing constructs also come in block variants, `with test`, `with test_raises[exctype]`, `with test_signals[exctype]`. + +As usual in test frameworks, the testing constructs behave somewhat like `assert`, with the difference that a failure or error will not abort the whole unit, unless explicitly asked to do so. There is no return value; upon success, the testing constructs return `None`. Upon failure (test assertion not satisfied) or error (unexpected exception or signal), the failure or error is reported, and further tests continue running. + +All the variants of the testing constructs catch any uncaught exceptions and signals from inside the test expression or block. Any unexpected uncaught exception or signal is considered an error. + +Because `unpythonic.test.fixtures` is, by design, a minimalistic *no-framework* (cf. "NoSQL"), it is up to you to define - in your custom test runner - whether having any failures, errors or warnings should lead to the whole test suite failing. Whether the program's exit code is zero, is important e.g. for GitHub's CI workflows. + +For example, in `unpythonic`'s own tests, warnings do not cause the test suite to fail, but errors and failures do. The top-level [`runtests.py`](../runtests.py) is a complete test runner using the reusable `unpythonic.test.runner` module: + +```python +import os +from unpythonic.test.runner import discover_testmodules, run + +import mcpyrate.activate # noqa: F401 + +testsets = [("my tests", discover_testmodules(os.path.join("mypackage", "tests")))] +if not run(testsets): + raise SystemExit(1) +``` + +`discover_testmodules` finds `test_*.py` files in a directory and returns dotted module names. `run` wraps the session/testset/import pattern, with automatic version-suffix gating (e.g. `test_foo_3_11.py` is skipped with a warning on Python < 3.11). + +#### Important: bytecode cache pitfall + +**Never compile `.py` files in a macro-enabled project** using `py_compile`, `python -m compileall`, pip's `--compile` flag, or any other mechanism that bypasses the macro expander. These tools produce `.pyc` files that do not contain macro-expanded code, which will break macro imports at run time. + +The symptom is typically `ImportError: cannot import name 'macros' from 'mcpyrate.quotes'` (or similar). This happens because the stale `.pyc` is loaded instead of the `.py` source, so the macro expander never runs. + +To fix this, clean the bytecode caches: + +```bash +macropython -c mypackage +``` -By default, the `test[expr]` macro asserts that the value of `expr` is truthy. If you want to assert only that `expr` runs to completion normally, use `test[returns_normally(expr)]`. +This removes all `__pycache__` directories under the given path. After cleaning, re-run your tests normally — the macro expander will recompile the source files correctly. -The test macros also come in block variants, `with test`, `with test_raises[exctype]`, `with test_signals[exctype]`. +#### Reading test results -As usual in test frameworks, the test constructs behave somewhat like `assert`, with the difference that a failure or error will not abort the whole unit (unless explicitly asked to do so). There is no return value; upon success, the test constructs return `None`. Upon failure (test assertion not satisfied) or error (unexpected exception or signal), the failure or error is reported, and further tests continue running. +The framework reports **Pass**, **Fail**, **Error**, and **Total** per testset, with optional **Warn** counts. These categories mean: -All the test variants catch any uncaught exceptions and signals from inside the test expression or block. Any unexpected uncaught exception or signal is considered an error. +- **Pass**: test assertion succeeded. +- **Fail**: test ran to completion, but the assertion was not satisfied. +- **Error**: test did not run to completion (unexpected exception or signal inside a `test[]` expression). This also includes intentional `error[]` signals — so a few errors from skip patterns (e.g. optional dependency not installed) may be normal. Check the actual error messages, not just the count. (Since 2.0.0, optional dependency skips use `warn[]` instead.) +- **Warn**: a human-initiated warning (via `warn[]` or `emit_warning()`). Warnings are not counted in the total, and do not cause the test suite to fail. -Because `unpythonic.test.fixtures` is, by design, a minimalistic *no-framework* (cf. "NoSQL"), it is up to you to define - in your custom test runner - whether having any failures, errors or warnings should lead to the whole test suite failing (whether the program's exit code is zero is important e.g. for GitHub's CI workflows). For example, in `unpythonic`'s own tests (see the very short [`runtests.py`](../runtests.py)), warnings do not cause the test suite to fail, but errors and failures do. +Nested testsets show hierarchy with indentation and asterisk depth (`**`, `****`, `******`, etc.). Counts propagate upward — the top-level summary reflects all tests across all testsets. #### Testing syntax quick reference @@ -1790,7 +2387,7 @@ Because `unpythonic.test.fixtures` is, by design, a minimalistic *no-framework* ```python from unpythonic.syntax import (macros, test, test_raises, test_signals, - fail, error, warn, the, expand_testing_macros_first) + fail, error, warn, the, expect, expand_testing_macros_first) from unpythonic.test.fixtures import (session, testset, returns_normally, catch_signals, terminate) ``` @@ -1803,6 +2400,9 @@ from unpythonic.test.fixtures import session, testset def runtests(): with testset("something 1"): + test[...] + test_raises[TypeError, ...] + test_raises[ValueError, ...] ... with testset("something 2"): ... @@ -1813,9 +2413,9 @@ if __name__ == '__main__': # pragma: no cover runtests() ``` -The if-main idiom allows running this test module individually, but it is tagged with `# pragma: no cover`, so that the coverage reporter won't yell about it when the module is run by the test runner as part of the complete test suite (which, incidentally, is also a good opportunity to measure coverage). +The if-main idiom allows running this test module individually, but it is tagged with `# pragma: no cover`, so that the coverage reporter will not yell about it when the module is run by the test runner as part of the complete test suite (which, incidentally, is also a good opportunity to [measure coverage](../measure_coverage.sh)). -If you want to ensure that testing macros expand before anything else - including your own code-walking block macros (when you have tests inside the body) - import the macro `expand_testing_macros_first`, and put a `with expand_testing_macros_first` around the affected code. (See [Expansion order](#expansion-order), below.) +If you want to ensure that testing macros expand before anything else - including your own code-walking block macros (when you have tests inside the body of a `with` block that invokes a code-walking block macro) - import the macro `expand_testing_macros_first`, and put a `with expand_testing_macros_first` around the affected code. (See [Expansion order](#expansion-order), below.) **Sessions and testsets**: @@ -1826,7 +2426,7 @@ with session(name): with testset(name): ... - with testset(name): + with testset(name): # nested testset ... with testset(name): @@ -1834,11 +2434,11 @@ with session(name): ... ``` -Each `name` above is human-readable and optional. The purpose of the naming feature is to improve [scannability](https://www.teachingenglish.org.uk/article/scanning) of the testing report for the human reader. +Each `name` above is human-readable and optional. The purpose of the naming feature is to improve [scannability](https://www.teachingenglish.org.uk/article/scanning) of the testing report, and of the unit test source code, for the human reader. Note that even if `name` is omitted, the parentheses are still mandatory, because `session` and `testset` are just garden variety context managers that must be instantiated in order for them to perform their jobs. -A session implicitly introduces a top-level testset, for convenience. +A session implicitly introduces a top-level testset, for convenience - so if you only a have a few tests and don't want to group them, you do not need to use `with testset` at all. Testsets can be nested arbitrarily deep. @@ -1848,13 +2448,13 @@ Additional tools for code using **conditions and restarts**: The `catch_signals` context manager controls the signal barrier of `with testset` and the `test` family of syntactic constructs. It is provided for writing tests for code that uses conditions and restarts. -Used as `with catch_signals(False)`, it disables the signal barrier. Within the dynamic extent of the block, an uncaught signal (in the sense of `unpythonic.conditions.signal` and its sisters) is not considered an error. This can be useful, because sometimes leaving a signal uncaught is the right thing to do. See [`unpythonic.tests.test_conditions`](../unpythonic/tests/test_conditions.py) for examples. +Used as `with catch_signals(False)`, it disables the signal barrier for the dynamic extent of the block. When the barrier is disabled, an uncaught signal (in the sense of `unpythonic.signal` and its sisters) is not considered as an error. This can be useful, because sometimes leaving a signal uncaught is the right thing to do. See [`unpythonic.tests.test_conditions`](../unpythonic/tests/test_conditions.py) for examples. -It can be nested. Used as `with catch_signals(True)`, it re-enables the barrier, if currently disabled. +The `with catch_signals` construct can be nested. Used as `with catch_signals(True)`, it re-enables the barrier, if currently disabled, for the dynamic extent of that inner `with catch_signals` block. When a `with catch_signals` block exits, the previous state of the signal barrier is automatically restored. -**Expression** forms: +**Expression** forms - complete list: ```python test[expr] @@ -1870,31 +2470,32 @@ error[message] warn[message] ``` -Inside a `test`, the helper macro `the[]` is available to mark interesting subexpressions inside `expr`, for failure and error reporting. An `expr` may contain an arbitrary number of `the[]`. By default, if `expr` is a comparison, the leftmost term is automatically marked (so that e.g. `test[x < 3]` will automatically report the value of `x` if the test fails); otherwise nothing. The default is only used if there is no explicit `the[]` inside `expr`. +Inside a `test[]`, the helper macro `the[]` is available to mark one or more interesting subexpressions inside `expr`, for failure and error reporting. An `expr` may contain an arbitrary number of `the[]`. By default, if `expr` is a comparison, the leftmost term is implicitly marked (so that e.g. `test[x < 3]` will automatically report the value of `x` if the test fails); otherwise nothing. The default is only used when there is **no** explicit `the[]` inside `expr`. The constructs `test_raises`, `test_signals`, `fail`, `error` and `warn` do **not** support `the[]`. Tests can be nested; this is sometimes useful as an explicit signal barrier. -Note the macros `error[]` and `warn[]` have nothing to do with the functions with the same name in `unpythonic.conditions`. The macros are part of the test framework; the functions with the same name are signaling protocols of the conditions and restarts system. Following the usual naming conventions in both systems, this naming conflict is unfortunately what we get. +Note that the testing constructs `error[]` and `warn[]`, which are macros, have nothing to do with the functions with the same name in the module `unpythonic.conditions`. The macros are part of the test framework; the functions with the same name are signaling protocols of the conditions and restarts system. Following the usual naming conventions separately in both systems, this naming conflict is unfortunately what we get. -**Block** forms: +**Block** forms - complete list: ```python with test: body ... + # no `expect[]`; assert just that the block completes normally with test: body ... - return expr + expect[expr] # assert that `expr` is truthy with test[message]: body ... with test[message]: body ... - return expr + expect[expr] with test_raises[exctype]: body ... @@ -1909,6 +2510,8 @@ with test_signals[exctype, message]: ... ``` +In a `with test:` block, `expect[expr]` (added in v2.2.0) is the way to declare the expression whose truthiness is asserted. Use it at most once per block. Earlier versions used `return expr` for this; that form still works but emits a `DeprecationWarning`, and using both `expect[]` and `return` in the same block is a `SyntaxError`. `return` will be un-hijacked in a future major release so it regains its standard Python meaning. + In `with test`, the `the[]` helper macro is available. It can be used to mark any number of expressions and/or subexpressions in the block body. The constructs `with test_raises`, `with test_signals` do **not** support `the[]`. @@ -1928,7 +2531,7 @@ with yourblockmacro: # outside-in Here the `...` may be edited by `yourblockmacro` before `test[]` sees it. (It likely **will** be edited, since this pattern will commonly appear in the tests for `yourblockmacro`, where the whole point is to have the `...` depend on what `yourblockmacro` outputs.) -If you need testing macros to expand before anything else even in this scenario (so you can more clearly see where in the unexpanded source code a particular expression came from), you can do this: +If you need testing macros to expand before anything else even in this scenario (so you can more clearly see where in the unexpanded source code a particular expression in a failing/erroring test came from), you can do this: ```python from unpythonic.syntax import macros, expand_testing_macros_first @@ -1938,9 +2541,9 @@ with expand_testing_macros_first: test[...] ``` -The `expand_testing_macros_first` macro is itself a code-walking block macro that does as it says on the tin. The testing macros are identified by scanning the bindings of the current macro expander; names don't matter, so it respects as-imports. +The `expand_testing_macros_first` macro is itself a code-walking block macro that does as it says on the tin. The testing macros are identified by scanning the bindings of the current macro expander; names do not matter, so it respects as-imports. -This does imply that `your_block_macro` will then receive the expanded form of `test[...]` as input, but that's macros for you. You'll have to choose which is more important: seeing the unexpanded code in error messages, or receiving unexpanded `test[]` expressions in `yourblockmacro`. +This does imply that `yourblockmacro` will then receive the expanded form of `test[...]` as input, but that's macros for you. You will have to choose which is more important: seeing the unexpanded code in error messages, or receiving unexpanded `test[]` expressions in `yourblockmacro`. #### `with test`: test blocks @@ -1950,13 +2553,13 @@ In `unpythonic.test.fixtures`, **a test block is implicitly lifted into a functi By default, a `with test` block asserts just that it completes normally. If you instead want to assert that an expression is truthy, use `return expr` to terminate the implicit function and return the value of the desired `expr`. The return value is passed to the test asserter for checking that it is truthy. -(Another way to view the default behavior is that the `with test` macro injects a `return True` at the end of the block, if there is no `return`. This is actually how the default behavior is implemented.) +Another way to view the default behavior is that the `with test` macro injects a `return True` at the end of the block to terminate the implicit function, if there is no explicit `return`. This is actually how the default behavior is implemented. -The `with test_raises[exctype]` and `with test_signals[exctype]` blocks assert that the block raises (respectively, signals) the declared exception (condition) type. These blocks are implicitly lifted into functions, too, but they do not check the return value. For them, **not** raising/signaling the declared exception/condition type is considered a test failure. Raising/signaling some other (hence unexpected) exception/condition type is considered an error. +The `with test_raises[exctype]` and `with test_signals[exctype]` blocks assert that the block raises (respectively, signals) the declared exception type. These blocks are implicitly lifted into functions, too, but they do not check the return value. For them, **not** raising/signaling the declared exception type is considered a test failure. Raising/signaling some other (hence unexpected) exception type is considered an error. #### `the`: capture the value of interesting subexpressions -The point of `unpythonic.test.fixtures` is to make testing macro-enabled Python as frictionless as reasonably possible. +The point of `unpythonic.test.fixtures` is to make testing macro-enabled Python as frictionless as reasonably possible. Thus we provide this convenience feature. Inside a `test[]` expression, or anywhere within the code in a `with test` block, the `the[]` macro can be used to declare any number of subexpressions as interesting, for capturing the source code and value into the test failure message, which is shown if the test fails. Each `the[]` captures one subexpression (as many times as it is evaluated, in the order evaluated). @@ -1964,7 +2567,9 @@ Because test macros expand outside-in, the source code is captured before any ne By default (if no explicit `the[]` is present), `test[]` implicitly inserts a `the[]` for the leftmost term if the top-level expression is a comparison (common use case), and otherwise does not capture anything. -When nothing is captured, if the test fails, the value of the whole expression is shown. Of course, you'll then already know the value is falsey, but there's still the possibly useful distinction of whether it's, say, `False`, `None`, `0` or `[]`. +If you want to instead capture *the RHS*, use an explicit `the[]`. For example, `test["green tea" == the[vert]]` captures the value of `vert` upon failure. + +When nothing is captured, if the test fails, the value of the whole expression is shown. Of course, you will then already know the value is falsey, but there is still the possibly useful distinction of whether it is, say, `False`, `None`, `0` or `[]`. A `test[]` or `with test` can have any number of subexpressions marked as `the[]`. It is possible to even nest a `the[]` inside another `the[]`, if you need the value of some subexpression as well as one of *its* subexpressions. The captured values are gathered, in the order they were evaluated (by Python's standard evaluation rules), into a list that is shown upon test failure. @@ -1974,25 +2579,36 @@ In case of nested `test[]` or nested `with test`, each `the[...]` is understood The `the[]` mechanism is smart enough to skip reporting trivialities for literals, such as `(1, 2, 3) = (1, 2, 3)` in `test[4 in the[(1, 2, 3)]]`, or `4 = 4` in `test[4 in (1, 2, 3)]`. In the second case, note the implicit `the[]` on the LHS, because `in` is a comparison operator. -If nothing but such trivialities were captured, the failure message will instead report the value of the whole expression. (The captures still remain inspectable in the exception instance.) +Because the implicit `the[]` wraps the leftmost term *as-written*, for a compound LHS such as `test[reply["status"] == "ok"]` the captured subexpression is `reply["status"]`, not the whole `reply`. If you would rather see the full container on failure (e.g. to read a `"reason"` field the server attached alongside `"status": "failed"`), wrap it explicitly: `test[the[reply]["status"] == "ok"]`. Note that adding any explicit `the[]` disables the implicit LHS capture, so in the latter form only `reply` is captured, not both `reply` and `reply["status"]`. The choice between the two forms is a debugging-granularity judgment: leaf captures are enough when the leaf is self-explanatory (`timer.dt == 0.0`), whereas wrapping the container is better when the leaf value alone is lossy. + +##### Common `the[]` mistakes + +A few anti-patterns recur often enough to call out explicitly. Each captures something less useful than the form it should be: -To make testing/debugging macro code more convenient, the `the[]` mechanism automatically unparses an AST value into its source code representation for display in the test failure message. This is meant for debugging macro utilities, to which a test case hands some quoted code (i.e. code lifted into its AST representation using mcpyrate's `q[]` macro). See [`unpythonic.syntax.test.test_letdoutil`](unpythonic/syntax/test/test_letdoutil.py) for some examples. (Note the unparsing is done for display only; the raw value remains inspectable in the exception instance.) +- **`test[the["X" in out]]`** — wraps the *whole* `in` expression, so the capture is the boolean result. On failure, the message tells you the assertion was false but does not show `out`. Use `test["X" in the[out]]` to capture `out` itself. +- **`test[the[X == Y]]`** — same shape with `==`: captures the boolean result. Use `test[X == Y]`; auto-capture wraps the LHS for you. +- **`test[the[X] == "Y"]`** — redundant: auto-capture already wraps the LHS. Use `test[X == "Y"]`. Reach for explicit `the[]` only when you want a *different* term captured than the LHS (see the compound-LHS discussion above). +- **`test[the[a] < b < c]`** — for a chained comparison, only `a` is captured, so a failure between `b` and `c` shows neither's value. Wrap every term you would want to see: `test[the[a] < the[b] < the[c]]`. -**CAUTION**: The source code is back-converted from the AST representation; hence its surface syntax may look slightly different to the original (e.g. extra parentheses). See ``mcpyrate.unparse``. +If nothing but such trivialities were captured, the failure message will instead report the value of the whole expression. The captures still remain inspectable in the exception instance. -**CAUTION**: The name of the `the[]` construct was inspired by Common Lisp, but the semantics are completely different. Common Lisp's `THE` is a return-type declaration (pythonistas would say *return-type annotation*), meant as a hint for the compiler to produce performance-optimized compiled code (see [chapter 32 of Peter Seibel's Practical Common Lisp](http://www.gigamonkeys.com/book/conclusion-whats-next.html)), whereas our `the[]` captures a value for test reporting. The only common factors are the name, and that neither construct changes the semantics of the marked code, much. In `unpythonic.test.fixtures`, the reason behind picking this name was that it doesn't change the flow of the source code as English that much, specifically to suggest, between the lines, that it doesn't change the semantics much. The reasoning behind CL's `THE` may be similar. +To make testing/debugging macro code more convenient, the `the[]` mechanism automatically unparses an AST value into its source code representation for display in the test failure message. This is meant for debugging macro utilities, to which a test case hands some quoted code (i.e. code lifted into its AST representation using mcpyrate's `q[]` macro). See [`unpythonic.syntax.tests.test_letdoutil`](unpythonic/syntax/tests/test_letdoutil.py) for some examples. Note the unparsing is done for display only; the raw value remains inspectable in the exception instance. + +**CAUTION**: The source code is back-converted from the AST representation; hence its surface syntax may look slightly different to the original (e.g. extra parentheses). See `mcpyrate.unparse`. + +**CAUTION**: The name of the `the[]` construct was inspired by Common Lisp, but that is where the similarities end. The `THE` construct of Common Lisp is a return-type declaration (pythonistas would say *return-type annotation*), meant as a hint for the compiler to produce performance-optimized compiled code. See [chapter 32 in Practical Common Lisp by Peter Seibel](http://www.gigamonkeys.com/book/conclusion-whats-next.html). In contrast, our `the[]` captures a value for test reporting. The only common factors are the name, and that neither construct changes the semantics of the marked code, much. In `unpythonic.test.fixtures`, the reason behind picking this name was that it does not change the flow of the source code as English that much, specifically to suggest, between the lines, that it does not change the semantics much. The reasoning behind CL's `THE` may be similar, but I have not researched its etymology. #### Test sessions and testsets The `with session()` in the example session above is optional. The human-readable session name is also optional, used for display purposes only. The session serves two roles: it provides an exit point for `terminate`, and defines an implicit top-level `testset`. -Tests can optionally be grouped into testsets. Each `testset` tallies passed, failed and errored tests within it, and displays the totals when it exits. Testsets can be named and nested. +Tests can optionally be grouped into testsets. Each `testset` tallies passed, failed and errored tests within it, and displays the totals when the context exits. Testsets can be named and nested. -It is useful to have at least one `testset` (the implicit top-level one established by `with session` is sufficient), because the `testset` mechanism forms one half of the test framework. It is possible to use the test macros without a `testset`, but that is only intended for building alternative test frameworks. +It is useful to have at least one `testset` (the implicit top-level one established by `with session` is fine), because the `testset` mechanism forms fully one half of the test framework. It is technically possible to use the testing macros without a `testset`, but that is only intended for building alternative test frameworks. Testsets also provide an option to locally install a `postproc` handler that gets a copy of each failure or error in that testset (and by default, any of its inner testsets), after the failure or error has been printed. In nested testsets, the dynamically innermost `postproc` wins. A failure is an instance of `unpythonic.test.fixtures.TestFailure`, an error is an instance of `unpythonic.test.fixtures.TestError`, and a warning is an instance of `unpythonic.test.fixtures.TestWarning`. All three inherit from `unpythonic.test.fixtures.TestingException`. Beside the human-readable message, these exception types contain attributes with programmatically inspectable information about what happened. -If you want to set a default global `postproc`, which is used when no local `postproc` is in effect, this too is configured in the `TestConfig` bunch of constants in `unpythonic.test.fixtures`. +If you want to set a default global `postproc`, which is used when no local `postproc` is in effect, this is configured in the `TestConfig` bunch of constants in `unpythonic.test.fixtures`. The `with testset` construct comes with one other important feature. The nearest dynamically enclosing `with testset` **catches any stray exceptions or signals** that occur within its dynamic extent, but outside a test construct. @@ -2000,7 +2616,25 @@ In case of an uncaught signal, the error is reported, and the testset resumes. In case of an uncaught exception, the error is reported, and the testset terminates, because the exception model does not support resuming. -Catching of uncaught *signals*, in both the low-level `test` constructs and the high-level `testset`, can be disabled using `with catch_signals(False)`. This is useful in testing code that uses conditions and restarts; sometimes allowing a signal (e.g. from `unpythonic.conditions.warn`) to remain uncaught is the right thing to do. +Chained exceptions are reported with their full context. Both the explicit `raise X from Y` form and Python's implicit chaining inside an `except` handler appear in the report: + +```python +with testset("raise from"): + try: + raise ValueError + except ValueError as e: + raise RuntimeError from e + +with testset("implicit chaining"): + try: + raise ValueError + except ValueError: + raise RuntimeError +``` + +In both testsets the report shows the inner `ValueError` and the outer `RuntimeError`, with the appropriate "above exception was the direct cause of" / "during handling of the above exception" link between them. + +Catching of uncaught *signals*, in both the low-level `test` constructs and the high-level `testset`, can be disabled using `with catch_signals(False)`. This is useful in testing code that uses conditions and restarts; sometimes allowing a signal (e.g. from `unpythonic.warn` in the conditions-and-restarts system) to remain uncaught is the right thing to do. #### Producing unconditional failures, errors, and warnings @@ -2010,15 +2644,15 @@ The helper macros `fail[message]`, `error[message]` and `warn[message]` uncondit - `error[...]` if some part of your tests is unable to run. - `warn[...]` if some tests are temporarily disabled and need future attention, e.g. for syntactic compatibility to make the code run for now on an old Python version. -Currently (v0.14.3), warnings produced by `warn[]` are not counted in the total number of tests run. But you can still get the warning count from the separate counter `unpythonic.test.fixtures.tests_warned` (see `unpythonic.collections.box`; basically you can `b.get()` or `unbox(b)` to read the value currently inside a box). +Currently (v0.14.3), warnings produced by `warn[]` are not counted in the total number of tests run. But you can still get the warning count from the separate counter `unpythonic.test.fixtures.tests_warned` (see `unpythonic.box`; basically you can `b.get()` or `unbox(b)` to read the value currently inside a box). #### Advanced: building a custom test framework -If `unpythonic.test.fixtures` does not fit your needs and you want to experiment with creating your own framework, the test asserter macros are reusable. For reference, their implementations can be found in `unpythonic.syntax.testingtools`. They refer to a few objects in `unpythonic.test.fixtures`; consider these a common ground that is not strictly part of the surrounding framework. +If `unpythonic.test.fixtures` does not fit your needs and you want to experiment with creating your own framework, the test asserter macros are reusable. Their implementations can be found in `unpythonic.syntax.testingtools`. They refer to a few objects in `unpythonic.test.fixtures`; consider these a common ground that is not strictly part of the surrounding framework. Start by reading the docstring of the `test` macro, which documents some low-level details. -Set up a condition handler to intercept test failures and errors. These will be signaled via `cerror`, using the conditions and restarts mechanism. See `unpythonic.conditions`. Report the failure/error in any way you desire, and then invoke the `proceed` restart (from your condition handler) to let testing continue. +Set up a condition handler to intercept test failures and errors. These will be signaled via `cerror`, using the conditions and restarts mechanism. See the module `unpythonic.conditions`. Report the failure/error in any way you desire, and then invoke the `proceed` restart (from your condition handler) to let testing continue. Look at the implementation of `testset` as an example. @@ -2026,34 +2660,40 @@ Look at the implementation of `testset` as an example. Because `unpythonic` is effectively a language extension, the standard options were not applicable. -The standard library's [`unittest`](https://docs.python.org/3/library/unittest.html) fails with `unpythonic` due to technical reasons related to `unpythonic`'s unfortunate choice of module names. The `unittest` framework chokes if a module in a library exports anything that has the same name as the module itself, and the library's top-level init then `from`-imports that construct into its namespace, causing the *module reference*, that was [implicitly brought in](http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html#the-submodules-are-added-to-the-package-namespace-trap) by the `from`-import itself, to be overwritten with what was explicitly imported: a reference to the construct that has the same name as the module. (Bad naming on my part, yes, but we're stuck with it at least until v0.15.0. As of v0.14.3, I see no reason to cross that particular bridge yet.) +The standard library's [`unittest`](https://docs.python.org/3/library/unittest.html) fails with `unpythonic` due to technical reasons related to `unpythonic`'s unfortunate choice of module names. The `unittest` framework crashes if a module in a library exports anything that has the same name as the module itself, and the library's top-level init then `from`-imports that construct into its namespace, causing the *module reference*, that was [implicitly brought in](http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html#the-submodules-are-added-to-the-package-namespace-trap) by the `from`-import itself, to be overwritten with what was explicitly imported: a reference to the construct that has the same name as the module. This is bad naming on my part, yes, but as of v0.15.0, I see no reason to cross that particular bridge yet. + +Also, in my opinion, `unittest` is overly verbose to use; automated tests are already a particularly verbose kind of program, even if the testing syntax is minimal. Eliminating extra verbosity encourages writing more tests. -Also, in my opinion, `unittest` is overly verbose to use; automated tests are already a particularly verbose kind of program, even if the testing syntax is minimal. +[Pytest](https://docs.pytest.org/en/latest/), on the other hand, provides compact syntax by hijacking the assert statement, but its import hook (to provide that syntax) cannot coexist with a macro expander, which also needs to install a (different) import hook. Pytest is also fairly complex. -[Pytest](https://docs.pytest.org/en/latest/), on the other hand, provides compact syntax by hijacking the assert statement, but its import hook (to provide that syntax) can't coexist with a macro expander, which also needs to install a different import hook. It's also fairly complex. +The central functional requirement for whatever would be used for testing `unpythonic` was to be able to *easily* deal with macro-enabled Python. No hoops to jump through, compared to testing regular Python, in order to be able to test all of `unpythonic` (including `unpythonic.syntax`) in a uniform way. -The central functional requirement for whatever would be used for testing `unpythonic` was to be able to easily deal with macro-enabled Python. No hoops to jump through, compared to testing regular Python, in order to be able to test all of `unpythonic` (including `unpythonic.syntax`) in a uniform way. +Also, if I was going to build my own framework, it would be nice for it to work seamlessly with code that uses conditions and restarts - since those are part of `unpythonic`, but not standard Python. -Simple and minimalistic would be a bonus. As of v0.14.3, the whole test framework is about 1.8k SLOC, counting docstrings, comments and blanks; under 700 SLOC if counting only active code lines. Add another 800 SLOC (all) / 200 SLOC (active code lines) for the machinery that implements conditions and restarts. +Simple and minimalistic would be a bonus. As of v0.15.0, the whole test framework is about 1.8k SLOC, counting docstrings, comments and blanks; under 700 SLOC if counting only active code lines. Add another 1k SLOC (all) / 200 SLOC (active code lines) for the machinery that implements conditions and restarts. -The framework will likely still evolve a bit as I find more holes in the [UX](https://en.wikipedia.org/wiki/User_experience) - which so far has led to features such as `the[]` and AST value auto-unparsing - but most of the desired functionality is already there. For example, I consider pytest-style implicit fixtures and a central test discovery system as outside the scope of this system. +The framework will likely still evolve a bit as I find more holes in the [UX](https://en.wikipedia.org/wiki/User_experience) - which so far has led to features such as `the[]` and AST value auto-unparsing - but most of the desired functionality is already present and working fine. For example, I consider pytest-style implicit fixtures and a central test discovery system as outside the scope of this framework. It does make the code shorter, but is perhaps slightly too much magic. -It's clear that `unpythonic.test.fixtures` is not going to replace `pytest`, nor does it aim to do so - [any more than Chuck Moore's Forth-based VLSI tools](https://yosefk.com/blog/my-history-with-forth-stack-machines.html) were intended to replace the commercial [VLSI](https://en.wikipedia.org/wiki/Very_Large_Scale_Integration) offerings. +It is clear that `unpythonic.test.fixtures` is not going to replace `pytest`, nor does it aim to do so - [any more than Chuck Moore's Forth-based VLSI tools](https://yosefk.com/blog/my-history-with-forth-stack-machines.html) were intended to replace the commercial [VLSI](https://en.wikipedia.org/wiki/Very_Large_Scale_Integration) offerings. What we have is small, simple, custom-built for its purpose (works well with macro-enabled Python; integrates with conditions and restarts), arguably somewhat pedagogic (demonstrates how to build a test framework in under 700 active SLOC), and importantly, works just fine. #### Etymology and roots -[Test fixture](https://en.wikipedia.org/wiki/Test_fixture) *is an environment used to consistently test some item, device, or piece of software*. In automated tests, it is typically a piece of code that is reused within the test suite of a project, to perform initialization and/or teardown tasks common to several test cases. +A [test fixture](https://en.wikipedia.org/wiki/Test_fixture) is defined as *an environment used to consistently test some item, device, or piece of software*. In automated tests, it is typically a piece of code that is reused within the test suite of a project, to perform initialization and/or teardown tasks common to several test cases. -A test framework can be reused across many different projects, and the error-catching and reporting code, if anything, is something that is shared across all test cases. Also, following our naming scheme, it had to be called `unpythonic.test.something`, and `fixtures` just happened to fit the theme. +A test framework can be reused across many different projects, and the error-catching and reporting code, if anything, is something that is shared across all test cases. Also, following our naming scheme, the framework had to be called `unpythonic.test.something`, and `fixtures` just happened to fit the theme. Inspired by [Julia](https://julialang.org/)'s standard-library [`Test` package](https://docs.julialang.org/en/v1/stdlib/Test/), and [chapter 9 of Peter Seibel's Practical Common Lisp](http://www.gigamonkeys.com/book/practical-building-a-unit-test-framework.html). -### ``dbg``: debug-print expressions with source code +### `dbg`: debug-print expressions with source code + +**Changed in v0.15.0.** *We now use the [`mcpyrate`](https://github.com/Technologicat/mcpyrate/) macro expander instead of `macropy`. Updated the REPL note below.* + +*Also, `dbgprint_expr` is now a dynvar.* -**Changed in 0.14.2.** The `dbg[]` macro now works in the REPL, too. You can use `mcpyrate.repl.console` (a.k.a. `macropython -i` in the shell) or the IPython extension `mcpyrate.repl.iconsole`. +**Changed in 0.14.2.** *The `dbg[]` macro now works in the REPL, too. You can use `mcpyrate.repl.console` (a.k.a. `macropython -i` in the shell) or the IPython extension `mcpyrate.repl.iconsole`.* [DRY](https://en.wikipedia.org/wiki/Don't_repeat_yourself) out your [qnd](https://en.wiktionary.org/wiki/quick-and-dirty) debug printing code. Both block and expression variants are provided: @@ -2076,7 +2716,7 @@ z = dbg[25 + 17] # --> [file.py:15] (25 + 17): 42 assert z == 42 # surrounding an expression with dbg[...] doesn't alter its value ``` -**In the block variant**, just like in ``nb``, a custom print function can be supplied as the first positional argument. This avoids transforming any uses of built-in ``print``: +**In the block variant**, just like in `nb`, a custom print function can be supplied as the first positional argument. This avoids transforming any uses of built-in `print`: ```python prt = lambda *args, **kwargs: print(*args) @@ -2093,13 +2733,13 @@ with dbg[prt]: ``` -The reference to the custom print function (i.e. the argument to the ``dbg`` block) **must be a bare name**. Support for methods may or may not be added in a future version. +The reference to the custom print function (i.e. the argument to the `dbg` block) **must be a bare name**. Support for methods may or may not be added in a future version. -**In the expr variant**, to customize printing, just assign a function to the dynvar ``dbgprint_expr`` via `with dyn.let(dbgprint_expr=...)`. If no custom printer is set, a default implementation is used. +**In the expr variant**, to customize printing, just assign a function to the dynvar `dbgprint_expr` via `with dyn.let(dbgprint_expr=...)`. If no custom printer is set, a default implementation is used. -For details on implementing custom debug print functions, see the docstrings of ``unpythonic.syntax.dbgprint_block`` and ``unpythonic.syntax.dbgprint_expr``, which provide the default implementations. +For details on implementing custom debug print functions, see the docstrings of `unpythonic.syntax.dbgprint_block` and `unpythonic.syntax.dbgprint_expr`, which provide the default implementations. -**CAUTION**: The source code is back-converted from the AST representation; hence its surface syntax may look slightly different to the original (e.g. extra parentheses). See ``mcpyrate.unparse``. +**CAUTION**: The source code is back-converted from the AST representation; hence its surface syntax may look slightly different to the original (e.g. extra parentheses). See `mcpyrate.unparse`. Inspired by the [dbg macro in Rust](https://doc.rust-lang.org/std/macro.dbg.html). @@ -2107,9 +2747,9 @@ Inspired by the [dbg macro in Rust](https://doc.rust-lang.org/std/macro.dbg.html Stuff that didn't fit elsewhere. -### ``nb``: silly ultralight math notebook +### `nb`: silly ultralight math notebook -Mix regular code with math-notebook-like code in a ``.py`` file. To enable notebook mode, ``with nb``: +Mix regular code with math-notebook-like code in a `.py` file. To enable notebook mode, `with nb`: ```python from unpythonic.syntax import macros, nb @@ -2129,9 +2769,9 @@ with nb[pprint]: assert _ == 3 * x * y ``` -Expressions at the top level auto-assign the result to ``_``, and auto-print it if the value is not ``None``. Only expressions do that; for any statement that is not an expression, ``_`` retains its previous value. +Expressions at the top level auto-assign the result to `_`, and auto-print it if the value is not `None`. Only expressions do that; for any statement that is not an expression, `_` retains its previous value. -A custom print function can be supplied as the first positional argument to ``nb``. This is useful with SymPy (and [latex-input](https://github.com/clarkgrubb/latex-input) to use α, β, γ, ... as actual variable names). +A custom print function can be supplied as the first positional argument to `nb`. This is useful with SymPy (and [latex-input](https://github.com/clarkgrubb/latex-input) to use α, β, γ, ... as actual variable names). Obviously not intended for production use, although is very likely to work anywhere. @@ -2141,20 +2781,41 @@ Is this just a set of macros, a language extension, or a compiler for a new lang ### The xmas tree combo -The macros in ``unpythonic.syntax`` are designed to work together, but some care needs to be taken regarding the order in which they expand. This complexity unfortunately comes with any pick-and-mix-your-own-language kit, because some features inevitably interact. For example, it is possible to lazify [continuation-enabled](https://en.wikipedia.org/wiki/Continuation-passing_style) code, but running the transformations the other way around produces nonsense. +The macros in `unpythonic.syntax` are designed to work together, but some care needs to be taken regarding the order in which they expand. This complexity unfortunately comes with any pick-and-mix-your-own-language kit, because some features inevitably interact. For example, it is possible to lazify [continuation-enabled](https://en.wikipedia.org/wiki/Continuation-passing_style) code, but running the transformations the other way around produces nonsense. -For simplicity, **the block macros make no attempt to prevent invalid combos** (unless there is a specific technical reason to do that for some particular combination). Be careful; e.g. don't nest several ``with tco`` blocks (lexically), that won't work. +The correct **xmas tree invocation** is: -The **AST edits** performed by the block macros are designed to run **in the following order (leftmost first)**: +```python +with prefix, autoreturn, quicklambda, multilambda, envify, lazify, namedlambda, autoref, autocurry, tco: + ... +``` + +Here `tco` can be replaced with `continuations`, if needed. + +We have taken into account that: + + - Outside-in: `prefix`, `autoreturn`, `quicklambda`, `multilambda` + - Two-pass: `envify`, `lazify`, `namedlambda`, `autoref`, `autocurry`, `tco`/`continuations` + +[The dialect examples](dialects.md) use this ordering. + +For simplicity, **the block macros make no attempt to prevent invalid combos**, unless there is a specific technical reason to do that for some particular combination. Be careful; e.g. do not nest several `with tco` blocks (lexically), that will not work. + +As an example of a specific technical reason, the `tco` macro skips already expanded `with continuations` blocks lexically contained within the `with tco`. This allows the [Lispython dialect](dialects/lispython.md) to support `continuations`. + + +#### AST edit order vs. macro invocation order + +The **AST edits** performed by the block macros are designed to run in the following order (leftmost first): ``` -prefix > autoreturn, quicklambda > multilambda > continuations or tco > ... +prefix > nb > autoreturn, quicklambda > multilambda > continuations or tco > ... ... > autocurry > namedlambda, autoref > lazify > envify ``` -The ``let_syntax`` (and ``abbrev``) block may be placed anywhere in the chain; just keep in mind what it does. +The `let_syntax` (and `abbrev`) block may be placed anywhere in the chain; just keep in mind what it does. -The ``dbg`` block can be run at any position after ``prefix`` and before ``tco`` (or ``continuations``). (It must be able to see function calls in Python's standard format, for detecting calls to the print function.) +The `dbg` block can be run at any position after `prefix` and before `tco` (or `continuations`). It must be able to see function calls in Python's standard format, for detecting calls to the print function. The correct ordering for **block macro invocations** - which is the actual user-facing part - is somewhat complicated by the fact that some of the above are two-pass macros. Consider this artificial example, where `mac` is a two-pass macro: @@ -2164,13 +2825,18 @@ with mac: ... ``` -The invocation `with mac` is *lexically on the outside*, thus the macro expander sees it first. The expansion order is then: +The invocation `with mac` is *lexically on the outside*, thus the macro expander sees it first. The expansion order then becomes: 1. First pass (outside in) of `with mac`. 2. Explicit recursion by `with mac`. This expands the `with cheese`. 3. Second pass (inside out) of `with mac`. -So, for example, even though `lazify` must *perform its AST editing* after `autocurry`, it is actually a two-pass macro. The first pass (outside in) only performs some preliminary analysis; the actual lazification happens in the second pass (inside out). So the correct invocation comboing these two is `with lazify, autocurry`. Similarly, `with lazify, continuations` is correct, even though the CPS transformation must occur first; these are both two-pass macros that perform their edits in the inside-out pass. See [the dialect examples](../unpythonic/dialects/) for combo invocations that are known to work. +So, for example, even though `lazify` must *perform its AST edits* after `autocurry`, it happens to be a two-pass macro. The first pass (outside in) only performs some preliminary analysis; the actual lazification happens in the second pass (inside out). So the correct invocation comboing these two is `with lazify, autocurry`. Similarly, `with lazify, continuations` is correct, even though the CPS transformation must occur first; these are both two-pass macros that perform their edits in the inside-out pass. + +Further details on individual block macros can be found in our [notes on macros](design-notes.md#detailed-notes-on-macros). + + +#### Single-line vs. multiline invocation format Example combo in the single-line format: @@ -2179,7 +2845,7 @@ with autoreturn, lazify, tco: ... ``` -In the multiline format: +The same combo in the multiline format: ```python with autoreturn: @@ -2188,15 +2854,12 @@ with autoreturn: ... ``` -Of these, `autoreturn` expands outside-in, while `lazify` and `tco` are both two-pass macros. - -We aim to improve the macro docs in the future. For now, to see if something is a two-pass macro, grep the codebase for `expander.visit`; that is the *explicit recursion* mentioned above, and means that within that function, anything below that line will run in the inside-out pass. See [the `mcpyrate` manual](https://github.com/Technologicat/mcpyrate/blob/master/doc/main.md#expand-macros-inside-out). +In MacroPy (which was used up to v0.14.3), there sometimes were [differences](https://github.com/azazel75/macropy/issues/21) between the behavior of the single-line and multi-line invocation format, but in `mcpyrate` (which is used by v0.15.0 and later), they should behave the same. -See our [notes on macros](../doc/design-notes.md#detailed-notes-on-macros) for more information. +With `mcpyrate`, there is still [a minor difference](https://github.com/Technologicat/mcpyrate/issues/3) if there are at least three nested macro invocations, and a macro is scanning the tree for another macro invocation; then the tree looks different depending on whether the single-line or the multi-line format was used. The differences in that are as one would expect knowing [how `with` statements look like](https://greentreesnakes.readthedocs.io/en/latest/nodes.html#With) in the Python AST. The reason the difference manifests only for three or more macro invocations is that `mcpyrate` pops the macro that is being expanded before it hands over the tree to the macro code; hence if there are only two, the inner tree will have only one "context manager" in its `with`. -**NOTE**: In MacroPy, there sometimes were [differences](https://github.com/azazel75/macropy/issues/21) between the behavior of the single-line and multi-line invocation format, but in `mcpyrate`, they should behave the same. +**NOTE** to the curious, and to future documentation maintainers: To see if something is a two-pass macro, grep the codebase for `expander.visit_recursively`; that is the *explicit recursion* mentioned above, and means that within that function, anything below that line will run in the inside-out pass. See [the `mcpyrate` manual](https://github.com/Technologicat/mcpyrate/blob/master/doc/main.md#expand-macros-inside-out). -With `mcpyrate`, there is still [a minor difference](https://github.com/Technologicat/mcpyrate/issues/3) if there are at least three nested macro invocations, and a macro is scanning the tree for another macro invocation; then the tree looks different depending on whether the single-line or the multi-line format was used. The differences in that are as one would expect knowing [how `with` statements look like](https://greentreesnakes.readthedocs.io/en/latest/nodes.html#With) in the Python AST. The reason the difference manifests only for three or more macro invocations is that `mcpyrate` pops the macro that is being expanded before it hands over the tree to the macro code; hence if there are only two, the inner tree will have only one "context manager" in its `with`. ### Emacs syntax highlighting @@ -2237,12 +2900,12 @@ Tested with `anaconda-mode`. #### How to use (for Emacs beginners) -If you use the [Spacemacs](http://spacemacs.org/) kit, the right place to insert the snippet is into the function `dotspacemacs/user-config`. Here's [my spacemacs.d](https://github.com/Technologicat/spacemacs.d/) for reference; the snippet is in `prettify-symbols-config.el`, and it's invoked from `dotspacemacs/user-config` in `init.el`. +If you use the [Spacemacs](http://spacemacs.org/) kit, the right place to insert the snippet is into the function `dotspacemacs/user-config`. Here's [my spacemacs.d](https://github.com/Technologicat/spacemacs.d/) for reference; the snippet is in `prettify-symbols-config.el`, and it is invoked from `dotspacemacs/user-config` in `init.el`. In a basic Emacs setup, the snippet goes into the `~/.emacs` startup file, or if you have an `.emacs.d/` directory, then into `~/.emacs.d/init.el`. ### This is semantics, not syntax! -[Strictly speaking](https://stackoverflow.com/questions/17930267/what-is-the-difference-between-syntax-and-semantics-of-programming-languages), ``True``. We just repurpose Python's existing syntax to give it new meanings. However, in [the Racket reference](https://docs.racket-lang.org/reference/), **a** *syntax* designates a macro, in contrast to a *procedure* (regular function). We provide syntaxes in this particular sense. The name ``unpythonic.syntax`` is also shorter to type than ``unpythonic.semantics``, less obscure, and close enough to convey the intended meaning. +[Strictly speaking](https://stackoverflow.com/questions/17930267/what-is-the-difference-between-syntax-and-semantics-of-programming-languages), `True`. We just repurpose Python's existing syntax to give it new meanings. However, in [the Racket reference](https://docs.racket-lang.org/reference/), **a** *syntax* designates a macro, in contrast to a *procedure* (regular function). We provide syntaxes in this particular sense. The name `unpythonic.syntax` is also shorter to type than `unpythonic.semantics`, less obscure, and close enough to convey the intended meaning. If you want custom *syntax* proper, or want to package a set of block macros as a custom language that extends Python, then you may be interested in our sister project [`mcpyrate`](https://github.com/Technologicat/mcpyrate). diff --git a/doc/readings.md b/doc/readings.md index 80200ac0..8ded43a7 100644 --- a/doc/readings.md +++ b/doc/readings.md @@ -7,6 +7,7 @@ - [REPL server](repl.md) - [Troubleshooting](troubleshooting.md) - [Design notes](design-notes.md) +- [Essays](essays.md) - **Additional reading** - [Contribution guidelines](../CONTRIBUTING.md) @@ -83,10 +84,19 @@ The common denominator is programming. Some relate to language design, some to c - [PyPy3](http://pypy.org/), fast, JIT-ing Python 3 that's mostly a drop-in replacement for CPythons 3.6 and 3.7. As of April 2021, support for 3.8 is in the works. Macro expanders (`macropy`, `mcpyrate`) work, too. -- [Brython](https://brython.info/): Python 3 in the browser, as a replacement for JavaScript. - - No separate compile step - the compiler is implemented in JS. Including a script tag of type text/python invokes it. - - Doesn't have the `ast` module, so no way to run macro expanders. - - Also quite a few other parts are missing, understandably. Keep in mind the web client is rather different as an environment from the server side or the desktop. So for new apps, Brython is ok, but if you have some existing Python code you want to move into the browser, it might or might not work, depending on what your code needs. +- [Pyodide](https://github.com/pyodide/pyodide): Python with the scientific stack, compiled to WebAssembly. + - [Docs](https://pyodide.org/en/stable/). + - [Online REPL](https://pyodide.org/en/stable/console.html). + - Has **the scientific Python stack**, and also supports **any pure-Python PyPI wheel**. + - The `ast` module works. This should be able to run `mcpyrate` and `unpythonic` in the browser! + +- Historical Python-in-the-browser efforts: + - [Brython](https://brython.info/): Python 3 in the browser, as a replacement for JavaScript. + - No separate compile step - the compiler is implemented in JS. Including a script tag of type text/python invokes it. + - Doesn't have the `ast` module, so no way to run macro expanders. + - Also quite a few other parts are missing, understandably. Keep in mind the web client is rather different as an environment from the server side or the desktop. So for new apps, Brython is ok, but if you have some existing Python code you want to move into the browser, it might or might not work, depending on what your code needs. + - [PyPy.js](http://pypyjs.org/): PyPy python interpreter, compiled for the web via [emscripten](http://emscripten.org/), with a custom JIT backend that emits [asm.js](http://asmjs.org/) code at runtime. + - Last updated in 2015, no longer working. - Counterpoint: [Eric Torreborre (2019): When FP does not save us](https://medium.com/barely-functional/when-fp-does-not-save-us-92b26148071f) @@ -149,7 +159,7 @@ The common denominator is programming. Some relate to language design, some to c - A special `uninitialized` value (which the paper calls ☠) is needed, because Scope - in the sense of controlling lexical name resolution - is a static (purely lexical) concept, but whether a particular name (once lexically resolved) has been initialized (or, say, whether it has been deleted) is a dynamic (run-time) feature. (I would say "property", if that word didn't have an entirely different technical meaning in Python.) - Our `continuations` macro essentially does what the authors call *a standard [CPS](https://en.wikipedia.org/wiki/Continuation-passing_style) transformation*, plus some technical details due to various bits of impedance mismatch. -- [John Shutt's blog](https://fexpr.blogspot.com/) contains many interesting posts on programming language design. He's the author of the [Kernel](https://web.cs.wpi.edu/~jshutt/kernel.html) Lisp dialect. Some pickings from the blog: +- [John Shutt's blog](https://fexpr.blogspot.com/) contains many interesting posts on programming language design. He [was](http://lambda-the-ultimate.org/node/5623) the author of the [Kernel](https://web.cs.wpi.edu/~jshutt/kernel.html) Lisp dialect. Some pickings from his blog: - [Fexpr (2011)](https://fexpr.blogspot.com/2011/04/fexpr.html). - The common wisdom that macros were a better choice is misleading. - [Bypassing no-go theorems (2013)](https://fexpr.blogspot.com/2013/07/bypassing-no-go-theorems.html). @@ -157,6 +167,7 @@ The common denominator is programming. Some relate to language design, some to c - [Abstractive power (2013)](https://fexpr.blogspot.com/2013/12/abstractive-power.html). - [Where do types come from? (2011)](https://fexpr.blogspot.com/2011/11/where-do-types-come-from.html). - [Continuations and term-rewriting calculi (2014)](https://fexpr.blogspot.com/2014/03/continuations-and-term-rewriting-calculi.html). + - [Interpreted programming languages (2016)](https://fexpr.blogspot.com/2016/08/interpreted-programming-languages.html) - Discussion of Kernel on LtU: [Decomposing lambda - the Kernel language](http://lambda-the-ultimate.org/node/1680). - [Walid Taha 2003: A Gentle Introduction to Multi-stage Programming](https://www.researchgate.net/publication/221024597_A_Gentle_Introduction_to_Multi-stage_Programming) @@ -168,6 +179,52 @@ The common denominator is programming. Some relate to language design, some to c - [Types vs. traits for dispatch](https://discourse.julialang.org/t/types-vs-traits-for-dispatch/46296) (discussion) - We have a demonstration in [unpythonic.tests.test_dispatch](../unpythonic/tests/test_dispatch.py). +- [Pascal Costanza's Highly Opinionated Guide to Lisp (2013)](http://www.p-cos.net/lisp/guide.html) + +- [Peter Seibel (2005): Practical Common Lisp](https://gigamonkeys.com/book/) + - This book is an excellent introduction that walks through Common Lisp, including some advanced features. It is also useful for non-lispers to take home interesting ideas from CL. + +- R. Kent Dybvig, Simon Peyton Jones, Amr Sabry (2007). A Monadic Framework for Delimited Continuations. Journal of functional programming, 17(6), 687-730. Preprint [here](https://legacy.cs.indiana.edu/~dyb/pubs/monadicDC.pdf). + - Particularly approachable explanation of delimited continuations. + - Could try building that for `unpythonic` in a future version. + +- [Wat: Concurrency and Metaprogramming for JS](https://github.com/manuel/wat-js) + - [pywat: Interpreter of the Wat language written in Python](https://github.com/piokuc/pywat) + - [Example of Wat in Manuel Simoni's blog (2013)](http://axisofeval.blogspot.com/2013/05/green-threads-in-browser-in-20-lines-of.html) + +- [Richard P. Gabriel, Kent M. Pitman (2001): Technical Issues of Separation in Function Cells and Value Cells](https://dreamsongs.com/Separation.html) + - A discussion of [Lisp-1 vs. Lisp-2](https://en.wikipedia.org/wiki/Lisp-1_vs._Lisp-2), particularly of historical interest. + - Summary: Lisp-1 often leads to more readable code than Lisp-2, but by the time this became clear, for Common Lisp that train had already sailed. The authors suggest that instead of fixing CL with a backward compatibility breaking change, future Lisps would do well to take lessons learned from both Scheme and Common Lisp. In my own opinion, [Racket](https://racket-lang.org/) indeed has. + - Interestingly, there are more namespaces in Lisps than just values and functions, so, as the authors note, the popular names "Lisp-1" and "Lisp-2" are actually misnomers. For example, the labels for the Common Lisp construct `TAGBODY`/`GO` live in their own namespace. + - If explained using Python terminology, a Common Lisp symbol instance essentially has one attribute for each namespace, that stores the value bound to that symbol in that namespace. + +- [`hoon`: The C of Functional Programming](https://urbit.org/docs/hoon/) + - Interesting take on an alternative computing universe where the functional camp won systems programming. These people have built [a whole operating system](https://github.com/urbit/urbit) on a Turing-complete non-lambda automaton, Nock. + - For my take, see [the opinion piece in Essays](essays.md#hoon-the-c-of-functional-programming). + - Judging by the docs, `hoon` is definitely ha-ha-only-serious, but I am not sure of whether it is serious-serious. See the comments to [the entry on Manuel Simoni's blog](http://axisofeval.blogspot.com/2015/07/what-i-learned-about-urbit-so-far.html) - some people do think `hoon` is actually useful. + - Technical points: + - `hoon` does not have syntactic macros. The reason given in the docs is the same as sometimes heard in the Python community - having a limited number of standard control structures, you always know what you are looking at. + - Interestingly, `hoon` has uniform support for *wide* and *tall* modes; it does not use parentheses, but uses a single space (in characteristic `hoon` fashion, termed an *ace*) versus multiple spaces (respectively, a *gap*). "Multiple spaces" allows also newlines, like in LaTeX. So [SRFI-110](https://srfi.schemers.org/srfi-110/srfi-110.html) is not the only attempt at a two-mode uniform grouping syntax. + +- *Ab initio* programming language efforts: + - `hoon`, see separate entry above. + - [Arc](http://www.paulgraham.com/arc.html) by Paul Graham and Robert Morris. + - [Discussion on](https://news.ycombinator.com/item?id=10535364) the Nile programming language developed by Ian Piumarta, Alan Kay, et al. + - Especially the low-level [Maru](https://www.piumarta.com/software/maru/) language by Ian Piumarta seems interesting. + - *Maru is a symbolic expression evaluator that can compile its own implementation language.* + - It compiles s-expressions to IA32 machine code, and has a metacircular evaluator implemented in less than 2k SLOC. It bootstraps from C. + +- [LtU: Why is there no widely accepted progress for 50 years?](http://lambda-the-ultimate.org/node/5590) + - Discussion on how programming languages *have* improved. + - Contains interesting viewpoints, such as dmbarbour's suggestion that much of modern hardware is essentially "compiled" from a hardware description language such as VHDL. + +- [Matthew Might: First-class (run-time) macros and meta-circular evaluation](https://matt.might.net/articles/metacircular-evaluation-and-first-class-run-time-macros/) + - *First-class macros are macros that can be bound to variables, passed as arguments and returned from functions. First-class macros expand and evaluate syntax at run-time.* + +- Useful concepts for programming language design: + - [Cognitive dimensions of notations](https://en.wikipedia.org/wiki/Cognitive_dimensions_of_notations) + - [System quality attributes](https://en.wikipedia.org/wiki/List_of_system_quality_attributes) + # Python-related FP resources @@ -188,6 +245,7 @@ Python clearly wants to be an impure-FP language. A decorator with arguments *is - [pyrsistent: Persistent/Immutable/Functional data structures for Python](https://github.com/tobgu/pyrsistent) - [pampy: Pattern matching for Python](https://github.com/santinic/pampy) (pure Python, no AST transforms!) + - Note that Python got [native support for pattern matching in 3.10](https://docs.python.org/3/whatsnew/3.10.html#pep-634-structural-pattern-matching) using the `match`/`case` construct. - [List of languages that compile to Python](https://github.com/vindarel/languages-that-compile-to-python) including Hy, a Lisp (in the [Lisp-2](https://en.wikipedia.org/wiki/Lisp-1_vs._Lisp-2) family) that can use Python libraries. diff --git a/doc/repl.md b/doc/repl.md index 6c101be6..d253e928 100644 --- a/doc/repl.md +++ b/doc/repl.md @@ -7,6 +7,7 @@ - **REPL server** - [Troubleshooting](troubleshooting.md) - [Design notes](design-notes.md) +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index 1693ea59..fb0d027e 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -7,6 +7,7 @@ - [REPL server](repl.md) - **Troubleshooting** - [Design notes](design-notes.md) +- [Essays](essays.md) - [Additional reading](readings.md) - [Contribution guidelines](../CONTRIBUTING.md) @@ -19,6 +20,8 @@ - [Cannot import the name `macros`?](#cannot-import-the-name-macros) - [But I did run my program with `macropython`?](#but-i-did-run-my-program-with-macropython) - [I'm hacking a macro inside a module in `unpythonic.syntax`, and my changes don't take?](#im-hacking-a-macro-inside-a-module-in-unpythonicsyntax-and-my-changes-dont-take) + - [Both `unpythonic` and library `x` provide language-extension feature `y`. Which is better?](#both-unpythonic-and-library-x-provide-language-extension-feature-y-which-is-better) + - [How to list the whole public API, and only the public API?](#how-to-list-the-whole-public-api-and-only-the-public-api) @@ -32,7 +35,7 @@ On the other hand, `unpythonic` is a kitchen-sink language extension, and half o If you intend to **use** `unpythonic.syntax` or `unpythonic.dialects`, or if you intend to **develop** `unpythonic` (specifically: to be able to run its test suite), then you will need a macro expander. -As of v0.15.0, specifically you'll need [`mcpyrate`](https://github.com/Technologicat/mcpyrate). +As of v0.15.0, specifically you will need [`mcpyrate`](https://github.com/Technologicat/mcpyrate). ### Why `mcpyrate` and not MacroPy? @@ -44,7 +47,7 @@ Beside the advanced features, the reason we use `mcpyrate` is that the `unpython ### Cannot import the name `macros`? -In `mcpyrate`-based programs, there is no run-time object named `macros`, so failing to import that usually means that, for some reason, the macro expander was not active. +In `mcpyrate`-based programs, there is no run-time object named `macros`, so failing to import that usually means that, for some reason, the macro expander is not enabled. Macro-enabled, `mcpyrate`-based programs expect to be run with `macropython` (included in the [`mcpyrate` PyPI package](https://pypi.org/project/mcpyrate/)) instead of bare `python3`. @@ -68,16 +71,91 @@ This will force a recompile of the `.py` files the next time they are loaded. Th ### I'm hacking a macro inside a module in `unpythonic.syntax`, and my changes don't take? -This is also likely due to a stale bytecode cache. As of `mcpyrate` 3.4.0, macro re-exports, used by `unpythonic.syntax.__init__`, may confuse the macro-dependency analyzer that determines bytecode cache validity. +This is also likely due to a stale bytecode cache. As of `mcpyrate` 3.4.0, macro re-exports, used by `unpythonic.syntax.__init__`, are not seen by the macro-dependency analyzer that determines bytecode cache validity. -The thing to realize here is that as per macropythonic tradition, in `mcpyrate`, a function being a macro is a property of its **use site**, not of its definition site. So how do we re-export a macro? We simply re-export the macro function, like we would do for any other function. +The important point to realize here is that as per macropythonic tradition, in `mcpyrate`, a function being a macro is a property of its **use site**, not of its definition site. So how do we re-export a macro? We simply re-export the macro function, like we would do for any other function. -Importantly, the import to make that re-export happen does not look like a macro-import. This is the right way to do it, since we want to make the object (macro function) available for clients to import, **not** establish bindings in the macro expander *for compiling the module `unpythonic.syntax.__init__` itself*. (The latter is what a macro-import does - it establishes macro bindings *for the module it lexically appears in*.) +The import to make that re-export happen does not look like a macro-import. This is the right way to do it, since we want to make the object (macro function) available for clients to import, **not** establish bindings in the macro expander *for compiling the module `unpythonic.syntax.__init__` itself*. (The latter is what a macro-import does - it establishes macro bindings *for the module it lexically appears in*.) -The problem is, the macro-dependency analyzer only looks at the macro-import dependency graph, not the full dependency graph, so when analyzing the user program (e.g. a unit test module in `unpythonic.syntax.tests`), it doesn't notice that the macro definition has changed. +The problem is, the macro-dependency analyzer only looks at the macro-import dependency graph, not the full dependency graph, so when analyzing the user program (e.g. a unit test module in `unpythonic.syntax.tests`), it does not scan the re-export that points to the changed macro definition. I might modify the `mcpyrate` analyzer in the future, but doing so will make the dependency scan a lot slower than it needs to be in most circumstances, because a large majority of imports in Python have nothing to do with macros. For now, we just note that this issue mainly concerns developers of large macro packages (such as `unpythonic.syntax`) that need to split - for factoring reasons - their macro definitions into separate modules, while presenting all macros to the user in one interface module. This issue does not affect the development of macro-using programs, or any programs where macros are imported from their original definition site (like they always were with MacroPy). Try clearing the bytecode cache in `unpythonic/`; this will force a recompile. + + +### Both `unpythonic` and library `x` provide language-extension feature `y`. Which is better? + +The point of having these features in `unpythonic` is integration, and a consistent API. So if you need only one specific language-extension feature, then a library that concentrates on that particular feature is likely a good choice. If you need the kitchen sink, too, then it's better to use our implementation, since our implementations of the various features are designed to work together. + +In some cases (e.g. the condition system), our implementation may offer extra features not present in the original library that inspired it. + +In other cases (e.g. multiple dispatch), the *other* implementation may be better (e.g. runs much faster). + + +### How to list the whole public API, and only the public API? + +In short, use Python's introspection capabilities. There are some subtleties here; below are some ready-made recipes. + +To view **the public API of a given submodule**: + +```python +import sys +print(sys.modules["unpythonic.collections"].__all__) # for example +``` + +If the `__all__` attribute for some submodule is missing, that submodule has no public API. + +For most submodules, you could just + +```python +print(unpythonic.collections.__all__) # for example +``` + +but there are some public API symbols in `unpythonic` that have the same name as a submodule. In these cases, the object overrides the submodule in the top-level namespace of `unpythonic`. So, for example, for `unpythonic.llist`, the second approach fails because `unpythonic.llist` points to a function, not to a module. Therefore, the first approach is preferable, as it always works. + +To view **the whole public API**, grouped by submodule: + +```python +import sys + +import unpythonic + +submodules = [name for name in dir(unpythonic) + if f"unpythonic.{name}" in sys.modules] + +for name in submodules: + module = sys.modules[f"unpythonic.{name}"] + if hasattr(module, "__all__"): # has a public API? + print("=" * 79) + print(f"Public API of 'unpythonic.{name}':") + print(module.__all__) +``` + +Note that even if you examine the API grouped by submodule, `unpythonic` guarantees all of its public API symbols to be present in the top-level namespace, too, so when you actually import the symbols, you can import them from the top-level namespace. (Actually, the macros expect you to do so, to recognize uses of various `unpythonic` constructs when analyzing code.) + +**Do not** do this to retrieve the submodules: + +```python +import types +submodules_wrong = [name for name in dir(unpythonic) + if issubclass(type(getattr(unpythonic, name)), types.ModuleType)] +``` + +for the same reason as above; in this variant, any submodules that have the same name as an object will be missing from the list. + +To view **the whole public API** available in the top-level namespace: + +```python +import types + +import unpythonic + +non_module_names = [name for name in dir(unpythonic) + if not issubclass(type(getattr(unpythonic, name)), types.ModuleType)] +print(non_module_names) +``` + +Now be very very careful: for the same reason as above, for the correct semantics we must use `issubclass(..., types.ModuleType)`, not `... in sys.modules`. Here we want to list each symbol in the top-level namespace of `unpythonic` that does not point to a module; **including** any objects that override a module in the top-level namespace. diff --git a/flake8rc b/flake8rc deleted file mode 100644 index 23a1ff37..00000000 --- a/flake8rc +++ /dev/null @@ -1,30 +0,0 @@ -[flake8] -# ignore silly style items -ignore = - # too complex (mcgabe) - C901, - # overhanging indent - E126, - # continuation line over-indented for visual indent - E127, - # block comment should start with # - E265, - # expected 1 blank line, found 0 - E301, - # expected 2 blank lines before def - E302, - # expected 2 blank lines after def - E305, - # expected blank line before nested def - E306, - # module level import not at top of file (can cause problems when autopep8 applies it without thinking) - E402, - # line too long >79 chars - E501, - # multiple statements on one line (def) - E704, - # do not assign a lambda expression, use a def (because autopep8 applies it blindly) - E731, - # line break after binary operator - W504 -exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,node_modules,instance,00_stuff,00_old diff --git a/makedist.sh b/makedist.sh index 338298d3..b6c03991 100755 --- a/makedist.sh +++ b/makedist.sh @@ -1,2 +1,2 @@ #!/bin/bash -python3 setup.py sdist bdist_wheel +pdm build diff --git a/measure_coverage.sh b/measure_coverage.sh index 2b3bfc4c..bf051c6e 100755 --- a/measure_coverage.sh +++ b/measure_coverage.sh @@ -6,7 +6,7 @@ # https://coverage.readthedocs.io/en/coverage-5.2.1/#quick-start echo -ne "Measuring...\n" -coverage run --source=. -m runtests +pdm run coverage run --source=. -m runtests echo -ne "Generating report...\n" -coverage html +pdm run coverage html echo -ne "Done. Open htmlcov/index.html in your browser to view.\n" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2e28e2bf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,158 @@ +[project] +name = "unpythonic" +description = "Supercharge your Python with parts of Lisp and Haskell." +authors = [ + { name = "Juha Jeronen", email = "juha.m.jeronen@gmail.com" }, +] +requires-python = ">=3.10,<3.16" + +# the `read` function and long_description_content_type from setup.py are no longer needed, +# modern build tools like pdm/hatch already know how to handle markdown if you point them at a .md file +# they will set the long_description and long_description_content_type for you +readme = "README.md" + +license = "BSD-2-Clause" +license-files = ["LICENSE.md"] + +# This tells whichever build backend you use (pdm in our case) to run its own mechanism to find the version +# of the project and plug it into the metadata +# details for how we instruct pdm to find the version are in the `[tool.pdm.version]` section below +dynamic = ["version"] + +dependencies = [ + "mcpyrate>=4.3.0", + "sympy>=1.13" +] +keywords=["functional-programming", "language-extension", "syntactic-macros", + "tail-call-optimization", "tco", "continuations", "currying", "lazy-evaluation", + "dynamic-variable", "macros", "lisp", "scheme", "racket", "haskell"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +[project.urls] +Repository = "https://github.com/Technologicat/unpythonic" + +[dependency-groups] +dev = [ + "ruff>=0.14.0", + "flake8", + "autopep8", + "importmagic", + "epc", + "jedi>=0.19.2", + # For local pre-release sanity checks: `python -m build --sdist` + # exercises the same sdist path that CI runs on tag push. + "build", + # For validating .github/workflows/*.yml during local dev (not used at runtime). + "pyyaml>=6.0.3", +] + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + +[tool.pdm.version] +# the `file` source tells pdm to look for a line in a file that matches the regex `__version__ = ".*"` +# The regex parse is fairly robust, it can handle arbitray whitespace and comments +source = "file" +path = "unpythonic/__init__.py" + +[tool.pdm.build] +# we don't need to explicitly inclue `mcpyrate.repl`. Unlink with setuptools, pdm automatically includes +# all packages and modules in the source tree pointed to by `includes`, minus any paths matching `excludes` +includes = ["unpythonic"] +excludes = ["**/tests", "**/__pycache__"] + +# note the exclusion of an equivalent to zip_safe. I used to think that zip_safe was a core python metadata flag +# telling pip and other python tools not to include the package in any kind of zip-import or zipapp file. +# I was wrong. zip_safe is a setuptools-specific flag that tells setuptools to not include the package in a bdist_egg +# Since bdist_eggs are no longer really used by anything and have been completely supplanted by wheels, zip_safe has no meaningful effect. +# The effect i think you hoped to achieve with zip_safe is achieved by excluding `__pycache__` folders from +# the built wheels, using the `excludes` field in the `[tool.pdm.build]` section above. + +# most python tools at this point, including mypy, have support for sourcing configuration from pyproject.toml +# making the setup.cfg file unnecessary +[tool.ruff] +line-length = 130 +target-version = "py314" +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + ".venv", + # PEP 798 comprehension unpacking, which ruff 0.15.10 cannot handle, and a syntax + # error cannot be silenced with `# noqa`. Two separate obstacles: it rejects the + # list form as too new for `target-version` above, and it cannot parse the dict, + # set and generator forms at all ("Expected `}`, found `for`"). Retry dropping + # this once ruff both parses those and is told a new enough target. + "unpythonic/syntax/tests/test_comprehension_unpacking_3_15.py", +] + +[tool.ruff.lint] +select = ["E", "W", "F", "SIM"] +ignore = [ + # pycodestyle + "E203", # whitespace before ':' — needed for slice alignment + "E265", # block comment should start with '# ' — commented-out code, markers + "E301", # expected 1 blank line — blank lines are semantic paragraph breaks + "E302", # expected 2 blank lines before def — same + "E305", # expected 2 blank lines after end — same + "E306", # expected blank line before nested def — same + "E402", # module level import not at top — conditional/deferred imports + "E501", # line too long — advisory, not enforced + "E731", # lambda assignment — closures are idiomatic in this codebase + # flake8-simplify + "SIM102", # collapsible if — nested ifs often represent distinct semantic guards + "SIM105", # contextlib.suppress — try/except/pass is more flexible and explicit + "SIM108", # ternary instead of if/else — often less readable, no real gain + "SIM114", # combine if branches — match-casing style; autofix would damage semantics + "SIM117", # combine with statements — nesting shows parent/child; also mcpyrate AST differences + "SIM118", # in-dict-keys — explicit .keys() marks the variable as a dictlike + "SIM300", # yoda conditions — natural reading order preferred + "SIM910", # dict.get with None default — explicit None documents programmer intent + # Note: SIM103 (return condition directly) is intentionally NOT ignored here. + # It is enabled as an advisory — CI runs it in a non-failing second pass. + # The two sites where it fires have been reviewed and carry a site-local + # `# noqa: SIM103` with the reason, so the advisory pass is currently empty + # and anything it reports from now on is new. +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "F403"] # re-exports via star-import + +[tool.mypy] +show_error_codes = true + +[tool.coverage.run] +# Coverage signal is about which lines of *production code* run. +# This project uses its own macro-aware test framework +# (`unpythonic.test.fixtures`) rather than pytest, and the runner +# already reports pass/fail/error per testset — coverage analysis +# of the test files themselves would add noise without insight. +# +# Excluding tests also sidesteps coverage.py's source-parser +# choking on macro-using test files whose unexpanded source is +# not legal standard Python (e.g. `nonlocal x` after `x = None` +# in test_conts.py, which the `continuations` macro splits into +# a separate function at expansion time). +source = ["unpythonic"] +omit = [ + "*/tests/*", +] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4fe57592..00000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -mcpyrate>=3.5.0 -sympy>=1.4 diff --git a/runtests.py b/runtests.py index e928983e..38233e5d 100644 --- a/runtests.py +++ b/runtests.py @@ -1,58 +1,30 @@ # -*- coding: utf-8 -*- -"""Run all tests for `unpythonic`. +"""Run all tests for ``unpythonic``. The test framework uses macros, but this top-level script does not. This can be -run under regular `python3` (i.e. does not need the `macropython` wrapper from -`mcpyrate`). +run under regular ``python3`` (i.e. does not need the ``macropython`` wrapper +from ``mcpyrate``). """ import os -import re import sys -from importlib import import_module -from unpythonic.test.fixtures import session, testset, tests_errored, tests_failed -from unpythonic.collections import unbox +from unpythonic.test.runner import discover_testmodules, run import mcpyrate.activate # noqa: F401 -def listtestmodules(path): - testfiles = listtestfiles(path) - testmodules = [modname(path, fn) for fn in testfiles] - return list(sorted(testmodules)) - -def listtestfiles(path, prefix="test_", suffix=".py"): - return [fn for fn in os.listdir(path) if fn.startswith(prefix) and fn.endswith(suffix)] - -def modname(path, filename): # some/dir/mod.py --> some.dir.mod - modpath = re.sub(os.path.sep, r".", path) - themod = re.sub(r"\.py$", r"", filename) - return ".".join([modpath, themod]) - def main(): - with session(): - # All folders containing unit tests are named `tests` (plural). - # - # The testing framework is called `unpythonic.test.fixtures`, - # so it lives in the only subfolder in the project that is named - # `test` (singular). - testsets = (("regular code", (listtestmodules(os.path.join("unpythonic", "tests")) + - listtestmodules(os.path.join("unpythonic", "net", "tests")))), - ("macros", listtestmodules(os.path.join("unpythonic", "syntax", "tests"))), - ("dialects", listtestmodules(os.path.join("unpythonic", "dialects", "tests")))) - for tsname, modnames in testsets: - with testset(tsname): - for m in modnames: - # Wrap each module in its own testset to protect the umbrella testset - # against ImportError as well as any failures at macro expansion time. - with testset(m): - # TODO: We're not inside a package, so we currently can't use a relative import. - # TODO: So we just hope this resolves to the local `unpythonic` source code, - # TODO: not to an installed copy of the library. - mod = import_module(m) - mod.runtests() - all_passed = (unbox(tests_failed) + unbox(tests_errored)) == 0 - return all_passed + # All folders containing unit tests are named `tests` (plural). + # + # The testing framework is called `unpythonic.test.fixtures`, + # so it lives in the only subfolder in the project that is named + # `test` (singular). + testsets = [("regular code", (discover_testmodules(os.path.join("unpythonic", "tests")) + + discover_testmodules(os.path.join("unpythonic", "net", "tests")) + + discover_testmodules(os.path.join("unpythonic", "monads", "tests")))), + ("macros", discover_testmodules(os.path.join("unpythonic", "syntax", "tests"))), + ("dialects", discover_testmodules(os.path.join("unpythonic", "dialects", "tests")))] + return run(testsets) if __name__ == '__main__': if not main(): diff --git a/setup.py b/setup.py deleted file mode 100644 index 573ce4ff..00000000 --- a/setup.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -# -"""setuptools-based setup.py for unpythonic. - -Tested on Python 3.8. - -Usage as usual with setuptools: - python3 setup.py build - python3 setup.py sdist - python3 setup.py bdist_wheel --universal - python3 setup.py install - -For details, see - http://setuptools.readthedocs.io/en/latest/setuptools.html#command-reference -or - python3 setup.py --help - python3 setup.py --help-commands - python3 setup.py --help bdist_wheel # or any command -""" - -import ast -import os - -from setuptools import setup # type: ignore[import] - - -def read(*relpath, **kwargs): # https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-setup-script - with open(os.path.join(os.path.dirname(__file__), *relpath), - encoding=kwargs.get('encoding', 'utf8')) as fh: - return fh.read() - -# Extract __version__ from the package __init__.py -# (since it's not a good idea to actually run __init__.py during the build process). -# -# http://stackoverflow.com/questions/2058802/how-can-i-get-the-version-defined-in-setup-py-setuptools-in-my-package -# -init_py_path = os.path.join("unpythonic", "__init__.py") -version = None -try: - with open(init_py_path) as f: - for line in f: - if line.startswith("__version__"): - module = ast.parse(line, filename=init_py_path) - expr = module.body[0] - assert isinstance(expr, ast.Assign) - v = expr.value - if type(v) is ast.Constant: # Python 3.8+ - # mypy understands `isinstance(..., ...)` but not `type(...) is ...`, - # and we want to match on the exact type, not any subclass that might be - # added in some future Python version. - assert isinstance(v, ast.Constant) - version = v.value - elif type(v) is ast.Str: - assert isinstance(v, ast.Str) # mypy - version = v.s - break -except FileNotFoundError: - pass -if not version: - raise RuntimeError(f"Version information not found in {init_py_path}") - -######################################################### -# Call setup() -######################################################### - -setup( - name="unpythonic", - version=version, - packages=["unpythonic", "unpythonic.syntax"], - provides=["unpythonic"], - keywords=["functional-programming", "language-extension", "syntactic-macros", - "tail-call-optimization", "tco", "continuations", "currying", "lazy-evaluation", - "dynamic-variable", "macros", "lisp", "scheme", "racket", "haskell"], - install_requires=[], # mcpyrate is optional for us, so we can't really put it here even though we recommend it. - python_requires=">=3.6,<3.10", - author="Juha Jeronen", - author_email="juha.m.jeronen@gmail.com", - url="https://github.com/Technologicat/unpythonic", - description="Supercharge your Python with parts of Lisp and Haskell.", - long_description=read("README.md"), - long_description_content_type="text/markdown", - license="BSD", - platforms=["Linux"], - classifiers=["Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - zip_safe=False # macros are not zip safe, because the zip importer fails to find sources. -) diff --git a/unpythonic/__init__.py b/unpythonic/__init__.py index d3ed076f..cf6874a8 100644 --- a/unpythonic/__init__.py +++ b/unpythonic/__init__.py @@ -7,7 +7,7 @@ for a trip down the rabbit hole. """ -__version__ = '0.15.0' +__version__ = '2.4.1-dev' from .amb import * # noqa: F401, F403 from .arity import * # noqa: F401, F403 @@ -17,6 +17,7 @@ from .dispatch import * # noqa: F401, F403 from .dynassign import * # noqa: F401, F403 from .ec import * # noqa: F401, F403 +from .environ import override as environ_override # noqa: F401 from .excutil import * # noqa: F401, F403 from .fix import * # noqa: F401, F403 from .fold import * # noqa: F401, F403 @@ -26,7 +27,7 @@ from .gmemo import * # noqa: F401, F403 from .gtco import * # noqa: F401, F403 from .it import * # noqa: F401, F403 -from .let import * # no guarantees on evaluation order (before Python 3.6), nice syntax # noqa: F401, F403 +from .let import * # noqa: F401, F403 # As of 0.15.0, lispylet is nowadays primarily a code generation target API for macros. from .lispylet import (let as ordered_let, letrec as ordered_letrec, # noqa: F401 @@ -36,12 +37,12 @@ from .llist import * # noqa: F401, F403 from .mathseq import * # noqa: F401, F403 from .misc import * # noqa: F401, F403 -from .numutil import * # noqa: F401, F403 from .seq import * # noqa: F401, F403 from .singleton import * # noqa: F401, F403 from .slicing import * # noqa: F401, F403 from .symbol import * # noqa: F401, F403 from .tco import * # noqa: F401, F403 +from .timeutil import * # noqa: F401, F403 from .typecheck import * # noqa: F401, F403 # -------------------------------------------------------------------------------- @@ -58,3 +59,8 @@ _init_module() del _init_module from .funutil import * # noqa: F401, F403 + +from .numutil import _init_module +_init_module() +del _init_module +from .numutil import * # noqa: F401, F403 diff --git a/unpythonic/amb.py b/unpythonic/amb.py index 45e8cd3b..7c45e8b7 100644 --- a/unpythonic/amb.py +++ b/unpythonic/amb.py @@ -14,7 +14,7 @@ - Presents the source code in the same order as it actually runs. -The implementation is based on the List monad. This is a hack with the bare +The implementation is based on the list monad. This is a hack with the bare minimum of components to make it work, complete with a semi-usable syntax. If you use `mcpyrate`: @@ -33,13 +33,15 @@ __all__ = ["forall", "choice", "insist", "deny"] from collections import namedtuple +from collections.abc import Callable, Iterable +from typing import Any from .arity import arity_includes, UnknownArity -from .llist import nil # we need a sentinel, let's recycle the existing one +from .monads.list import List -Assignment = namedtuple("Assignment", "k v") +Choice = namedtuple("Choice", "k v") -def choice(**binding): +def choice(**binding: Iterable) -> Choice: """Make a nondeterministic choice. Example:: @@ -50,16 +52,24 @@ def choice(**binding): if len(binding) != 1: raise ValueError(f"Expected exactly one name=iterable pair, got {len(binding)} with values {binding}") for k, v in binding.items(): # just one but we don't know its name - return Assignment(k, v) + return Choice(k, v) # Hacky code generator, because Python has ``eval`` but no syntactic macros. # For a cleaner solution based on AST transformation with macros, # see unpythonic.syntax.forall. -def forall(*lines): +def forall(*lines: Choice | Callable) -> tuple: """Nondeterministically evaluate lines. + *Nondeterministic* here is the `amb` sense, not the modern colloquial one: + nothing is stochastic, and results do not vary between runs. Every branch is + explored and every solution returned - multiversal rather than random. + This is essentially a bastardized variant of Haskell's do-notation, - specialized for the List monad. + specialized for the list monad. + + **Prefer the macro version**, ``unpythonic.syntax.forall``, when the macro + layer is available: it is the same feature with a clean design, whereas this + one is complicated by having to avoid macros. See ``doc/design-notes.md``. Examples:: @@ -83,8 +93,8 @@ def forall(*lines): - All choices are evaluated, depth first, and set of results is returned as a tuple. - - If a line returns an iterable, it is implicitly converted into a List - monad containing the same items. + - If a line returns an iterable, it is implicitly converted into a + list monad containing the same items. - This applies also to the RHS of a ``choice``. @@ -94,11 +104,11 @@ def forall(*lines): This allows easily returning a tuple (as one result item) from the computation, as in the above pythagorean triples example. - - If a line returns a single item, it is wrapped into a singleton List - (a List containing that one item). + - If a line returns a single item, it is wrapped into a singleton + list monad (a MonadicList containing that one item). - The final result (containing all the results) is converted from - List monad to tuple for output. + the list monad to tuple for output. - The values currently picked by the choices are bound to names in the environment. To access it, use a ``lambda e: ...`` like in @@ -115,25 +125,27 @@ def forall(*lines): bind = " >> " seq = ".then" - class env: - def __init__(self): - self.names = set() - def assign(self, k, v): + class Scope: + def __init__(self) -> None: + self.names: set[str] = set() + def assign(self, k: str, v: Any) -> None: + """Assign value ``v`` to name ``k`` in this ``Scope``.""" self.names.add(k) setattr(self, k, v) - # simulate lexical closure property for env attrs - # - freevars: set of names that "fall in" from a surrounding lexical scope - def close_over(self, freevars): + def close_over(self, freevars: set[str]) -> None: + """Simulate lexical closure property for scope attrs. + + ``freevars``: set of names that "fall in" from a surrounding scope. + """ names_to_clear = {k for k in self.names if k not in freevars} for k in names_to_clear: delattr(self, k) self.names = freevars.copy() # stuff used inside the eval - e = env() - def begin(*exprs): # args eagerly evaluated by Python - # begin(e1, e2, ..., en): - # perform side effects e1, e2, ..., e[n-1], return the value of en. + e = Scope() + def begin(*exprs: Any) -> Any: # args eagerly evaluated by Python + """begin(e1, e2, ..., en): perform side effects e1, e2, ..., e[n-1], return the value of en.""" return exprs[-1] allcode = "" @@ -144,7 +156,7 @@ def begin(*exprs): # args eagerly evaluated by Python is_first = (j == 0) is_last = (j == len(lines) - 1) - if isinstance(item, Assignment): + if isinstance(item, Choice): name, body = item else: name, body = None, item @@ -175,7 +187,7 @@ def begin(*exprs): # args eagerly evaluated by Python begin_is_open = False # monadic-bind or sequence to the next item, leaving only the appropriate - # names defined in the env (so that we get proper lexical scoping + # names defined in the scope (so that we get proper lexical scoping # even though we use an imperative stateful object to implement it) if not is_last: if name: @@ -199,8 +211,11 @@ def begin(*exprs): # args eagerly evaluated by Python mlst = eval(allcode, {"e": e, "bodys": bodys, "begin": begin, "monadify": monadify}) return tuple(mlst) -def monadify(value, unpack=True): - """Pack value into a monadic list if it is not already. +# -------------------------------------------------------------------------------- +# This low-level machinery is shared with the macro version, `unpythonic.syntax.forall`. + +def monadify(value: Any, unpack: bool = True) -> "MonadicList": + """Pack ``value`` into a monadic list if it is not already. If ``unpack=True``, an iterable ``value`` is unpacked into the created monadic list instance; if ``False``, the whole iterable is packed as one item. @@ -212,155 +227,14 @@ def monadify(value, unpack=True): return MonadicList.from_iterable(value) except TypeError: pass # fall through - return MonadicList(value) # unit(List, value) - -class MonadicList: # TODO: This if anything is **the** place to use @typed. - """A monadic list.""" - def __init__(self, *elts): - """The unit operator. Lift value(s) into a MonadicList. - - *elts: a or [a] - returns: M a - """ - # Accept the sentinel nil as a special **item** that, when passed to - # the List constructor, produces an empty list. - if len(elts) == 1 and elts[0] is nil: - self.x = () - else: - self.x = elts - - def __rshift__(self, f): - """Monadic bind; standard notation ">>=" in Haskell. - - self: M a - f: a -> M b - returns: M b - - Generally speaking, bind is defined as:: - m >> f = m.fmap(f).join() - - Specifically for `MonadicList`, bind is `flatmap`. - """ - # bind ma f = join (fmap f ma) - return self.fmap(f).join() - # done manually, essentially List.from_iterable(flatmap(lambda elt: f(elt), self.x)) - #return List.from_iterable(result for elt in self.x for result in f(elt)) - - def then(self, f): - """Sequence, a.k.a. "then"; standard notation ">>" in Haskell. - - Like `bind`, but discarding the input `a`. - - self: M a - f : M b - returns: M b - """ - cls = self.__class__ - if not isinstance(f, cls): - raise TypeError(f"Expected a List monad, got {type(f)} with value {repr(f)}") - return self >> (lambda _: f) - - @classmethod - def guard(cls, b): - """Allow a branch of the computation to continue only if `b` is truthy. - - b: bool - returns: M b - - How to use: - - The type of `guard` is (bool -> M b). You'll want to wrap it in a function - that takes in an `a`; then `guard` outputs the `M b`, as expected by monadic - bind, so that you can bind your MonadicList `m` to your guard function. - - The call to `guard` produces a dummy `MonadicList`, which will be non-blank - (with exactly one item) if `b` is truthy, and blank if `b` is falsey. - - Use `.then(...)` just after the `guard` to discard the dummy, and replace with - the actual output you want. The value (that passed the guard) from the original - `MonadicList` is still live in the current scope. - - If you just want to filter, just `MonadicList(x)` it (recall that here the - constructor stands for the `unit` operator). - - When an input doesn't pass the guard, the blank output from `guard` automatically - cancels the rest of that branch of the computation. - """ - if b: - return cls(True) # List with one element; value not intended to be actually used. - return cls() # 0-element List; short-circuit this branch of the computation. - - # make List iterable so that "for result in f(elt)" works (when f outputs a List monad) - def __iter__(self): - return iter(self.x) - def __len__(self): - return len(self.x) - def __getitem__(self, i): - return self.x[i] - - def __eq__(self, other): - if other is self: - return True - if len(self) != len(other): - return False - return other == self.x - - def __add__(self, other): - """Concatenation of MonadicList, for convenience.""" - if not isinstance(other, MonadicList): - raise TypeError(f"Expected a monadic list, got {type(other)} with value {repr(other)}") - cls = self.__class__ - return cls.from_iterable(self.x + other.x) - - def __repr__(self): # pragma: no cover - clsname = self.__class__.__name__ - return f"{clsname}{self.x}" - - @classmethod - def from_iterable(cls, iterable): - """Convenience method: turn an iterable into a MonadicList. - - Eager; the input iterable will be iterated over in its entirety - to produce the list. If it is consumable, it will be consumed. - """ - try: - return cls(*iterable) - except TypeError: # maybe a generator; try forcing it before giving up. - return cls(*tuple(iterable)) - - def copy(self): - """Return a copy of this MonadicList.""" - cls = self.__class__ - return cls(*self.x) - - @classmethod - def lift(cls, f): - """Lift a regular function into a List-producing one. - - f: a -> b - returns: a -> M b - """ - return lambda x: cls(f(x)) - - def fmap(self, f): - """The map operator. - - self: M a - f: a -> b - returns: M b - """ - cls = self.__class__ - return cls.from_iterable(f(elt) for elt in self.x) - - def join(self): - """The join operator. Flatten nested self. - - x: M (M a) - returns: M a - """ - cls = self.__class__ - if not all(isinstance(elt, cls) for elt in self.x): - raise TypeError(f"Expected a nested List monad, got {type(self.x)} with value {self.x}") - # list of lists - concat them - return cls.from_iterable(elt for sublist in self.x for elt in sublist) + return MonadicList(value) # unit: varargs form — singleton list containing value + +# TODO(3.0.0): remove this deprecated alias. Users should import `List` +# directly from `unpythonic.monads`. +MonadicList = List insist = MonadicList.guard # retroactively require expr to be True -def deny(v): +def deny(v: Any) -> Any: """Opposite of `insist`. End a branch of the computation if `v` is truthy.""" return insist(not v) diff --git a/unpythonic/arity.py b/unpythonic/arity.py index ec4e3b9e..7f730429 100644 --- a/unpythonic/arity.py +++ b/unpythonic/arity.py @@ -11,17 +11,21 @@ "resolve_bindings", "resolve_bindings_partial", "tuplify_bindings", "UnknownArity"] -from collections import OrderedDict +from collections.abc import Callable import copy -from inspect import signature, Parameter, ismethod, BoundArguments, _empty +from inspect import signature, Parameter, Signature, ismethod, BoundArguments, _empty import itertools import operator +from typing import Any, Literal + +# TODO: When floor bumps to 3.12, use `type _FuncKind = ...` (PEP 695). +_FuncKind = Literal["function", "instancemethod", "classmethod", "staticmethod"] class UnknownArity(ValueError): """Raised when the arity of a function cannot be inspected.""" # HACK: some built-ins report incorrect arities (0, 0) at least in Python 3.4 -# TODO: re-test on 3.8 and on PyPy3 (3.7), just to be sure. +# TODO: re-test on 3.8, 3.9, 3.10, 3.11, 3.12 and on PyPy3 (3.8 and later), just to be sure. # # Full list of built-ins: # https://docs.python.org/3/library/functions.html @@ -30,7 +34,7 @@ class UnknownArity(ValueError): # # Note this doesn't cover methods such as list.append, or any other parts # of the standard library. -_infty = float("+inf") +_infty: float = float("+inf") _builtin_arities = { # inspectable, but reporting incorrectly bool: (1, 1), # bool(x) bytes: (0, 3), # see help(bytes) @@ -157,7 +161,7 @@ class UnknownArity(ValueError): operator.itruediv: (2, 2), operator.ixor: (2, 2)} -def getfunc(f): # public as of v0.14.3+ +def getfunc(f: Callable[..., Any] | staticmethod | classmethod) -> tuple[Callable[..., Any], _FuncKind]: # public as of v0.14.3+ """Given a function or method, return the underlying function. Return value is a tuple ``(function, kind)``, where ``kind`` is one of @@ -202,13 +206,13 @@ def getfunc(f): # public as of v0.14.3+ raw_function = f return (raw_function, kind) -def arities(f): +def arities(f: Callable[..., Any]) -> tuple[int | float, int | float]: """Inspect f's minimum and maximum positional arity. This uses inspect.signature; note that the signature of builtin functions cannot be inspected. This is worked around to some extent, but e.g. methods of built-in classes (such as ``list``) might not be inspectable - (at least on CPython < 3.7). + (at least on old CPython < 3.7). For bound methods, ``self`` or ``cls`` does not count toward the arity, because these are passed implicitly by Python. Note a `@classmethod` becomes @@ -276,7 +280,7 @@ def arities(f): except (TypeError, ValueError) as e: # likely an uninspectable method of a builtin raise UnknownArity(*e.args) -def required_kwargs(f): +def required_kwargs(f: Callable[..., Any]) -> set[str]: """Return a set containing the names of required name-only arguments of `f`. *Required* means the parameter has no default. @@ -288,7 +292,7 @@ def required_kwargs(f): """ return _kwargs(f, optionals=False) -def optional_kwargs(f): +def optional_kwargs(f: Callable[..., Any]) -> set[str]: """Return a set containing the names of optional name-only arguments of `f`. *Optional* means the parameter has a default. @@ -300,13 +304,13 @@ def optional_kwargs(f): """ return _kwargs(f, optionals=True) -def _kwargs(f, optionals=True): +def _kwargs(f: Callable[..., Any], optionals: bool = True) -> set[str]: f, _ = getfunc(f) # Integration with the multiple-dispatch system (multimethods). from .dispatch import isgeneric, list_methods # circular import if isgeneric(f): - thekwargs = {} + thekwargs: set[str] = set() for (thecallable, type_signature) in list_methods(f): thekwargs.update(_kwargs(thecallable, optionals=optionals)) return thekwargs @@ -321,7 +325,7 @@ def _kwargs(f, optionals=True): except (TypeError, ValueError) as e: raise UnknownArity(*e.args) -def kwargs(f): +def kwargs(f: Callable[..., Any]) -> tuple[set[str], set[str]]: """Like Racket's (procedure-keywords). Return two sets: the first contains the `required_kwargs` of ``f``, @@ -331,7 +335,7 @@ def kwargs(f): """ return (required_kwargs(f), optional_kwargs(f)) -def arity_includes(f, n): +def arity_includes(f: Callable[..., Any], n: int) -> bool: """Check whether f's positional arity includes n. I.e., return whether ``f()`` can be called with ``n`` positional arguments. @@ -339,23 +343,23 @@ def arity_includes(f, n): lower, upper = arities(f) return lower <= n <= upper -def resolve_bindings_partial(f, *args, **kwargs): +def resolve_bindings_partial(f: Callable[..., Any], *args: Any, **kwargs: Any) -> BoundArguments: """Like `resolve_bindings`, but use `inspect.Signature.bind_partial`. That is, it is acceptable for some parameters of `f` not to have a binding. """ return _resolve_bindings(f, args, kwargs, _partial=True) -def resolve_bindings(f, *args, **kwargs): +def resolve_bindings(f: Callable[..., Any], *args: Any, **kwargs: Any) -> BoundArguments: """Resolve parameter bindings established by `f` when called with the given args and kwargs. This is an inspection tool, which does not actually call `f`. This is useful for memoizers and other similar decorators that need a canonical representation of `f`'s parameter bindings. - **NOTE**: As of v0.15.0, this is a thin wrapper on top of `inspect.Signature.bind`, - which was added in Python 3.5. In `unpythonic` 0.14.2 and 0.14.3, we used to have - our own implementation of the parameter binding algorithm (that ran also on Python 3.4), - but it is no longer needed, since now we support only Python 3.6 and later. + **NOTE**: This is a thin wrapper on top of `inspect.Signature.bind`, which was added in Python 3.5. + In `unpythonic` 0.14.2 and 0.14.3, we used to have our own implementation of the parameter binding + algorithm (that ran also on Python 3.4), but it is no longer needed, since as of v0.15.3, + we support only Python 3.8 and later. The only thing we do beside call `inspect.Signature.bind` is that we apply default values (from the definition of `f`) automatically. @@ -411,7 +415,7 @@ def f(a): """ return _resolve_bindings(f, args, kwargs, _partial=False) -def _resolve_bindings(f, args, kwargs, *, _partial): +def _resolve_bindings(f: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any], *, _partial: bool) -> BoundArguments: thesignature = signature(f) if _partial: bound_arguments = thesignature.bind_partial(*args, **kwargs) @@ -420,7 +424,7 @@ def _resolve_bindings(f, args, kwargs, *, _partial): bound_arguments.apply_defaults() return bound_arguments -def tuplify_bindings(bound_arguments): +def tuplify_bindings(bound_arguments: BoundArguments) -> tuple[tuple[str, Any], ...]: """Convert the return value of `resolve_bindings` into a hashable form. This is useful for memoizers and similar use cases, which need to use a @@ -429,15 +433,15 @@ def tuplify_bindings(bound_arguments): `bound_arguments` is an `inspect.BoundArguments` object. In our return value, `bound_arguments.arguments` itself, as well as the value of - the `**kwargs` parameter contained in it, if any, are converted from `OrderedDict` - to `tuple` using `tuple(od.items())`. + the `**kwargs` parameter contained in it, if any, are converted from `dict` + to `tuple` using `tuple(d.items())`. The result is hashable, if all the passed arguments are. See `resolve_bindings` for an example. """ - def tuplify(ordereddict): - return tuple(ordereddict.items()) + def tuplify(d: dict[str, Any]) -> tuple[tuple[str, Any], ...]: + return tuple(d.items()) # Tuplify the **kwargs dict. # @@ -460,28 +464,32 @@ def tuplify(ordereddict): return tuplify(thearguments) -# This is `inspect.Signature.bind` from Python 3.8.5, modified for our purposes so we can determine +# This is `inspect.Signature._bind` from Python 3.14, modified for our purposes so we can determine # unbound *and extra* arguments (both positional and by-name) without raising a `TypeError`. # We need this for kwargs support in `curry`, because we want to pass through unmatched args and kwargs # (which otherwise trigger a `TypeError`). # # This is only for `curry`; all other code uses the standard implementation. # -# Used under the PSF license. Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; All Rights Reserved -def _bind(thesignature, args, kwargs, *, partial): +# Lines where we diverge from stdlib are marked with `[unpythonic]`. +# Commented-out code shows the original stdlib behavior at each such point. +# +# Used under the PSF license. Copyright (c) 2001-2025 Python Software Foundation; All Rights Reserved +def _bind(thesignature: Signature, args: tuple[Any, ...], kwargs: dict[str, Any], *, partial: bool) -> tuple[BoundArguments, tuple[Parameter, ...], tuple[tuple[Any, ...], dict[str, Any]]]: """Private method. Don't use directly.""" - arguments = OrderedDict() + arguments = {} parameters = iter(thesignature.parameters.values()) parameters_ex = () arg_vals = iter(args) - # These are added for `unpythonic`. - unbound_parameters = [] - extra_args = [] - extra_kwargs = OrderedDict() + pos_only_param_in_kwargs = [] + + # [unpythonic] These collect what stdlib would reject with TypeError. + unbound_parameters: list[Parameter] = [] + extra_args: list[Any] = [] + extra_kwargs: dict[str, Any] = {} kwargs = copy.copy(kwargs) # the caller might need the original later while True: @@ -504,10 +512,13 @@ def _bind(thesignature, args, kwargs, *, partial): break elif param.name in kwargs: if param.kind == Parameter.POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + if param.default is _empty: + msg = f'missing a required positional-only argument: {param.name!r}' + raise TypeError(msg) + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == Parameter.VAR_KEYWORD or @@ -524,8 +535,13 @@ def _bind(thesignature, args, kwargs, *, partial): parameters_ex = (param,) break else: - # msg = 'missing a required argument: {arg!r}' - # msg = msg.format(arg=param.name) + # [unpythonic] Collect instead of raising: + # if param.kind == Parameter.KEYWORD_ONLY: + # argtype = ' keyword-only' + # else: + # argtype = '' + # msg = 'missing a required{argtype} argument: {arg!r}' + # msg = msg.format(arg=param.name, argtype=argtype) # raise TypeError(msg) from None unbound_parameters.append(param) else: @@ -533,12 +549,14 @@ def _bind(thesignature, args, kwargs, *, partial): try: param = next(parameters) except StopIteration: + # [unpythonic] Collect instead of raising: # raise TypeError('too many positional arguments') from None extra_args.append(arg_val) else: if param.kind in (Parameter.VAR_KEYWORD, Parameter.KEYWORD_ONLY): # Looks like we have no parameter for this positional # argument + # [unpythonic] Collect instead of raising: # raise TypeError( # 'too many positional arguments') from None extra_args.append(arg_val) @@ -584,26 +602,30 @@ def _bind(thesignature, args, kwargs, *, partial): # arguments. if (not partial and param.kind != Parameter.VAR_POSITIONAL and param.default is _empty): + # [unpythonic] Collect instead of raising: # raise TypeError('missing a required argument: {arg!r}'. # format(arg=param_name)) from None unbound_parameters.append(param) else: - if param.kind == Parameter.POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: + # [unpythonic] Collect instead of raising: # raise TypeError( # 'got an unexpected keyword argument {arg!r}'.format( # arg=next(iter(kwargs)))) diff --git a/unpythonic/assignonce.py b/unpythonic/assignonce.py index 0267df9e..0488108f 100644 --- a/unpythonic/assignonce.py +++ b/unpythonic/assignonce.py @@ -3,11 +3,23 @@ __all__ = ["assignonce"] +from typing import Any + from .env import env as _envcls class assignonce(_envcls): """Environment with assign-once names. + **Reach for this only when you want the assign-once discipline itself.** + The macro layer supports plain ``env`` far more thoroughly, so choosing + ``assignonce`` costs you that support. + + Note ``env.finalize()`` is *not* a substitute, and the two guarantees are + orthogonal: ``finalize()`` freezes the *set of names* (no additions, no + deletions) while leaving existing bindings rebindable, whereas this class + fixes each binding's *first value* while leaving the set of names open. + Pick by which of the two you actually need. + In Scheme terms, this makes ``define`` and ``set!`` look different:: with assignonce() as e: @@ -23,13 +35,24 @@ class assignonce(_envcls): e.set("foo", "tavern") e.foo = "quux" # AttributeError """ - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if name in self._reserved_names or name not in self: return super().__setattr__(name, value) else: raise AttributeError(f"name {repr(name)} is already defined") - def set(self, name, value): + def __delattr__(self, name: str) -> None: + """Forbid `del e.foo` on a defined name. + + Otherwise the assign-once contract could be bypassed via + ``del e.foo; e.foo = new_value``. Use ``.set(name, value)`` for + explicit rebinding instead. + """ + if name not in self._reserved_names and name in self: + raise AttributeError(f"name {repr(name)} is defined; deletion not allowed in an assign-once environment (use .set() to rebind)") + super().__delattr__(name) + + def set(self, name: str, value: Any) -> Any: """Rebind an existing name to a new value.""" env = self._env if name not in env: diff --git a/unpythonic/collections.py b/unpythonic/collections.py index 9d544168..8a02bf98 100644 --- a/unpythonic/collections.py +++ b/unpythonic/collections.py @@ -11,13 +11,15 @@ from itertools import repeat from abc import abstractmethod from collections import abc -from collections.abc import (Container, Iterable, Hashable, Sized, - Sequence, Mapping, Set, +from collections.abc import (Callable, Container, Iterable, Hashable, + ItemsView, Iterator, KeysView, Sized, + Sequence, Mapping, Set, ValuesView, MutableSequence, MutableMapping, MutableSet, MappingView) from inspect import isclass from operator import lt, le, ge, gt import threading +from typing import Any # Some of these are used only to detect (and perhaps mogrify) our own cat food in `mogrify`. # @@ -27,22 +29,25 @@ from .env import env from .dynassign import _Dyn from .funutil import Values +from .it import drop from .llist import cons, Nil -from .misc import getattrrec +from .misc import getattrrec, CountingIterator -def get_abcs(cls): +def get_abcs(cls: type) -> set[type]: """Return a set of the collections.abc superclasses of cls (virtuals too).""" return {v for k, v in vars(abc).items() if isclass(v) and issubclass(cls, v)} # TODO: allow multiple input container args in mogrify, like map does (also support longest, fillvalue) # OTOH, that's assuming an ordered iterable... so maybe not for general containers? # TODO: move to unpythonic.it? This is a spork... -def mogrify(func, container): +def mogrify(func: Callable, container: Any) -> Any: """In-place recursive map for mutable containers. Recurse on container, apply func to each atom. Containers can be nested, with an arbitrary combination of types. + If `container` is actually an atom (not a container), just apply func to it. + Containers are detected by checking for instances of ``collections.abc`` superclasses (also virtuals are ok). @@ -80,7 +85,7 @@ def mogrify(func, container): Any **immutable** container encountered is transformed into a new copy, just like in ``map``. """ - def doit(x): + def doit(x: Any) -> Any: if isinstance(x, Values): new_rets = doit(x.rets) new_kwrets = doit(x.kwrets) @@ -196,19 +201,19 @@ def f(b): for the particular situation. This class just makes the programmer's intent more explicit. """ - def __init__(self, x=None): + def __init__(self, x: Any = None) -> None: self.x = x - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover return f"box({repr(self.x)})" - def __contains__(self, x): + def __contains__(self, x: Any) -> bool: return self.x == x - def __iter__(self): + def __iter__(self) -> Iterator: return (x for x in (self.x,)) - def __len__(self): + def __len__(self) -> int: return 1 - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return other == self.x - def set(self, x): + def set(self, x: Any) -> Any: """Store a new value in the box, replacing the old one. As a convenience, returns the new value. @@ -218,17 +223,17 @@ def set(self, x): """ self.x = x return x - def __lshift__(self, x): + def __lshift__(self, x: Any) -> Any: """Syntactic sugar for storing a new value. `b << 42` is the same as `b.set(42)`. - (Note that for `env`, the `<<` operator returns the *environment* so it - can be chained to make several assignments, but that doesn't make sense - for a `box`, so we just return the new value.) + Note that for `env`, the `<<` operator returns the *environment* so it + can be chained to make several assignments; but that doesn't make sense + for a `box`, so we just return the new value. """ return self.set(x) - def get(self): + def get(self) -> Any: """Return the value currently in the box. The syntactic sugar for `b.get()` is `unbox(b)`. @@ -245,34 +250,34 @@ class ThreadLocalBox(box): the initial contents of the box in all threads. (Note what this implies if that `x` happens to be mutable.) """ - def __init__(self, x=None): + def __init__(self, x: Any = None) -> None: self.storage = threading.local() self._default = x - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover """**WARNING**: the repr shows only the content seen by the current thread.""" return f"ThreadLocalBox({repr(self.get())})" - def __contains__(self, x): + def __contains__(self, x: Any) -> bool: return self.get() == x - def __iter__(self): + def __iter__(self) -> Iterator: return (x for x in (self.get(),)) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return other == self.get() - def set(self, x): + def set(self, x: Any) -> Any: self.storage.x = x return x - def __lshift__(self, x): + def __lshift__(self, x: Any) -> Any: return self.set(x) - def get(self): + def get(self) -> Any: if hasattr(self.storage, "x"): # default overridden in this thread? return self.storage.x return self._default - def setdefault(self, x): + def setdefault(self, x: Any) -> None: """Change the default object.""" self._default = x - def getdefault(self): + def getdefault(self) -> Any: """Get the default object.""" return self._default - def clear(self): + def clear(self) -> None: """Remove the value in the box in this thread, thus unshadowing the default.""" if hasattr(self.storage, "x"): del self.storage.x @@ -289,27 +294,30 @@ class Some: In a way, `Some` is a relative of `box`: it's an **immutable** single-item container. It supports `.get` and `unbox`, but no `<<` or `.set`. + + It is also the logical opposite of a bare `None`, also syntactically: + `Some(...) is not None`. """ - def __init__(self, x=None): + def __init__(self, x: Any = None) -> None: self.x = x - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover return f"Some({repr(self.x)})" - def __contains__(self, x): + def __contains__(self, x: Any) -> bool: return self.x == x - def __iter__(self): + def __iter__(self) -> Iterator: return (x for x in (self.x,)) - def __len__(self): + def __len__(self) -> int: return 1 - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return other == self.x - def get(self): + def get(self) -> Any: """Return the value currently in the `Some`. The syntactic sugar for `b.get()` is `unbox(b)`. """ return self.x -def unbox(b): +def unbox(b: "box | Some") -> Any: """Return the value from inside the box b. Syntactic sugar for `b.get()`. @@ -364,12 +372,12 @@ class Shim: Here `Shim(box, fallback)` is foldr's `op(elt, acc)`. """ - def __init__(self, thebox, fallback=None): + def __init__(self, thebox: box, fallback: "box | Any | None" = None) -> None: if not isinstance(thebox, box): raise TypeError(f"Expected box, got {type(thebox)} with value {repr(thebox)}") self._shim_box = thebox self._shim_fallback = fallback - def __getattr__(self, k): + def __getattr__(self, k: str) -> Any: thing = unbox(self._shim_box) fallback = self._shim_fallback if not fallback or hasattr(thing, k): @@ -377,7 +385,7 @@ def __getattr__(self, k): # fallback and not hasattr(thing, k) otherthing = unbox(fallback) if isinstance(fallback, box) else fallback return getattr(otherthing, k) - def __setattr__(self, k, v): + def __setattr__(self, k: str, v: Any) -> None: if k in ("_shim_box", "_shim_fallback"): return super().__setattr__(k, v) thing = unbox(self._shim_box) @@ -406,6 +414,12 @@ class frozendict: As usual, this does **not** protect from mutating the values themselves, if they happen to be mutable objects (such as containers). + "Frozen" refers to the **mapping**: there is no ``__setitem__``, + ``__delitem__``, ``update``, etc. As with any plain Python class, + instance **attributes** (``d.foo = ...``) are not frozen — only + contents accessed via the mapping protocol are. The use case is + immutable data, not a sealed object. + Any ``m`` used in the initialization of a ``frozendict`` is shallow-copied to make sure the bindings in the ``frozendict`` do not change even if the original is later mutated. @@ -423,7 +437,7 @@ class frozendict: """ # Make the empty frozendict() a singleton, but allow invoking the constructor # multiple times, always returning the same instance. - def __new__(cls, *ms, **bindings): + def __new__(cls, *ms: Mapping, **bindings: Any) -> "frozendict": if not ms and not bindings: global _the_empty_frozendict if _the_empty_frozendict is None: @@ -435,7 +449,7 @@ def __new__(cls, *ms, **bindings): # https://github.com/Technologicat/unpythonic/issues/55 # https://docs.python.org/3/library/pickle.html#object.__getnewargs_ex__ # https://docs.python.org/3/library/pickle.html#object.__getnewargs__ - def __getnewargs__(self): + def __getnewargs__(self) -> tuple: if self is not _the_empty_frozendict: # In our case it doesn't matter what the value is, as long as there is one, # because `__new__` uses the *presence* of any args to know the instance is @@ -444,7 +458,7 @@ def __getnewargs__(self): return ("nonempty",) return () - def __init__(self, *ms, **bindings): + def __init__(self, *ms: Mapping, **bindings: Any) -> None: """Arguments: ms: mappings; optional @@ -468,10 +482,10 @@ def __init__(self, *ms, **bindings): self._data.update(bindings) @wraps(dict.__repr__) - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover return f"frozendict({self._data.__repr__()})" - def __hash__(self): + def __hash__(self) -> int: return hash(frozenset(self.items())) # Provide any read-access parts of the dict API. @@ -483,31 +497,31 @@ def __hash__(self): # https://docs.python.org/3/library/collections.abc.html # https://docs.python.org/3/reference/datamodel.html#emulating-container-types @wraps(dict.__getitem__) - def __getitem__(self, k): + def __getitem__(self, k: Hashable) -> Any: return self._data.__getitem__(k) @wraps(dict.__iter__) - def __iter__(self): + def __iter__(self) -> Iterator[Hashable]: return self._data.__iter__() @wraps(dict.__len__) - def __len__(self): + def __len__(self) -> int: return self._data.__len__() @wraps(dict.__contains__) - def __contains__(self, k): + def __contains__(self, k: Hashable) -> bool: return self._data.__contains__(k) @wraps(dict.keys) - def keys(self): + def keys(self) -> KeysView: return self._data.keys() @wraps(dict.items) - def items(self): + def items(self) -> ItemsView: return self._data.items() @wraps(dict.values) - def values(self): + def values(self) -> ValuesView: return self._data.values() @wraps(dict.get) - def get(self, k, *d): + def get(self, k: Hashable, *d: Any) -> Any: return self._data.get(k, *d) @wraps(dict.__eq__) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return other == self._data # Register virtual ABCs for our collections (like the builtins have). @@ -544,35 +558,35 @@ class MutableSequenceView(SequenceView): classes that implement ``MutableSequenceView`` must account for this. """ @abstractmethod - def __setitem__(self, k, v): + def __setitem__(self, k: int | slice, v: Any) -> None: pass # pragma: no cover @abstractmethod - def reverse(self): + def reverse(self) -> None: pass # pragma: no cover # ----------------------------------------------------------------------------- -class _StrReprEqMixin: - def _lowlevel_repr(self): # pragma: no cover +class _SequenceStrReprEqMixin: + """Mixin providing __str__, __repr__, and __eq__ for sequence-like views.""" + def _lowlevel_repr(self) -> Sequence: # pragma: no cover cls = type(getattrrec(self, "seq")) # de-onionize ctor = tuple if hasattr(cls, "_make") else cls # slice of namedtuple -> tuple return ctor(x for x in self) - def __str__(self): # pragma: no cover + def __str__(self) -> str: # pragma: no cover return str(self._lowlevel_repr()) - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover return f"{self.__class__.__name__}({self._lowlevel_repr()!r})" - def __eq__(self, other): + def __eq__(self, other: Sequence) -> bool: if other is self: return True + if not isinstance(other, Sequence): + raise TypeError(f"Cannot compare {type(self).__name__} with {type(other).__name__}") if len(self) != len(other): return False - for v1, v2 in zip(self, other): - if v1 != v2: - return False - return True + return all(v1 == v2 for v1, v2 in zip(self, other)) -class roview(SequenceView, _StrReprEqMixin): +class roview(SequenceView, _SequenceStrReprEqMixin): """Read-only live view into a sequence. Supports slicing (also recursively, i.e. can be sliced again). @@ -610,7 +624,7 @@ class roview(SequenceView, _StrReprEqMixin): http://stackoverflow.com/q/3485475/can-i-create-a-view-on-a-python-list """ - def __init__(self, sequence, s=None): + def __init__(self, sequence: Sequence, s: slice | None = None) -> None: """If s is None, view the whole input. If s is a slice, view that slice. The slice can also be specified later by subscripting with a slice @@ -619,35 +633,36 @@ def __init__(self, sequence, s=None): """ if s is None: s = slice(None, None, None) - self.seq = sequence - self.slice = s - self._seql = None + self.seq: Sequence = sequence + self.slice: slice = s + self._seql: int | None = None + self._cache: tuple[Sequence, range] | None = None - def __iter__(self): + def __iter__(self) -> Iterator: data, r = self._update_cache() - def view_iterator(): + def view_iterator() -> Iterator: for j in r: yield data[j] return view_iterator() - def __len__(self): + def __len__(self) -> int: _, r = self._update_cache() return len(r) - def _update_cache(self): + def _update_cache(self) -> tuple[Sequence, range]: seql = len(self.seq) if seql != self._seql: self._seql = seql self._cache = self._range() return self._cache - def _range(self): # return underlying sequence, current range of all elements of self in it - def buildr(seq): + def _range(self) -> tuple[Sequence, range]: # return underlying sequence, current range of all elements of self in it + def buildr(seq: Sequence) -> tuple[Sequence, range]: if not isinstance(seq, (roview, view)): return seq, range(len(seq)) data, r = buildr(seq.seq) return data, r[seq.slice] return buildr(self) - def __getitem__(self, k): + def __getitem__(self, k: int | slice) -> Any: if isinstance(k, slice): if k == slice(None, None, None): # v[:] return self @@ -702,7 +717,7 @@ class view(roview, MutableSequenceView): assert v == [1, 2, 3, 4, 5] assert lst == [42, 0, 1, 2, 3, 4, 5] """ - def __init__(self, sequence, s=None): + def __init__(self, sequence: MutableSequence, s: slice | None = None) -> None: # some fandango because MutableSequenceView is not a MutableSequence for technical reasons. if isinstance(sequence, SequenceView): if not isinstance(sequence, MutableSequenceView): @@ -710,7 +725,7 @@ def __init__(self, sequence, s=None): elif isinstance(sequence, Sequence) and not isinstance(sequence, MutableSequence): raise TypeError("cannot create writable view into a read-only sequence") super().__init__(sequence, s) - def __setitem__(self, k, v): + def __setitem__(self, k: int | slice, v: Any) -> None: data, r = self._update_cache() if isinstance(k, slice): # TODO: would be nicer if we could convert a range into a slice, then just data[rk] = v. @@ -728,13 +743,13 @@ def __setitem__(self, k, v): if k >= n or k < -n: raise IndexError("view assigment index out of range") data[r[k]] = v - def reverse(self): + def reverse(self) -> None: self[::-1] = [x for x in self] # ----------------------------------------------------------------------------- # Inherit from Sequence, because we want the default implementations of e.g. count, index to be found in the MRO. -class ShadowedSequence(Sequence, _StrReprEqMixin): +class ShadowedSequence(Sequence, _SequenceStrReprEqMixin): """Sequence with some elements shadowed by those from another sequence. Or in other words, a functionally updated view of a sequence. Or somewhat @@ -743,35 +758,53 @@ class ShadowedSequence(Sequence, _StrReprEqMixin): Essentially, ``out[k] = v[index_in_slice(k, ix)] if in_slice(k, ix) else seq[k]``, but doesn't actually allocate ``out``. - ``ix`` may be integer (if ``v`` represents one item only) or slice (if ``v`` - is intended as a sequence). The default ``None`` means ``out[k] = seq[k]`` + ``ix`` may be integer (if ``v`` represents one item only) or ``slice`` (if ``v`` + is intended as a sequence). The default ``ix=None`` means ``out[k] = seq[k]`` with no shadower. + + If ``ix`` is a ``slice``, then: + + - If the replacement specification requires reading ``v`` backwards, + and/or if you plan to iterate over the ``ShadowedSequence`` more + than once, then ``v`` must implement ``collections.abc.Sequence``, + i.e. it must have ``__len__`` and ``__getitem__`` methods. + + - If the replacement specification only needs reading ``v`` forwards, + **AND** if you plan to read the ``ShadowedSequence`` only once (e.g. + as part of a `fupdate` or `fup` operation), then it is sufficient + for ``v`` to implement only ``collections.abc.Iterator``, i.e. the + ``__iter__`` and ``__next__`` methods only. """ - def __init__(self, seq, ix=None, v=None): + def __init__(self, seq: Sequence, ix: int | slice | None = None, v: Any = None) -> None: if ix is not None and not isinstance(ix, (slice, int)): raise TypeError(f"ix: expected slice or int, got {type(ix)} with value {ix}") - self.seq = seq - self.ix = ix - self.v = v + if not isinstance(seq, Sequence): + raise TypeError(f"seq: expected a sequence, got {type(seq)} with value {seq}") + if isinstance(ix, slice) and not isinstance(v, (Sequence, Iterable)): + raise TypeError(f"v: when ix is a slice, v must be a sequence or an iterable; got {type(v)} with value {v}") + self.seq: Sequence = seq + self.ix: int | slice | None = ix + self.v: Any = v + self._v_it: CountingIterator | None = None # Provide __iter__ (even though implemented using len() and __getitem__()) # so that our __getitem__ can raise IndexError when needed, without it # getting caught by the genexpr in unpythonic.fup.fupdate when it builds # the output sequence. - def __iter__(self): + def __iter__(self) -> Iterator: if self.ix is None: # allow no-op ShadowedSequences since the repr suggests one could do that return iter(self.seq) n = len(self) getone = self._getone - def ShadowedSequenceIterator(): + def ShadowedSequenceIterator() -> Iterator: for j in range(n): yield getone(j) return ShadowedSequenceIterator() - def __len__(self): + def __len__(self) -> int: return len(self.seq) - def __getitem__(self, k): + def __getitem__(self, k: int | slice) -> Any: if self.ix is None: # allow no-op ShadowedSequences since the repr suggests one could do that return self.seq[k] n = len(self) @@ -786,7 +819,7 @@ def __getitem__(self, k): raise IndexError("ShadowedSequence index out of range") return self._getone(k) - def _getone(self, k): + def _getone(self, k: int) -> Any: ix = self.ix n = len(self) if in_slice(k, ix, n): @@ -794,12 +827,36 @@ def _getone(self, k): return self.v # just one item # we already know k is in ix, so skip validation for speed. i = _index_in_slice(k, ix, n, _validate=False) - if i >= len(self.v): - raise IndexError(f"Replacement sequence too short; attempted to access index {i} with len {len(self.v)} (items: {self.v})") - return self.v[i] + if isinstance(self.v, Sequence): + if i >= len(self.v): + raise IndexError(f"Replacement sequence too short; attempted to access index {i} with len {len(self.v)} (items: {self.v})") + return self.v[i] + elif isinstance(self.v, Iterable): + if not self._v_it: + self._v_it = CountingIterator(self.v) + if i < self._v_it.count: + # Special case for `unpythonic.gmemo._MemoizedGenerator`, + # to support reverse-walking the start of a memoized infinite replacement + # that was created using `imemoize`/`fimemoize`/`gmemoize`. + # It has the `__len__` and `__getitem__` methods, but does + # **not** support the full `collections.abc.Sequence` API. + # At this point, the memo contains all the items accessed or dropped so far. + bare_it = self._v_it._it + if all(hasattr(bare_it, name) for name in ("__len__", "__getitem__")): + assert i < len(bare_it) # because we counted them! + return bare_it[i] + raise IndexError(f"Trying to read an already consumed item of a non-sequence iterable; attempted to access index {i} with {self._v_it.count} items already consumed.") + n_skip = i - self._v_it.count + assert n_skip >= 0 + if n_skip: + # NOTE: If the iterable is memoized, the items we drop here will enter the memo. + self._v_it = drop(n_skip, self._v_it) + return next(self._v_it) + else: + assert False return self.seq[k] # not in slice -def in_slice(i, s, length=None): +def in_slice(i: int, s: int | slice, length: int | None = None) -> bool: """Return whether the int i is in the slice s. For convenience, ``s`` may be int instead of slice; then return @@ -810,7 +867,7 @@ def in_slice(i, s, length=None): (if ``s.start`` or ``s.stop`` is ``None``). If ``length is None``, negative or missing ``s.start`` or ``s.stop`` may raise - ValueError. (A negative ``s.step`` by itself does not need ``l``.) + ValueError. (A negative ``s.step`` by itself does not need ``length``.) """ if not isinstance(s, (slice, int)): raise TypeError(f"s must be slice or int, got {type(s)} with value {s}") @@ -828,7 +885,7 @@ def in_slice(i, s, length=None): on_grid = (i - start) % step == 0 return at_or_after_start and on_grid and before_stop -def index_in_slice(i, s, length=None): +def index_in_slice(i: int, s: int | slice, length: int | None = None) -> int | None: """Return the index of the int i in the slice s, or None if i is not in s. (I.e. how-manyth item of the slice the index i is.) @@ -839,24 +896,24 @@ def index_in_slice(i, s, length=None): # efficiency: allow skipping the validation check for call sites # that have already checked with in_slice(). -def _index_in_slice(i, s, length=None, _validate=True): +def _index_in_slice(i: int, s: int | slice, length: int | None = None, _validate: bool = True) -> int | None: if (not _validate) or in_slice(i, s, length): wrap = _make_negidx_converter(length) start, _, step = _canonize_slice(s, length, wrap) return (wrap(i) - start) // step -def _make_negidx_converter(length): +def _make_negidx_converter(length: int | None) -> Callable[[int | None], int | None]: if length is not None: if not isinstance(length, int): raise TypeError(f"length must be int, got {type(length)} with value {length}") if length <= 0: raise ValueError(f"length must be an int >= 1, got {length}") - def apply_conversion(k): + def apply_conversion(k: int) -> int: return k % length else: - def apply_conversion(k): + def apply_conversion(k: int) -> int: raise ValueError("Need length to interpret negative indices") - def convert(k): + def convert(k: int | None) -> int | None: if k is not None: if not isinstance(k, int): # This is not triggered in the current code because the outer @@ -870,9 +927,10 @@ def convert(k): if length is not None and not -length <= k <= length: raise IndexError(f"Should have -length <= k <= length, but length = {length}, and k = {k}") return apply_conversion(k) if k < 0 else k + return None # passthrough for missing slice components return convert -def _canonize_slice(s, length=None, wrap=None): # convert negatives, inject defaults. +def _canonize_slice(s: slice, length: int | None = None, wrap: Callable | None = None) -> tuple[int, int, int]: # convert negatives, inject defaults. if not isinstance(s, slice): # Not triggered in the current code, because this is an internal function # and `in_slice` already checks; but let's be careful in case this is later diff --git a/unpythonic/conditions.py b/unpythonic/conditions.py index 56907434..46d081a8 100644 --- a/unpythonic/conditions.py +++ b/unpythonic/conditions.py @@ -57,14 +57,16 @@ "available_restarts", "available_handlers", "restarts", "with_restarts", "handlers", - "ControlError", + "ControlError", "ConditionProtocol", "resignal_in", "resignal"] import threading from collections import deque, namedtuple +from collections.abc import Callable, Generator from functools import partial from operator import itemgetter import contextlib +from typing import Any, NoReturn, Protocol, TypeVar import warnings from .collections import box, unbox @@ -72,8 +74,21 @@ from .excutil import equip_with_traceback from .misc import namelambda, safeissubclass +# TODO: When floor bumps to 3.12, use inline `[T]` syntax on `with_restarts` +# and `resignal_in` (PEP 695), and `type _ExcMapping = ...` for the mapping +# type repeated in `_resignal_handler`, `resignal_in`, and `resignal`. +T = TypeVar('T') + +class ConditionProtocol(Protocol): + """The call signature shared by error-handling protocols (`signal`, `error`, `cerror`, `warn`). + + A custom protocol is any callable satisfying this interface. + """ + def __call__(self, condition: BaseException | type[BaseException], + *, cause: BaseException | type[BaseException] | None = ...) -> Any: ... + _stacks = threading.local() -def _ensure_stacks(): # per-thread init +def _ensure_stacks() -> None: # per-thread init for x in ("restarts", "handlers"): if not hasattr(_stacks, x): setattr(_stacks, x, deque()) @@ -88,7 +103,7 @@ class ControlError(Exception): when no handler handles the signal. """ -def signal(condition, *, cause=None, protocol=None): +def signal(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None = None, protocol: ConditionProtocol | None = None) -> BaseException: """Signal a condition. Signaling a condition works similarly to raising an exception (pass an @@ -123,10 +138,10 @@ def signal(condition, *, cause=None, protocol=None): The return value is the input `condition`, canonized to an instance (even if originally, an exception *type* was passed to `signal`), with its `__cause__` and `__protocol__` attributes filled in, - and with a traceback attached (on Python 3.7+). For example, the - `error` protocol uses the return value to chain the unhandled signal - properly into a `ControlError` exception; as a result, the error report - looks like a standard exception chain, with nice-looking tracebacks. + and with a traceback attached. For example, the `error` protocol + uses the return value to chain the unhandled signal properly into + a `ControlError` exception; as a result, the error report looks + like a standard exception chain, with nice-looking tracebacks. If you want to error out on unhandled conditions, see `error`, which is otherwise the same as `signal`, except it raises if `signal` would have @@ -162,9 +177,8 @@ def signal(condition, *, cause=None, protocol=None): You can signal any exception or warning object, both builtins and any custom ones. - On Python 3.7 and later, the exception object representing the signaled - condition is equipped with a traceback, just like a raised exception. - On Python 3.6 this is not possible, so the traceback is `None`. + The exception object representing the signaled condition is equipped + with a traceback, just like a raised exception. """ # Since the handler is called normally, we don't unwind the call stack, # remaining inside the `signal()` call in the low-level code. @@ -182,7 +196,7 @@ def signal(condition, *, cause=None, protocol=None): protocol = protocol or signal condition = _prepare_signal_instance(condition, cause=cause, protocol=protocol, stacklevel=3) - def accepts_arg(f): + def accepts_arg(f: Callable[..., Any]) -> bool: try: if arity_includes(f, 1): return True @@ -200,7 +214,7 @@ def accepts_arg(f): # `error()` uses this return value; this allows us to provide a unified format for tracebacks. return condition -def _prepare_signal_instance(condition, *, cause, protocol, stacklevel): +def _prepare_signal_instance(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None, protocol: ConditionProtocol, stacklevel: int) -> BaseException: """Canonize a condition, and populate its technical data.""" # Consistency with behavior of exceptions in Python: # Even if a class is raised, as in `raise StopIteration`, the `raise` statement @@ -208,7 +222,7 @@ def _prepare_signal_instance(condition, *, cause, protocol, stacklevel): # special handling for the "class raised" case. # https://docs.python.org/3/reference/simple_stmts.html#the-raise-statement # https://stackoverflow.com/questions/19768515/is-there-a-difference-between-raising-exception-class-and-exception-instance/19768732 - def canonize(exc, err_reason): + def canonize(exc: BaseException | type[BaseException] | None, err_reason: str) -> BaseException | None: if exc is None: return None if isinstance(exc, BaseException): # "signal(SomeError())" @@ -218,23 +232,19 @@ def canonize(exc, err_reason): return exc() # instantiate with no args, like `raise` does except TypeError: # "issubclass() arg 1 must be a class" pass - error(ControlError(f"Only exceptions and subclasses of Exception can {err_reason}; got {type(condition)} with value {repr(condition)}.")) + error(ControlError(f"Only instances (derived too) and subclasses of BaseException can {err_reason}; got {type(condition)} with value {repr(condition)}.")) condition = canonize(condition, "be signaled") cause = canonize(cause, "act as the cause of another signal") condition.__cause__ = cause condition.__protocol__ = protocol - # Embed a stack trace in the signal, like Python does for raised exceptions. - # This only works on Python 3.7 and later, because we need to create a traceback object in pure Python code. - try: - condition = equip_with_traceback(condition, stacklevel=stacklevel) - except NotImplementedError: # pragma: no cover - pass # well, we tried! + # Embed a stack trace in the signal, like Python does for raised exceptions. This API was added in Python 3.7. + condition = equip_with_traceback(condition, stacklevel=stacklevel) return condition -def invoke(name_or_restart, *args, **kwargs): +def invoke(name_or_restart: "str | BoundRestart", *args: Any, **kwargs: Any) -> NoReturn: """Invoke a restart currently in scope. Known as `INVOKE-RESTART` in Common Lisp. `name_or_restart` can be the name of a restart, or a restart object returned @@ -334,7 +344,7 @@ def invoke(name_or_restart, *args, **kwargs): set of constant args/kwargs. """ -def invoker(restart_name, *args, **kwargs): +def invoker(restart_name: str, *args: Any, **kwargs: Any) -> Callable[..., NoReturn]: """Create a handler that just invokes the named restart. The args and kwargs are "frozen" into the created handler by closure, and @@ -422,13 +432,13 @@ def invoker(restart_name, *args, **kwargs): return the_invoker class _Stacked: # boilerplate - def __init__(self, bindings): + def __init__(self, bindings: Any) -> None: _ensure_stacks() self.e = bindings - def __enter__(self): + def __enter__(self) -> "_Stacked": self.dq.appendleft(self.e) return self - def __exit__(self, exctype, excvalue, traceback): + def __exit__(self, exctype: type[BaseException] | None, excvalue: BaseException | None, traceback: Any) -> None: self.dq.popleft() class Restarts(_Stacked): @@ -436,7 +446,7 @@ class Restarts(_Stacked): # because `with restarts` tells apart instances by their `id`. # The `with restarts` form packs the arguments once, then we pass # through that dictionary instance as-is. - def __init__(self, bindings): + def __init__(self, bindings: dict[str, Callable[..., Any]]) -> None: """bindings: dictionary of name (str) -> callable""" for n, c in bindings.items(): if not (isinstance(n, str) and callable(c)): @@ -490,7 +500,7 @@ class handlers(_Stacked): """ # This thin wrapper around `_Stacked` is all we need to provide # the `with handlers` form. - def __init__(self, *bindings): + def __init__(self, *bindings: tuple[type[BaseException] | tuple[type[BaseException], ...], Callable[..., Any]]) -> None: """binding: (cls, callable)""" for t, c in bindings: if not (((isinstance(t, tuple) and all(safeissubclass(x, BaseException) for x in t)) or @@ -500,15 +510,15 @@ def __init__(self, *bindings): super().__init__(bindings) self.dq = _stacks.handlers -class InvokeRestart(Exception): - def __init__(self, restart, *args, **kwargs): # e is the context +class InvokeRestart(BaseException): + def __init__(self, restart: "BoundRestart", *args: Any, **kwargs: Any) -> None: self.restart, self.a, self.kw = restart, args, kwargs # message when uncaught self.args = ("unpythonic.conditions: internal error: uncaught InvokeRestart",) - def __call__(self): + def __call__(self) -> Any: return self.restart.function(*self.a, **self.kw) -def _find_handlers(cls): # 0..n (though 0 is an error, handled at the calling end) +def _find_handlers(cls: type[BaseException]) -> Generator[Callable[..., Any], None, None]: # 0..n (though 0 is an error, handled at the calling end) _ensure_stacks() for e in _stacks.handlers: for t, handler in e: # t: tuple or type @@ -520,7 +530,7 @@ def _find_handlers(cls): # 0..n (though 0 is an error, handled at the calling e yield handler BoundRestart = namedtuple("BoundRestart", ["name", "function", "context"]) -def find_restart(name): # exactly 1 (most recently bound wins) +def find_restart(name: str) -> "BoundRestart | None": # exactly 1 (most recently bound wins) """Look up a restart. Known as `FIND-RESTART` in Common Lisp. If the named restart is currently in (dynamic) scope, return an opaque @@ -537,8 +547,9 @@ def find_restart(name): # exactly 1 (most recently bound wins) for e in _stacks.restarts: if name in e: return BoundRestart(name, e[name], e) + return None # no matching restart found -def available_restarts(): +def available_restarts() -> list[tuple[str, Callable[..., Any]]]: """Return a sorted list of restarts currently in scope. Name shadowing is respected; for each unique name, the return value @@ -558,7 +569,7 @@ def available_restarts(): out.append((name, restart)) return list(sorted(out, key=itemgetter(0))) -def available_handlers(): +def available_handlers() -> list[tuple[type[BaseException], Callable[..., Any]]]: """Like available_restarts, but for handlers. As in `available_restarts`, shadowing is respected. In this case the most @@ -581,7 +592,7 @@ def available_handlers(): return list(sorted(out, key=lambda x: x[0].__name__)) @contextlib.contextmanager -def restarts(**bindings): +def restarts(**bindings: Callable[..., Any]) -> Generator[box, None, None]: """Provide restarts. Known as `RESTART-CASE` in Common Lisp. Roughly, restarts can be thought of as canned error recovery strategies. @@ -682,7 +693,7 @@ def restarts(**bindings): else: raise # unwind this level of call stack, propagate outwards -def with_restarts(**bindings): +def with_restarts(**bindings: Callable[..., Any]) -> Callable[[Callable[[], T]], T]: """Alternate syntax. Use restarts with a `def` code block instead of a `with`. The def'd name is replaced by the unboxed result, so you can return a value @@ -717,7 +728,7 @@ def dostuff(): If you'd like to use a `with` statement instead of a parametric decorator and a `def`, see the `restarts` form. """ - def call_with_restarts(f): + def call_with_restarts(f: Callable[[], T]) -> T: """Call `f`, while providing the restarts stored in this closure. Invoking such a restart terminates `f`, and instead of its normal @@ -731,7 +742,7 @@ def call_with_restarts(f): # Common Lisp standard error handling protocols, building on the `signal` function. # Pythonified to add the `cause` argument. -def error(condition, *, cause=None): +def error(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None = None) -> NoReturn: """Like `signal`, but raise `ControlError` if the condition is not handled. Note **raise**, not **signal**. Keep in mind the original Common Lisp @@ -750,7 +761,7 @@ def error(condition, *, cause=None): """ _error(condition, cause=cause, protocol=error) -def cerror(condition, *, cause=None): +def cerror(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None = None) -> None: """Like `error`, but allow a handler to instruct the caller to ignore the error. `cerror` internally establishes a restart named `proceed`, which can be @@ -786,7 +797,7 @@ def __init__(self, value): with restarts(proceed=(lambda: None)): # just for control, no return value _error(condition, cause=cause, protocol=cerror) -def _error(condition, *, cause, protocol): +def _error(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None, protocol: ConditionProtocol) -> NoReturn: # The return value is canonized to an instance (even if `condition` was an exception *type*), # and importantly, it has a nice-looking traceback that points to this line here. # If the signal goes unhandled, Python's exception system will want to show that traceback @@ -800,7 +811,7 @@ def _error(condition, *, cause, protocol): # TODO: And do we want to raise ControlError, or the original condition? raise ControlError("Unhandled error condition") from condition -def warn(condition, *, cause=None): +def warn(condition: BaseException | type[BaseException], *, cause: BaseException | type[BaseException] | None = None) -> None: """Like `signal`, but emit a warning if the condition is not handled. For emitting the warning, we use Python's standard `warnings.warn` mechanism. @@ -864,22 +875,28 @@ def __init__(self, value): # Library to application signal type auto-conversion -def _resignal_handler(mapping, condition): +def _resignal_handler(mapping: dict[type[BaseException] | tuple[type[BaseException], ...], type[BaseException] | BaseException], condition: BaseException) -> None: """Remap a condition instance to another condition type. `mapping`: dict-like, `{LibraryExc0: ApplicationExc0, ...}` - Each `LibraryExc` must be a signal type. + Each `LibraryExc` must be an exception type or a tuple of + exception types. It will be matched using `isinstance`. + + Each `ApplicationExc` can be an exception type or an exception + instance. If an instance, then that exact instance is signaled + as the converted signal. - Each `ApplicationExc` can be a condition type or an instance. - If an instance, then that exact instance is signaled as the - converted condition. + `condition`: the exception instance that was signaled, and is to + be converted (if it matches an entry in `mapping`). + When converted, it is automatically chained into + an `ApplicationExc` signal. - `libraryexc`: the signal instance to convert. It is - automatically chained into `ApplicationExc`. + Conversions in `mapping` are tried in the order specified; hence, + just like in `with handlers`, place more specific types first. - This function never returns normally. If no key in the mapping - matches, this delegates to the next outer handler. + If no key in the mapping matches, this delegates to the next outer + signal handler. """ for LibraryExc, ApplicationExc in mapping.items(): if isinstance(condition, LibraryExc): @@ -891,7 +908,7 @@ def _resignal_handler(mapping, condition): resignaler(ApplicationExc, cause=condition) # cancel and delegate to the next outer handler -def resignal_in(body, mapping): +def resignal_in(body: Callable[[], T], mapping: dict[type[BaseException] | tuple[type[BaseException], ...], type[BaseException] | BaseException]) -> T: """Remap condition types in an expression. Like `unpythonic.excutil.reraise_in` (which see), but for conditions. @@ -927,7 +944,7 @@ def resignal_in(body, mapping): return body() @contextlib.contextmanager -def resignal(mapping): +def resignal(mapping: dict[type[BaseException] | tuple[type[BaseException], ...], type[BaseException] | BaseException]) -> Generator[None, None, None]: """Remap condition types. Context manager. Like `unpythonic.excutil.reraise` (which see), but for conditions. diff --git a/unpythonic/dialects/__init__.py b/unpythonic/dialects/__init__.py index 67d6d7df..04d83bbe 100644 --- a/unpythonic/dialects/__init__.py +++ b/unpythonic/dialects/__init__.py @@ -8,10 +8,12 @@ We provide these dialects mainly to demonstrate how to use that subsystem to customize Python beyond what a local macro expander can do. -For examples of how to use the dialects, see the unit tests. +For examples of how to use these particular dialects, see the unit tests. """ # re-exports -from .lispython import Lispython # noqa: F401 -from .listhell import Listhell # noqa: F401 -from .pytkell import Pytkell # noqa: F401 +from .befunge import * # noqa: F401, F403 +from .bf import * # noqa: F401, F403 +from .lispython import * # noqa: F401, F403 +from .listhell import * # noqa: F401, F403 +from .pytkell import * # noqa: F401, F403 diff --git a/unpythonic/dialects/befunge.py b/unpythonic/dialects/befunge.py new file mode 100644 index 00000000..30c45e08 --- /dev/null +++ b/unpythonic/dialects/befunge.py @@ -0,0 +1,307 @@ +# -*- coding: utf-8 -*- +"""befunge: Befunge-93 as a Python dialect. + +Activate via:: + + \"\"\"Hello from Befunge!\"\"\" + + from unpythonic.dialects.befunge import dialects, Befunge + + "!egnufeB morf olleH">:#,_@ + +The body of a `befunge`-dialect file is parsed as a Befunge-93 playfield +and run by a runtime interpreter shipped in this module. + +Unlike `unpythonic.dialects.bf`, the compiled output is a thin shim — the +dialect's `transform_source` wraps the playfield text in a single call to +`run`. Befunge's semantics live in this module's interpreter, by necessity: + +- The IP moves in two dimensions on a fixed 80×25 toroidal playfield. +- `p` (put) lets a program rewrite its own cells at runtime, so no + static analysis of the playfield can be sound. +- Control flow is fundamentally IP-driven: `?` random direction, + `_`/`|` direction-from-stack, `#` skip-next, string mode (`"`). + +Where `bf`'s `compile` produces structured Python that mirrors the program, +`befunge`'s `transform_source` produces only an interpreter invocation. +That contrast is itself the pedagogic value of having two dialects. + +I/O operator names differ from `bf`. In Befunge: + +- `.` pops and prints an integer (followed by a space, per spec) +- `,` pops and prints a character (`chr(value & 0xFF)`) +- `&` reads a whitespace-delimited integer from stdin and pushes it +- `~` reads one character from stdin and pushes its `ord` + +EOF on `&` and `~` pushes 0 (matches `bf`'s convention). + +Errors + +- `SyntaxError` — source-level malformation at `Playfield(src)` construction + (more than 25 rows, any line longer than 80 columns). +- `IndexError` — runtime out-of-grid access via `g`/`p`. The IP itself + never goes out of grid; its motion wraps toroidally. +- `UnknownOpcodeError` — runtime: the IP visited a cell whose byte value + isn't a recognized Befunge command. Subclasses `RuntimeError`. + +Stack and cells + +- Stack: unbounded Python `int`s. Underflow on pop returns 0 (per spec). +- Playfield cells: bytes (0–255). `p` masks the stored value to a byte; + `g` returns the byte value. + +Random direction (`?`) + +The `?` command picks a direction with `random.Random`, an instance kept +per `run` call. ``run(src, *, seed=...)`` lets tests pin the RNG for +deterministic output; ``seed=None`` uses OS entropy. +""" + +__all__ = ["Befunge", "Playfield", "UnknownOpcodeError", "run"] + +import random +import sys + +from mcpyrate.dialects import Dialect, split_at_dialectimport + + +WIDTH = 80 +HEIGHT = 25 + + +class UnknownOpcodeError(RuntimeError): + """Raised when the Befunge interpreter visits a cell with no + recognized command, including bytes written by `p` that don't + map to any opcode. + + Subclasses ``RuntimeError`` so blanket runtime-error catchers still + work; the specific class enables targeted ``except UnknownOpcodeError``. + """ + + +class Playfield: + """Strict Befunge-93 playfield: 80×25 cells, byte-valued. + + Cells outside the grid raise ``IndexError`` on both read and write. + The grid is fixed; the IP wraps toroidally during motion (handled + by `run`, not by this class). + + Exposed for unit testing the layout and out-of-bounds policy in + isolation from the interpreter loop. + """ + WIDTH = WIDTH + HEIGHT = HEIGHT + + def __init__(self, src: str = "") -> None: + lines = src.splitlines() + # Strip leading and trailing entirely-blank lines. In-line leading + # spaces on a non-blank line are preserved — those are no-op cells. + while lines and not lines[0].strip(): + lines.pop(0) + while lines and not lines[-1].strip(): + lines.pop() + if len(lines) > HEIGHT: + raise SyntaxError( + f"befunge: program exceeds {HEIGHT}-row grid (got {len(lines)} rows)" + ) + for k, ln in enumerate(lines): + if len(ln) > WIDTH: + raise SyntaxError( + f"befunge: line {k} exceeds {WIDTH}-column grid (got {len(ln)} cols)" + ) + # Pad to HEIGHT rows of WIDTH bytes; default-blank cells read as ord(' '). + self._cells = bytearray(b" " * (WIDTH * HEIGHT)) + for y, ln in enumerate(lines): + for x, ch in enumerate(ln): + self._cells[y * WIDTH + x] = ord(ch) & 0xFF + + def __getitem__(self, xy: tuple) -> int: + x, y = xy + if not (0 <= x < WIDTH and 0 <= y < HEIGHT): + raise IndexError(f"befunge: cell ({x}, {y}) out of grid") + return self._cells[y * WIDTH + x] + + def __setitem__(self, xy: tuple, value: int) -> None: + x, y = xy + if not (0 <= x < WIDTH and 0 <= y < HEIGHT): + raise IndexError(f"befunge: cell ({x}, {y}) out of grid") + self._cells[y * WIDTH + x] = value & 0xFF + + +_DIRECTIONS = ((1, 0), (-1, 0), (0, 1), (0, -1)) # E, W, S, N + + +def run(src: str, *, seed: int = None) -> None: + """Run a Befunge-93 program. + + `src` is the playfield text. Output goes to ``sys.stdout``; input + is read from ``sys.stdin``. + + `seed` (keyword-only, default ``None``) seeds the RNG used by `?`. + Pass an explicit integer for deterministic output in tests; leave at + ``None`` for normal nondeterminism via OS entropy. + """ + rng = random.Random(seed) + pf = Playfield(src) + stack: list = [] + + def push(v: int) -> None: + stack.append(v) + + def pop() -> int: + return stack.pop() if stack else 0 + + x, y = 0, 0 + dx, dy = 1, 0 + string_mode = False + + while True: + cell = pf[(x, y)] + ch = chr(cell) + + if string_mode: + if ch == '"': + string_mode = False + else: + push(cell) + elif ch == '@': + return + elif "0" <= ch <= "9": + push(int(ch)) + elif ch == "+": + b = pop() + a = pop() + push(a + b) + elif ch == "-": + b = pop() + a = pop() + push(a - b) + elif ch == "*": + b = pop() + a = pop() + push(a * b) + elif ch == "/": + # Per spec: division by zero pushes 0. + b = pop() + a = pop() + push(a // b if b != 0 else 0) + elif ch == "%": + b = pop() + a = pop() + push(a % b if b != 0 else 0) + elif ch == "!": + push(0 if pop() != 0 else 1) + elif ch == "`": + b = pop() + a = pop() + push(1 if a > b else 0) + elif ch == ">": + dx, dy = 1, 0 + elif ch == "<": + dx, dy = -1, 0 + elif ch == "v": + dx, dy = 0, 1 + elif ch == "^": + dx, dy = 0, -1 + elif ch == "?": + dx, dy = rng.choice(_DIRECTIONS) + elif ch == "_": + dx, dy = (1, 0) if pop() == 0 else (-1, 0) + elif ch == "|": + dx, dy = (0, 1) if pop() == 0 else (0, -1) + elif ch == '"': + string_mode = True + elif ch == ":": + v = pop() + push(v) + push(v) + elif ch == "\\": + b = pop() + a = pop() + push(b) + push(a) + elif ch == "$": + pop() + elif ch == ".": + sys.stdout.write(f"{pop()} ") + elif ch == ",": + sys.stdout.write(chr(pop() & 0xFF)) + elif ch == "&": + # Read a whitespace-delimited integer from stdin. + # EOF or unparseable input pushes 0. + c = sys.stdin.read(1) + while c and c.isspace(): + c = sys.stdin.read(1) + if not c: + push(0) + else: + buf = c + while True: + c = sys.stdin.read(1) + if not c or c.isspace(): + break + buf += c + try: + push(int(buf)) + except ValueError: + push(0) + elif ch == "~": + c = sys.stdin.read(1) + push(ord(c) if c else 0) + elif ch == "#": + # Trampoline: skip the next cell. Advance one extra step now. + x = (x + dx) % WIDTH + y = (y + dy) % HEIGHT + elif ch == "p": + py = pop() + px = pop() + v = pop() + pf[(px, py)] = v + elif ch == "g": + py = pop() + px = pop() + push(pf[(px, py)]) + elif ch == " ": + pass + else: + raise UnknownOpcodeError( + f"befunge: unknown command at ({x}, {y}): {ch!r}" + ) + + x = (x + dx) % WIDTH + y = (y + dy) % HEIGHT + + +def _strip_leading_blank_lines(text: str) -> str: + lines = text.splitlines(keepends=True) + while lines and not lines[0].strip(): + lines.pop(0) + return "".join(lines) + + +class Befunge(Dialect): + """Befunge-93 as a whole-module source-to-source transformer. + + Text before the dialect-import line is passed through unchanged + (keeps the encoding declaration and module docstring intact); text + after it is treated as a Befunge-93 playfield and embedded verbatim + into a call to `run`. Any other dialect-imports in the module are + preserved so further dialect processing can find them. + + Leading entirely-blank lines in the body are stripped before the + playfield is built — without this, the blank line that typically + follows the dialect-import would become row 0 (all spaces), the IP + would walk a no-op row toroidally forever, and the program would + never reach its first instruction. + """ + def transform_source(self, text): + r = split_at_dialectimport(text, type(self).__name__, self.lineno) + if r is None: + return text + prologue, other, body = r + body = _strip_leading_blank_lines(body) + shim = ( + "from unpythonic.dialects.befunge import run\n" + f"run({body!r})\n" + ) + return prologue + "".join(other) + shim diff --git a/unpythonic/dialects/bf.py b/unpythonic/dialects/bf.py new file mode 100644 index 00000000..5acbce92 --- /dev/null +++ b/unpythonic/dialects/bf.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +"""bf: the classical human-incomprehensible automaton as a Python dialect. + +This module provides a `bf` → Python source-to-source compiler, wrapped +as an `mcpyrate` dialect so that a file ending in ``.py`` can contain +a `bf` program directly:: + + from unpythonic.dialects.bf import dialects, BF + + ++++++++[>++++++++<-]>+. + +Running the file (under `macropython`, or by `import`) compiles the body to +Python, then executes the result. The same compiler is also available as a +plain function:: + + from unpythonic.dialects import bf + print(bf.compile(bf_program_str)) + +The qualified `bf.compile` form is recommended over `from ... import compile` +to avoid shadowing the builtin in the importer's namespace. + +This prints the Python that the dialect would run. Useful for the pedagogic +side of things — reading a non-trivial `bf` program by rewriting it in a +language a human can actually read. + +Design + +- **Cells**: 8-bit wrapping, `dict[int, int]` with `collections.defaultdict` + semantics, implemented by the `Tape` class. The tape auto-extends in both + directions; untouched cells read as zero. + +- **Folding**: consecutive identical commands collapse (`+++` → `tape[ptr] += + 3`). No cancellation of opposites (`+-`, `><` do **not** annihilate). + What you wrote is what you get, just collapsed where collapse is lossless. + +- **Loops**: `[` → `while tape[ptr]:` plus indent; `]` → dedent. An empty + loop body (only comments or nothing) gets a `pass` to keep the output + parseable as Python. + +- **I/O**: `.` writes `chr(tape[ptr])` to `stdout`; `,` reads one character + from `stdin`. On EOF, `,` stores 0 in the current cell. + +- **Comments**: classical `bf` treats any non-command character as a no-op. + We preserve the text: consecutive runs of non-command characters compile + into Python `# ...` comments, positioned where they appeared in the source. + If an author-written comment already begins with `# ` (or just `#`), one + leading `#` is stripped before the compiler prepends its own, so + ``# real comment`` and ``real comment`` both come out as ``# real comment`` + in the output. + +- **`reset`**: a line whose stripped content is exactly ``reset`` compiles to + ``tape.clear(); ptr = 0``. This lets several `bf` programs share a file. + +- **Blank lines**: passed through, with consecutive blanks collapsed to one. +""" + +__all__ = ["BF", "Tape", "compile"] + +from collections import defaultdict + +from mcpyrate.dialects import Dialect, split_at_dialectimport + + +class Tape(defaultdict): + """The `bf` Turing tape. + + A `defaultdict[int, int]` that masks assigned values to 0–255, giving the + canonical 8-bit wrapping cells while leaving the pointer unbounded in + either direction. + """ + def __init__(self): + super().__init__(int) + + def __setitem__(self, key, value): + super().__setitem__(key, value & 0xFF) + + +def compile(src: str) -> str: + """Compile a `bf` program to Python source. + + `src` is the raw `bf` program text (no surrounding Python, no dialect + import). The returned string is self-contained, runnable Python — it + imports `Tape` from this module, initialises state, and performs the + operations the `bf` program describes. + + Shadows ``builtins.compile`` if imported by name. Recommended use + is via the module: ``from unpythonic.dialects import bf`` then + ``bf.compile(src)``. + """ + INDENT = " " + lines_out = [] + indent = 0 + loop_stack = [] # indices into lines_out of open `while tape[ptr]:` lines + + def emit(s: str = "") -> None: + lines_out.append(INDENT * indent + s if s else "") + + def emit_run(cmd: str, count: int) -> None: + if cmd == "+": + emit(f"tape[ptr] += {count}") + elif cmd == "-": + emit(f"tape[ptr] -= {count}") + elif cmd == ">": + emit(f"ptr += {count}") + elif cmd == "<": + emit(f"ptr -= {count}") + + def emit_comment(buf: str) -> None: + text = buf.strip() + if not text: + return + # Strip one author-written leading `#` so we don't double it. + if text.startswith("# "): + text = text[2:] + elif text.startswith("#"): + text = text[1:] + text = text.strip() + if not text: + return + emit(f"# {text}") + + # Prelude + emit("from sys import stdin, stdout") + emit("from unpythonic.dialects.bf import Tape") + emit("tape = Tape()") + emit("ptr = 0") + emit() + prev_blank = True + + for raw_line in src.splitlines(): + stripped = raw_line.strip() + + if stripped == "reset": + if indent != 0: + raise SyntaxError("bf: `reset` is only valid at top level (outside all `[...]` loops)") + emit("# reset") + emit("tape.clear()") + emit("ptr = 0") + prev_blank = False + continue + + if not stripped: + if not prev_blank: + emit() + prev_blank = True + continue + + cur_char = None # last seen +/-/>/< command char in the current run + cur_count = 0 + comment_buf = "" + + for ch in raw_line: + if ch in "+-><": + if comment_buf: + emit_comment(comment_buf) + comment_buf = "" + if ch == cur_char: + cur_count += 1 + else: + if cur_char is not None: + emit_run(cur_char, cur_count) + cur_char = ch + cur_count = 1 + elif ch in "[].,": + if comment_buf: + emit_comment(comment_buf) + comment_buf = "" + if cur_char is not None: + emit_run(cur_char, cur_count) + cur_char = None + cur_count = 0 + if ch == "[": + emit("while tape[ptr]:") + loop_stack.append(len(lines_out) - 1) + indent += 1 + elif ch == "]": + if not loop_stack: + raise SyntaxError("bf: unmatched `]`") + while_idx = loop_stack.pop() + # Python requires a non-empty suite; emit `pass` if the + # loop body contained only comments (or nothing at all). + has_stmt = any( + ln.strip() and not ln.strip().startswith("#") + for ln in lines_out[while_idx + 1:] + ) + if not has_stmt: + emit("pass") + indent -= 1 + elif ch == ".": + emit("stdout.write(chr(tape[ptr])); stdout.flush()") + else: # ch == "," + emit('tape[ptr] = ord(stdin.read(1) or "\\x00")') + else: + if cur_char is not None: + emit_run(cur_char, cur_count) + cur_char = None + cur_count = 0 + comment_buf += ch + + if cur_char is not None: + emit_run(cur_char, cur_count) + if comment_buf: + emit_comment(comment_buf) + + prev_blank = False + + if loop_stack: + raise SyntaxError("bf: unmatched `[`") + + while lines_out and lines_out[-1] == "": + lines_out.pop() + + return "\n".join(lines_out) + "\n" + + +class BF(Dialect): + """Brainfuck as a whole-module source-to-source transformer. + + Text before the dialect-import line is passed through unchanged (keeps + the encoding declaration and module docstring intact); text after it + is treated as `bf` source and compiled via `compile`. Any other + dialect-imports in the module are preserved so that further dialect + processing can find them. + """ + def transform_source(self, text): + r = split_at_dialectimport(text, type(self).__name__, self.lineno) + if r is None: + return text + prologue, other, body = r + return prologue + "".join(other) + compile(body) diff --git a/unpythonic/dialects/lispython.py b/unpythonic/dialects/lispython.py index 32c50cf4..94d56204 100644 --- a/unpythonic/dialects/lispython.py +++ b/unpythonic/dialects/lispython.py @@ -4,7 +4,7 @@ Powered by `mcpyrate` and `unpythonic`. """ -__all__ = ["Lispython"] +__all__ = ["Lispython", "Lispy"] __version__ = '2.0.0' @@ -28,7 +28,7 @@ def transform_ast(self, tree): # tree is an ast.Module with q as template: __lang__ = "Lispython" # noqa: F841, just provide it to user code. from unpythonic.syntax import (macros, tco, autoreturn, # noqa: F401, F811 - multilambda, quicklambda, namedlambda, f, + multilambda, quicklambda, namedlambda, fn, where, let, letseq, letrec, dlet, dletseq, dletrec, @@ -37,7 +37,46 @@ def transform_ast(self, tree): # tree is an ast.Module let_syntax, abbrev, block, expr, cond) from unpythonic import cons, car, cdr, ll, llist, nil, prod, dyn, Values # noqa: F401, F811 - with autoreturn, quicklambda, multilambda, tco, namedlambda: + with autoreturn, quicklambda, multilambda, namedlambda, tco: __paste_here__ # noqa: F821, just a splicing marker. - tree.body = splice_dialect(tree.body, template, "__paste_here__") + + # Beginning with 3.6.0, `mcpyrate` makes available the source location info + # of the dialect-import that imported this dialect. + tree.body = splice_dialect(tree.body, template, "__paste_here__", + reference=self.location_ref) + + return tree + + +class Lispy(Dialect): + """**Pythonistas rejoice!** + + O language like Lisp, like Python! + Semantic changes sensibly carry, + Python's primary virtue vindicate. + Ire me not with implicit imports, + Let my IDE label mistakes. + """ + + def transform_ast(self, tree): # tree is an ast.Module + with q as template: + __lang__ = "Lispy" # noqa: F841, just provide it to user code. + from unpythonic.syntax import (macros, tco, autoreturn, # noqa: F401, F811 + multilambda, quicklambda, namedlambda) + # The important point is none of these expect the user code to look like + # anything but regular Python, so IDEs won't yell about undefined names; + # just the semantics are slightly different. + # + # Even if the user code uses `fn[]` (to make `quicklambda` actually do anything), + # that macro must be explicitly imported. It works, because `splice_dialect` + # hoists macro-imports from the top level of the user code into the top level + # of the template. + with autoreturn, quicklambda, multilambda, namedlambda, tco: + __paste_here__ # noqa: F821, just a splicing marker. + + # Beginning with 3.6.0, `mcpyrate` makes available the source location info + # of the dialect-import that imported this dialect. + tree.body = splice_dialect(tree.body, template, "__paste_here__", + reference=self.location_ref) + return tree diff --git a/unpythonic/dialects/listhell.py b/unpythonic/dialects/listhell.py index 35ece7d4..ebbea55b 100644 --- a/unpythonic/dialects/listhell.py +++ b/unpythonic/dialects/listhell.py @@ -15,7 +15,7 @@ class Listhell(Dialect): def transform_ast(self, tree): # tree is an ast.Module - with q as template: + with q as template: # noqa: F823 -- `q` is a macro-injected name __lang__ = "Listhell" # noqa: F841, just provide it to user code. from unpythonic.syntax import macros, prefix, q, u, kw, autocurry # noqa: F401, F811 # Auxiliary syntax elements for the macros @@ -23,5 +23,10 @@ def transform_ast(self, tree): # tree is an ast.Module from unpythonic import composerc as compose # compose from Right, Currying # noqa: F401 with prefix, autocurry: __paste_here__ # noqa: F821, just a splicing marker. - tree.body = splice_dialect(tree.body, template, "__paste_here__") + + # Beginning with 3.6.0, `mcpyrate` makes available the source location info + # of the dialect-import that imported this dialect. + tree.body = splice_dialect(tree.body, template, "__paste_here__", + reference=self.location_ref) + return tree diff --git a/unpythonic/dialects/pytkell.py b/unpythonic/dialects/pytkell.py index d676388a..c5565835 100644 --- a/unpythonic/dialects/pytkell.py +++ b/unpythonic/dialects/pytkell.py @@ -39,5 +39,10 @@ def transform_ast(self, tree): # tree is an ast.Module from unpythonic import cons, car, cdr, ll, llist, nil # noqa: F401 with lazify, autocurry: __paste_here__ # noqa: F821, just a splicing marker. - tree.body = splice_dialect(tree.body, template, "__paste_here__") + + # Beginning with 3.6.0, `mcpyrate` makes available the source location info + # of the dialect-import that imported this dialect. + tree.body = splice_dialect(tree.body, template, "__paste_here__", + reference=self.location_ref) + return tree diff --git a/unpythonic/dialects/tests/test_befunge.py b/unpythonic/dialects/tests/test_befunge.py new file mode 100644 index 00000000..47044b6f --- /dev/null +++ b/unpythonic/dialects/tests/test_befunge.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +"""Test the befunge dialect: Playfield class, run, and dialect activation.""" + +import io +from contextlib import redirect_stdout + +from mcpyrate.compiler import create_module, run as run_module + +from ...syntax import macros, test, test_raises, the # noqa: F401 +from ...test.fixtures import session, testset + +from ...misc import redirect_stdin +from ..befunge import Befunge, Playfield, UnknownOpcodeError, run # noqa: F401 + + +def _capture(src, *, seed=None, stdin=None): + """Run a Befunge program; return captured stdout.""" + buf = io.StringIO() + if stdin is not None: + with redirect_stdout(buf), redirect_stdin(stdin): + run(src, seed=seed) + else: + with redirect_stdout(buf): + run(src, seed=seed) + return buf.getvalue() + + +def runtests(): + with testset("Playfield class"): + # Default: 80x25 of spaces. + pf = Playfield() + test[pf[(0, 0)] == ord(" ")] + test[pf[(79, 24)] == ord(" ")] + + # Source shorter than 25 lines pads. + pf = Playfield("ab\ncd") + test[pf[(0, 0)] == ord("a")] + test[pf[(1, 0)] == ord("b")] + test[pf[(0, 1)] == ord("c")] + test[pf[(1, 1)] == ord("d")] + test[pf[(2, 0)] == ord(" ")] # right-pad + test[pf[(0, 24)] == ord(" ")] # bottom-pad + + # Trailing blank lines stripped before count. + pf = Playfield("a" + "\n" * 30) # 1 row of "a", 30 trailing blanks + test[pf[(0, 0)] == ord("a")] + + # >25 rows raises SyntaxError. + too_tall = "\n".join(["x"] * 26) + test_raises[SyntaxError, Playfield(too_tall)] + + # Line >80 cols raises SyntaxError. + too_wide = "x" * 81 + test_raises[SyntaxError, Playfield(too_wide)] + + # OOB access (read and write). + pf = Playfield("hi") + test_raises[IndexError, pf[(80, 0)]] + test_raises[IndexError, pf[(0, 25)]] + test_raises[IndexError, pf[(-1, 0)]] + test_raises[IndexError, pf[(0, -1)]] + + def _oob_write(): + pf[(80, 0)] = 42 + test_raises[IndexError, _oob_write()] + + # In-bounds write masks to byte. + pf[(0, 0)] = 0x1FF + test[pf[(0, 0)] == 0xFF] + pf[(0, 0)] = -1 + test[pf[(0, 0)] == 0xFF] + + with testset("run: arithmetic"): + # 9 5 - . → push 9, push 5, subtract, print int → "4 " + test[_capture("95-.@") == "4 "] + # Add, multiply, mod. + test[_capture("23+.@") == "5 "] + test[_capture("23*.@") == "6 "] + test[_capture("73%.@") == "1 "] + # Division by zero pushes 0 (per spec). + test[_capture("50/.@") == "0 "] + test[_capture("50%.@") == "0 "] + # Logical not. + test[_capture("0!.@") == "1 "] + test[_capture("5!.@") == "0 "] + # Greater-than. + test[_capture("53`.@") == "1 "] + test[_capture("35`.@") == "0 "] + + with testset("run: stack ops"): + # : duplicate + test[_capture("5:..@") == "5 5 "] + # \ swap + test[_capture("12\\..@") == "1 2 "] + # $ discard + test[_capture("12$.@") == "1 "] + # Stack underflow returns 0. + test[_capture(".@") == "0 "] + test[_capture("+.@") == "0 "] # 0 + 0 + + with testset("run: string mode and char output"): + # "A", prints 'A'. + test[_capture('"A",@') == "A"] + # Multi-char string mode. + test[_capture('"!iH",,,@') == "Hi!"] + + with testset("run: trampoline #"): + # # skips the next cell. Here, skip a `9` that would push 9. + test[_capture("1#9.@") == "1 "] + + with testset("run: conditional direction _ and |"): + # _ pops; 0 → east, nonzero → west. + # `0_>1.@`: push 0, _ pops 0 → east, > east, push 1, print "1 ", halt. + test[_capture("0_>1.@") == "1 "] + + # | pops; 0 → south, nonzero → north. + # IP travels south to a v that keeps it going south, hitting `2.@`. + prog = ( + "v\n" + ">0|\n" + " v\n" + " 2\n" + " .\n" + " @\n" + ) + test[_capture(prog) == "2 "] + + with testset("run: ? random direction (seeded determinism)"): + # Seed reaches the rng: same seed, same output. + # Program `?@` halts whichever direction is chosen (toroidal wrap + # eventually reaches @ at (1, 0) when going east). + out_a = _capture("?@", seed=42) + out_b = _capture("?@", seed=42) + test[the[out_a] == out_b] + + # ? without seed: just verify it runs without error. + # No assertion on output (nondeterministic). + _capture("?@") + + with testset("run: p (put) and g (get)"): + # Round-trip: put 65 at (3, 1), then get it back, print as char. + # Stack layout for p: ..., v, x, y. We need: 65 (val), 3 (x), 1 (y). + # Push 65 = '6' '5' '*' '+' won't be exact... Use string mode. + # "A" pushes 65. Then push 3 (x), push 1 (y). p stores at (3, 1). + # Then push 3 (x), push 1 (y). g reads, push value. , prints char. + prog = '"A"31p31g,@' + test[_capture(prog) == "A"] + + # OOB g → IndexError. Push x=99 (=9*11=99), y=0. Need x>=80. + # 99* = 9, 9, * = push 81. Then 0, g → reads (81, 0). OOB. + test_raises[IndexError, run("99*0g.@")] + + # OOB p → IndexError. Push v=0, x=81, y=0. + test_raises[IndexError, run("099*0p@")] + + with testset("run: toroidal IP wrap"): + # IP at (3, 0) going east hits @ before wrapping. + # To exercise wrap: program at column 0..N where the @ is at col N, + # but the IP needs to traverse. + # Simplest: `v\n@` — IP at (0,0) `v` south, (0,1) `@` halt. + # Wrap test: program at (0, 0) is `<`, redirect west. IP wraps to + # (79, 0). Need @ somewhere on the wrap path. + # `<` at col 0 → IP goes west, wraps to col 79. + # Need to halt eventually. Place @ at col 79. + prog = "<" + " " * 78 + "@" # 80 chars: < at 0, spaces, @ at 79 + # IP at (0,0) `<` west, wrap to (79, 0) `@` halt. + # Output: nothing. + test[_capture(prog) == ""] + + # Vertical wrap. + # IP at (0, 0) `^` north, wrap to (0, 24). + # Place @ at (0, 24). Build 25-line program. + prog = "^\n" + "\n".join([" "] * 23) + "\n@" + test[_capture(prog) == ""] + + with testset("run: @ halts"): + # Trivial halt; second @ never reached. + test[_capture("@,@") == ""] + + with testset("run: unknown opcode"): + # `Z` is not a Befunge command. Has to be reached as an instruction + # (not in string mode). So just put it at (0, 0). + test_raises[UnknownOpcodeError, run("Z")] + + with testset("run: & integer input and ~ char input"): + # & reads whitespace-delimited int. + test[_capture("&.@", stdin=io.StringIO("42\n")) == "42 "] + # & EOF pushes 0. + test[_capture("&.@", stdin=io.StringIO("")) == "0 "] + # & with non-int: pushes 0. + test[_capture("&.@", stdin=io.StringIO("abc\n")) == "0 "] + + # ~ reads one char. + test[_capture("~,@", stdin=io.StringIO("X")) == "X"] + # ~ EOF pushes 0. + test[_capture("~.@", stdin=io.StringIO("")) == "0 "] + + with testset("run: Hello from Befunge!"): + # Family-tradition Hello World, exercising string mode, the :#,_@ + # print loop, and toroidal westward re-entry through `>`. + program = '"!egnufeB morf olleH">:#,_@' + test[_capture(program) == "Hello from Befunge!"] + + with testset("Befunge dialect activation"): + # A blank line between the dialect-import and the program is the + # natural form. Without leading-blank-line stripping, that blank + # line would become row 0 (all spaces) and the IP would loop + # toroidally on it forever instead of reaching the program. + src = ( + 'from unpythonic.dialects.befunge import dialects, Befunge\n' + '\n' + '"!egnufeB morf olleH">:#,_@\n' + ) + mod = create_module("_befunge_dialect_activation_test") + buf = io.StringIO() + with redirect_stdout(buf): + run_module(src, mod) + test[buf.getvalue() == "Hello from Befunge!"] + + +if __name__ == '__main__': + with session(__file__): + runtests() diff --git a/unpythonic/dialects/tests/test_bf.py b/unpythonic/dialects/tests/test_bf.py new file mode 100644 index 00000000..1c3e4eb7 --- /dev/null +++ b/unpythonic/dialects/tests/test_bf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +"""Test the bf dialect: Tape class, bf.compile, and dialect activation.""" + +import io +from contextlib import redirect_stdout + +from mcpyrate.compiler import create_module, run + +from ...syntax import macros, test, test_raises, the # noqa: F401 +from ...test.fixtures import session, testset + +from ...misc import redirect_stdin +from ..bf import BF, Tape # noqa: F401 +from .. import bf # for bf.compile (qualified to avoid shadowing builtins.compile) + + +def _print_string_program(s): + """Hand-build a simple bf program that writes `s` using only cell 0. + + Not optimal (no multiplication loops), but unambiguously correct and + exercises run-folding on long `+` / `-` sequences. + """ + parts = [] + cur = 0 + for ch in s: + diff = ord(ch) - cur + if diff > 0: + parts.append("+" * diff) + elif diff < 0: + parts.append("-" * (-diff)) + parts.append(".") + cur = ord(ch) + return "".join(parts) + + +def _run_bf(src): + """Compile `src` and exec it, returning the captured stdout.""" + buf = io.StringIO() + code = bf.compile(src) + ns = {} + with redirect_stdout(buf): + exec(compile(code, "", "exec"), ns) + return buf.getvalue() + + +def runtests(): + with testset("Tape class"): + t = Tape() + # Default zero. + test[t[0] == 0] + test[t[999] == 0] + test[t[-5] == 0] + + # Assignment masks to 0-255. + t[0] = 256 + test[t[0] == 0] + t[0] = 257 + test[t[0] == 1] + t[0] = -1 + test[t[0] == 255] + + # += / -= go through __setitem__, so they wrap too. + t2 = Tape() + t2[0] += 300 + test[t2[0] == 44] # 300 & 0xFF + t2[0] = 0 + t2[0] -= 1 + test[t2[0] == 255] + + # clear() resets. + t2[7] = 42 + t2.clear() + test[t2[7] == 0] + + with testset("bf.compile: folding"): + out = bf.compile("+++") + test["tape[ptr] += 3" in the[out]] + out = bf.compile("-----") + test["tape[ptr] -= 5" in the[out]] + out = bf.compile(">>>>") + test["ptr += 4" in the[out]] + out = bf.compile("<<") + test["ptr -= 2" in the[out]] + + with testset("bf.compile: no cancellation of opposites"): + # `+-` does not cancel: two separate runs, honest compilation. + out = bf.compile("+-") + test["tape[ptr] += 1" in the[out]] + test["tape[ptr] -= 1" in the[out]] + # `><` same. + out = bf.compile("><") + test["ptr += 1" in the[out]] + test["ptr -= 1" in the[out]] + + with testset("bf.compile: loops"): + out = bf.compile("[+]") + test["while tape[ptr]:" in the[out]] + test["tape[ptr] += 1" in the[out]] + + # Empty loop body becomes `pass` (Python requires a non-empty suite). + out = bf.compile("[]") + test["while tape[ptr]:" in the[out]] + test["pass" in the[out]] + + # Comment-only body also needs `pass`. + out = bf.compile("[ comment only ]") + test["# comment only" in the[out]] + test["pass" in the[out]] + + # Nested loops. + out = bf.compile("[[+]]") + # Two while statements, with deeper indent on the inner one. + test[out.count("while tape[ptr]:") == 2] + test[" tape[ptr] += 1" in the[out]] # 8-space indent (nested) + + with testset("bf.compile: I/O"): + out = bf.compile(".") + test["stdout.write(chr(tape[ptr]))" in the[out]] + out = bf.compile(",") + test["stdin.read(1)" in the[out]] + # EOF convention: empty string fallback to "\x00". + test['"\\x00"' in the[out]] + + with testset("bf.compile: comments"): + # Non-command text on its own line becomes a Python comment. + out = bf.compile("hello world\n+") + test["# hello world" in the[out]] + test["tape[ptr] += 1" in the[out]] + + # Inline comment between command runs. + out = bf.compile("+++ move right >>>") + lines = out.splitlines() + # Expect: `tape[ptr] += 3`, then `# move right`, then `ptr += 3`. + idx_plus = next(i for i, ln in enumerate(lines) if "tape[ptr] += 3" in ln) + idx_cmt = next(i for i, ln in enumerate(lines) if "# move right" in ln) + idx_gt = next(i for i, ln in enumerate(lines) if "ptr += 3" in ln) + test[the[idx_plus] < the[idx_cmt] < the[idx_gt]] + + # Author-written `#` comment does not get doubled. + out = bf.compile("# a note\n+") + test["# a note" in the[out]] + test["# # a note" not in the[out]] + + with testset("bf.compile: reset"): + out = bf.compile("+\nreset\n+") + # Reset emits a labeled block. + test["# reset" in the[out]] + test["tape.clear()" in the[out]] + test["ptr = 0" in the[out]] + # Both `+` commands are present. + test[out.count("tape[ptr] += 1") == 2] + + # reset inside a loop is an error. + test_raises[SyntaxError, bf.compile("[\nreset\n]")] + + # reset as substring of a longer word does NOT trigger. + out = bf.compile("# we may reset here eventually\n+") + test["tape.clear()" not in the[out]] + + with testset("bf.compile: errors"): + test_raises[SyntaxError, bf.compile("[")] + test_raises[SyntaxError, bf.compile("]")] + test_raises[SyntaxError, bf.compile("[[]")] + + with testset("bf.compile: execution — classic P-printer"): + # `++++++++[>++++++++++<-]>.` — the standard building block. + # Sets cell 1 to 8 * 10 = 80, then prints chr(80) = 'P'. + out = _run_bf("++++++++[>++++++++++<-]>.") + test[out == "P"] + + with testset("bf.compile: execution — single-cell string printer"): + out = _run_bf(_print_string_program("Hi!")) + test[out == "Hi!"] + + # The marquee test — rewards the curious CI-log reader. + out = _run_bf(_print_string_program("Hello from bf!")) + test[out == "Hello from bf!"] + + with testset("bf.compile: execution — reset between programs"): + # Two programs in one file, separated by `reset`. + # First prints 'A' (65), second prints 'B' (66). + src = "+" * 65 + ".\nreset\n" + "+" * 66 + "." + out = _run_bf(src) + test[out == "AB"] + + with testset("bf.compile: execution — input with EOF"): + # `,.` reads one char and echoes it. + code = bf.compile(",.") + ns = {} + # Feed one char, then EOF. + buf = io.StringIO() + with redirect_stdout(buf), redirect_stdin(io.StringIO("Z")): + exec(compile(code, "", "exec"), ns) + test[buf.getvalue() == "Z"] + + # Empty stdin → EOF → cell stays 0 → `.` prints chr(0). + buf = io.StringIO() + with redirect_stdout(buf), redirect_stdin(io.StringIO("")): + exec(compile(code, "", "exec"), ns) + test[buf.getvalue() == "\x00"] + + with testset("BF dialect activation"): + # Run a small bf-in-Python program through the full dialect pipeline. + src = ("from unpythonic.dialects.bf import dialects, BF\n" + "\n" + + _print_string_program("Hello from bf!")) + mod = create_module("_bf_dialect_activation_test") + buf = io.StringIO() + with redirect_stdout(buf): + run(src, mod) + test[buf.getvalue() == "Hello from bf!"] + + +if __name__ == '__main__': + with session(__file__): + runtests() diff --git a/unpythonic/dialects/tests/test_lispy.py b/unpythonic/dialects/tests/test_lispy.py new file mode 100644 index 00000000..8c5f8aed --- /dev/null +++ b/unpythonic/dialects/tests/test_lispy.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""Test the Lispy dialect. + +Like Lispython, but more pythonic: nothing is imported implicitly, +except the macros injected by the dialect template (to perform the +whole-module semantic changes at macro expansion time). +""" + +from ...dialects import dialects, Lispy # noqa: F401 + +from ...syntax import macros, test, the # noqa: F401 +from ...test.fixtures import session, testset + +from ...syntax import macros, continuations, call_cc, letrec, fn, local, cond # noqa: F401, F811 +from ...syntax import _ # optional, makes IDEs happy +from ...funutil import Values + +def runtests(): + print(f"Hello from {__lang__}!") # noqa: F821, the dialect template defines it. + + # auto-TCO (both in defs and lambdas), implicit return in tail position + with testset("implicit tco, implicit autoreturn"): + def fact(n): + def f(k, acc): + if k == 1: + return acc # "return" still available for early return + f(k - 1, k * acc) + f(n, acc=1) + test[fact(4) == 24] + fact(5000) # no crash (and correct result, since Python uses bignums transparently) + + t = letrec[[evenp << (lambda x: (x == 0) or oddp(x - 1)), # noqa: F821 + oddp << (lambda x:(x != 0) and evenp(x - 1))] in # noqa: F821 + evenp(10000)] # no crash # noqa: F821 + test[t is True] + + # lambdas are named automatically + with testset("implicit namedlambda"): + square = lambda x: x**2 + test[square(3) == 9] + test[square.__name__ == "square"] + + # the underscore (in Lispy, the `fn` macro must be imported explicitly) + cube = fn[_**3] + test[cube(3) == 27] + test[cube.__name__ == "cube"] + + my_mul = fn[_ * _] + test[my_mul(2, 3) == 6] + test[my_mul.__name__ == "my_mul"] + + # lambdas can have multiple expressions and local variables + # + # If you need to return a literal list from a lambda, use an extra set of + # brackets; the outermost brackets always enable multiple-expression mode. + # + with testset("implicit multilambda"): + # In Lispy, the `local` macro must be imported explicitly. + # `local[name << value]` makes a local variable in a multilambda (or in any `do[]` environment). + mylam = lambda x: [local[y << 2 * x], # noqa: F821 + y + 1] # noqa: F821 + test[mylam(10) == 21] + + a = lambda x: [local[t << x % 2], # noqa: F821 + cond[t == 0, "even", # noqa: F821 + t == 1, "odd", + None]] # cond[] requires an else branch + test[a(2) == "even"] + test[a(3) == "odd"] + + # MacroPy #21; namedlambda must be in its own with block in the + # dialect implementation or the particular combination of macros + # invoked by Lispy will fail (uncaught jump, __name__ not set). + # + # With `mcpyrate` this shouldn't matter, but we're keeping the example. + with testset("autonamed letrec lambdas, multiple-expression let body"): + t = letrec[[evenp << (lambda x: (x == 0) or oddp(x - 1)), # noqa: F821 + oddp << (lambda x:(x != 0) and evenp(x - 1))] in # noqa: F821 + [local[x << evenp(100)], # noqa: F821, multi-expression let body is a do[] environment + (x, evenp.__name__, oddp.__name__)]] # noqa: F821 + test[t == (True, "evenp", "oddp")] + + with testset("integration with continuations"): + with continuations: # has TCO; should be skipped by the implicit `with tco` inserted by the dialect + k = None # kontinuation + def setk(*args, cc): + nonlocal k + k = cc # current continuation, i.e. where to go after setk() finishes + Values(*args) # multiple-return-values + def doit(): + lst = ['the call returned'] + *more, = call_cc[setk('A')] + lst + list(more) + test[doit() == ['the call returned', 'A']] + # We can now send stuff into k, as long as it conforms to the + # signature of the assignment targets of the "call_cc". + test[k('again') == ['the call returned', 'again']] + test[k('thrice', '!') == ['the call returned', 'thrice', '!']] + + # We must have some statement here to make the implicit autoreturn happy, + # because the continuations testset is the last one, and the top level of + # a `with continuations` block is not allowed to have a `return`. + pass + +if __name__ == '__main__': + with session(__file__): + runtests() diff --git a/unpythonic/dialects/tests/test_lispython.py b/unpythonic/dialects/tests/test_lispython.py index 9e3cba42..4085b07f 100644 --- a/unpythonic/dialects/tests/test_lispython.py +++ b/unpythonic/dialects/tests/test_lispython.py @@ -8,7 +8,7 @@ from ...syntax import macros, continuations, call_cc # noqa: F401, F811 -# `unpythonic` is effectively `lispython`'s stdlib; not everything gets imported by default. +# `unpythonic` is effectively Lispython's stdlib; not everything gets imported by default. from ...fold import foldl # Of course, all of Python's stdlib is available too. @@ -73,11 +73,15 @@ def f(k, acc): test[square(3) == 9] test[square.__name__ == "square"] - # the underscore (NOTE: due to this, "f" is a reserved name in lispython) - cube = f[_**3] # noqa: F821 + # the underscore (NOTE: due to this, "fn" is a reserved name in Lispython) + cube = fn[_**3] # noqa: F821 test[cube(3) == 27] test[cube.__name__ == "cube"] + my_mul = fn[_ * _] # noqa: F821 + test[my_mul(2, 3) == 6] + test[my_mul.__name__ == "my_mul"] + # lambdas can have multiple expressions and local variables # # If you need to return a literal list from a lambda, use an extra set of @@ -96,8 +100,8 @@ def f(k, acc): test[a(3) == "odd"] # MacroPy #21; namedlambda must be in its own with block in the - # dialect implementation or this particular combination will fail - # (uncaught jump, __name__ not set). + # dialect implementation or the particular combination of macros + # invoked by Lispython will fail (uncaught jump, __name__ not set). # # With `mcpyrate` this shouldn't matter, but we're keeping the example. with testset("autonamed letrec lambdas, multiple-expression let body"): @@ -120,7 +124,7 @@ def f(k, acc): test[x == 3] with testset("integration with continuations"): - with continuations: # should be skipped by the implicit tco inserted by the dialect + with continuations: # has TCO; should be skipped by the implicit `with tco` inserted by the dialect k = None # kontinuation def setk(*args, cc): nonlocal k diff --git a/unpythonic/dialects/tests/test_listhell.py b/unpythonic/dialects/tests/test_listhell.py index 6d1f8283..8959c9ad 100644 --- a/unpythonic/dialects/tests/test_listhell.py +++ b/unpythonic/dialects/tests/test_listhell.py @@ -57,7 +57,7 @@ def f(*, a, b): # in case of duplicate name across kws, rightmost wins test[(f, kw(a="hi there"), kw(b="foo"), kw(b="bar")) == (q, "hi there", "bar")] # noqa: F821 - # give *args with unpythonic.fun.apply, like in Lisps: + # give *args with unpythonic.apply, like in Lisps: with testset("starargs with apply()"): lst = [1, 2, 3] def g(*args, **kwargs): diff --git a/unpythonic/dialects/tests/test_pytkell.py b/unpythonic/dialects/tests/test_pytkell.py index 94c9a77a..98628af7 100644 --- a/unpythonic/dialects/tests/test_pytkell.py +++ b/unpythonic/dialects/tests/test_pytkell.py @@ -8,9 +8,12 @@ from ...test.fixtures import session, testset from ...syntax import macros, continuations, call_cc, tco # noqa: F401, F811 +from ...syntax import macros, monadic_do # noqa: F401, F811 +from ...monads import Maybe, Writer, List from ...funutil import Values from ...misc import timer +from math import sqrt from types import FunctionType from operator import add, mul @@ -74,8 +77,8 @@ def f(a, b): test[f(1, 2) == (1, 2)] test[(flip(f))(1, 2) == (2, 1)] # NOTE flip reverses all (doesn't just flip the first two) # noqa: F821 - # # TODO: this doesn't work, because curry sees f's arities as (2, 2) (kwarg handling!) - # test[(flip(f))(1, b=2) == (1, 2)] # b -> kwargs + # flip reverses only those arguments that are passed *positionally* + test[(flip(f))(1, b=2) == (1, 2)] # b -> kwargs # noqa: F821 # http://www.cse.chalmers.se/~rjmh/Papers/whyfp.html with testset("iterables"): @@ -113,6 +116,15 @@ def f(a, b): # # pythagorean triples with testset("nondeterministic evaluation"): + # TODO: This is very slow in Pytkell; investigate whether the cause is `lazify`, `autocurry`, or both. + # + # Running the same code in a macro-enabled IPython (i.e. without Pytkell), there is no noticeable delay + # after you press enter, before it gives the result. If you want to try it, you'll need to: + # + # %load_ext mcpyrate.repl.iconsole + # from unpythonic.syntax import macros, forall, test + # from unpythonic import insist + # pt = forall[z << range(1, 21), # hypotenuse # noqa: F821 x << range(1, z + 1), # shorter leg # noqa: F821 y << range(x, z + 1), # longer leg # noqa: F821 @@ -145,7 +157,7 @@ def f(a, b): test[last(take(1001, s(0, 0.001, ...))) == 1] # noqa: F821 # iterables returned by s() support infix math - # (to add infix math support to some other iterable, m(iterable)) + # (to add infix math support to some other iterable, imathify(iterable)) c = s(1, 3, ...) + s(2, 4, ...) # noqa: F821 test[tuple(take(5, c)) == (3, 7, 11, 15, 19)] # noqa: F821 test[tuple(take(5, c)) == (23, 27, 31, 35, 39)] # consumed! # noqa: F821 @@ -200,14 +212,74 @@ def f(k, acc): if k == 1: return acc return f(k - 1, k * acc) - return f(n, 1) # TODO: doesn't work as f(n, acc=1) due to curry's kwarg handling + return f(n, acc=1) test[fact(4) == 24] + # **CAUTION**: Pytkell is slow, because so much happens at run time. On an i7-4710MQ: + # + # - The performance test below, `fact(5000)`, completes in about 500ms. + # + # **Without** Pytkell, using a macro-enabled IPython session: + # + # - `fact(5000)` with the same definition (the `with tco` block above) completes in about 15ms. + # - `prod(range(1, 5001))` completes in about 7ms. (This is `unpythonic.prod`, which uses + # `unpythonic`'s custom fold implementation.) + # - The simplest thing that works: + # n = 1 + # for k in range(1, 5001): + # n *= k + # completes in about 5ms. print("Performance...") with timer() as tictoc: - fact(5000) # no crash, but Pytkell is a bit slow + fact(5000) # no crash print(" Time taken for factorial of 5000: {:g}s".format(tictoc.dt)) + # No kell is complete without its monads. + with testset("monadic do-notation"): + # `nil` is available from the Pytkell dialect template (module-level import). + + # Maybe — sqrt chain. In Pytkell's auto-lazy world, the chain + # still evaluates eagerly at the bind points (since the receiver + # of >> needs to be an actual monad to dispatch). + def maybe_sqrt(x): + if x < 0: + return Maybe(nil) # noqa: F821 -- `nil` is in the Pytkell dialect + return Maybe(sqrt(x)) + + with monadic_do[Maybe] as root4: + [a := maybe_sqrt(16), + b := maybe_sqrt(a), + Maybe(b)] + test[root4 == Maybe(2.0)] + + with monadic_do[Maybe] as bad: + [a := maybe_sqrt(-1), + b := maybe_sqrt(a), + Maybe(b)] + test[bad == Maybe(nil)] # noqa: F821 -- `nil` is in the Pytkell dialect + + # List — classical Pythagorean triples. + def r(lo, hi): + return List.from_iterable(range(lo, hi)) + + with monadic_do[List] as pt: + [z := r(1, 21), + x := r(1, z + 1), + y := r(x, z + 1), + List.guard(x * x + y * y == z * z), + List((x, y, z))] + test[tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20))] + + # Writer — logged computation. + with monadic_do[Writer] as w: + [a := Writer(10, "start; "), + b := Writer(a + 1, "+1; "), + Writer(b * 2, "doubled; ")] + value, log = w.data + test[value == 22] + test[log == "start; +1; doubled; "] + if __name__ == '__main__': with session(__file__): runtests() diff --git a/unpythonic/dispatch.py b/unpythonic/dispatch.py index a0268ecc..5f06e414 100644 --- a/unpythonic/dispatch.py +++ b/unpythonic/dispatch.py @@ -234,6 +234,10 @@ def example(): See the limitations in `unpythonic.typecheck` for which features of the `typing` module are supported and which are not. + + Code using the `with lazify` macro cannot usefully use `@generic` or `@typed`, + because all arguments of each function call will be wrapped in a promise + (`unpythonic.lazyutil.Lazy`) that carries no type information on its contents. """ return _setup(_function_fullname(f), f) @@ -299,6 +303,12 @@ def typed(f): Once a `@typed` function has been created, no more multimethods can be attached to it. + + **CAUTION**: + + Code using the `with lazify` macro cannot usefully use `@generic` or `@typed`, + because all arguments of each function call will be wrapped in a promise + (`unpythonic.lazyutil.Lazy`) that carries no type information on its contents. """ s = generic(f) del s._register # remove the ability to register more methods diff --git a/unpythonic/dynassign.py b/unpythonic/dynassign.py index 8fce2455..1520eeca 100644 --- a/unpythonic/dynassign.py +++ b/unpythonic/dynassign.py @@ -5,7 +5,9 @@ import threading from collections import ChainMap -from collections.abc import Container, Sized, Iterable, Mapping +from collections.abc import Container, ItemsView, Iterator, KeysView, Sized, Iterable, Mapping, ValuesView +from types import TracebackType +from typing import Any from .singleton import Singleton @@ -16,7 +18,7 @@ _mainthread_stack = [] _mainthread_lock = threading.RLock() -def _getstack(): +def _getstack() -> list[dict[str, Any]]: if threading.current_thread() is threading.main_thread(): return _mainthread_stack if not hasattr(_L, "_stack"): @@ -29,20 +31,20 @@ def _getstack(): _L._stack = _mainthread_stack.copy() return _L._stack -def _getobservers(): +def _getobservers() -> dict[int, "_DynLiveView"]: if not hasattr(_L, "_observers"): _L._observers = {} return _L._observers class _EnvBlock(object): - def __init__(self, bindings): + def __init__(self, bindings: dict[str, Any]) -> None: self.bindings = bindings - def __enter__(self): + def __enter__(self) -> None: if self.bindings: # optimization, skip pushing an empty scope _getstack().append(self.bindings) for o in _getobservers().values(): o._refresh() - def __exit__(self, t, v, tb): + def __exit__(self, exctype: type[BaseException] | None, excvalue: BaseException | None, traceback: TracebackType | None) -> None: if self.bindings: _getstack().pop() for o in _getobservers().values(): @@ -51,14 +53,14 @@ def __exit__(self, t, v, tb): # We need multiple observer instances, because dynamic scope stacks are thread-local. # If they weren't, this could be a singleton and the __del__ method wouldn't be needed. class _DynLiveView(ChainMap): - def __init__(self): + def __init__(self) -> None: super().__init__(self) self._refresh() _getobservers()[id(self)] = self # TODO: __del__ most certainly runs during test_dynassign (as can be # evidenced by placing a debug print inside it), but coverage fails # to report it as covered. - def __del__(self): # pragma: no cover + def __del__(self) -> None: # pragma: no cover # No idea how, but our REPL server can trigger a KeyError here # if the user views `help()`, which causes the client to get stuck. # Then pressing `q` in the server console to quit the help, and then @@ -73,7 +75,7 @@ def __del__(self): # pragma: no cover del _getobservers()[id(self)] except KeyError: pass - def _refresh(self): + def _refresh(self) -> None: self.maps = list(reversed(_getstack())) + [_global_dynvars] class _Dyn(Singleton): @@ -143,7 +145,7 @@ def main(): # it doesn't matter that the default `__setstate__` clobbers the `__dict__` # of the singleton instance at unpickle time. - def _resolve(self, name): + def _resolve(self, name: str) -> dict[str, Any]: # Essentially asdict() and look up, but without creating the ChainMap # every time _resolve() is called. for scope in reversed(_getstack()): @@ -153,12 +155,12 @@ def _resolve(self, name): return _global_dynvars raise AttributeError(f"dynamic variable {repr(name)} is not defined") - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: """Read the value of a dynamic binding.""" scope = self._resolve(name) return scope[name] - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: """Update an existing dynamic binding. The update occurs in the closest enclosing dynamic scope that has @@ -173,7 +175,7 @@ def __setattr__(self, name, value): scope = self._resolve(name) scope[name] = value - def let(self, **bindings): + def let(self, **bindings: Any) -> _EnvBlock: """Introduce dynamic bindings. Context manager; usage is ``with dyn.let(name=value, ...):`` @@ -189,7 +191,7 @@ def let(self, **bindings): """ return _EnvBlock(bindings) - def update(self, **bindings): + def update(self, **bindings: Any) -> None: """Mass-update existing dynamic bindings. For each binding, the update occurs in the closest enclosing dynamic @@ -202,7 +204,7 @@ def update(self, **bindings): caution applies. Use carefully, if at all. """ # validate, and resolve scopes (let AttributeError propagate) - def doit(): + def doit() -> None: scopes = {k: self._resolve(k) for k in bindings} for k, v in bindings.items(): scope = scopes[k] @@ -216,7 +218,7 @@ def doit(): doit() # membership test (in, not in) - def __contains__(self, name): + def __contains__(self, name: str) -> bool: try: getattr(self, name) return True @@ -224,7 +226,7 @@ def __contains__(self, name): return False # iteration - def asdict(self): + def asdict(self) -> _DynLiveView: """Return a view of dyn as a ``collections.ChainMap``. When new dynamic scopes begin or old ones exit, its ``.maps`` attribute @@ -232,40 +234,40 @@ def asdict(self): """ return _DynLiveView() - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self.asdict()) # no __next__, iterating over dict. # Mapping - def items(self): + def items(self) -> ItemsView[str, Any]: """Abbreviation for asdict().items().""" return self.asdict().items() - def keys(self): + def keys(self) -> KeysView[str]: return self.asdict().keys() - def values(self): + def values(self) -> ValuesView[Any]: return self.asdict().values() - def get(self, k, default=None): - return self[k] if k in self else default - def __eq__(self, other): # dyn is a singleton, but its contents can be compared to another mapping. + def get(self, k: str, default: Any = None) -> Any: + return self[k] if k in self else default # noqa: SIM401 -- this IS the .get() implementation + def __eq__(self, other: Any) -> bool: # dyn is a singleton, but its contents can be compared to another mapping. return other == self.asdict() - def __len__(self): + def __len__(self) -> int: return len(self.asdict()) # subscripting - def __getitem__(self, k): + def __getitem__(self, k: str) -> Any: return getattr(self, k) - def __setitem__(self, k, v): + def __setitem__(self, k: str, v: Any) -> None: setattr(self, k, v) # pretty-printing - def __repr__(self): # pragma: no cover + def __repr__(self) -> str: # pragma: no cover bindings_list = [f"{k}={repr(self[k])}" for k in self] bindings_str = ", ".join(bindings_list) return f"" dyn = _Dyn() -def make_dynvar(**bindings): +def make_dynvar(**bindings: Any) -> None: """Create a dynamic variable and set its default value. The default value is used when ``dyn`` is queried for the value outside the diff --git a/unpythonic/ec.py b/unpythonic/ec.py index c612aa31..be8a77da 100644 --- a/unpythonic/ec.py +++ b/unpythonic/ec.py @@ -26,12 +26,17 @@ __all__ = ["throw", "catch", "call_ec"] +from collections.abc import Callable from functools import wraps +from typing import Any, NoReturn, TypeVar from .regutil import register_decorator -# from .symbol import gensym +from .symbol import gensym -def throw(value, tag=None, allow_catchall=True): +F = TypeVar('F', bound=Callable) +T = TypeVar('T') + +def throw(value: Any, tag: Any = None, allow_catchall: bool = True) -> NoReturn: """Escape to a dynamically surrounding ``@catch``. Essentially this just raises an ``Escape`` instance with the given arguments. @@ -59,12 +64,12 @@ def throw(value, tag=None, allow_catchall=True): """ raise Escape(value, tag, allow_catchall) -class Escape(Exception): +class Escape(BaseException): """Exception that essentially represents the invocation of an escape continuation. Constructor parameters: see ``throw()``. """ - def __init__(self, value, tag=None, allow_catchall=True): + def __init__(self, value: Any, tag: Any = None, allow_catchall: bool = True) -> None: self.value = value self.tag = tag self.allow_catchall = allow_catchall @@ -72,7 +77,7 @@ def __init__(self, value, tag=None, allow_catchall=True): # Error message when uncaught self.args = ("Not within the dynamic extent of a @catch",) -def catch(tags=None, catch_untagged=True): +def catch(tags: Any = None, catch_untagged: bool = True) -> Callable[[F], F]: """Decorator. Mark function as exitable by ``throw(value)``. In Lisp terms, this essentially captures the escape continuation (ec) @@ -182,14 +187,14 @@ def s(loop, acc=0, i=0): else: # single tag tags = set((tags,)) - def shouldcatch(e): + def shouldcatch(e: Escape) -> bool: return ((tags is None and e.allow_catchall) or (catch_untagged and e.tag is None) or (tags is not None and e.tag is not None and e.tag in tags)) - def decorator(f): + def decorator(f: F) -> F: @wraps(f) - def catchpoint(*args, **kwargs): + def catchpoint(*args: Any, **kwargs: Any) -> Any: try: return f(*args, **kwargs) except Escape as e: @@ -201,7 +206,7 @@ def catchpoint(*args, **kwargs): return decorator @register_decorator(priority=80) -def call_ec(f): +def call_ec(f: Callable[..., T]) -> T: """Decorator. Call with escape continuation (call/ec). Parameters: @@ -249,7 +254,9 @@ def inner(): Similar usage is valid for named functions, too. """ # Create a process-wide unique id to tag the ec: - anchor = object() # gensym("anchor"), but object() is much faster, and we don't need a label, or pickle support. + # If this ever becomes a performance bottleneck, object() is faster + # (no UUID allocation) at the cost of losing debug readability. + anchor = gensym("anchor") uid = id(anchor) # Closure property important here. "ec" itself lives as long as someone # retains a reference to it. It's a first-class value; the callee could @@ -260,7 +267,7 @@ def inner(): # if it is raised. ec_valid = True # First-class ec like in Lisps. What's first-class in Python? Functions! - def ec(value): + def ec(value: Any) -> NoReturn: if not ec_valid: raise RuntimeError("Cannot escape after the dynamic extent of the call_ec invocation.") # Be catchable only by our own catch point. @@ -268,7 +275,7 @@ def ec(value): try: # Set up a tagged catch point that catches only the ec we just set up. @catch(uid, catch_untagged=False) - def wrapper(): + def wrapper() -> T: return f(ec) return wrapper() finally: diff --git a/unpythonic/env.py b/unpythonic/env.py index eac43868..f780085c 100644 --- a/unpythonic/env.py +++ b/unpythonic/env.py @@ -3,7 +3,9 @@ __all__ = ["env"] -from collections.abc import Container, Sized, Iterable, Mapping, MutableMapping +from collections.abc import Container, Sized, Iterable, ItemsView, Iterator, KeysView, Mapping, MutableMapping, ValuesView +from types import TracebackType +from typing import Any from .lazyutil import passthrough_lazy_args # co-operate with unpythonic.syntax.lazify; this is essentially a binding construct, @@ -12,6 +14,12 @@ class env: """Environment for let-like constructs. + **Import as** ``from unpythonic.env import env``. This module shares its name + with this class, and the top-level package does not re-export the class, so + ``from unpythonic import env`` gives you the *module* instead - after which + ``env(x=1)`` fails with ``TypeError: 'module' object is not callable``, which + does not hint at the cause. + Names must be identifiers (see str.isidentifier()), even when introduced by subscripting the env instance. @@ -52,22 +60,26 @@ class env: """ # do not allow bindings that would break functionality. _reserved_names = ("set", "clear", "finalize", "_env", "_finalized", - "_direct_write", "_reserved_names") - _direct_write = ("_env", "_finalized") - - def __init__(self, **bindings): - self._env = {} - self._finalized = False # "let" sets this once env setup done + "_reserved_names") + + # For pickle support, since unpickling calls `__new__` but not `__init__`. + # If `self._env` is not present, `__getattr__` will crash with an infinite loop. So create it as early as possible. + def __new__(cls, **kwargs: Any) -> "env": + instance = super().__new__(cls) + # Bypass our locked-down `__setattr__` to install the internal slots once. + object.__setattr__(instance, "_env", {}) + object.__setattr__(instance, "_finalized", False) # `finalize()` flips this + instance.__init__(**kwargs) + return instance + + def __init__(self, **bindings: Any) -> None: for name, value in bindings.items(): setattr(self, name, value) # item access by name # https://docs.python.org/3/reference/datamodel.html#object.__setattr__ # https://docs.python.org/3/reference/datamodel.html#object.__getattr__ - def __setattr__(self, name, value): - # TODO: doesn't protect against client code writing to the _direct_write names. - if name in self._direct_write: # hook to allow creating internal variables directly in self - return super().__setattr__(name, value) + def __setattr__(self, name: str, value: Any) -> None: if name in self._reserved_names: raise AttributeError(f"cannot overwrite reserved name {repr(name)}; complete list: {self._reserved_names}") if self._finalized and name not in self: @@ -78,7 +90,7 @@ def __setattr__(self, name, value): # value = self._wrap(name, value) # for "e.x << value" rebind syntax. self._env[name] = value # make all other attrs else live inside _env - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: # Block invalid names in subscripting (which redirects here). if not name.isidentifier(): raise ValueError(f"{repr(name)} is not a valid identifier") @@ -87,7 +99,7 @@ def __getattr__(self, name): raise AttributeError(f"name {repr(name)} is not defined") return e[name] - def __delattr__(self, name): + def __delattr__(self, name: str) -> None: if not name.isidentifier(): # Can happen through __delitem__. raise ValueError(f"{repr(name)} is not a valid identifier") if self._finalized: @@ -98,44 +110,44 @@ def __delattr__(self, name): del e[name] # membership test (in, not in) - def __contains__(self, k): + def __contains__(self, k: str) -> bool: return self._env.__contains__(k) # iteration - def __iter__(self): + def __iter__(self) -> Iterator[str]: return self._env.__iter__() # no __next__, iterating over dict. # Mapping - def items(self): + def items(self) -> ItemsView[str, Any]: """Like dict.items().""" return self._env.items() - def keys(self): + def keys(self) -> KeysView[str]: return self._env.keys() - def values(self): + def values(self) -> ValuesView[Any]: return self._env.values() - def get(self, k, default=None): - return self[k] if k in self else default - def __eq__(self, other): + def get(self, k: str, default: Any = None) -> Any: + return self[k] if k in self else default # noqa: SIM401 -- this IS the .get() implementation + def __eq__(self, other: Any) -> bool: return other == self._env - def __len__(self): + def __len__(self) -> int: return len(self._env) # MutableMapping - def pop(self, k, *default): + def pop(self, k: str, *default: Any) -> Any: if self._finalized: raise TypeError(f"deleting bindings from a finalized environment not allowed; attempted to delete {repr(k)}") return self._env.pop(k, *default) - def popitem(self): + def popitem(self) -> tuple[str, Any]: if self._finalized: raise TypeError("deleting bindings from a finalized environment not allowed") return self._env.popitem() - def clear(self): + def clear(self) -> None: if self._finalized: raise TypeError("clearing a finalized environment not allowed") return self._env.clear() - def update(self, *mapping, **bindings): + def update(self, *mapping: Mapping, **bindings: Any) -> None: """See `dict.update` for the signature.""" if mapping: if len(mapping) > 1: @@ -146,26 +158,26 @@ def update(self, *mapping, **bindings): if self._finalized and any(k not in self for k in bindings): raise AttributeError("adding new bindings to a finalized environment is not allowed") return self._env.update(*mapping, **bindings) - def setdefault(self, k, *default): + def setdefault(self, k: str, *default: Any) -> Any: if self._finalized and k not in self: raise AttributeError(f"name {repr(k)} is not defined; adding new bindings to a finalized environment is not allowed") return self._env.setdefault(k, *default) # subscripting - def __getitem__(self, k): + def __getitem__(self, k: str) -> Any: return getattr(self, k) - def __setitem__(self, k, v): + def __setitem__(self, k: str, v: Any) -> None: setattr(self, k, v) - def __delitem__(self, k): + def __delitem__(self, k: str) -> None: delattr(self, k) # context manager - def __enter__(self): + def __enter__(self) -> "env": return self - def __exit__(self, exctype, excvalue, traceback): + def __exit__(self, exctype: type[BaseException] | None, excvalue: BaseException | None, traceback: TracebackType | None) -> None: self._env.clear() # on context exit, clear even if we are a finalized env # pretty-printing @@ -175,7 +187,7 @@ def __repr__(self): # pragma: no cover return f"" # other - def set(self, name, value): + def set(self, name: str, value: Any) -> Any: """Convenience method to allow assignment in expression contexts. Like Scheme's set! function. Only rebinding is allowed. @@ -187,11 +199,11 @@ def set(self, name, value): return self._set(name, value) # for co-operation with the do[] macro: internal function with no already-defined check. - def _set(self, name, value): + def _set(self, name: str, value: Any) -> Any: setattr(self, name, value) return value # for convenience - def __lshift__(self, arg): + def __lshift__(self, arg: tuple[str, Any]) -> "env": """Alternative syntax for assignment. ``e << ("x", 42)`` is otherwise the same as ``e.set("x", 42)``, except @@ -203,7 +215,7 @@ def __lshift__(self, arg): self.set(name, value) return self - def finalize(self): + def finalize(self) -> "env": """Finalize environment. This stops the instance from accepting any more new bindings, @@ -211,8 +223,16 @@ def finalize(self): Existing bindings can still be given new values even in a finalized environment. + + **Changed in v2.3.1**: returns `self`, so that it can be chained:: + + e = env(x=42).finalize() + + Earlier versions returned `None`. """ - self._finalized = True + # Bypass our own `__setattr__`, which would refuse `_finalized` as a reserved name. + object.__setattr__(self, "_finalized", True) + return self # For rebind syntax: "e.foo << newval" --> "e.foo.__lshift__(newval)", # so foo.__lshift__() must be set up to rebind e.foo. diff --git a/unpythonic/environ.py b/unpythonic/environ.py new file mode 100644 index 00000000..60a78eb1 --- /dev/null +++ b/unpythonic/environ.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +"""Utilities for working with OS environment variables.""" + +__all__ = ["override"] + +from collections.abc import Iterator +import contextlib +import os +import threading + +_lock = threading.RLock() + +@contextlib.contextmanager +def override(**bindings: str) -> Iterator[None]: + """Context manager: temporarily override OS environment variable(s). + + When the ``with`` block exits, the previous state of the environment + is restored. If a variable was unset before entry, it is removed + again on exit. + + Thread-safe: concurrent overrides from different threads are serialised + by a module-level ``RLock``, so only one set of overrides is active at + a time. Same-thread nesting is supported (the lock is reentrant). + + Example:: + + import os + from unpythonic import environ_override + os.environ["MY_VAR"] = "original" + with environ_override(MY_VAR="temporary", OTHER="added"): + print(os.environ["MY_VAR"]) # "temporary" + print(os.environ["OTHER"]) # "added" + print(os.environ["MY_VAR"]) # "original" + print("OTHER" in os.environ) # False + """ + with _lock: + old = {k: os.environ[k] for k in bindings if k in os.environ} + try: + os.environ.update(bindings) + yield + finally: + for k in bindings: + if k in old: + os.environ[k] = old[k] + else: + os.environ.pop(k, None) diff --git a/unpythonic/excutil.py b/unpythonic/excutil.py index a09c1ec6..03b8432c 100644 --- a/unpythonic/excutil.py +++ b/unpythonic/excutil.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- """Exception-related utilities.""" -__all__ = ["raisef", "tryf", +__all__ = ["raisef", "tryf", "withf", "equip_with_traceback", "async_raise", "reraise_in", "reraise"] -from contextlib import contextmanager +from collections.abc import Callable, Iterable, Iterator, Mapping +from contextlib import contextmanager, ExitStack import sys import threading +from typing import Any, NoReturn from types import TracebackType # For async_raise only. Note `ctypes.pythonapi` is not an actual module; @@ -24,10 +26,33 @@ ctypes = None PyThreadState_SetAsyncExc = None +# Key: exception type or tuple of exception types (matched with isinstance). +# Value: exception type or instance to raise as the replacement. +ExcSpec = type[BaseException] | tuple[type[BaseException], ...] +ExcMapping = Mapping[ExcSpec, type[BaseException] | BaseException] + from .arity import arity_includes, UnknownArity -def raisef(exc, *, cause=None): +def _accepts_arity(f: Callable, n: int) -> bool: + """Whether `f` can be called with `n` positional arguments. + + Used by `tryf` and `withf` to dispatch between an n-arg call form + (passing values to the user-supplied callback) and a 0-arg thunk + form (callback ignores values). + + On `UnknownArity` (e.g. an uninspectable C callable), returns + `True` — we need to choose *something* as the default, and the + n-arg form is the more flexible choice. Any real mismatch + surfaces as a `TypeError` at the call. + """ + try: + return arity_includes(f, n) + except UnknownArity: # well, we tried! # pragma: no cover + return True + + +def raisef(exc: BaseException | type[BaseException], *, cause: BaseException | None = None) -> NoReturn: """``raise`` as a function, to make it possible for lambdas to raise exceptions. Example:: @@ -54,7 +79,7 @@ def raisef(exc, *, cause=None): else: raise exc -def tryf(body, *handlers, elsef=None, finallyf=None): +def tryf(body: Callable[[], Any], *handlers: tuple, elsef: Callable[[], Any] | None = None, finallyf: Callable[[], Any] | None = None) -> Any: """``try``/``except``/``finally`` as a function. This allows lambdas to handle exceptions. @@ -91,15 +116,7 @@ def tryf(body, *handlers, elsef=None, finallyf=None): you can also just create an ``env`` at an appropriate point, and store them there. """ - def accepts_arg(f): - try: - if arity_includes(f, 1): - return True - except UnknownArity: # pragma: no cover - return True # just assume it - return False - - def isexceptiontype(exc): + def isexceptiontype(exc: Any) -> bool: try: if issubclass(exc, BaseException): return True @@ -129,13 +146,13 @@ def isexceptiontype(exc): if isinstance(excspec, tuple): # tuple of exception types # this is safe, exctype is always a class at this point. if any(issubclass(exctype, t) for t in excspec): - if accepts_arg(handler): + if _accepts_arity(handler, 1): return handler(exception) else: return handler() else: # single exception type if issubclass(exctype, excspec): - if accepts_arg(handler): + if _accepts_arity(handler, 1): return handler(exception) else: return handler() @@ -147,7 +164,56 @@ def isexceptiontype(exc): if finallyf is not None: finallyf() -def equip_with_traceback(exc, stacklevel=1): # Python 3.7+ +def withf(cms: Any, body: Callable[..., Any]) -> Any: + """``with`` as a function. + + This allows lambdas to use context managers. + + ``cms`` is either a single context manager, or a sequence of context + managers ``(cm1, cm2, ...)``. A sequence is entered left-to-right and + exited in reverse, analogously to ``with cm1, cm2, ...:``. + + A bare context manager (one whose runtime type defines ``__enter__``) + is treated as a 1-element sequence; the explicit tuple is optional in + that case. + + ``body`` represents the body of the ``with`` block. The arity is + auto-detected: + + - If ``body`` accepts as many positional arguments as there are + context managers, it receives the as-values, in order. This is + the analogue of ``with cm1 as x, cm2 as y: body(x, y)``. + + - If ``body`` is a thunk (takes no positional arguments), the + as-values are discarded. Useful for context managers used purely + for their side effects, such as ``with lock: ...``. + + The return value of ``withf`` is whatever ``body`` returns. (Lispily, + `with` is an expression here, even though Python's statement form + is value-less.) + + Exceptions raised inside ``body`` are passed to the context manager's + ``__exit__`` as usual; if not suppressed there, they propagate out of + ``withf``. + """ + if hasattr(type(cms), "__enter__"): + cms = (cms,) + elif not isinstance(cms, Iterable): + raise TypeError(f"cms must be a context manager or an iterable of context managers, got {type(cms)} with value {repr(cms)}") + else: + cms = tuple(cms) + for cm in cms: + if not hasattr(type(cm), "__enter__"): + raise TypeError(f"Each item in cms must be a context manager, got {type(cm)} with value {repr(cm)}") + + n = len(cms) + with ExitStack() as stack: + values = [stack.enter_context(cm) for cm in cms] + if _accepts_arity(body, n): + return body(*values) + return body() + +def equip_with_traceback(exc: BaseException, stacklevel: int = 1) -> BaseException: # Python 3.7+ """Given an exception instance exc, equip it with a traceback. `stacklevel` is the starting depth below the top of the call stack, @@ -166,10 +232,6 @@ def equip_with_traceback(exc, stacklevel=1): # Python 3.7+ The return value is `exc`, with its traceback set to the produced traceback. - Python 3.7 and later only. - - When not supported, raises `NotImplementedError`. - This is useful mainly in special cases, where `raise` cannot be used for some reason, and a manually created exception instance needs a traceback. (The `signal` function in the conditions-and-restarts system uses this.) @@ -207,27 +269,24 @@ def equip_with_traceback(exc, stacklevel=1): # Python 3.7+ break # Python 3.7+ allows creating `types.TracebackType` objects in Python code. - try: - tracebacks = [] - nxt = None # tb_next should point toward the level where the exception occurred. - for frame in frames: # walk from top of call stack toward the root - tb = TracebackType(nxt, frame, frame.f_lasti, frame.f_lineno) - tracebacks.append(tb) - nxt = tb - if tracebacks: - tb = tracebacks[-1] # root level - else: - tb = None - except TypeError as err: # Python 3.6 or earlier - raise NotImplementedError("Need Python 3.7 or later to create traceback objects") from err - return exc.with_traceback(tb) # Python 3.7+ + tracebacks = [] + nxt = None # tb_next should point toward the level where the exception occurred. + for frame in frames: # walk from top of call stack toward the root + tb = TracebackType(nxt, frame, frame.f_lasti, frame.f_lineno) + tracebacks.append(tb) + nxt = tb + if tracebacks: + tb = tracebacks[-1] # root level + else: + tb = None + return exc.with_traceback(tb) # TODO: To reduce the risk of spaghetti user code, we could require a non-main thread's entrypoint to declare # via a decorator that it's willing to accept asynchronous exceptions, and check that mark here, making this # mechanism strictly opt-in. The decorator could inject an `asyncexc_ok` attribute to the Thread object; # that's enough to prevent accidental misuse. # OTOH, having no such mechanism is the simpler design. -def async_raise(thread_obj, exception): +def async_raise(thread_obj: threading.Thread, exception: type[BaseException] | BaseException) -> None: """Raise an exception in another thread. thread_obj: `threading.Thread` object @@ -323,7 +382,7 @@ def async_raise(thread_obj, exception): PyThreadState_SetAsyncExc(ctypes.c_long(target_tid), ctypes.c_long(0)) raise SystemError("PyThreadState_SetAsyncExc failed, broke the interpreter state.") -def reraise_in(body, mapping): +def reraise_in(body: Callable[[], Any], mapping: ExcMapping) -> Any: """Remap exception types in an expression. This allows conveniently converting library exceptions to application @@ -363,7 +422,7 @@ def reraise_in(body, mapping): _reraise_handler(mapping, libraryexc) @contextmanager -def reraise(mapping): +def reraise(mapping: ExcMapping) -> Iterator[None]: """Remap exception types. Context manager. This allows conveniently converting library exceptions to application @@ -400,7 +459,7 @@ def reraise(mapping): except BaseException as libraryexc: _reraise_handler(mapping, libraryexc) -def _reraise_handler(mapping, libraryexc): +def _reraise_handler(mapping: ExcMapping, libraryexc: BaseException) -> NoReturn: """Remap an exception instance to another exception type. `mapping`: dict-like, `{LibraryExc0: ApplicationExc0, ...}` diff --git a/unpythonic/fix.py b/unpythonic/fix.py index 92066ae9..ed0a9b45 100644 --- a/unpythonic/fix.py +++ b/unpythonic/fix.py @@ -52,9 +52,11 @@ __all__ = ["fix", "fixtco"] +from collections.abc import Callable import typing # we use typing.NoReturn as a special value at runtime import threading from functools import wraps +from typing import Any from .fun import const, memoize from .tco import trampolined, _jump @@ -63,7 +65,7 @@ from .regutil import register_decorator _L = threading.local() -def _get_threadlocals(): +def _get_threadlocals() -> "env": if not hasattr(_L, "_data"): # TCO info forms a stack to support nested TCO chains (during a # TCO chain, regular call, which then calls another TCO chain). @@ -71,7 +73,7 @@ def _get_threadlocals(): return _L._data @register_decorator(priority=40, istco=False) # same priority as @fixtco -def fix(bottom=typing.NoReturn, memo=True): +def fix(bottom: Any = typing.NoReturn, memo: bool = True) -> Callable: """Break recursion cycles. Parametric decorator. This is sometimes useful for recursive pattern-matching definitions. For an @@ -147,7 +149,7 @@ def f(...): return _fix(bottom, memo, tco=False) @register_decorator(priority=40, istco=True) # same priority as @trampolined -def fixtco(bottom=typing.NoReturn, memo=True): +def fixtco(bottom: Any = typing.NoReturn, memo: bool = True) -> Callable: """TCO-enabled version of @fix. On top of performing the duties of `fix`, this parametric decorator applies @@ -206,7 +208,7 @@ def f(k): # OTOH, maybe that's not needed, since by definition, a decorator overwrites the name. # So returning the decorated version would be just fine. # -def _fix(bottom=typing.NoReturn, memo=True, *, tco): +def _fix(bottom: Any = typing.NoReturn, memo: bool = True, *, tco: bool) -> Callable: # Being a class, typing.NoReturn is technically callable (to construct an # instance), but because it's an abstract class, the call raises TypeError. # We want to use the class itself as a data value, so we special-case it. diff --git a/unpythonic/fold.py b/unpythonic/fold.py index 3f6a0cb1..6d680214 100644 --- a/unpythonic/fold.py +++ b/unpythonic/fold.py @@ -18,17 +18,23 @@ "prod", "running_minmax", "minmax"] +from collections.abc import Callable, Iterable, Iterator from functools import partial from itertools import zip_longest from operator import mul +from typing import Any, TypeVar #from collections import deque +from .funutil import Values #from .it import first, last, rev from .it import last, rev +from .symbol import sym + +T = TypeVar('T') # Require at least one iterable to make this work seamlessly with curry. We take # this approach with any new function families the standard library doesn't provide. -def scanl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def scanl(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> Iterator[T]: """Scan (a.k.a. accumulate). Like ``itertools.accumulate``, but supports multiple input iterables. @@ -68,7 +74,7 @@ def scanl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): acc = proc(*(xs + (acc,))) yield acc -def scanr(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def scanr(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> Iterator[T]: """Dual of scanl; scan from the right. Example:: @@ -159,7 +165,7 @@ def append_tuple(a, b, acc): # yield from subgen # sustain the chain # return _scanr_recurser() -def scanl1(proc, iterable, init=None): +def scanl1(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> Iterator[T]: """scanl for a single iterable, with optional init. If ``init is None``, use the first element from the iterable. @@ -180,20 +186,20 @@ def scanl1(proc, iterable, init=None): try: init = next(it) except StopIteration: - def empty_iterable(): + def empty_iterable() -> Iterator[T]: yield from () return empty_iterable() return scanl(proc, init, it) -_uselast = object() # sentinel -def scanr1(proc, iterable, init=None): +_uselast = sym("_uselast") # sentinel +def scanr1(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> Iterator[T]: """Dual of scanl1. If ``init is None``, use the last element from the iterable. """ return scanr(proc, _uselast if init is None else init, iterable) -def foldl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def foldl(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> T | None: """Racket-like foldl that supports multiple input iterables. At least one iterable (``iterable0``) is required. More are optional. @@ -210,14 +216,14 @@ def foldl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): return last(scanl(proc, init, iterable0, *iterables, longest=longest, fillvalue=fillvalue)) -def foldr(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def foldr(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> T | None: """Dual of foldl; fold from the right.""" # if using the haskelly result ordering in scanr, then first(...); # if ordering results as they are computed, then last(...) return last(scanr(proc, init, iterable0, *iterables, longest=longest, fillvalue=fillvalue)) -def reducel(proc, iterable, init=None): +def reducel(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> T | None: """Foldl for a single iterable, with optional init. If ``init is None``, use the first element from the iterable. @@ -225,7 +231,7 @@ def reducel(proc, iterable, init=None): Like ``functools.reduce``, but uses ``proc(elt, acc)`` like Racket.""" return last(scanl1(proc, iterable, init)) -def reducer(proc, iterable, init=None): +def reducer(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> T | None: """Dual of reducel. If ``init is None``, use the last element from the iterable. @@ -234,7 +240,7 @@ def reducer(proc, iterable, init=None): # if ordering results as they are computed, then last(...) return last(scanr1(proc, iterable, init)) -def rscanl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def rscanl(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> Iterator[T]: """Reverse each input, then scanl. For multiple input iterables, the notion of *corresponding elements* @@ -245,11 +251,11 @@ def rscanl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): return scanl(proc, init, rev(iterable0), *(rev(s) for s in iterables), longest=longest, fillvalue=fillvalue) -def rscanl1(proc, iterable, init=None): +def rscanl1(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> Iterator[T]: """Reverse the input, then scanl1.""" return scanl1(proc, rev(iterable), init) -def rfoldl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): +def rfoldl(proc: Callable[..., T], init: T, iterable0: Iterable, *iterables: Iterable, longest: bool = False, fillvalue: Any = None) -> T | None: """Reverse each input, then foldl. For multiple input iterables, the notion of *corresponding elements* @@ -260,11 +266,11 @@ def rfoldl(proc, init, iterable0, *iterables, longest=False, fillvalue=None): return foldl(proc, init, rev(iterable0), *(rev(s) for s in iterables), longest=longest, fillvalue=fillvalue) -def rreducel(proc, iterable, init=None): +def rreducel(proc: Callable[..., T], iterable: Iterable[T], init: T | None = None) -> T | None: """Reverse the input, then reducel.""" return reducel(proc, rev(iterable), init) -def unfold1(proc, init): +def unfold1(proc: Callable[[T], tuple[Any, T] | None], init: T) -> Iterator: """Generate a sequence corecursively. The counterpart of foldl. Returns a generator. @@ -297,29 +303,34 @@ def step2(k): # x0, x0 + 2, x0 + 4, ... value, state = result yield value -def unfold(proc, *inits): +def unfold(proc: Callable[..., Values | None], *inits: Any, **kwinits: Any) -> Iterator: """Like unfold1, but for n-in-(1+n)-out proc. The current state is unpacked to the argument list of ``proc``. - It must return either ``(value, *newstates)``, or ``None`` to signify - that the sequence ends. + It must return either a ``Values`` object where the first positional + return value is the ``value`` to be yielded at this iteration, and + anything else is state to be unpacked to the args/kwargs of ``proc`` + at the next iteration; or a bare ``None`` to signify that the sequence ends. If your state is something simple such as one number, see ``unfold1``. Example:: def fibo(a, b): - return (a, b, a + b) + return Values(a, a=b, b=a + b) assert (tuple(take(10, unfold(fibo, 1, 1))) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55)) """ - states = inits + state = Values(*inits, **kwinits) while True: - result = proc(*states) + result = proc(*state.rets, **state.kwrets) if result is None: break - value, *states = result + if not isinstance(result, Values): + raise TypeError(f"Expected `None` (to terminate) or a `Values` (to continue), got {type(result)} with value {repr(result)}") + value, *rets = result.rets # unpack the first positional return value, keep the rest + state = Values(*rets, **result.kwrets) yield value # This is **not** how to make a right map; the result is exactly the same @@ -353,14 +364,14 @@ def fibo(a, b): # return args # return mapr(identity, *iterables) -def prod(iterable, start=1): +def prod(iterable: Iterable[int | float], start: int | float = 1) -> int | float: """Like the builtin sum, but compute the product. This is a fold operation. """ return reducel(mul, iterable, init=start) -def running_minmax(iterable): +def running_minmax(iterable: Iterable[T]) -> Iterator[tuple[T, T]]: """Return a generator extracting a running `(min, max)` from `iterable`. The iterable is iterated just once. @@ -377,10 +388,10 @@ def running_minmax(iterable): try: first = next(it) except StopIteration: # behave like `unpack` and `window` on empty input - def empty_iterable(): + def empty_iterable() -> Iterator[tuple[T, T]]: yield from () return empty_iterable() - def mm(elt, acc): + def mm(elt: T, acc: tuple[T, T]) -> tuple[T, T]: a, b = acc if elt < a: a = elt @@ -389,7 +400,7 @@ def mm(elt, acc): return a, b return scanl(mm, (first, first), it) -def minmax(iterable): +def minmax(iterable: Iterable[T]) -> tuple[T, T] | tuple[None, None]: """Extract `(min, max)` from `iterable`, iterating it just once. If `iterable` is empty, return `(None, None)`. diff --git a/unpythonic/fploop.py b/unpythonic/fploop.py index 84692c4a..185e9bf1 100644 --- a/unpythonic/fploop.py +++ b/unpythonic/fploop.py @@ -26,7 +26,9 @@ def iter(loop, i=0): __all__ = ["looped", "looped_over", "breakably_looped", "breakably_looped_over"] +from collections.abc import Callable, Iterable from functools import partial +from typing import Any from .ec import call_ec from .arity import arity_includes, UnknownArity @@ -34,7 +36,7 @@ def iter(loop, i=0): from .regutil import register_decorator @register_decorator(priority=50, istco=True) -def looped(body): +def looped(body: Callable) -> Any: """Decorator to make a functional loop and run it immediately. This essentially chains @trampolined and @call, with some extra magic. @@ -113,7 +115,7 @@ def s(acc, i): """ # The magic parameter that, when called, inserts itself into the # positional args of the jump target. - def loop(*args, **kwargs): + def loop(*args: Any, **kwargs: Any) -> _jump: # Pass the original non-trampolined body; it is sufficient # to have one trampoline at the top level. return _jump(body, (loop,) + args, kwargs) # already packed args, inst directly. @@ -126,7 +128,7 @@ def loop(*args, **kwargs): return tb(loop) # like @call, run the (now trampolined) body. @register_decorator(priority=50, istco=True) -def breakably_looped(body): +def breakably_looped(body: Callable) -> Any: """Functionally loop over an iterable. Like ``@looped``, but the client now gets two positionally passed magic parameters: @@ -163,8 +165,8 @@ def result(loop, brk, acc=0, i=0): print(result) """ @call_ec - def result(brk): - def loop(*args, **kwargs): + def result(brk: Callable) -> Any: + def loop(*args: Any, **kwargs: Any) -> _jump: return _jump(body, (loop, brk) + args, kwargs) # already packed args, inst directly. try: if not arity_includes(body, 2): @@ -176,7 +178,7 @@ def loop(*args, **kwargs): return result @register_decorator(priority=50, istco=True) -def looped_over(iterable, acc=None): # decorator factory +def looped_over(iterable: Iterable, acc: Any = None) -> Callable[[Callable], Any]: # decorator factory """Functionally loop over an iterable. Like ``@looped``, but the client now gets three positionally passed magic parameters: @@ -243,12 +245,12 @@ def out(loop, x, acc): assert s == 45 """ # Decorator that plays the role of @call, with "iterable" bound by closure. - def run(body): + def run(body: Callable) -> Any: it = iter(iterable) oldacc = acc # keep track of the last seen value for acc # The magic parameter that, when called, inserts the implicit parameters # into the positional args of the jump target. Runs between iterations. - def loop(*args, **kwargs): + def loop(*args: Any, **kwargs: Any) -> Any: nonlocal oldacc newacc = args[0] if len(args) else oldacc oldacc = newacc @@ -272,7 +274,7 @@ def loop(*args, **kwargs): return run @register_decorator(priority=50, istco=True) -def breakably_looped_over(iterable, acc=None): # decorator factory +def breakably_looped_over(iterable: Iterable, acc: Any = None) -> Callable[[Callable], Any]: # decorator factory """Functionally loop over an iterable. Like ``@looped_over``, but with *continue* and *break* functionality. @@ -313,12 +315,12 @@ def s(loop, x, acc, cnt, brk): return loop(acc + x) assert s == 35 """ - def run(body): + def run(body: Callable) -> Any: it = iter(iterable) @call_ec - def result(brk): + def result(brk: Callable) -> Any: oldacc = acc - def loop(*args, **kwargs): + def loop(*args: Any, **kwargs: Any) -> Any: nonlocal oldacc newacc = args[0] if len(args) else oldacc oldacc = newacc diff --git a/unpythonic/fun.py b/unpythonic/fun.py index e1c0c70b..01a54e90 100644 --- a/unpythonic/fun.py +++ b/unpythonic/fun.py @@ -7,7 +7,8 @@ Memoize is typical FP (Racket has it in mischief), and flip comes from Haskell. """ -__all__ = ["memoize", "curry", "iscurried", +__all__ = ["memoize", + "partial", "curry", "iscurried", "flip", "rotate", "apply", "identity", "const", "notf", "andf", "orf", @@ -18,9 +19,14 @@ "withself"] from collections import namedtuple +from collections.abc import Callable from functools import wraps, partial as functools_partial from inspect import signature -from typing import get_type_hints +from threading import RLock +from typing import Any, TypeVar, get_type_hints + +F = TypeVar('F', bound=Callable) +T = TypeVar('T') from .arity import (_resolve_bindings, tuplify_bindings, _bind) from .fold import reducel @@ -32,13 +38,25 @@ from .regutil import register_decorator from .symbol import sym -# we use @passthrough_lazy_args (and handle possible lazy args) to support unpythonic.syntax.lazify. -from .lazyutil import passthrough_lazy_args, islazy, force, force1, maybe_force_args +# We use `@passthrough_lazy_args` and `maybe_force_args` to support unpythonic.syntax.lazify. +from .lazyutil import passthrough_lazy_args, islazy, force, maybe_force_args + +# -------------------------------------------------------------------------------- + +#def memoize_simple(f): # essential idea, without exception handling or thread-safety. +# memo = {} +# @wraps(f) +# def memoized(*args, **kwargs): +# k = tuplify_bindings(resolve_bindings(f, *args, **kwargs)) +# if k not in memo: +# memo[k] = f(*args, **kwargs) +# return memo[k] +# return memoized _success = sym("_success") _fail = sym("_fail") @register_decorator(priority=10) -def memoize(f): +def memoize(f: F) -> F: """Decorator: memoize the function f. All of the args and kwargs of ``f`` must be hashable. @@ -49,18 +67,35 @@ def memoize(f): **CAUTION**: ``f`` must be pure (no side effects, no internal state preserved between invocations) for this to make any sense. + + Beginning with v0.15.0, `memoize` is thread-safe even when the same memoized + function instance is called concurrently from multiple threads. Exactly one + thread will compute the result. If `f` is recursive, the thread that acquired + the lock is the one that is allowed to recurse into the memoized `f`. """ + # One lock per use site of `memoize`. We use an `RLock` to allow recursive calls + # to the memoized `f` in the thread that acquired the lock. + lock = RLock() memo = {} @wraps(f) def memoized(*args, **kwargs): k = tuplify_bindings(_resolve_bindings(f, args, kwargs, _partial=False)) - if k not in memo: - try: - result = (_success, maybe_force_args(f, *args, **kwargs)) - except BaseException as err: - result = (_fail, err) - memo[k] = result # should yell separately if k is not a valid key - kind, value = memo[k] + try: # EAFP to eliminate TOCTTOU. + kind, value = memo[k] + except KeyError: + # But we still need to be careful to avoid race conditions. + with lock: + if k not in memo: + # We were the first thread to acquire the lock. + try: + result = (_success, maybe_force_args(f, *args, **kwargs)) + except BaseException as err: + result = (_fail, err) + memo[k] = result # should yell separately if k is not a valid key + else: + # Some other thread acquired the lock before us. + pass + kind, value = memo[k] if kind is _fail: raise value return value @@ -68,15 +103,7 @@ def memoized(*args, **kwargs): memoized = passthrough_lazy_args(memoized) return memoized -#def memoize_simple(f): # essential idea, without exception handling -# memo = {} -# @wraps(f) -# def memoized(*args, **kwargs): -# k = tuplify_bindings(resolve_bindings(f, *args, **kwargs)) -# if k not in memo: -# memo[k] = f(*args, **kwargs) -# return memo[k] -# return memoized +# -------------------------------------------------------------------------------- # Parameter naming is consistent with `functools.partial`. # @@ -85,7 +112,7 @@ def memoized(*args, **kwargs): # latest application winning. We must resist the temptation to override that behavior here, # because there are other places in the stdlib, particularly `inspect._signature_get_partial` # (as of Python 3.8), that expect the standard semantics. -def partial(func, *args, **kwargs): +def partial(func: Callable, *args: Any, **kwargs: Any) -> Callable: """Type-checking `functools.partial`. This is a wrapper that type-checks the arguments against the type annotations @@ -130,7 +157,7 @@ def partial(func, *args, **kwargs): _extract_self_or_cls(thecallable, args)), _partial=True) - else: # Not `@generic` or `@typed`; just a function that has type annotations. + else: # Not `@generic` or `@typed`; just a function that might have type annotations. # It's not very unpythonic-ic to provide this since we already have `@typed` for this use case, # but it's much more pythonic, if the type-checking `partial` works properly for code that does # not opt in to `unpythonic`'s multiple-dispatch subsystem. @@ -153,20 +180,22 @@ def partial(func, *args, **kwargs): # `functools.partial` already handles chaining partial applications, so send only the new args/kwargs to it. return functools_partial(func, *args, **kwargs) -make_dynvar(curry_context=[]) -@passthrough_lazy_args -def _currycall(f, *args, **kwargs): - """Co-operate with unpythonic.syntax.curry. +# -------------------------------------------------------------------------------- - In a ``with autocurry`` block, we need to call `f` also when ``f()`` has - transformed to ``curry(f)``, but definitions can be curried as usual. +#def curry_simple(f): # essential idea, without any extra features +# min_arity, _ = arities(f) +# @wraps(f) +# def curried(*args, **kwargs): +# if len(args) < min_arity: +# return curry(partial(f, *args, **kwargs)) +# return f(*args, **kwargs) +# return curried - Hence we provide this separate mode to curry-and-call even if no args. +make_dynvar(curry_context=[]) - This mode no-ops when ``f`` is not inspectable, instead of raising - an ``unpythonic.arity.UnknownArity`` exception. - """ - return curry(f, *args, _curry_force_call=True, _curry_allow_uninspectable=True, **kwargs) +def iscurried(f: Any) -> bool: + """Return whether f is a curried function.""" + return hasattr(f, "_is_curried_function") @register_decorator(priority=8) @passthrough_lazy_args @@ -276,15 +305,8 @@ def f(x, y): assert f(y=2)(x=1) == (1, 2) - However, it is possible that the algorithm isn't perfect, so there may be small semantic - differences to regular one-step function calls. If you find any, please file an issue, - so these can at the very least be documented; and if doable with reasonable effort, - preferably fixed. - - It is still an error if **named** arguments are left over for an outer curry context. - Treating this case would require generalizing return values so that functions could - return named outputs. See: - https://github.com/Technologicat/unpythonic/issues/32 + If you notice any semantic differences in parameter binding when using `curry`, when compared + to regular one-step function calls, please file an issue. """ f = force(f) # lazify support: we need the value of f # trivial case first: interaction with call_ec and other replace-def-with-value decorators @@ -304,117 +326,17 @@ def fallback(): # what to do when inspection fails return maybe_force_args(f, *args, **kwargs) return f - # Try to fail-fast with uninspectable builtins. - try: - signature(f) - except ValueError as err: # inspection failed in inspect.signature()? - msg = err.args[0] - if "no signature found" in msg: - return fallback() - raise - - # TODO: To make `curry` pay-as-you-go, look for opportunities to speed this up - # for non-`@generic` functions. Currently this more general `curry` for v0.15.0 - # (that handles kwargs correctly) can be even 50% slower than the more limited one - # (based on positional arity only) that was in v0.14.3. - - # actions - _call = sym("_call") - _call_with_passthrough = sym("_call_with_passthrough") - _keep_currying = sym("_keep_currying") - Analysis = namedtuple("Analysis", ["bound_arguments", "unbound_parameters", "extra_args", "extra_kwargs"]) - def analyze_parameter_bindings(f, args, kwargs): - # `functools.partial()` doesn't remove an already-set kwarg from the signature (as seen by - # `inspect.signature`), but `functools.partial` objects have a `keywords` attribute, which - # contains what we want. - # - # To support kwargs properly, we must compute argument bindings anyway, so we also use the - # `func` and `args` attributes. This allows us to compute the bindings of all arguments - # against the original function. - if isinstance(f, functools_partial): - function = f.func - collected_args = f.args + args - collected_kwargs = {**f.keywords, **kwargs} - else: - function = f - collected_args = args - collected_kwargs = kwargs - - def _bind_arguments(thecallable): - # For this check we look for a complete match, hence `_partial=False`. - bound_arguments, unbound_parameters, (extra_args, extra_kwargs) = _bind(signature(thecallable), - collected_args, - collected_kwargs, - partial=False) - return Analysis(bound_arguments, unbound_parameters, extra_args, extra_kwargs) - - # `@generic` functions have several call signatures, so we must aggregate the results - # in a sensible way. For non-generics, there's just one call signature. - if not isgeneric(function): - # For non-generics, the curry-time type check occurs when we later call `partial`, - # so we don't need to do that here. We just compute the bindings of arguments to parameters. - analysis = _bind_arguments(function) - if not analysis.unbound_parameters and not analysis.extra_args and not analysis.extra_kwargs: - return _call, analysis - elif not analysis.unbound_parameters and (analysis.extra_args or analysis.extra_kwargs): - return _call_with_passthrough, analysis - assert analysis.unbound_parameters - return _keep_currying, analysis - - # Curry resolver for `@generic`/`@typed` (generic functions, multimethods, multiple dispatch). - # - # Iterate over multimethods, once per step: - # - # 1. If there is an exact match (all parameters bound, type check passes, no extra - # `args`/`kwargs`), call it. - # 2. If there is a complete match (all parameters bound, type check passes), but - # with extra `args`/`kwargs` (that cannot be accepted by the call signature), - # call it, arranging passthrough for the extra `args`/`kwargs`. - # 3. If there is at least one partial match (type check passes for bound arguments, - # unbound parameters remain), keep currying. In this case extra `args`/`kwargs`, - # if any, do not matter. This will fall into case 1 or 2 above after we get - # additional `args`/`kwargs` to complete a match. - # - # If none of the above match, we know at least one parameter got a binding - # that fails the type check. Raise `TypeError`. - # - # In steps 1 and 2, we use the same lookup order as the multiple dispatcher does; - # the first matching multimethod wins. Actual dispatch is still done by the dispatcher; - # we only compute the bindings to determine which case above the call falls into. - # - # `@typed` is a special case of `@generic` with just one multimethod registered. - # The resulting behavior is the same as for a non-generic function, because the - # above algorithm reduces to that. - - # We can't use the public `list_methods` here, because on OOP methods, - # decorators live on the unbound method (raw function). Thus we must - # extract `self`/`cls` from the arguments of the call (for linked - # dispatcher lookup in the MRO). - multimethods = _list_multimethods(function, - _extract_self_or_cls(function, - collected_args)) - # Step 1: exact match - for thecallable, type_signature in multimethods: - analysis = _bind_arguments(thecallable) - if not analysis.unbound_parameters and not analysis.extra_args and not analysis.extra_kwargs: - if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): - return _call, analysis - # Step 2: complete match, with extra args/kwargs - for thecallable, type_signature in multimethods: - analysis = _bind_arguments(thecallable) - if not analysis.unbound_parameters and (analysis.extra_args or analysis.extra_kwargs): - if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): - return _call_with_passthrough, analysis - # Step 3: partial match - for thecallable, type_signature in multimethods: - analysis = _bind_arguments(thecallable) - if analysis.unbound_parameters: - if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): - return _keep_currying, analysis - # No matter which multimethod we pick, at least one parameter gets a binding - # that fails the type check. - _raise_multiple_dispatch_error(function, collected_args, collected_kwargs, - candidates=multimethods, _partial=True) + # Try to fail-fast with uninspectable builtins, even if no arguments were passed. + # (If we get arguments, there's no landmine, because calling the curried function + # will perform the signature analysis.) + if not (args or kwargs): + try: + signature(f) + except ValueError as err: # inspection failed in inspect.signature()? + msg = err.args[0] + if "no signature found" in msg: + return fallback() + raise @wraps(f) def curried(*args, **kwargs): @@ -423,8 +345,10 @@ def curried(*args, **kwargs): # In order to decide what to do when the curried function is called, we must first compute # the parameter bindings. All of `f`'s parameters must be bound (whether by position or by # name) before calling `f`. + # + # The parameter binding analysis result is needed for passthrough. try: - action, analysis = analyze_parameter_bindings(f, args, kwargs) + action, analysis = _decide_curry_action(f, args, kwargs) except ValueError as err: # inspection failed in inspect.signature()? msg = err.args[0] if "no signature found" in msg: @@ -460,7 +384,6 @@ def curried(*args, **kwargs): if now_result.rets: # `leftmost`, not `first`, for unambiguous stack traces. leftmost, *others = now_result.rets - leftmost = force1(leftmost) # Extra positional arguments (`later_args`) are passed through *on the right*. # Hence any further positional return values are inserted before them. @@ -482,7 +405,7 @@ def curried(*args, **kwargs): later_kwargs = {**later_kwargs, **now_result.kwrets} else: # The only return value is also the leftmost one. - leftmost = force1(now_result) + leftmost = now_result if callable(leftmost): pass else: @@ -532,20 +455,134 @@ def curried(*args, **kwargs): return maybe_force_args(curried, *args, **kwargs) return curried -def iscurried(f): - """Return whether f is a curried function.""" - return hasattr(f, "_is_curried_function") +@passthrough_lazy_args +def _currycall(f, *args, **kwargs): + """Co-operate with unpythonic.syntax.autocurry. -#def curry_simple(f): # essential idea, without any extra features -# min_arity, _ = arities(f) -# @wraps(f) -# def curried(*args, **kwargs): -# if len(args) < min_arity: -# return curry(partial(f, *args, **kwargs)) -# return f(*args, **kwargs) -# return curried + In a ``with autocurry`` block, we need to call `f` also when ``f()`` has + transformed to ``curry(f)``, but definitions can be curried as usual. + + Hence we provide this separate mode to curry-and-call even if no args. + + This mode no-ops when ``f`` is not inspectable, instead of raising + an ``unpythonic.arity.UnknownArity`` exception. + """ + return curry(f, *args, _curry_force_call=True, _curry_allow_uninspectable=True, **kwargs) + +# actions during currying +_call = sym("_call") +_call_with_passthrough = sym("_call_with_passthrough") +_keep_currying = sym("_keep_currying") + +_Analysis = namedtuple("_Analysis", ["bound_arguments", "unbound_parameters", "extra_args", "extra_kwargs"]) + +# For performance, it is important to have this function defined once at the top level +# of the module, instead of defining it as a closure each time `curry` is called. +def _decide_curry_action(f, args, kwargs): + """ Internal helper for `curry`. + + The `args` and `kwargs` are those added at this step of currying. + + We detect if `f` is a `functools.partial` object, and automatically extract + any previously supplied `args` and `kwargs` for analysis. + + Return value is `(action, analysis)`. See source code for details. + """ + # `functools.partial()` doesn't remove an already-set kwarg from the signature (as seen by + # `inspect.signature`), but `functools.partial` objects have a `keywords` attribute, which + # contains what we want. + # + # To support kwargs properly, we must compute argument bindings anyway, so we also use the + # `func` and `args` attributes. This allows us to compute the bindings of all arguments + # against the original function. + if isinstance(f, functools_partial): + function = f.func + collected_args = f.args + args + collected_kwargs = {**f.keywords, **kwargs} + else: + function = f + collected_args = args + collected_kwargs = kwargs -def flip(f): + def _bind_arguments(thecallable): + # For this check we look for a complete match, hence `_partial=False`. + bound_arguments, unbound_parameters, (extra_args, extra_kwargs) = _bind(signature(thecallable), + collected_args, + collected_kwargs, + partial=False) + return _Analysis(bound_arguments, unbound_parameters, extra_args, extra_kwargs) + + # `@generic` functions have several call signatures, so we must aggregate the results + # in a sensible way. For non-generics, there's just one call signature. + if not isgeneric(function): + # For non-generics, the curry-time type check occurs when we later call `partial`, + # so we don't need to do that here. We just compute the bindings of arguments to parameters. + analysis = _bind_arguments(function) + if not analysis.unbound_parameters and not analysis.extra_args and not analysis.extra_kwargs: + return _call, analysis + elif not analysis.unbound_parameters and (analysis.extra_args or analysis.extra_kwargs): + return _call_with_passthrough, analysis + assert analysis.unbound_parameters + return _keep_currying, analysis + + # Curry resolver for `@generic`/`@typed` (generic functions, multimethods, multiple dispatch). + # + # Iterate over multimethods, once per step: + # + # 1. If there is an exact match (all parameters bound, type check passes, no extra + # `args`/`kwargs`), call it. + # 2. If there is a complete match (all parameters bound, type check passes), but + # with extra `args`/`kwargs` (that cannot be accepted by the call signature), + # call it, arranging passthrough for the extra `args`/`kwargs`. + # 3. If there is at least one partial match (type check passes for bound arguments, + # unbound parameters remain), keep currying. In this case extra `args`/`kwargs`, + # if any, do not matter. This will fall into case 1 or 2 above after we get + # additional `args`/`kwargs` to complete a match. + # + # If none of the above match, we know at least one parameter got a binding + # that fails the type check. Raise `TypeError`. + # + # In steps 1 and 2, we use the same lookup order as the multiple dispatcher does; + # the first matching multimethod wins. Actual dispatch is still done by the dispatcher; + # we only compute the bindings to determine which case above the call falls into. + # + # `@typed` is a special case of `@generic` with just one multimethod registered. + # The resulting behavior is the same as for a non-generic function, because the + # above algorithm reduces to that. + + # We can't use the public `list_methods` here, because on OOP methods, + # decorators live on the unbound method (raw function). Thus we must + # extract `self`/`cls` from the arguments of the call (for linked + # dispatcher lookup in the MRO). + multimethods = _list_multimethods(function, + _extract_self_or_cls(function, + collected_args)) + # Step 1: exact match + for thecallable, type_signature in multimethods: + analysis = _bind_arguments(thecallable) + if not analysis.unbound_parameters and not analysis.extra_args and not analysis.extra_kwargs: + if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): + return _call, analysis + # Step 2: complete match, with extra args/kwargs + for thecallable, type_signature in multimethods: + analysis = _bind_arguments(thecallable) + if not analysis.unbound_parameters and (analysis.extra_args or analysis.extra_kwargs): + if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): + return _call_with_passthrough, analysis + # Step 3: partial match + for thecallable, type_signature in multimethods: + analysis = _bind_arguments(thecallable) + if analysis.unbound_parameters: + if not _get_argument_type_mismatches(type_signature, analysis.bound_arguments): + return _keep_currying, analysis + # No matter which multimethod we pick, at least one parameter gets a binding + # that fails the type check. + _raise_multiple_dispatch_error(function, collected_args, collected_kwargs, + candidates=multimethods, _partial=True) + +# -------------------------------------------------------------------------------- + +def flip(f: Callable[..., T]) -> Callable[..., T]: """Decorator: flip (reverse) the positional arguments of f.""" @wraps(f) def flipped(*args, **kwargs): @@ -554,7 +591,7 @@ def flipped(*args, **kwargs): flipped = passthrough_lazy_args(flipped) return flipped -def rotate(k): +def rotate(k: int) -> Callable[[Callable[..., T]], Callable[..., T]]: """Decorator (factory): cycle positional arg slots of f to the right by k places. Negative values cycle to the left. @@ -585,8 +622,10 @@ def rotated(*args, **kwargs): return rotated return rotate_k +# -------------------------------------------------------------------------------- + @passthrough_lazy_args -def apply(f, arg0, *more, **kwargs): +def apply(f: Callable[..., T], arg0: Any, *more: Any, **kwargs: Any) -> T: """Scheme/Racket-like apply. Not really needed since Python has *, but included for completeness. @@ -609,9 +648,11 @@ def apply(f, arg0, *more, **kwargs): lst = tuple(more[-1]) return maybe_force_args(f, *(args + lst), **kwargs) +# -------------------------------------------------------------------------------- + # Not marking this as lazy-aware works better with continuations (since this # is the default cont, and return values should be values, not lazy[]) -def identity(*args, **kwargs): +def identity(*args: Any, **kwargs: Any) -> Any: """Identity function. Accepts any args and kwargs, and returns them. @@ -640,7 +681,7 @@ def identity(*args, **kwargs): # In lazify, return values are always just values, so we have to force args # to compute the return value; as a shortcut, just don't mark this as lazy. -def const(*args, **kwargs): +def const(*args: Any, **kwargs: Any) -> Callable[..., Any]: """Constant function. Returns a function that accepts any arguments (also kwargs) @@ -673,13 +714,17 @@ def constant(*a, **kw): return ret return constant -def notf(f): # Racket: negate +# -------------------------------------------------------------------------------- + +def notf(f: Callable[..., Any]) -> Callable[..., bool]: # Racket: negate """Return a function that returns the logical not of the result of f. Examples:: assert notf(lambda x: 2*x)(3) is False assert notf(lambda x: 2*x)(0) is True + + In Racket, this is known as `negate`. """ def negated(*args, **kwargs): return not maybe_force_args(f, *args, **kwargs) @@ -687,7 +732,7 @@ def negated(*args, **kwargs): negated = passthrough_lazy_args(negated) return negated -def andf(*fs): # Racket: conjoin +def andf(*fs: Callable[..., Any]) -> Callable[..., Any]: # Racket: conjoin """Return a function that conjoins calls to fs with "and". Each function in ``fs`` is called with the same ``args`` and ``kwargs``, @@ -701,6 +746,8 @@ def andf(*fs): # Racket: conjoin assert andf(lambda x: isinstance(x, int), lambda x: x % 2 == 0)(42) is True assert andf(lambda x: isinstance(x, int), lambda x: x % 2 == 0)(43) is False + + In Racket, this is known as `conjoin`. """ @passthrough_lazy_args def conjoined(*args, **kwargs): @@ -712,7 +759,7 @@ def conjoined(*args, **kwargs): return b return conjoined -def orf(*fs): # Racket: disjoin +def orf(*fs: Callable[..., Any]) -> Callable[..., Any]: # Racket: disjoin """Return a function that disjoins calls to fs with "or". Each function in ``fs`` is called with the same ``args`` and ``kwargs``, @@ -728,6 +775,8 @@ def orf(*fs): # Racket: disjoin assert orf(isstr, iseven)(42) is True assert orf(isstr, iseven)("foo") is True assert orf(isstr, iseven)(None) is False # neither condition holds + + In Racket, this is known as `disjoin`. """ @passthrough_lazy_args def disjoined(*args, **kwargs): @@ -739,6 +788,8 @@ def disjoined(*args, **kwargs): return False return disjoined +# -------------------------------------------------------------------------------- + def _make_compose1(direction): """Make a function that composes functions from an iterable. @@ -930,8 +981,10 @@ def composelci(iterable): """Like composelc, but read the functions from an iterable.""" return composeli(map(curry, iterable)) +# -------------------------------------------------------------------------------- + # Helpers to insert one-in-one-out functions into multi-arg compose chains -def tokth(k, f): +def tokth(k: int, f: Callable) -> Callable[..., "Values"]: """Return a function to apply f to args[k], pass the rest through. The output is a `Values`. Named arguments are passed through as-is. @@ -958,7 +1011,7 @@ def apply_f_to_kth_arg(*args, **kwargs): apply_f_to_kth_arg = passthrough_lazy_args(apply_f_to_kth_arg) return apply_f_to_kth_arg -def to1st(f): +def to1st(f: Callable) -> Callable[..., "Values"]: """Return a function to apply f to first item in args, pass the rest through. Example:: @@ -971,15 +1024,15 @@ def mymap_one(f, sequence): """ return tokth(0, f) # this is just a partial() but we want to provide a docstring. -def to2nd(f): +def to2nd(f: Callable) -> Callable[..., "Values"]: """Return a function to apply f to second item in args, pass the rest through.""" return tokth(1, f) -def tolast(f): +def tolast(f: Callable) -> Callable[..., "Values"]: """Return a function to apply f to last item in args, pass the rest through.""" return tokth(-1, f) -def to(*specs): +def to(*specs: tuple[int, Callable]) -> Callable[..., "Values"]: """Return a function to apply f1, ..., fn to items in args, pass the rest through. The specs are processed sequentially in the given order (allowing also @@ -997,8 +1050,10 @@ def to(*specs): """ return composeli(tokth(k, f) for k, f in specs) +# -------------------------------------------------------------------------------- + @register_decorator(priority=80) -def withself(f): +def withself(f: Callable) -> Callable: """Decorator. Allow a lambda to refer to itself. This is essentially the Y combinator trick packaged as a decorator. diff --git a/unpythonic/funutil.py b/unpythonic/funutil.py index ccdf05c8..ce2e8340 100644 --- a/unpythonic/funutil.py +++ b/unpythonic/funutil.py @@ -13,11 +13,34 @@ # HACK: break dependency loop llist -> fun -> funutil -> collections -> llist _init_done = False frozendict = sym("frozendict") # doesn't matter what the value is, will be overwritten later -def _init_module(): # called by unpythonic.__init__ when otherwise done +def _init_module() -> None: # called by unpythonic.__init__ when otherwise done global frozendict, _init_done from .collections import frozendict _init_done = True +def _maybe_unpack_values(args, kwargs): + """Expand any `Values` in `args` (left-to-right, in place). + + Each `Values` encountered contributes its `rets` to the positional arguments + and its `kwrets` to the keyword arguments. Across multiple `Values` and the + caller's explicit `kwargs`, rightmost wins per unique keyword (explicit + `kwargs` are syntactically last, so they override). + + Used by `call` and `callwith`. Bails out cheaply if no `Values` is present. + """ + if not any(isinstance(a, Values) for a in args): + return args, kwargs + new_args = [] + new_kwargs = {} + for a in args: + if isinstance(a, Values): + new_args.extend(a.rets) + new_kwargs.update(a.kwrets) + else: + new_args.append(a) + new_kwargs.update(kwargs) + return tuple(new_args), new_kwargs + # Only the single-argument form (just f) of the "call" decorator is supported by unpythonic.syntax.util.sort_lambda_decorators. # # This is as it should be; if given any arguments beside f, the call doesn't conform @@ -91,7 +114,34 @@ def _(): Note that in the multi-break case, ``x`` and ``y`` are no longer in scope outside the block, since the block is a function. + + **Values unpacking**: + + Any ``Values`` in the positional arguments is unpacked in place, + left-to-right: its ``rets`` splice into the positional arguments, + its ``kwrets`` merge into the keyword arguments. Across multiple + ``Values`` and the explicit ``kwargs``, rightmost wins per unique + keyword name (explicit ``kwargs`` are syntactically last, so they + override). Mirrors the spread/merge semantics of Python's + ``[*a, *b, c]`` and ``{**a, **b}``. + + ``Values`` is the protocol unpythonic uses for multiple positional + and named return values; this lets you take a ``Values`` produced by + one function and apply it as the arguments to another. + + Examples:: + + v = Values(1, 2, x=3) + assert call(lambda a, b, x: (a, b, x), v) == (1, 2, 3) + + # Spread anywhere, mixed with regular args. + assert call(lambda a, b, c: (a, b, c), 1, Values(2, 3)) == (1, 2, 3) + + # Spread-and-override: kwrets contribute defaults, explicit kwargs win. + defaults = Values(timeout=30, retries=3) + # call(api, defaults, retries=5) → api(timeout=30, retries=5) """ + args, kwargs = _maybe_unpack_values(args, kwargs) # return f(*args, **kwargs) return maybe_force_args(force(f), *args, **kwargs) # support unpythonic.syntax.lazify @@ -191,7 +241,22 @@ def mul3(a, b, c): *Function application with $* in http://learnyouahaskell.com/higher-order-functions + + **Values unpacking**: + + Any ``Values`` in the positional arguments is unpacked when + ``callwith`` is invoked (so the closure captures already-expanded + args/kwargs). Same spread-in-place / rightmost-wins semantics as + ``call``; see its docstring. + + Example:: + + v = Values(2, 3) + def myadd(a, b): + return a + b + assert callwith(v)(myadd) == 5 """ + args, kwargs = _maybe_unpack_values(args, kwargs) def applyfrozenargsto(f): return maybe_force_args(force(f), *args, **kwargs) return applyfrozenargsto @@ -224,7 +289,7 @@ class Values: Accordingly, various parts of `unpythonic` that deal with function composition use the `Values` abstraction; particularly `curry`, and - the `compose` and `pipe` families. + the `compose` and `pipe` families, and the `with continuations` macro. **Behavior**: @@ -270,8 +335,8 @@ def g(): assert "x" in result # `in` looks in the named part assert result["x"] == 3 assert result.get("x", None) == 3 - assert result.get("y", None) == None - assert tuple(results.keys()) == ("x",) # also `values()`, `items()` + assert result.get("y", None) is None + assert tuple(result.keys()) == ("x",) # also `values()`, `items()` def h(): return Values(1, 2, x=3) @@ -360,7 +425,7 @@ def values(self): return self.kwrets.values() def get(self, k, default=None): """Dict-like `get` for the named part.""" - return self[k] if k in self else default + return self[k] if k in self else default # noqa: SIM401 -- this IS the .get() implementation # comparison def __eq__(self, other): diff --git a/unpythonic/fup.py b/unpythonic/fup.py index d8876104..1f28f01f 100644 --- a/unpythonic/fup.py +++ b/unpythonic/fup.py @@ -3,11 +3,15 @@ __all__ = ["fupdate"] +from collections.abc import Iterable, Sequence from copy import copy +from typing import Any, TypeVar from .collections import frozendict, ShadowedSequence -def fupdate(target, indices=None, values=None, **bindings): +T = TypeVar('T') + +def fupdate(target: T, indices: "int | slice | Sequence[int | slice] | None" = None, values: Any = None, **bindings: Any) -> T: """Return a functionally updated copy of a sequence or a mapping. The input can be mutable or immutable; it does not matter. @@ -105,7 +109,7 @@ def fupdate(target, indices=None, values=None, **bindings): if indices is not None and bindings: raise ValueError("Cannot use both indices and bindings.") if indices is not None: - def make_output(seq): + def make_output(seq: Iterable) -> T: cls = type(target) ctor = cls._make if hasattr(cls, "_make") else cls # namedtuple support gen = (x for x in seq) diff --git a/unpythonic/gmemo.py b/unpythonic/gmemo.py index 30607a5c..d767f475 100644 --- a/unpythonic/gmemo.py +++ b/unpythonic/gmemo.py @@ -6,14 +6,18 @@ __all__ = ["gmemoize", "imemoize", "fimemoize"] +from collections.abc import Callable, Generator, Iterable, Iterator from functools import wraps from threading import RLock +from typing import Any, TypeVar from .arity import resolve_bindings, tuplify_bindings from .regutil import register_decorator from .symbol import sym -def gmemoize(gfunc): +F = TypeVar('F', bound=Callable) + +def gmemoize(gfunc: F) -> F: """Decorator: produce memoized generator instances. Similar to ``itertools.tee``, but the whole sequence is kept in memory @@ -91,9 +95,9 @@ def some_evens(n): # drop n first terms See also ``imemoize``, ``fimemoize``. """ - memos = {} + memos: dict[tuple, tuple] = {} @wraps(gfunc) - def gmemoized(*args, **kwargs): + def gmemoized(*args: Any, **kwargs: Any) -> "_MemoizedGenerator": k = tuplify_bindings(resolve_bindings(gfunc, *args, **kwargs)) if k not in memos: # underlying generator instance, memo instance, lock instance @@ -105,16 +109,17 @@ def gmemoized(*args, **kwargs): _fail = sym("_fail") class _MemoizedGenerator: """Wrapper that manages one memoized sequence. Co-operates with gmemoize.""" - def __init__(self, g, memo, lock): + def __init__(self, g: Generator, memo: list, lock: RLock) -> None: self.g = g self.memo = memo # each instance for the same g gets the same memo self.lock = lock - self.j = 0 # current position in memo - def __repr__(self): + self.j: int = 0 # current position in memo + def __repr__(self) -> str: return f"<_MemoizedGenerator object {self.g.__name__} at 0x{id(self):x}>" - def __iter__(self): + # Support the `collections.abc.Iterable` API + def __iter__(self) -> "_MemoizedGenerator": return self - def __next__(self): + def __next__(self) -> Any: j = self.j memo = self.memo with self.lock: @@ -131,8 +136,30 @@ def __next__(self): if kind is _fail: raise value return value - -def imemoize(iterable): + # Support a subset of the `collections.abc.Sequence` API for already-computed items + def __len__(self) -> int: + return len(self.memo) + def __getitem__(self, k: int | slice) -> Any: + if not isinstance(k, (int, slice)): + raise TypeError(f"Expected an int or slice index, got {type(k)} with value {repr(k)}") + length = len(self.memo) + if isinstance(k, slice): + # For slices where at least one item raises an exception, we raise the + # exception that is encountered first when walking the slice. + lst = [] + for kind, value in self.memo[k]: + if kind is _fail: + raise value + lst.append(value) + return lst + if k >= length or k < -length: + raise IndexError(f"memoized generator index out of range; got {k}, with {len(self.memo)} items currently available") + kind, value = self.memo[k] + if kind is _fail: + raise value + return value + +def imemoize(iterable: Iterable) -> Callable: """Memoize an iterable. Return a gfunc with no parameters which, when called, returns a generator @@ -161,11 +188,13 @@ def imemoize(iterable): If you need to take arguments to create the iterable, see ``fimemoize``. """ - # The lambda is the gfunc; decorate it with gmemoize and return that. - return gmemoize(lambda: (yield from iterable)) + @gmemoize + def iterable_as_gfunc() -> Iterator: + yield from iterable + return iterable_as_gfunc @register_decorator(priority=10) -def fimemoize(ifactory): +def fimemoize(ifactory: F) -> F: """Like imemoize, but for cases where creating the iterable needs arguments. ``ifactory`` is a function, which takes any number of positional or keyword @@ -205,7 +234,7 @@ def some_evens(n): # gfunc! assert last(some_evens(25)) == last(some_evens(25)) """ @wraps(ifactory) - def gfunc(*args, **kwargs): + def gfunc(*args: Any, **kwargs: Any) -> Iterator: yield from ifactory(*args, **kwargs) return gmemoize(gfunc) # return gmemoize(lambda *a, **kw: (yield from ifactory(*a, **kw))) diff --git a/unpythonic/gtco.py b/unpythonic/gtco.py index b5cccf38..f5e2da25 100644 --- a/unpythonic/gtco.py +++ b/unpythonic/gtco.py @@ -3,10 +3,14 @@ __all__ = ["gtco", "gtrampolined"] +from collections.abc import Callable, Generator from functools import wraps from inspect import isgenerator +from typing import Any, TypeVar -def gtco(generator): +F = TypeVar('F', bound=Callable) + +def gtco(generator: Generator) -> Generator: """Low-level function: run a generator with TCO enabled. In the generator, use ``return`` to tail-chain to the next generator. @@ -34,7 +38,7 @@ def march(): except TypeError: return x # passthrough -def gtrampolined(gfunc): +def gtrampolined(gfunc: F) -> F: """Decorator for generator functions (i.e. definitions of generators). Decorating the definition avoids the need to use ``gtco`` at call time. @@ -49,16 +53,16 @@ def ones(): last(take(10000, ones())) # no crash """ @wraps(gfunc) - def trampolining_gfunc(*args, **kwargs): + def trampolining_gfunc(*args: Any, **kwargs: Any) -> "_TrampolinedGenerator": generator = gfunc(*args, **kwargs) return _TrampolinedGenerator(generator) # inject a trampoline return trampolining_gfunc class _TrampolinedGenerator: """Wrapper to inject the gtco() call to the generator g returned by gfunc.""" - def __init__(self, g): + def __init__(self, g: Generator) -> None: self.g = g - def __iter__(self): + def __iter__(self) -> Generator: return gtco(iter(self.g)) # start the trampoline # no __next__, because __iter__ redirects; # this wrapper is never actually iterated over. diff --git a/unpythonic/it.py b/unpythonic/it.py index 80e4da94..a6ff6bbd 100644 --- a/unpythonic/it.py +++ b/unpythonic/it.py @@ -23,20 +23,25 @@ "flatten", "flatten1", "flatten_in", "iterate", "iterate1", "partition", - "partition_int", "inn", "iindex", "find", "window", "chunked", - "within", "fixpoint", + "within", "interleave", "subset", "powerset", "allsame"] from builtins import map as stdlib_map +from collections.abc import Callable, Iterable, Iterator from operator import itemgetter from itertools import tee, islice, zip_longest, starmap, chain, filterfalse, groupby, takewhile from collections import deque +from typing import Any, TypeVar -def rev(iterable): +from .funutil import Values + +T = TypeVar('T') + +def rev(iterable: Iterable[T]) -> Iterable[T]: """Reverse an iterable. If a sequence, the return value is ``reversed(iterable)``. @@ -56,7 +61,7 @@ def rev(iterable): except TypeError: return reversed(tuple(iterable)) -def map(function, iterable0, *iterables): +def map(function: Callable[..., T], iterable0: Iterable, *iterables: Iterable) -> Iterator[T]: """Curry-friendly map. Thin wrapper around Python's builtin ``map``, making it mandatory to @@ -74,7 +79,7 @@ def map(function, iterable0, *iterables): # When completing an existing set of functions (map, zip, zip_longest), # consistency wins over curry-friendliness. -def map_longest(func, *iterables, fillvalue=None): +def map_longest(func: Callable[..., T], *iterables: Iterable, fillvalue: Any = None) -> Iterator[T]: """Like map, but terminate on the longest input. In the input to ``func``, missing elements (after end of shorter inputs) @@ -86,7 +91,7 @@ def map_longest(func, *iterables, fillvalue=None): # with the terminology used at the call site. yield from starmap(func, zip_longest(*iterables, fillvalue=fillvalue)) -def rmap(func, *iterables): +def rmap(func: Callable[..., T], *iterables: Iterable) -> Iterator[T]: """Like map, but from the right. For multiple inputs with different lengths, ``rmap`` syncs the **right** ends. @@ -111,7 +116,7 @@ def rmap(func, *iterables): """ yield from map(func, *(rev(s) for s in iterables)) -def rzip(*iterables): +def rzip(*iterables: Iterable[T]) -> Iterator[tuple[T, ...]]: """Like zip, but from the right. For multiple inputs with different lengths, ``rzip`` syncs the **right** ends. @@ -134,15 +139,15 @@ def rzip(*iterables): """ yield from zip(*(rev(s) for s in iterables)) -def rmap_longest(func, *iterables, fillvalue=None): +def rmap_longest(func: Callable[..., T], *iterables: Iterable, fillvalue: Any = None) -> Iterator[T]: """Like rmap, but terminate on the longest input.""" yield from map_longest(func, *(rev(s) for s in iterables), fillvalue=fillvalue) -def rzip_longest(*iterables, fillvalue=None): +def rzip_longest(*iterables: Iterable, fillvalue: Any = None) -> Iterator[tuple]: """Like rzip, but terminate on the longest input.""" yield from zip_longest(*(rev(s) for s in iterables), fillvalue=fillvalue) -def mapr(proc, *iterables): +def mapr(proc: Callable[..., T], *iterables: Iterable) -> Iterator[T]: """Like map, but from the right. For multiple inputs with different lengths, ``mapr`` syncs the **left** ends. @@ -150,7 +155,7 @@ def mapr(proc, *iterables): """ yield from rev(map(proc, *iterables)) -def zipr(*iterables): +def zipr(*iterables: Iterable[T]) -> Iterator[tuple[T, ...]]: """Like zip, but from the right. For multiple inputs with different lengths, ``zipr`` syncs the **left** ends. @@ -158,11 +163,11 @@ def zipr(*iterables): """ yield from rev(zip(*iterables)) -def mapr_longest(proc, *iterables, fillvalue=None): +def mapr_longest(proc: Callable[..., T], *iterables: Iterable, fillvalue: Any = None) -> Iterator[T]: """Like mapr, but terminate on the longest input.""" yield from rev(map_longest(proc, *iterables, fillvalue=fillvalue)) -def zipr_longest(*iterables, fillvalue=None): +def zipr_longest(*iterables: Iterable, fillvalue: Any = None) -> Iterator[tuple]: """Like zipr, but terminate on the longest input.""" yield from rev(zip_longest(*iterables, fillvalue=fillvalue)) @@ -186,7 +191,7 @@ def zipr_longest(*iterables, fillvalue=None): # return _mapr(identity, iterable0, *iterables, # longest=longest, fillvalue=fillvalue) -def flatmap(f, iterable0, *iterables): +def flatmap(f: Callable[..., Iterable[T]], iterable0: Iterable, *iterables: Iterable) -> Iterator[T]: """Map, then concatenate results. At least one iterable (``iterable0``) is required. More are optional. @@ -221,7 +226,7 @@ def sum_and_diff(a, b): # for xs in map(f, iterable0, *iterables): # yield from xs -def uniqify(iterable, *, key=None): +def uniqify(iterable: Iterable[T], *, key: Callable[[T], Any] | None = None) -> Iterator[T]: """Skip duplicates in iterable. Returns a generator that yields unique items from iterable, preserving @@ -246,7 +251,7 @@ def uniqify(iterable, *, key=None): seen_add(k) yield e -def uniq(iterable, *, key=None): +def uniq(iterable: Iterable[T], *, key: Callable[[T], Any] | None = None) -> Iterator[T]: """Like uniqify, but for consecutive duplicates only. Named after the *nix utility. @@ -256,7 +261,7 @@ def uniq(iterable, *, key=None): # the outer map retrieves the item from the subiterator in (key, subiterator). yield from map(next, map(itemgetter(1), groupby(iterable, key))) -def take(n, iterable): +def take(n: int, iterable: Iterable[T]) -> Iterator[T]: """Return an iterator that yields the first n items of iterable, then stops. Stops earlier if ``iterable`` has fewer than ``n`` items. @@ -269,7 +274,7 @@ def take(n, iterable): raise ValueError(f"expected n >= 0, got {n}") return islice(iter(iterable), n) -def drop(n, iterable): +def drop(n: int | None, iterable: Iterable[T]) -> Iterator[T]: """Skip the first n elements of iterable, then yield the rest. If ``n`` is ``None``, consume the iterable until it runs out. @@ -288,7 +293,7 @@ def drop(n, iterable): next(islice(it, n, n), None) # advance it to empty slice starting at n return it -def split_at(n, iterable): +def split_at(n: int, iterable: Iterable[T]) -> tuple[Iterator[T], Iterator[T]]: """Split iterable at position n. Returns a pair of iterators ``(first_part, second_part)``. @@ -312,7 +317,7 @@ def split_at(n, iterable): ia, ib = tee(iter(iterable)) return take(n, ia), drop(n, ib) -def unpack(n, iterable, *, k=None, fillvalue=None): +def unpack(n: int, iterable: Iterable[T], *, k: int | None = None, fillvalue: Any = None) -> tuple: # TODO: use TypeVarTuple for the return type once floor bumps to Python 3.11+ """From iterable, return the first n elements, and the kth tail. Lazy generalization of sequence unpacking, works also for infinite iterables. @@ -360,7 +365,7 @@ def unpack(n, iterable, *, k=None, fillvalue=None): out.append(next(it)) except StopIteration: # had fewer than n items remaining out += [fillvalue] * (n - len(out)) - def empty_iterable(): + def empty_iterable() -> Iterator[T]: yield from () tl = empty_iterable() break @@ -372,14 +377,14 @@ def empty_iterable(): out.append(tl) return tuple(out) -def tail(iterable): +def tail(iterable: Iterable[T]) -> Iterator[T]: """Return an iterator pointing to the tail of iterable. Same as ```drop(1, iterable)```. """ return drop(1, iterable) -def butlast(iterable): +def butlast(iterable: Iterable[T]) -> Iterator[T]: """Yield all items from iterable, except the last one (if iterable is finite). Return a generator. @@ -389,7 +394,7 @@ def butlast(iterable): """ return butlastn(1, iterable) -def butlastn(n, iterable): +def butlastn(n: int, iterable: Iterable[T]) -> Iterator[T]: """Yield all items from iterable, except the last n (if iterable is finite). Return a generator. @@ -411,15 +416,15 @@ def butlastn(n, iterable): except StopIteration: return -def first(iterable, *, default=None): +def first(iterable: Iterable[T], *, default: T | None = None) -> T | None: """Like nth, but return the first item.""" return nth(0, iterable, default=default) -def second(iterable, *, default=None): +def second(iterable: Iterable[T], *, default: T | None = None) -> T | None: """Like nth, but return the second item.""" return nth(1, iterable, default=default) -def nth(n, iterable, *, default=None): +def nth(n: int, iterable: Iterable[T], *, default: T | None = None) -> T | None: """Return the item at position n from an iterable. The ``default`` is returned if there are fewer than ``n + 1`` items. @@ -434,7 +439,7 @@ def nth(n, iterable, *, default=None): except StopIteration: return default -def last(iterable, *, default=None): +def last(iterable: Iterable[T], *, default: T | None = None) -> T | None: """Return the last item from an iterable. We consume the iterable until it runs out of items, then return the @@ -447,7 +452,7 @@ def last(iterable, *, default=None): d = deque(iterable, maxlen=1) # C speed return d.pop() if d else default -def lastn(n, iterable): +def lastn(n: int, iterable: Iterable[T]) -> Iterator[T]: """Yield the last n items from an iterable. We consume the iterable until it runs out of items, then return a generator @@ -461,7 +466,7 @@ def lastn(n, iterable): d = deque(iterable, maxlen=n) # C speed yield from d -def scons(x, iterable): +def scons(x: T, iterable: Iterable[T]) -> Iterator[T]: """Prepend one element to the start of an iterable, return new iterable. Same as ``itertools.chain((x,), iterable)``. The point is sometimes it is @@ -472,7 +477,7 @@ def scons(x, iterable): """ return chain((x,), iterable) -def pad(n, fillvalue, iterable): +def pad(n: int, fillvalue: Any, iterable: Iterable[T]) -> Iterator[Any]: """Pad iterable with copies of fillvalue so its length is at least ``n``. Examples:: @@ -557,26 +562,34 @@ def flatten_in(iterable, pred=None): else: yield e -def iterate1(f, x): +def iterate1(f: Callable[[T], T], x: T) -> Iterator[T]: """Return an infinite generator yielding x, f(x), f(f(x)), ...""" while True: yield x x = f(x) -def iterate(f, *args): +def iterate(f: Callable[..., Values], *args: Any, **kwargs: Any) -> Iterator[Values]: """Multiple-argument version of iterate1. - The function ``f`` should return a tuple or list of as many elements as it - takes positional arguments; this will be unpacked to the argument list in - the next call. + The initial ``args`` and ``kwargs`` are packed into a ``Values`` object, + which we will below denote as ``x``. When calling ``f``, ``x`` is unpacked + to its args/kwargs. - Or in other words, yield args, f(*args), f(*f(*args)), ... + The function ``f`` must return a ``Values`` object in the same shape + as it takes args and kwargs; this then becomes the new ``x``. + + Using this notation, this function behaves exactly like ``iterate1``: + the return value of ``iterate`` is an infinite generator that yields + x, f(x), f(f(x)), ... """ + x = Values(*args, **kwargs) while True: - yield args - args = f(*args) + yield x + x = f(*x.rets, **x.kwrets) + if not isinstance(x, Values): + raise TypeError(f"Expected a `Values`, got {type(x)} with value {repr(x)}") -def partition(pred, iterable): +def partition(pred: Callable[[T], bool], iterable: Iterable[T]) -> tuple[Iterator[T], Iterator[T]]: """Partition an iterable to entries satifying and not satisfying a predicate. Return two generators, ``(false-items, true-items)``, where each generator @@ -594,7 +607,7 @@ def partition(pred, iterable): It will eventually run out of memory storing all the odd numbers "to be read later".) - Not to be confused with `unpythonic.it.partition_int`, which partitions + Not to be confused with `unpythonic.numutil.partition_int`, which partitions a (small) positive integer to smaller integers, in all possible ways, such that those integers sum to the original one. """ @@ -602,64 +615,7 @@ def partition(pred, iterable): t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) -def partition_int(n, lower=1, upper=None): - """Yield all ordered sequences of smaller positive integers that sum to `n`. - - `n` must be an integer >= 1. - - `lower` is an optional lower limit for each member of the sum. Each member - of the sum must be `>= lower`. - - (Most of the splits are a ravioli consisting mostly of ones, so it is much - faster to not generate such splits than to filter them out from the result. - The default value `lower=1` generates everything.) - - `upper` is, similarly, an optional upper limit; each member of the sum - must be `<= upper`. The default `None` means no upper limit (effectively, - in that case `upper=n`). - - It must hold that `1 <= lower <= upper <= n`. - - Not to be confused with `unpythonic.it.partition`, which partitions an - iterable based on a predicate. - - **CAUTION**: The number of possible partitions grows very quickly with `n`, - so in practice this is only useful for small numbers, or with a lower limit - that is not too much smaller than `n / 2`. A possible use case for this - function is to determine the number of letters to allocate for each - component of an anagram that may consist of several words. - - See: - https://en.wikipedia.org/wiki/Partition_(number_theory) - """ - # sanity check the preconditions, fail-fast - if not isinstance(n, int): - raise TypeError(f"n must be integer; got {type(n)} with value {repr(n)}") - if not isinstance(lower, int): - raise TypeError(f"lower must be integer; got {type(lower)} with value {repr(lower)}") - if upper is not None and not isinstance(upper, int): - raise TypeError(f"upper must be integer; got {type(upper)} with value {repr(upper)}") - upper = upper if upper is not None else n - if n < 1: - raise ValueError(f"n must be positive; got {n}") - if lower < 1 or upper < 1 or lower > n or upper > n or lower > upper: - raise ValueError(f"it must hold that 1 <= lower <= upper <= n; got lower={lower}, upper={upper}") - - def _partition(n): - for k in range(min(n, upper), lower - 1, -1): - m = n - k - if m == 0: - yield (k,) - else: - out = [] - for item in _partition(m): - out.append((k,) + item) - for term in out: - yield term - - return _partition(n) # instantiate the generator - -def inn(x, iterable): +def inn(x: T, iterable: Iterable[T]) -> bool: """Contains-check (``x in iterable``) with automatic termination. ``iterable`` may be infinite. @@ -726,7 +682,7 @@ def primes(): pred = (lambda elt: elt <= x) if d > 0 else (lambda elt: elt >= x) return x in takewhile(pred, it) -def iindex(x, iterable): +def iindex(x: T, iterable: Iterable[T]) -> int: """Like list.index, but for a general iterable. Note that just like ``x in iterable``, this will not terminate if ``iterable`` @@ -741,7 +697,7 @@ def iindex(x, iterable): return j raise ValueError(f"{x} is not in iterable") -def find(predicate, iterable, default=None): +def find(predicate: Callable[[T], bool], iterable: Iterable[T], default: T | None = None) -> T | None: """Return the first item matching `predicate` in `iterable`, or `default` if no match. If you need all matching items, just use the builtin `filter` or a comprehension; @@ -749,7 +705,7 @@ def find(predicate, iterable, default=None): """ return next(filter(predicate, iterable), default) -def window(n, iterable): +def window(n: int, iterable: Iterable[T]) -> Iterator[tuple[T, ...]]: """Sliding length-n window iterator for a general iterable. Acts like ``zip(s, s[1:], ..., s[n-1:])`` for a sequence ``s``, but the input @@ -770,10 +726,10 @@ def window(n, iterable): try: xs.append(next(it)) except StopIteration: - def empty_iterable(): + def empty_iterable() -> Iterator[T]: yield from () return empty_iterable() - def windowed(): + def windowed() -> Iterator[tuple[T, ...]]: while True: yield tuple(xs) xs.popleft() @@ -783,7 +739,7 @@ def windowed(): return return windowed() -def chunked(n, iterable): +def chunked(n: int, iterable: Iterable[T]) -> Iterator[Iterator[T]]: """Split an iterable into constant-length chunks. Conceptually, whereas ``window`` slides its stencil through which the @@ -809,7 +765,7 @@ def chunked(n, iterable): if n < 2: raise ValueError(f"expected n >= 2, got {n}") it = iter(iterable) - def chunker(): + def chunker() -> Iterator[Iterator[T]]: try: while True: cit = islice(it, n) @@ -819,7 +775,7 @@ def chunker(): return return chunker() -def within(tol, iterable): +def within(tol: float, iterable: Iterable[T]) -> Iterator[T]: """Yield items from iterable until successive items are close enough. Items are yielded until `abs(a - b) <= tol` for successive items @@ -839,43 +795,7 @@ def within(tol, iterable): yield b return -def fixpoint(f, x0, tol=0): - """Compute the (arithmetic) fixed point of f, starting from the initial guess x0. - - (Not to be confused with the logical fixed point with respect to the - definedness ordering.) - - The fixed point must be attractive for this to work. See the Banach - fixed point theorem. - https://en.wikipedia.org/wiki/Banach_fixed-point_theorem - - If the fixed point is attractive, and the values are represented in - floating point (hence finite precision), the computation should - eventually converge down to the last bit (barring roundoff or - catastrophic cancellation in the final few steps). Hence the default tol - of zero. - - CAUTION: an arbitrary function from ℝ to ℝ **does not** necessarily - have a fixed point. Limit cycles and chaotic behavior of `f` will cause - non-termination. Keep in mind the classic example: - https://en.wikipedia.org/wiki/Logistic_map - - Examples:: - from math import cos, sqrt - from unpythonic import fixpoint, ulp - c = fixpoint(cos, x0=1) - - # Actually "Newton's" algorithm for the square root was already known to the - # ancient Babylonians, ca. 2000 BCE. (Carl Boyer: History of mathematics) - def sqrt_newton(n): - def sqrt_iter(x): # has an attractive fixed point at sqrt(n) - return (x + n / x) / 2 - return fixpoint(sqrt_iter, x0=n / 2) - assert abs(sqrt_newton(2) - sqrt(2)) <= ulp(1.414) - """ - return last(within(tol, iterate1(f, x0))) - -def interleave(*iterables): +def interleave(*iterables: Iterable[T]) -> Iterator[T]: """Interleave items from several iterables. Generator. Example:: @@ -887,7 +807,7 @@ def interleave(*iterables): class ShortestInputEnded(Exception): pass iters = [iter(it) for it in iterables] - def roundrobin(): + def roundrobin() -> Iterator[T]: for it in iters: try: x = next(it) @@ -900,7 +820,7 @@ def roundrobin(): except ShortestInputEnded: return -def subset(part, whole): +def subset(part: Iterable, whole: Iterable) -> bool: """Test whether `part` is a subset of `whole`. Both must be iterable. Note consumable iterables will be consumed @@ -915,7 +835,7 @@ def subset(part, whole): """ return all(elt in whole for elt in part) -def powerset(iterable): +def powerset(iterable: Iterable[T]) -> Iterator[tuple[T, ...]]: """Yield the powerset of a general iterable. The powerset is the set of all subsets of items taken from the iterable. @@ -1000,10 +920,11 @@ def total_num_items(ld): yield from t bag.extend(t) -def allsame(iterable): +def allsame(iterable: Iterable) -> bool: """Return whether all elements of an iterable are the same. - The test uses `!=` to compare. + The test uses `!=` to compare, and short-circuits at the + first item that is different. If `iterable` is empty, the return value is `True` (like for `all`). @@ -1016,7 +937,4 @@ def allsame(iterable): x0 = next(it) except StopIteration: return True # like all(()) is True - for x in it: - if x != x0: - return False - return True + return all(x == x0 for x in it) diff --git a/unpythonic/lazyutil.py b/unpythonic/lazyutil.py index 7058798c..b54553ee 100644 --- a/unpythonic/lazyutil.py +++ b/unpythonic/lazyutil.py @@ -15,7 +15,7 @@ # HACK: break dependency loop llist -> fun -> lazyutil -> collections -> llist _init_done = False jump = sym("jump") # doesn't matter what the value is, will be overwritten later -def _init_module(): # called by unpythonic.__init__ when otherwise done +def _init_module() -> None: # called by unpythonic.__init__ when otherwise done global mogrify, jump, _init_done from .collections import mogrify from .tco import jump diff --git a/unpythonic/let.py b/unpythonic/let.py index e457a46e..81ff1e97 100644 --- a/unpythonic/let.py +++ b/unpythonic/let.py @@ -3,13 +3,17 @@ __all__ = ["let", "letrec", "dlet", "dletrec", "blet", "bletrec"] +from collections.abc import Callable from functools import wraps +from typing import Any, TypeVar from .arity import arity_includes, UnknownArity from .env import env as _envcls from .funutil import call -def let(body, **bindings): +F = TypeVar('F', bound=Callable) + +def let(body: Callable, **bindings: Any) -> Any: """``let`` expression. In ``let``, the bindings are independent (do not see each other); only @@ -80,7 +84,7 @@ def let(body, **bindings): """ return _let("let", body, **bindings) -def letrec(body, **bindings): +def letrec(body: Callable, **bindings: Any) -> Any: """``letrec`` expression. Like ``let``, but bindings can see each other. To make a binding use the @@ -106,25 +110,8 @@ def letrec(body, **bindings): body=lambda e: e.b * e.f(1)) # --> 84 - **CAUTION**: - - Simple values (non-callables) may depend on earlier definitions - in the same letrec **only in Python 3.6 and later**. - - Until Python 3.6, initialization of the bindings occurs - **in an arbitrary order**, because of the ``kwargs`` mechanism. - See PEP 468: - - https://www.python.org/dev/peps/pep-0468/ - - In Python < 3.6, in the first example above, trying to reference ``env.a`` - on the RHS of ``b`` may get either the ``lambda e: ...``, or the value ``1``, - depending on whether the binding ``a`` has been initialized at that point or not. - - If you need left-to-right initialization of bindings in Python < 3.6, - see ``unpythonic.lispylet``. - - The following applies regardless of Python version. + Simple values (non-callables) may depend on earlier definitions + in the same letrec. A callable value may depend on **any** binding, also later ones. This allows mutually recursive functions:: @@ -151,9 +138,9 @@ def letrec(body, **bindings): L = [1, 1, 3, 1, 3, 2, 3, 2, 2, 2, 4, 4, 1, 2, 3] print(u(L)) # [1, 3, 2, 4] - Works also in Python < 3.6, because here ``see`` is a callable. Hence, ``e.seen`` - doesn't have to exist when the *definition* of ``see`` is evaluated; it only has to - exist when ``e.see(x)`` is *called*. + Note that ``see`` is a callable. Hence, strictly speaking it doesn't matter + if ``e.seen`` exists when the *definition* of ``see`` is evaluated; it only + has to exist when ``e.see(x)`` is *called*. Parameters: `body`: function @@ -173,7 +160,7 @@ def letrec(body, **bindings): """ return _let("letrec", body, **bindings) -def dlet(**bindings): +def dlet(**bindings: Any) -> Callable[[F], F]: """``let`` decorator. For let-over-def; think *let over lambda* in Lisp:: @@ -198,7 +185,7 @@ def counter(*, env=None): """ return _dlet("let", **bindings) -def dletrec(**bindings): +def dletrec(**bindings: Any) -> Callable[[F], F]: """``letrec`` decorator. Like ``dlet``, but with ``letrec`` instead of ``let``:: @@ -211,7 +198,7 @@ def bar(a, *, env): """ return _dlet("letrec", **bindings) -def blet(**bindings): +def blet(**bindings: Any) -> Callable[[Callable], Any]: """``let`` block. This chains ``@dlet`` and ``@call``:: @@ -223,13 +210,13 @@ def result(*, env): """ return _blet("let", **bindings) -def bletrec(**bindings): +def bletrec(**bindings: Any) -> Callable[[Callable], Any]: """``letrec`` block. This chains ``@dletrec`` and ``@call``.""" return _blet("letrec", **bindings) -def _let(mode, body, **bindings): +def _let(mode: str, body: Callable | None, **bindings: Any) -> Any: assert mode in ("let", "letrec") # Important for Python 3.6+, which preserves ordering of kwargs (PEP 468): # @@ -271,21 +258,21 @@ def _let(mode, body, **bindings): # decorator factory: almost as fun as macros? # _envname is for co-operation with the dlet macro. -def _dlet(mode, _envname="env", **bindings): - def deco(body): +def _dlet(mode: str, _envname: str = "env", **bindings: Any) -> Callable[[F], F]: + def deco(body: F) -> F: # evaluate env only once, when the function def runs # (to preserve state between calls to the decorated function) env = _let(mode, body=None, **bindings) @wraps(body) - def withenv(*args, **kwargs): + def withenv(*args: Any, **kwargs: Any) -> Any: kwargs_with_env = kwargs.copy() kwargs_with_env[_envname] = env return body(*args, **kwargs_with_env) return withenv return deco -def _blet(mode, _envname="env", **bindings): +def _blet(mode: str, _envname: str = "env", **bindings: Any) -> Callable[[Callable], Any]: dlet_deco = _dlet(mode, _envname, **bindings) - def deco(body): + def deco(body: Callable) -> Any: return call(dlet_deco(body)) return deco diff --git a/unpythonic/lispylet.py b/unpythonic/lispylet.py index da9c1516..e1a78ed6 100644 --- a/unpythonic/lispylet.py +++ b/unpythonic/lispylet.py @@ -3,13 +3,17 @@ __all__ = ["let", "letrec", "dlet", "dletrec", "blet", "bletrec"] +from collections.abc import Callable from functools import wraps +from typing import Any, TypeVar from .arity import arity_includes, UnknownArity from .env import env as _envcls from .funutil import call -def let(bindings, body): +F = TypeVar('F', bound=Callable) + +def let(bindings: tuple[tuple[str, Any], ...], body: Callable) -> Any: """``let`` expression. In ``let``, the bindings are independent (do not see each other); only @@ -86,7 +90,7 @@ def let(bindings, body): """ return _let(bindings, body) -def letrec(bindings, body): +def letrec(bindings: tuple[tuple[str, Any], ...], body: Callable) -> Any: """``letrec`` expression. Like ``let``, but bindings can see each other. To make a binding use the @@ -160,7 +164,7 @@ def letrec(bindings, body): """ return _let(bindings, body, mode="letrec") -def dlet(bindings): +def dlet(bindings: tuple[tuple[str, Any], ...]) -> Callable[[F], F]: """``let`` decorator. For let-over-def; think *let over lambda* in Lisp:: @@ -185,7 +189,7 @@ def counter(*, env): """ return _dlet(bindings) -def dletrec(bindings): +def dletrec(bindings: tuple[tuple[str, Any], ...]) -> Callable[[F], F]: """``letrec`` decorator. Like ``dlet``, but with ``letrec`` instead of ``let``:: @@ -198,7 +202,7 @@ def bar(a, *, env): """ return _dlet(bindings, mode="letrec") -def blet(bindings): +def blet(bindings: tuple[tuple[str, Any], ...]) -> Callable[[Callable], Any]: """``let`` block. This chains ``@dlet`` and ``@call``:: @@ -210,7 +214,7 @@ def result(*, env): """ return _blet(bindings) -def bletrec(bindings): +def bletrec(bindings: tuple[tuple[str, Any], ...]) -> Callable[[Callable], Any]: """``letrec`` block. This chains ``@dletrec`` and ``@call``.""" @@ -218,7 +222,7 @@ def bletrec(bindings): # Core idea based on StackOverflow answer by divs1210 (2017), # used under the MIT license. https://stackoverflow.com/a/44737147 -def _let(bindings, body, *, env=None, mode="let"): +def _let(bindings: tuple[tuple[str, Any], ...], body: Callable | None, *, env: _envcls | None = None, mode: str = "let") -> Any: assert mode in ("let", "letrec") env = env or _envcls() @@ -250,19 +254,19 @@ def _let(bindings, body, *, env=None, mode="let"): return _let(more, body, env=env, mode=mode) # loop # _envname is for co-operation with the dlet macro. -def _dlet(bindings, mode="let", _envname="env"): # let and letrec decorator factory - def deco(body): +def _dlet(bindings: tuple[tuple[str, Any], ...], mode: str = "let", _envname: str = "env") -> Callable[[F], F]: # let and letrec decorator factory + def deco(body: F) -> F: env = _let(bindings, body=None, mode=mode) # set up env, don't run yet @wraps(body) - def withenv(*args, **kwargs): + def withenv(*args: Any, **kwargs: Any) -> Any: kwargs_with_env = kwargs.copy() kwargs_with_env[_envname] = env return body(*args, **kwargs_with_env) return withenv return deco -def _blet(bindings, mode="let", _envname="env"): +def _blet(bindings: tuple[tuple[str, Any], ...], mode: str = "let", _envname: str = "env") -> Callable[[Callable], Any]: dlet_deco = _dlet(bindings, mode, _envname) - def deco(body): + def deco(body: Callable) -> Any: return call(dlet_deco(body)) return deco diff --git a/unpythonic/llist.py b/unpythonic/llist.py index a8a56d92..995e69a6 100644 --- a/unpythonic/llist.py +++ b/unpythonic/llist.py @@ -5,43 +5,61 @@ """ from abc import ABCMeta, abstractmethod -from collections.abc import Iterable, Iterator +from collections.abc import Callable, Generator, Iterable, Iterator +from dataclasses import FrozenInstanceError from itertools import zip_longest +from typing import Any from .fun import composer1i from .fold import foldr, foldl from .it import rev from .singleton import Singleton -# from .symbol import gensym +from .symbol import gensym + +_fill = gensym("fill") # explicit list better for tooling support -_exports = ["cons", "nil", - "LinkedListIterator", "LinkedListOrCellIterator", "TailIterator", - "BinaryTreeIterator", "ConsIterator", - "car", "cdr", - "caar", "cadr", "cdar", "cddr", - "caaar", "caadr", "cadar", "caddr", "cdaar", "cdadr", "cddar", "cdddr", - "caaaar", "caaadr", "caadar", "caaddr", "cadaar", "cadadr", "caddar", "cadddr", - "cdaaar", "cdaadr", "cdadar", "cdaddr", "cddaar", "cddadr", "cdddar", "cddddr", - "ll", "llist", "lreverse", "lappend", "lzip"] +__all__ = ["FrozenAttributeError", + "cons", "nil", + "LinkedListIterator", "LinkedListOrCellIterator", "TailIterator", + "BinaryTreeIterator", "ConsIterator", + "car", "cdr", + "caar", "cadr", "cdar", "cddr", + "caaar", "caadr", "cadar", "caddr", "cdaar", "cdadr", "cddar", "cdddr", + "caaaar", "caaadr", "caadar", "caaddr", "cadaar", "cadadr", "caddar", "cadddr", + "cdaaar", "cdaadr", "cdadar", "cdaddr", "cddaar", "cddadr", "cdddar", "cddddr", + "ll", "llist", "lreverse", "lappend", "lzip"] #from itertools import product, repeat #_ads = lambda n: product(*repeat("ad", n)) #_c2r = ["c{}{}r".format(*x) for x in _ads(2)] #_c3r = ["c{}{}{}r".format(*x) for x in _ads(3)] #_c4r = ["c{}{}{}{}r".format(*x) for x in _ads(4)] -#_exports.extend(_c2r) -#_exports.extend(_c3r) -#_exports.extend(_c4r) -__all__ = _exports +#__all__.extend(_c2r) +#__all__.extend(_c3r) +#__all__.extend(_c4r) + +class FrozenAttributeError(TypeError, FrozenInstanceError): + """Raised on a write/delete attempt against a frozen-instance type. + + Multiply-inherits from `TypeError` (the legacy unpythonic <= 2.x base) + and `dataclasses.FrozenInstanceError` (the standard-library convention + since Python 3.7, itself a subclass of `AttributeError`). Either + `except` clause catches it. + + Compatibility shim: lets unpythonic align with the stdlib idiom + without breaking user code that catches `TypeError`. The `TypeError` + base will be dropped in 3.0.0 (see issue #35), at which point this + class becomes a plain `FrozenInstanceError` and likely goes away. + """ class Nil(Singleton): """The empty linked list. Singleton.""" # support the iterator protocol so we can say tuple(nil) --> () - def __iter__(self): + def __iter__(self) -> "Nil": return self - def __next__(self): + def __next__(self) -> Any: raise StopIteration() - def __repr__(self): + def __repr__(self) -> str: return "nil" nil = Nil() @@ -61,20 +79,20 @@ class ConsIterator(metaclass=ABCMeta): For usage examples see the predefined iterators in ``unpythonic.llist``. """ @abstractmethod - def __init__(self, startcell, walker): + def __init__(self, startcell: "cons", walker: Callable[["cons"], Generator]) -> None: if not isinstance(startcell, cons): raise TypeError(f"Expected a cons, got {type(startcell)} with value {startcell}") self.walker = iter(walker(startcell)) # iter() needed to support gtrampolined generators - def __iter__(self): + def __iter__(self) -> "ConsIterator": return self - def __next__(self): + def __next__(self) -> Any: return next(self.walker) Iterable.register(ConsIterator) Iterator.register(ConsIterator) class LinkedListIterator(ConsIterator): """Iterator for linked lists built from cons cells.""" - def __init__(self, head, _fullerror=True): + def __init__(self, head: "cons", _fullerror: bool = True) -> None: def walker(head): cell = head while cell is not nil: @@ -94,7 +112,7 @@ class LinkedListReverseIterator(LinkedListIterator): Computes the reversed list at init time, so it can then be walked forward. Cost O(n). """ - def __init__(self, head, _fullerror=True): + def __init__(self, head: "cons", _fullerror: bool = True) -> None: self._data = lreverse(head) super().__init__(self._data, _fullerror) @@ -103,7 +121,7 @@ class LinkedListOrCellIterator(ConsIterator): Default iteration strategy. Useful for sequence unpacking of cons and ll. """ - def __init__(self, head, _fullerror=True): + def __init__(self, head: "cons", _fullerror: bool = True) -> None: def walker(head): cell = head while cell is not nil: @@ -127,7 +145,7 @@ class TailIterator(ConsIterator): # for member() TailIterator(ll(1, 2, 3)) --> ll(1, 2, 3), ll(2, 3), ll(3) """ - def __init__(self, head): + def __init__(self, head: "cons") -> None: def walker(head): cell = head while cell is not nil: @@ -140,7 +158,7 @@ def walker(head): class BinaryTreeIterator(ConsIterator): """Iterator for binary trees built from cons cells.""" - def __init__(self, root): + def __init__(self, root: "cons") -> None: # def walker(cell): # FP, call stack overflow for deep trees # for x in (cell.car, cell.cdr): # if isinstance(x, cons): @@ -180,7 +198,7 @@ class JackOfAllTradesIterator(ConsIterator): If you want the ace for a particular trade, use the specific iterator for the specific kind of cons structure you have. """ - def __init__(self, root): + def __init__(self, root: "cons") -> None: # @gtrampolined # def walker(cell): # FP, tail-recursive in the cdr half only # if isinstance(cell.car, cons): @@ -212,21 +230,21 @@ class cons: Iterable. Default is to iterate as a linked list. """ - def __init__(self, v1, v2): - self.car = v1 - self.cdr = v2 - self._immutable = True - def __setattr__(self, k, v): - if hasattr(self, "_immutable"): - raise TypeError("'cons' object does not support item assignment") - super().__setattr__(k, v) - def __iter__(self): + def __init__(self, v1: Any, v2: Any) -> None: + # Bypass our locked-down `__setattr__` to populate the read-only fields once. + object.__setattr__(self, "car", v1) + object.__setattr__(self, "cdr", v2) + def __setattr__(self, k: str, v: Any) -> None: + raise FrozenAttributeError(f"'cons' object does not support attribute assignment; tried to set {k!r}") + def __delattr__(self, k: str) -> None: + raise FrozenAttributeError(f"'cons' object does not support attribute deletion; tried to delete {k!r}") + def __iter__(self) -> LinkedListOrCellIterator: """Return iterator with default iteration scheme: single cell or list.""" return LinkedListOrCellIterator(self) - def __reversed__(self): + def __reversed__(self) -> LinkedListReverseIterator: """For lists. Caution: O(n), works by building a reversed list.""" return LinkedListReverseIterator(self) - def __repr__(self): + def __repr__(self) -> str: """Representation in pythonic notation. Suitable for ``eval`` if all elements are.""" @@ -239,7 +257,7 @@ def __repr__(self): result_list = (repr(self.car), repr(self.cdr)) result_str = ", ".join(result_list) return f"cons({result_str})" - def lispyrepr(self): # TODO: maybe rename or alias this to `__str__`? + def lispyrepr(self) -> str: # TODO: maybe rename or alias this to `__str__`? """Representation in Lisp-like dot notation.""" try: result_list = [repr(x) for x in LinkedListIterator(self, _fullerror=False)] @@ -248,44 +266,40 @@ def lispyrepr(self): # TODO: maybe rename or alias this to `__str__`? result_list = (r(self.car), ".", r(self.cdr)) result_str = " ".join(result_list) return f"({result_str})" - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if other is self: return True if isinstance(other, cons): try: # duck test linked lists ia, ib = (LinkedListIterator(x) for x in (self, other)) - fill = object() # gensym("fill"), but object() is much faster, and we don't need a label, or pickle support. - for a, b in zip_longest(ia, ib, fillvalue=fill): - if a != b: - return False - return True + return all(a == b for a, b in zip_longest(ia, ib, fillvalue=_fill)) except TypeError: return self.car == other.car and self.cdr == other.cdr return False - def __hash__(self): + def __hash__(self) -> int: try: # duck test linked list tpl = tuple(LinkedListIterator(self)) except TypeError: tpl = (self.car, self.cdr) return hash(tpl) -def _car(x): +def _car(x: cons) -> Any: return _typecheck(x).car -def _cdr(x): +def _cdr(x: cons) -> Any: return _typecheck(x).cdr -def _typecheck(x): +def _typecheck(x: Any) -> cons: if not isinstance(x, cons): raise TypeError(f"Expected a cons, got {type(x)} with value {x}") return x -def _build_accessor(name): +def _build_accessor(name: str) -> Callable[[cons], Any]: spec = name[1:-1] f = {'a': _car, 'd': _cdr} return composer1i(f[char] for char in spec) -def car(x): +def car(x: cons) -> Any: """Return the first half of a cons cell.""" return _car(x) -def cdr(x): +def cdr(x: cons) -> Any: """Return the second half of a cons cell.""" return _cdr(x) @@ -320,7 +334,7 @@ def cdr(x): cdddar = _build_accessor("cdddar") cddddr = _build_accessor("cddddr") -def ll(*elts): +def ll(*elts: Any) -> "cons | Nil": """Make a linked list with the given elements. ``ll(...)`` plays the same role as ``[...]`` or ``(...)`` for lists or tuples, @@ -334,7 +348,7 @@ def ll(*elts): """ return llist(elts) -def llist(iterable): +def llist(iterable: Iterable) -> "cons | Nil": """Make a linked list from iterable. ``llist(...)`` plays the same role as ``list(...)`` or ``tuple(...)`` for @@ -360,7 +374,7 @@ def llist(iterable): return iterable._data return lreverse(rev(iterable)) -def lreverse(iterable): +def lreverse(iterable: Iterable) -> "cons | Nil": """Reverse an iterable, loading the result into a linked list. If you have a linked list and want an iterator instead, use ``reversed(l)``. @@ -368,13 +382,13 @@ def lreverse(iterable): """ return foldl(cons, nil, iterable) -def lappend(*ls): +def lappend(*ls: "cons | Nil") -> "cons | Nil": """Append the given linked lists left-to-right.""" - def lappend_two(l1, l2): + def lappend_two(l1: "cons | Nil", l2: "cons | Nil") -> "cons | Nil": return foldr(cons, l2, l1) return foldr(lappend_two, nil, ls) -def member(x, l): +def member(x: Any, l: cons) -> "cons | bool": # noqa: E741 -- standard Lisp name for a linked list """Walk linked list l and check if item x is in it. Returns: @@ -385,7 +399,7 @@ def member(x, l): return t return False -def lzip(*ls): +def lzip(*ls: "cons | Nil") -> "cons | Nil": """Zip linked lists, producing a linked list of linked lists. Built-in zip() works too, but produces tuples. diff --git a/unpythonic/mathseq.py b/unpythonic/mathseq.py index 8d2e041c..f499bc82 100644 --- a/unpythonic/mathseq.py +++ b/unpythonic/mathseq.py @@ -18,27 +18,36 @@ (currently, the Fibonacci numbers and the prime numbers). """ -__all__ = ["s", "imathify", "gmathify", +__all__ = ["s", "imathify", "gmathify", "slift1", "slift2", "sadd", "ssub", "sabs", "spos", "sneg", "sinvert", "smul", "spow", "struediv", "sfloordiv", "smod", "sdivmod", "sround", "strunc", "sfloor", "sceil", "slshift", "srshift", "sand", "sxor", "sor", "cauchyprod", "diagonal_reduce", - "fibonacci", "primes"] + "fibonacci", "triangular", "primes"] +from collections.abc import Callable, Iterable, Iterator from itertools import repeat, takewhile, count from functools import wraps -from operator import (add as primitive_add, mul as primitive_mul, - pow as primitive_pow, mod as primitive_mod, - floordiv as primitive_floordiv, truediv as primitive_truediv, - sub as primitive_sub, - neg as primitive_neg, pos as primitive_pos, - and_ as primitive_and, xor as primitive_xor, or_ as primitive_or, - lshift as primitive_lshift, rshift as primitive_rshift, - invert as primitive_invert, - lt as primitive_lt, le as primitive_le, - eq as primitive_eq, ne as primitive_ne, - ge as primitive_ge, gt as primitive_gt) +from operator import (add as atom_add, mul as atom_mul, + pow as atom_pow, mod as atom_mod, + floordiv as atom_floordiv, truediv as atom_truediv, + sub as atom_sub, + neg as atom_neg, pos as atom_pos, + and_ as atom_and, xor as atom_xor, or_ as atom_or, + lshift as atom_lshift, rshift as atom_rshift, + invert as atom_invert, + lt as atom_lt, le as atom_le, + eq as atom_eq, ne as atom_ne, + ge as atom_ge, gt as atom_gt) + +from typing import Any, Literal, TypeVar + +# TODO: When floor bumps to 3.12, use inline `[T]` syntax on `slift1` +# and `slift2` (PEP 695). Also consider making `imathify` generic +# (`class imathify[T]`) — currently impractical because element types +# are determined at runtime and arithmetic mixes types. +T = TypeVar('T') from .it import take, rev, window from .gmemo import imemoize, gmemoize @@ -57,7 +66,12 @@ class _NoSuchType: mpf = _NoSuchType mpf_almosteq = None -def _numsign(x): +try: + import sympy +except ImportError: # pragma: no cover, optional at runtime, but installed at development time. + sympy = None + +def _numsign(x: Any) -> int: """The sign function, for numeric inputs.""" if x == 0: return 0 @@ -65,7 +79,7 @@ def _numsign(x): try: from sympy import log as _symlog, Expr as _symExpr, sign as _symsign - def log(x, b=None): + def log(x: Any, b: Any = None) -> Any: """The logarithm function. Works for both numeric and symbolic (`SymPy.Expr`) inputs. @@ -82,7 +96,7 @@ def log(x, b=None): return math_log(x, b) else: return math_log(x) - def sign(x): + def sign(x: Any) -> Any: """The sign function. Works for both numeric and symbolic (`SymPy.Expr`) inputs. @@ -96,7 +110,7 @@ def sign(x): _symExpr = _NoSuchType -def s(*spec): +def s(*spec: Any) -> "imathify": """Create a lazy mathematical sequence. The sequence is returned as a generator object that supports infix math @@ -263,13 +277,30 @@ def s(*spec): """ origspec = spec # for error messages - def is_almost_int(x): + def is_almost_int(x: Any) -> bool: try: + if sympy and isinstance(x, sympy.Expr): + x = sympy.N(x) return almosteq(float(round(x)), x) except TypeError: # likely a SymPy expression that didn't simplify to a number return False - def analyze(*spec): # raw spec (part before '...' if any) --> description + def analyze(*spec: Any) -> tuple[str, Any, Any | None]: + """Classify a raw sequence spec (the elements before ``...``) into a description. + + Returns ``(seqtype, x0, k)`` where: + + - ``seqtype``: ``"const"``, ``"arith"``, ``"geom"``, or ``"power"`` + - ``x0``: initial value (first element) + - ``k``: sequence parameter — ``None`` for const, common difference ``d`` + for arith, common ratio ``r`` for geom, exponent ``p`` for power + + Requires 1–3 spec elements to identify the sequence type. More elements + are accepted if consistent (checked by analyzing overlapping triplets). + + Cyclic sequences and ``Ellipsis`` handling are done by the caller (``s()``) + before ``analyze`` is called; this function only sees the numeric elements. + """ n = len(spec) if n == 1: a0 = spec[0] @@ -314,7 +345,7 @@ def analyze(*spec): # raw spec (part before '...' if any) --> description else: # more elements are optional but must be consistent data = [analyze(*triplet) for triplet in window(3, spec)] seqtypes, x0s, ks = zip(*data) - def isconst(xs): + def isconst(xs: tuple[Any, ...]) -> bool: first, *rest = xs return all(almosteq(x, first) for x in rest) if not isconst(seqtypes) or not isconst(ks): @@ -323,36 +354,48 @@ def isconst(xs): raise SyntaxError(f"Inconsistent specification '{origspec}'") return data[0] - # final term handler for finite sequences - compute how many terms we should generate in total infty = float("inf") - def nofterms(desc, elt): # return total number of terms in sequence or False + def nofterms(desc: tuple[str, Any, Any | None], elt: Any) -> int | float | bool: + """Compute total number of terms for a finite sequence with a final element. + + ``desc`` is a sequence descriptor ``(seqtype, x0, k)`` as returned by + ``analyze``. ``elt`` is the final element specified by the user. + + Returns the total term count (``int``), ``float("+inf")`` if the length + cannot be determined (constant sequence matching its own value), or + ``False`` if ``elt`` does not belong to the described sequence. + + For geometric and power sequences, an alternating-sign parity check + ensures ``elt`` has the correct sign for its position. + """ seqtype, x0, k = desc if seqtype == "const": if elt == x0: - return infty # cannot determine how many items in a '...''d constant sequence + return infty elif seqtype == "arith": - # elt = x0 + a*k --> a = (elt - x0) / k - a = (elt - x0) / k + a = (elt - x0) / k # elt = x0 + a*k if is_almost_int(a) and a > 0: return int(1 + round(a)) # fencepost elif seqtype == "geom": - # elt = x0*(k**a) --> k**a = (elt/x0) --> a = logk(elt/x0) - a = log(abs(elt / x0), abs(k)) + a = log(abs(elt / x0), abs(k)) # elt = x0*(k**a) if is_almost_int(a) and a > 0: - if not almosteq(x0 * (k**a), elt): # check parity of final term, could be an alternating sequence + if not almosteq(x0 * (k**a), elt): # parity check for alternating sequences return False return int(1 + round(a)) else: # seqtype == "power": - # elt = x0**(k**a) --> k**a = logx0 elt --> a = logk (logx0 elt) - a = log(log(abs(elt), abs(x0)), abs(k)) + a = log(log(abs(elt), abs(x0)), abs(k)) # elt = x0**(k**a) if is_almost_int(a) and a > 0: - if not almosteq(x0**(k**a), elt): # parity + if not almosteq(x0**(k**a), elt): # parity check return False return int(1 + round(a)) return False - # v0.14.3+: cyclic infinite sequences - def iscyclic(spec): + def iscyclic(spec: tuple[Any, ...]) -> bool: + """Check whether ``spec`` describes a cyclic sequence. + + A cyclic spec has a ``list`` as its last element, marking the repeating + cycle: ``(*initials, [*repeats])``. The list must be non-empty. + """ assert len(spec) >= 1 *maybe_initial, maybe_repeating = spec if isinstance(maybe_repeating, list): @@ -361,18 +404,18 @@ def iscyclic(spec): return True return False - # analyze the specification - if Ellipsis not in spec: # convenience fallback - if iscyclic(spec): + # Analyze the specification. We parse from the right, peeling off the trailing elements to determine which case we're in. + if Ellipsis not in spec: # no `...` — convenience fallback, explicit enumeration of all elements. + if iscyclic(spec): # a finite sequence can't be cyclic raise SyntaxError("Expected final ... for cyclic sequence.") return imathify(x for x in spec) - else: + else: # has a `...` + # Peel off the last element to see where the `...` is. *spec, last = spec - if last is Ellipsis: + if last is Ellipsis: # s(a0, a1, ...) or s([*repeats], ...) — infinite sequence. if not spec: - raise SyntaxError(f"Expected s(a0, a1, ...), s(a0, a1, ..., an), s([*repeats], ...), or s(*initials, [*repeats], ...); got '{origspec}'") + raise SyntaxError(f"Expected s(a0, a1, ...) or s(a0, a1, ..., an), s([*repeats], ...), or s(*initials, [*repeats], ...); got '{origspec}'") assert spec # not empty - # v0.14.3+: cyclic infinite sequences if iscyclic(spec): seqtype = "cyclic" *initial, repeating = spec @@ -380,7 +423,8 @@ def iscyclic(spec): else: seqtype, x0, k = analyze(*spec) n = infty - else: + else: # s(a0, a1, ..., an) — finite sequence with final element `last`. + # Peel off the `...` (now second-to-last) and analyze the formula. *spec, dots = spec if not (dots is Ellipsis and spec): raise SyntaxError(f"Expected s(a0, a1, ...) or s(a0, a1, ..., an), s([*repeats], ...), or s(*initials, [*repeats], ...); got '{origspec}'") @@ -397,7 +441,7 @@ def iscyclic(spec): if seqtype == "const": return imathify(repeat(x0) if n is infty else repeat(x0, n)) elif seqtype == "cyclic": - def cyclic(): + def cyclic() -> Iterator[Any]: yield from initial while True: yield from repeating @@ -405,7 +449,7 @@ def cyclic(): elif seqtype == "arith": # itertools.count doesn't avoid accumulating roundoff error for floats, so we implement our own. # This should be, for any j, within 1 ULP of the true result. - def arith(): + def arith() -> Iterator[Any]: j = 0 while True: yield x0 + j * k @@ -413,7 +457,7 @@ def arith(): return imathify(arith() if n is infty else take(n, arith())) elif seqtype == "geom": if isinstance(k, _symExpr) or abs(k) >= 1: - def geoimathify(): + def geom() -> Iterator[Any]: j = 0 while True: yield x0 * (k**j) @@ -425,22 +469,22 @@ def geoimathify(): # Note that 1/(1/3) --> 3.0 even for floats, so we don't actually # need to modify the detection algorithm to account for this. kinv = 1 / k - def geoimathify(): + def geom() -> Iterator[Any]: j = 0 while True: yield x0 / (kinv**j) j += 1 - return imathify(geoimathify() if n is infty else take(n, geoimathify())) + return imathify(geom() if n is infty else take(n, geom())) else: # seqtype == "power": if isinstance(k, _symExpr) or abs(k) >= 1: - def power(): + def power() -> Iterator[Any]: j = 0 while True: yield x0**(k**j) j += 1 else: kinv = 1 / k - def power(): + def power() -> Iterator[Any]: j = 0 while True: yield x0**(1 / (kinv**j)) @@ -452,26 +496,34 @@ def power(): class imathify: """Endow any iterable with infix math support (termwise). - The original iterable is saved to an attribute, and ``m.__iter__`` redirects - to it. No caching is performed, so performing a math operation on the m'd + The original iterable is saved to an attribute, and ``imathify.__iter__`` redirects + to it. No caching is performed, so performing a math operation on the imathified iterable will still consume the iterable (if it is consumable, for example a generator). This adds infix math only; to apply a function (e.g. ``sin``) termwise to - an iterable, use the comprehension syntax or ``map``, as usual. + an iterable, use ``slift1`` (or ``slift2`` for binary operations):: + + from math import sin + ssin = slift1(sin) + sinseq = ssin(s(1, 2, ...)) + + Or, for one-off use, wrap a generator expression in ``imathify``:: + + sinseq = imathify(sin(x) for x in a) The mathematical sequences (Python-technically, iterables) returned by - ``s()`` are automatically m'd, as is the result of any infix arithmetic - operation performed on an already m'd iterable. + ``s()`` are automatically imathified, as is the result of any infix arithmetic + operation performed on an already imathified iterable. **CAUTION**: When an operation meant for general iterables is applied to an m'd iterable, the math support vanishes (because the operation returns a - general iterable, not an m'd one), but can be restored by m'ing again. + general iterable, not an imathified one), but can be restored by m'ing again. **NOTE**: The function versions of the operations (``sadd`` etc.) work on - general iterables (so you don't need to ``m`` their inputs), and return - an m'd iterable. The ``m`` operation is only needed for infix math, to make - arithmetic-heavy code more readable. + general iterables (so you don't need to ``imathify`` their inputs), and return + an imathified iterable. The ``imathify`` operation is only needed for infix math, + to make arithmetic-heavy code more readable. Examples:: @@ -498,77 +550,77 @@ class imathify: https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types """ - def __init__(self, iterable): + def __init__(self, iterable: Iterable[Any]) -> None: self._g = iterable - def __iter__(self): + def __iter__(self) -> Iterator[Any]: return iter(self._g) - def __add__(self, other): + def __add__(self, other: Any) -> "imathify": return sadd(self, other) - def __radd__(self, other): + def __radd__(self, other: Any) -> "imathify": return sadd(other, self) - def __sub__(self, other): + def __sub__(self, other: Any) -> "imathify": return ssub(self, other) - def __rsub__(self, other): + def __rsub__(self, other: Any) -> "imathify": return ssub(other, self) - def __abs__(self): + def __abs__(self) -> "imathify": return sabs(self) - def __pos__(self): + def __pos__(self) -> "imathify": return spos(self) - def __neg__(self): + def __neg__(self) -> "imathify": return sneg(self) - def __invert__(self): + def __invert__(self) -> "imathify": return sinvert(self) - def __mul__(self, other): + def __mul__(self, other: Any) -> "imathify": return smul(self, other) - def __rmul__(self, other): + def __rmul__(self, other: Any) -> "imathify": return smul(other, self) - def __truediv__(self, other): + def __truediv__(self, other: Any) -> "imathify": return struediv(self, other) - def __rtruediv__(self, other): + def __rtruediv__(self, other: Any) -> "imathify": return struediv(other, self) - def __floordiv__(self, other): + def __floordiv__(self, other: Any) -> "imathify": return sfloordiv(self, other) - def __rfloordiv__(self, other): + def __rfloordiv__(self, other: Any) -> "imathify": return sfloordiv(other, self) - def __divmod__(self, other): + def __divmod__(self, other: Any) -> "imathify": return sdivmod(self, other) - def __rdivmod__(self, other): + def __rdivmod__(self, other: Any) -> "imathify": return sdivmod(other, self) - def __mod__(self, other): + def __mod__(self, other: Any) -> "imathify": return smod(self, other) - def __rmod__(self, other): + def __rmod__(self, other: Any) -> "imathify": return smod(other, self) - def __pow__(self, other, *mod): - return spow(self, other, *mod) - def __rpow__(self, other): + def __pow__(self, other: Any, mod: int | None = None) -> "imathify": + return spow(self, other, mod) + def __rpow__(self, other: Any) -> "imathify": return spow(other, self) - def __round__(self, *ndigits): - return sround(self, *ndigits) - def __trunc__(self): + def __round__(self, ndigits: int | None = None) -> "imathify": + return sround(self, ndigits) + def __trunc__(self) -> "imathify": return strunc(self) - def __floor__(self): + def __floor__(self) -> "imathify": return sfloor(self) - def __ceil__(self): + def __ceil__(self) -> "imathify": return sceil(self) - def __lshift__(self, other): + def __lshift__(self, other: Any) -> "imathify": return slshift(self, other) - def __rlshift__(self, other): + def __rlshift__(self, other: Any) -> "imathify": return slshift(other, self) - def __rshift__(self, other): + def __rshift__(self, other: Any) -> "imathify": return srshift(self, other) - def __rrshift__(self, other): + def __rrshift__(self, other: Any) -> "imathify": return srshift(other, self) - def __and__(self, other): + def __and__(self, other: Any) -> "imathify": return sand(self, other) - def __rand__(self, other): + def __rand__(self, other: Any) -> "imathify": return sand(other, self) - def __xor__(self, other): + def __xor__(self, other: Any) -> "imathify": return sxor(self, other) - def __rxor__(self, other): + def __rxor__(self, other: Any) -> "imathify": return sxor(other, self) - def __or__(self, other): + def __or__(self, other: Any) -> "imathify": return sor(self, other) - def __ror__(self, other): + def __ror__(self, other: Any) -> "imathify": return sor(other, self) # Can't do this because each of these conversion operators must return an # instance of that primitive type. @@ -580,20 +632,22 @@ def __ror__(self, other): # return sint(self) # def __float__(self): # return sfloat(self) - def __lt__(self, other): + def __lt__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return slt(self, other) - def __le__(self, other): + def __le__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return sle(self, other) - def __eq__(self, other): + def __eq__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return seq(self, other) - def __ne__(self, other): + def __ne__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return sne(self, other) - def __ge__(self, other): + def __ge__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return sge(self, other) - def __gt__(self, other): + def __gt__(self, other: Any) -> "imathify": # type: ignore[override] # termwise, not scalar return sgt(self, other) -def gmathify(gfunc): +Iterable.register(imathify) + +def gmathify(gfunc: Callable[..., Iterable[Any]]) -> Callable[..., imathify]: """Decorator: make gfunc imathify() the returned generator instances. Return a new gfunc, which passes all its arguments to the original ``gfunc``. @@ -606,7 +660,7 @@ def gmathify(gfunc): assert last(take(5, a() + a())) == 10 """ @wraps(gfunc) - def mathify(*args, **kwargs): + def mathify(*args: Any, **kwargs: Any) -> imathify: return imathify(gfunc(*args, **kwargs)) return mathify @@ -614,18 +668,57 @@ def mathify(*args, **kwargs): # We expose the full set of "imathify" operators also as functions à la the ``operator`` module. # Prefix "s", short for "mathematical Sequence". # https://docs.python.org/3/library/operator.html +# +# But first, let's define some factories. -# The *settings mechanism is used by round and pow. -# These are recursive to support iterables containing iterables (e.g. an iterable of math sequences). -def _make_termwise_stream_unop(op, *settings): - def stream_op(a): - if hasattr(a, "__iter__"): +def slift1(op: Callable[..., T], *settings: Any) -> Callable[[Iterable[T] | T], imathify | T]: + """Lift a scalar unary operation to work termwise on iterables. + + Returns a function that, given an iterable, lazily applies ``op`` to + each element and returns an imathified generator. Scalar inputs + are passed through to ``op`` directly. Recurses into nested iterables. + + Any extra ``settings`` are appended to each call to ``op``, e.g. + ``slift1(round, 2)`` gives termwise ``round(x, 2)``. + + Example:: + + from math import sin + ssin = slift1(sin) + result = ssin(s(1, 2, 3, ...)) # termwise sin + + All the built-in ``s``-prefixed unary operators (``sabs``, ``sneg``, ...) + are defined using this mechanism. + """ + def stream_op(a: Iterable[T] | T) -> imathify | T: + if isinstance(a, Iterable): return imathify(stream_op(x) for x in a) return op(a, *settings) return stream_op -def _make_termwise_stream_binop(op, *settings): - def stream_op(a, b): - isiterable = [hasattr(x, "__iter__") for x in (a, b)] + +def slift2(op: Callable[..., T], *settings: Any) -> Callable[[Iterable[T] | T, Iterable[T] | T], imathify | T]: + """Lift a scalar binary operation to work termwise on iterables. + + Returns a function that, given two inputs (either or both iterables), + lazily applies ``op`` termwise and returns an imathified generator. + When both inputs are iterables, ``zip`` semantics apply (terminates at + the shorter). When one input is scalar, it is broadcast. Recurses into + nested iterables. + + Any extra ``settings`` are appended to each call to ``op``, e.g. + ``slift2(pow, 5)`` gives termwise ``pow(a, b, 5)``. + + Example:: + + from math import atan2 + satan2 = slift2(atan2) + result = satan2(s(1, 2, 3, ...), s(4, 5, 6, ...)) # termwise atan2 + + All the built-in ``s``-prefixed binary operators (``sadd``, ``smul``, ...) + are defined using this mechanism. + """ + def stream_op(a: Iterable[T] | T, b: Iterable[T] | T) -> imathify | T: + isiterable = [isinstance(x, Iterable) for x in (a, b)] if all(isiterable): # it's very convenient here that zip() terminates when the shorter input runs out. return imathify(stream_op(x, y) for x, y in zip(a, b)) @@ -639,40 +732,42 @@ def stream_op(a, b): return op(a, b, *settings) return stream_op -sadd = _make_termwise_stream_binop(primitive_add) +# With these factories, the operators are just: + +sadd = slift2(atom_add) sadd.__doc__ = """Termwise a + b when one or both are iterables.""" -ssub = _make_termwise_stream_binop(primitive_sub) +ssub = slift2(atom_sub) ssub.__doc__ = """Termwise a - b when one or both are iterables.""" -sabs = _make_termwise_stream_unop(abs) +sabs = slift1(abs) sabs.__doc__ = """Termwise abs(a) for an iterable.""" -spos = _make_termwise_stream_unop(primitive_pos) +spos = slift1(atom_pos) spos.__doc__ = """Termwise +a for an iterable.""" -sneg = _make_termwise_stream_unop(primitive_neg) +sneg = slift1(atom_neg) sneg.__doc__ = """Termwise -a for an iterable.""" -smul = _make_termwise_stream_binop(primitive_mul) +smul = slift2(atom_mul) smul.__doc__ = """Termwise a * b when one or both are iterables.""" -_pow = _make_termwise_stream_binop(primitive_pow) # 2-arg form -def spow(a, b, *mod): +_spow = slift2(atom_pow) # 2-arg form +def spow(a: Any, b: Any, mod: int | None = None) -> Any: """Termwise a ** b when one or both are iterables. An optional third argument is supported, and passed through to the built-in ``pow`` function. """ - op = _make_termwise_stream_binop(pow, mod[0]) if mod else _pow - return op(a, b) + stream_op = slift2(pow, mod) if mod is not None else _spow + return stream_op(a, b) -struediv = _make_termwise_stream_binop(primitive_truediv) +struediv = slift2(atom_truediv) struediv.__doc__ = """Termwise a / b when one or both are iterables.""" -sfloordiv = _make_termwise_stream_binop(primitive_floordiv) +sfloordiv = slift2(atom_floordiv) sfloordiv.__doc__ = """Termwise a // b when one or both are iterables.""" -smod = _make_termwise_stream_binop(primitive_mod) +smod = slift2(atom_mod) smod.__doc__ = """Termwise a % b when one or both are iterables.""" -sdivmod = _make_termwise_stream_binop(divmod) +sdivmod = slift2(divmod) sdivmod.__doc__ = """Termwise (a // b, a % b) when one or both are iterables.""" -_round = _make_termwise_stream_unop(round) # 1-arg form -def sround(a, *ndigits): +_sround = slift1(round) # 1-arg form +def sround(a: Any, ndigits: int | None = None) -> Any: """Termwise round(a) for an iterable. An optional second argument is supported, and passed through to the @@ -683,28 +778,28 @@ def sround(a, *ndigits): https://docs.python.org/3/library/functions.html#round """ - op = _make_termwise_stream_unop(round, ndigits[0]) if ndigits else _round - return op(a) + stream_op = slift1(round, ndigits) if ndigits is not None else _sround + return stream_op(a) -strunc = _make_termwise_stream_unop(trunc) +strunc = slift1(trunc) strunc.__doc__ = """Termwise math.trunc(a) for an iterable.""" -sfloor = _make_termwise_stream_unop(floor) +sfloor = slift1(floor) sfloor.__doc__ = """Termwise math.floor(a) for an iterable.""" -sceil = _make_termwise_stream_unop(ceil) +sceil = slift1(ceil) sceil.__doc__ = """Termwise math.ceil(a) for an iterable.""" # bit twiddling operations -slshift = _make_termwise_stream_binop(primitive_lshift) +slshift = slift2(atom_lshift) slshift.__doc__ = """Termwise a << b when one or both are iterables.""" -srshift = _make_termwise_stream_binop(primitive_rshift) +srshift = slift2(atom_rshift) srshift.__doc__ = """Termwise a >> b when one or both are iterables.""" -sand = _make_termwise_stream_binop(primitive_and) +sand = slift2(atom_and) sand.__doc__ = """Termwise a & b when one or both are iterables.""" -sxor = _make_termwise_stream_binop(primitive_xor) +sxor = slift2(atom_xor) sxor.__doc__ = """Termwise a ^ b when one or both are iterables.""" -sor = _make_termwise_stream_binop(primitive_or) +sor = slift2(atom_or) sor.__doc__ = """Termwise a | b when one or both are iterables.""" -sinvert = _make_termwise_stream_unop(primitive_invert) +sinvert = slift1(atom_invert) sinvert.__doc__ = """Termwise ~a for an iterable. Note this is a bitwise invert, which is usually not what you want. @@ -719,36 +814,37 @@ def sround(a, *ndigits): # Can't do this because each of these conversion operators must return an # instance of that primitive type. # -# sbool = _make_termwise_stream_unop(bool) +# sbool = slift1(bool) # sbool.__doc__ = """Termwise bool(a) for an iterable.""" -# scomplex = _make_termwise_stream_unop(complex) +# scomplex = slift1(complex) # scomplex.__doc__ = """Termwise complex(a) for an iterable.""" -# sint = _make_termwise_stream_unop(int) +# sint = slift1(int) # sint.__doc__ = """Termwise int(a) for an iterable.""" -# sfloat = _make_termwise_stream_unop(float) +# sfloat = slift1(float) # sfloat.__doc__ = """Termwise float(a) for an iterable.""" -slt = _make_termwise_stream_binop(primitive_lt) +slt = slift2(atom_lt) slt.__doc__ = """Termwise a < b when one or both are iterables.""" -sle = _make_termwise_stream_binop(primitive_le) +sle = slift2(atom_le) sle.__doc__ = """Termwise a <= b when one or both are iterables.""" -seq = _make_termwise_stream_binop(primitive_eq) +seq = slift2(atom_eq) seq.__doc__ = """Termwise a == b when one or both are iterables.""" -sne = _make_termwise_stream_binop(primitive_ne) +sne = slift2(atom_ne) sne.__doc__ = """Termwise a != b when one or both are iterables.""" -sge = _make_termwise_stream_binop(primitive_ge) +sge = slift2(atom_ge) sge.__doc__ = """Termwise a >= b when one or both are iterables.""" -sgt = _make_termwise_stream_binop(primitive_gt) +sgt = slift2(atom_gt) sgt.__doc__ = """Termwise a > b when one or both are iterables.""" # ----------------------------------------------------------------------------- -def cauchyprod(a, b, *, require="any"): +def cauchyprod(a: Iterable[Any], b: Iterable[Any], *, + require: Literal["all", "any"] = "any") -> imathify: """Cauchy product of two (possibly infinite) iterables. Defined by:: - c[k] = suimathify(a[j] * b[k-j], j = 0, 1, ..., k), k = 0, 1, ... + c[k] = sum_j imathify(a[j] * b[k-j], j = 0, 1, ..., k), k = 0, 1, ... As a table:: @@ -771,12 +867,15 @@ def cauchyprod(a, b, *, require="any"): """ return diagonal_reduce(a, b, require=require, combine=smul, reduce=sum) -def diagonal_reduce(a, b, *, combine, reduce, require="any"): +def diagonal_reduce(a: Iterable[Any], b: Iterable[Any], *, + combine: Callable[[Iterable[Any], Iterable[Any]], Iterable[Any]], + reduce: Callable[[Iterable[Any]], Any], + require: Literal["all", "any"] = "any") -> imathify: """Diagonal combination-reduction for two (possibly infinite) iterables. Defined by:: - c[k] = reduce(combine(a[j], b[k-j]), j = 0, 1, ..., k), k = 0, 1, ... + c[k] = reduce_j(combine(a[j], b[k-j]), j = 0, 1, ..., k), k = 0, 1, ... As a table:: @@ -794,7 +893,7 @@ def diagonal_reduce(a, b, *, combine, reduce, require="any"): The Cauchy product is the special case with ``combine=smul, reduce=sum``. - The output is automatically m'd so that it supports infix arithmetic. + The output is automatically imathified so that it supports infix arithmetic. The operations: @@ -854,14 +953,13 @@ def diagonal_reduce(a, b, *, combine, reduce, require="any"): is not formed, because the terms ``a[0]*b[2]`` and ``a[2]*b[0]`` (that would contribute to it in the infinite case) cannot be formed from length-2 inputs. """ - # TODO: Python 3.8+: test for the appropriate `typing` Protocol instead? - if not all(hasattr(x, "__iter__") for x in (a, b)): + if not all(isinstance(x, Iterable) for x in (a, b)): raise TypeError(f"Expected two iterables, got {type(a)}, {type(b)}") if require not in ("all", "any"): raise ValueError(f"require must be 'all' or 'any'; got '{require}'") ga = imemoize(a) gb = imemoize(b) - def diagonal(): + def diagonal() -> Iterator[Any]: n = 1 # how many terms to take from a and b; output index k = n - 1 while True: xs, ys = (tuple(take(n, g())) for g in (ga, gb)) @@ -880,15 +978,37 @@ def diagonal(): # ----------------------------------------------------------------------------- -def fibonacci(): +def fibonacci() -> imathify: """Return the Fibonacci numbers 1, 1, 2, 3, 5, 8, ... as a lazy sequence.""" - def fibos(): + def fibos() -> Iterator[int]: a, b = 1, 1 while True: yield a a, b = b, a + b return imathify(fibos()) +def triangular() -> imathify: + """Return the triangular numbers 1, 3, 6, 10, ... as a lazy sequence. + + Etymology:: + + x + x x + x x x + x x x x + ... + """ + # We could just use Gauss's result n * (n + 1) / 2 (which can be proved by induction), + # but this algorithm is trivially correct. + def _triangular() -> Iterator[int]: + s = 1 # running total + r = 2 # places in the next row of the triangle + while True: + yield s + s += r + r += 1 + return imathify(_triangular()) + # See test_gmemo.py for history. This is an FP-ized sieve of Eratosthenes. # # This version wins in speed for moderate n (1e5) on typical architectures where @@ -897,16 +1017,16 @@ def fibos(): # larger as n grows (so memory transfers dominate for large n). That strategy # seems faster for n ~ 1e3, though. @gmemoize -def _primes(): +def _primes() -> Iterator[int]: yield 2 for n in count(start=3, step=2): if not any(n % p == 0 for p in takewhile(lambda x: x * x <= n, _primes())): yield n @gmemoize -def _fastprimes(): - memo = [] - def primes(): +def _fastprimes() -> Iterator[int]: + memo: list[int] = [] + def primes() -> Iterator[int]: memo.append(2) yield 2 for n in count(start=3, step=2): @@ -915,7 +1035,7 @@ def primes(): yield n return primes() -def primes(optimize="speed"): +def primes(optimize: Literal["memory", "speed"] = "speed") -> imathify: """Return the prime numbers 2, 3, 5, 7, 11, 13, ... as a lazy sequence. FP sieve of Eratosthenes with memoization. diff --git a/unpythonic/misc.py b/unpythonic/misc.py index 757db587..7a21c898 100644 --- a/unpythonic/misc.py +++ b/unpythonic/misc.py @@ -8,20 +8,28 @@ "Popper", "CountingIterator", "slurp", "callsite_filename", - "safeissubclass"] + "safeissubclass", + "maybe_open", "redirect_stdin", + "UnionFilter", + "si_prefix"] +from collections.abc import Callable, Iterable, Iterator +import contextlib from copy import copy from functools import partial -from itertools import count -import inspect -from queue import Empty -from sys import version_info -from time import monotonic -from types import CodeType, FunctionType, LambdaType +import logging +import pathlib +from queue import Empty, Queue +import sys +from time import perf_counter +from typing import Any, IO, TypeVar +from types import FunctionType, LambdaType, TracebackType + +F = TypeVar('F', bound=Callable) from .regutil import register_decorator -def pack(*args): +def pack(*args: Any) -> tuple: """Multi-argument constructor for tuples. In other words, the inverse of tuple unpacking, as a function. @@ -52,7 +60,7 @@ def p(loop, item, acc): return args # pretty much like in Lisps, (define (list . args) args) @register_decorator(priority=5) # allow sorting by unpythonic.syntax.sort_lambda_decorators -def namelambda(name): +def namelambda(name: str) -> Callable[[F], F]: """Rename a function. Decorator. This can be used to give a lambda a meaningful name, which is especially @@ -86,7 +94,7 @@ def namelambda(name): Note the inner lambda does not see the outer's new name. """ - def rename(f): + def rename(f: F) -> F: if not isinstance(f, (LambdaType, FunctionType)): # TODO: Can't raise TypeError; @fploop et al. do-it-now-and-replace-def-with-result # TODO: decorators need to do this. @@ -98,30 +106,7 @@ def rename(f): f.__name__ = name idx = f.__qualname__.rfind('.') f.__qualname__ = f"{f.__qualname__[:idx]}.{name}" if idx != -1 else name - # __code__.co_name is read-only, but there's a types.CodeType constructor - # that we can use to re-create the code object with the new name. - # (This is no worse than what the stdlib's Lib/modulefinder.py already does.) - co = f.__code__ - # https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py - # https://www.python.org/dev/peps/pep-0570/ - # https://docs.python.org/3/library/types.html#types.CodeType - # https://docs.python.org/3/library/inspect.html#types-and-members - if version_info >= (3, 8, 0): # Python 3.8+: positional-only parameters - f.__code__ = CodeType(co.co_argcount, co.co_posonlyargcount, co.co_kwonlyargcount, - co.co_nlocals, co.co_stacksize, co.co_flags, - co.co_code, co.co_consts, co.co_names, - co.co_varnames, co.co_filename, - name, - co.co_firstlineno, co.co_lnotab, co.co_freevars, - co.co_cellvars) - else: - f.__code__ = CodeType(co.co_argcount, co.co_kwonlyargcount, - co.co_nlocals, co.co_stacksize, co.co_flags, - co.co_code, co.co_consts, co.co_names, - co.co_varnames, co.co_filename, - name, - co.co_firstlineno, co.co_lnotab, co.co_freevars, - co.co_cellvars) + f.__code__ = f.__code__.replace(co_name=name) return f return rename @@ -141,21 +126,30 @@ class timer: for _ in range(int(1e7)): pass """ - def __init__(self, p=False): + def __init__(self, p: bool = False) -> None: """p: if True, print the delta-t when done. Regardless of ``p``, the result is always accessible as the ``dt``. """ self.p = p - def __enter__(self): - self.t0 = monotonic() + def __enter__(self) -> "timer": + # `perf_counter`, not `monotonic`: the former is documented as "a + # clock with the highest available resolution to measure a short + # duration" and is backed by `QueryPerformanceCounter` (~100 ns) on + # Windows, whereas `monotonic` is backed there by the ~16 ms + # tick-counter and would record `dt = 0` for microsecond-scale + # blocks (e.g. a PyPy-JIT'd tight loop). Both are monotonic; we + # only give up the "comparable across processes" guarantee of + # `monotonic`, which `timer` does not need since it only measures + # a dynamic extent in wall-clock time within a single process. + self.t0 = perf_counter() return self - def __exit__(self, exctype, excvalue, traceback): - self.dt = monotonic() - self.t0 + def __exit__(self, exctype: type[BaseException] | None, excvalue: BaseException | None, traceback: TracebackType | None) -> None: + self.dt = perf_counter() - self.t0 if self.p: print(self.dt) -def getattrrec(object, name, *default): +def getattrrec(object: Any, name: str, *default: Any) -> Any: """Extract the underlying data from an onion of wrapper objects. ``r = object.name``, and then get ``r.name`` recursively, as long as @@ -170,7 +164,7 @@ def getattrrec(object, name, *default): o = getattr(o, name, *default) return o -def setattrrec(object, name, value): +def setattrrec(object: Any, name: str, value: Any) -> None: """Inject data into the innermost layer in an onion of wrapper objects. See also ``getattrrec``. @@ -234,7 +228,7 @@ class Popper: Named after Karl Popper. """ - def __init__(self, seq): + def __init__(self, seq: Iterable[Any]) -> None: """seq: input container. Must support either ``popleft()`` or ``pop(0)``. Fully duck-typed. At least ``collections.deque`` and any @@ -242,9 +236,9 @@ def __init__(self, seq): """ self.seq = seq self._pop = seq.popleft if hasattr(seq, "popleft") else partial(seq.pop, 0) - def __iter__(self): + def __iter__(self) -> "Popper": return self - def __next__(self): + def __next__(self) -> Any: if self.seq: return self._pop() raise StopIteration @@ -253,19 +247,22 @@ def __next__(self): class CountingIterator: """Iterator that counts how many elements it has yielded. - The count stops updating when the original iterable raises StopIteration. + Wraps the original iterator of `iterable`. Simply use + `CountingIterator(iterable)` in place of `iter(iterable)`. + + The count stops updating when the original iterator raises StopIteration. """ - def __init__(self, iterable): + def __init__(self, iterable: Iterable[Any]) -> None: self._it = iter(iterable) - self.count = 0 - def __iter__(self): + self.count: int = 0 + def __iter__(self) -> "CountingIterator": return self - def __next__(self): + def __next__(self) -> Any: x = next(self._it) # let StopIteration propagate self.count += 1 return x -def slurp(queue): +def slurp(queue: Queue) -> list: """Slurp all items currently on a queue.Queue into a list. This retrieves items from the queue until it is empty, populates a list with them @@ -284,29 +281,201 @@ def slurp(queue): pass return out -def callsite_filename(): +_CALLSITE_TRANSPARENT = frozenset(( + "maybe_force_args", # lazify + "curried", "curry", "_currycall", # autocurry + "call", "callwith", # manual use of misc utils +)) + +def callsite_filename() -> str: """Return the filename of the call site, as a string. Useful as a building block for debug utilities and similar. - The filename is grabbed from the call stack using `inspect`. - This works also in the REPL (where `__file__` is undefined). + Skips over our own call-helpers (`call`, `callwith`, `curry` and + friends, lazify's `maybe_force_args`), so the *user's* call site is + reported. Works also in the REPL (where `__file__` is undefined). """ - stack = inspect.stack() - for k in count(start=1): # ignore callsite_filename() itself - framerecord = stack[k] - # ignore our call-helpers - if framerecord.function not in ("maybe_force_args", # lazify - "curried", "curry", "_currycall", # autocurry - "call", "callwith"): # manual use of misc utils - frame = framerecord.frame - filename = frame.f_code.co_filename - return filename - -def safeissubclass(cls, cls_or_tuple): + # We walk via `sys._getframe` rather than `inspect.stack`. `inspect.stack` + # calls `inspect.getframeinfo` for every frame on the way and reads source + # context lines around `f_lineno`, which raises `TypeError` if any frame + # in the walk has `f_lineno is None`. PyPy 3.11 / macOS / Windows hits + # exactly that: at least one frame on the way out of a `test[]` macro + # invocation reports `f_lineno = None`. Linux PyPy and CPython don't. + # We never use line info here; only `f_code.co_filename`. + frame = sys._getframe(1) # skip callsite_filename itself + while frame is not None: + if frame.f_code.co_name not in _CALLSITE_TRANSPARENT: + return frame.f_code.co_filename + frame = frame.f_back + raise RuntimeError("callsite_filename: no eligible frame on the call stack") + +def safeissubclass(cls: Any, cls_or_tuple: type | tuple[type, ...]) -> bool: """Like issubclass, but if `cls` is not a class, swallow the `TypeError` and return `False`.""" try: return issubclass(cls, cls_or_tuple) except TypeError: # "issubclass() arg 1 must be a class" pass return False + +# -------------------------------------------------------------------------------- +# I/O utilities + +@contextlib.contextmanager +def maybe_open(filename: str | pathlib.Path | None, + mode: str, + fallback: IO, + **kwargs) -> Iterator[IO]: + """Context manager: open a file, or use a fallback stream. + + Adapter that lets you always syntactically write + ``with maybe_open(...) as f:`` even when the target is a + standard stream like ``sys.stdin`` or ``sys.stdout``. + + ``filename``: path to open (``str`` or ``pathlib.Path``). + If ``None``, yield ``fallback`` instead. + ``mode``: as in the builtin ``open``. + ``fallback``: stream to use when ``filename is None``. Typical values + are ``sys.stdin`` (reading) and ``sys.stdout`` or + ``sys.stderr`` (writing). + ``**kwargs``: passed through to ``open``. + """ + if filename is not None: + with open(filename, mode, **kwargs) as f: + yield f + else: + yield fallback + + +class redirect_stdin(contextlib._RedirectStream): + """Context manager that feeds ``sys.stdin`` from *target*. + + The third sibling: the standard library ships + `contextlib.redirect_stdout` (Python 3.4) and + `contextlib.redirect_stderr` (Python 3.5), but not + `redirect_stdin`. This fills the gap, sharing machinery with its + stdlib siblings so the behavior matches exactly — including the + per-instance stack that supports nested re-entry on the same + instance. + + Like its stdlib siblings, this redirects the global ``sys.stdin`` + and is **not** safe under concurrent use from multiple threads; + parallel redirects from different threads will stomp on each other. + For tests (the primary use case), single-threaded use is the norm. + + Example:: + + from io import StringIO + from unpythonic import redirect_stdin + + with redirect_stdin(StringIO("42\\n")): + value = input() # reads "42" + """ + _stream = "stdin" + + +# -------------------------------------------------------------------------------- +# Logging utilities + +class UnionFilter(logging.Filter): + """A ``logging.Filter`` that matches if *any* sub-filter matches. + + The standard library provides ``logging.Filter`` for a single logger-name + prefix, but no OR combinator. ``UnionFilter`` fills the gap:: + + import logging + from unpythonic import UnionFilter + for handler in logging.root.handlers: + handler.addFilter(UnionFilter(logging.Filter("myapp.core"), + logging.Filter("myapp.io"))) + """ + def __init__(self, *filters: logging.Filter) -> None: + self.filters = filters + + def filter(self, record: logging.LogRecord) -> bool: + return any(f.filter(record) for f in self.filters) + +# -------------------------------------------------------------------------------- +# Number formatting + +def si_prefix(number: int | float, precision: int = 2, binary: bool = False, + separator: str = " ", always_separate: bool = False) -> str: + """Format a number with an SI decimal or IEC binary prefix. + + Returns a string like ``"1.50 k"``, ``"23.40 M"``, ``"500.00 m"`` + (milli), or ``"42.00"`` (no prefix for magnitudes in [1, base)). + + ``number``: the value to format (``int`` or ``float``). + ``precision``: decimal places (default 2). + ``binary``: if ``True``, use IEC binary prefixes (Ki, Mi, Gi, ...) + with base 1024 instead of SI decimal prefixes with + base 1000. Sub-unity binary prefixes (mi, µi, ni, ...) + follow the same convention. + ``separator``: what to put between the number and the prefix. + Defaults to a space, per SI. Pass ``""`` for the + compact form some UIs prefer (``"1.50k"``). + ``always_separate``: if ``True``, emit the separator even when there + is no prefix, so that ``"42.00 "`` lines up with + ``"1.50 k"``. + + The last one is for callers appending a unit of measurement — watts, + bytes, hertz. ``f"{si_prefix(n)}W"`` gives ``"1.50 kW"`` but + ``"42.00W"``, because in the second case there is no prefix to + separate from; ``always_separate=True`` makes the spacing uniform so + the unit can simply be appended:: + + f"{si_prefix(1536, binary=True, always_separate=True)}B" # "1.50 KiB" + f"{si_prefix(512, binary=True, always_separate=True)}B" # "512.00 B" + + Negative numbers and zero are handled correctly. + + In decimal mode (the default), both positive prefixes (k through Q) + and negative prefixes (m through q) are supported. The micro prefix + is ``µ`` (U+00B5 MICRO SIGN). + + Examples:: + + si_prefix(1500) # "1.50 k" + si_prefix(2_500_000) # "2.50 M" + si_prefix(0.0015) # "1.50 m" + si_prefix(0.0000025) # "2.50 µ" + si_prefix(-1500) # "-1.50 k" + si_prefix(42) # "42.00" + si_prefix(1536, binary=True) # "1.50 Ki" + si_prefix(2_621_440, binary=True) # "2.50 Mi" + si_prefix(0.5, binary=True) # "512.00 mi" + si_prefix(1500, separator="") # "1.50k" + + ``separator`` and ``always_separate`` were added in v2.3.0. + """ + if binary: + base = 1024 + large = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi', 'Ri', 'Qi') + small = ('mi', 'µi', 'ni', 'pi', 'fi', 'ai', 'zi', 'yi', 'ri', 'qi') + else: + base = 1000 + large = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y', 'R', 'Q') + small = ('m', 'µ', 'n', 'p', 'f', 'a', 'z', 'y', 'r', 'q') + def render(value: int | float, prefix: str) -> str: + # The empty prefix is the only case where the separator is in question: with nothing to separate + # from, SI writes the bare number, while a caller appending a unit wants the spacing kept uniform. + if prefix or always_separate: + return f"{value:.{precision}f}{separator}{prefix}" + return f"{value:.{precision}f}" + + if number == 0: + return render(0, "") + sign = -1 if number < 0 else 1 + magnitude = abs(number) + if magnitude >= 1: + for prefix in large: + if magnitude < base: + return render(sign * magnitude, prefix) + magnitude /= base + return render(sign * magnitude, large[-1]) + else: + for prefix in small: + magnitude *= base + if magnitude >= 1: + return render(sign * magnitude, prefix) + return render(sign * magnitude, small[-1]) diff --git a/unpythonic/monads/__init__.py b/unpythonic/monads/__init__.py new file mode 100644 index 00000000..ecf0fc82 --- /dev/null +++ b/unpythonic/monads/__init__.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +"""Monads for unpythonic. + +A monad is really just a design pattern, describable as: + +- chaining of operations with custom processing between steps, or +- generalization of function composition. + +The OO(F)P-ish approach taken here uses the class constructor for each +monad as its ``unit`` (in Haskell: ``return``), and spells bind as ``>>`` +via ``__rshift__``. (In Python the standard Haskell bind symbol ``>>=`` +maps to ``__irshift__``, which is an in-place operation that does not +chain, so we can't use that.) + +The general pattern: wrap an initial value with unit, then send it +through a sequence of monadic functions using bind. Each function in the +chain must use the same type of monad for the chain to compose. + +**Start here**: ``Maybe`` and ``List`` are perhaps the most important to +understand first — they're straightforward *container* monads. Move on +to ``Writer`` for another container-ish example, then ``State`` and +``Reader`` for the more mind-bending *computation* monads. + +Seven monads plus the two base classes: + +- ``Monad``, ``LiftableMonad`` — the base classes +- ``Identity`` — pedagogical no-op +- ``Maybe`` — simple short-circuiting on "nothing" +- ``Either``, ``Left``, ``Right`` — short-circuiting with a carried error +- ``List`` — non-deterministic / multivalued computation +- ``Writer`` — pure-functional audit log +- ``State`` — threading a state value through a pure chain +- ``Reader`` — reading from a shared immutable environment + +plus: + +- ``liftm``, ``liftm2``, ``liftm3`` — lift regular functions into monadic ones + +The subpackage is **not** re-exported at the top level of ``unpythonic`` — +import directly as ``from unpythonic.monads import Maybe``, etc. This is +because ``unpythonic`` is mostly lispy, not haskelly, and some of these +constructs have names that could be confusing in the top-level namespace. + +For do-notation syntax over any of these monads, see the macro +``from unpythonic.syntax import monadic_do``. + +**Approachable explanations**: + +- http://blog.sigfpe.com/2006/08/you-could-have-invented-monads-and.html +- http://nikgrozev.com/2013/12/10/monads-in-15-minutes/ +- https://stackoverflow.com/questions/44965/what-is-a-monad +- https://www.stephanboyer.com/post/9/monads-part-1-a-design-pattern +- https://www.stephanboyer.com/post/10/monads-part-2-impure-computations +- https://www.stephanboyer.com/post/83/super-quick-intro-to-monads +- http://learnyouahaskell.com/functors-applicative-functors-and-monoids + +**Further reading — other Python monad libraries**: + +- https://github.com/dbrattli/OSlash +- https://github.com/justanr/pynads +- https://bitbucket.org/jason_delaat/pymonad/ +- https://github.com/dpiponi/Monad-Python +- http://www.valuedlessons.com/2008/01/monads-in-python-with-nice-syntax.html + +This subpackage is ported from the teaching code at +https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/monads.py +""" + +from .abc import * # noqa: F401, F403 +from .core import * # noqa: F401, F403 + +from .identity import * # noqa: F401, F403 +from .maybe import * # noqa: F401, F403 +from .either import * # noqa: F401, F403 +from .list import * # noqa: F401, F403 +from .writer import * # noqa: F401, F403 +from .state import * # noqa: F401, F403 +from .reader import * # noqa: F401, F403 diff --git a/unpythonic/monads/abc.py b/unpythonic/monads/abc.py new file mode 100644 index 00000000..b421fec9 --- /dev/null +++ b/unpythonic/monads/abc.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +"""Monad base classes. + +Two-level split: + +- ``Monad``: the base class all monads inherit from. Requires ``__init__`` + (unit), ``fmap``, ``join``. Provides default implementations of + ``__rshift__`` (bind) and ``then`` (sequence) based on ``fmap`` + ``join``. + +- ``LiftableMonad(Monad)``: adds ``lift``, i.e. ``(a -> b) -> (a -> M b)``. + Used by monads where lift is well-defined in the usual "compose with unit" + sense (``Identity``, ``Maybe``, ``Either``, ``List``, ``Writer``). ``State`` + and ``Reader`` inherit from ``Monad`` directly — their ``lift`` is not + well-defined in that shape. + +Following unpythonic's duck-first philosophy (see ``unpythonic.slicing.Sliced`` +as the model), abstract methods are marked with ``@abstractmethod`` as an +intent marker for documentation; the classes are not strict ABCs. Enforcement +is soft: the decorator tells the reader what to implement, but instantiating +an incomplete subclass will not fail until an unimplemented method is called. +""" + +__all__ = ["Monad", "LiftableMonad"] + +from abc import abstractmethod +from collections.abc import Callable + + +class Monad: + """Base class for monads. + + A **must-override** method is tagged ``@abstractmethod`` and the docstring + says so. Other methods are concrete; override only for efficiency or if a + particular monad genuinely needs different semantics. + + Must override: + + - ``__init__`` (the unit operation): wrap a plain value into a monadic one. + Type: ``unit: a -> M a``. Not tagged ``@abstractmethod`` because every + Python class has its own ``__init__``; the contract is by convention. + + - ``fmap(self, f)``: apply ``f: a -> b`` inside the monad, returning + ``M b``. Type: ``fmap: M a -> (a -> b) -> M b``. + + - ``join(self)``: flatten a nested monadic value. + Type: ``join: M (M a) -> M a``. + + Provided (override only if needed): + + - ``__rshift__(self, f)`` (bind, Haskell ``>>=``): default + ``bind ma f = join (fmap f ma)``. Override e.g. for ``Writer``, which + bypasses ``fmap`` to avoid double-logging. + + - ``then(self, other)`` (sequence, Haskell ``>>``): default + ``self >> (lambda _: other)``. Rarely worth overriding. + + **Python note**. The usual Haskell bind symbol is ``>>=``, but in Python + that maps to ``__irshift__``, which is an in-place operation and does not + chain. We use ``>>`` (``__rshift__``) instead, consistent with the + teaching-code monads this subpackage is ported from. + """ + + @abstractmethod + def fmap(self, f: Callable) -> "Monad": + """The map operator. **Must override.** + + ``fmap: M a -> (a -> b) -> M b`` + + Apply the regular function ``f: a -> b`` to the value(s) inside this + monadic container, returning a new monadic value of the same type. + """ + ... + + @abstractmethod + def join(self) -> "Monad": + """The join operator. **Must override.** + + ``join: M (M a) -> M a`` + + Flatten a doubly-wrapped monadic value into a singly-wrapped one. + """ + ... + + def __rshift__(self, f: Callable) -> "Monad": + """Monadic bind (Haskell ``>>=``, spelled ``>>`` in Python). + + ``bind: M a -> (a -> M b) -> M b`` + + Default: ``bind ma f = join (fmap f ma)``. Override for efficiency + (e.g. ``Writer`` implements bind directly to avoid double-logging + via ``fmap``). + """ + return self.fmap(f).join() + + def then(self, other: "Monad") -> "Monad": + """Monadic sequence (Haskell ``>>``, spelled ``.then`` in Python). + + ``then: M a -> M b -> M b`` + + Like bind, but discarding the input value; yields ``other`` regardless + of what's inside ``self`` (subject to the monad's short-circuit rules, + e.g. ``Maybe(Empty).then(x) is Maybe(Empty)``). + """ + cls = self.__class__ + if not isinstance(other, cls): + raise TypeError(f"Expected a {cls.__name__}, got {type(other)} with value {other!r}") + return self >> (lambda _: other) + + +class LiftableMonad(Monad): + """A monad with a well-defined ``lift`` operation. + + Adds ``lift``, which promotes a regular function ``f: a -> b`` into a + monad-producing one ``a -> M b``. Default implementation: compose with + unit, i.e. ``lift f = lambda x: cls(f(x))``. + + Inherits from ``Monad``; the usual ``fmap``/``join``/unit contract still + applies. + + ``State`` and ``Reader`` deliberately do **not** inherit from this class — + their ``lift`` is not well-defined in the compose-with-unit sense. Use + ``Monad`` directly for those. + """ + + @classmethod + def lift(cls, f: Callable) -> Callable: + """Lift a regular function into a monad-producing one. + + ``lift: (a -> b) -> (a -> M b)`` + + Default: ``lift f = lambda x: cls(f(x))``, i.e. compose with unit. + Override if the monad needs a different construction (e.g. ``Writer`` + produces a log entry as part of the lift). + """ + return lambda x: cls(f(x)) diff --git a/unpythonic/monads/core.py b/unpythonic/monads/core.py new file mode 100644 index 00000000..ce01b07b --- /dev/null +++ b/unpythonic/monads/core.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +"""Monadic helpers that don't belong to any single monad. + +``liftm`` and its arity variants ``liftm2``, ``liftm3`` turn a regular +multi-argument function into a monadic one. See Haskell ``Control.Monad`` +for the originals (which go up to ``liftM8``). These three cover the +common cases; if more are ever needed, the pattern is obvious. + +Note the slight but important distinction between ``lift`` (on +``LiftableMonad``) and ``liftm`` here:: + + lift: f: (a -> b) -> lifted: (a -> M b) + liftm: f: (a -> r) -> lifted: (M a -> M r) + liftm2: f: ((a, b) -> r) -> lifted: ((M a, M b) -> M r) + liftm3: f: ((a, b, c) -> r) -> lifted: ((M a, M b, M c) -> M r) + +(Type signatures: each letter stands for a type such as int, str, .... +For example, ``f: (a -> r)`` means ``f`` is a function that takes a +single input parameter of type ``a`` and returns a value of type ``r``. +``M a`` roughly means "monad containing data of type ``a``".) + +Why the ``M`` in the input of ``liftm``'s result? Because in ``liftm`` +the *lifted* function binds, whereas ``lift`` expects the use site to +do that. + +Don't worry if this doesn't make sense at first — return to these +details once you've played with a few monad examples. The important +practical distinction: ``liftm`` takes monadic input and binds +internally; ``lift`` takes a plain value, wraps it, and hands you +something you then bind. +""" + +__all__ = ["liftm", "liftm2", "liftm3"] + +from collections.abc import Callable +from functools import wraps + +from .abc import Monad + + +def liftm(M: type, f: Callable) -> Callable: + """Lift a unary function into a monadic one. + + ``liftm: f: (a -> r) -> lifted: (M a -> M r)`` + + Given a regular function ``f: a -> r``, produce a function that takes + one monadic argument ``M a`` and returns ``M r``. The lifted function + binds internally using ``>>``. + + The first parameter ``M`` (the monad type) is fixed per call site and + changes rarely, so the signature is curry-friendly — use + ``partial(liftm, Maybe)`` to get a Maybe-specific lifter. + """ + @wraps(f) + def lifted(Mx: Monad) -> Monad: + if not isinstance(Mx, M): + raise TypeError(f"argument: expected monad {M}, got {type(Mx)} with data {Mx!r}") + return Mx >> (lambda x: + M(f(x))) + return lifted + + +def liftm2(M: type, f: Callable) -> Callable: + """Lift a binary function into a monadic one. + + ``liftm2: f: ((a, b) -> r) -> lifted: ((M a, M b) -> M r)`` + + Like `liftm`, but for two-argument ``f``. + """ + @wraps(f) + def lifted(Mx: Monad, My: Monad) -> Monad: + if not isinstance(Mx, M): + raise TypeError(f"first argument: expected monad {M}, got {type(Mx)} with data {Mx!r}") + if not isinstance(My, M): + raise TypeError(f"second argument: expected monad {M}, got {type(My)} with data {My!r}") + return Mx >> (lambda x: + My >> (lambda y: # noqa: E128 -- monadic style + M(f(x, y)))) + return lifted + + +def liftm3(M: type, f: Callable) -> Callable: + """Lift a ternary function into a monadic one. + + ``liftm3: f: ((a, b, c) -> r) -> lifted: ((M a, M b, M c) -> M r)`` + + Like `liftm`, but for three-argument ``f``. + """ + @wraps(f) + def lifted(Mx: Monad, My: Monad, Mz: Monad) -> Monad: + if not isinstance(Mx, M): + raise TypeError(f"first argument: expected monad {M}, got {type(Mx)} with data {Mx!r}") + if not isinstance(My, M): + raise TypeError(f"second argument: expected monad {M}, got {type(My)} with data {My!r}") + if not isinstance(Mz, M): + raise TypeError(f"third argument: expected monad {M}, got {type(Mz)} with data {Mz!r}") + return Mx >> (lambda x: + My >> (lambda y: # noqa: E128 -- monadic style + Mz >> (lambda z: # noqa: E128 -- monadic style + M(f(x, y, z))))) + return lifted diff --git a/unpythonic/monads/either.py b/unpythonic/monads/either.py new file mode 100644 index 00000000..d0acf6ab --- /dev/null +++ b/unpythonic/monads/either.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +"""The Either monad — Maybe's richer sibling. + +Where ``Maybe`` says "present or absent," ``Either`` says "succeeded or +failed, and here's what failed" — it carries an error value down the +short-circuit path instead of just a ``Nothing``. + +By convention, ``Right`` is the success path (pun intended: *right* also +means correct) and ``Left`` is the failure path. Binding through a +``Left`` short-circuits the rest of the chain, preserving the error:: + + from unpythonic.monads import Left, Right + + result = Right(10) >> (lambda x: Right(x + 1)) + # result == Right(11) + + result = Left("boom") >> (lambda x: Right(x + 1)) + # result == Left("boom") + +``Left`` and ``Right`` are sibling subclasses of ``Either``. Use +``Left`` / ``Right`` directly at the construction site; ``Either`` +itself is abstract. +""" + +__all__ = ["Either", "Left", "Right"] + +from collections.abc import Callable +from typing import Any + +from .abc import LiftableMonad + + +class Either(LiftableMonad): + """Abstract base for ``Left`` and ``Right``. + + Do not instantiate directly — use ``Left(err)`` for failure and + ``Right(val)`` for success. + + Overrides ``then`` from ``Monad`` so that ``self >> (lambda _: other)`` + accepts any ``Either`` on the RHS, not only the exact same subclass. + That is, ``Right(1).then(Left("boom"))`` works (and returns + ``Left("boom")``, since the right-hand side is the next step of the + computation). + """ + + def __init__(self, value: Any) -> None: + if type(self) is Either: + raise TypeError("Either is abstract; use Left(err) or Right(val)") + self.value = value + + @classmethod + def lift(cls, f: Callable) -> Callable: + """Lift into ``Right`` (the success path). ``Left``-lifting doesn't make sense.""" + return lambda x: Right(f(x)) + + def then(self, other: "Either") -> "Either": + if not isinstance(other, Either): + raise TypeError(f"Expected an Either, got {type(other)} with value {other!r}") + return self >> (lambda _: other) + + @classmethod + def guard(cls, b: Any, err: Any = "guard failed") -> "Either": + """Turn a boolean into a pass/short-circuit token. + + ``b`` truthy → dummy ``Right``; falsy → ``Left(err)``. Use ``.then`` + after to replace the dummy with the real result. + """ + if b: + return Right(True) + return Left(err) + + def __eq__(self, other: Any) -> bool: + if other is self: + return True + if not isinstance(other, Either): + return NotImplemented + return type(self) is type(other) and self.value == other.value + + def __hash__(self) -> int: + return hash((type(self), self.value)) + + def __repr__(self) -> str: # pragma: no cover + return f"{type(self).__name__}({self.value!r})" + + +class Left(Either): + """The failure path. Binding through a ``Left`` short-circuits.""" + + def fmap(self, f: Callable) -> "Either": + # Short-circuit: preserve the error; don't apply f. + return self + + def join(self) -> "Either": + # Short-circuit monad, same convention as ``Maybe(nil).join()``: + # there's no nested monad to unwrap (the payload is an error value, + # not an Either), and even in Haskell's typed form the Either monad + # instance has ``join (Left e) = Left e``. Return ``self`` so bind + # through Left stays Left. + return self + + +class Right(Either): + """The success path. Binding through a ``Right`` proceeds.""" + + def fmap(self, f: Callable) -> "Either": + return Right(f(self.value)) + + def join(self) -> "Either": + if not isinstance(self.value, Either): + raise TypeError(f"Expected a nested Either, got {type(self.value)} with data {self.value!r}") + return self.value diff --git a/unpythonic/monads/identity.py b/unpythonic/monads/identity.py new file mode 100644 index 00000000..463089ac --- /dev/null +++ b/unpythonic/monads/identity.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +"""The identity monad. + +Cf. the identity function. This is a no-op — just regular function +composition dressed as a monad. Its value is pedagogical: it shows the +monad structure in its simplest form, and serves as a sanity reference +when building other monads. +""" + +__all__ = ["Identity"] + +from collections.abc import Callable +from typing import Any + +from .abc import LiftableMonad + + +class Identity(LiftableMonad): + """The identity monad. + + Binding through ``Identity`` is the same as ordinary function + composition: ``Identity(x) >> f == f(x)`` (where ``f: a -> M b``). + + Usage:: + + from unpythonic.monads import Identity + + result = Identity(2) >> (lambda x: Identity(x + 1)) + # result == Identity(3) + """ + + def __init__(self, x: Any) -> None: + """Unit: wrap a plain value ``x: a`` into ``Identity a``.""" + self.x = x + + def fmap(self, f: Callable) -> "Identity": + """``fmap: Identity a -> (a -> b) -> Identity b``""" + cls = self.__class__ + return cls(f(self.x)) + + def join(self) -> "Identity": + """``join: Identity (Identity a) -> Identity a``""" + cls = self.__class__ + if not isinstance(self.x, cls): + raise TypeError(f"Expected a nested {cls.__name__}, got {type(self.x)} with data {self.x!r}") + return self.x + + def __eq__(self, other: Any) -> bool: + if other is self: + return True + if not isinstance(other, Identity): + return NotImplemented + return self.x == other.x + + def __hash__(self) -> int: + return hash((Identity, self.x)) + + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}({self.x!r})" diff --git a/unpythonic/monads/list.py b/unpythonic/monads/list.py new file mode 100644 index 00000000..922ba3b3 --- /dev/null +++ b/unpythonic/monads/list.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +"""The List monad — multivalued computations. + +One of the most genuinely useful monads in Python. Binding through a +``List`` is essentially ``flatMap``: each value in the list becomes a +sub-computation that produces its own list of results, and all the +sub-results are concatenated into a single flat list. + +The classical motivating example is McCarthy's *amb* operator +(non-deterministic choice) — which can be expressed as combining +``List`` monads in a do-notation. + +This module replaces the implementation that previously lived as +``MonadicList`` in ``unpythonic.amb``. ``amb.MonadicList`` is kept as +a deprecated alias, which will be removed in 3.0.0. + +**Constructor style**. The varargs form ``List(1, 2, 3)`` is primary +because it makes monadic unit the class itself: ``unit x = List(x)`` +(singleton list). ``List.from_iterable(xs)`` is the iterable form. + +**Empty lists**. The sentinel ``nil`` from ``unpythonic.llist`` is +accepted as a single-argument special case: ``List(nil)`` constructs +an empty list. This is analogous to Maybe's use of ``nil`` for +``Nothing``, and supports the ``liftm2``-style "no result" signaling +without needing a dedicated Empty singleton of our own. +""" + +__all__ = ["List"] + +from collections.abc import Callable, Iterable, Iterator, Sequence, Sized +from typing import Any + +from ..llist import nil + +from .abc import LiftableMonad + + +class List(LiftableMonad): + """The list monad.""" + + def __init__(self, *elts: Any) -> None: + """Construct a ``List`` from the given elements. + + Usage:: + + List() # empty + List(1) # singleton — the monadic unit + List(1, 2, 3) # three elements + List(nil) # also empty — sentinel form, convenient for + # liftm2-style "no result" signaling + + Use `from_iterable` to build a List from an existing iterable. + """ + # sentinel: a single-argument call with `nil` means "empty list." + # This is analogous to Maybe's convention (Maybe(nil) = Nothing), + # and lets liftm2/3-style constructions produce empty results + # without needing a separate Empty singleton of our own. + if len(elts) == 1 and elts[0] is nil: + self.x: tuple = () + else: + self.x = elts + + def fmap(self, f: Callable) -> "List": + """``fmap: List a -> (a -> b) -> List b`` + + Applies ``f`` to each element; result is a list of the same length. + """ + cls = self.__class__ + return cls.from_iterable(f(elt) for elt in self.x) + + def join(self) -> "List": + """``join: List (List a) -> List a`` + + Concatenates a list of lists into a single flat list. + """ + cls = self.__class__ + if not all(isinstance(elt, cls) for elt in self.x): + raise TypeError(f"Expected a nested {cls.__name__}, got {self.x!r}") + return cls.from_iterable(elt for sublist in self.x for elt in sublist) + + @classmethod + def guard(cls, b: Any) -> "List": + """Turn a boolean into a pass/short-circuit token for list monad filtering. + + ``b`` truthy → singleton dummy list (continues the branch); + ``b`` falsy → empty list (short-circuits this branch). Pair with + ``.then`` to yield the real result on success. + """ + if b: + return cls(True) # non-empty; value isn't used + return cls() # empty — short-circuits this branch + + @classmethod + def from_iterable(cls, iterable: Iterable) -> "List": + """Construct a ``List`` from an existing iterable. Eager.""" + # avoid the varargs special-case for single-nil by constructing directly + instance = cls.__new__(cls) + instance.x = tuple(iterable) + return instance + + # `unpythonic.collections.mogrify` uses `cls._make(iterable)` when available + # to reconstruct sequence-like containers element-by-element (matching the + # namedtuple convention). Without this, mogrify would fall back to + # `cls(iterable)` = varargs, which packs the whole iterable as a single + # element. This hook preserves correct behavior under ``lazify`` and other + # places that recursively rebuild containers. + _make = from_iterable + + def copy(self) -> "List": + """Return a shallow copy of this list.""" + return self.__class__.from_iterable(self.x) + + # Sequence ABC interface — registered below. + def __iter__(self) -> Iterator: + return iter(self.x) + + def __len__(self) -> int: + return len(self.x) + + def __getitem__(self, i: int) -> Any: + return self.x[i] + + def __reversed__(self) -> Iterator: + return reversed(self.x) + + def __contains__(self, value: Any) -> bool: + return value in self.x + + def index(self, value: Any) -> int: + return self.x.index(value) + + def count(self, value: Any) -> int: + return self.x.count(value) + + def __eq__(self, other: Any) -> bool: + if other is self: + return True + if isinstance(other, List): + return self.x == other.x + # Accept comparison against plain sequences for convenience. + try: + return len(self) == len(other) and all(a == b for a, b in zip(self.x, other)) + except TypeError: + return NotImplemented + + def __hash__(self) -> int: + return hash((List, self.x)) + + def __add__(self, other: "List") -> "List": + """Concatenation of Lists.""" + if not isinstance(other, List): + raise TypeError(f"Expected a List, got {type(other)} with value {other!r}") + return self.__class__.from_iterable(self.x + other.x) + + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}{self.x}" + + +# Register as a virtual subclass of the Sequence ABCs — matches the old +# MonadicList behavior so `isinstance(List(...), Sequence)` is True. +for _abscls in (Iterable, Sized, Sequence): + _abscls.register(List) +del _abscls diff --git a/unpythonic/monads/maybe.py b/unpythonic/monads/maybe.py new file mode 100644 index 00000000..62db5785 --- /dev/null +++ b/unpythonic/monads/maybe.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +"""The Maybe monad. + +Sketch of how to implement an exception system in pure FP. Not really +needed in Python for that purpose — Python has real exceptions — but a +clean informative example of a short-circuiting monad, and occasionally +handy in its own right when you want to thread "maybe a value, maybe +nothing" through a pipeline without crufting up the happy path with +explicit None checks. + +**Future improvement**: a proper Maybe in Haskell is an ADT (algebraic +data type) with two data constructors, ``Just x`` and ``Nothing``. We +could use mcpyrate (syntactic macros) together with ``@generic`` +(multiple-dispatch) to approximate that shape — case classes ``Just`` +and ``Nothing`` sharing a ``Maybe`` supertype, with pattern matching. +Not done here; the in-band encoding below is the direct port of the +teaching code. + +**Conventions** (no user-facing ``Just``/``Nothing`` wrapper classes — +just ``Maybe(value)``): + +- ``Maybe(x)`` for ``x is not nil`` wraps a present value. +- ``Maybe(nil)`` represents absence (``nil`` from ``unpythonic.llist`` + is unpythonic's project-wide "nothing" sentinel, chosen to avoid + proliferating null singletons). + +Trade-off: This encoding cannot wrap ``nil`` itself as a present value. +In all other cases this yields better UX vs. demanding a ``Some(...)`` +wrapper per value. +""" + +__all__ = ["Maybe"] + +from collections.abc import Callable +from typing import Any + +from ..llist import nil + +from .abc import LiftableMonad + + +class Maybe(LiftableMonad): + """The Maybe monad. ``Maybe(x)`` is ``Just x``; ``Maybe(nil)`` is ``Nothing``. + + Binding through ``Nothing`` short-circuits the rest of the chain:: + + from unpythonic.llist import nil + from unpythonic.monads import Maybe + + # happy path: one bind at a time walks the chain + result = Maybe(10) >> (lambda x: Maybe(x + 1)) + # result == Maybe(11) + + # short-circuit: the remaining lambdas are never called + result = Maybe(nil) >> (lambda x: Maybe(x + 1)) + # result == Maybe(nil) + """ + + def __init__(self, x: Any) -> None: + """Unit: wrap ``x: a`` into ``Maybe a``. + + Pass ``nil`` (from ``unpythonic.llist``) to construct ``Nothing``. + """ + self.x = x + + def fmap(self, f: Callable) -> "Maybe": + """``fmap: Maybe a -> (a -> b) -> Maybe b``. Preserves ``Nothing``.""" + if self.x is nil: + return self + cls = self.__class__ + return cls(f(self.x)) + + def join(self) -> "Maybe": + """``join: Maybe (Maybe a) -> Maybe a``. Preserves ``Nothing``.""" + if self.x is nil: + return self + cls = self.__class__ + if not isinstance(self.x, cls): + raise TypeError(f"Expected a nested {cls.__name__}, got {type(self.x)} with data {self.x!r}") + return self.x + + @classmethod + def guard(cls, b: Any) -> "Maybe": + """Turn a boolean into a pass/short-circuit token. + + ``guard: bool -> Maybe b`` + + When ``b`` is truthy, returns a dummy ``Just``; when falsy, returns + ``Nothing``. Typical use: ``Maybe(x) >> (lambda v: Maybe.guard(v > 0).then(Maybe(v)))`` + — the ``.then`` discards the guard's dummy and yields the value on + success, ``Nothing`` on failure. + """ + if b: + return cls(True) # dummy Just; the value isn't used + return cls(nil) # Nothing + + def __eq__(self, other: Any) -> bool: + if other is self: + return True + if not isinstance(other, Maybe): + return NotImplemented + # nil is a singleton; `is` comparison would also work, but == is fine too. + return self.x == other.x + + def __hash__(self) -> int: + return hash((Maybe, self.x)) + + def __repr__(self) -> str: + # Round-trippable: eval(repr(m)) reconstructs the Maybe (given nil is in scope). + return f"Maybe({self.x!r})" + + def __str__(self) -> str: # pragma: no cover + # Haskell-flavored display for humans: "Nothing" / "Just x". + if self.x is nil: + return "Nothing" + return f"Just {self.x!r}" diff --git a/unpythonic/monads/reader.py b/unpythonic/monads/reader.py new file mode 100644 index 00000000..8e3f5b6c --- /dev/null +++ b/unpythonic/monads/reader.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +"""The Reader monad — a read-only shared environment. + +**Mind-bending parts inside.** + +Something between a container and a computation. On the one hand, +``Reader e a`` is essentially just the function type ``e -> a`` with a +monad API wrapped around it; on the other, like ``State``, the +environment only becomes bound when we ``.run`` the Reader — until +then, everything is just planning. + +A ``Reader e a`` wraps a function ``e -> a``, where ``e`` is some +environment (configuration, dependency-injection context, etc.). Binding +threads a single environment ``e`` through the chain; each sub-computation +can ``.ask()`` for the environment and do something with it. + +Does **not** inherit from ``LiftableMonad`` — the teaching code leaves +``Reader.lift`` unimplemented, and there's no canonical shape for it. + +Based on: + +- https://wiki.haskell.org/Monads_as_containers +- https://www.mjoldfield.com/atelier/2014/08/monads-reader.html +- https://blog.ssanj.net/posts/2014-09-23-A-Simple-Reader-Monad-Example.html +- https://stackoverflow.com/questions/14178889/what-is-the-purpose-of-the-reader-monad +""" + +__all__ = ["Reader"] + +from collections.abc import Callable +from typing import Any + +from .abc import Monad + + +class Reader(Monad): + """The Reader monad. Wraps a function ``e -> a``. + + **What bind does**: taking a computation that may read from the + environment before producing a value of type ``a``, and a function + from values of type ``a`` to computations that may read from the + environment before returning a value of type ``b``, and composing + these — yielding a computation that may read from the (shared) + environment before returning a value of type ``b``. + + Uses the default ``Monad.__rshift__`` (``fmap . join``); no override + needed, the generic definition fits Reader perfectly. + + Usage:: + + from unpythonic.monads import Reader + + # A config-reading chain. + config = {"multiplier": 3, "offset": 10} + + chain = (Reader.asks(lambda e: e["multiplier"]) + >> (lambda m: Reader.asks(lambda e: e["offset"]) + >> (lambda o: Reader.unit(m * 5 + o)))) + + result = chain.run(config) + # result == 25 + """ + + def __init__(self, f: Callable) -> None: + """Wrap a reader function ``f: e -> a``. + + Essentially, ``Reader e a = (e -> a)``, with a thin monad wrapper. + """ + if not callable(f): + raise TypeError(f"Expected a callable e -> a, got {f!r}") + self.r = f + + @classmethod + def unit(cls, x: Any) -> "Reader": + """Unit: ``a -> Reader e a``. Ignores the environment.""" + return cls(lambda _: x) + + def run(self, env: Any) -> Any: + """Run the reader against an environment ``env: e``. Returns ``a``.""" + return self.r(env) + + @classmethod + def ask(cls) -> "Reader": + """Yield the environment itself as the data value. ``-> Reader e e``.""" + return cls(lambda env: env) + + @classmethod + def asks(cls, f: Callable) -> "Reader": + """Apply ``f: e -> a`` to the environment; yield ``a`` as data.""" + return cls.ask() >> (lambda env: cls.unit(f(env))) + + def local(self, f: Callable) -> "Reader": + """Run this computation in an ``f``-modified environment. ``f: e -> e``.""" + return self.__class__(lambda env: self.run(f(env))) + + def fmap(self, f: Callable) -> "Reader": + """``fmap: Reader e a -> (a -> b) -> Reader e b``""" + cls = self.__class__ + return cls(lambda env: f(self.run(env))) + + def join(self) -> "Reader": + """``join: Reader e (Reader e a) -> Reader e a`` + + Given a reader that yields another reader, run the outer reader to + get the inner, then run the inner with the same environment. + """ + cls = self.__class__ + return cls(lambda env: self.run(env).run(env)) + + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}({self.r!r})" diff --git a/unpythonic/monads/state.py b/unpythonic/monads/state.py new file mode 100644 index 00000000..eb5fc2d9 --- /dev/null +++ b/unpythonic/monads/state.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- +"""The State monad — threading a state value through a pure computation. + +**Warning**: mind-bending material. + +In Python, in the same vein as ``unfold()``, we don't really *need* the +State monad for its basic uses — generators already handle implicit +state nicely (though they use genuine destructive imperative updates, +whereas this doesn't). But it's worth studying, because in the process +we see a different way of thinking about monads. + +Where the container-style monads (``Identity``, ``Maybe``, ``List``, etc.) +wrap a *value*, a ``State`` wraps a *computation*: a function +``s -> (a, s)`` that takes an input state, produces a data value, and +returns a new state. The main idea is **monads as computation** rather +than monads as containers. + +**How it's used** — two alternating phases: + +1. State processor ``s -> (a, s)``: old state in; a data value and new + state out. +2. The code at the use site: do something with the data value ``a``, + then tell phase 1 which state processor to run next. + +The state ``s`` only becomes bound when the composed chain starts +running — and we start the chain only after we're done composing it. +In the call to ``.run(s0)``, we give the chain the initial state it +will start in; then the monad does the plumbing required to pass the +state across the state-processor calls, in a functional (FP) manner. +Just like in an FP loop, there is no mutation, but in effect, the state +changes (via fresh instances). Until the chain runs, everything is, so +to speak, just hypothetical — planning what we'll do once we get our +hands on an initial state value. This is an important difference from +the data-container monads. + +**On the three-chainee puzzle** (one of the most difficult points to +grasp at first): at first glance, it would seem the state processor in +the middle of a chain ``A >> B >> C`` runs twice — once as the second +operation of the first State instance, and again as the first operation +of the second State instance. But actually that's wrong. Binding is +essentially function composition, and we return the composed function. +Hence ``A >> B`` becomes a new composed state processor — call it ``D`` +— and the chain is transformed into ``D >> C``. At this point, *nothing +has actually run yet*; we are just planning what to do by building +composed functions. Now the second bind composes a new state processor +out of ``D`` and ``C``. When we eventually ``.run`` the chain, running +``D`` internally runs both ``A`` and ``B``, so each of ``A``, ``B``, +``C`` runs exactly once — as they should. + +The monad is, in effect, *shunting the state value around the code that +is only interested in the data*, and delivering the state only where +it's actually needed — into the actual state processors. + +**Type invariant**: in Haskell, the type of the state value stays the +same in a chain, whereas the type of the data value may change. Python +doesn't enforce that, but readers familiar with the typed version will +expect it. + +Based on: + +- http://brandon.si/code/the-state-monad-a-tutorial-for-the-confused/ +- https://wiki.haskell.org/Monads_as_computation +- https://en.wikibooks.org/wiki/Haskell/Understanding_monads/State +- https://wiki.haskell.org/State_Monad + +Does **not** inherit from ``LiftableMonad`` — ``lift f = a -> M b`` doesn't +have a useful shape for State (the lifted function would need to choose +what to do with the state; there's no canonical answer). +""" + +__all__ = ["State"] + +from collections.abc import Callable +from typing import Any + +from .abc import Monad + + +class State(Monad): + """The State monad. Wraps a state-processor function ``s -> (a, s)``. + + **Constructor vs. unit**: ``State(f)`` wraps an existing processor; + ``State.unit(a)`` wraps the value ``a`` as a state-ignoring processor + (``lambda s: (a, s)``). They are genuinely different, unlike in most + other monads where unit is just the constructor. + + Usage:: + + from unpythonic.monads import State + + # A counter: reads state, bumps it, returns previous value as data + bump = State(lambda s: (s, s + 1)) + + chain = (bump + >> (lambda a: bump + >> (lambda b: bump + >> (lambda c: State.unit((a, b, c)))))) + + result, final_state = chain.run(10) + # result == (10, 11, 12) + # final_state == 13 + """ + + def __init__(self, f: Callable) -> None: + """Wrap a state-processor function ``f: s -> (a, s)``.""" + if not callable(f): + raise TypeError(f"Expected a callable s -> (a, s), got {f!r}") + self.processor = f + + @classmethod + def unit(cls, a: Any) -> "State": + """Unit: ``a -> State(s -> (a, s))``. The state-ignoring processor.""" + return cls(lambda s: (a, s)) + + def run(self, s: Any) -> tuple: + """Run the wrapped processor starting from state ``s``. Returns ``(a, s')``.""" + return self.processor(s) + + def eval(self, s: Any) -> Any: + """Run and return just the data value (discarding the final state).""" + value, _ = self.run(s) + return value + + def exec(self, s: Any) -> Any: + """Run and return just the final state (discarding the data value).""" + _, final_state = self.run(s) + return final_state + + def __rshift__(self, f: Callable) -> "State": + """Monadic bind. Composes state processors. + + ``bind: State(s -> (a, s)) -> (a -> State(s -> (b, s))) -> State(s -> (b, s))`` + + Overridden (rather than using the Monad default of ``fmap . join``) + because direct composition is much clearer here than going through + the ``(M a)``-wrapping round trip. See the module docstring + references for a detailed derivation. + + Here ``f`` is expected to be ``a -> State(s -> (b, s))``: it takes + a *data value* (not a state value!) and returns a state processor. + What's this crazy kind of function? Somewhat similarly to "lambda + as a code block" in Lisp, it's not really a function in the usual + sense (though formally it is one) — it's the code block that the + chain binds into. It's a thing to be performed *between* two + processings of the state. So it makes sense that it takes the data + value (the ``a`` part of the result of the current state + processor), does something with it, and then tells us what to do + next — i.e. provides a new state processor. + + The beauty: the user-level code block *doesn't even see* the state + value. It only gets the data value of the result, just as if + computing with plain functions that need no state. The monad + shunts the state value around, delivering it only where it's + actually needed — into the actual state processors. + + See also the ``wrap`` / ``unwrap`` comments at + https://en.wikibooks.org/wiki/Haskell/Understanding_monads/State + """ + def composed(s: Any) -> tuple: + value, s_prime = self.run(s) # apply current processor + # Take the contained data value from inside the monad (= the + # data result of our wrapped computation) and send it to the + # user's code block. The block gives us a new State monad, + # which wraps the next state processor to run. + next_processor = f(value) + return next_processor.run(s_prime) # then apply the new processor + return State(composed) + + @classmethod + def get(cls) -> "State": + """Return the current state value as the data part. ``-> State(s -> (s, s))``.""" + return cls(lambda s: (s, s)) + + @classmethod + def put(cls, s: Any) -> "State": + """Replace the state with ``s``; yield ``None`` as data. ``s -> State(s -> (None, s))``.""" + return cls(lambda _: (None, s)) + + @classmethod + def modify(cls, f: Callable) -> "State": + """Apply ``f: s -> s`` to the state; yield ``None`` as data.""" + return cls.get() >> (lambda s: cls.put(f(s))) + + @classmethod + def gets(cls, f: Callable) -> "State": + """Run ``f: s -> a`` on the state; yield ``a`` as data, state unchanged.""" + return cls.get() >> (lambda s: cls.unit(f(s))) + + def fmap(self, f: Callable) -> "State": + """``fmap: State(s -> (a, s)) -> (a -> b) -> State(s -> (b, s))``""" + return self >> (lambda a: State.unit(f(a))) + + def join(self) -> "State": + """``join: State(s -> (State(s -> (a, s)), s)) -> State(s -> (a, s))`` + + Plain-words derivation: given ``mm : State(s -> (State(s -> (a, s)), s))``, + run the outer state function to get ``(inner_m, s')``, then run the inner + with ``s'`` — standard "thread the state" pattern. + """ + def joined(s: Any) -> tuple: + inner_m, s_prime = self.run(s) # outer yields (inner State, new state) + if not isinstance(inner_m, State): + raise TypeError( + f"Expected a nested State, got {type(inner_m)} with value {inner_m!r}" + ) + return inner_m.run(s_prime) # run inner with the threaded state + return State(joined) + + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}({self.processor!r})" diff --git a/unpythonic/monads/tests/__init__.py b/unpythonic/monads/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/unpythonic/monads/tests/test_monads.py b/unpythonic/monads/tests/test_monads.py new file mode 100644 index 00000000..bec01494 --- /dev/null +++ b/unpythonic/monads/tests/test_monads.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +"""Tests for the pure-Python monad subpackage.""" + +from math import sqrt + +from ...syntax import macros, test, test_raises, the # noqa: F401 +from ...test.fixtures import session, testset + +from ...llist import nil + +from .. import ( + Monad, LiftableMonad, + liftm, liftm2, liftm3, + Identity, Maybe, Either, Left, Right, List, Writer, State, Reader, +) + + +def runtests(): + with testset("Monad / LiftableMonad base classes"): + # Every monad subclass inherits from Monad. + for M in (Identity, Maybe, Either, Left, Right, List, Writer, State, Reader): + test[issubclass(M, Monad)] + + # The Liftable subset. + for M in (Identity, Maybe, Either, Left, Right, List, Writer): + test[issubclass(M, LiftableMonad)] + + # State and Reader are NOT LiftableMonad. + for M in (State, Reader): + test[not issubclass(M, LiftableMonad)] + + # isinstance works for concrete monad values. + test[isinstance(Identity(5), Monad)] + test[isinstance(Maybe(nil), LiftableMonad)] + test[isinstance(State.unit(1), Monad)] + test[not isinstance(State.unit(1), LiftableMonad)] + + # Default bind (fmap . join) actually fires for a minimal subclass. + # Use Identity as a stand-in — its __rshift__ comes from Monad (no override). + out = Identity(3) >> (lambda x: Identity(x + 7)) + test[out == Identity(10)] + + # Default then fires similarly. + out2 = Identity(1).then(Identity(99)) + test[out2 == Identity(99)] + + # then's type check rejects cross-monad sequencing. + test_raises[TypeError, Identity(1).then(Maybe(5))] + + with testset("liftm, liftm2, liftm3"): + lifted1 = liftm(Maybe, lambda x: x + 1) + test[lifted1(Maybe(5)) == Maybe(6)] + test[lifted1(Maybe(nil)) == Maybe(nil)] # short-circuit preserved + + # liftm requires a monadic argument. + test_raises[TypeError, lifted1(5)] + + lifted2 = liftm2(Identity, lambda x, y: x * y) + test[lifted2(Identity(3), Identity(4)) == Identity(12)] + test_raises[TypeError, lifted2(3, Identity(4))] + test_raises[TypeError, lifted2(Identity(3), 4)] + + lifted3 = liftm3(Identity, lambda x, y, z: x + y + z) + test[lifted3(Identity(1), Identity(2), Identity(3)) == Identity(6)] + test_raises[TypeError, lifted3(1, Identity(2), Identity(3))] + test_raises[TypeError, lifted3(Identity(1), 2, Identity(3))] + test_raises[TypeError, lifted3(Identity(1), Identity(2), 3)] + + with testset("Identity"): + test[Identity(42) == Identity(42)] + test[(Identity(2) >> (lambda x: Identity(x + 1))) == Identity(3)] + test[Identity(5).fmap(lambda x: x * 10) == Identity(50)] + test[Identity(Identity(7)).join() == Identity(7)] + test_raises[TypeError, Identity(5).join()] # not nested + test[Identity.lift(lambda x: x + 100)(5) == Identity(105)] + + with testset("Maybe"): + # Happy path + test[(Maybe(10) >> (lambda x: Maybe(x + 1))) == Maybe(11)] + + # Short-circuit: Nothing propagates; lambda never called + called = [] + def watcher(x): + called.append(x) + return Maybe(x + 1) + result = Maybe(nil) >> watcher + test[result == Maybe(nil)] + test[called == []] # watcher never invoked + + # fmap preserves Nothing + test[Maybe(nil).fmap(lambda x: x * 2) == Maybe(nil)] + test[Maybe(5).fmap(lambda x: x * 2) == Maybe(10)] + + # join + test[Maybe(Maybe(7)).join() == Maybe(7)] + test[Maybe(nil).join() == Maybe(nil)] + test_raises[TypeError, Maybe(5).join()] # not nested + + # guard + test[Maybe.guard(True).then(Maybe(42)) == Maybe(42)] + test[Maybe.guard(False).then(Maybe(42)) == Maybe(nil)] + + # lift + test[Maybe.lift(lambda x: x + 1)(5) == Maybe(6)] + + # Classical sqrt chain (via Maybe) + def maybe_sqrt(x): + if x < 0: + return Maybe(nil) + return Maybe(sqrt(x)) + test[Maybe(16) >> maybe_sqrt >> maybe_sqrt == Maybe(2.0)] + test[Maybe(-1) >> maybe_sqrt >> maybe_sqrt == Maybe(nil)] + + with testset("Either / Left / Right"): + # Construction + test[Right(42) == Right(42)] + test[Left("err") == Left("err")] + test[Right(42) != Left(42)] # different branches, same value + test_raises[TypeError, Either(5)] # abstract + + # Happy path + test[(Right(10) >> (lambda x: Right(x + 1))) == Right(11)] + + # Short-circuit + test[Left("boom") >> (lambda x: Right(x + 1)) == Left("boom")] + + # Left doesn't invoke the lambda + called = [] + Left("err") >> (lambda x: (called.append(x), Right(x))[1]) + test[called == []] + + # fmap + test[Right(5).fmap(lambda x: x * 2) == Right(10)] + test[Left("err").fmap(lambda x: x * 2) == Left("err")] + + # join + test[Right(Right(7)).join() == Right(7)] + test[Right(Left("nested err")).join() == Left("nested err")] + test[Left("outer").join() == Left("outer")] + test_raises[TypeError, Right(5).join()] # not nested + + # lift (always produces Right) + test[Either.lift(lambda x: x + 1)(5) == Right(6)] + test[Right.lift(lambda x: x + 1)(5) == Right(6)] + + # Cross-subclass then (Right.then(Left) works) + test[Right(1).then(Left("replace")) == Left("replace")] + test[Left("err").then(Right(5)) == Left("err")] # short-circuit wins + + # guard + test[Either.guard(True).then(Right(42)) == Right(42)] + test[Either.guard(False, "bad").then(Right(42)) == Left("bad")] + + with testset("List"): + test[List() == List()] + test[List(1, 2, 3) == List(1, 2, 3)] + test[List(nil) == List()] # sentinel form = empty + test[List.from_iterable(range(3)) == List(0, 1, 2)] + + # fmap / bind / join + test[List(1, 2, 3).fmap(lambda x: x * 10) == List(10, 20, 30)] + test[(List(1, 2, 3) >> (lambda x: List(x, x * 10))) == List(1, 10, 2, 20, 3, 30)] + test[List(List(1, 2), List(3)).join() == List(1, 2, 3)] + + # guard / filter + filtered = List(1, 2, 3, 4) >> (lambda x: + List.guard(x % 2 == 0).then(List(x))) + test[filtered == List(2, 4)] + + # Pythagorean triples (the canonical List-monad example) + def r(low, high): + return List.from_iterable(range(low, high)) + pt = r(1, 21) >> (lambda z: + r(1, z + 1) >> (lambda x: + r(x, z + 1) >> (lambda y: + List.guard(x * x + y * y == z * z).then( + List((x, y, z)))))) + test[tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20))] + + # Sequence protocol + from collections.abc import Sequence + test[isinstance(List(1, 2, 3), Sequence)] + test[List(1, 2, 3)[1] == 2] + test[2 in List(1, 2, 3)] + test[List(1, 2, 3) + List(4, 5) == List(1, 2, 3, 4, 5)] + + # lift + test[List.lift(lambda x: x + 1)(5) == List(6)] + + with testset("Writer"): + # Basic chain with log accumulation + result = (Writer(10) + >> (lambda x: Writer(x + 1, "added 1; ")) + >> (lambda y: Writer(y * 2, "doubled; "))) + test[result.data == (22, "added 1; doubled; ")] + + # fmap is transparent (doesn't add log) + test[Writer(5, "start; ").fmap(lambda x: x * 10).data == (50, "start; ")] + + # join + test[Writer(Writer(7, "inner"), "outer").join().data == (7, "outerinner")] + test_raises[TypeError, Writer(5).join()] + + # tell + tr = Writer(10, "step1; ").then(Writer.tell("step2; ")) + test[tr.data == (None, "step1; step2; ")] + + # lift (doesn't auto-log) + test[Writer.lift(lambda x: x + 1)(5).data == (6, "")] + + with testset("State"): + bump = State(lambda s: (s, s + 1)) + + # Basic chain + chain = (bump + >> (lambda a: bump + >> (lambda b: bump + >> (lambda c: State.unit((a, b, c)))))) + data, final = chain.run(10) + test[data == (10, 11, 12)] + test[final == 13] + + # eval / exec + test[chain.eval(10) == (10, 11, 12)] + test[chain.exec(10) == 13] + + # get / put / modify / gets + test[State.get().run(42) == (42, 42)] + test[State.put(99).run(5) == (None, 99)] + test[State.modify(lambda s: s * 2).run(7) == (None, 14)] + test[State.gets(lambda s: s + 1).run(10) == (11, 10)] + + # fmap + test[State.unit(5).fmap(lambda x: x * 10).run("anything") == (50, "anything")] + + # join (the plain-words-derivation case) + def inner(s): + return (s * 10, s + 1) + def outer(s): + return (State(inner), s + 100) + nested = State(outer) + # outer(0) -> (inner_state, 100); inner(100) -> (1000, 101) + test[nested.join().run(0) == (1000, 101)] + + # join rejects non-nested + test_raises[TypeError, State.unit(5).join().run(0)] + + # State does NOT have lift + test[not hasattr(State, "lift") or State.lift is not LiftableMonad.__dict__.get("lift")] + + # Constructor rejects non-callables + test_raises[TypeError, State(42)] + + with testset("Reader"): + config = {"multiplier": 3, "offset": 10} + + chain = (Reader.asks(lambda e: e["multiplier"]) + >> (lambda m: Reader.asks(lambda e: e["offset"]) + >> (lambda o: Reader.unit(m * 5 + o)))) + test[chain.run(config) == 25] + + # ask / asks / unit + test[Reader.ask().run("env") == "env"] + test[Reader.asks(lambda e: e.upper()).run("hello") == "HELLO"] + test[Reader.unit(42).run("ignored") == 42] + + # local: modify the environment for a sub-computation + test[Reader.ask().local(lambda e: e * 2).run(5) == 10] + + # fmap / join + test[Reader.unit(5).fmap(lambda x: x * 10).run(None) == 50] + nested = Reader(lambda e: Reader(lambda e2: e + e2)) + test[nested.join().run(3) == 6] + + # Reader does NOT have lift + test[not hasattr(Reader, "lift") or Reader.lift is not LiftableMonad.__dict__.get("lift")] + + # Constructor rejects non-callables + test_raises[TypeError, Reader(42)] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/monads/writer.py b/unpythonic/monads/writer.py new file mode 100644 index 00000000..1cd51832 --- /dev/null +++ b/unpythonic/monads/writer.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""The Writer monad — pure-functional debug/audit log. + +A ``Writer w a`` wraps a pair ``(value, log)``. Binding threads the value +through the chain while concatenating logs. The log can be any type that +supports ``+`` and has a sensible empty value — the defaults assume a +``str`` log (empty ``""``). + +Classical use: produce a computation result along with a trace of what was +done, without resorting to mutable state or ``print`` side-effects. +""" + +__all__ = ["Writer"] + +from collections.abc import Callable +from typing import Any + +from .abc import LiftableMonad + + +class Writer(LiftableMonad): + """The Writer monad. ``Writer(value, log)``; log defaults to ``""``. + + Usage:: + + from unpythonic.monads import Writer + + result = (Writer(10) + >> (lambda x: Writer(x + 1, f"added 1 to {x}; ")) + >> (lambda y: Writer(y * 2, f"doubled {y}; "))) + value, log = result.data + # value == 22 + # log == "added 1 to 10; doubled 11; " + + Use the classmethod ``Writer.tell(msg)`` to add a log entry without + touching the value: ``writer_a.then(Writer.tell(msg))`` appends ``msg`` + to the log and passes ``writer_a``'s value through (actually: yields + ``None`` as the value of the ``tell`` step; use ``.then`` to replace + with the real value on the next step). + + **Semantics note**. ``fmap`` does **not** add a log entry of its own — + the teaching code did, which in turn forced a manual override of bind + to avoid double-logging. Here we keep fmap transparent so the default + bind (``fmap . join``) from the ``Monad`` base works as-is. + """ + + def __init__(self, value: Any, log: Any = "") -> None: + """Unit: wrap ``value: a`` with an optional ``log: w`` (default empty string).""" + self.data = (value, log) + + def fmap(self, f: Callable) -> "Writer": + """``fmap: Writer w a -> (a -> b) -> Writer w b``. Log passes through unchanged.""" + value, log = self.data + cls = self.__class__ + return cls(f(value), log) + + def join(self) -> "Writer": + """``join: Writer w (Writer w a) -> Writer w a``. Concatenates outer + inner logs.""" + cls = self.__class__ + if not isinstance(self.data[0], cls): + raise TypeError( + f"Expected a nested {cls.__name__}, got {type(self.data[0])} with data {self.data[0]!r}" + ) + inner, outer_log = self.data + inner_value, inner_log = inner.data + return cls(inner_value, outer_log + inner_log) + + @classmethod + def tell(cls, log_entry: Any) -> "Writer": + """Emit a log entry and yield a dummy value. + + ``tell: w -> Writer w None`` + + Use with ``.then`` to interleave logging into a chain: e.g. + ``computation.then(Writer.tell("done; "))`` yields a Writer whose + value is ``None`` and whose log has ``"done; "`` appended. + """ + return cls(None, log_entry) + + def __eq__(self, other: Any) -> bool: + if other is self: + return True + if not isinstance(other, Writer): + return NotImplemented + return self.data == other.data + + def __hash__(self) -> int: + return hash((Writer, self.data)) + + def __repr__(self) -> str: # pragma: no cover + return f"{self.__class__.__name__}{self.data!r}" diff --git a/unpythonic/net/__init__.py b/unpythonic/net/__init__.py index 2b3ca9c9..a16495b0 100644 --- a/unpythonic/net/__init__.py +++ b/unpythonic/net/__init__.py @@ -10,5 +10,12 @@ """ from .msg import * -from .ptyproxy import * +try: + from .ptyproxy import * +except ModuleNotFoundError: + import logging + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + logger.info("`unpythonic.net.ptyproxy` could not be loaded, the REPL server will not be available. Usually this is harmless; most applications do not need the REPL server.") + PTYSocketProxy = None from .util import * diff --git a/unpythonic/net/client.py b/unpythonic/net/client.py index 8cd85d53..720247cd 100644 --- a/unpythonic/net/client.py +++ b/unpythonic/net/client.py @@ -32,13 +32,30 @@ for a remote tab completer, and a separate client-side `input()` loop.) """ -import readline # noqa: F401, input() uses the readline module if it has been loaded. +import platform import socket import select import sys import re import time +# NOTE: `readline` is imported lazily inside `connect()`, not at module top. +# Two reasons: +# +# 1. POSIX stdlib ships `readline`; Windows does not (the stdlib module is +# GNU-readline-only). A module-level `import readline` therefore makes +# this whole module unimportable on Windows — even though `connect()` +# is the only thing that needs it. Keeping the import inside the +# function lets callers on non-POSIX platforms at least import the +# module (useful for test collection, docs, and for Windows support). +# +# 2. A three-tier fallback is applied at the import site: stdlib +# `readline` first, then third-party `pyreadline3` (a Windows drop-in +# with a compatible API surface), and finally `None` (degrade +# gracefully — the REPL loop still works, the user just loses history +# and tab completion). The same pattern is used in +# `mcpyrate.repl.macropython` and `raven.librarian.minichat`. + from .msg import MessageDecoder from .util import socketsource, ReceiveBuffer from .common import ApplevelProtocolMixin @@ -153,6 +170,33 @@ def connect(host, repl_port, control_port): connection immediately. (The server should be smart enough to notice the client is gone, and clean up any relevant resources.) """ + _connect(host, repl_port, control_port) + + +# The real implementation. `connect` is a thin shim; this one takes an +# extra `_input` hook so the tier 1 test suite can drive the REPL loop +# through a scripted fake `input()` without monkey-patching `builtins.input` +# globally — which would also hijack the in-process server's +# `InteractiveConsole.raw_input` path and break the test. +# +# The pattern (public shim + private `_name` impl with extra kwargs) is +# used elsewhere in the unpythonic fleet when a public signature should +# stay clean but tests need a seam. +def _connect(host, repl_port, control_port, _input=None): + if _input is None: + import builtins + _input = builtins.input + + # Three-tier readline loading. See module-level comment for rationale. + try: + import readline # noqa: F401, side effect: enable GNU readline in input() + except ImportError: + try: + import pyreadline3 as readline # type: ignore # noqa: F401 + except ImportError: + readline = None + _has_readline = readline is not None + class SessionExit(Exception): pass try: @@ -170,8 +214,22 @@ class SessionExit(Exception): # Set up remote tab completion, using a custom completer for readline. # https://stackoverflow.com/questions/35115208/is-there-any-way-to-combine-readline-rlcompleter-and-interactiveconsole-in-pytho - readline.set_completer(controller.complete) - readline.parse_and_bind("tab: complete") # TODO: do we need to call this, PyPy doesn't support it? + if _has_readline: + readline.set_completer(controller.complete) + # macOS ships `readline` backed by `libedit`, which speaks a + # different `parse_and_bind` dialect than GNU readline. Detect + # by platform to keep tab completion working on Macs. See + # https://stackoverflow.com/questions/7116038/python-repl-tab-completion-on-macos + if platform.system() == "Darwin": # macOS + readline.parse_and_bind("bind ^I rl_complete") + else: # Linux, Windows (pyreadline3) + readline.parse_and_bind("tab: complete") # PyPy ignores this, but not needed there. + else: + # No readline at all: the REPL loop still works through plain + # `input()`, but the user loses history and tab completion. + print("unpythonic.net.client: `readline` unavailable — command history and tab completion are disabled.\n" + " On Windows, `pip install pyreadline3` restores both.", + file=sys.stderr) with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: # remote REPL session sock.connect((host, repl_port)) # TODO: IPv6 support @@ -250,7 +308,7 @@ def read_more_input(): # "R", "E" (but evaluate remotely) try: - inp = input(prompt) + inp = _input(prompt) sock.sendall((inp + "\n").encode("utf-8")) except EOFError: print("unpythonic.net.client: Ctrl+D pressed, asking server to disconnect.") diff --git a/unpythonic/net/ptyproxy.py b/unpythonic/net/ptyproxy.py index 7e870caf..dbbd78ac 100644 --- a/unpythonic/net/ptyproxy.py +++ b/unpythonic/net/ptyproxy.py @@ -1,28 +1,79 @@ -"""PTY/socket proxy. Useful for serving terminal applications for remote use.""" +# -*- coding: utf-8; -*- +"""PTY/socket proxy. Useful for serving terminal applications for remote use. -import os -import tty -import termios -import select -import threading +This module defines `PTYSocketProxy`, an abstract base class that plugs a +bidirectional byte channel between a network socket and in-process code that +wants to look like it's running behind a terminal. The concrete implementation +is chosen at construction time based on platform: + + - **POSIX**: `ptyproxy_posix.PosixPTYSocketProxy` uses a real pseudo-terminal + (`os.openpty` + raw mode on master). `os.isatty()` returns `True` inside + code that reads the slave. + - **Windows**: `ptyproxy_windows.WindowsPTYSocketProxy` uses + `socket.socketpair()` as the master/slave byte channel. `os.isatty()` + returns `False` (the framework itself doesn't care, but user code inside + a REPL session *may*). + +Instantiating `PTYSocketProxy(...)` dispatches to the right subclass +automatically — you don't need to import the backend module yourself. +`isinstance(obj, PTYSocketProxy)` works transparently for both backends. + +In-tree consumer: `unpythonic.net.server`, which plugs the slave side into a +`code.InteractiveConsole` so a remote client can drive an in-process REPL +(sharing the server's state — the whole point of `unpythonic.net.server`, +which is a hot-patching back door, not a pseudo-shell). The class itself is +general, though: anything that wants "code in this process that looks like +it's behind a tty from a socket's point of view" can use it. +""" + +import platform +from abc import ABC, abstractmethod __all__ = ["PTYSocketProxy"] -# What this does for us in a remote REPL session in unpythonic.net.server is that: -# >>> import os -# >>> os.isatty(sys.stdin.fileno()) -# True -# whereas without the PTY, the same code returns False. -# -class PTYSocketProxy: - """Plug a PTY between a network socket and Python code that expects to run in a terminal. - Generally, having a PTY enables the "interactive" features of some *nix terminal apps. +class PTYSocketProxy(ABC): + """Plug a (P)TY between a network socket and in-process code that expects a terminal. + + Having a (pseudo-)terminal enables the "interactive" features of terminal + applications. This class differs from many online examples in that **we do + not** use `pty.spawn`; the code running on the slave side doesn't need to + be a separate process, and instead runs on a thread in the same process + as the server. + + **Construction**: call `PTYSocketProxy(sock, on_socket_disconnect, + on_slave_disconnect)` — dispatch to `PosixPTYSocketProxy` or + `WindowsPTYSocketProxy` happens inside `__new__`, based on + `platform.system()`. Direct instantiation of a specific subclass also + works (e.g. for tests that want to force a backend). + + **Callbacks**: + + `on_socket_disconnect`, if set, is a one-argument callable called when + an EOF is detected on the socket. It receives the `PTYSocketProxy` + instance, and can e.g. `proxy.write_to_master(some_disconnect_command)` + to tell the software connected on the slave side to exit. - This is different from many online examples in that **we do not** use `pty.spawn`, - so the code that runs on the PTY slave side doesn't need to be a separate process. + What that command is, is up to the protocol your specific software + speaks, so we just provide a general mechanism to send it. In other + words, you get a disconnect event for free, but you need to know how + to tell your specific software to end the session when that event fires. - Based on solution by SO user gowenfawr: + `on_slave_disconnect`, if set, is a similar callable called when an + EOF is detected on the slave side. + + **Public interface** (all abstract, implemented by subclasses): + + - `start()` — begin forwarding traffic in a daemon thread. + - `stop()` — shut down and release resources. + - `write_to_master(data)` — inject bytes on the master side; they + appear as input on the slave. + - `open_slave_streams()` — context manager yielding text + `(rfile, wfile)` over the slave side, suitable for wiring into + `code.InteractiveConsole`. + - `name` — human-readable slave-side name for log messages. + + Based on a solution by SO user gowenfawr: https://stackoverflow.com/questions/48781155/how-to-connect-inet-socket-to-pty-device-in-python On PTYs in Python and in general, see: @@ -32,81 +83,93 @@ class PTYSocketProxy: https://terminallabs.com/blog/a-better-cli-passthrough-in-python/ http://man7.org/linux/man-pages/man7/pty.7.html """ - def __init__(self, sock, on_socket_disconnect=None, on_slave_disconnect=None): - """Open the PTY. The slave FD becomes available as `self.slave`. - - `on_socket_disconnect`, if set, is a one-argument callable that is called - when an EOF is detected on the socket. It receives the `PTYSocketProxy` - instance and can e.g. `os.write(proxy.master, some_disconnect_command)` - to tell the software connected on the slave side to exit. - - What the command is, is up to the protocol your specific software - speaks, so we just provide a general mechanism to send a command to it. - In other words, you get a disconnect event for free, but you need to - know how to tell your specific software to end the session when that - event fires. - - `on_slave_disconnect`, if set, is a similar callable that is called when - an EOF is detected on the PTY slave. - - **NOTE**: `slave` is a raw file descriptor (just a small integer), - not a Python stream. If you need a stream, `open()` the file descriptor - (twice if you need to read *and* write; make sure to set `closefd` to - `False`, as `PTYSocketProxy` will manage the closing). + + def __new__(cls, *args, **kwargs): + # When the abstract base class itself is instantiated, dispatch to the + # platform-specific concrete subclass. Explicit subclass instantiation + # (cls is a subclass, not PTYSocketProxy itself) bypasses the dispatch + # and just runs normally — useful for tests that want a specific + # backend regardless of platform. + # + # `__new__` returning a subclass instance causes Python to still call + # `__init__` on it (because the returned object is-a `cls`), and the + # lookup resolves to the subclass's `__init__` — so `*args, **kwargs` + # land in the right place without us needing to forward them manually. + if cls is PTYSocketProxy: + if platform.system() == "Windows": + from .ptyproxy_windows import WindowsPTYSocketProxy + cls = WindowsPTYSocketProxy + else: + from .ptyproxy_posix import PosixPTYSocketProxy + cls = PosixPTYSocketProxy + return super().__new__(cls) + + def __enter__(self): + """Enable ``with PTYSocketProxy(...) as proxy:`` usage. + + The context manager just guarantees `stop()` runs on exit from the + ``with`` block — whether the body completes normally, raises, or is + interrupted. This is the recommended way to use the proxy, so that + the master/slave transport cannot leak on exceptional paths. + + The `start()` call is deliberately *not* pulled into `__enter__`, + because some callers may want to do setup between construction and + the start of forwarding. Call `proxy.start()` explicitly inside the + ``with`` body. """ - # master is the "pty side", slave is the "tty side". - master, slave = os.openpty() - tty.setraw(master, termios.TCSANOW) # http://man7.org/linux/man-pages/man3/termios.3.html - self.sock = sock - self.master, self.slave = master, slave - self.on_socket_disconnect = on_socket_disconnect - self.on_slave_disconnect = on_slave_disconnect - self._terminated = True - self._thread = None + return self + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + return False # don't suppress exceptions + + @abstractmethod def start(self): - """Start forwarding traffic between the PTY master and the socket.""" - if self._thread: - raise RuntimeError("Already running.") - - # Note we use raw fds (file descriptors) and the low-level os.read, os.write functions, - # which bypass all niceties file objects have. - # https://docs.python.org/3/library/os.html - def forward_traffic(): - mypoll = select.poll() - mypoll.register(self.sock, select.POLLIN) - mypoll.register(self.master, select.POLLIN) - while not self._terminated: - try: - fdlist = mypoll.poll(1000) - for fd, event in fdlist: - if fd == self.master: - request = os.read(fd, 4096) - if len(request) == 0: # disconnect by PTY slave - self.on_slave_disconnect(self) - return - self.sock.send(request) - else: - request = self.sock.recv(4096) - if len(request) == 0: # disconnect by client behind socket - self.on_socket_disconnect(self) - return - os.write(self.master, request) - except ConnectionResetError: - self.on_socket_disconnect(self) - return - - self._terminated = False - self._thread = threading.Thread(target=forward_traffic, name=f"PTY on {os.ttyname(self.slave)}", daemon=True) - self._thread.start() + """Start forwarding traffic between the master endpoint and the socket.""" + @abstractmethod def stop(self): - """Shut down. This also closes the PTY.""" - if self._thread: - self._terminated = True - self._thread.join() - self._thread = None - os.close(self.master) - self.master = None - os.close(self.slave) - self.slave = None + """Shut down. Also closes the master/slave transport. + + **Must be idempotent**: calling `stop()` on a proxy that was never + started, or calling it twice, must be safe. This is load-bearing + for `__exit__` — the context manager always calls `stop()`, even + if the caller also called it explicitly inside the ``with`` body. + """ + + @abstractmethod + def write_to_master(self, data): + """Write raw bytes to the master side, as if typed by the client. + + Bytes written here appear on the slave side as input, so e.g. + `proxy.write_to_master(b"quit()\\n")` injects a line of input into + whatever code is reading the slave stream. Useful to programmatically + tell a REPL (or other terminal application on the slave side) to exit. + + `data` must be a `bytes` object. + """ + + @abstractmethod + def open_slave_streams(self, encoding="utf-8"): + """Context manager yielding ``(rfile, wfile)`` text streams over the slave side. + + Both streams are closed on exit from the ``with`` block; the + underlying slave transport itself remains managed by the proxy + and is closed by `stop()`. + + The returned streams are suitable for wiring into + `code.InteractiveConsole` as its input/output. + + Concrete implementations should decorate with + `@contextlib.contextmanager`; this declaration is just the contract. + """ + + @property + @abstractmethod + def name(self): + """Human-readable name of the slave side, for log messages. + + On POSIX this is the tty name (`os.ttyname`); on Windows it's a + synthetic identifier, since no tty is involved. Safe to read after + `stop()` — subclasses cache it up front. + """ diff --git a/unpythonic/net/ptyproxy_posix.py b/unpythonic/net/ptyproxy_posix.py new file mode 100644 index 00000000..a16c7644 --- /dev/null +++ b/unpythonic/net/ptyproxy_posix.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8; -*- +"""POSIX backend for `PTYSocketProxy`. See `ptyproxy.py` for the public interface.""" + +import contextlib +import os +import tty +import termios +import select +import threading + +from .ptyproxy import PTYSocketProxy + +__all__ = ["PosixPTYSocketProxy"] + + +# What this does for us in a remote REPL session in `unpythonic.net.server` is: +# >>> import os +# >>> os.isatty(sys.stdin.fileno()) +# True +# whereas without the PTY, the same code returns False. On Windows, where no +# real pty is available, that property is lost — see `ptyproxy_windows`. +class PosixPTYSocketProxy(PTYSocketProxy): + """POSIX implementation of `PTYSocketProxy` using `os.openpty`. + + See the `PTYSocketProxy` base class for the public interface contract. + """ + + def __init__(self, sock, on_socket_disconnect=None, on_slave_disconnect=None): + # master is the "pty side", slave is the "tty side". + master, slave = os.openpty() + # Transactional: if anything between here and the end of __init__ + # raises, we own two open fds and the caller will never get a + # reference to close them. Release them before re-raising. + try: + tty.setraw(master, termios.TCSANOW) # http://man7.org/linux/man-pages/man3/termios.3.html + # `os.ttyname` is cached up front so `self.name` still works + # after `stop()` has closed the slave fd — callers use it in + # log messages during teardown. Also part of the transaction: + # if the slave fd is somehow already invalid, fail early. + self._name = os.ttyname(slave) + except BaseException: + try: + os.close(master) + except OSError: + pass + try: + os.close(slave) + except OSError: + pass + raise + self.sock = sock + self.master, self.slave = master, slave + self.on_socket_disconnect = on_socket_disconnect + self.on_slave_disconnect = on_slave_disconnect + self._terminated = True + self._thread = None + + @property + def name(self): + return self._name + + def write_to_master(self, data): + os.write(self.master, data) + + @contextlib.contextmanager + def open_slave_streams(self, encoding="utf-8"): + # `closefd=False` on both: the raw slave fd is owned by this proxy + # and will be released by `stop()`, not by the stream wrappers. + with contextlib.ExitStack() as stack: + wfile = stack.enter_context(open(self.slave, "wt", encoding=encoding, closefd=False)) + rfile = stack.enter_context(open(self.slave, "rt", encoding=encoding, closefd=False)) + yield rfile, wfile + + def start(self): + if self._thread: + raise RuntimeError("Already running.") + + # Note we use raw fds (file descriptors) and the low-level os.read, os.write functions, + # which bypass all niceties file objects have. + # https://docs.python.org/3/library/os.html + def forward_traffic(): + mypoll = select.poll() + mypoll.register(self.sock, select.POLLIN) + mypoll.register(self.master, select.POLLIN) + while not self._terminated: + try: + fdlist = mypoll.poll(1000) + for fd, event in fdlist: + if fd == self.master: + request = os.read(fd, 4096) + if len(request) == 0: # disconnect by PTY slave + self.on_slave_disconnect(self) + return + self.sock.send(request) + else: + request = self.sock.recv(4096) + if len(request) == 0: # disconnect by client behind socket + self.on_socket_disconnect(self) + return + os.write(self.master, request) + except ConnectionResetError: + self.on_socket_disconnect(self) + return + + self._terminated = False + self._thread = threading.Thread(target=forward_traffic, name=f"PTY on {os.ttyname(self.slave)}", daemon=True) + self._thread.start() + + def stop(self): + # Decoupled and idempotent: the fd teardown runs regardless of + # whether the forwarding thread was ever started, and each close + # is guarded so a failure on one fd doesn't leak the other. + if self._thread is not None: + self._terminated = True + self._thread.join() + self._thread = None + if self.master is not None: + try: + os.close(self.master) + except OSError: + pass + self.master = None + if self.slave is not None: + try: + os.close(self.slave) + except OSError: + pass + self.slave = None diff --git a/unpythonic/net/ptyproxy_windows.py b/unpythonic/net/ptyproxy_windows.py new file mode 100644 index 00000000..3aaabb80 --- /dev/null +++ b/unpythonic/net/ptyproxy_windows.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8; -*- +"""Windows backend for `PTYSocketProxy`. See `ptyproxy.py` for the public interface. + +No real pseudo-terminal is involved: we use `socket.socketpair()` as a pair +of connected loopback sockets standing in for the pty master/slave +endpoints. The forwarding loop is identical in shape to the POSIX backend +(byte shovelling between `sock` and `master`), just with `select.select` +instead of `select.poll` (Windows has no `poll` for sockets) and with +socket methods instead of raw fd `os.read`/`os.write`. + +**What we lose compared to a real PTY**: `os.isatty()` on code running +against the slave side returns `False`. The framework itself (`code. +InteractiveConsole`, `unpythonic.net.server`) does not depend on +`isatty()`; user code *inside* a REPL session that checks +`sys.stdin.isatty()` will see the Windows result. This is the one +documented wart of the Windows port — see the 2.0.x CHANGELOG entry. + +**Why not ConPTY / pywinpty**: ConPTY is architected around launching a +*child process* attached to a pseudoconsole. There is no supported +"attach my own process's existing thread to this pseudoconsole" +primitive, and spawning a subprocess per REPL session would defeat the +whole point of `unpythonic.net.server`, which is to let a remote client +inspect and hot-patch state in the *host* Python process — that requires +the REPL to run in the same process as the server. + +The right question isn't "how do we get ConPTY", it's "what do we +actually need". The answer: two connected bidirectional byte streams. +`socket.socketpair()` provides exactly that, stdlib-only, with lines +of code that mirror the POSIX backend almost 1:1. + +**Why the Windows backend also works on POSIX**: `socket.socketpair()` is +available on every platform Python supports. The Windows-specific +constraint is that it returns AF_INET loopback sockets there (POSIX +defaults to AF_UNIX, which is also fine). This lets us unit-test +`WindowsPTYSocketProxy` on a Linux/macOS dev machine by explicit +instantiation, without needing a Windows box. +""" + +import contextlib +import select +import socket +import threading + +from .ptyproxy import PTYSocketProxy + +__all__ = ["WindowsPTYSocketProxy"] + + +class WindowsPTYSocketProxy(PTYSocketProxy): + """Windows implementation of `PTYSocketProxy` using `socket.socketpair()`. + + See the `PTYSocketProxy` base class for the public interface contract. + """ + + def __init__(self, sock, on_socket_disconnect=None, on_slave_disconnect=None): + # No `openpty`; a connected socketpair stands in for the pty + # master/slave endpoints. Both ends are full-duplex sockets, so + # "master" and "slave" are labels for roles, not transport + # distinctions — unlike on POSIX where master/slave have + # asymmetric kernel-level semantics. + master, slave = socket.socketpair() + # Transactional: if anything between here and the end of __init__ + # raises, we own two open sockets and the caller will never get + # a reference to close them. Release them before re-raising. + try: + # Synthetic name for log messages — no `ttyname` equivalent + # here. Low-order bits of `id(self)` give a short, + # human-readable tag that distinguishes concurrent proxies. + self._name = f"(socketpair#{id(self) & 0xffff:04x})" + except BaseException: + try: + master.close() + except OSError: + pass + try: + slave.close() + except OSError: + pass + raise + self.sock = sock + self.master, self.slave = master, slave + self.on_socket_disconnect = on_socket_disconnect + self.on_slave_disconnect = on_slave_disconnect + self._terminated = True + self._thread = None + + @property + def name(self): + return self._name + + def write_to_master(self, data): + self.master.sendall(data) + + @contextlib.contextmanager + def open_slave_streams(self, encoding="utf-8"): + # `socket.makefile` uses a reference-counting scheme: each call + # increments `_io_refs` on the underlying socket, and closing the + # wrapper decrements it. The raw socket is only closed once + # `_io_refs` hits zero *and* `socket.close()` has been called + # explicitly. So closing the wfile/rfile wrappers here does NOT + # close the underlying slave socket — that's left to `stop()`, + # matching the POSIX backend's `closefd=False` semantics. + # + # `buffering=1` on the writer = line buffering. This matters + # because socket writers default to block-buffered (~8 KB), which + # would stall REPL prompts until enough bytes accumulated. With + # line buffering, every `\n` flushes — and `builtins.input()` + # also explicitly calls `sys.stdout.flush()` before reading, so + # bare prompts (no trailing newline) also reach the client + # promptly. + # + # `newline=""` on the writer disables `\n` → `os.linesep` + # translation — a CRITICAL Windows fix, because `os.linesep` is + # `\r\n` there, and the default `newline=None` would translate + # every `\n` the application writes into `\r\n` on the wire. + # That would pollute the client's display with stray `\r`s and + # potentially break the prompt-detection / session-ID-parsing + # regex on `net.client`. On POSIX the setting is a no-op (since + # `os.linesep == "\n"`), so it's also safe to run on Linux — + # and crucially it means the Linux test suite validates exactly + # the same code path that Windows will execute. + # + # The reader uses default `newline=None` (universal newlines), + # which returns `\n`-terminated lines regardless of the actual + # on-wire ending — exactly what `code.InteractiveConsole` + # expects from `sys.stdin.readline()`. + with contextlib.ExitStack() as stack: + wfile = stack.enter_context(self.slave.makefile("w", buffering=1, encoding=encoding, newline="")) + rfile = stack.enter_context(self.slave.makefile("r", encoding=encoding)) + yield rfile, wfile + + def start(self): + if self._thread: + raise RuntimeError("Already running.") + + # Windows has no `select.poll` for sockets (Python's `select.poll` + # exists on Windows but only for a limited file-descriptor set — + # sockets are not supported). `select.select` handles sockets on + # all platforms, so we use that here. The 1-second timeout is the + # same as the POSIX `poll(1000)` — it bounds the latency for + # `stop()` to notice `self._terminated` flipping. + def forward_traffic(): + while not self._terminated: + try: + rs, _ws, _es = select.select([self.sock, self.master], [], [], 1.0) + for s in rs: + if s is self.master: + request = self.master.recv(4096) + if len(request) == 0: # disconnect by slave-side code + self.on_slave_disconnect(self) + return + self.sock.send(request) + else: + request = self.sock.recv(4096) + if len(request) == 0: # disconnect by client behind socket + self.on_socket_disconnect(self) + return + self.master.send(request) + except ConnectionResetError: + self.on_socket_disconnect(self) + return + + self._terminated = False + self._thread = threading.Thread(target=forward_traffic, name=f"PTY on {self._name}", daemon=True) + self._thread.start() + + def stop(self): + # Decoupled and idempotent: the socket teardown runs regardless + # of whether the forwarding thread was ever started, and each + # close is guarded so a failure on one socket doesn't leak the + # other. + if self._thread is not None: + self._terminated = True + self._thread.join() + self._thread = None + if self.master is not None: + try: + self.master.close() + except OSError: + pass + self.master = None + if self.slave is not None: + try: + self.slave.close() + except OSError: + pass + self.slave = None diff --git a/unpythonic/net/server.py b/unpythonic/net/server.py index e1cd14ec..2ea7364f 100644 --- a/unpythonic/net/server.py +++ b/unpythonic/net/server.py @@ -119,7 +119,6 @@ import rlcompleter # yes, just rlcompleter without readline; backend for remote tab completion. import threading import sys -import os import time import socketserver import atexit @@ -133,7 +132,8 @@ from code import InteractiveConsole as Console from ..collections import ThreadLocalBox, Shim -from ..misc import async_raise, namelambda +from ..excutil import async_raise +from ..misc import namelambda from ..symbol import sym from .util import ReuseAddrThreadingTCPServer, socketsource @@ -444,22 +444,26 @@ def handle(self): # https://docs.python.org/3/library/socketserver.html#socketserver.StreamRequestHandler def on_socket_disconnect(adaptor): - server_print(f"PTY on {os.ttyname(adaptor.slave)} for client {client_address_str} disconnected by client.") - os.write(adaptor.master, "quit()\n".encode("utf-8")) # as if this text arrived from the socket + server_print(f"PTY on {adaptor.name} for client {client_address_str} disconnected by client.") + adaptor.write_to_master(b"quit()\n") # as if this text arrived from the socket def on_slave_disconnect(adaptor): - server_print(f"PTY on {os.ttyname(adaptor.slave)} for client {client_address_str} disconnected by PTY slave.") - adaptor = PTYSocketProxy(self.request, on_socket_disconnect, on_slave_disconnect) - adaptor.start() - server_print(f"PTY on {os.ttyname(adaptor.slave)} for client {client_address_str} opened.") - - # fdopen the slave side of the PTY to get file objects to work with. - # Be sure not to close the fd when exiting, it is managed by PTYSocketProxy. - # - # Note we can open the slave side in text mode, so these streams can behave - # exactly like standard input and output. The proxying between the master side - # and the network socket runs in binary mode inside PTYSocketProxy. - with open(adaptor.slave, "wt", encoding="utf-8", closefd=False) as wfile: - with open(adaptor.slave, "rt", encoding="utf-8", closefd=False) as rfile: + server_print(f"PTY on {adaptor.name} for client {client_address_str} disconnected by PTY slave.") + # `with PTYSocketProxy(...)` guarantees `stop()` runs on exit — + # whether the body completes normally, raises, or is interrupted. + # Crucially, this covers the paths where `adaptor.start()` or + # `open_slave_streams()` itself raises, which a bare try/finally + # around `adaptor.stop()` inside the body would miss. + with PTYSocketProxy(self.request, on_socket_disconnect, on_slave_disconnect) as adaptor: + adaptor.start() + server_print(f"PTY on {adaptor.name} for client {client_address_str} opened.") + + # Open the slave side as a pair of text streams, so these behave + # exactly like standard input and output. The proxying between the + # master side and the network socket runs in binary mode inside + # PTYSocketProxy. Stream teardown is managed by this inner context + # manager; the underlying slave transport itself is managed by + # PTYSocketProxy and closed by the outer `with`. + with adaptor.open_slave_streams() as (rfile, wfile): # Set up the input and output streams for the thread we are running in. # We use ThreadingTCPServer, so each connection gets its own thread. # Here we just send the relevant object into each thread-local box. @@ -489,10 +493,8 @@ def on_slave_disconnect(adaptor): self.console.interact(banner=None, exitmsg="Bye.") except SystemExit: # Close the connection upon server process exit. pass - finally: - server_print(f"Closing PTY on {os.ttyname(adaptor.slave)} for {client_address_str}.") - adaptor.stop() - server_print(f"Closing REPL session {self.session_id} for {client_address_str}.") + server_print(f"Closing PTY on {adaptor.name} for {client_address_str}.") + server_print(f"Closing REPL session {self.session_id} for {client_address_str}.") except BaseException as err: # yes, SystemExit and KeyboardInterrupt, too. server_print(err) finally: @@ -605,7 +607,13 @@ def start(locals, bind="127.0.0.1", repl_port=1337, control_port=8128, banner=No _server_instance = (server, server_thread, cserver, cserver_thread) atexit.register(stop) - return bind, repl_port, control_port + # Return the **actual** bound ports, not the values the caller passed in. + # Matters when the caller asks for port 0 (let the kernel pick a free port); + # `server.server_address[1]` is the only way to retrieve the kernel's choice. + # On a regular fixed-port start, these just echo the input values. + actual_repl_port = server.server_address[1] + actual_control_port = cserver.server_address[1] + return bind, actual_repl_port, actual_control_port def stop(): diff --git a/unpythonic/net/tests/fixtures.py b/unpythonic/net/tests/fixtures.py index c727279c..e37c81c7 100644 --- a/unpythonic/net/tests/fixtures.py +++ b/unpythonic/net/tests/fixtures.py @@ -1,12 +1,9 @@ # -*- coding: utf-8; -*- """Unit test fixtures for network code.""" -import threading import socket -from time import sleep +import threading -# Server bind address for testing. -addrspec = ("127.0.0.1", 7777) def nettest(server_recv_func, client_send_func): """Server receives and the client sends. @@ -16,32 +13,61 @@ def nettest(server_recv_func, client_send_func): client_send_func: 1-arg callable; take socket, send data into it. No return value. """ - # TODO: IPv6 support + # Bind to port 0 so the kernel picks a free port for us, then read the + # actual port back via `getsockname()`. Using a hardcoded port causes + # mysterious `OSError: Address already in use` failures when another + # process (or another test) happens to hold the port. + # + # We call `bind()` and `listen()` in the main thread, synchronously, + # *before* spawning either worker thread. That's what makes the + # fixture race-free without any explicit readiness signal: by the + # time the client thread calls `connect()`, the listening socket + # already exists, and the kernel will queue the incoming connection + # on the accept backlog until the server thread gets around to + # calling `accept()`. No `threading.Event` needed — the TCP stack + # is already the synchronization primitive. + server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_sock.bind(("127.0.0.1", 0)) + server_sock.listen() + addrspec = server_sock.getsockname() + + # Exceptions captured and re-raised in the main thread after the worker + # threads join. Previously these were swallowed into a `print(err)`, + # which buried the real cause of any failure in the test output and + # made the test subsequently `IndexError` on the empty `result` list. + errors = [] result = [] + def recv_server(): try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(addrspec) - sock.listen() - conn, addr = sock.accept() - with conn: - data = server_recv_func(conn) - result.append(data) - except Exception as err: - print(err) + conn, _addr = server_sock.accept() + try: + data = server_recv_func(conn) + result.append(data) + finally: + conn.close() + except BaseException as err: + errors.append(err) + def send_client(): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.connect(addrspec) client_send_func(sock) - except Exception as err: - print(err) + except BaseException as err: + errors.append(err) + ts = threading.Thread(target=recv_server) tc = threading.Thread(target=send_client) - ts.start() - sleep(0.05) - tc.start() - ts.join() - tc.join() + try: + ts.start() + tc.start() + ts.join() + tc.join() + finally: + server_sock.close() + + if errors: + raise errors[0] return result[0] diff --git a/unpythonic/net/tests/test_client.py b/unpythonic/net/tests/test_client.py new file mode 100644 index 00000000..49cb1d18 --- /dev/null +++ b/unpythonic/net/tests/test_client.py @@ -0,0 +1,565 @@ +# -*- coding: utf-8; -*- +"""Tier 1 REPL tests for `unpythonic.net.client` / `unpythonic.net.server`. + +In-process, single-test-process. The REPL server runs on `127.0.0.1:0` +in a daemon thread; the client's interactive loop runs in the main test +thread, driven by a `scripted_repl` context manager that captures +stdout/stderr and feeds a pre-scripted input sequence through a fake +`input()` piped in via the `_input` parameter of `client._connect`. + +Why `_input` rather than monkey-patching `builtins.input` globally (as +`mcpyrate/test/test_126_repl.py` does): the server's `InteractiveConsole` +also calls `builtins.input` internally — it needs to, because on the +session thread `sys.stdin` is a `Shim(_threadlocal_stdin)` pointing at +the PTY slave and the only way `code.InteractiveConsole.raw_input` +reaches that is via `input(prompt)`. A global monkey-patch would +hijack the server's input path too, and the test would hang. The +`_input` seam lets us replace the client-side `input()` without +touching the server-side one. + +The `scripted_repl` helper is the two-REPL-in-one-process sibling of +`mcpyrate/test/test_126_repl.py` (committed there as `0fee81b`), which +is the simpler "one REPL in the process" version. The shape diverges +intentionally for load-bearing architectural reasons: + + * mcpyrate's version monkey-patches `builtins.input` and replaces + `sys.stdout` / `sys.stderr` with `StringIO` — simple, correct for + a single in-process `MacroConsole`. + + * This version cannot do either. `unpythonic.net.server` also runs + an `InteractiveConsole` in the same process (on a session thread), + which *also* calls `builtins.input`, so a global patch would + hijack the server. And the server installs a + `Shim(_threadlocal_stdout)` as `sys.stdout` to route per-thread to + each session's PTY slave — replacing `sys.stdout` globally would + kill that routing and the client would hang forever waiting for a + prompt that never arrives. + + * So this version (a) exposes `fake_input` as a seam the caller + threads through `_connect(_input=...)`, and (b) mutates the + **main-thread slot** of `server._threadlocal_stdout/stderr` via + `ThreadLocalBox.__lshift__`, which leaves session-thread routing + untouched. + +Grep for "scripted_repl" across the fleet if you need to cross-check +the pattern in a third project; keep both versions in mind, pick the +simpler one unless you hit the two-REPL constraint. + +Cross-platform: `unpythonic.net` runs on MS Windows too, via the +`socket.socketpair`-based `WindowsPTYSocketProxy` backend. The test +suite runs the full integration tests on every platform in the CI +matrix. A platform-conditional testset (`tier 1: Windows backend via +server (POSIX-only smoke)`) force-runs the Windows backend on a POSIX +dev machine as extra insurance that the Windows code path is covered +without waiting for Windows CI. + +**Tier 1 is zero coverage of readline itself.** The `_input` seam in +`client._connect(..., _input=...)` replaces the entire `input()` +pathway before `readline` is ever entered, so the client-side +line editor, history, tab-completer binding (as rendered to the user +via key events), and interrupt-during-input are not "partially +covered" — they are not covered at all. Tier 1 is a protocol and +plumbing test, not a terminal-UX test. A regression in a +`readline.parse_and_bind` call, in the custom remote completer +wiring, or in the SIGINT-during-readline path would pass tier 1 +silently. Validation of those behaviours currently requires a human +at a real terminal; a subprocess + `pexpect` tier 2 would be the +principled fix if a real-world regression ever made the case for it. +""" + +import contextlib +import io +import platform +import socket +import sys +import threading +import time +import types + +from ...syntax import macros, test, the, warn # noqa: F401 +from ...test.fixtures import session, testset + +from ..msg import MessageDecoder +from ..ptyproxy import PTYSocketProxy +from ..util import socketsource +from ..common import ApplevelProtocolMixin + + +@contextlib.contextmanager +def scripted_repl(script): + """Drive the client's interactive REPL through a pre-scripted input sequence. + + `script` is an iterable of strings, each one line the user would + type (no trailing newlines). When the script is exhausted, the + next `input()` call raises `EOFError` — which is how a normal REPL + exits on Ctrl+D, and how the `unpythonic.net` client sends `quit()` + to the server for a clean disconnect. + + On exit from the `with` block, `captured.stdout` and `captured. + stderr` are materialized to plain strings. Materialization happens + in `finally`, so it runs even on test failure and the interface is + consistent between the success and failure paths. + + The fake `input` is yielded as `captured.fake_input` so the caller + can pass it into `client._connect(..., _input=captured.fake_input)`. + + **Must be used inside a `test_repl_server()` context.** The naive + approach of `sys.stdout = StringIO()` would be *wrong* here, because + once `unpythonic.net.server` is running, `sys.stdout` is a + `Shim(_threadlocal_stdout)` that routes writes per-thread — each + session thread writes to its own PTY slave. A global reassignment + of `sys.stdout` would replace that Shim and kill the server's PTY + routing, so the session thread's eval results would go nowhere and + the client would block forever waiting for a prompt that never + arrives. (Ask me how I know.) + + The right layer is the `ThreadLocalBox` that backs the Shim: we + override the **main thread's** slot in `server._threadlocal_stdout`, + which leaves the session threads' slots untouched. Client writes + (which run in the main thread, through the same Shim) then land in + our `StringIO`; server writes (which run in session threads) still + land in the PTY slave. On exit, we `clear()` the main-thread slot + so the box falls back to its default (the real stdout). + + Usage:: + + with test_repl_server() as (rport, cport): + with scripted_repl(["2 + 3"]) as captured: + client._connect(host, rport, cport, _input=captured.fake_input) + assert "5" in captured.stdout + """ + # Imported locally to keep this helper usable only when the server + # module has been loaded. `server._threadlocal_stdout/stderr` only + # exist (as ThreadLocalBoxes inside a Shim) once the module has run + # its top-level code. Importing at module top would work too, but + # the local import makes the dependency explicit right here. + from .. import server + + lines = iter(script) + + def fake_input(prompt=""): + # Echo the prompt into the captured stream so tests that care + # about prompt text can see it. A real tty would also echo. + sys.stdout.write(prompt) + sys.stdout.flush() + try: + line = next(lines) + except StopIteration: + raise EOFError # REPL's normal exit path (Ctrl+D) + # Echo the "typed" line too, matching real tty behaviour. + sys.stdout.write(line + "\n") + return line + + captured = types.SimpleNamespace( + stdout=io.StringIO(), + stderr=io.StringIO(), + fake_input=fake_input, + ) + try: + # Main-thread override only — see docstring. + server._threadlocal_stdout << captured.stdout + server._threadlocal_stderr << captured.stderr + yield captured + finally: + # Remove the main-thread override so the box falls back to its + # default (the real stdout). We don't just reassign the + # previous value, because there wasn't one in this thread before + # we started — the box was holding its default. + server._threadlocal_stdout.clear() + server._threadlocal_stderr.clear() + # Materialize live StringIO → plain str so assertions after the + # `with` see strings, not file-like objects. Runs on the + # failure path too. + captured.stdout = captured.stdout.getvalue() + captured.stderr = captured.stderr.getvalue() + + +@contextlib.contextmanager +def test_repl_server(): + """Start an `unpythonic.net.server` on `127.0.0.1:0` for the duration of the test. + + Yields `(repl_port, control_port)` — the kernel-assigned port numbers + returned by `server.start()`. The context manager guarantees + `server.stop()` runs on exit (even on test failure), so the next + test gets a clean `_server_instance = None` state. + + Uses `banner=""` to keep the server banner out of captured stdout, + since tier-1 tests want to assert on eval results, not boilerplate. + """ + # Imported inside the function so the test module can be collected + # on MS Windows (the platform check below will skip tests cleanly, + # but the `import` of `..server` must not explode at module load). + from .. import server + bind, rport, cport = server.start( + locals={}, + bind="127.0.0.1", + repl_port=0, + control_port=0, + banner="", + ) + try: + yield rport, cport + finally: + server.stop() + + +def _wait_for_port(host, port, timeout=2.0): + """Retry `socket.create_connection` until the server is accepting connections. + + `ReuseAddrThreadingTCPServer.__init__` binds and listens synchronously, + so in theory the server is ready by the time `server.start()` returns. + In practice there is still a race with `serve_forever` picking up the + first accept — we've seen the first connection occasionally get a + connection-refused on a loaded machine. A couple of retries with a + small backoff absorbs that. + """ + deadline = time.monotonic() + timeout + last_err = None + while time.monotonic() < deadline: + try: + sock = socket.create_connection((host, port), timeout=0.5) + sock.close() + return + except (ConnectionRefusedError, OSError) as err: + last_err = err + time.sleep(0.01) + raise RuntimeError(f"Server at {host}:{port} did not become ready within {timeout}s: {last_err}") + + +def _run_cleanup_contract_suite(proxy_cls): + """Run the four cleanup-contract sub-tests against a specific backend class. + + Called once per backend (POSIX and Windows) from the top-level + `tier 1: ptyproxy cleanup contract` testset. Factoring into a helper + keeps the tests DRY while still exercising both backends — the + Windows backend works on POSIX too (via `socket.socketpair`), + so we can test it on a Linux dev box without waiting for Windows CI. + + Resource-close verification uses the attribute contract + (`master`/`slave` become `None` after teardown). We don't poke at the + raw fd with `os.fstat` because the fd type differs between backends + (POSIX: int, Windows: socket). + """ + with testset("stop-before-start releases resources"): + sock = socket.socket() + try: + proxy = proxy_cls(sock) + test[proxy.master is not None] + test[proxy.slave is not None] + proxy.stop() + # Latent bug before fix: `stop()` was gated on + # `if self._thread:` and did nothing if `start()` had + # never been called, leaking both fds. + test[proxy.master is None] + test[proxy.slave is None] + finally: + sock.close() + + with testset("double stop is idempotent"): + sock = socket.socket() + try: + proxy = proxy_cls(sock) + proxy.stop() + proxy.stop() # must not raise + test[proxy.master is None] + test[proxy.slave is None] + finally: + sock.close() + + with testset("exception in `with` body triggers cleanup"): + sock = socket.socket() + proxy_captured = None + caught = False + try: + with proxy_cls(sock) as proxy: + proxy_captured = proxy + raise RuntimeError("simulated crash in with body") + except RuntimeError: + caught = True + # The exception must have propagated out of the `with` — the + # context manager returns False from __exit__, i.e. does not + # suppress. + test[caught] + # …and `stop()` must have run via __exit__, releasing the fds. + test[proxy_captured.master is None] + test[proxy_captured.slave is None] + sock.close() + + with testset("name readable after stop()"): + sock = socket.socket() + try: + proxy = proxy_cls(sock) + cached_name = proxy.name # whatever the backend chose + proxy.stop() + # `name` is cached at construction time so log messages + # in a teardown `finally:` block can still reference it + # after the underlying slave transport is gone. + test[proxy.name == cached_name] + finally: + sock.close() + + +def runtests(): + # Cleanup contract tests — cross-platform, run before the Windows + # early-return below. These exercise `PTYSocketProxy` construction, + # teardown, and context-manager semantics without touching the + # server/client roundtrip. + with testset("tier 1: ptyproxy cleanup contract"): + # Windows backend: works on any platform (`socket.socketpair` is + # cross-platform), so we always run it. On a POSIX dev box this + # gives us real coverage of the Windows backend code without + # waiting for Windows CI to light up. + with testset("Windows backend (socketpair)"): + from ..ptyproxy_windows import WindowsPTYSocketProxy + _run_cleanup_contract_suite(WindowsPTYSocketProxy) + + # POSIX backend: uses `os.openpty`, `termios`, `tty` — imports + # blow up on Windows. Gated. + if platform.system() != "Windows": + with testset("POSIX backend (openpty)"): + from ..ptyproxy_posix import PosixPTYSocketProxy + _run_cleanup_contract_suite(PosixPTYSocketProxy) + + with testset("dispatch picks the right backend"): + sock = socket.socket() + try: + proxy = PTYSocketProxy(sock) + if platform.system() == "Windows": + from ..ptyproxy_windows import WindowsPTYSocketProxy + test[type(proxy) is WindowsPTYSocketProxy] + else: + from ..ptyproxy_posix import PosixPTYSocketProxy + test[type(proxy) is PosixPTYSocketProxy] + proxy.stop() + finally: + sock.close() + + from .. import client + + if platform.system() != "Windows": + with testset("tier 1: Windows backend via server (POSIX-only smoke)"): + # Cross-platform validation trick: on POSIX, force the server + # to use `WindowsPTYSocketProxy` instead of the native POSIX + # backend, then run a minimal full-REPL roundtrip through it. + # This exercises the Windows backend's `forward_traffic` + # thread (select.select + socketpair), `open_slave_streams` + # (sock.makefile text I/O with line buffering), and + # `write_to_master` (sock.sendall) under real REPL load — + # all on a Linux dev machine. + # + # On Windows itself the native backend *is* Windows, so this + # force-smoke is redundant: the full integration testset + # below already exercises `WindowsPTYSocketProxy` through + # the same code path. Hence the guard. + from .. import server as _server_module + from ..ptyproxy_windows import WindowsPTYSocketProxy + _original_backend = _server_module.PTYSocketProxy + _server_module.PTYSocketProxy = WindowsPTYSocketProxy + try: + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + with scripted_repl(["7 * 8"]) as captured: + client._connect("127.0.0.1", rport, cport, _input=captured.fake_input) + test["56" in the[captured.stdout]] + finally: + _server_module.PTYSocketProxy = _original_backend + + with testset("tier 1: full-client ↔ server roundtrip"): + with testset("basic arithmetic roundtrip"): + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + with scripted_repl(["2 + 3"]) as captured: + client._connect("127.0.0.1", rport, cport, _input=captured.fake_input) + # The server eval result "5" must appear in captured + # client stdout (which also contains the session banner, + # the prompt, and the echoed input). + test["5" in the[captured.stdout]] + + with testset("multi-line function definition"): + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + with scripted_repl([ + "def f():", + " return 42", + "", + "f()", + ]) as captured: + client._connect("127.0.0.1", rport, cport, _input=captured.fake_input) + test["42" in the[captured.stdout]] + + with testset("syntax error recovery"): + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + with scripted_repl([ + "this is : not valid python $$$", + "1 + 1", + ]) as captured: + client._connect("127.0.0.1", rport, cport, _input=captured.fake_input) + combined = captured.stdout + captured.stderr + # The server must report the SyntaxError … + test["SyntaxError" in the[combined]] + # … and the session must survive the bad line: the + # following good line's result still shows up. + test["2" in the[captured.stdout]] + + with testset("clean disconnect on EOF"): + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + with scripted_repl([]) as captured: + client._connect("127.0.0.1", rport, cport, _input=captured.fake_input) + # No traceback should escape on the clean path; the + # client's own "Session closed." message should appear. + test["Traceback" not in the[captured.stderr]] + test["Session closed" in the[captured.stdout]] + + with testset("tier 1: netcat-mode raw socket"): + with test_repl_server() as (rport, cport): # noqa: F841 -- we only need rport here + _wait_for_port("127.0.0.1", rport) + # Talk to the REPL port directly, without using the + # `unpythonic.net.client`. This exercises the + # netcat-compat path on the server: no control channel, no + # pairing, no handshake — just raw line-oriented I/O through + # the PTY. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.connect(("127.0.0.1", rport)) + sock.settimeout(3.0) + + def recv_until(needle, max_wait=3.0): + """Read from the socket until `needle` (bytes) appears, or timeout.""" + buf = b"" + deadline = time.monotonic() + max_wait + while needle not in buf: + if time.monotonic() >= deadline: + break + try: + chunk = sock.recv(4096) + except socket.timeout: + break + if not chunk: + break + buf += chunk + return buf + + # Drain the banner/prompt header so the eval result is + # the next thing we see. + recv_until(b">>>> ") + sock.sendall(b"2 + 3\n") + tail = recv_until(b">>>> ", max_wait=3.0) + test[b"5" in the[tail]] + # Close politely. The server's `on_socket_disconnect` + # writes `"quit()\n"` to the PTY master, which tears + # down the session without crashing the server thread. + + with testset("tier 1: control-channel RPC"): + with test_repl_server() as (rport, cport): # noqa: F841 -- we only need cport here + _wait_for_port("127.0.0.1", cport) + # Talk to the control port directly using the app-level + # protocol. This bypasses the REPL loop entirely — it + # tests only the DescribeServer / TabComplete RPC surface. + + class _ProbeClient(ApplevelProtocolMixin): + def __init__(self, sock): + self.sock = sock + self.decoder = MessageDecoder(socketsource(sock)) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as csock: + csock.connect(("127.0.0.1", cport)) + probe = _ProbeClient(csock) + + # DescribeServer: must return status=ok and a prompts dict. + probe._send({"command": "DescribeServer"}) + reply = probe._recv() + # `the[reply]` (not `the[reply["status"]]`) so a failure + # shows the whole reply dict, including any "reason" + # field the server may have set — more actionable than + # just seeing `reply["status"] == "failed"`. + test[the[reply]["status"] == "ok"] + test["ps1" in the[reply["prompts"]]] + test["ps2" in the[reply["prompts"]]] + + # TabComplete: ask for completions of "pri" in state 0. + # `rlcompleter.Completer` over an empty namespace will + # still find builtins like `print`. + probe._send({"command": "TabComplete", "text": "pri", "state": 0}) + reply = probe._recv() + test[the[reply]["status"] == "ok"] + test[the[reply]["result"] is not None] + test["print" in the[reply["result"]]] + + with testset("tier 1 stretch: sequential reconnect"): + # Start a server once, then connect / disconnect / reconnect with + # the full client. Regression check for session teardown hygiene: + # if `ConsoleSession` or `PTYSocketProxy` leaks resources on exit, + # the second connect is where it would show. + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + + with scripted_repl(["10 * 11"]) as captured1: + client._connect("127.0.0.1", rport, cport, _input=captured1.fake_input) + test["110" in the[captured1.stdout]] + + with scripted_repl(["20 * 21"]) as captured2: + client._connect("127.0.0.1", rport, cport, _input=captured2.fake_input) + test["420" in the[captured2.stdout]] + + with testset("tier 1 stretch: two concurrent clients"): + # Two real client loops in parallel threads, one server. The + # server supports multiple simultaneous REPL sessions (each gets + # its own thread via ThreadingTCPServer), and each session has + # its own `_threadlocal_stdout`/`_threadlocal_stderr` slot. So + # as long as the session threads don't stomp on each other, both + # clients should get their own results back without cross-talk. + # + # IMPORTANT: each client thread runs `_connect` from its own + # thread; the `scripted_repl` helper, however, overrides only + # the *main* thread's slot in `_threadlocal_stdout`. So for + # this test we can't use `scripted_repl` — we drive the client + # threads directly, and assert on values the server evals + # produced, by having each thread stash its result list. + # + # We sidestep the stdout-capture issue entirely: each client + # uses a plain fake_input that doesn't need captured output, + # and we just assert via the return path (inputs fed, clean + # exit, no exception crossed the thread boundary). + with test_repl_server() as (rport, cport): + _wait_for_port("127.0.0.1", rport) + _wait_for_port("127.0.0.1", cport) + + thread_errors = [] + + def run_one_client(script_lines): + lines = iter(script_lines) + def inp(prompt=""): + try: + return next(lines) + except StopIteration: + raise EOFError + try: + client._connect("127.0.0.1", rport, cport, _input=inp) + except BaseException as err: # pragma: no cover + thread_errors.append(err) + + # Two disjoint scripts; we assert on the fact that both + # clients reach clean exit (EOFError → quit() → SessionExit) + # without raising. Getting this far means the server + # demuxed both sessions correctly. + t1 = threading.Thread(target=run_one_client, args=(["100 + 23"],)) + t2 = threading.Thread(target=run_one_client, args=(["200 + 46"],)) + t1.start() + t2.start() + t1.join(timeout=10.0) + t2.join(timeout=10.0) + test[not the[t1.is_alive()]] # didn't time out — `not` is unary, the[] required + test[not the[t2.is_alive()]] + test[thread_errors == []] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/net/tests/test_msg.py b/unpythonic/net/tests/test_msg.py index 66f6dd2e..c4a1f031 100644 --- a/unpythonic/net/tests/test_msg.py +++ b/unpythonic/net/tests/test_msg.py @@ -7,8 +7,8 @@ from .fixtures import nettest -from ..msg import encodemsg, MessageDecoder -from ..util import bytessource, streamsource, socketsource +from ..msg import encodemsg, decodemsg, MessageDecoder +from ..util import ReceiveBuffer, bytessource, streamsource, socketsource def runtests(): with testset("sans-IO"): @@ -79,6 +79,42 @@ def runtests(): test[decoder.decode() == b"hello again"] test[decoder.decode() is None] + with testset("decodemsg (free function form)"): + # `MessageDecoder` wraps `decodemsg` and manages the `ReceiveBuffer` + # internally; the tests above exercise both via the class-based path. + # `decodemsg` itself is also in `msg.__all__` as a public free-function + # entry point, so it deserves a direct test that doesn't route through + # the class. + with testset("basic roundtrip"): + buf = ReceiveBuffer() + source = bytessource(encodemsg(b"hello world")) + test[decodemsg(buf, source) == b"hello world"] + # Subsequent call: source is exhausted, returns None. + test[decodemsg(buf, source) is None] + + with testset("multiple messages with stream synchronization"): + # Junk between the two messages must be discarded; both messages + # must decode in order. Exercises the same invariants as the + # `MessageDecoder` tests above, via the free-function API. + bio = BytesIO() + bio.write(encodemsg(b"first")) + bio.write(b"junk junk junk") + bio.write(encodemsg(b"second")) + bio.seek(0, SEEK_SET) + buf = ReceiveBuffer() + source = streamsource(bio) + test[decodemsg(buf, source) == b"first"] + test[decodemsg(buf, source) == b"second"] + test[decodemsg(buf, source) is None] + + with testset("binary-safe payload"): + # Messages may contain arbitrary bytes, including the sync-byte + # value (0xFF) inside the payload. + payload = bytes(range(256)) + buf = ReceiveBuffer() + source = bytessource(encodemsg(payload)) + test[decodemsg(buf, source) == payload] + with testset("with TCP sockets"): def server1(sock): decoder = MessageDecoder(socketsource(sock)) diff --git a/unpythonic/net/tests/test_util.py b/unpythonic/net/tests/test_util.py index c600db79..07050029 100644 --- a/unpythonic/net/tests/test_util.py +++ b/unpythonic/net/tests/test_util.py @@ -1,11 +1,13 @@ # -*- coding: utf-8; -*- -from ...syntax import macros, test, warn # noqa: F401 +import socketserver + +from ...syntax import macros, test, test_raises, warn # noqa: F401 from ...test.fixtures import session, testset from .fixtures import nettest -from ..util import recvall, netstringify +from ..util import ReceiveBuffer, ReuseAddrThreadingTCPServer, recvall, netstringify def runtests(): with testset("netstringify"): @@ -16,6 +18,126 @@ def runtests(): client = lambda sock: [sock.sendall(b"x" * 512), sock.sendall(b"x" * 512)] test[len(nettest(server, client)) == 1024] + with testset("ReuseAddrThreadingTCPServer"): + # Direct regression guard for a latent bug that hit us once already: + # the class used to override `server_bind()` to set `SO_REUSEADDR`, + # but the override silently dropped the + # `self.server_address = self.socket.getsockname()` refresh from + # stdlib `TCPServer.server_bind`. That line is what updates + # `server_address` to reflect the kernel-assigned port when you + # bind to port 0. Without it, `server.server_address[1]` reports + # 0 even though the socket is listening on a real port. + # + # The fix (commit `4243ded`) was to delete the custom override + # and just set `allow_reuse_address = True` as a class attribute, + # letting stdlib's `server_bind` do both the sockopt AND the + # `server_address` refresh. This testset guards the fix. + + class _NullHandler(socketserver.BaseRequestHandler): + def handle(self): + pass + + with testset("bind to port 0 returns the actually-bound port"): + server = ReuseAddrThreadingTCPServer(("127.0.0.1", 0), _NullHandler) + try: + actual_port = server.server_address[1] + # The bug manifested as `actual_port == 0`; with the fix, + # we see a real kernel-assigned port (1024..65535 range, + # but we only check non-zero since the exact port is + # non-deterministic). + test[actual_port != 0] + test[actual_port > 0] + finally: + server.server_close() + + with testset("allow_reuse_address is enabled"): + # The class attribute form: stdlib `TCPServer.server_bind` + # checks this and calls `setsockopt(SO_REUSEADDR, 1)` when + # truthy. We verify the class attribute is in place — the + # actual sockopt is exercised transitively by the fact that + # the integration tests run repeatedly without "Address + # already in use" errors. + test[ReuseAddrThreadingTCPServer.allow_reuse_address is True] + + with testset("ReceiveBuffer"): + # `ReceiveBuffer` is a thin `BytesIO` wrapper with message-protocol + # semantics: append bytes as more arrive on a transport, `getvalue()` + # to inspect what's there, `set()` to replace (typically after a + # message boundary has been consumed and the remainder needs to + # stay in the buffer for the next message). It's part of + # `unpythonic.net.util.__all__` — a public API used internally by + # `MessageDecoder` and externally by at least one fleet-outside + # consumer (`raven.common.netutil`). + + with testset("construction"): + # Default: empty buffer. + test[ReceiveBuffer().getvalue() == b""] + # Initial contents populate the buffer. + test[ReceiveBuffer(b"hello").getvalue() == b"hello"] + # `getvalue()` is non-destructive — calling twice returns + # the same bytes, buffer still has them afterwards. + buf = ReceiveBuffer(b"abc") + test[buf.getvalue() == b"abc"] + test[buf.getvalue() == b"abc"] + + with testset("append"): + buf = ReceiveBuffer() + buf.append(b"hello") + test[buf.getvalue() == b"hello"] + # Multiple appends accumulate in order. + buf.append(b" ") + buf.append(b"world") + test[buf.getvalue() == b"hello world"] + # Empty append is a no-op. + buf.append(b"") + test[buf.getvalue() == b"hello world"] + # `append` returns self — chainable. + test[buf.append(b"!") is buf] + test[buf.getvalue() == b"hello world!"] + + with testset("set replaces contents"): + buf = ReceiveBuffer(b"old contents") + buf.set(b"new") + test[buf.getvalue() == b"new"] + # set() with empty bytes clears the buffer. + buf.set(b"") + test[buf.getvalue() == b""] + # `set` also returns self. + test[ReceiveBuffer().set(b"x") is not None] + + with testset("set + append: position is at end, not zero"): + # This is a subtle but documented property: `set(new_contents)` + # must leave the internal stream position at the *end* of the + # new contents, so a subsequent `append` continues from where + # `set` left off. The naive refactor `self._buffer = + # BytesIO(new_contents)` puts the position at 0 and the next + # write would overwrite — this test guards that regression. + buf = ReceiveBuffer() + buf.set(b"abc") + buf.append(b"def") + test[buf.getvalue() == b"abcdef"] + # Also via the __init__ path, which delegates to set(). + buf2 = ReceiveBuffer(b"foo") + buf2.append(b"bar") + test[buf2.getvalue() == b"foobar"] + + with testset("type errors"): + # `ReceiveBuffer` rejects non-`bytes` inputs strictly — `bytearray`, + # `memoryview`, and `str` are all refused even though `BytesIO` + # itself would accept some of them. This is deliberate: message- + # protocol code expects immutable `bytes` boundaries and a + # `bytearray` input could mutate under the buffer's feet. + test_raises[TypeError, ReceiveBuffer().append("not bytes")] + test_raises[TypeError, ReceiveBuffer().append(bytearray(b"nope"))] + test_raises[TypeError, ReceiveBuffer().append(memoryview(b"nope"))] + test_raises[TypeError, ReceiveBuffer().append(42)] + test_raises[TypeError, ReceiveBuffer().append(None)] + test_raises[TypeError, ReceiveBuffer().set("not bytes")] + test_raises[TypeError, ReceiveBuffer().set(bytearray(b"nope"))] + # Construction delegates to `set`, so the same check applies. + test_raises[TypeError, ReceiveBuffer("not bytes")] + test_raises[TypeError, ReceiveBuffer(bytearray(b"nope"))] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/net/util.py b/unpythonic/net/util.py index 541a6c09..4d56dc84 100644 --- a/unpythonic/net/util.py +++ b/unpythonic/net/util.py @@ -16,11 +16,16 @@ # https://docs.python.org/3/library/socketserver.html#socketserver.ThreadingMixIn # https://docs.python.org/3/library/socketserver.html#socketserver.TCPServer class ReuseAddrThreadingTCPServer(socketserver.ThreadingTCPServer): - def server_bind(self): - """Custom server_bind ensuring the socket is available for rebind immediately.""" - # from https://stackoverflow.com/a/18858817 - self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.socket.bind(self.server_address) + """`ThreadingTCPServer` with `SO_REUSEADDR` enabled. + + Setting `allow_reuse_address = True` is the stdlib-blessed way to get + `SO_REUSEADDR` on the listening socket — `TCPServer.server_bind` already + sets the sockopt *and* refreshes `self.server_address` from + `socket.getsockname()` after binding, which is important when binding + to port 0 (kernel-assigned port): the caller can then read the actual + bound port from `server.server_address[1]`. + """ + allow_reuse_address = True # We could achieve the same result using a `unpythonic.collections.box` to diff --git a/unpythonic/numutil.py b/unpythonic/numutil.py index f573f262..91f5884c 100644 --- a/unpythonic/numutil.py +++ b/unpythonic/numutil.py @@ -1,10 +1,28 @@ # -*- coding: utf-8 -*- """Low-level utilities for numerics.""" -__all__ = ["almosteq", "ulp"] +__all__ = ["almosteq", "ulp", + "fixpoint", + "partition_int", "partition_int_triangular", "partition_int_custom"] +from collections.abc import Callable, Generator, Iterable +from itertools import takewhile from math import floor, log2 import sys +from typing import TypeVar + +T = TypeVar('T') + +from .it import iterate1, last, within, rev +from .symbol import sym + +# HACK: break dependency loop mathseq -> numutil -> mathseq +_init_done = False +triangular = sym("triangular") # doesn't matter what the value is, will be overwritten later +def _init_module() -> None: # called by unpythonic.__init__ when otherwise done + global triangular, _init_done + from .mathseq import triangular + _init_done = True class _NoSuchType: pass @@ -18,7 +36,7 @@ class _NoSuchType: # TODO: Overhaul `almosteq` in v0.16.0, should work like mpf for consistency. -def almosteq(a, b, tol=1e-8): +def almosteq(a: float, b: float, tol: float = 1e-8) -> bool: """Almost-equality that supports several formats. The tolerance ``tol`` is used for the builtin ``float`` and ``mpmath.mpf``. @@ -52,7 +70,7 @@ def almosteq(a, b, tol=1e-8): return d / min(abs(a) + abs(b), max_float) < tol -def ulp(x): # Unit in the Last Place +def ulp(x: float) -> float: # Unit in the Last Place """Given a float x, return the unit in the last place (ULP). This is the numerical value of the least-significant bit, as a float. @@ -65,3 +83,162 @@ def ulp(x): # Unit in the Last Place # m_min = abs. value represented by a mantissa of 1.0, with the same exponent as x has m_min = 2**floor(log2(abs(x))) return m_min * eps + + +def fixpoint(f: Callable[[T], T], x0: T, tol: float = 0) -> T: + """Compute the (arithmetic) fixed point of f, starting from the initial guess x0. + + (Not to be confused with the logical fixed point with respect to the + definedness ordering.) + + The fixed point must be attractive for this to work. See the Banach + fixed point theorem. + https://en.wikipedia.org/wiki/Banach_fixed-point_theorem + + If the fixed point is attractive, and the values are represented in + floating point (hence finite precision), the computation should + eventually converge down to the last bit (barring roundoff or + catastrophic cancellation in the final few steps). Hence the default tol + of zero. + + CAUTION: an arbitrary function from ℝ to ℝ **does not** necessarily + have a fixed point. Limit cycles and chaotic behavior of `f` will cause + non-termination. Keep in mind the classic example: + https://en.wikipedia.org/wiki/Logistic_map + + Examples:: + from math import cos, sqrt + from unpythonic import fixpoint, ulp + c = fixpoint(cos, x0=1) + + # Actually "Newton's" algorithm for the square root was already known to the + # ancient Babylonians, ca. 2000 BCE. (Carl Boyer: History of mathematics) + # Concerning naming, see also https://en.wikipedia.org/wiki/Stigler's_law_of_eponymy + def sqrt_newton(n): + def sqrt_iter(x): # has an attractive fixed point at sqrt(n) + return (x + n / x) / 2 + return fixpoint(sqrt_iter, x0=n / 2) + assert abs(sqrt_newton(2) - sqrt(2)) <= ulp(1.414) + """ + return last(within(tol, iterate1(f, x0))) + + +def partition_int(n: int, lower: int = 1, upper: int | None = None) -> Generator[tuple[int, ...], None, None]: + """Yield all ordered sequences of smaller positive integers that sum to `n`. + + `n` must be an integer >= 1. + + `lower` is an optional lower limit for each member of the sum. Each member + of the sum must be `>= lower`. + + (Most of the splits are a ravioli consisting mostly of ones, so it is much + faster to not generate such splits than to filter them out from the result. + The default value `lower=1` generates everything.) + + `upper` is, similarly, an optional upper limit; each member of the sum + must be `<= upper`. The default `None` means no upper limit (effectively, + in that case `upper=n`). + + It must hold that `1 <= lower <= upper <= n`. + + Not to be confused with `unpythonic.it.partition`, which partitions an + iterable based on a predicate. + + **CAUTION**: The number of possible partitions grows very quickly with `n`, + so in practice this is only useful for small numbers, or with a lower limit + that is not too much smaller than `n / 2`. A possible use case for this + function is to determine the number of letters to allocate for each + component of an anagram that may consist of several words. + + See: + https://en.wikipedia.org/wiki/Partition_(number_theory) + """ + # sanity check the preconditions, fail-fast + if not isinstance(n, int): + raise TypeError(f"n must be integer; got {type(n)} with value {repr(n)}") + if not isinstance(lower, int): + raise TypeError(f"lower must be integer; got {type(lower)} with value {repr(lower)}") + if upper is not None and not isinstance(upper, int): + raise TypeError(f"upper must be integer; got {type(upper)} with value {repr(upper)}") + upper = upper if upper is not None else n + if n < 1: + raise ValueError(f"n must be positive; got {n}") + if lower < 1 or upper < 1 or lower > n or upper > n or lower > upper: + raise ValueError(f"it must hold that 1 <= lower <= upper <= n; got lower={lower}, upper={upper}") + + return partition_int_custom(n, range(min(n, upper), lower - 1, -1)) # instantiate the generator + +def partition_int_triangular(n: int, lower: int = 1, upper: int | None = None) -> Generator[tuple[int, ...], None, None]: + """Like `partition_int`, but allow only triangular numbers in the result. + + Triangular numbers are 1, 3, 6, 10, ... + + This function answers the timeless question: if I have `n` stackable plushies, + what are the possible stack configurations? Example:: + + configurations = partition_int_triangular(78, lower=10) + print(frozenset(tuple(sorted(c)) for c in configurations)) + + Result:: + + frozenset({(10, 10, 10, 10, 10, 28), + (10, 10, 15, 15, 28), + (15, 21, 21, 21), + (21, 21, 36), + (78,)}) + + Here `lower` sets the minimum number of plushies to allocate for one stack. + """ + if not isinstance(n, int): + raise TypeError(f"n must be integer; got {type(n)} with value {repr(n)}") + if not isinstance(lower, int): + raise TypeError(f"lower must be integer; got {type(lower)} with value {repr(lower)}") + if upper is not None and not isinstance(upper, int): + raise TypeError(f"upper must be integer; got {type(upper)} with value {repr(upper)}") + upper = upper if upper is not None else n + if n < 1: + raise ValueError(f"n must be positive; got {n}") + if lower < 1 or upper < 1 or lower > n or upper > n or lower > upper: + raise ValueError(f"it must hold that 1 <= lower <= upper <= n; got lower={lower}, upper={upper}") + + triangulars_upto_n = takewhile(lambda m: m <= n, + triangular()) + return partition_int_custom(n, rev(filter(lambda m: lower <= m <= upper, + triangulars_upto_n))) + +def partition_int_custom(n: int, components: Iterable[int]) -> Generator[tuple[int, ...], None, None]: + """Partition an integer in a custom way. + + `n`: integer to partition. + `components`: iterable of ints; numbers that are allowed to appear + in the partitioning result. Each number `m` must + satisfy `1 <= m <= n`. + + Will be forced into a `tuple` internally; hence, + only finite iterables are supported. + + See `partition_int`, `partition_triangular`. + """ + if not isinstance(n, int): + raise TypeError(f"n must be integer; got {type(n)} with value {repr(n)}") + if n < 1: + raise ValueError(f"n must be positive; got {n}") + components = tuple(components) + invalid_components = [not isinstance(x, int) for x in components] + if any(invalid_components): + raise TypeError(f"each component must be an integer; got invalid components {invalid_components}") + invalid_components = [not (1 <= x <= n) for x in components] + if any(invalid_components): + raise ValueError(f"each component x must be 1 <= x <= n; got n = {n}, with invalid components {invalid_components}") + def rec(components): + for k in components: + m = n - k + if m == 0: + yield (k,) + else: + out = [] + for item in partition_int_custom(m, tuple(x for x in components if x <= m)): + out.append((k,) + item) + for term in out: + yield term + return rec(components) diff --git a/unpythonic/regutil.py b/unpythonic/regutil.py index 575af6b6..33012db2 100644 --- a/unpythonic/regutil.py +++ b/unpythonic/regutil.py @@ -21,13 +21,18 @@ # would require its __init__.py to run first, but it in turn expects pretty # much all of the regular code to be already initialized. +from collections.abc import Callable +from typing import TypeVar + +F = TypeVar('F', bound=Callable) + # These names must be bound exactly once, as anyone may from-import them. -decorator_registry = [] -all_decorators = set() -tco_decorators = set() +decorator_registry: list[tuple[float, str]] = [] +all_decorators: set[str] = set() +tco_decorators: set[str] = set() # Basic idea shamelessly stolen from MacroPy's macro registry. -def register_decorator(priority=0.0, istco=False): +def register_decorator(priority: float = 0.0, istco: bool = False) -> Callable[[F], F]: """Decorator that registers a custom decorator for the syntax machinery. Unknown decorators cannot be reordered robustly, hence ``sort_lambda_decorators`` diff --git a/unpythonic/seq.py b/unpythonic/seq.py index d2b79cb2..234db0e5 100644 --- a/unpythonic/seq.py +++ b/unpythonic/seq.py @@ -8,6 +8,8 @@ "do", "do0", "assign"] from collections import namedtuple +from collections.abc import Callable +from typing import Any from .arity import arity_includes, UnknownArity from .dynassign import dyn @@ -18,7 +20,7 @@ from .symbol import sym # sequence side effects in a lambda -def begin(*vals): +def begin(*vals: Any) -> Any: """Racket-like begin: return the last value. Eager; bodys already evaluated by Python when this is called. @@ -33,7 +35,7 @@ def begin(*vals): """ return vals[-1] if len(vals) else None -def begin0(*vals): # eager, bodys already evaluated when this is called +def begin0(*vals: Any) -> Any: # eager, bodys already evaluated when this is called """Racket-like begin0: return the first value. Eager; bodys already evaluated by Python when this is called. @@ -48,7 +50,7 @@ def begin0(*vals): # eager, bodys already evaluated when this is called """ return vals[0] if len(vals) else None -def lazy_begin(*bodys): +def lazy_begin(*bodys: Callable[[], Any]) -> Any: """Racket-like begin: run bodys in sequence, return the last return value. Lazy; each body must be a thunk (0-argument function), to delay its evaluation @@ -73,7 +75,7 @@ def lazy_begin(*bodys): body() return last() -def lazy_begin0(*bodys): +def lazy_begin0(*bodys: Callable[[], Any]) -> Any: """Racket-like begin0: run bodys in sequence, return the first return value. Lazy; each body must be a thunk (0-argument function), to delay its evaluation @@ -101,7 +103,7 @@ def lazy_begin0(*bodys): # sequence one-input, one-output functions @passthrough_lazy_args -def pipe1(value0, *bodys): +def pipe1(value0: Any, *bodys: Callable[[Any], Any]) -> Any: """Perform a sequence of operations on an initial value. Bodys are applied left to right. @@ -176,18 +178,18 @@ class piped1: Eager; apply each function immediately and store the new value. """ - def __init__(self, x): + def __init__(self, x: Any) -> None: """Set up a pipe and load the initial value x into it.""" self._x = x - def __or__(self, f): + def __or__(self, f: Callable[[Any], Any] | sym) -> "piped1 | Any": """Pipe the value through the one-argument function f. - Return a ``piped`` object, for chainability. + Return a ``piped1`` object, for chainability. As the only exception, if ``f`` is the sentinel ``exitpipe``, return the current value (thus exiting the pipe). - A new ``piped`` object is created at each step of piping; + A new ``piped1`` object is created at each step of piping; the "update" is purely functional, nothing is overwritten. Examples:: @@ -208,29 +210,29 @@ def __repr__(self): # pragma: no cover @passthrough_lazy_args class lazy_piped1: - """Like piped, but apply the functions later. + """Like piped1, but apply the functions later. This matters if the initial value is mutable: - - ``piped`` computes immediately and stores a copy of the new result + - ``piped1`` computes immediately and stores a copy of the new result at each step. Any updates to the initial value are not seen by the pipeline. - - ``lazy_piped`` just sets up a computation, and performs it when eventually + - ``lazy_piped1`` just sets up a computation, and performs it when eventually piped into ``exitpipe``. The computation always looks up the latest state of the initial value. - Another way to say this is that ``lazy_piped`` looks up the initial value + Another way to say this is that ``lazy_piped1`` looks up the initial value dynamically, at get time. """ - def __init__(self, x, *, _funcs=None): + def __init__(self, x: Any, *, _funcs: tuple | None = None) -> None: """Set up a lazy pipe and load the initial value x into it. The ``_funcs`` parameter is for internal use. """ self._x = x self._funcs = force(_funcs or ()) - def __or__(self, f): + def __or__(self, f: Callable[[Any], Any] | sym) -> "lazy_piped1 | Any": """Pipe the value into f; but just plan to do so, don't perform it yet. To run the stored computation, pipe into ``exitpipe``. @@ -251,7 +253,7 @@ def append_succ(lis): def nextfibo(state): a, b = state fibos.append(a) # store result by side effect - return (b, a + b) # new state, handed to next function in the pipe + return (b, a + b) # new state, handed to the next function in the pipe p = lazy_piped1((1, 1)) # load initial state into a lazy pipe for _ in range(10): # set up pipeline p = p | nextfibo @@ -276,7 +278,7 @@ def __repr__(self): # pragma: no cover return f"" @passthrough_lazy_args -def pipe(values0, *bodys): +def pipe(values0: Any, *bodys: Callable) -> Any: """Like pipe1, but with arbitrary number of inputs/outputs at each step. The only restriction is that the call and return signatures must match: @@ -347,7 +349,7 @@ def pipe(values0, *bodys): return xs @passthrough_lazy_args -def pipec(values0, *bodys): +def pipec(values0: Any, *bodys: Callable) -> Any: """Like pipe, but curry each function before piping. Useful with the passthrough in ``curry``. Each function only needs to @@ -368,21 +370,25 @@ class piped: The only restriction is that the call and return signatures must match: each function must take those positional/named arguments the previous one returns. Use a `Values` object to denote multiple-return-values, and/or - named return values. + named return values (named return values are sent in to the next function + as named arguments). """ - def __init__(self, *xs, **kws): + def __init__(self, *xs: Any, **kws: Any) -> None: """Set up a pipe and load the initial values xs and kws into it. The inputs are automatically packed into a `Values`. """ self._xs = Values(*xs, **kws) - def __or__(self, f): + def __or__(self, f: Callable[..., Any] | sym) -> "piped | Any": """Pipe the values through the function f. If the data currently in the pipe is a `Values`, it is unpacked to the args and kwargs of `f`. Otherwise, we feed the data to `f` as a single positional argument. + As the only exception, if ``f`` is the sentinel ``exitpipe``, + return the current value (thus exiting the pipe). + Example:: f = lambda x, y: Values(2*x, y+1) @@ -431,10 +437,10 @@ def nextfibo(a, b): # now two arguments p = lazy_piped(1, 1) for _ in range(10): p = p | nextfibo - assert p | exitpipe == Values(a=89, b=144) # final state + assert p | exitpipe == Values(a=89, b=144) # run; check final state assert fibos == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55] """ - def __init__(self, *xs, _funcs=None, **kws): + def __init__(self, *xs: Any, _funcs: tuple | None = None, **kws: Any) -> None: """Set up a lazy pipe and load the initial values xs and kws into it. The inputs are automatically packed into a `Values`. @@ -443,7 +449,7 @@ def __init__(self, *xs, _funcs=None, **kws): """ self._xs = Values(*xs, **kws) self._funcs = force(_funcs or ()) - def __or__(self, f): + def __or__(self, f: Callable[..., Any] | sym) -> "lazy_piped | Any": """Pipe the values into f; but just plan to do so, don't perform it yet. When f is `exitpipe`, perform the planned computation. @@ -482,7 +488,7 @@ def __repr__(self): # pragma: no cover # do(): improved begin() that can name intermediate results and refer to them DoAssign = namedtuple("DoAssign", "name value") -def assign(**binding): +def assign(**binding: Any) -> "DoAssign": """Bind a name to a value inside a do(). Re-using a previous name overwrites. @@ -508,7 +514,7 @@ def assign(**binding): for k, v in binding.items(): return DoAssign(k, v) -def do(*items): +def do(*items: Any) -> Any: """Haskell-ish do, but without any monadic magic. Run ``items`` sequentially. Optionally, locally bind a name to each result, @@ -574,7 +580,7 @@ def do(*items): consistent for all of the expressions. """ e = env() - def maybe_call(v): + def maybe_call(v: Any) -> Any: if callable(v): try: if not arity_includes(v, 1): @@ -591,7 +597,7 @@ def maybe_call(v): item = maybe_call(item) # perform side effects return item # return the final value -def do0(*items): +def do0(*items: Any) -> Any: """Like do, but return the value of the first item. Examples:: diff --git a/unpythonic/singleton.py b/unpythonic/singleton.py index 8cf74262..27e58826 100644 --- a/unpythonic/singleton.py +++ b/unpythonic/singleton.py @@ -112,6 +112,7 @@ __all__ = ["Singleton"] import threading +from typing import Any from weakref import WeakValueDictionary _instances = WeakValueDictionary() @@ -124,7 +125,7 @@ # we override `__call__` **in the metaclass**, in order to override calls # of the class (i.e. constructor invocations). class ThereCanBeOnlyOne(type): - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> "Singleton": # For consistency with single-thread behavior, don't let more than one # `__call__` run concurrently. This eliminates a race when many threads # try to instantiate the singleton, guaranteeing only one of them will @@ -166,7 +167,7 @@ class Singleton(metaclass=ThereCanBeOnlyOne): """ # We allow extra args so that __init__ can have them, but ignore them in the # super __new__ call, since our super is `object`, which takes no extra args. - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> "Singleton": # What we want to do: # if cls not in _instances: # _instances[cls] = super().__new__(cls) diff --git a/unpythonic/slicing.py b/unpythonic/slicing.py index 3aea8e8e..bcddb363 100644 --- a/unpythonic/slicing.py +++ b/unpythonic/slicing.py @@ -1,15 +1,24 @@ # -*- coding: utf-8 -*- """Operations on sequences with native slice syntax. Syntactic sugar, pure Python.""" -__all__ = ["islice", "fup"] +__all__ = ["islice", "Sliced", "fup", "FupTarget", "Fuppable"] +from abc import abstractmethod +from collections.abc import Iterable, Iterator, Sequence from itertools import islice as islicef +from typing import Any from .fup import fupdate from .it import first, lastn, butlastn from .misc import CountingIterator -def islice(iterable): +class Sliced: + """Tag type for the return value of ``islice``. Subscript to perform slicing.""" + @abstractmethod + def __getitem__(self, k: int | slice) -> "Iterator | Any": + ... + +def islice(iterable: Iterable) -> Sliced: """Use itertools.islice with slice syntax, with some bonus features. Usage:: @@ -22,6 +31,9 @@ def islice(iterable): start or stop will force the iterable, because that is the only way to know its length. + The desired elements are held in an internal buffer until they are yielded + by iterating over the `islice`. + - A single index (negative also allowed) is interpreted as a length-1 islice starting at that index. The slice is then immediately evaluated and the item is returned. @@ -51,9 +63,9 @@ def islice(iterable): **CAUTION**: ``step``, if present, must be positive. """ # manually curry to take indices later, but expect them in subscript syntax to support slicing - class islice1: + class islice1(Sliced): """Subscript me to perform the slicing.""" - def __getitem__(self, k): + def __getitem__(self, k: int | slice) -> Iterator | Any: if isinstance(k, tuple): raise TypeError(f"multidimensional indexing not supported, got {k}") if isinstance(k, slice): @@ -101,7 +113,19 @@ def __getitem__(self, k): # return first(islicef(iterable, k, k + 1)) # return islice1() -def fup(seq): +class Fuppable: + """Ready to be fupped. Left-shift (``<<``) with values to perform the update.""" + @abstractmethod + def __lshift__(self, v: Any) -> Sequence: + ... + +class FupTarget: + """The target sequence of a ``fup``. Subscript to select where to fup it.""" + @abstractmethod + def __getitem__(self, k: int | slice) -> Fuppable: + ... + +def fup(seq: Sequence) -> FupTarget: """Functionally update a sequence. Usage:: @@ -124,14 +148,14 @@ def fup(seq): Named after the sound a sequence makes when it is hit by a functional update. """ # two-phase manual curry, first expect a subscript, then an lshift. - class fup1: + class fup1(FupTarget): """Subscript me to specify index or slice where to fupdate.""" - def __getitem__(self, k): + def __getitem__(self, k: int | slice) -> Fuppable: if isinstance(k, tuple): raise TypeError(f"multidimensional indexing not supported, got {k}") - class fup2: + class fup2(Fuppable): """Left-shift me with values to perform the fupdate.""" - def __lshift__(self, v): + def __lshift__(self, v: Any) -> Sequence: return fupdate(seq, k, v) return fup2() return fup1() diff --git a/unpythonic/symbol.py b/unpythonic/symbol.py index a9dccaaf..063427e8 100644 --- a/unpythonic/symbol.py +++ b/unpythonic/symbol.py @@ -13,12 +13,12 @@ import uuid # Symbol registry. Used for tracking symbol object identities within the same process. -_symbols = WeakValueDictionary() +_symbols: WeakValueDictionary[str, "sym"] = WeakValueDictionary() _symbols_update_lock = threading.Lock() # Gensyms go into a separate registry, to make name conflicts with named symbols # impossible, even if someone grabs one of the UUIDs and uses it as a name. -_gensyms = WeakValueDictionary() +_gensyms: WeakValueDictionary[uuid.UUID, "gsym"] = WeakValueDictionary() _gensyms_update_lock = threading.Lock() class Symbol: @@ -35,7 +35,7 @@ class sym(Symbol): In plain English: a lightweight, human-readable, process-wide unique marker, that can be quickly compared to another such marker by object identity. - name: any hashable, typically str. + name: str The human-readable name of the symbol. Maps to the object identity. Example:: @@ -58,7 +58,7 @@ class sym(Symbol): CAUTION: If you're familiar with JavaScript's `Symbol` and looking for that, see `gensym`. """ - def __new__(cls, name): # This covers unpickling, too. + def __new__(cls, name: str) -> "sym": # This covers unpickling, too. # What we want to do: # if name not in _symbols: # _symbols[name] = super().__new__(cls) @@ -79,7 +79,7 @@ def __new__(cls, name): # This covers unpickling, too. instance = _symbols[name] return instance - def __init__(self, name): + def __init__(self, name: str) -> None: self.name = name # Pickle support. The default `__setstate__` (writing to `self.__dict__`) @@ -89,12 +89,12 @@ def __init__(self, name): # Note we don't `sys.intern` the name *strings*; if we did, we'd need a # custom `__setstate__` to redo that upon unpickling, since for `pickle` # a string is a string, whether the original was interned or not. - def __getnewargs__(self): + def __getnewargs__(self) -> tuple: return (self.name,) - def __str__(self): + def __str__(self) -> str: return self.name - def __repr__(self): + def __repr__(self) -> str: return f'sym("{self.name}")' @@ -110,7 +110,7 @@ class gsym(Symbol): label: str The human-readable label, shown in `str` and `repr`. """ - def __new__(cls, uid, label): + def __new__(cls, uid: uuid.UUID, label: str) -> "gsym": try: return _gensyms[uid] except KeyError: @@ -124,20 +124,20 @@ def __new__(cls, uid, label): instance = _gensyms[uid] return instance - def __init__(self, uid, label): + def __init__(self, uid: uuid.UUID, label: str) -> None: self.uid = uid self.label = label - def __getnewargs__(self): + def __getnewargs__(self) -> tuple: return (self.uid, self.label) - def __str__(self): + def __str__(self) -> str: return f"gensym#{self.label}:{self.uid}" - def __repr__(self): + def __repr__(self) -> str: return f'gsym("{self.label}", {repr(self.uid)})' -def gensym(label): +def gensym(label: str) -> gsym: """Create an uninterned symbol. The return value is the only time you'll see that symbol object; take good diff --git a/unpythonic/syntax/__init__.py b/unpythonic/syntax/__init__.py index 8a78ddda..38203ce2 100644 --- a/unpythonic/syntax/__init__.py +++ b/unpythonic/syntax/__init__.py @@ -58,13 +58,13 @@ # def mymacrointerface(tree, *, expander, *kw): # # perform your outside-in processing here # -# tree = expander.visit(tree) # recurse explicitly +# tree = expander.visit_recursively(tree) # recurse explicitly # # # perform your inside-out processing here # # return tree # -# If the line `tree = expander.visit(tree)` is omitted, the macro expands outside-in. +# If the line `tree = expander.visit_recursively(tree)` is omitted, the macro expands outside-in. # Note this default is different from MacroPy's! # TODO: 0.16: With `mcpyrate` we could start looking at values, not names, when the aim is to detect hygienically captured `unpythonic` constructs. See use sites of `isx`; refer to `mcpyrate.quotes.is_captured_value` and `mcpyrate.quotes.lookup_value`. @@ -81,7 +81,8 @@ # TODO: 0.16: AST pattern matching for `mcpyrate`? Would make destructuring easier. A writable representation (auto-viewify) is a pain to build, though... -# TODO: Far future: Change decorator macro invocations to use [] instead of () to pass macro arguments. Requires Python 3.9, so the earliest time to do this is when 3.9 becomes the minimum Python version for `unpythonic`. +# Parenthesis syntax for decorator macro arguments is deprecated; bracket syntax is preferred. +# Parenthesis syntax is kept for backward compatibility. from .autocurry import * # noqa: F401, F403 from .autoref import * # noqa: F401, F403 @@ -92,6 +93,8 @@ from .lazify import * # noqa: F401, F403 from .letdo import * # noqa: F401, F403 from .letsyntax import * # noqa: F401, F403 +from .monadic_do import * # noqa: F401, F403 +from .multishot import * # noqa: F401, F403 from .nb import * # noqa: F401, F403 from .prefix import * # noqa: F401, F403 from .tailtools import * # noqa: F401, F403 diff --git a/unpythonic/syntax/astcompat.py b/unpythonic/syntax/astcompat.py deleted file mode 100644 index 4dd444e7..00000000 --- a/unpythonic/syntax/astcompat.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -"""Conditionally import AST node types only supported by recent enough Python versions (3.7+).""" - -__all__ = ["NamedExpr", - "Num", "Str", "Bytes", "NameConstant", "Ellipsis", - "Index", "ExtSlice", - "getconstant"] - -import ast - -from ..symbol import gensym - -_NoSuchNodeType = gensym("_NoSuchNodeType") - -# -------------------------------------------------------------------------------- -# New AST node types - -# Minimum language version supported by this module is Python 3.6. - -# No new AST node types in Python 3.7. - -try: # Python 3.8+ - from ast import NamedExpr # a.k.a. walrus operator ":=" -except ImportError: # pragma: no cover - NamedExpr = _NoSuchNodeType - -# No new AST node types in Python 3.9. - -# TODO: any new AST node types in Python 3.10? (release expected in October 2021) - -# -------------------------------------------------------------------------------- -# Deprecated AST node types - -try: # Python 3.8+, https://docs.python.org/3/whatsnew/3.8.html#deprecated - from ast import Num, Str, Bytes, NameConstant, Ellipsis -except ImportError: # pragma: no cover - Num = Str = Bytes = NameConstant = Ellipsis = _NoSuchNodeType - -try: # Python 3.9+, https://docs.python.org/3/whatsnew/3.9.html#deprecated - from ast import Index, ExtSlice - # We ignore the internal classes Suite, Param, AugLoad, AugStore, - # which were never used in Python 3.x. -except ImportError: # pragma: no cover - Index = ExtSlice = _NoSuchNodeType - -# -------------------------------------------------------------------------------- -# Compatibility functions - -def getconstant(tree): - """Given an AST node `tree` representing a constant, return the contained raw value. - - This encapsulates the AST differences between Python 3.8+ and older versions. - - There are no `setconstant` or `makeconstant` counterparts, because you can - just create an `ast.Constant` in Python 3.6 and later. The parser doesn't - emit them until Python 3.8, but Python 3.6+ compile `ast.Constant` just fine. - """ - if type(tree) is ast.Constant: # Python 3.8+ - return tree.value - # up to Python 3.7 - elif type(tree) is ast.NameConstant: # up to Python 3.7 # pragma: no cover - return tree.value - elif type(tree) is ast.Num: # pragma: no cover - return tree.n - elif type(tree) in (ast.Str, ast.Bytes): # pragma: no cover - return tree.s - elif type(tree) is ast.Ellipsis: # `ast.Ellipsis` is the AST node type, `builtins.Ellipsis` is `...`. # pragma: no cover - return ... - raise TypeError(f"Not an AST node representing a constant: {type(tree)} with value {repr(tree)}") # pragma: no cover diff --git a/unpythonic/syntax/autocurry.py b/unpythonic/syntax/autocurry.py index 2dca5984..0124c3b5 100644 --- a/unpythonic/syntax/autocurry.py +++ b/unpythonic/syntax/autocurry.py @@ -7,11 +7,14 @@ from mcpyrate.quotes import macros, q, a, h # noqa: F401 +from mcpyrate.astcompat import TypeAlias from mcpyrate.quotes import is_captured_value from mcpyrate.walkers import ASTTransformer from .util import (suggest_decorator_index, isx, has_curry, sort_lambda_decorators) +from ..dynassign import dyn + # CAUTION: unpythonic.syntax.lambdatools.namedlambda depends on the exact names # "curryf" and "currycall" to detect an auto-curried expression with a final lambda. from ..fun import curry as curryf, _currycall as currycall @@ -68,9 +71,8 @@ def add3(a, b, c): if syntax == "block" and kw['optional_vars'] is not None: raise SyntaxError("autocurry does not take an as-part") # pragma: no cover - tree = expander.visit(tree) - - return _autocurry(block_body=tree) + with dyn.let(_macro_expander=expander): + return _autocurry(block_body=tree) _iscurry = lambda name: name in ("curry", "currycall") @@ -84,19 +86,35 @@ def transform(self, tree): if is_captured_value(tree): return tree + # Python 3.12+: leave `type` statements alone (autocurrying a type declaration makes no sense) + if type(tree) is TypeAlias: + return tree + hascurry = self.state.hascurry - # Curry all calls; except as a small optimization, skip `Values(...)`, - # which accepts any args and kwargs, so currying it does not make sense. - # (It represents multiple-return-values in `unpythonic`.) - if type(tree) is Call and not isx(tree.func, "Values"): - if has_curry(tree): # detect decorated lambda with manual curry - # the lambda inside the curry(...) is the next Lambda node we will descend into. - hascurry = True - if not isx(tree.func, _iscurry): - tree.args = [tree.func] + tree.args - tree.func = q[h[currycall]] - if hascurry: # this must be done after the edit because the edit changes the children - self.generic_withstate(tree, hascurry=True) + if type(tree) is Call: + # Don't auto-curry some calls we know not to need it. This is both a performance optimization + # and allows other macros (particularly `lazify`) to be able to see the original calls. + # (It also generates cleaner expanded output.) + # - `Values(...)` accepts any args and kwargs, so currying it does not make sense. + # - `(chain_conts(cc1, cc2))(...)` handles a return value in `with continuations`. + # This has the effect that in `with continuations`, the tail-calls to continuation + # functions won't be curried, but perhaps that's ok. This allows the Pytkell dialect's + # `with lazify, autocurry` combo to work with an inner `with continuations`. + if (isx(tree.func, "Values") or + (type(tree.func) is Call and isx(tree.func.func, "chain_conts"))): + # However, *do* auto-curry in the positional and named args of the call. + tree.args = self.visit(tree.args) + tree.keywords = self.visit(tree.keywords) + return tree + else: # general case + if has_curry(tree): # detect decorated lambda with manual curry + # the lambda inside the curry(...) is the next Lambda node we will descend into. + hascurry = True + if not isx(tree.func, _iscurry): + tree.args = [tree.func] + tree.args + tree.func = q[h[currycall]] + if hascurry: # this must be done after the edit because the edit changes the children + self.generic_withstate(tree, hascurry=True) elif type(tree) in (FunctionDef, AsyncFunctionDef): if not any(isx(item, _iscurry) for item in tree.decorator_list): # no manual curry already @@ -117,5 +135,6 @@ def transform(self, tree): return self.generic_visit(tree) + block_body = dyn._macro_expander.visit_recursively(block_body) newbody = AutoCurryTransformer(hascurry=False).visit(block_body) return sort_lambda_decorators(newbody) diff --git a/unpythonic/syntax/autoref.py b/unpythonic/syntax/autoref.py index 8fdbc407..0d4e9723 100644 --- a/unpythonic/syntax/autoref.py +++ b/unpythonic/syntax/autoref.py @@ -13,7 +13,6 @@ from mcpyrate.quotes import is_captured_value from mcpyrate.walkers import ASTTransformer -from .astcompat import getconstant from .nameutil import isx from .util import ExpandedAutorefMarker from .letdoutil import isdo, islet, ExpandedDoView, ExpandedLetView @@ -151,10 +150,20 @@ def autoref(tree, *, args, syntax, expander, **kw): @passthrough_lazy_args def _autoref_resolve(args): - *objs, s = [force1(x) for x in args] + """Perform an autoref lookup in a `with autoref` block. + + `args`: list [obj0, ..., objN, attrname] + + Each `obj` is tried, left to right, and the first one that + `hasattr(obj, attrname)` wins. The return value is the tuple + `(True, getattr(obj, attrname))`. + + If no obj matches, the return value is `(False, None)`. + """ + *objs, attrname = [force1(x) for x in args] for o in objs: - if hasattr(o, s): - return True, force1(getattr(o, s)) + if hasattr(o, attrname): + return True, force1(getattr(o, attrname)) return False, None def _autoref(block_body, args, asname): @@ -164,7 +173,7 @@ def _autoref(block_body, args, asname): if not block_body: raise SyntaxError("expected at least one statement inside the 'with autoref' block") # pragma: no cover - block_body = dyn._macro_expander.visit(block_body) + block_body = dyn._macro_expander.visit_recursively(block_body) # second pass, inside-out @@ -224,7 +233,7 @@ def transform(self, tree): elif isinstance(tree, ExpandedAutorefMarker): self.generic_withstate(tree, referents=referents + [tree.varname]) elif isautoreference(tree): # generated by an inner already expanded autoref block - thename = getconstant(get_resolver_list(tree)[-1]) + thename = get_resolver_list(tree)[-1].value if thename in referents: # This case is tricky to trigger, so let's document it here. This code: # diff --git a/unpythonic/syntax/dbg.py b/unpythonic/syntax/dbg.py index 13f0891f..68abff2c 100644 --- a/unpythonic/syntax/dbg.py +++ b/unpythonic/syntax/dbg.py @@ -103,12 +103,12 @@ def dbg(tree, *, args, syntax, expander, **kw): if syntax == "block" and kw['optional_vars'] is not None: raise SyntaxError("dbg (block mode) does not take an as-part") # pragma: no cover - tree = expander.visit(tree) - - if syntax == "expr": - return _dbg_expr(tree) - else: # syntax == "block": - return _dbg_block(body=tree, args=args) + # Expand inside-out. + with dyn.let(_macro_expander=expander): + if syntax == "expr": + return _dbg_expr(tree) + else: # syntax == "block": + return _dbg_block(body=tree, args=args) def dbgprint_block(ks, vs, *, filename=None, lineno=None, sep=", ", **kwargs): """Default debug printer for the ``dbg`` macro, block variant. @@ -213,6 +213,9 @@ def _dbg_block(body, args): pfunc = q[h[dbgprint_block]] pname = "print" # override standard print function within this block + # TODO: Do we really need to expand inside-out here? + body = dyn._macro_expander.visit_recursively(body) + class DbgBlockTransformer(ASTTransformer): def transform(self, tree): if is_captured_value(tree): @@ -223,7 +226,7 @@ def transform(self, tree): values = q[t[tree.args]] tree.args = [names, values] # can't use inspect.stack in the printer itself because we want the line number *before macro expansion*. - lineno = tree.lineno if hasattr(tree, "lineno") else None + lineno = getattr(tree, "lineno", None) # may be absent on 3.10–3.12; None on 3.13+ tree.keywords += [keyword(arg="filename", value=q[h[callsite_filename]()]), keyword(arg="lineno", value=q[u[lineno]])] tree.func = pfunc @@ -231,7 +234,10 @@ def transform(self, tree): return DbgBlockTransformer().visit(body) def _dbg_expr(tree): - ln = q[u[tree.lineno]] if hasattr(tree, "lineno") else q[None] + # TODO: Do we really need to expand inside-out here? + tree = dyn._macro_expander.visit_recursively(tree) + + ln = q[u[getattr(tree, "lineno", None)]] filename = q[h[callsite_filename]()] # Careful here! We must `h[]` the `dyn`, but not `dbgprint_expr` itself, # because we want to look up that attribute dynamically. diff --git a/unpythonic/syntax/forall.py b/unpythonic/syntax/forall.py index 95e4337e..641a8467 100644 --- a/unpythonic/syntax/forall.py +++ b/unpythonic/syntax/forall.py @@ -11,6 +11,7 @@ from .letdoutil import isenvassign, UnexpandedEnvAssignView from ..amb import monadify +from ..dynassign import dyn from ..misc import namelambda from ..amb import insist, deny # for re-export only # noqa: F401 @@ -24,24 +25,30 @@ def forall(tree, *, syntax, expander, **kw): Example:: # pythagorean triples - pt = forall[z << range(1, 21), # hypotenuse - x << range(1, z+1), # shorter leg - y << range(x, z+1), # longer leg + pt = forall[z := range(1, 21), # hypotenuse + x := range(1, z+1), # shorter leg + y := range(x, z+1), # longer leg insist(x*x + y*y == z*z), (x, y, z)] assert tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), (8, 15, 17), (9, 12, 15), (12, 16, 20)) + + The classic spelling ``name << iterable``, from v0.15.0 to v0.15.2, is still accepted. """ if syntax != "expr": raise SyntaxError("forall is an expr macro only") # pragma: no cover - tree = expander.visit(tree) - - return _forall(exprs=tree) + # Inside-out macro. + with dyn.let(_macro_expander=expander): + return _forall(exprs=tree) def _forall(exprs): if type(exprs) is not Tuple: # pragma: no cover, let's not test macro expansion errors. raise SyntaxError("forall body: expected a sequence of comma-separated expressions") # pragma: no cover + + # Expand inside-out to easily support lexical scoping. + exprs = dyn._macro_expander.visit_recursively(exprs) + itemno = 0 def build(lines, tree): if not lines: diff --git a/unpythonic/syntax/lambdatools.py b/unpythonic/syntax/lambdatools.py index 4c6e23a9..e3589772 100644 --- a/unpythonic/syntax/lambdatools.py +++ b/unpythonic/syntax/lambdatools.py @@ -3,7 +3,7 @@ __all__ = ["multilambda", "namedlambda", - "f", + "fn", "_", "quicklambda", "envify"] @@ -14,6 +14,7 @@ from mcpyrate.quotes import macros, q, u, n, a, h # noqa: F401 from mcpyrate import gensym +from mcpyrate.astcompat import NamedExpr from mcpyrate.expander import MacroExpander from mcpyrate.quotes import is_captured_value from mcpyrate.splicing import splice_expression @@ -21,10 +22,10 @@ from mcpyrate.walkers import ASTTransformer from ..dynassign import dyn -from ..misc import namelambda from ..env import env +from ..misc import namelambda +from ..symbol import sym -from .astcompat import getconstant, Str, NamedExpr from .letdo import _implicit_do, _do from .letdoutil import islet, isenvassign, UnexpandedLetView, UnexpandedEnvAssignView, ExpandedDoView from .nameutil import getname @@ -97,6 +98,10 @@ def namedlambda(tree, *, syntax, expander, **kw): let syntax supported by unpythonic (here using the haskelly let-in just as an example). + The last two are env-assignment, which accepts ``f := (lambda ...: ...)`` as well; both spellings are + named. That looks like the walrus above and is a different thing - what tells them apart is whether an + unpythonic environment is in scope, not the operator. + Support for other forms of assignment might or might not be added in a future version. @@ -124,64 +129,71 @@ def namedlambda(tree, *, syntax, expander, **kw): with dyn.let(_macro_expander=expander): return _namedlambda(block_body=tree) -def f(tree, *, syntax, expander, **kw): +def fn(tree, *, syntax, expander, **kw): """[syntax, expr] Underscore notation (quick lambdas) for Python. Usage:: - f[body] + fn[body] - The ``f[]`` macro creates a lambda. Each underscore in ``body`` + The ``fn[]`` macro creates a lambda. Each underscore in ``body`` introduces a new parameter. Example:: - func = f[_ * _] + func = fn[_ * _] expands to:: func = lambda a0, a1: a0 * a1 - The underscore is interpreted magically by ``f[]``; but ``_`` itself - is not a macro, and has no special meaning outside ``f[]``. The underscore - does **not** need to be imported for ``f[]`` to recognize it. + The underscore is interpreted magically by ``fn[]``; but ``_`` itself + is not a macro, and has no special meaning outside ``fn[]``. The underscore + does **not** need to be imported for ``fn[]`` to recognize it. - The macro does not descend into any nested ``f[]``. + But if you want to make your IDE happy, there is a symbol named ``_`` in + `unpythonic.syntax` you can import to silence any "undefined name" errors + regarding the use of ``_``. It is a regular run-time object, not a macro. + + The macro does not descend into any nested ``fn[]``. """ if syntax != "expr": raise SyntaxError("f is an expr macro only") # pragma: no cover # What's my name in the current expander? (There may be several names.) # https://github.com/Technologicat/mcpyrate/blob/master/doc/quasiquotes.md#hygienic-macro-recursion - bindings = extract_bindings(expander.bindings, f) + bindings = extract_bindings(expander.bindings, fn) mynames = list(bindings.keys()) - return _f(tree, mynames) + return _fn(tree, mynames) + +_ = sym("_") # for those who want to make their IDEs happy def quicklambda(tree, *, syntax, expander, **kw): - """[syntax, block] Make ``f`` quick lambdas expand first. + """[syntax, block] Make ``fn`` quick lambdas expand first. To be able to transform correctly, the block macros in ``unpythonic.syntax`` that transform lambdas (e.g. ``multilambda``, ``tco``) need to see all ``lambda`` definitions written with Python's standard ``lambda``. - However, the ``f`` macro uses the syntax ``f[...]``, which (to the analyzer) + However, the ``fn`` macro uses the syntax ``f[...]``, which (to the analyzer) does not look like a lambda definition. This macro changes the expansion - order, forcing any ``f[...]`` lexically inside the block to expand before + order, forcing any ``fn[...]`` lexically inside the block to expand before any other macros do. - Any expression of the form ``f[...]``, where ``f`` is any name bound in the - current macro expander to the macro `unpythonic.syntax.f`, is understood as - a quick lambda. (In plain English, this respects as-imports of the macro ``f``.) + Any expression of the form ``fn[...]``, where ``fn`` is any name bound in the + current macro expander to the macro `unpythonic.syntax.fn`, is understood as + a quick lambda. (In plain English, this respects as-imports of the macro ``fn``.) Example - a quick multilambda:: - from unpythonic.syntax import macros, multilambda, quicklambda, f, local + from unpythonic.syntax import macros, multilambda, quicklambda, fn, local + from unpythonic.syntax import _ # optional, makes IDEs happy with quicklambda, multilambda: - func = f[[local[x << _], - local[y << _], - x + y]] + func = fn[[local[x := _], + local[y := _], + x + y]] assert func(1, 2) == 3 (This is of course rather silly, as an unnamed argument can only be mentioned @@ -200,7 +212,7 @@ def quicklambda(tree, *, syntax, expander, **kw): # the original expander. Thus it leaves all other macros alone. This is the # official `mcpyrate` way to immediately expand only some particular macros # inside the current macro invocation. - bindings = extract_bindings(expander.bindings, f) + bindings = extract_bindings(expander.bindings, fn) return MacroExpander(bindings, expander.filename).visit(tree) def envify(tree, *, syntax, expander, **kw): @@ -259,7 +271,7 @@ def iscurrywithfinallambda(tree): return type(tree.args[-1]) is Lambda # Detect an autocurry from an already expanded "with autocurry". - # CAUTION: These must match what unpythonic.syntax.curry.autocurry uses in its output. + # CAUTION: These must match what unpythonic.syntax.autocurry.autocurry uses in its output. currycall_name = "currycall" iscurryf = lambda name: name in ("curryf", "curry") # auto or manual curry in a "with autocurry" def isautocurrywithfinallambda(tree): @@ -333,7 +345,7 @@ def transform(self, tree): else: tree.value = self.visit(tree.value) return tree - elif type(tree) is NamedExpr: # f := lambda ...: ... (Python 3.8+, added in unpythonic 0.15) + elif type(tree) is NamedExpr: # f := lambda ...: ... (Python 3.8+, added in unpythonic 0.15.0) tree.value, thelambda, match = nameit(getname(tree.target), tree.value) if match: thelambda.body = self.visit(thelambda.body) @@ -360,8 +372,8 @@ def transform(self, tree): if k is None: # {..., **d, ...} tree.values[j] = self.visit(v) else: - if type(k) in (Constant, Str): # Python 3.8+: ast.Constant - thename = getconstant(k) + if type(k) is Constant: + thename = k.value tree.values[j], thelambda, match = nameit(thename, v) if match: thelambda.body = self.visit(thelambda.body) @@ -376,7 +388,7 @@ def transform(self, tree): # outside in: transform in unexpanded let[] forms newbody = NamedLambdaTransformer().visit(block_body) - newbody = dyn._macro_expander.visit(newbody) + newbody = dyn._macro_expander.visit_recursively(newbody) # inside out: transform in expanded autocurry newbody = NamedLambdaTransformer().visit(newbody) @@ -411,7 +423,7 @@ def transform(self, tree): # # Used under the MIT license. # Copyright (c) 2013-2018, Li Haoyi, Justin Holmgren, Alberto Berti and all the other contributors. -def _f(tree, mynames=()): +def _fn(tree, mynames=()): class UnderscoreTransformer(ASTTransformer): def transform(self, tree): if is_captured_value(tree): @@ -437,15 +449,13 @@ def _envify(block_body): # first pass, outside-in userlambdas = detect_lambda(block_body) - block_body = dyn._macro_expander.visit(block_body) + # Expand inside-out to easily support lexical scoping. + block_body = dyn._macro_expander.visit_recursively(block_body) # second pass, inside-out def getargs(tree): # tree: FunctionDef, AsyncFunctionDef, Lambda a = tree.args - if hasattr(a, "posonlyargs"): # Python 3.8+: positional-only parameters - allargs = a.posonlyargs + a.args + a.kwonlyargs - else: - allargs = a.args + a.kwonlyargs + allargs = a.posonlyargs + a.args + a.kwonlyargs argnames = [x.arg for x in allargs] if a.vararg: argnames.append(a.vararg.arg) @@ -520,10 +530,13 @@ def isourupdate(thecall): newvalue = self.visit(view.value) return q[a[envset](u[view.name], a[newvalue])] # transform references to currently active bindings + # x --> e14.x + # It doesn't matter if this hits an already expanded inner `with envify`, + # because the gensymmed environment name won't be in our bindings, and the "x" + # has become the `attr` in an `Attribute` node. elif type(tree) is Name and tree.id in bindings.keys(): - # We must be careful to preserve the Load/Store/Del context of the name. - # The default lets `mcpyrate` fix it later. - ctx = tree.ctx if hasattr(tree, "ctx") else None + # Preserve the Load/Store/Del context of the name. + ctx = getattr(tree, "ctx", None) out = deepcopy(bindings[tree.id]) out.ctx = ctx return out diff --git a/unpythonic/syntax/lazify.py b/unpythonic/syntax/lazify.py index 78c533c2..1b60fd7d 100644 --- a/unpythonic/syntax/lazify.py +++ b/unpythonic/syntax/lazify.py @@ -9,6 +9,7 @@ from mcpyrate.quotes import macros, q, u, a, h # noqa: F401 +from mcpyrate.astcompat import TypeAlias from mcpyrate.astfixers import fix_ctx from mcpyrate.quotes import capture_as_macro, is_captured_value from mcpyrate.unparser import unparse @@ -310,7 +311,7 @@ def f(lst): with lazify: lst = [] for x in range(3): - lst.append(let[[y << x] in lazy[y]]) + lst.append(let[[y := x] in lazy[y]]) print(lst[0]) # 0 print(lst[1]) # 1 print(lst[2]) # 2 @@ -360,15 +361,15 @@ def add2first(a, b, c): def f(a, b): return a - assert let[[c << 42, - d << 1/0] in f(c)(d)] == 42 - assert letrec[[c << 42, - d << 1/0, - e << 2*c] in f(e)(d)] == 84 - - assert letrec[[c << 42, - d << 1/0, - e << 2*c] in [local[x << f(e)(d)], + assert let[[c := 42, + d := 1/0] in f(c)(d)] == 42 + assert letrec[[c := 42, + d := 1/0, + e := 2*c] in f(e)(d)] == 84 + + assert letrec[[c := 42, + d := 1/0, + e := 2*c] in [local[x := f(e)(d)], x/4]] == 21 Works also with continuations. Rules: @@ -565,7 +566,7 @@ def _is_literal_container(tree, maps_only=False): if type(tree) is Dict: return True # Not reached in case of `lazyrec`, similarly as above. - if type(tree) is Call and any(isx(tree.func, s) for s in _ctorcalls_map): + if type(tree) is Call and any(isx(tree.func, s) for s in _ctorcalls_map): # noqa: SIM103 -- last of several parallel guard clauses; collapsing only this one hides that shape return True return False @@ -576,10 +577,10 @@ def _is_literal_container(tree, maps_only=False): # it is too easy to accidentally set up an infinite recursion. # # This is ok: -# force1(lst)[0] = (10 * (force1(lst()[0]) if isinstance(lst, Lazy1) else force1(lst[0]))) +# force1(lst)[0] = (10 * (force1(lst()[0]) if isinstance(lst, Lazy) else force1(lst[0]))) # # but this blows up (by infinite recursion) later when we eventually force lst[0]: -# force1(lst)[0] = Lazy1(lambda: (10 * (force1(lst()[0]) if isinstance(lst, Lazy1) else force1(lst[0])))) +# force1(lst)[0] = Lazy(lambda: (10 * (force1(lst()[0]) if isinstance(lst, Lazy) else force1(lst[0])))) # # We **could** solve this by forcing and capturing the current value before assigning, # instead of allowing the RHS to refer to a lazy list element. But on the other hand, @@ -599,7 +600,10 @@ def _lazify(body): # Expand any inner macro invocations. Particularly, this expands away any `lazyrec[]` and `lazy[]` # so they become easier to work with. We also know that after this, any `Subscript` is really a # subscripting operation and not a macro invocation. - body = dyn._macro_expander.visit(body) + # + # We must explicitly use recursive mode to ensure we get rid of all macro invocations, because + # we may be running inside a `with step_expansion`, which uses the expand-once-only mode. + body = dyn._macro_expander.visit_recursively(body) # `lazify`'s analyzer needs the `ctx` attributes in `tree` to be filled in correctly. body = fix_ctx(body, copy_seen_nodes=False) # TODO: or maybe copy seen nodes? @@ -645,6 +649,10 @@ def f(tree): # else forcing_mode == "off" return tree + # Python 3.12+: leave `type` statements alone (lazifying a type declaration makes no sense) + elif type(tree) is TypeAlias: + return tree + elif type(tree) in (FunctionDef, AsyncFunctionDef, Lambda): if type(tree) is Lambda and id(tree) not in userlambdas: return self.generic_visit(tree) # ignore macro-introduced lambdas (but recurse inside them) @@ -714,30 +722,49 @@ def transform_starred(tree, dstarred=False): thelambda.body = self.visit(thelambda.body) return tree - # namelambda() is used by let[] and do[] - # Lazy() is a strict function, takes a lambda, constructs a Lazy object - # _autoref_resolve doesn't need any special handling - # Values() doesn't need any special handling + # Don't lazify in calls to some specific functions we know to be strict. + # Some of these are performance optimizations; others must be left as-is + # for other macros to be able to see the original calls. (It also generates + # cleaner expanded output.) + # - `namelambda` (emitted by `let[]`, `do[]`, and `test[]`) + # - All known container constructor calls (listed in `_ctorcalls_all`). + # - `Lazy` takes a lambda, constructs a `Lazy` object; if we're calling `Lazy`, + # the expression is already lazy. + # - `_autoref_resolve` does the name lookup in `with autoref` blocks. + # + # Don't lazify in calls to return-value utilities, because return values + # are never implicitly lazy in `unpythonic`. + # - `Values` constructs a multiple-return-values and/or named return values. + # - `(chain_conts(cc1, cc2))(args)` handles a return value in `with continuations`. elif (isdo(tree) or is_decorator(tree.func, "namelambda") or any(isx(tree.func, s) for s in _ctorcalls_all) or isx(tree.func, _expanded_lazy_name) or isx(tree.func, "_autoref_resolve") or - isx(tree.func, "Values")): - # here we know the operator (.func) to be one of specific names; - # don't transform it to avoid confusing lazyrec[] (important if this - # is an inner call in the arglist of an outer, lazy call, since it - # must see any container constructor calls that appear in the args) + isx(tree.func, "Values") or + (type(tree.func) is Call and isx(tree.func.func, "chain_conts"))): + # Here we know the operator (.func) to be one of specific names; + # don't transform it to avoid confusing `lazyrec[]`. + # + # This is especially important, if this is an inner call in the + # arglist of an outer, lazy call, since it must see any container + # constructor calls that appear in the args. + # + # But *do* transform in the positional and named args of the call; + # doing so generates the code to force any promises that are passed + # to the function being called. # # TODO: correct forcing mode for recursion? We shouldn't need to forcibly use "full", # since maybe_force_args() already fully forces any remaining promises # in the args when calling a strict function. + # NOTE v0.15.0: In practice, using whatever is the currently active mode seems to be fine. tree.args = self.visit(tree.args) tree.keywords = self.visit(tree.keywords) return tree - else: + else: # general case thefunc = self.visit(tree.func) + # Lazify the arguments of the call. adata = [] for x in tree.args: if type(x) is Starred: # *args in Python 3.5+ diff --git a/unpythonic/syntax/letdo.py b/unpythonic/syntax/letdo.py index f435cb41..42ea4dc5 100644 --- a/unpythonic/syntax/letdo.py +++ b/unpythonic/syntax/letdo.py @@ -26,8 +26,7 @@ FunctionDef, Return, AsyncFunctionDef, arguments, arg, - Load) -import sys + Store, Del) from mcpyrate.quotes import macros, q, u, n, a, t, h # noqa: F401 @@ -92,7 +91,7 @@ def where(tree, *, syntax, **kw): Usage:: - let[body, where[k0 << v0, ...]] + let[body, where[k0 := v0, ...]] Only meaningful for declaring the bindings in a let-where, for all expression-form let constructs: `let`, `letseq`, `letrec`, `let_syntax`, @@ -100,7 +99,7 @@ def where(tree, *, syntax, **kw): """ if syntax != "name": raise SyntaxError("where (unpythonic.syntax.letdo.where) is a name macro only") # pragma: no cover - raise SyntaxError("where (unpythonic.syntax.letdo.where) is only meaningful in a let[body, where[k0 << v0, ...]]") # pragma: no cover + raise SyntaxError("where (unpythonic.syntax.letdo.where) is only meaningful in a let[body, where[k0 := v0, ...]]") # pragma: no cover @parametricmacro def let(tree, *, args, syntax, expander, **kw): @@ -110,18 +109,18 @@ def let(tree, *, args, syntax, expander, **kw): Usage:: - let[k0 << v0, ...][body] - let[k0 << v0, ...][[body0, ...]] + let[k0 := v0, ...][body] + let[k0 := v0, ...][[body0, ...]] where ``body`` is an expression. The names bound by ``let`` are local; they are available in ``body``, and do not exist outside ``body``. Alternative haskelly syntax is also available:: - let[[k0 << v0, ...] in body] - let[[k0 << v0, ...] in [body0, ...]] - let[body, where[k0 << v0, ...]] - let[[body0, ...], where[k0 << v0, ...]] + let[[k0 := v0, ...] in body] + let[[k0 := v0, ...] in [body0, ...]] + let[body, where[k0 := v0, ...]] + let[[body0, ...], where[k0 := v0, ...]] For a body with multiple expressions, use an extra set of brackets, as shown above. This inserts a ``do``. Only the outermost extra brackets @@ -133,9 +132,14 @@ def let(tree, *, args, syntax, expander, **kw): Each ``name`` in the same ``let`` must be unique. - Rebinding of let-bound variables inside `body` is supported with `unpythonic` - env-assignment syntax, ``x << 42``. This is an expression, performing the - assignment, and returning the new value. + Starting at v0.15.3, rebinding of let-bound variables inside `body` + is supported using the walrus assignment syntax, ``x := 42``. + The new syntax is preferred, but the old one is still available + for backward compatibility. + + From v0.15.0 to v0.15.2, rebinding of let-bound variables inside `body` + is supported with `unpythonic` env-assignment syntax, ``x << 42``. + This is an expression, performing the assignment, and returning the new value. In a multiple-expression body, also an internal definition context exists for local variables that are not part of the ``let``; see ``do`` for details. @@ -210,9 +214,9 @@ def dlet(tree, *, args, syntax, expander, **kw): Example:: - @dlet[x << 0] + @dlet[x := 0] def count(): - x << x + 1 + (x := x + 1) return x assert count() == 1 assert count() == 2 @@ -222,7 +226,7 @@ def count(): ``let`` environment *for the entirety of that lexical scope*. (This is modeled after Python's standard scoping rules.) - **CAUTION**: assignment to the let environment is ``name << value``; + **CAUTION**: assignment to the let environment is ``name := value``; the regular syntax ``name = value`` creates a local variable in the lexical scope of the ``def``. """ @@ -240,9 +244,9 @@ def dletseq(tree, *, args, syntax, expander, **kw): Example:: - @dletseq[x << 1, - x << x + 1, - x << x + 2] + @dletseq[x := 1, + x := x + 1, + x := x + 2] def g(a): return a + x assert g(10) == 14 @@ -259,8 +263,8 @@ def dletrec(tree, *, args, syntax, expander, **kw): Example:: - @dletrec[evenp << (lambda x: (x == 0) or oddp(x - 1)), - oddp << (lambda x: (x != 0) and evenp(x - 1))] + @dletrec[evenp := (lambda x: (x == 0) or oddp(x - 1)), + oddp := (lambda x: (x != 0) and evenp(x - 1))] def f(x): return evenp(x) assert f(42) is True @@ -280,7 +284,7 @@ def blet(tree, *, args, syntax, expander, **kw): Example:: - @blet[x << 21] + @blet[x := 21] def result(): return 2 * x assert result == 42 @@ -297,9 +301,9 @@ def bletseq(tree, *, args, syntax, expander, **kw): Example:: - @bletseq[x << 1, - x << x + 1, - x << x + 2] + @bletseq[x := 1, + x := x + 1, + x := x + 2] def result(): return x assert result == 4 @@ -316,8 +320,8 @@ def bletrec(tree, *, args, syntax, expander, **kw): Example:: - @bletrec[evenp << (lambda x: (x == 0) or oddp(x - 1)), - oddp << (lambda x: (x != 0) and evenp(x - 1))] + @bletrec[evenp := (lambda x: (x == 0) or oddp(x - 1)), + oddp := (lambda x: (x != 0) and evenp(x - 1))] def result(): return evenp(42) assert result is True @@ -376,14 +380,14 @@ def _let_expr_impl(bindings, body, mode): # (It is important we expand at least that immediately after, to resolve its local variables, # because those may have the same lexical names as some of the let-bindings.) body = _implicit_do(body) - body = dyn._macro_expander.visit(body) + body = dyn._macro_expander.visit_recursively(body) if not bindings: # Optimize out a `let` with no bindings. The macro layer cannot trigger # this case, because our syntaxes always require at least one binding. # So this check is here just to protect against use with no bindings directly # from other syntax transformers, which in theory could attempt anything. return body # pragma: no cover - bindings = dyn._macro_expander.visit(bindings) + bindings = dyn._macro_expander.visit_recursively(bindings) names, values = zip(*[b.elts for b in bindings]) # --> (k1, ..., kn), (v1, ..., vn) names = [getname(k, accept_attr=False) for k in names] # any duplicates will be caught by env at run-time @@ -414,14 +418,15 @@ def _letlike_transform(tree, envname, lhsnames, rhsnames, setter, dowrap=True): """Common transformations for let-like operations. Namely:: + x := val --> e.set('x', val) x << val --> e.set('x', val) x --> e.x (when x appears in load context) # ... -> lambda e: ... (applied if dowrap=True) - lhsnames: names to recognize on the LHS of x << val as belonging to this env + lhsnames: names to recognize on the LHS of env-assignment (`x := val` or `x << val`) as belonging to this env rhsnames: names to recognize anywhere in load context as belonging to this env - These are separate mainly for ``do[]``, so that we can have new bindings + The LHS/RHS names are separate mainly for ``do[]``, so that we can have new bindings take effect only in following exprs. setter: function, (k, v) --> v, side effect to set e.k to v @@ -433,7 +438,7 @@ def _letlike_transform(tree, envname, lhsnames, rhsnames, setter, dowrap=True): return tree def _transform_envassignment(tree, lhsnames, envset): - """x << val --> e.set('x', val) (for names bound in this environment)""" + """`x := val` or `x << val` --> `e.set('x', val)` (for names bound in this environment)""" # names_in_scope: according to Python's standard binding rules, see scopeanalyzer.py. # Variables defined in let envs are thus not listed in `names_in_scope`. def transform(tree, names_in_scope): @@ -446,12 +451,14 @@ def transform(tree, names_in_scope): return scoped_transform(tree, callback=transform) def _transform_name(tree, rhsnames, envname): - """x --> e.x (in load context; for names bound in this environment)""" + """`x` --> `e.x` (in load context; for names bound in this environment)""" # names_in_scope: according to Python's standard binding rules, see scopeanalyzer.py. # Variables defined in let envs are thus not listed in `names_in_scope`. def transform(tree, names_in_scope): # This transformation is deceptively simple, hence requires some comment: # + # - The goal is to transform read accesses to let variables, `x` --> `e.x`. + # # - Attributes (and Subscripts) work, because we are called again for # the `value` part of the `Attribute` (or `Subscript`) node, which # then gets transformed if it's a `Name` matching our rules. @@ -467,12 +474,10 @@ def transform(tree, names_in_scope): # in those parts of code where it is used, so an outer let will # leave it alone. if type(tree) is Name and tree.id in rhsnames and tree.id not in names_in_scope: - hasctx = hasattr(tree, "ctx") # macro-created nodes might not have a ctx. - if hasctx and type(tree.ctx) is not Load: # let variables are rebound using `<<`, not `=`. + if type(getattr(tree, "ctx", None)) in (Store, Del): # Skip assignments and deletes. return tree attr_node = q[n[f"{envname}.{tree.id}"]] - if hasctx: - attr_node.ctx = tree.ctx + attr_node.ctx = getattr(tree, "ctx", None) return attr_node return tree return scoped_transform(tree, callback=transform) @@ -510,13 +515,13 @@ def _let_decorator_impl(bindings, body, mode, kind): assert kind in ("decorate", "call") if type(body) not in (FunctionDef, AsyncFunctionDef): raise SyntaxError("Expected a function definition to decorate") # pragma: no cover - body = dyn._macro_expander.visit(body) + body = dyn._macro_expander.visit_recursively(body) if not bindings: # Similarly as above, this cannot trigger from the macro layer no # matter what that layer does. This is here to optimize away a `dlet` # with no bindings, when used directly from other syntax transformers. return body # pragma: no cover - bindings = dyn._macro_expander.visit(bindings) + bindings = dyn._macro_expander.visit_recursively(bindings) names, values = zip(*[b.elts for b in bindings]) # --> (k1, ..., kn), (v1, ..., vn) names = [getname(k, accept_attr=False) for k in names] # any duplicates will be caught by env at run-time @@ -551,20 +556,20 @@ def _let_decorator_impl(bindings, body, mode, kind): def _dletseq_impl(bindings, body, kind): # What we want: # - # @dletseq[x << 1, - # x << x + 1, - # x << x + 2] + # @dletseq[x := 1, + # x := x + 1, + # x := x + 2] # def g(*args, **kwargs): # return x # assert g() == 4 # # --> # - # @dlet[x << 1] + # @dlet[x := 1] # def g(*args, **kwargs, e1): # original args from tree go to the outermost def - # @dlet[x << x + 1] # on RHS, important for e1.x to be in scope + # @dlet[x := x + 1] # on RHS, important for e1.x to be in scope # def g2(*, e2): - # @dlet[x << x + 2] + # @dlet[x := x + 2] # def g3(*, e3): # expansion proceeds from inside out # return e3.x # original args travel here by the closure property # return g3() @@ -582,10 +587,8 @@ def _dletseq_impl(bindings, body, kind): userargs = body.args # original arguments to the def fname = body.name - noargs = arguments(args=[], kwonlyargs=[], vararg=None, kwarg=None, + noargs = arguments(args=[], posonlyargs=[], kwonlyargs=[], vararg=None, kwarg=None, defaults=[], kw_defaults=[]) - if sys.version_info >= (3, 8, 0): # Python 3.8+: positional-only arguments - noargs.posonlyargs = [] iname = gensym(f"{fname}_inner") body.args = noargs body.name = iname @@ -625,7 +628,7 @@ def local(tree, *, syntax, **kw): Usage:: - local[name << value] + local[name := value] Only meaningful in a ``do[...]``, ``do0[...]``, or an implicit ``do`` (extra bracket syntax). @@ -637,7 +640,7 @@ def local(tree, *, syntax, **kw): on the RHS. This means that if you want, you can declare a local ``x`` that takes its - initial value from a nonlocal ``x``, by ``local[x << x]``. Here the ``x`` + initial value from a nonlocal ``x``, by ``local[x := x]``. Here the ``x`` on the RHS is the nonlocal one (since the declaration has not yet taken effect), and the ``x`` on the LHS is the name given to the new local variable that only exists inside the ``do``. Any references to ``x`` in any further @@ -680,14 +683,14 @@ def do(tree, *, syntax, expander, **kw): Example:: - do[local[x << 42], + do[local[x := 42], print(x), - x << 23, + x := 23, x] This is sugar on top of ``unpythonic.seq.do``, but with some extra features. - - To declare and initialize a local name, use ``local[name << value]``. + - To declare and initialize a local name, use ``local[name := value]``. The operator ``local`` is syntax, not really a function, and it only exists inside a ``do``. There is also an operator ``delete`` @@ -702,7 +705,7 @@ def do(tree, *, syntax, expander, **kw): - Names declared within the same ``do`` must be unique. Re-declaring the same name is an expansion-time error. - - To assign to an already declared local name, use ``name << value``. + - To assign to an already declared local name, use ``name := value``. **local name declarations** @@ -711,7 +714,7 @@ def do(tree, *, syntax, expander, **kw): result = [] let((lst, []))[do[result.append(lst), # the let "lst" - local[lst << lst + [1]], # LHS: do "lst", RHS: let "lst" + local[lst := lst + [1]], # LHS: do "lst", RHS: let "lst" result.append(lst)]] # the do "lst" assert result == [[], [1]] @@ -753,14 +756,14 @@ def do(tree, *, syntax, expander, **kw): uses, the ambiguity does not arise. The transformation inserts not only the word ``do``, but also the outermost brackets. For example:: - let[x << 1, - y << 2][[ + let[x := 1, + y := 2][[ [x, y]]] transforms to:: - let[x << 1, - y << 2][do[[ # "do[" is inserted between the two opening brackets + let[x := 1, + y := 2][do[[ # "do[" is inserted between the two opening brackets [x, y]]]] # and its closing "]" is inserted here which already gets rid of the ambiguity. @@ -770,24 +773,24 @@ def do(tree, *, syntax, expander, **kw): Macros are expanded in an inside-out order, so a nested ``let`` shadows names, if the same names appear in the ``do``:: - do[local[x << 17], - let[x << 23][ + do[local[x := 17], + let[x := 23][ print(x)], # 23, the "x" of the "let" print(x)] # 17, the "x" of the "do" The reason we require local names to be declared is to allow write access to lexically outer environments from inside a ``do``:: - let[x << 17][ - do[x << 23, # no "local[...]"; update the "x" of the "let" - local[y << 42], # "y" is local to the "do" + let[x := 17][ + do[x := 23, # no "local[...]"; update the "x" of the "let" + local[y := 42], # "y" is local to the "do" print(x, y)]] With the extra bracket syntax, the latter example can be written as:: - let[x << 17][[ - x << 23, - local[y << 42], + let[x := 17][[ + x := 23, + local[y := 42], print(x, y)]] It's subtly different in that the first version has the do-items in a tuple, @@ -833,11 +836,11 @@ def transform(self, tree): expr = islocaldef(tree) if expr: if not isenvassign(expr): - raise SyntaxError("local[...] takes exactly one expression of the form 'name << value'") # pragma: no cover + raise SyntaxError("local[...] takes exactly one expression of the form 'name := value' or 'name << value'") # pragma: no cover view = UnexpandedEnvAssignView(expr) self.collect(view.name) - view.value = self.visit(view.value) # nested local[] (e.g. from `do0[local[y << 5],]`) - return expr # `local[x << 21]` --> `x << 21`; compiling *that* makes the env-assignment occur. + view.value = self.visit(view.value) # nested local[] (e.g. from `do0[local[y := 5],]`) + return expr # `local[x := 21]` --> `x := 21`; compiling *that* makes the env-assignment occur. return tree # don't recurse! c = LocaldefCollector() tree = c.visit(tree) @@ -918,7 +921,8 @@ def _do0(tree): raise SyntaxError("do0 body: expected a sequence of comma-separated expressions") # pragma: no cover elts = tree.elts # Use `local[]` and `do[]` as hygienically captured macros. - newelts = [q[a[_our_local][_do0_result << a[elts[0]]]], # noqa: F821, local[] defines it inside the do[]. + # + newelts = [q[a[_our_local][_do0_result := a[elts[0]]]], # noqa: F821, local[] defines it inside the do[]. *elts[1:], q[_do0_result]] # noqa: F821 return q[a[_our_do][t[newelts]]] # do0[] is also just a do[] diff --git a/unpythonic/syntax/letdoutil.py b/unpythonic/syntax/letdoutil.py index 892d8ddc..aceac13e 100644 --- a/unpythonic/syntax/letdoutil.py +++ b/unpythonic/syntax/letdoutil.py @@ -7,29 +7,25 @@ "ExpandedLetView", "ExpandedDoView"] from ast import (Call, Name, Subscript, Compare, In, - Tuple, List, Constant, BinOp, LShift, Lambda) -import sys + Tuple, List, Constant, BinOp, LShift, Lambda, + copy_location) from mcpyrate import unparse +from mcpyrate.astcompat import NamedExpr from mcpyrate.core import Done -from .astcompat import getconstant, Str from .nameutil import isx, getname letf_name = "letter" # must match what ``unpythonic.syntax.letdo._let_expr_impl`` uses in its output. dof_name = "dof" # name must match what ``unpythonic.syntax.letdo.do`` uses in its output. -currycall_name = "currycall" # output of ``unpythonic.syntax.curry`` +currycall_name = "currycall" # output of ``unpythonic.syntax.autocurry`` def _get_subscript_slice(tree): assert type(tree) is Subscript - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - return tree.slice - return tree.slice.value + return tree.slice def _set_subscript_slice(tree, newslice): # newslice: AST assert type(tree) is Subscript - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - tree.slice = newslice - tree.slice.value = newslice + tree.slice = newslice def _canonize_macroargs_node(macroargs): # We do this like `mcpyrate.expander.destructure_candidate` does, # except that we also destructure a list. @@ -37,21 +33,46 @@ def _canonize_macroargs_node(macroargs): return macroargs.elts return [macroargs] # anything that doesn't have at least one comma at the top level -def canonize_bindings(elts, letsyntax_mode=False): # public as of v0.14.3+ - """Wrap a single binding without container into a length-1 `list`. +# For analysis of let-bindings and env-assignments. +def _isname(tree): + """Return whether `tree` is a lexical name. + + The actual `ast.Name` may be wrapped in a `mcpyrate.core.Done`, which is produced + by expanded `@namemacro`s; we accept a `Done` containing an `ast.Name`, too. + + We don't accept hygienic captures, since those correspond to values, not names. + """ + return type(tree) is Name or (isinstance(tree, Done) and _isname(tree.body)) +def _isbindingtarget(tree, letsyntax_mode): + """Return whether `tree` is a valid target for a let-binding or env-assignment. - Pass through multiple bindings as-is. + letsyntax_mode: used by let_syntax to allow template definitions. + This allows, beside a bare name `k`, the formats `k(a0, ...)` and `k[a0, ...]` + to appear in the variable-name position. + """ + return (_isname(tree) or + (letsyntax_mode and ((type(tree) is Call and _isname(tree.func)) or + (type(tree) is Subscript and _isname(tree.value))))) + +def canonize_bindings(elts, letsyntax_mode=False): # public as of v0.14.3+ + """Convert any `let` bindings format supported by `unpythonic` into a canonical format. Yell if the input format is invalid. + The canonical format is a `list` of `ast.Tuple`:: + + [Tuple(elts=[k0, v0]), ...] + elts: `list` of bindings, one of:: + [k0 := v0, ...] # v0.15.3+: new env-assignment syntax, preferred + [k := v] # v0.15.3+ + [k0 << v0, ...] # v0.15.0+: previous env-assignment syntax + [k << v] # v0.15.0+ + [[k0, v0], ...] # v0.15.0+: accept also brackets (for consistency) + [[k, v]] # v0.15.0+ [(k0, v0), ...] # multiple bindings contained in a tuple [(k, v),] # single binding contained in a tuple also ok [k, v] # special single binding format, missing tuple container - [[k0, v0], ...] # v0.15.0+: accept also brackets (for consistency) - [[k, v]] # v0.15.0+ - [k0 << v0, ...] # v0.15.0+: accept also env-assignment syntax - [k << v] # v0.15.0+ where the ks and vs are AST nodes. @@ -59,43 +80,55 @@ def canonize_bindings(elts, letsyntax_mode=False): # public as of v0.14.3+ This allows, beside a bare name `k`, the formats `k(a0, ...)` and `k[a0, ...]` to appear in the variable-name position. """ - def isname(tree): - # Note we don't accept hygienic captures. - # The `Done` may be produced by expanded `@namemacro`s. - return type(tree) is Name or (isinstance(tree, Done) and isname(tree.body)) - def isbindingtarget(tree): - return (isname(tree) or - (letsyntax_mode and ((type(tree) is Call and isname(tree.func)) or - (type(tree) is Subscript and isname(tree.value))))) def iskvpairbinding(lst): - return len(lst) == 2 and isbindingtarget(lst[0]) - def isenvassignbinding(tree): - if not (type(tree) is BinOp and type(tree.op) is LShift): - return False - return isbindingtarget(tree.left) + return len(lst) == 2 and _isbindingtarget(lst[0], letsyntax_mode) - if len(elts) == 1 and isenvassignbinding(elts[0]): # [k << v] - return [Tuple(elts=[elts[0].left, elts[0].right])] + if len(elts) == 1: + if isenvassign(elts[0], letsyntax_mode) is LShift: # [k << v] + return [Tuple(elts=[elts[0].left, elts[0].right])] + if isenvassign(elts[0], letsyntax_mode) is NamedExpr: # [k := v] + return [Tuple(elts=[elts[0].target, elts[0].value])] if len(elts) == 2 and iskvpairbinding(elts): # [k, v] return [Tuple(elts=elts)] # TODO: `mcpyrate`: just `q[t[elts]]`? if all((type(b) is Tuple and iskvpairbinding(b.elts)) for b in elts): # [(k0, v0), ...] return elts if all((type(b) is List and iskvpairbinding(b.elts)) for b in elts): # [[k0, v0], ...] return [Tuple(elts=b.elts) for b in elts] - if all((isenvassign(b) and isbindingtarget(b.left)) for b in elts): # [k0 << v0, ...] - return [Tuple(elts=[b.left, b.right]) for b in elts] - raise SyntaxError("expected bindings to be `(k0, v0), ...`, `[k0, v0], ...`, or `k0 << v0, ...`, or a single `k, v`, or `k << v`") # pragma: no cover + if all(isenvassign(b, letsyntax_mode) for b in elts): # [k0 << v0, ...] or [k0 := v0, ...] + out = [] + for b in elts: + if isenvassign(b, letsyntax_mode) is LShift: + out.append(Tuple(elts=[b.left, b.right])) + else: # NamedExpr + out.append(Tuple(elts=[b.target, b.value])) + return out + raise SyntaxError("expected bindings to be `k0 := v0, ...`, `k0 << v0, ...`, `[k0, v0], ...`, or `(k0, v0), ...`, or a single `k := v`, `k << v`, or `k, v`") # pragma: no cover + +def isenvassign(tree, letsyntax_mode=False): + """Detect whether tree is an unpythonic ``env`` assignment. + + Starting at v0.15.3: new env-assignment syntax ``name := value`` is recommended. -def isenvassign(tree): - """Detect whether tree is an unpythonic ``env`` assignment, ``name << value``. + From v0.15.0 to v0.15.2, env-assignment used the syntax ``name << value``. + This is still available for backward compatibility. - The only way this differs from a general left-shift is that the LHS must be - an ``ast.Name``. + Return value is one of the constants: + `NamedExpr`: `tree` is an env-assignment, with modern syntax. + `LShift`: `tree` is an env-assignment, with classic syntax, + `False`: `tree` is not an env-assignment, + + The only way this differs from a left-shift or the usual kind of walrus assignment + is that the LHS must be an ``ast.Name``. + + letsyntax_mode: used by let_syntax to allow template definitions. + This allows, beside a bare name `k`, the formats `k(a0, ...)` and `k[a0, ...]` + to appear in the variable-name position. """ - if not (type(tree) is BinOp and type(tree.op) is LShift): - return False - # The `Done` may be produced by expanded `@namemacro`s. - return type(tree.left) is Name or (isinstance(tree.left, Done) and type(tree.body) is Name) + if type(tree) is BinOp and type(tree.op) is LShift and _isbindingtarget(tree.left, letsyntax_mode): + return LShift + if type(tree) is NamedExpr and _isbindingtarget(tree.target, letsyntax_mode): # added in 0.15.3 + return NamedExpr + return False # TODO: This would benefit from macro destructuring in the expander. # TODO: See https://github.com/Technologicat/mcpyrate/issues/3 @@ -146,30 +179,30 @@ def islet(tree, expanded=True): elif not isx(tree.func, letf_name): return False mode = [kw.value for kw in tree.keywords if kw.arg == "mode"] - assert len(mode) == 1 and type(mode[0]) in (Constant, Str) - mode = getconstant(mode[0]) + assert len(mode) == 1 and type(mode[0]) is Constant + mode = mode[0].value kwnames = [kw.arg for kw in tree.keywords] if "_envname" in kwnames: return (f"{kind}_decorator", mode) # this call was generated by _let_decorator_impl else: return (f"{kind}_expr", mode) # this call was generated by _let_expr_impl - # dlet[k0 << v0, ...] (usually in a decorator list) + # dlet[k0 := v0, ...] (usually in a decorator list) deconames = ("dlet", "dletseq", "dletrec", "blet", "bletseq", "bletrec") if type(tree) is Subscript and type(tree.value) is Name: # could be a Subscript decorator (Python 3.9+) s = tree.value.id if any(s == x for x in deconames): return ("decorator", s) - if type(tree) is Call and type(tree.func) is Name: # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + if type(tree) is Call and type(tree.func) is Name: # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) s = tree.func.id if any(s == x for x in deconames): return ("decorator", s) # otherwise we should have an expr macro invocation - if not type(tree) is Subscript: + if type(tree) is not Subscript: return False # Note we don't care about the bindings format here. - # let[k0 << v0, ...][body] - # let(k0 << v0, ...)[body] + # let[k0 := v0, ...][body] + # let(k0 := v0, ...)[body] # ^^^^^^^^^^^^^^^^^^ macro = tree.value exprnames = ("let", "letseq", "letrec", "let_syntax", "abbrev") @@ -177,7 +210,7 @@ def islet(tree, expanded=True): s = macro.value.id if any(s == x for x in exprnames): return ("lispy_expr", s) - elif type(macro) is Call and type(macro.func) is Name: # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + elif type(macro) is Call and type(macro.func) is Name: # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) s = macro.func.id if any(s == x for x in exprnames): return ("lispy_expr", s) @@ -185,8 +218,8 @@ def islet(tree, expanded=True): elif type(macro) is Name: s = macro.id if any(s == x for x in exprnames): - # let[k0 << v0, ...][body] - # let(k0 << v0, ...)[body] + # let[k0 := v0, ...][body] + # let(k0 := v0, ...)[body] # ^^^^ expr = _get_subscript_slice(tree) h = _ishaskellylet(expr) @@ -201,19 +234,19 @@ def _ishaskellylet(tree): In other words, detect the part inside the brackets in:: - let[[k0 << v0, ...] in body] - let[body, where[k0 << v0, ...]] + let[[k0 := v0, ...] in body] + let[body, where[k0 := v0, ...]] To detect the full expression including the ``let[]``, use ``islet`` instead. """ - # let[[k0 << v0, ...] in body] - # let[(k0 << v0, ...) in body] + # let[[k0 := v0, ...] in body] + # let[(k0 := v0, ...) in body] def maybeiscontentofletin(tree): return (type(tree) is Compare and len(tree.ops) == 1 and type(tree.ops[0]) is In and type(tree.left) in (List, Tuple)) - # let[body, where[k0 << v0, ...]] - # let[body, where(k0 << v0, ...)] + # let[body, where[k0 := v0, ...]] + # let[body, where(k0 := v0, ...)] def maybeiscontentofletwhere(tree): return type(tree) is Tuple and len(tree.elts) == 2 and type(tree.elts[1]) in (Call, Subscript) @@ -272,7 +305,7 @@ def isdo(tree, expanded=True): return False # TODO: detect also do[] with a single expression inside? (now requires a comma) - if not type(_get_subscript_slice(tree)) is Tuple: + if type(_get_subscript_slice(tree)) is not Tuple: return False return tree.value.id @@ -280,10 +313,10 @@ def isdo(tree, expanded=True): # ----------------------------------------------------------------------------- class UnexpandedEnvAssignView: - """Destructure an env-assignment, writably. + """Destructure an unexpanded env-assignment, writably. If ``tree`` cannot be interpreted as an unpythonic ``env`` assignment - of the form ``name << value``, then ``TypeError`` is raised. + of the form ``name := value`` or ``name << value``, then ``TypeError`` is raised. For easy in-place modification of both ``name`` and ``value``. Use before the env-assignment is expanded away (so, before the ``let[]`` or ``do[]`` @@ -303,7 +336,7 @@ class UnexpandedEnvAssignView: ``value``: the thing being assigned, as an AST. - Writing to either attribute updates the original. + Writing to either attribute updates the original, preserving the syntax (`:=` or `<<`). """ def __init__(self, tree): if not isenvassign(tree): @@ -311,21 +344,34 @@ def __init__(self, tree): self._tree = tree def _getname(self): - return getname(self._tree.left, accept_attr=False) + if isenvassign(self._tree) is LShift: + return getname(self._tree.left, accept_attr=False) + else: # NamedExpr + return getname(self._tree.target, accept_attr=False) def _setname(self, newname): if not isinstance(newname, str): raise TypeError(f"expected str for new name, got {type(newname)} with value {repr(newname)}") + if isenvassign(self._tree) is LShift: + targetnode = self._tree.left + else: # NamedExpr + targetnode = self._tree.target # The `Done` may be produced by expanded `@namemacro`s. - if isinstance(self._tree.left, Done): - self._tree.left.body.id = newname + if isinstance(targetnode, Done): + targetnode.body.id = newname else: - self._tree.left.id = newname + targetnode.id = newname name = property(fget=_getname, fset=_setname, doc="The name of the assigned var, as an str. Writable.") def _getvalue(self): - return self._tree.right + if isenvassign(self._tree) is LShift: + return self._tree.right + else: # NamedExpr + return self._tree.value def _setvalue(self, newvalue): - self._tree.right = newvalue + if isenvassign(self._tree) is LShift: + self._tree.right = newvalue + else: # NamedExpr + self._tree.value = newvalue value = property(fget=_getvalue, fset=_setvalue, doc="The value of the assigned var, as an AST. Writable.") class UnexpandedLetView: @@ -339,30 +385,32 @@ class UnexpandedLetView: **Supported formats**:: - dlet[k0 << v0, ...] # decorator - let[k0 << v0, ...][body] # lispy expression - let[[k0 << v0, ...] in body] # haskelly expression - let[body, where[k0 << v0, ...]] # haskelly expression, inverted + dlet[k0 := v0, ...] # decorator + let[k0 := v0, ...][body] # lispy expression + let[[k0 := v0, ...] in body] # haskelly expression + let[body, where[k0 := v0, ...]] # haskelly expression, inverted In addition, we also support *just the bracketed part* of the haskelly formats. This is to make it easier for the macro interface to destructure these forms (for sending into the ``let`` syntax transformer). So these forms are supported, too:: - [k0 << v0, ...] in body - (body, where[k0 << v0, ...]) + [k0 := v0, ...] in body + (body, where[k0 := v0, ...]) Finally, in any of these, the bindings subform can actually be in any of the formats: - [k0 << v0, ...] # preferred, v0.15.0+ + [k0 := v0, ...] # preferred, v0.15.3+ + [k0 << v0, ...] # preferred, v0.15.0 to v0.15.2 (k0 << v0, ...) [[k0, v0], ...] [(k0, v0), ...] ([k0, v0], ...) ((k0, v0), ...) k, v - k << v # preferred for a single binding, v0.15.0+ + k := v # preferred for a single binding, v0.15.3+ + k << v # preferred for a single binding, v0.15.0 to v0.15.2 This is a data abstraction that hides the detailed structure of the AST, since there are many alternate syntaxes that can be used for a ``let`` @@ -445,7 +493,7 @@ def _getbindings(self): # ^^^^^^^^^^ thetree = self._tree.value - if type(thetree) is Call: # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + if type(thetree) is Call: # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) return canonize_bindings(thetree.args) # Subscript theargs = _get_subscript_slice(thetree) @@ -474,7 +522,7 @@ def _setbindings(self, newbindings): # ^^^^^^^^^^ thetree = self._tree.value - if type(thetree) is Call: # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + if type(thetree) is Call: # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) thetree.args = newbindings return _set_subscript_slice(thetree, Tuple(elts=newbindings)) @@ -676,8 +724,8 @@ def _setbindings(self, newbindings): raise NotImplementedError("changing the number of items currently not supported by this view (do that before the let[] expands)") # pragma: no cover for newb in newbindings.elts: newk, newv = newb.elts - if type(newk) not in (Constant, Str): # Python 3.8+: ast.Constant - raise TypeError("ExpandedLetView: let: each key must be an ast.Constant or an ast.Str") # pragma: no cover + if type(newk) is not Constant: + raise TypeError("ExpandedLetView: let: each key must be an ast.Constant") # pragma: no cover # Abstract away the namelambda(...). We support both "with autocurry" and bare formats: # currycall(letter, bindings, currycall(currycall(namelambda, "let_body"), curryf(lambda e: ...))) # letter(bindings, namelambda("let_body")(lambda e: ...)) @@ -689,7 +737,7 @@ def _setbindings(self, newbindings): for oldb, newb in zip(thebindings.elts, newbindings.elts): oldk, thev = oldb.elts newk, newv = newb.elts - newk_string = getconstant(newk) # Python 3.8+: ast.Constant + newk_string = newk.value if type(newv) is not Lambda: raise TypeError("ExpandedLetView: letrec: each value must be of the form `lambda e: ...`") # pragma: no cover if curried: @@ -706,12 +754,11 @@ def _setbindings(self, newbindings): # update name in the namelambda(...) thev.func.args[0] = Constant(value=f"letrec_binding_{newk_string}") # Python 3.8+: ast.Constant # Macro-generated nodes may be missing source location information, - # in which case we let `mcpyrate` fix it later. + # in which case we let `mcpyrate` fix it later. `ast.copy_location` + # copies whichever of `lineno`/`col_offset`/`end_lineno`/`end_col_offset` + # are present on `oldb`, leaving the rest unset. # This is mainly an issue for the unit tests of this module, which macro-generate the "old" data. - if hasattr(oldb, "lineno") and hasattr(oldb, "col_offset"): - newelts.append(Tuple(elts=[newk, thev], lineno=oldb.lineno, col_offset=oldb.col_offset)) - else: - newelts.append(Tuple(elts=[newk, thev])) + newelts.append(copy_location(Tuple(elts=[newk, thev]), oldb)) thebindings.elts = newelts else: thebindings.elts = newbindings.elts diff --git a/unpythonic/syntax/letsyntax.py b/unpythonic/syntax/letsyntax.py index b0acc118..2b4e59a5 100644 --- a/unpythonic/syntax/letsyntax.py +++ b/unpythonic/syntax/letsyntax.py @@ -4,6 +4,16 @@ # at macro expansion time. If you're looking for regular run-time let et al. macros, # see letdo.py. +# TODO: Coverage of code using `with block` and `with expr` is not reported correctly. +# +# TODO: As this is a toy macro system within the real macro system, that is to be expected; +# TODO: `mcpyrate` goes to some degree of trouble to produce correct coverage reporting for +# TODO: the real macro system, and we haven't duplicated that effort here. +# +# TODO: With `mcpyrate`, we don't really need `let_syntax` and `abbrev` anymore, so we could +# TODO: actually remove them; but their tests exercise some code paths that would otherwise +# TODO: remain untested. As of v0.15.0, we're keeping them for now. + __all__ = ["let_syntax", "abbrev", "expr", "block"] from mcpyrate.quotes import macros, q, a # noqa: F401 @@ -11,7 +21,6 @@ from ast import Name, Call, Subscript, Tuple, Starred, Expr, With from copy import deepcopy from functools import partial -import sys from mcpyrate import parametricmacro from mcpyrate.quotes import is_captured_value @@ -33,16 +42,20 @@ def let_syntax(tree, *, args, syntax, expander, **kw): **Expression variant**:: - let_syntax[lhs << rhs, ...][body] - let_syntax[lhs << rhs, ...][[body0, ...]] + let_syntax[lhs := rhs, ...][body] + let_syntax[lhs := rhs, ...][[body0, ...]] Alternative haskelly syntax:: - let_syntax[[lhs << rhs, ...] in body] - let_syntax[[lhs << rhs, ...] in [body0, ...]] + let_syntax[[lhs := rhs, ...] in body] + let_syntax[[lhs := rhs, ...] in [body0, ...]] - let_syntax[body, where[lhs << rhs, ...]] - let_syntax[[body0, ...], where[lhs << rhs, ...]] + let_syntax[body, where[lhs := rhs, ...]] + let_syntax[[body0, ...], where[lhs := rhs, ...]] + + The classic spelling ``lhs << rhs``, from v0.15.0 to v0.15.2, is still accepted - and for a + *template* it is the only spelling there is, because a template's LHS is a subscript and Python's + own grammar rejects ``(f[a] := ...)``. So the template examples below keep ``<<`` deliberately. **Block variant**:: @@ -148,7 +161,7 @@ def abbrev(tree, *, args, syntax, expander, **kw): Because this variant expands before any macros in the body, it can locally rename other macros, e.g.:: - abbrev[m << macrowithverylongname][ + abbrev[m := macrowithverylongname][ m[tree1] if m[tree2] else m[tree3]] **CAUTION**: Because ``abbrev`` expands outside-in, and does not respect @@ -194,12 +207,14 @@ def block(tree, *, syntax, **kw): # -------------------------------------------------------------------------------- # Syntax transformers -# let_syntax[lhs << rhs, ...][body] -# let_syntax[lhs << rhs, ...][[body0, ...]] -# let_syntax[[lhs << rhs, ...] in body] -# let_syntax[[lhs << rhs, ...] in [body0, ...]] -# let_syntax[body, where[lhs << rhs, ...]] -# let_syntax[[body0, ...], where[lhs << rhs, ...]] +# let_syntax[lhs := rhs, ...][body] +# let_syntax[lhs := rhs, ...][[body0, ...]] +# let_syntax[[lhs := rhs, ...] in body] +# let_syntax[[lhs := rhs, ...] in [body0, ...]] +# let_syntax[body, where[lhs := rhs, ...]] +# let_syntax[[body0, ...], where[lhs := rhs, ...]] +# +# `lhs << rhs` also works; for a template it is the only spelling, `(f[a] := ...)` being a SyntaxError. # # This transformer takes destructured input, with the bindings subform # and the body already extracted, and supplied separately. @@ -225,8 +240,8 @@ def register_bindings(): target.append((name, args, value, "expr")) if expand_inside: - bindings = dyn._macro_expander.visit(bindings) - body = dyn._macro_expander.visit(body) + bindings = dyn._macro_expander.visit_recursively(bindings) + body = dyn._macro_expander.visit_recursively(body) register_bindings() body = _substitute_templates(templates, body) body = _substitute_barenames(barenames, body) @@ -318,7 +333,7 @@ def isbinding(tree): if type(ctxmanager) is Subscript and type(ctxmanager.value) is Name and ctxmanager.value.id == mode: return mode, "template" # expr(...), block(...) - # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) if type(ctxmanager) is Call and type(ctxmanager.func) is Name and ctxmanager.func.id == mode: return mode, "template" return False @@ -330,7 +345,7 @@ def isbinding(tree): # `let_syntax` mode (expand_inside): respect lexical scoping of nested `let_syntax`/`abbrev` expanded = False if expand_inside and (is_let_syntax(stmt) or is_abbrev(stmt)): - stmt = dyn._macro_expander.visit(stmt) + stmt = dyn._macro_expander.visit_recursively(stmt) expanded = True stmt = _substitute_templates(templates, stmt) @@ -341,14 +356,14 @@ def isbinding(tree): check_stray_blocks_and_exprs(value) # before expanding it! if expand_inside and not expanded: - value = dyn._macro_expander.visit(value) + value = dyn._macro_expander.visit_recursively(value) target = templates if args else barenames target.append((name, args, value, mode)) else: check_stray_blocks_and_exprs(stmt) # before expanding it! if expand_inside and not expanded: - stmt = dyn._macro_expander.visit(stmt) + stmt = dyn._macro_expander.visit_recursively(stmt) new_block_body.append(stmt) new_block_body = eliminate_ifones(new_block_body) @@ -359,10 +374,7 @@ def isbinding(tree): # ----------------------------------------------------------------------------- def _get_subscript_args(tree): - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - theslice = tree.slice - else: - theslice = tree.slice.value + theslice = tree.slice if type(theslice) is Tuple: args = theslice.elts else: @@ -379,7 +391,7 @@ def _analyze_lhs(tree): elif type(tree) is Subscript and type(tree.value) is Name: # template f[x, ...] name = tree.value.id args = [a.id for a in _get_subscript_args(tree)] - # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) elif type(tree) is Call and type(tree.func) is Name: # template f(x, ...) name = tree.func.id if any(type(a) is Starred for a in tree.args): # *args (Python 3.5+) @@ -435,8 +447,8 @@ def _substitute_templates(templates, tree): def isthisfunc(tree): if type(tree) is Subscript and type(tree.value) is Name and tree.value.id == name: return True - # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 - if type(tree) is Call and type(tree.func) is Name and tree.func.id == name: + # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) + if type(tree) is Call and type(tree.func) is Name and tree.func.id == name: # noqa: SIM103 -- a second, deprecated syntax recognized separately; `or`-ing the two would strand the comment above and merge two distinct recognitions into one expression return True return False def subst(tree): diff --git a/unpythonic/syntax/monadic_do.py b/unpythonic/syntax/monadic_do.py new file mode 100644 index 00000000..6ef73aad --- /dev/null +++ b/unpythonic/syntax/monadic_do.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +"""Monadic do-notation as a block macro. + +Syntax:: + + with monadic_do[M] as result: + [x := mx, + y := my(x), + M.guard(...), + M.unit(x + y)] + +The body is a single list literal. Each item corresponds to one line of +a Haskell do-block. The **last item** is the final monadic expression +(any expression of type ``M a``, matching Haskell's last-line-of-do). +All **earlier items** are binds: + +- ``name := mexpr`` — monadic bind: the unwrapped value is bound to + ``name`` for subsequent lines. +- ``name << mexpr`` — legacy alternative for ``:=`` (same shapes + ``letdoutil`` recognizes for ``let[]``). +- a bare ``mexpr`` — sequencing-only (Haskell's ``do { mx; ... }``): the + result is threaded but discarded. The short-circuit behavior of the + monad still applies (``Maybe(nil)``, ``Left``, empty ``List`` all + cancel the rest of the chain). + +Expands to a nested lambda-bind chain:: + + result = mx >> (lambda x: my(x) >> (lambda _: M.guard(...) >> (lambda _: M.unit(x + y)))) + +**Placement in the xmas tree**: always the innermost ``with``. Its body +shape (a single list-literal statement) forbids lexically wrapping other +``with`` blocks inside it, and outer two-pass macros (``lazify``, +``continuations``, ``tco``, ``autocurry``, etc.) expand inner macros +between their two passes, which means they will correctly see and edit +the expanded bind chain. +""" + +__all__ = ["monadic_do"] + +from ast import List, Name, NamedExpr, BinOp, LShift, Expr, Assign, Store, arg, expr + +from mcpyrate.quotes import macros, q, a, n # noqa: F401 + +from mcpyrate import parametricmacro + +from ..dynassign import dyn + +from .letdoutil import canonize_bindings + + +@parametricmacro +def monadic_do(tree, *, args, syntax, expander, **kw): + """[syntax, block] Monadic do-notation. + + See module docstring for usage, placement, and expansion. + """ + if syntax != "block": + raise SyntaxError("monadic_do is a block macro only") # pragma: no cover + + # Require exactly one macro argument: the monad type. + if len(args) != 1: + raise SyntaxError( + f"monadic_do expects exactly one macro argument (the monad type), got {len(args)}" + ) # pragma: no cover + + # Require the `as` binding — this is where the result lands. + result_var = kw.get("optional_vars", None) + if result_var is None: + raise SyntaxError( + "monadic_do requires an as-binding: `with monadic_do[M] as result:`" + ) # pragma: no cover + if type(result_var) is not Name: + raise SyntaxError( + "monadic_do's as-binding must be a single name" + ) # pragma: no cover + + with dyn.let(_macro_expander=expander): + return _monadic_do(block_body=tree, monad_type=args[0], result_name=result_var.id) + + +def _monadic_do(block_body: list, monad_type: expr, result_name: str) -> list: + # Expand inner macros first (outside-in), just like `forall` and `autoref` do. + block_body = dyn._macro_expander.visit_recursively(block_body) + + # Body must be exactly one statement, an Expr wrapping a List literal. + if len(block_body) != 1: + raise SyntaxError( + f"monadic_do body must be a single list-literal statement, got {len(block_body)} statements" + ) # pragma: no cover + stmt = block_body[0] + if type(stmt) is not Expr or type(stmt.value) is not List: + raise SyntaxError( + "monadic_do body must be a single list literal `[bind, ..., final_expr]`" + ) # pragma: no cover + + items = stmt.value.elts + if not items: + raise SyntaxError( + "monadic_do body list must have at least one item (the final monadic expression)" + ) # pragma: no cover + + # Split: all but the last are binds; the last is the final monadic expression. + *binding_items, final_expr = items + + # Normalize bare expressions in the binds as synthetic `_ := expr` so they + # look like sequencing-only bindings to `canonize_bindings`. Matches Haskell's + # do-notation where a bare expression line is sequence-only (>>, not >>=). + normalized = [ + item if _is_binding_form(item) else NamedExpr(target=Name(id="_", ctx=Store()), value=item) + for item in binding_items + ] + + # Parse via letdoutil — accepts := and <<. + if normalized: + canonical = canonize_bindings(normalized) # [Tuple(elts=[Name(k), v]), ...] + pairs = [(t.elts[0].id, t.elts[1]) for t in canonical] + else: + pairs = [] + + # Build the bind chain, innermost-first: + # final_expr + # mz >> (lambda z: final_expr) + # my >> (lambda y: mz >> (lambda z: final_expr)) + # mx >> (lambda x: my >> (lambda y: mz >> (lambda z: final_expr))) + body = final_expr + for name, mexpr in reversed(pairs): + # lambda : + lam = q[lambda: a[body]] + lam.args.args = [arg(arg=name)] + # >> + body = q[a[mexpr] >> a[lam]] + + # Final assignment: ` = `. This is a statement; we replace + # the entire `with` body with it. + assignment = Assign(targets=[Name(id=result_name, ctx=Store())], value=body) + return [assignment] + + +def _is_binding_form(item) -> bool: + """Return True if *item* is ``name := expr`` or ``name << expr`` (a let-style binding).""" + if type(item) is NamedExpr and type(item.target) is Name: + return True + if type(item) is BinOp and type(item.op) is LShift and type(item.left) is Name: # noqa: SIM103 -- keep cases visually separate + return True + return False diff --git a/unpythonic/syntax/multishot.py b/unpythonic/syntax/multishot.py new file mode 100644 index 00000000..2acf06ba --- /dev/null +++ b/unpythonic/syntax/multishot.py @@ -0,0 +1,682 @@ +# -*- coding: utf-8 -*- +"""Multi-shot generators. + +A `@multishot` function is a generator-shaped construct whose execution +state is captured *as a continuation* at every `myield`, so it can be +resumed from any earlier `myield` arbitrarily many times — branching +execution into independent timelines. + +Built on top of `call_cc[]` / `get_cc()`. Only meaningful inside a +`with continuations:` block (this is enforced by `call_cc[]`, which the +expansion of `myield` produces). + +For attribution: the one-shot vs. multi-shot continuation distinction +goes back at least to Bruggeman, Waddell & Dybvig 1996 ("Representing +control in the presence of one-shot continuations"). Racket's docs are +the canonical reference for current usage: +https://docs.racket-lang.org/reference/cont.html + +Public surface: + + - `multishot` — decorator macro that turns a `def` into a multi-shot + generator. Inside, the four `myield` forms are recognized and rewritten. + - `myield` — name/expr macro for the four yield variants: + + Multi-shot yield Returns `k` expects Single-shot analog + + myield k no argument yield + myield[expr] (k, value) no argument yield expr + var = myield k one argument var = yield + var = myield[expr] (k, value) one argument var = yield expr + + - `MultishotIterator` — adapter that makes a `@multishot` conform to + a subset of Python's generator protocol, plus `copy.copy()` for forking. + +See `doc/macros.md` for the user-facing documentation. +""" + +import ast +from functools import partial + +from mcpyrate.quotes import macros, q, n, a, h # noqa: F401 + +from mcpyrate import namemacro, gensym +from mcpyrate.quotes import is_captured_value +from mcpyrate.utils import extract_bindings, rename +from mcpyrate.walkers import ASTTransformer + +from ..fun import identity +from ..misc import safeissubclass + +from .scopeanalyzer import isnewscope +from .tailtools import macros, call_cc # noqa: F401, F811 -- macro-import: makes `h[call_cc]` a hygienic *macro* reference +from .tailtools import get_cc, iscontinuation + + +__all__ = ["multishot", "myield", "myield_from", "MultishotIterator"] + + +# -------------------------------------------------------------------------------- +# `myield` — name/expr macro + +def myield_function(tree, syntax, **kw): + """[syntax, name/expr] Yield from a multi-shot generator. + + Only meaningful at the top level of a function decorated with `@multishot`. + Outside that context, raises `SyntaxError` at macro-expansion time. + + For details, see `multishot`. + """ + if syntax not in ("name", "expr"): + raise SyntaxError("myield is a name and expr macro only") # pragma: no cover + + # Allow `myield` in non-Load contexts so the name can be assigned to / del'd + # without spuriously triggering the macro (mostly defensive — `multishot` + # itself recognizes the patterns it needs before this macro runs). + if type(getattr(tree, "ctx", None)) in (ast.Store, ast.Del): + return tree + + # `myield` is not really a macro; it's a marker that `@multishot` looks for + # and rewrites away. If a `myield` survives to reach the expander, it was + # placed somewhere `@multishot` couldn't see it (outside `@multishot`, or + # inside a nested scope that `@multishot` deliberately doesn't recurse into). + raise SyntaxError("myield may only appear at the top level of a `@multishot` generator") + + +myield = namemacro(myield_function) + + +# -------------------------------------------------------------------------------- +# `myield_from` — name/expr macro + +def myield_from_function(tree, syntax, **kw): + """[syntax, name/expr] Delegate to another `@multishot` generator. + + Multi-shot analog of `yield from`. Drives the inner multi-shot generator, + re-yielding each of its values to the outer's caller. On inner exhaustion + (`StopIteration`), execution continues in the outer body. The two forms:: + + myield_from[expr] # statement; inner's StopIteration value discarded + var = myield_from[expr] # statement; inner's StopIteration value bound to var + + Only meaningful at the top level of a `@multishot` function decorated within + a `with continuations:` block. Forwards `send` and `throw` from the outer's + caller into the inner; while delegating, `outer_mi.gi_yieldfrom` returns + the inner `MultishotIterator`. + + For details, see `multishot`. + """ + if syntax not in ("name", "expr"): + raise SyntaxError("myield_from is a name and expr macro only") # pragma: no cover + + if type(getattr(tree, "ctx", None)) in (ast.Store, ast.Del): + return tree + + raise SyntaxError("myield_from may only appear at the top level of a `@multishot` " + "generator, as `myield_from[expr]` or `var = myield_from[expr]`") + + +myield_from = namemacro(myield_from_function) + + +# -------------------------------------------------------------------------------- +# Expansion of `myield_from[expr]` / `var = myield_from[expr]` +# +# Architecture: +# +# - **Capture rest-of-outer at outer's top level**, before the iteration +# begins, via `_rest = call_cc[get_cc()]`. The first pass through (when +# `_rest` is a continuation) tail-calls the driver, passing `_rest`; the +# second pass (when the driver has invoked `_rest(value)`) sees `_rest` as +# the inner's `StopIteration` value and falls through to the post- +# `myield_from` code in outer's body. +# +# - **The driver itself uses cut-the-tail** (`cc = identity` + return tuple) +# to escape each `(captured_cc, inner_value)` to the user. Because the +# driver is *tail-called* from outer (no nested trampoline started), and +# the helper is *tail-called* via `call_cc[helper(...)]`, every step shares +# the trampoline that `mi._k()` set up. The cut-the-tail escape therefore +# reaches the user's `mi._k()` return — not a nested helper's frame. +# +# - **Resume after exhaustion via the captured rest-cc**: when inner raises +# `StopIteration`, the driver does `return _rest_k(stopvalue)` (tail call), +# resuming outer's body just after the rest-cc-capture point. +# +# - **`MultishotIterator` for inner**: convenient wrapper for `send`/`throw` +# forwarding (we delegate to its protocol methods); it also lets +# `gi_yieldfrom` surface the inner iterator via a stamp on the captured cc. +# +# Limitations of this v1: send/throw do reach the inner; `gi_yieldfrom` +# tracks correctly while delegating; multi-shot fork during delegation is +# inherited from the multi-shot semantics of the inner and the captured cc +# stamping. See `doc/macros.md` for the user-facing description. + +def _build_myield_from_expansion(arg, target): + """Build the AST list for a `myield_from` invocation. + + See the section comment above for the architecture. `arg` is the AST of + the inner-multishot-call expression (e.g., `inner()`). `target` is the + assignment target for `var = myield_from[...]`, or `None` for statement + form (inner's `StopIteration` value discarded). + """ + inner_mi_name = gensym("_inner_mi") + yieldf_name = gensym("_yieldf") + drive_name = gensym("_drive") + rest_name = gensym("_rest") + + with q as quoted: + _INNER_MI_ = h[MultishotIterator](a[arg]) + + def _YIELDF_(_value, _inner_mi, *, cc): + # Cut-the-tail: capture cc as `_k` (this is the at-call-cc + # continuation, which is "rest of `_drive` after the call_cc"), + # stamp it for `gi_yieldfrom`, then locally `cc = identity` so + # the trampolined return delivers the (k, value) tuple straight + # to whoever started the trampoline — i.e., the user's `mi._k()`. + _k = cc + _k._yieldfrom_inner = _inner_mi + cc = h[identity] + return (_k, _value) + + def _DRIVE_(_inner_mi, _rest_k, _value=None, _is_throw=False): + try: + if _is_throw: + _x = _inner_mi.throw(_value) + else: + _x = _inner_mi.send(_value) + except h[StopIteration] as _stopit: + # Inner exhausted. Resume outer's body via the captured + # rest-cc; outer continues at the post-`myield_from` code. + return _rest_k(_stopit.value) + _sent = h[call_cc][_YIELDF_(_x, _inner_mi)] + if isinstance(_sent, BaseException) or h[safeissubclass](_sent, BaseException): + return _DRIVE_(_inner_mi, _rest_k, _sent, True) + return _DRIVE_(_inner_mi, _rest_k, _sent) + + # `_REST_ = call_cc[get_cc()]` is the multi-shot analog of Racket's + # `(let/cc return ...)`: `_REST_` is bound to "rest of outer" as a + # continuation. The first pass captures it; the driver later invokes + # it with inner's `StopIteration` value to fall through to the rest. + _REST_ = h[call_cc][h[get_cc]()] + if h[iscontinuation](_REST_): + return _DRIVE_(_INNER_MI_, _REST_) + # control reaches here when inner exhausted; `_REST_` holds inner's + # `StopIteration` value (or `None` if inner returned without a value). + + rename("_INNER_MI_", inner_mi_name, quoted) + rename("_YIELDF_", yieldf_name, quoted) + rename("_DRIVE_", drive_name, quoted) + rename("_REST_", rest_name, quoted) + + if target is not None: + with q as quoted_assign: + a[target] = n[rest_name] + quoted = quoted + quoted_assign + + return quoted + + +# -------------------------------------------------------------------------------- +# `@multishot` — decorator macro + +def multishot(tree, syntax, expander, **kw): + """[syntax, decorator] Make a function into a multi-shot generator. + + Only meaningful inside a `with continuations:` block — required, not + auto-wrapped. The expansion of `myield` produces `call_cc[get_cc()]`, + which `with continuations` then turns into the actual continuation + machinery; outside `with continuations`, that step fails with the + standard `call_cc[]` SyntaxError. + + Multi-shot yield is spelled `myield`. The use site of `@multishot` + must macro-import `myield` too, so that this macro knows which name + you've bound it under. + + There are four variants:: + + Multi-shot yield Returns `k` expects Single-shot analog + + myield k no argument yield + myield[expr] (k, value) no argument yield expr + var = myield k one argument var = yield + var = myield[expr] (k, value) one argument var = yield expr + + To resume, call `k`. In cases where `k` expects an argument, that + argument is the value to send into `var`. + + Important differences from standard Python generators: + + - A multi-shot generator may be resumed from any `myield` arbitrarily + many times, in any order. There is no concept of a single paused + activation; each continuation is a function (technically a closure). + + When a multi-shot generator "myields", it returns just like a + normal function, technically terminating its execution. But it + gives you a continuation closure that you can call to resume + execution just after that particular `myield`. + + The state lives in the closure cells of the continuation. The + continuations are nested, so for a given activation, any locals + in the already-executed part remain alive as long as at least + one reference to a relevant continuation closure exists. + + "Nested" implies that re-invoking an earlier continuation branches + execution into an independent timeline — but only for *locals*. + Each resume gets a fresh activation record, so locals diverge; + closure cells captured before the resume point are shared, so a + mutation through one (a `nonlocal`, a mutable argument, or + module-level state) is visible to every timeline reached from the + same fork point. + + - `myield` is a *statement*, and it may only appear at the top level + of a `@multishot` function definition (limitation of the underlying + `call_cc[]`). Use inside lambdas, comprehensions, or nested `def`s + is rejected at macro-expansion time. + + Usage:: + + with continuations: + @multishot + def f(): + # Stop and return a continuation `k` that resumes just after this `myield`. + myield + + # Stop and return the tuple `(k, 42)`. + myield[42] + + # Stop and return a continuation `k`. Upon resuming `k`, + # set the local `k` to the value sent in. + k = myield + + # Stop and return the tuple `(k, 42)`. Upon resuming `k`, + # set the local `k` to the value sent in. + k = myield[42] + + # Instantiate the multi-shot generator (like calling a gfunc). + # There is always an implicit bare `myield` at the beginning. + k0 = f() + + # Start; run up to the explicit bare `myield`; receive new continuation. + k1 = k0() + + # Continue to `myield[42]`; receive new continuation and the `42`. + k2, x2 = k1() + + # Continue to `k = myield`; receive new continuation. + k3 = k2() + + # Send `23` as the value of `k`; continue to `k = myield[42]`. + k4, x4 = k3(23) + + # Send `17` as the value of `k`; continue to the end. + # Reaching the end raises `StopIteration` (as with a regular generator). + # `return value` inside `@multishot` raises `StopIteration(value)`. + + # Re-invoke an earlier continuation: + k2, x2 = k1() + + For ergonomic generator-shaped consumption, wrap the initial continuation + in a `MultishotIterator`. + """ + if syntax != "decorator": + raise SyntaxError("multishot is a decorator macro only") # pragma: no cover + if type(tree) is not ast.FunctionDef: + raise SyntaxError("@multishot supports `def` only") + + # Detect the name(s) under which `myield` is macro-imported (handles as-imports). + macro_bindings = extract_bindings(expander.bindings, myield_function) + if not macro_bindings: + raise SyntaxError("The use site of `@multishot` must macro-import `myield`, too.") + names_of_myield = list(macro_bindings.keys()) + + # `myield_from` is optional; only present if the user macro-imported it. + macro_bindings_from = extract_bindings(expander.bindings, myield_from_function) + names_of_myield_from = set(macro_bindings_from.keys()) + + def is_myield_name(node): + return type(node) is ast.Name and node.id in names_of_myield + def is_myield_expr(node): + return type(node) is ast.Subscript and is_myield_name(node.value) + def is_myield_from_expr(node): + # `myield_from[expr]` parses as a Subscript with `value` being the + # `myield_from` name. Mirrors how `myield[expr]` is recognized. + return (type(node) is ast.Subscript + and type(node.value) is ast.Name + and node.value.id in names_of_myield_from) + def getslice(subscript_node): + return subscript_node.slice + + class MultishotYieldTransformer(ASTTransformer): + def transform(self, tree): + if is_captured_value(tree): + return tree + if isnewscope(tree): + return tree + + # `myield_from[expr]` / `var = myield_from[expr]` — handled before + # `myield` shapes since both macros share the user-namespace symbol family. + if names_of_myield_from: + if (type(tree) is ast.Assign and len(tree.targets) == 1 + and is_myield_from_expr(tree.value)): + return _build_myield_from_expansion(getslice(tree.value), + target=tree.targets[0]) + if type(tree) is ast.Expr and is_myield_from_expr(tree.value): + return _build_myield_from_expansion(getslice(tree.value), target=None) + + # `k = myield[value]` + if type(tree) is ast.Assign and is_myield_expr(tree.value): + if len(tree.targets) != 1: + raise SyntaxError("expected exactly one assignment target in `k = myield[expr]`") + var = tree.targets[0] + value = getslice(tree.value) + with q as quoted: + a[var] = h[call_cc][h[get_cc]()] + if h[iscontinuation](a[var]): + return a[var], a[value] + elif isinstance(a[var], BaseException) or h[safeissubclass](a[var], BaseException): + raise a[var] + return quoted + + # `k = myield` + elif type(tree) is ast.Assign and is_myield_name(tree.value): + if len(tree.targets) != 1: + raise SyntaxError("expected exactly one assignment target in `k = myield`") + var = tree.targets[0] + with q as quoted: + a[var] = h[call_cc][h[get_cc]()] + if h[iscontinuation](a[var]): + return a[var] + elif isinstance(a[var], BaseException) or h[safeissubclass](a[var], BaseException): + raise a[var] + return quoted + + # `myield[value]` + elif type(tree) is ast.Expr and is_myield_expr(tree.value): + var = q[n[gensym("k")]] + value = getslice(tree.value) + with q as quoted: + a[var] = h[call_cc][h[get_cc]()] + if h[iscontinuation](a[var]): + return h[partial](a[var], None), a[value] + elif isinstance(a[var], BaseException) or h[safeissubclass](a[var], BaseException): + raise a[var] + return quoted + + # `myield` + elif type(tree) is ast.Expr and is_myield_name(tree.value): + var = q[n[gensym("k")]] + with q as quoted: + a[var] = h[call_cc][h[get_cc]()] + if h[iscontinuation](a[var]): + return h[partial](a[var], None) + elif isinstance(a[var], BaseException) or h[safeissubclass](a[var], BaseException): + raise a[var] + return quoted + + return self.generic_visit(tree) + + class ReturnToRaiseStopIterationTransformer(ASTTransformer): + def transform(self, tree): + if is_captured_value(tree): + return tree + if isnewscope(tree): + return tree + + if type(tree) is ast.Return: + if tree.value is None: + with q as quoted: + raise h[StopIteration] + return quoted + with q as quoted: + raise h[StopIteration](a[tree.value]) + return quoted + + return self.generic_visit(tree) + + # Make the multishot generator raise `StopIteration` when it finishes via + # any `return`. First make the implicit bare `return` explicit, then rewrite. + # This must happen before transforming `myield`, to avoid breaking tail-calling + # of the continuations. + if type(tree.body[-1]) is not ast.Return: + with q as quoted: + return + tree.body.extend(quoted) + tree.body = ReturnToRaiseStopIterationTransformer().visit(tree.body) + + # Inject a bare `myield` resume point at the beginning of the function body. + # When the multishot is initially called, the arguments are bound, and the + # caller gets a continuation back; resuming that continuation actually starts + # executing the function body. Mirrors a Python generator's first-`next` shape. + tree.body.insert(0, ast.Expr(value=ast.Name(id=names_of_myield[0]))) + + tree.body = MultishotYieldTransformer().visit(tree.body) + + return tree + + +# -------------------------------------------------------------------------------- +# `MultishotIterator` — generator-protocol adapter + +def _continuation_code(k): + """Extract the `__code__` of a continuation, unwrapping `partial` if present.""" + func = k.func if isinstance(k, partial) else k + return getattr(func, "__code__", None) + + +def _step(k, mode, value=None): + """Advance a continuation by one step. ``mode`` ∈ {"next", "send", "throw"}. + + ``"next"`` is treated as ``"send"`` with ``value=None`` — **matching the + standard generator protocol**, where ``next(gen)`` is defined to be + ``gen.send(None)``. + + For ``"send"``, if ``k`` is partial-wrapped (came from a bare ``myield`` or + ``myield[expr]`` that doesn't bind a local), the sent value is dropped and + the continuation advances normally — **also matching the standard generator + protocol**, where ``gen.send(value)`` against a bare ``yield`` also discards + the value silently. For raw-form continuations (from ``var = myield`` or + ``var = myield[expr]``), the value is bound to ``var``. + + For ``"throw"``, the partial is unwrapped to inject the exception directly + into the underlying continuation; the partial's pre-applied ``None`` would + otherwise cause an arity mismatch. + + Returns whatever the continuation returns (typically a ``(next_k, value)`` + tuple, or it raises if the continuation raises). + """ + if mode == "throw": + underlying = k.func if isinstance(k, partial) else k + return underlying(value) + # mode in ("next", "send"); next ≡ send(None) + if mode == "next": + value = None + return k() if isinstance(k, partial) else k(value) + + +class MultishotIterator: + """Adapt a `@multishot` generator to a subset of Python's generator protocol. + + Example:: + + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + mi = MultishotIterator(g()) + assert [x for x in mi] == [1, 2, 3] + + Beyond the standard subset, `MultishotIterator` supports `copy.copy(mi)`, + which forks the iterator at its current continuation. The fork shares the + current continuation; subsequent advances get fresh activation records, so + the forks' locals diverge. Closure cells captured before the fork point + are shared, so a mutation through one (a `nonlocal`, a mutable argument, + or module-level state) in one fork is visible to the others. Forks are + independent timelines for *locals* only, not for state reached through + closure cells. (Unlike standard generators, multi-shot generators support + `copy.copy()`.) + + `copy.deepcopy(mi)` raises `TypeError` — the continuation closes over caller + state we can't meaningfully deep-copy. Use `copy.copy(mi)` to fork. + + The current continuation is stored as `self.k` (read/write, type-checked). + Overwriting `self.k` re-opens a closed iterator. + + Supported subset of the generator protocol: + + - `iter(mi)`, `next(mi)`, `mi.send(value)` + - `mi.throw(exc)`, `mi.close()` + - `mi.gi_code` — the `__code__` of the current continuation, or `None` + when closed. **Use this as the liveness signal**, not `gi_frame`. + - `mi.gi_frame` — **always `None`**. A multi-shot generator has no + paused frame; state lives in the closure cells of the continuation. + The standard-generator idiom `gen.gi_frame is None ↔ exhausted` does + *not* apply here. + - `mi.gi_running` — **always `False`**. Nothing is ever paused. + - `mi.gi_yieldfrom` — currently always `None` (delegation via + `myield_from` is not yet implemented). + + Not supported: + + - `yield from` across a standard generator and a multi-shot generator + (semantic mismatch — standard generators have paused state, multi-shots + don't; cannot be papered over). + - Pickling — continuations are closures. + - Async (`__aiter__`, `asend`, etc.). + """ + def __init__(self, k): + self._k = None + self._closed = False + self.k = k + + # `self.k` — type-checked, fail-fast. + @property + def k(self): + return self._k + @k.setter + def k(self, k): + if not (iscontinuation(k) or (isinstance(k, partial) and iscontinuation(k.func))): + raise TypeError( + f"expected `k` to be a continuation or a partially applied continuation, got {k!r}" + ) + self._k = k + self._closed = False + + # Generator-protocol introspection. + @property + def gi_frame(self): + return None + + @property + def gi_code(self): + if self._closed: + return None + return _continuation_code(self._k) + + @property + def gi_running(self): + return False + + @property + def gi_yieldfrom(self): + if self._closed: + return None + # `_yieldfrom_inner` is stamped on the captured continuation by the + # `myield_from` helper. The continuation is raw there (not partial- + # wrapped), so a direct attribute read suffices — but check the + # underlying function too for robustness in case a future code path + # ever returns a partial-wrapped continuation from `myield_from`. + underlying = self._k.func if isinstance(self._k, partial) else self._k + return getattr(underlying, "_yieldfrom_inner", None) + + # Internal: drives one step via `_step`, updates `self._k` from the + # returned `(next_k, value)`, and surfaces the value to the caller. + # Used by `__next__`, `send`, and `throw`. + def _advance(self, mode, value=None): + assert mode in ("next", "send", "throw") + if self._closed: + raise StopIteration + try: + result = _step(self._k, mode, value) + except StopIteration: + self._closed = True + raise + if isinstance(result, tuple): + self.k, x = result + else: + self.k, x = result, None + return x + + # Generator API. + def __iter__(self): + return self + + def __next__(self): + return self._advance("next") + + def send(self, value): + return self._advance("send", value) + + def throw(self, exc): + # Re-enters the current continuation, making it raise `exc`. If the + # body catches and reaches another `myield`, the new continuation + # becomes the current one and we return the next yielded value + # (matching the standard generator protocol). If the exception isn't + # caught, it propagates out of this call. + return self._advance("throw", exc) + + def close(self): + # https://docs.python.org/3/reference/expressions.html#generator.close + # Bypass `_advance` here: close has different semantics (it injects + # `GeneratorExit` and accepts `StopIteration` as a clean exit), and + # `_advance` would short-circuit on the pre-set `_closed` flag. + if self._closed: + return + self._closed = True + try: + _step(self._k, "throw", GeneratorExit) + except GeneratorExit: + return # body let the close exception propagate (expected) + except StopIteration: + return # body caught GeneratorExit and exited cleanly + # Body caught `GeneratorExit` and `myield`ed another value — disallowed, + # mirroring the standard generator protocol. + raise RuntimeError("@multishot generator attempted to `myield` a value while it was being closed") + + # Forking — the multi-shot superpower exposed through the stdlib `copy` protocol. + def __copy__(self): + """Return a fork of this iterator at the current continuation. + + Both iterators share the current continuation. Each subsequent + advance gets a fresh activation record, so the forks' locals + diverge. Closure cells captured before the fork point are shared, + so a mutation through one (a `nonlocal`, a mutable argument, or + module-level state) in one fork is visible to the others. Forks + are independent timelines for *locals* only, not for state reached + through closure cells. + + This is the multi-shot semantics, not a quirk of `copy.copy()`; + the same applies to plain re-invocation of an earlier continuation. + + If this iterator is closed, the fork is also closed (the underlying + continuation is preserved, so the fork can be re-opened by assigning + to `.k`). + """ + forked = MultishotIterator(self._k) + forked._closed = self._closed + return forked + + def __deepcopy__(self, memo): + raise TypeError( + "multi-shot iterators cannot be deep-copied; use copy.copy() to fork" + ) + + def __del__(self): + # Mirror generator GC semantics. Mostly cosmetic for multishots + # (no paused frame to clean up), but politely closes the iterator. + try: + self.close() + except Exception: + pass diff --git a/unpythonic/syntax/nameutil.py b/unpythonic/syntax/nameutil.py index caf43993..f0088574 100644 --- a/unpythonic/syntax/nameutil.py +++ b/unpythonic/syntax/nameutil.py @@ -9,7 +9,6 @@ "is_unexpanded_expr_macro", "is_unexpanded_block_macro"] from ast import Name, Attribute, Subscript, Call, With -import sys from mcpyrate.core import Done from mcpyrate.quotes import is_captured_macro, is_captured_value, lookup_macro @@ -108,7 +107,7 @@ def is_unexpanded_expr_macro(macrofunction, expander, tree): **CAUTION**: This function doesn't currently support detecting macros that take macro arguments. """ - if not type(tree) is Subscript: + if type(tree) is not Subscript: return False maybemacro = tree.value @@ -124,11 +123,7 @@ def is_unexpanded_expr_macro(macrofunction, expander, tree): # extract the expr macro = expander.isbound(name_node.id) if macro is macrofunction: - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - body = tree.slice - else: - body = tree.slice.value - return body + return tree.slice return False @@ -144,7 +139,7 @@ def is_unexpanded_block_macro(macrofunction, expander, tree): **CAUTION**: This function doesn't currently support several macros in the same `with`. """ - if not type(tree) is With: + if type(tree) is not With: return False ctxmanager = tree.items[0].context_expr # optvars = tree.items[0].optional_vars # as-part @@ -154,7 +149,7 @@ def is_unexpanded_block_macro(macrofunction, expander, tree): # discard args if any if type(maybemacro) is Subscript: maybemacro = maybemacro.value - # parenthesis syntax for macro arguments TODO: Python 3.9+: remove once we bump minimum Python to 3.9 + # Parenthesis syntax for macro arguments (deprecated; kept for backward compatibility) elif type(maybemacro) is Call: maybemacro = maybemacro.func diff --git a/unpythonic/syntax/nb.py b/unpythonic/syntax/nb.py index 3a0e0863..39ab6c13 100644 --- a/unpythonic/syntax/nb.py +++ b/unpythonic/syntax/nb.py @@ -47,20 +47,26 @@ def nb(tree, *, args, syntax, **kw): def _nb(body, args): p = args[0] if args else q[h[print]] # custom print function hook - with q as newbody: # pragma: no cover, quoted only. + with q as newbody: _ = None - theprint = a[p] + theprint = lambda value: h[_print_and_passthrough](a[p], value) for stmt in body: - # We ignore statements (because no return value), and, - # test[] and related expressions from our test framework. - # Those don't return a value either, and play a role - # similar to the `assert` statement. + # We ignore statements (because no return value), and, test[] and related + # expressions from our test framework. Those have no meaningful return value + # either, and play a role similar to the `assert` statement. if type(stmt) is not Expr or istestmacro(stmt.value): newbody.append(stmt) continue - with q as newstmts: # pragma: no cover, quoted only. + with q as newstmts: _ = a[stmt.value] if _ is not None: theprint(_) newbody.extend(newstmts) return newbody + +# Work together with `autoreturn`. If the implicit print appears in tail position, +# the passthrough will return the value that was printed, so that when `autoreturn` +# transforms the code into `return theprint(_)`, it still works fine. +def _print_and_passthrough(printer, value): + printer(value) + return value diff --git a/unpythonic/syntax/prefix.py b/unpythonic/syntax/prefix.py index 7f9a31f0..b5dcfc6a 100644 --- a/unpythonic/syntax/prefix.py +++ b/unpythonic/syntax/prefix.py @@ -7,7 +7,6 @@ __all__ = ["prefix", "q", "u", "kw"] from ast import Call, Starred, Tuple, Load, Subscript -import sys from mcpyrate.quotes import macros, q, u, a, t # noqa: F811, F401 @@ -86,6 +85,9 @@ def prefix(tree, *, syntax, **kw): # noqa: F811 Current limitations: + - The `q`, `u` and `kw` macros cannot be renamed by as-importing; + `with prefix` expects them to have their original names. + - passing ``*args`` and ``**kwargs`` not supported. Workarounds: ``call(...)``; Python's usual function call syntax. @@ -108,7 +110,7 @@ def prefix(tree, *, syntax, **kw): # noqa: F811 # operators compiled away by `prefix`), but the "q[]" we use as a macro in # this module is the quasiquote operator from `mcpyrate.quotes`. # -# This `def` doesn't overwrite the macro `q`, because the `def` runs at run time. +# This `def` doesn't overwrite the `mcpyrate` quasiquote macro `q`, because the `def` runs at run time. # The expander does not try to expand this `q` as a macro, because `def q(...)` # is not a valid macro invocation even when the name `q` has been imported as a macro. @namemacro @@ -122,8 +124,8 @@ def q(tree, *, syntax, **kw): # noqa: F811 def u(tree, *, syntax, **kw): # noqa: F811 """[syntax, name] Unquote operator. Only meaningful in a tuple inside a prefix block.""" if syntax != "name": - raise SyntaxError("q (unpythonic.syntax.prefix.q) is a name macro only") # pragma: no cover - raise SyntaxError("q (unpythonic.syntax.prefix.q) is only valid in a tuple inside a `with prefix` block") # pragma: no cover, not meant to hit the expander + raise SyntaxError("u (unpythonic.syntax.prefix.u) is a name macro only") # pragma: no cover + raise SyntaxError("u (unpythonic.syntax.prefix.u) is only valid in a tuple inside a `with prefix` block") # pragma: no cover, not meant to hit the expander # TODO: This isn't a perfect solution, because there is no "call" macro kind. # TODO: We currently trigger the error on any appearance of the name `kw` outside a valid context. @@ -144,6 +146,8 @@ def kw(tree, *, syntax, **kw): # noqa: F811 # -------------------------------------------------------------------------------- def _prefix(block_body): + # TODO: Should change these to query the expander to allow renaming by as-imports. + # TODO: How to do that can be found in the implementation of `quicklambda`. isquote = lambda tree: getname(tree, accept_attr=False) == "q" isunquote = lambda tree: getname(tree, accept_attr=False) == "u" iskwargs = lambda tree: type(tree) is Call and getname(tree.func, accept_attr=False) == "kw" @@ -189,11 +193,7 @@ def transform(self, tree): # Expr # Subscript if type(tree) is Subscript: - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - body = tree.slice - else: - body = tree.slice.value - + body = tree.slice if type(body) is Tuple: # Skip the transformation of the expr tuple itself, but transform its elements. # This skips the transformation of the macro argument tuple, too, because diff --git a/unpythonic/syntax/scopeanalyzer.py b/unpythonic/syntax/scopeanalyzer.py index 50b16c2f..6aaa7463 100644 --- a/unpythonic/syntax/scopeanalyzer.py +++ b/unpythonic/syntax/scopeanalyzer.py @@ -71,12 +71,16 @@ "scoped_transform", "get_lexical_variables", "get_names_in_store_context", - "get_names_in_del_context"] + "get_names_in_del_context", + "extract_args", + "collect_globals", + "collect_nonlocals"] from ast import (Name, Tuple, Lambda, FunctionDef, AsyncFunctionDef, ClassDef, Import, ImportFrom, Try, ListComp, SetComp, GeneratorExp, DictComp, Store, Del, Global, Nonlocal) +from mcpyrate.astcompat import TryStar, MatchStar, MatchMapping, MatchAs from mcpyrate.core import Done from mcpyrate.walkers import ASTTransformer, ASTVisitor @@ -212,35 +216,15 @@ def get_lexical_variables(tree, collect_locals=True): raise TypeError(f"Expected a tree representing a lexical scope, got {type(tree)}") if type(tree) in (Lambda, FunctionDef, AsyncFunctionDef): - a = tree.args - allargs = a.args + a.kwonlyargs - if hasattr(a, "posonlyargs"): # Python 3.8+: positional-only arguments - allargs += a.posonlyargs - argnames = [x.arg for x in allargs] - if a.vararg: - argnames.append(a.vararg.arg) - if a.kwarg: - argnames.append(a.kwarg.arg) - + argnames = extract_args(tree) fname = [] localvars = [] nonlocals = [] if type(tree) in (FunctionDef, AsyncFunctionDef): fname = [tree.name] - if collect_locals: localvars = list(uniqify(get_names_in_store_context(tree.body))) - - class NonlocalsCollector(ASTVisitor): - def examine(self, tree): - if type(tree) in (Global, Nonlocal): - for x in tree.names: - self.collect(x) - if not isnewscope(tree): - self.generic_visit(tree) - nc = NonlocalsCollector() - nc.visit(tree.body) - nonlocals = nc.collected + nonlocals = collect_nonlocals(tree.body) + collect_globals(tree.body) return list(uniqify(fname + argnames + localvars)), list(uniqify(nonlocals)) @@ -306,8 +290,8 @@ def get_names_in_store_context(tree): This includes: - - Any ``Name`` in store context (such as on the LHS of an `Assign` - or `NamedExpr` node) + - Any ``Name`` in store context (such as on the LHS of an `Assign`, + `NamedExpr` (Python 3.8+), `TypeAlias` (Python 3.12+)) - The name of ``FunctionDef``, ``AsyncFunctionDef`` or``ClassDef`` @@ -317,8 +301,12 @@ def get_names_in_store_context(tree): - The exception name of any ``except`` handlers + - The exception name of any ``except*`` handlers (Python 3.11+) + - The names in the as-part of ``With`` + - The names bound in `match`/`case` patterns (Python 3.10+) + Duplicates may be returned; use ``set(...)`` or ``list(uniqify(...))`` on the output to remove them. @@ -349,7 +337,7 @@ def examine(self, tree): elif type(tree) in (Import, ImportFrom): for x in tree.names: self.collect(x.asname if x.asname is not None else x.name) - elif type(tree) is Try: + elif type(tree) in (Try, TryStar): # Python 3.11+: `try`/`except*` # https://docs.python.org/3/reference/compound_stmts.html#the-try-statement # # TODO: The `err` in `except SomeException as err` is only bound within the `except` block, @@ -361,13 +349,30 @@ def examine(self, tree): # TODO: `try`, even inside the `except` blocks, will be bound in the whole parent scope. for h in tree.handlers: self.collect(h.name) + # Python 3.10+: `match`/`case` captures are `MatchAs(name='x')` and + # `MatchStar(name='rest')` with bare strings (not `Name` nodes). The `name` + # is `None` for `_` (wildcard, doesn't capture). `Name` nodes in patterns are + # class references (e.g. `Point` in `case Point(x, y):`), not captures. + # + # `generic_visit` handles most match patterns automatically, since `MatchAs` + # and `MatchStar` nodes appear as children. The one exception is + # `MatchMapping.rest`, which is a bare string attribute (not an AST child). + elif type(tree) in (MatchAs, MatchStar): + if tree.name is not None: + self.collect(tree.name) + elif type(tree) is MatchMapping: + if tree.rest is not None: # `**rest` capture + self.collect(tree.rest) + + # Python 3.12+: `TypeAlias` uses a name in `Store` context on its LHS so it needs no special handling here. + # Same note as for for loops. # elif type(tree) in (With, AsyncWith): # for item in tree.items: # if item.optional_vars is not None: # self._collect_name_or_list(item.optional_vars) # macro-created nodes might not have a ctx, but our macros don't create lexical assignments. - if type(tree) is Name and hasattr(tree, "ctx") and type(tree.ctx) is Store: + if type(tree) is Name and type(getattr(tree, "ctx", None)) is Store: self.collect(tree.id) if not isnewscope(tree): self.generic_visit(tree) @@ -386,13 +391,61 @@ class DelNamesCollector(ASTVisitor): def examine(self, tree): # We want to detect things like "del x": # Delete(targets=[Name(id='x', ctx=Del()),]) - # We don't currently care about "del myobj.x" or "del mydict['x']" (these examples in Python 3.6): + # We don't currently care about "del myobj.x" or "del mydict['x']" (these old examples in Python 3.6): # Delete(targets=[Attribute(value=Name(id='myobj', ctx=Load()), attr='x', ctx=Del()),]) # Delete(targets=[Subscript(value=Name(id='mydict', ctx=Load()), slice=Index(value=Str(s='x')), ctx=Del()),]) - if type(tree) is Name and hasattr(tree, "ctx") and type(tree.ctx) is Del: + if type(tree) is Name and type(getattr(tree, "ctx", None)) is Del: self.collect(tree.id) if not isnewscope(tree): self.generic_visit(tree) nc = DelNamesCollector() nc.visit(tree) return nc.collected + +def extract_args(tree): + """Extract the parameter names from a `Lambda`, `FunctionDef`, or `AsyncFunctionDef` node. + + Return a `list` of bare `str`. + """ + if type(tree) not in (Lambda, FunctionDef, AsyncFunctionDef): + raise ValueError(f"Expected a function definition AST node, got {tree}") + a = tree.args + allargs = a.posonlyargs + a.args + a.kwonlyargs + argnames = [x.arg for x in allargs] + if a.vararg: + argnames.append(a.vararg.arg) + if a.kwarg: + argnames.append(a.kwarg.arg) + return argnames + +def collect_globals(tree): + """Collect the names of all names declared `global` in `tree`, stopping at scope boundaries. + + Return a `list` of bare `str`. + """ + class GlobalsCollector(ASTVisitor): + def examine(self, tree): + if type(tree) is Global: + for name in tree.names: + self.collect(name) + if not isnewscope(tree): + self.generic_visit(tree) + collector = GlobalsCollector() + collector.visit(tree) + return collector.collected + +def collect_nonlocals(tree): + """Collect the names of all names declared `nonlocal` in `tree`, stopping at scope boundaries. + + Return a `list` of bare `str`. + """ + class NonlocalsCollector(ASTVisitor): + def examine(self, tree): + if type(tree) is Nonlocal: + for name in tree.names: + self.collect(name) + if not isnewscope(tree): + self.generic_visit(tree) + collector = NonlocalsCollector() + collector.visit(tree) + return collector.collected diff --git a/unpythonic/syntax/tailtools.py b/unpythonic/syntax/tailtools.py index 981903dd..7a1d6742 100644 --- a/unpythonic/syntax/tailtools.py +++ b/unpythonic/syntax/tailtools.py @@ -5,27 +5,27 @@ __all__ = ["autoreturn", "tco", - "continuations", "call_cc"] + "continuations", "call_cc", "get_cc", "iscontinuation"] from functools import partial -from ast import (Lambda, FunctionDef, AsyncFunctionDef, +from ast import (Lambda, FunctionDef, AsyncFunctionDef, ClassDef, arguments, arg, keyword, List, Tuple, Call, Name, Starred, Constant, BoolOp, And, Or, - With, AsyncWith, If, IfExp, Try, Assign, Return, Expr, + With, AsyncWith, If, IfExp, Try, Match, Assign, Return, Expr, + Await, copy_location) -import sys from mcpyrate.quotes import macros, q, u, n, a, h # noqa: F401 from mcpyrate import gensym +from mcpyrate.astcompat import TryStar from mcpyrate.quotes import capture_as_macro, is_captured_value from mcpyrate.utils import NestingLevelTracker from mcpyrate.walkers import ASTTransformer, ASTVisitor -from .astcompat import getconstant, NameConstant from .ifexprs import aif, it from .letdoutil import isdo, islet, ExpandedLetView, ExpandedDoView from .util import (isx, isec, @@ -38,7 +38,6 @@ from ..fun import identity from ..funutil import Values from ..it import uniqify -from ..lazyutil import force1, passthrough_lazy_args from ..tco import trampolined, jump # In `continuations`, we use `aif` and `it` as hygienically captured macros. @@ -207,7 +206,7 @@ def oddp(x): def result(ec): ... - # use directly on a literal lambda + # use directly on a literal lambda (effectively, as a decorator) result = call_ec(lambda ec: ...) When macro expansion of the ``with tco`` block starts, names of escape @@ -348,6 +347,13 @@ def myfunc(a, b, cc): Inside a ``with continuations:`` block, the ``call_cc[]`` statement captures a continuation. (It is actually a macro, for technical reasons.) + Capturing a continuation introduces a scope boundary. The continuation + captured by `call_cc` (i.e. the rest of the function body after the + `call_cc` statement) is a new scope, and the assignment part of the + `call_cc` statement takes effect in that new scope. Under the hood, + the assignment from the `call_cc` is implemented as function parameters; + the continuation is a function. + For various possible program topologies that continuations may introduce, see the clarifying pictures under ``doc/`` in the source distribution. @@ -668,28 +674,49 @@ def transform(self, tree): if is_captured_value(tree): return tree # don't recurse! if type(tree) in (FunctionDef, AsyncFunctionDef): - tree.body[-1] = transform_tailstmt(tree.body[-1]) + newtail = TailStatementTransformer().visit(tree.body[-1]) + if isinstance(newtail, list): # replaced by more than one statement? + tree.body = tree.body[:-1] + newtail + else: + tree.body[-1] = newtail return self.generic_visit(tree) - def transform_tailstmt(tree): - # TODO: For/AsyncFor/While? - if type(tree) is If: - tree.body[-1] = transform_tailstmt(tree.body[-1]) - if tree.orelse: - tree.orelse[-1] = transform_tailstmt(tree.orelse[-1]) - elif type(tree) in (With, AsyncWith): - tree.body[-1] = transform_tailstmt(tree.body[-1]) - elif type(tree) is Try: - # We don't care about finalbody; typically used for unwinding only. - if tree.orelse: # tail position is in else clause if present - tree.orelse[-1] = transform_tailstmt(tree.orelse[-1]) - else: # tail position is in the body of the "try" - tree.body[-1] = transform_tailstmt(tree.body[-1]) - # additionally, tail position is in each "except" handler - for handler in tree.handlers: - handler.body[-1] = transform_tailstmt(handler.body[-1]) - elif type(tree) is Expr: - tree = Return(value=tree.value) - return tree + + class TailStatementTransformer(ASTTransformer): + def transform(self, tree): + # TODO: For/AsyncFor/While? + if type(tree) is If: + tree.body[-1] = self.visit(tree.body[-1]) + if tree.orelse: + tree.orelse[-1] = self.visit(tree.orelse[-1]) + elif type(tree) in (With, AsyncWith): + tree.body[-1] = self.visit(tree.body[-1]) + elif type(tree) in (Try, TryStar): # Python 3.11+: `try`/`except*` + # We don't care about finalbody; typically used for unwinding only. + if tree.orelse: # tail position is in else clause if present + tree.orelse[-1] = self.visit(tree.orelse[-1]) + else: # tail position is in the body of the "try" + tree.body[-1] = self.visit(tree.body[-1]) + # additionally, tail position is in each "except" handler + for handler in tree.handlers: + handler.body[-1] = self.visit(handler.body[-1]) + elif type(tree) is Match: # Python 3.10+: `match`/`case` + for case in tree.cases: + if case.body: + case.body[-1] = self.visit(case.body[-1]) + elif type(tree) in (FunctionDef, AsyncFunctionDef, ClassDef): # v0.15.0+ + # If the item in tail position is a named function definition + # or a class definition, it binds a name - that of the function/class. + # Return that object. + with q as quoted: + with a: + tree + return n[tree.name] + tree = quoted + elif type(tree) is Expr: # expr -> return expr + with q as quoted: + return a[tree.value] + tree = quoted[0] + return tree # This macro expands outside-in. Any nested macros should get clean standard Python, # not having to worry about implicit "return" statements. return AutoreturnTransformer().visit(block_body) @@ -701,7 +728,7 @@ def _tco(block_body): userlambdas = detect_lambda(block_body) known_ecs = list(uniqify(detect_callec(block_body))) - block_body = dyn._macro_expander.visit(block_body) + block_body = dyn._macro_expander.visit_recursively(block_body) # second pass, inside-out transform_retexpr = partial(_transform_retexpr) @@ -738,7 +765,6 @@ def chain_conts(cc1, cc2, with_star=False): # cc1=_pcc, cc2=cc """Internal function, used in code generated by the continuations macro.""" if with_star: # to be chainable from a tail call, accept a multiple-values arglist if cc1 is not None: - @passthrough_lazy_args def cc(*rets, **kwrets): return jump(cc1, cc=cc2, *rets, **kwrets) else: @@ -749,32 +775,13 @@ def cc(*rets, **kwrets): cc = cc2 else: # for inert data value returns (this produces the multiple-values arglist) if cc1 is not None: - @passthrough_lazy_args def cc(return_value): - # Return values are never implicitly lazy in `unpythonic`, - # so why we need to `force1` here requires a comment. - # - # In general, we should treat these `cc` functions as lazy, - # so they won't force their args. Those args here are a return value, - # but due to `continuations`, it's not just a return, but a call - # into the `cc` function. - # - # Thus, returning a `Values` from a continuation-enabled function, - # that `Values` ends up here (or in the other branch, with no `cc1`). - # Because it's *technically* an argument for a lazy function, it gets - # a `lazy[]` wrapper added by `with lazify`. - # - # To determine whether we have one or multiple return values, we must - # force that wrapper promise, without touching anything inside. - return_value = force1(return_value) if isinstance(return_value, Values): return jump(cc1, cc=cc2, *return_value.rets, **return_value.kwrets) else: return jump(cc1, return_value, cc=cc2) else: - @passthrough_lazy_args def cc(return_value): - return_value = force1(return_value) if isinstance(return_value, Values): return jump(cc2, *return_value.rets, **return_value.kwrets) else: @@ -790,7 +797,7 @@ class CallCcMarker(ContinuationsMarker): """AST marker denoting a `call_cc[]` invocation.""" -def _continuations(block_body): +def _continuations(block_body): # here be dragons. # This is a very loose pythonification of Paul Graham's continuation-passing # macros in On Lisp, chapter 20. # @@ -802,7 +809,7 @@ def _continuations(block_body): known_ecs = list(uniqify(detect_callec(block_body))) with _continuations_level.changed_by(+1): - block_body = dyn._macro_expander.visit(block_body) + block_body = dyn._macro_expander.visit_recursively(block_body) # second pass, inside-out @@ -886,31 +893,111 @@ def data_cb(tree): # transform an inert-data return value into a tail-call to c # specified inside the body of the macro invocation like PG's solution does. # Instead, we capture as the continuation all remaining statements (i.e. # those that lexically appear after the ``call_cc[]``) in the current block. - def iscallcc(tree): + def iscallccstatement(tree): if type(tree) not in (Assign, Expr): return False return isinstance(tree.value, CallCcMarker) - def split_at_callcc(body): + # owner: FunctionDef node, or `None` if the use site of the `call_cc` is not inside a function + def split_at_callcc(owner, body): if not body: return [], None, [] before, after = [], body while True: stmt, *after = after - if iscallcc(stmt): + if iscallccstatement(stmt): # after is always non-empty here (has at least the explicitified "return") # ...unless we're at the top level of the "with continuations" block if not after: raise SyntaxError("call_cc[] cannot appear as the last statement of a 'with continuations' block (no continuation to capture)") # pragma: no cover - # TODO: To support Python's scoping properly in assignments after the `call_cc`, - # TODO: we have to scan `before` for assignments to local variables (stopping at - # TODO: scope boundaries; use `unpythonic.syntax.scoping.get_names_in_store_context`, - # TODO: and declare those variables `nonlocal` in `after`. This way the binding - # TODO: will be shared between the original context and the continuation. - # See Politz et al 2013 (the "full monty" paper), section 4.2. + # after = patch_scoping(owner, before, stmt, after) # bad idea, DON'T DO THIS return before, stmt, after before.append(stmt) if not after: return before, None, [] + # Try to maintain an illusion of Python's standard scoping rules across the split + # into the parent context (`before`) and continuation closure (`after`). + # See Politz et al 2013 (the "full monty" paper), section 4.2. + # + # TODO: On second thought, this is a bad idea, DON'T DO THIS. + # + # The function `patch_scoping` is an experiment that implements propagation + # of the scope of variable definitions from the parent scope into the continuation, + # recursively. But: + # + # - Due to how the continuation machinery works, the continuation's + # parameters (assignment targets of the `call_cc`) **must** shadow + # the same names from the parent scope, if they happen to exist there. + # + # - There is no propagation from the continuation up the parent scope + # chain. That is, if a continuation declares a new local variable, the + # name won't become available to any of the parent contexts, even if + # those are part of the same original function (to which the + # continuation splitting was applied). Implementing this would require + # a second pass. + # + # - Without looking at the source code of the full module, it is not even + # possible to determine whether the top level of the with continuations + # block is inside a function or not. This has implications to `call_cc` + # invoked from the top level of the block: should the variables from + # the parent scope be declared `nonlocal` or `global`? + # + # It is much simpler and much more robust to just document that introducing a + # continuation introduces a scope boundary - that is a simple, transparent rule + # that is easy to work with. The behavior is no worse than how, in standard Python, + # comprehensions and generator expressions introduce a scope boundary. + # + # owner: FunctionDef node, or `None` if the use site of the `call_cc` is not inside a function + # def patch_scoping(owner, before, callcc, after): + # # Determine the names of all variables that should be made local to the continuation function. + # # In the unexpanded code, the continuation doesn't look like a new scope, so by appearances, + # # these will effectively break the usual scoping rules. Thus this set should be kept minimal. + # # To allow the machinery to actually work, at least the parameters of the continuation function + # # *must* be allowed to shadow names from the parent scope. + # targets, starget, ignored_condition, ignored_thecall, ignored_altcall = analyze_callcc(callcc) + # if not targets and not starget: + # targets = ["_ignored_arg"] # this must match what `make_continuation` does, below + # # The assignment targets of the `call_cc` become parameters of the continuation function. + # # Furthermore, a continuation function generated by `make_continuation` always takes + # # the `cc` and `_pcc` parameters. + # afterargs = targets + ([starget] or []) + ["cc", "_pcc"] + # afterlocals = afterargs + # + # if owner: + # # When `call_cc` is used inside a function, local variables of the + # # parent function (including parameters) become nonlocals in the + # # continuation. + # # + # # But only those that are not also locals of the continuation! + # # In that case, the local variable of the continuation overrides. + # # Locals of the continuation include its arguments, and any names in store context. + # beforelocals = set(extract_args(owner) + get_names_in_store_context(before)) + # afternonlocals = list(beforelocals.difference(afterlocals)) + # if afternonlocals: # TODO: Python 3.8: walrus assignment + # after.insert(0, Nonlocal(names=afternonlocals)) + # else: + # # When `call_cc` is used at the top level of `with continuations` block, + # # the variables at that level become globals in the continuation. + # # + # # TODO: This **CANNOT** always work correctly, because we would need to know + # # TODO: whether the `with continuations` block itself is inside a function or not. + # # TODO: So we just assume it's outside any function. + # beforelocals = set(get_names_in_store_context(before)) + # afternonlocals = list(beforelocals.difference(afterlocals)) + # if afternonlocals: # TODO: Python 3.8: walrus assignment + # after.insert(0, Global(names=afternonlocals)) + # + # # Nonlocals of the parent function remain nonlocals in the continuation. + # # When `owner is None`, `beforenonlocals` will be empty. + # beforenonlocals = collect_nonlocals(before) + # if beforenonlocals: # TODO: Python 3.8: walrus assignment + # after.insert(0, Nonlocal(names=beforenonlocals)) + # + # # Globals of parent are also globals in the continuation. + # beforeglobals = collect_globals(before) + # if beforeglobals: # TODO: Python 3.8: walrus assignment + # after.insert(0, Global(names=beforeglobals)) + # + # return after # we mutate; return it just for convenience # TODO: To support named return values (`kwrets` in a `Values` object) from the `call_cc`'d function, # TODO: we need to change the syntax to something that allows us to specify which names are meant to # TODO: capture the positional return values, and which ones the named return values. Doing so will @@ -949,14 +1036,14 @@ def maybe_starred(expr): # return [expr.id] or set starget raise SyntaxError(f"call_cc[]: expected an assignment or a bare expr, got {stmt}") # pragma: no cover # extract the function call(s) if not isinstance(stmt.value, CallCcMarker): # both Assign and Expr have a .value - assert False # we should get only valid call_cc[] invocations that pass the `iscallcc` test # pragma: no cover + assert False # we should get only valid call_cc[] invocations that pass the `iscallccstatement` test # pragma: no cover theexpr = stmt.value.body # discard the AST marker - if not (type(theexpr) in (Call, IfExp) or (type(theexpr) in (Constant, NameConstant) and getconstant(theexpr) is None)): + if not (type(theexpr) in (Call, IfExp) or (type(theexpr) is Constant and theexpr.value is None)): raise SyntaxError("the bracketed expression in call_cc[...] must be a function call, an if-expression, or None") # pragma: no cover def extract_call(tree): if type(tree) is Call: return tree - elif type(tree) in (Constant, NameConstant) and getconstant(tree) is None: + elif type(tree) is Constant and tree.value is None: return None else: raise SyntaxError("call_cc[...]: expected a function call or None") # pragma: no cover @@ -968,6 +1055,7 @@ def extract_call(tree): condition = altcall = None thecall = extract_call(theexpr) return targets, starget, condition, thecall, altcall + # owner: FunctionDef node, or `None` if the use site of the `call_cc` is not inside a function def make_continuation(owner, callcc, contbody): targets, starget, condition, thecall, altcall = analyze_callcc(callcc) @@ -1025,21 +1113,24 @@ def prepare_call(tree): body=q[n["cc"]], orelse=non) contarguments = arguments(args=[arg(arg=x) for x in targets], + posonlyargs=[], kwonlyargs=[arg(arg="cc"), arg(arg="_pcc")], vararg=(arg(arg=starget) if starget else None), kwarg=None, defaults=posargdefaults, kw_defaults=[q[h[identity]], maybe_capture]) - if sys.version_info >= (3, 8, 0): # Python 3.8+: positional-only arguments - contarguments.posonlyargs = [] funcdef = FDef(name=contname, args=contarguments, body=contbody, decorator_list=[], # patched later by transform_def returns=None) # return annotation not used here - # in the output stmts, define the continuation function... - newstmts = [funcdef] + # 0.15.1: tag the continuation function as a continuation, for introspection. + setcontflag = Assign(targets=[q[n[f"{contname}.is_continuation"]]], + value=q[True]) + + # in the output stmts, define the continuation function, set its is-continuation flag, ... + newstmts = [funcdef, setcontflag] if owner: # ...and tail-call it (if currently inside a def) def jumpify(tree): tree.args = [tree.func] + tree.args @@ -1067,30 +1158,45 @@ def transform(self, tree): if type(tree) in (FunctionDef, AsyncFunctionDef): tree.body = transform_callcc(tree, tree.body) return self.generic_visit(tree) + # owner: FunctionDef node, or `None` if the use site of the `call_cc` is not inside a function def transform_callcc(owner, body): # owner: FunctionDef or AsyncFunctionDef node, or None (top level of block) # body: list of stmts # we need to consider only one call_cc in the body, because each one # generates a new nested def for the walker to pick up. - before, callcc, after = split_at_callcc(body) + before, callcc, after = split_at_callcc(owner, body) if callcc: body = before + make_continuation(owner, callcc, contbody=after) return body # TODO: improve error reporting for stray call_cc[] invocations class StrayCallccChecker(ASTVisitor): def examine(self, tree): - if iscallcc(tree): - raise SyntaxError("call_cc[...] only allowed at the top level of a def or async def, or at the top level of the block; must appear as an expr or an assignment RHS") # pragma: no cover + if iscallccstatement(tree): + raise SyntaxError("call_cc[...] only allowed at the top level of a def, or at the top level of the block; must appear as an expr or an assignment RHS") # pragma: no cover if type(tree) in (Assign, Expr): v = tree.value if type(v) is Call and type(v.func) is Name and v.func.id == "call_cc": raise SyntaxError("call_cc(...) should be call_cc[...] (note brackets; it's a macro)") # pragma: no cover self.generic_visit(tree) + # TODO: Interaction of `continuations` with async functions is not implemented. + # So for robustness, we raise a syntax error for now. + class AsyncDefChecker(ASTVisitor): + def examine(self, tree): + if type(tree) is AsyncFunctionDef: + raise SyntaxError("`with continuations` does not currently support `async` functions") + elif type(tree) is AsyncWith: + raise SyntaxError("`with continuations` does not currently support `async` context managers") + elif type(tree) is Await: + raise SyntaxError("`with continuations` does not currently support `await`") + self.generic_visit(tree) + # ------------------------------------------------------------------------- # Main processing logic begins here # ------------------------------------------------------------------------- + AsyncDefChecker().visit(block_body) + # Disallow return at the top level of the block, because it would behave # differently depending on whether placed before or after the first call_cc[] # invocation. (Because call_cc[] internally creates a function and calls it.) @@ -1145,6 +1251,224 @@ def transform(self, tree): # (needed to support continuations in the Lispython dialect, since it applies tco globally.) return ExpandedContinuationsMarker(body=new_block_body) +def iscontinuation(x): + """Return whether the object `x` is a continuation function. + + This function can be used for inspection at run time. + + Continuation functions are created by `call_cc[...]` in a `with continuations` block. + """ + return callable(x) and hasattr(x, "is_continuation") and x.is_continuation + +# TODO: Do we need to account for `_pcc` here? Probably not, since this is defined at the +# TODO: top level of a module, not as a closure inside another function. +@trampolined +def get_cc(*args, cc): + """When used together with `call_cc[]`, capture and get the current continuation. + + This convenience function covers the common use case when working with + continuations, when you just want to snapshot the control state into a + local variable. + + In other words, this is what you want 99% of the time when you need `call_cc`. + + Or in yet other words, `get_cc` is the less antisocial little sister of `call_cc` + from an alternate timeline, and in this adventure the two work as a team. + + The `*args`, if any, are passed through. + + Usage:: + + with continuations: + ... + def dostuff(): + ... + + k = call_cc[get_cc()] + + # Now `k` is the continuation from this point on. + # You can do whatever you want with it! + # + # To invoke it, `k(k)` to always preserve the meaning + # of `k` in this part of the code. (See below.) + + ... + return k # maybe our caller wants to replay part of us later + + Any positional `*args` are passed through, so that you can also make a + continuation that takes additional arguments:: + + def domorestuff(): + ... + + k, x1, x2 = call_cc[get_cc(1, 2)] # -> k=cc, x1=1, x2=2 + + print(x1, x2) + return k + + k = domorestuff() + k(3, 4) + k(x1=3, x2=4) # same thing + + Important: in the `get_cc` call, the initial values for the additional + arguments, if any, must be passed positionally, due to `call_cc` syntax + limitations. However, when invoking the continuation, they can be passed + any way you want. + + As for how this works, you may have seen the following helper function + in Matthew Might's article on continuations by example: + + (define (current-continuation) + (call/cc (lambda (cc) (cc cc)))) + + The lambda is pretty much `get_cc`. We cannot factor away the `call/cc`, + because our `call_cc` is a macro that arranges for the actual capture to + happen at its use site (and it cannot affect any outer levels of the call + stack). + + + **CAUTION**: + + In `k = call_cc[get_cc()]`, the continuation is automatically assigned to + `k` only during the first run, i.e. (in the example) whenever `dostuff` is + called normally. + + By the rules of `unpythonic.syntax.call_cc`, the continuation function will + have parameters for whatever is on the left-hand side of the assignment; in + this case, there will be one parameter, `k`. + + When you invoke the continuation later, the name `k` inside the continuation + (i.e. in the code below the `call_cc` line) will point to whatever value you + sent into the continuation as its argument. + + To achieve least surprise, in 99% of cases, one should arrange things so that + in the continuation, the name `k` always actually points to the continuation, + no matter whether the code runs normally or via continuation invocation. + + Thus, unless there is a specific reason to do otherwise, the recommended way + to invoke the continuation is `k(k)` (giving it itself as the argument). + + Note this caution applies to any continuation that expects to take itself + as an argument; the `k = call_cc[get_cc()]` pattern is just a convenient + way to create such continuations. + + + **Comparison to Lisps**: + + The `k = call_cc[get_cc()]` pattern was inspired by The One True Way to use + `call/cc` in Lisp dialects that have multi-shot continuations, as well as the + `let/cc` construct in Racket. + + The One True Way is to use a one-argument lambda that is invoked immediately + by the `call/cc`: + + (define dostuff () + ... + (call/cc (lambda (k) + ;; ...now k is the continuation... + ... + k))) ;; return it just for the lulz + + The name `call/cc` (`call-with-current-continuation`) is a misnomer; the + purpose of the construct is not really to call a reusable function defined + somewhere else; used that way, it may seem an esoteric feature primarily + intended to confuse programmers. Instead, when combined with a lexical closure + as above, it exposes the continuation as a local variable - which is a + clean and useful technique for a variety of purposes (custom escapes, + generators, backtracking, ...). + + Racket abstracts this pattern into `let/cc`, which communicates the intent + more clearly: + + (define dostuff () + ... + (let/cc k + ;; ...now k is the continuation... + ... + k)) ;; return it just for the lulz + + (Racket has no `return` keyword - it does not need one, since you can + create one using `(let/cc return ...)`, scoping it to whichever block + you want.) + + In the Lisp examples above, `k` is the continuation starting with the next + expression after the `call/cc` or `let/cc` block (expression). + + In our `k = call_cc[get_cc()]` pattern, `k` is the rest of the function body + after the statement `k = call_cc[get_cc()]`. + + So in Lisps, invoking `k` inside the block performs an exit (think of a Python + `return` from that block), whereas in our implementation, doing so loops back + to the next statement just after the `call_cc`. + + There is a similarity between our `get_cc` and something that is possible + in Lisps: our continuation starts from the next statement that runs after + `k = call_cc[get_cc()]`. This is exactly how the `(current-continuation)` + function, mentioned at the beginning, works. + + + **Why `get_cc`?**: + + In Python, a function using all the features of the language cannot be + defined in an expression, so in most cases the (un)pythonic `call_cc` + must indeed call a function defined somewhere else. + + The question becomes, what should this function be? + + 1. To be useful at all, it should make it easier to program with continuations, + over arbitrary use of `call_cc`. + + 2. To promote a standard usage pattern, the function should be as general as + possible, so that we only ever need one. + + 3. For least surprise, the function should do as little as possible; + particularly, no side effects. + + 4. For familiarity, we should stay as close to The One True Way pattern as + possible. In the pattern, the lambda converts the call into a let-like + construct, which pythonifies into an assignment, `k = call_cc[...]`. + + 5. The only reason to use `call_cc` is when you want to get the continuation. + + The obvious solution is a function that just passes the continuation as an + argument into that very same continuation, without any side effects; this is + exactly what `get_cc` does. Thus we get the pattern `k = call_cc[get_cc()]`, + which arguably does exactly what it says on the tin. + """ + # If `get_cc` was defined inside a `with continuations` block, the definition + # could be just: + # + # def get_cc(*, cc): + # return cc + # + # because that means "send the value `cc` into the current continuation" + # (i.e. "escape into the current continuation with the value `cc`"), and + # `cc` is the current continuation. For a more detailed analysis in Scheme: + # + # https://stackoverflow.com/questions/57663699/returning-continuations-from-call-cc + # + # Since `get_cc` is not defined inside a `with continuations` block (so that + # we can easily provide it in the same module that defines the continuation + # machinery, without using multiphase compilation), we make the actual definition + # essentially as a handcrafted macro expansion. + # + # So when returning, we are expected to tail-call (i.e. TCO-jump into) the + # continuation function that was given to us, with our return value(s) becoming + # its argument(s). + # + # Below the first `cc` is the continuation function, and the second `cc` + # is the return value that we are sending into it. + # + # The `*args` are a passthrough so that e.g. `k, a, b = call_cc[get_cc(1, 2)]`; + # allows you to pass parameters into the continuation later. + # + # One often sees the pattern `(cc cc)` also in Lisps; for example, see + # the function `(current-continuation)` in Matthew Might's article on + # continuations by example: + # http://matt.might.net/articles/programming-with-continuations--exceptions-backtracking-search-threads-generators-coroutines/ + # + return jump(cc, cc, *args) + # ----------------------------------------------------------------------------- def _tco_transform_def(tree, *, preproc_cb): diff --git a/unpythonic/syntax/testingtools.py b/unpythonic/syntax/testingtools.py index 4d7edb9b..48fba902 100644 --- a/unpythonic/syntax/testingtools.py +++ b/unpythonic/syntax/testingtools.py @@ -4,7 +4,7 @@ See also `unpythonic.test.fixtures` for the high-level machinery. """ -__all__ = ["the", "test", +__all__ = ["the", "expect", "test", "test_signals", "test_raises", "fail", "error", "warn", "expand_testing_macros_first", @@ -20,7 +20,7 @@ from mcpyrate.walkers import ASTTransformer from ast import Tuple, Subscript, Name, Call, copy_location, Compare, arg, Return, parse, Expr, AST -import sys +import warnings from ..dynassign import dyn from ..env import env @@ -57,6 +57,7 @@ def the(tree, **kw): test[lower_limit < the[computeitem(...)]] test[lower_limit < the[computeitem(...)] < upper_limit] test[myconstant in the[computeset(...)]] + test["green tea" == the[vert]] especially if you need to capture several subexpressions:: @@ -64,18 +65,55 @@ def the(tree, **kw): Note the above rules mean that if there is just one interesting subexpression, and it is the leftmost term of a comparison, `the[...]` - is optional, although allowed (to explicitly document intent). + is redundant, although allowed (to explicitly document intent). These have the same effect:: test[the[computeitem(...)] in myitems] test[computeitem(...) in myitems] + **Common mistakes.** All of these come from marking the wrong thing, or + from not marking where a mark was needed. They fail the same way: the test + still passes or fails correctly, but the failure message reports something + useless:: + + test[the["X" in out]] # captures the *boolean*, so `out` is never shown + test[the[x == 42]] # same shape, same problem + test[4 in mycollection] # auto-capture takes the LHS - a literal, so useless + test[the[a] < b < c] # chained: only `a` captured, `b` and `c` invisible + + The fixes are, respectively: mark the term you actually want to see, drop + the mark and let auto-capture take the LHS, mark the collection rather than + leaning on auto-capture, and mark every term whose value would be worth + having:: + + test["X" in the[out]] + test[x == 42] + test[4 in the[mycollection]] + test[the[a] < the[b] < the[c]] + + Note the first and third are the same error approached from opposite sides - + over-marking and under-marking - and have the same fix. + + The question to ask is *what value would I want to see if this failed?*, then + mark that. The answer is sometimes the container rather than the leaf: in + ``test[the[response]["status"] == "ok"]``, auto-capture would report only + ``"failed"``, which is true and useless, whereas marking ``response`` shows + the whole dict. + The `the[...]` mark passes the value through, and does not affect the evaluation order of user code. A `test[...]` may have multiple `the[...]`; the captured values are gathered in a list that is shown upon test failure. + Trivial captures are skipped: a capture whose source code reads the same as + the `repr` of its value tells you nothing the assertion does not already + show, so it is dropped and the value of the whole expression is reported + instead. This is what happens with literals, and `test[4 in mycollection]` + above is why the rule exists - the auto-captured `4` would otherwise report + `4 = 4`. The suppression keeps the message honest, but the collection is + still not shown; only marking it does that. + In case of nested tests, each `the[...]` is understood as belonging to the lexically innermost surrounding test. @@ -83,6 +121,36 @@ def the(tree, **kw): """ raise SyntaxError("the[] is only meaningful inside a `test[]` or in a `with test` block") # pragma: no cover, not meant to hit the expander +def expect(tree, **kw): + """[syntax, expr] In a `with test` block, declare the expression whose value is checked. + + Only meaningful as a statement at the top level of a `with test:` (or + `with test[message]:`) block. Use exactly once per block:: + + with test: + x = compute() + expect[x > 0] + + with test["the answer"]: + answer = compute_answer() + expect[answer == 42] + + **Added in v2.2.0**. + + `expect[]` tells the test framework which expression's value should be + checked. Each `with test:` block declares its tested expression exactly + once, by exactly one form: either `expect[expr]` or `return expr`, not + both. The older `return expr` form continues to work but emits a + `DeprecationWarning`; it will be un-hijacked in a future major release + so that `return` inside `with test:` regains its standard Python meaning. + + The "result" capture rules apply to the expression inside `expect[]`, + just as they did to `return expr`. `the[]` marks may appear inside + `expect[]` (or anywhere else in the block); when none are present and + the expression is a comparison, the leftmost term is implicitly captured. + """ + raise SyntaxError("expect[] is only meaningful as a statement at the top level of `with test:`") # pragma: no cover, not meant to hit the expander + @parametricmacro def test(tree, *, args, syntax, expander, **kw): # noqa: F811 """[syntax, expr/block] Make a test assertion. For writing automated tests. @@ -158,32 +226,44 @@ def test(tree, *, args, syntax, expander, **kw): # noqa: F811 with test: body0 ... - return expr # optional + expect[expr] # optional with test[message]: body0 ... - return expr # optional + expect[expr] # optional The test block is automatically lifted into a function, so it introduces **a local scope**. Use the `nonlocal` or `global` declarations if you need to mutate something defined on the outside. - If there is a `return` at the top level of the block, that is the return - value from the test; it is what will be asserted. + If there is an `expect[expr]` at the top level of the block, the value of + `expr` is what will be asserted. There may be at most one `expect[]` per block. - If there is no `return`, the test asserts that the block completes normally, - just like a `test[returns_normally(...)]` does for an expression. + If `expect[]` is not present, the test asserts that the block completes + normally, just like `test[returns_normally(...)]` does for an expression. The asymmetry in syntax reflects the asymmetry between expressions and - statements in Python. Likewise, the fact that `with test` requires `return` - to return a value, but `test[...]` doesn't, is similar to the difference - between `def` and `lambda`. + statements in Python. Likewise, the fact that `with test` requires + `expect[]` to designate the tested expression, but `test[...]` doesn't, + is similar to the difference between `def` and `lambda`. + + In the block variant, the "result" capture rules apply to the expression + inside `expect[]`. To override, `the[]` marks can be used for capturing + the value of any expressions inside the block. The marks don't have to + be inside `expect[]`; they can appear anywhere. - In the block variant, the "result" capture rules apply to the return value - designated by `return`. To override, `the[]` marks can be used for capturing - the value of any expressions inside the block. The marks don't have to be - in the `return`; they can appear anywhere. + **Changed in v2.2.0**: + + Earlier versions of `unpythonic` used `return expr` instead of `expect[expr]`. + + Beginning with v2.2.0, the preferred syntax is `expect[expr]`. + `return expr` continues to work in this position (treated exactly like + `expect[expr]`) but emits a `DeprecationWarning`. `return` will be + un-hijacked in a future major release so that it regains its standard + Python meaning. + + Using both `expect[]` and `return` in the same block is a `SyntaxError`. **Failure and error signaling**: @@ -800,7 +880,7 @@ def _warn_expr(tree): def _test_expr(tree): # Note we want the line number *before macro expansion*, so we capture it now. - ln = q[u[tree.lineno]] if hasattr(tree, "lineno") else q[None] + ln = q[u[getattr(tree, "lineno", None)]] # may be absent on 3.10–3.12; None on 3.13+ filename = q[h[callsite_filename]()] asserter = q[h[unpythonic_assert]] @@ -823,7 +903,7 @@ def _test_expr(tree): # For this reason, we provide `with expand_testing_macros_first`, which # in itself is a code-walking block macro, whose only purpose is to force # `test[]` and its sisters to expand first.) - sourcecode = unparse(tree) + sourcecode = unparse(tree, color=True, expander=dyn._macro_expander) envname = gensym("e") # for injecting the captured value @@ -860,13 +940,19 @@ def _test_expr(tree): # These are used by `_test_expr` and `_test_block`. def _is_important_subexpr_mark(tree): return type(tree) is Subscript and type(tree.value) is Name and tree.value.id == "the" +def _is_expect_marker(stmt): + """True if `stmt` is `expect[expr]` as a top-level statement of a `with test:` block.""" + return (type(stmt) is Expr and + type(stmt.value) is Subscript and + type(stmt.value.value) is Name and + stmt.value.value.id == "expect") def _record_value(envname, sourcecode, value): envname.captured_values.append((sourcecode, value)) return value def _inject_value_recorder(envname, tree): # wrap tree with the the[] handler recorder = q[h[_record_value]] # TODO: stash hygienic value? return q[a[recorder](n[envname], - u[unparse(tree)], + u[unparse(tree, color=True, expander=dyn._macro_expander)], a[tree])] def _transform_important_subexpr(tree, envname): # The the[] mark mechanism is invoked outside-in, because for reporting, @@ -880,10 +966,7 @@ def transform(self, tree): if isunexpandedtestmacro(tree): return tree elif _is_important_subexpr_mark(tree): - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - thing = tree.slice - else: - thing = tree.slice.value + thing = tree.slice self.collect(thing) # or anything really; value not used, we just count them. # Handle any nested the[] subexpressions subtree = self.visit(thing) @@ -901,7 +984,7 @@ def _test_expr_raises(tree): return _test_expr_signals_or_raises(tree, "test_raises", q[h[unpythonic_assert_raises]]) def _test_expr_signals_or_raises(tree, syntaxname, asserter): - ln = q[u[tree.lineno]] if hasattr(tree, "lineno") else q[None] + ln = q[u[getattr(tree, "lineno", None)]] # may be absent on 3.10–3.12; None on 3.13+ filename = q[h[callsite_filename]()] # test_signals[exctype, expr, message] @@ -915,7 +998,7 @@ def _test_expr_signals_or_raises(tree, syntaxname, asserter): raise SyntaxError(f"Expected one of {syntaxname}[exctype, expr], {syntaxname}[exctype, expr, message]") # pragma: no cover # Same remark about outside-in source code capture as in `_test_expr`. - sourcecode = unparse(tree) + sourcecode = unparse(tree, color=True, expander=dyn._macro_expander) # Name our lambda to make the stack trace more understandable. # For consistency, the name matches that used by `_test_expr`. @@ -938,7 +1021,7 @@ def _test_block(block_body, args): first_stmt = block_body[0] # Note we want the line number *before macro expansion*, so we capture it now. - ln = q[u[first_stmt.lineno]] if hasattr(first_stmt, "lineno") else q[None] + ln = q[u[getattr(first_stmt, "lineno", None)]] # may be absent on 3.10–3.12; None on 3.13+ filename = q[h[callsite_filename]()] asserter = q[h[unpythonic_assert]] @@ -952,7 +1035,7 @@ def _test_block(block_body, args): raise SyntaxError('Expected `with test:` or `with test[message]:`') # pragma: no cover # Same remark about outside-in source code capture as in `_test_expr`. - sourcecode = unparse(block_body) + sourcecode = unparse(block_body, color=True, expander=dyn._macro_expander) envname = gensym("e") # for injecting the captured value @@ -976,6 +1059,40 @@ def _insert_funcname_here_(_insert_envname_here_): thefunc.args.args[0] = arg(arg=envname) # inject the gensymmed parameter name thefunc.body = block_body + # Recognize `expect[expr]` as a top-level statement of the block, and emit + # a `DeprecationWarning` for any top-level `return expr` (the deprecated + # form). `expect[]` and `return` cannot coexist in the same block — the + # block declares its tested expression exactly once, by exactly one form. + expect_indices = [] + return_indices = [] + for i, stmt in enumerate(thefunc.body): + if _is_expect_marker(stmt): + expect_indices.append(i) + elif type(stmt) is Return: + return_indices.append(i) + if len(expect_indices) > 1: + first_extra = thefunc.body[expect_indices[1]] + raise SyntaxError(f"at most one `expect[]` is allowed at the top level of `with test:` (got {len(expect_indices)}); extra at line {first_extra.lineno}") + if expect_indices and return_indices: + first_return = thefunc.body[return_indices[0]] + raise SyntaxError(f"`with test:` block has both `expect[]` and `return`; use one form, not both. Extra `return` at line {first_return.lineno}") + expander_filename = dyn._macro_expander.filename + for ridx in return_indices: + rstmt = thefunc.body[ridx] + warnings.warn_explicit( + "Using `return` to declare the tested expression in a `with test:` block is deprecated; use `expect[]` instead. The `return`-form will be un-hijacked in a future major release.", + DeprecationWarning, + expander_filename, + rstmt.lineno, + ) + if expect_indices: + eidx = expect_indices[0] + expect_subscript = thefunc.body[eidx].value + retval = expect_subscript.slice + new_return = Return(value=retval) + copy_location(new_return, thefunc.body[eidx]) + thefunc.body[eidx] = new_return + # Handle the return statement. # # We just check if there is at least one; if so, we don't need to do @@ -987,6 +1104,7 @@ def _insert_funcname_here_(_insert_envname_here_): if not the_exprs and type(retval) is Compare: # inject the implicit the[] on the LHS retval.left = _inject_value_recorder(envname, retval.left) + break else: # When there is no return statement at the top level of the `with test` block, # we inject a `return True` to satisfy the test when the injected function @@ -1009,7 +1127,7 @@ def _test_block_signals_or_raises(block_body, args, syntaxname, asserter): first_stmt = block_body[0] # Note we want the line number *before macro expansion*, so we capture it now. - ln = q[u[first_stmt.lineno]] if hasattr(first_stmt, "lineno") else q[None] + ln = q[u[getattr(first_stmt, "lineno", None)]] # may be absent on 3.10–3.12; None on 3.13+ filename = q[h[callsite_filename]()] # with test_raises[exctype, message]: @@ -1023,7 +1141,7 @@ def _test_block_signals_or_raises(block_body, args, syntaxname, asserter): raise SyntaxError(f'Expected `with {syntaxname}(exctype):` or `with {syntaxname}[exctype, message]:`') # pragma: no cover # Same remark about outside-in source code capture as in `_test_expr`. - sourcecode = unparse(block_body) + sourcecode = unparse(block_body, color=True, expander=dyn._macro_expander) testblock_function_name = gensym("_test_block") thetest = q[(a[asserter])(a[exctype], diff --git a/unpythonic/syntax/tests/selftest_testingtools.py b/unpythonic/syntax/tests/selftest_testingtools.py new file mode 100644 index 00000000..bde28b94 --- /dev/null +++ b/unpythonic/syntax/tests/selftest_testingtools.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +"""Self-test of the `unpythonic.test.fixtures` testing framework. + +The `test[]` macro allows writing unit tests for macro-enabled code, in a +compact assert-like syntax, while letting the rest of the tests run even +if some tests fail. This file exercises the *low-level machinery* — +the `unpythonic.conditions` plumbing under `test[]`, `test_signals[]`, +`test_raises[]`, etc. — using bare Python `assert` statements. Bare +`assert` is unavoidable here: we can't use `test[]` to verify `test[]`'s +own pass/fail dispatch (circular self-reference), and the `selftest_` +prefix keeps `runtests.py` from picking this module up via its +`test_*.py` discovery glob. + +For a worked, running demonstration of the framework's user-facing +features, see the example session in `unpythonic/test/fixtures.py`'s +module docstring, the "simple framework demo" in `README.md`, and the +"Test sessions and testsets" chapter in `doc/macros.md`. + +Running this self-test (uses relative macro-imports, so it must be +imported as a module rather than invoked as a script):: + + python -c "import mcpyrate.activate; from unpythonic.syntax.tests.selftest_testingtools import runtests; runtests()" +""" + +from ...syntax import macros, test, test_signals, test_raises, fail, error, warn, the, expect # noqa: F401 + +from functools import partial + +from ...test.fixtures import (session, testset, terminate, returns_normally, # noqa: F401 + tests_run, tests_failed, tests_errored, + TestFailure, TestError) + +from ...conditions import invoke, handlers, restarts, cerror # noqa: F401 +from ...excutil import raisef + +def runtests(): + # Low-level machinery. + + # Simple error reporter, just for a demonstration. + # + # If we don't need to configure which restart to invoke after the error has + # been reported, `report` could take just the `err` argument. + def report(the_restart, err): + # print(err, file=sys.stderr) # or log or whatever + invoke(the_restart) + report_and_proceed = partial(report, "proceed") + + # Basic usage. + # + # A `with handlers` block around the tests is mandatory. Without it, + # `test[]` will raise `ControlError` when the condition system detects that + # the cerror (correctable error, signaled by `test[]`) was not handled. + # + # (Only the client code can know what to do with the error, so `test[]` + # cannot automatically write the `with handlers` block for us.) + with handlers((TestFailure, report_and_proceed)): + test[2 + 2 == 5] # fails, but allows further tests to continue + test[2 + 2 == 4] + test[17 + 23 == 40, "my failure message"] + # One wouldn't normally use `assert` in a test module that uses `test[]`, + # but we have to test `test[]` itself somehow. + assert tests_run == 3 # we use the type pun that a box is equal to its content. + assert tests_failed == 1 + assert tests_errored == 0 + + # By setting up our own restart, we can skip the rest of a block of tests. + # + # The handler can be overridden locally. This works, because the + # dynamically most recently bound handler for the same signal type wins + # (see `unpythonic.conditions`). + # + # We can reset the counters by sending a new value into the box. + tests_failed << 0 + tests_errored << 0 + tests_run << 0 + report_and_skip = partial(report, "skip") + with handlers(((TestFailure, TestError), report_and_proceed)): + test[2 + 2 == 5] # fails, but allows further tests to continue + + with restarts(skip=(lambda: None)): # just for control, no return value + with handlers(((TestFailure, TestError), report_and_skip)): + test[2 + 2 == 6] # --> fails, skips the rest of this block + test[2 + 2 == 7] # not reached + + test[2 + 2 == 8] # fails, but allows further tests to continue + test[2 + 2 == 9] + assert tests_run == 4 + assert tests_failed == 4 + assert tests_errored == 0 + + # The test machinery counts an uncaught exception inside a test expr as an error + # (i.e. the test did not run to completion), not a failure. + tests_failed << 0 + tests_errored << 0 + tests_run << 0 + with handlers(((TestFailure, TestError), report_and_proceed)): + test[raisef(RuntimeError)] # errors out, but allows further tests to continue + test[2 + 2 == 4] + test[17 + 23 == 40, "my failure message"] + assert tests_run == 3 + assert tests_failed == 0 + assert tests_errored == 1 + + # Test the `the[]` marker, which changes which subexpression has its value + # captured for test failure message display purposes. + tests_failed << 0 + tests_errored << 0 + tests_run << 0 + with handlers(((TestFailure, TestError), report_and_proceed)): + count = 0 + def counter(): + nonlocal count + count += 1 + return count + test[counter() < counter()] + test[the[counter()] < counter()] + test[counter() < the[counter()]] # evaluation order not affected + assert tests_run == 3 + assert tests_failed == 0 + assert tests_errored == 0 + + # `expect[]` inside a `with test:` block — the runtime dispatch path. + # We test that the value of the expression inside `expect[expr]` is what + # gets asserted, and that both `the[]` capture rules (implicit LHS on a + # `Compare`, explicit) still apply, the same as they did for the + # deprecated `return expr` form. + tests_failed << 0 + tests_errored << 0 + tests_run << 0 + with handlers(((TestFailure, TestError), report_and_proceed)): + with test: + a = 21 + expect[a + a == 42] # passes + with test: + b = 1 + expect[b + b == 99] # fails + with test: + # No `expect[]` and no `return`: asserts the block completes normally. + log = [] + log.append("ran") + with test: + # Implicit-LHS capture: no explicit `the[]` anywhere in the block, + # and `expect[]` wraps a `Compare`, so the LHS is captured for + # failure reporting. (Effect is only visible on failure.) + c = 0 + expect[c == 0] + with test: + # Explicit `the[]` inside `expect[]` overrides implicit-LHS. + items = ["a", "b"] + expect["a" in the[items]] + assert tests_run == 5 + assert tests_failed == 1 + assert tests_errored == 0 + + # # If you want to proceed after most failures, but there is some particularly + # # critical test which, if it fails, should abort the rest of the whole unit, + # # you can override the handler locally: + # + # def die(err): + # print(err, file=sys.stderr) # or log or whatever + # sys.exit(255) + # + # with handlers(((TestFailure, TestError), report)): + # test[2 + 2 == 5] # fails, but allows further tests to continue + # + # with handlers(((TestFailure, TestError), die)): + # test[2 + 2 == 6] # --> die + # test[17 + 23 == 40, "my failure message"] # not reached + # + # # if this point was ever reached (currently it's not)... + # test[2 + 2 == 7] # ...this fails, but allows further tests to continue + # + # # This works, because the dynamically most recently bound handler for the + # # same signal type wins (see `unpythonic.conditions`). + # # + # # Similarly, if you want to skip the rest of a block of tests upon a failure: + # + # from unpythonic.conditions import restarts, invoker + # + # with handlers(((TestFailure, TestError), report)): + # test[2 + 2 == 5] # fails, but allows further tests to continue + # + # with restarts(skip=(lambda: None)): # just for control, no return value + # with handlers(((TestFailure, TestError), invoker("skip"))): + # test[2 + 2 == 6] # --> fails, skip the rest of this block + # test[17 + 23 == 40, "my failure message"] # not reached + # + # test[2 + 2 == 7] # fails, but allows further tests to continue + + print("All tests PASSED") + +# Note: no `if __name__ == '__main__': runtests()` — this module uses +# relative macro-imports, so running it as a script fails. Use the +# import-and-call incantation in the module docstring instead. diff --git a/unpythonic/syntax/tests/test_autocurry.py b/unpythonic/syntax/tests/test_autocurry.py index 325736bd..f5177bed 100644 --- a/unpythonic/syntax/tests/test_autocurry.py +++ b/unpythonic/syntax/tests/test_autocurry.py @@ -11,6 +11,8 @@ from ...llist import cons, nil, ll from ...collections import frozendict +# TODO: Add test that `autocurry` leaves `type` statements alone once we bump minimum language version to Python 3.12. + def runtests(): with testset("basic usage"): with autocurry: diff --git a/unpythonic/syntax/tests/test_autoret.py b/unpythonic/syntax/tests/test_autoret.py index d1e431f9..005e9654 100644 --- a/unpythonic/syntax/tests/test_autoret.py +++ b/unpythonic/syntax/tests/test_autoret.py @@ -16,8 +16,8 @@ def runtests(): # - if you need a loop in tail position to have a return value, # use an explicit return, or the constructs from unpythonic.fploop. # - any explicit return statements are left alone, so "return" can be used normally. - with autoreturn: - with testset("basic usage"): + with testset("basic usage"): + with autoreturn: def f(): "I'll just return this" test[f() == "I'll just return this"] @@ -26,7 +26,8 @@ def f2(): return "I'll just return this" # explicit return, not transformed test[f2() == "I'll just return this"] - with testset("if, elif, else"): + with testset("if, elif, else"): + with autoreturn: def g(x): if x == 1: "one" @@ -38,7 +39,8 @@ def g(x): test[g(2) == "two"] test[g(42) == "something else"] - with testset("except, else"): + with testset("except, else"): + with autoreturn: def h(x): try: if x == 1: @@ -50,7 +52,8 @@ def h(x): test[h(10) == 20] test[h(1) == "error"] - with testset("except, body of the try"): + with testset("except, body of the try"): + with autoreturn: def h2(x): try: if x == 1: @@ -61,12 +64,67 @@ def h2(x): test[h2(10) == 10] test[h2(1) == "error"] - with testset("with block"): + with testset("with block"): + with autoreturn: def ctx(): with env(x="hi") as e: # just need some context manager for testing, doesn't matter which e.x # tail position in a with block test[ctx() == "hi"] + with testset("function definition"): # v0.15.0+ + with autoreturn: + def outer(): + def inner(): + "inner function" + test[callable(outer())] # returned a function + test[outer()() == "inner function"] + + with testset("class definition"): # v0.15.0+ + with autoreturn: + def classdefiner(): + class InnerClassDefinition: + pass + test[isinstance(classdefiner(), type)] # returned a class + test[classdefiner().__name__ == "InnerClassDefinition"] + + with testset("match/case"): # Python 3.10+ + with autoreturn: + def classify(x): + match x: + case 1: + "one" + case 2: + "two" + case _: + "other" + test[classify(1) == "one"] + test[classify(2) == "two"] + test[classify(42) == "other"] + + def classify_nested(x): + match x: + case (a, b): + a + b + case [a, b, *rest]: + a + b + sum(rest) + case _: + 0 + test[classify_nested((3, 4)) == 7] + test[classify_nested([1, 2, 3, 4]) == 10] + test[classify_nested("nope") == 0] + + def classify_with_guard(x): + match x: + case n if n < 0: + "negative" + case 0: + "zero" + case n if n > 0: + "positive" + test[classify_with_guard(-5) == "negative"] + test[classify_with_guard(0) == "zero"] + test[classify_with_guard(7) == "positive"] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/syntax/tests/test_comprehension_unpacking_3_15.py b/unpythonic/syntax/tests/test_comprehension_unpacking_3_15.py new file mode 100644 index 00000000..3dcdcd02 --- /dev/null +++ b/unpythonic/syntax/tests/test_comprehension_unpacking_3_15.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +"""The macros pass through Python 3.15's new comprehension unpacking (PEP 798). + +These tests require Python 3.15+ because the unpacking syntax won't parse on +earlier versions. + +`lazify`, `autocurry`, `tco` and `continuations` have no comprehension-specific +handling, so the new forms reach their generic paths. Two shapes are new there: a +`Starred` in the element position of a list/set/generator comprehension, and a +`DictComp` whose `value` is `None` because the mapping expression sits in `key`. + +**These test semantics, not just values.** A test that only checked results would +pass even if `lazify` had quietly gone strict inside the new forms, or `autocurry` +had stopped currying there — which is exactly the way this could break without +anyone noticing. So each macro is exercised for the property that makes it that +macro: laziness by leaving a `1 / 0` unevaluated, currying by partially applying, +TCO by recursing deeper than the stack allows, and continuations by capturing one. + +TODO: Merge into the per-macro test modules when the floor bumps to Python 3.15+. +""" + +from ...syntax import macros, test, test_raises, the # noqa: F401 +from ...test.fixtures import session, testset + +from ...syntax import macros, lazify, autocurry, tco, continuations, call_cc # noqa: F401, F811 + + +def _pair(k): + return [k, k] + + +def _mapping(k): + return {k: k} + + +with lazify: + def _first(a, b): + return a + + def lazy_starred_list(ks): + # If laziness holds, `1 / 0` is never evaluated. + return [*_first(_pair(k), 1 / 0) for k in ks] + + def lazy_starred_set(ks): + return {*_first(_pair(k), 1 / 0) for k in ks} + + def lazy_starred_genexpr(ks): + return list((*_first(_pair(k), 1 / 0) for k in ks)) + + def lazy_dict_unpacking(ks): + return {**_first(_mapping(k), 1 / 0) for k in ks} + + def lazy_ordinary_dictcomp(ks): + # Control: the `k: v` form uses the `value` field the unpacking form leaves empty. + return {k: _first(_pair(k), 1 / 0) for k in ks} + + +with autocurry: + def _add3(a, b, c): + return a + b + c + + def curried_starred_list(ks): + return [*[_add3(1)(2)(k)] for k in ks] + + def curried_dict_unpacking(ks): + return {**{k: _add3(1, 2)(k)} for k in ks} + + +with tco: + def tco_deep_recursion(n, acc): + """Recurses deeper than the stack allows, so it only completes under TCO.""" + if n <= 0: + return acc + items = [*_pair(n) for _ in (1,)] + return tco_deep_recursion(n - 1, acc + len(items)) + + def tco_dict_unpacking(ks): + return {**_mapping(k) for k in ks} + + +with continuations: + def _ident(x): + return x + + def cc_starred_list(ks): + x = call_cc[_ident(ks)] + return [*_pair(k) for k in x] + + def cc_dict_unpacking(ks): + x = call_cc[_ident(ks)] + return {**_mapping(k) for k in x} + + +def runtests(): + with testset("lazify keeps its laziness inside the new comprehension forms"): + # Reaching a value at all means the unused `1 / 0` argument was never forced. + test[lazy_starred_list([1, 2]) == [1, 1, 2, 2]] + test[lazy_starred_set([1, 2]) == {1, 2}] + test[lazy_starred_genexpr([1, 2]) == [1, 1, 2, 2]] + test[lazy_dict_unpacking([1, 2]) == {1: 1, 2: 2}] + test[lazy_ordinary_dictcomp([1, 2]) == {1: [1, 1], 2: [2, 2]}] + + with testset("autocurry still curries inside the new comprehension forms"): + # Partial application, so a value comes back only if currying happened. + test[curried_starred_list([1, 2]) == [4, 5]] + test[curried_dict_unpacking([1, 2]) == {1: 4, 2: 5}] + + with testset("tco still optimizes a tail call whose body uses the new forms"): + # 5000 frames deep; without TCO this is a RecursionError. + test[tco_deep_recursion(5000, 0) == 10000] + test[tco_dict_unpacking([1, 2]) == {1: 1, 2: 2}] + + with testset("continuations survive the new comprehension forms"): + test[cc_starred_list([1, 2]) == [1, 1, 2, 2]] + test[cc_dict_unpacking([1, 2]) == {1: 1, 2: 2}] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_conts.py b/unpythonic/syntax/tests/test_conts.py index 34238797..0f04706a 100644 --- a/unpythonic/syntax/tests/test_conts.py +++ b/unpythonic/syntax/tests/test_conts.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- """Continuations (call/cc for Python).""" -from ...syntax import macros, test, test_raises, error # noqa: F401 +from ...syntax import macros, test, test_raises, error, fail # noqa: F401 from ...test.fixtures import session, testset, returns_normally from ...syntax import macros, continuations, call_cc, multilambda, autoreturn, autocurry, let # noqa: F401, F811 +from ...syntax import get_cc, iscontinuation +from ...collections import box, unbox from ...ec import call_ec from ...fploop import looped from ...fun import withself @@ -67,7 +69,7 @@ def h1(a, b): def h2(a, b): x, y = call_cc[f(a, b)] - return True or f(3, 4) + return True or f(3, 4) # noqa: SIM222 -- testing short-circuit with continuations test[h2(3, 4) is True] # "or" with 3 or more items (testing; handled differently internally) @@ -78,12 +80,12 @@ def h3(a, b): def h4(a, b): x, y = call_cc[f(a, b)] - return None or True or f(3, 4) + return None or True or f(3, 4) # noqa: SIM222 -- testing short-circuit with continuations test[h4(3, 4) is True] def h5(a, b): x, y = call_cc[f(a, b)] - return 42 or None or f(3, 4) + return 42 or None or f(3, 4) # noqa: SIM222 -- testing short-circuit with continuations test[h5(3, 4) == 42] # "and" @@ -94,7 +96,7 @@ def i1(a, b): def i2(a, b): x, y = call_cc[f(a, b)] - return False and f(3, 4) + return False and f(3, 4) # noqa: SIM223 -- testing short-circuit with continuations test[i2(3, 4) is False] # "and" with 3 or more items @@ -105,12 +107,12 @@ def i3(a, b): def i4(a, b): x, y = call_cc[f(a, b)] - return True and False and f(3, 4) + return True and False and f(3, 4) # noqa: SIM223 -- testing short-circuit with continuations test[i4(3, 4) is False] def i5(a, b): x, y = call_cc[f(a, b)] - return None and False and f(3, 4) + return None and False and f(3, 4) # noqa: SIM223 -- testing short-circuit with continuations test[i5(3, 4) is False] # combination of "and" and "or" @@ -403,7 +405,7 @@ def amb(lst, cc): ourcc = cc stack.append(lambda: amb(rest, cc=ourcc)) return first - def fail(): + def fail(): # noqa: F811, not redefining, the first one is a macro. if stack: f = stack.pop() return f() @@ -652,6 +654,247 @@ def s(loop, acc=0): test[tuple(out) == 2 * tuple(range(11))] test[s == 10] + # As of 0.15.1, the preferred way of working with continuations is as follows. + # + # The pattern `k = call_cc[get_cc()]` covers the 99% common case where you + # just want to snapshot and save the control state into a local variable. + # + # See docstring of `unpythonic.syntax.get_cc` for more. It's a regular function + # that works together with the `call_cc` macro. + with testset("get_cc, the less antisocial little sister of call_cc"): + with continuations: + def append_stuff_to(lst): + lst.append("one") + k = call_cc[get_cc()] + lst.append("two") + return k + + lst = [] + k = append_stuff_to(lst) + test[lst == ["one", "two"]] + # invoke the continuation + k(k) # send `k` back in as argument so it the continuation sees it as its local `k` + test[lst == ["one", "two", "two"]] + + # If your continuation needs to take arguments, `get_cc` can also make a parametric continuation: + with testset("get_cc with parametric continuation"): + with continuations: + def append_stuff_to(lst): + # Important: in the `get_cc` call, the initial values for + # the additional arguments, if any, must be passed positionally, + # due to `call_cc` syntax limitations. + k, x1, x2 = call_cc[get_cc(1, 2)] + lst.extend([x1, x2]) + return k + + lst = [] + k = append_stuff_to(lst) + test[lst == [1, 2]] + # invoke the continuation, sending both `k` and our additional arguments. + k(k, 3, 4) + test[lst == [1, 2, 3, 4]] + # When invoking the continuation, the additional arguments can be passed + # in any way allowed by Python. + k(k, x1=5, x2=6) + test[lst == [1, 2, 3, 4, 5, 6]] + + # You can also abuse `k` to pass an arbitrary object, if inside the + # continuation, you don't need a reference to the continuation itself. + # This is the lispy solution. + # + # Then you can `iscontinuation(k)` to check whether it is a continuation + # (first run, return value of `get_cc()`), or something else (second and + # further runs, a value sent in via the continuation). + # + # Whether this or the previous example is more pythonic is left as an + # exercise to the reader. + # + # In this solution, be careful, if you need to send in a continuation + # function for some reason. It is impossible to be 100% sure whether `k` + # is *the* continuation that should have been returned by *this* `get_cc`. + # If you need to send in a continuation function, box it (in a read-only + # `Some` box, even), to make it explicit that it's intended as data. + with testset("get_cc lispy style"): + with continuations: + # The pattern + # + # k = call_cc[get_cc()] + # if iscontinuation(k): + # return k + # + # creates a multi-shot resume point. See also `test_multishot.py` for the + # `@multishot` macro that automates this pattern. + def append_stuff_to(lst): + ... # could do something useful here (otherwise, why make a continuation?) + + k = call_cc[get_cc()] + + # <-- the resume point is here, with `k` set to "the return value of the `call_cc`", + # i.e. the continuation during the first run, and whatever was sent in during later runs. + + # In 0.15.1+, continuation functions created by the `call_cc[...]` macro are + # tagged, and can be detected using `unpythonic.syntax.iscontinuation`, which + # is a regular function: + if iscontinuation(k): # first run; just return the continuation + return k + + # invoked via continuation, now `k` is input data instead of a continuation + x1, x2 = k + lst.extend([x1, x2]) + return None + + lst = [] + k = append_stuff_to(lst) + k([1, 2]) # whatever object we send in becomes the local `k` in the continuation. + test[lst == [1, 2]] + k([3, 4]) + test[lst == [1, 2, 3, 4]] + + with testset("scoping, locals only"): + # This is the cleanest way to scope your local variables in continuations: + # just accept the fact that each continuation introduces a scope boundary. + with continuations: + def f(): + # Original function scope + x = None + + # Continuation 1 scope begins here + # (from the statement following `call_cc` onward, but including the `k1`) + k1 = call_cc[get_cc()] + if iscontinuation(k1): + # This `x` is local to continuation 1. + x = "cont 1 first time" + return k1, x + + # Continuation 2 scope begins here + k2 = call_cc[get_cc()] + if iscontinuation(k2): + # This `x` is local to continuation 2. + x = "cont 2 first time" + return k2, x + + # Still in continuation 2, so this is the `x` of continuation 2. + x = "cont 2 second time" + return None, x + + k1, x = f() + test[x == "cont 1 first time"] + k2, x = k1(None) # when resuming, send `None` as the new value of variable `k1` in continuation 1 + test[x == "cont 2 first time"] + k3, x = k2(None) + test[k3 is None] + test[x == "cont 2 second time"] + + k2, x = k1(None) # multi-shotting from earlier resume point + test[x == "cont 2 first time"] + + with testset("scoping, in presence of nonlocal"): + # It shouldn't matter in this particular example whether we declare the `x` + # in the continuations `nonlocal`, because once the parent returns, the + # only places that can access its locals *from that activation* are the + # continuation closures *created by that activation*. The point of this + # testset is to demonstrate that `nonlocal` *works* as expected — the + # continuation's `nonlocal x` reaches back to the parent's `x`, just as + # if the continuation were any ordinary nested closure. + with continuations: + def f(): + # Original function scope + x = None + + # Continuation 1 scope begins here + # (from the statement following `call_cc` onward, but including the `k1`) + k1 = call_cc[get_cc()] + nonlocal x # noqa: F811 -- macro splits body; post-expansion `nonlocal` is at the top of the continuation function, not a redefinition + if iscontinuation(k1): + # This is now the original `x`. + x = "cont 1 first time" + return k1, x + + # Continuation 2 scope begins here + k2 = call_cc[get_cc()] + nonlocal x # noqa: F811 -- as above + if iscontinuation(k2): + # This too is the original `x`. + x = "cont 2 first time" + return k2, x + + # Still the original `x`. + x = "cont 2 second time" + return None, x + + k1, x = f() + test[x == "cont 1 first time"] + k2, x = k1(None) # when resuming, send `None` as the new value of variable `k1` in continuation 1 + test[x == "cont 2 first time"] + k3, x = k2(None) + test[k3 is None] + test[x == "cont 2 second time"] + + k2, x = k1(None) # multi-shotting from earlier resume point + test[x == "cont 2 first time"] + + # If you need to scope like `nonlocal`, use the classic solution: box the value, + # so you have no need to overwrite the name; you can replace the thing in the box. + # + # (Classic from before `nonlocal` declarations were a thing. They were added in 3.0; + # for historical interest, see https://www.python.org/dev/peps/pep-3104/ ) + with testset("scoping, using a box"): + with continuations: + # poor man's execution trace + def make_tracing_box_updater(thebox, trace): + def update(value): + trace.append(f"old: {unbox(thebox)}") + thebox << value + trace.append(f"new: {unbox(thebox)}") + return value + return update + + # If we wanted to replace the list instance later, we could pass the list in a box, too. + def f(lst): + # Now there is just one `x`, which is the box; we just update the contents. + # Original function scope + x = box("f") + lst.append(f"initial: {unbox(x)}") + update = make_tracing_box_updater(x, lst) + + # Continuation 1 scope begins here + # (from the statement following `call_cc` onward, but including the `k1`) + k1 = call_cc[get_cc()] + if iscontinuation(k1): + return k1, update("k1 first") + update("k1 again") + + # Continuation 2 scope begins here + k2 = call_cc[get_cc()] + if iscontinuation(k2): + return k2, update("k2 first") + update("k2 again") + + return None, unbox(x) + + trace = [] + k1, x = f(trace) + test[x == "k1 first"] + test[trace == ['initial: f', 'old: f', 'new: k1 first']] + k2, x = k1(None) # when resuming, send `None` as the new value of variable `k1` in continuation 1 + test[x == "k2 first"] + test[trace == ['initial: f', 'old: f', 'new: k1 first', + 'old: k1 first', 'new: k1 again', 'old: k1 again', 'new: k2 first']] + k3, x = k2(None) + test[k3 is None] + test[x == "k2 again"] + test[trace == ['initial: f', 'old: f', 'new: k1 first', + 'old: k1 first', 'new: k1 again', 'old: k1 again', 'new: k2 first', + 'old: k2 first', 'new: k2 again']] + + k2, x = k1(None) # multi-shotting from earlier resume point + test[x == "k2 first"] + test[trace == ['initial: f', 'old: f', 'new: k1 first', + 'old: k1 first', 'new: k1 again', 'old: k1 again', 'new: k2 first', + 'old: k2 first', 'new: k2 again', + 'old: k2 again', 'new: k1 again', 'old: k1 again', 'new: k2 first']] + # ^^^^^^^^^^^^^^^ state as left by `k2` before the multi-shot + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/syntax/tests/test_conts_gen.py b/unpythonic/syntax/tests/test_conts_gen.py index 2b432a13..8101333f 100644 --- a/unpythonic/syntax/tests/test_conts_gen.py +++ b/unpythonic/syntax/tests/test_conts_gen.py @@ -16,9 +16,12 @@ See also the Racket version of this: https://github.com/Technologicat/python-3-scicomp-intro/blob/master/examples/beyond_python/generator.rkt + +For the `@multishot` macro that automates the multi-shot pattern, see +`test_multishot.py`. """ -from ...syntax import macros, test, test_raises # noqa: F401 +from ...syntax import macros, test, test_raises # noqa: F401, F811 from ...test.fixtures import session, testset from ...syntax import macros, continuations, call_cc, dlet, abbrev, let_syntax, block # noqa: F401, F811 @@ -26,7 +29,8 @@ from ...fploop import looped from ...fun import identity -#from mcpyrate.debug import macros, step_expansion # noqa: F811, F401 +from mcpyrate.debug import macros, step_expansion # noqa: F811, F401 + def runtests(): with testset("a basic generator"): @@ -178,7 +182,7 @@ def result(loop, i=0): x = g2() # noqa: F821 test[out == list(range(10))] - with testset("multi-shot generators"): + with testset("multi-shot generators with call_cc[]"): with continuations: with let_syntax: with block[value] as my_yield: # noqa: F821 @@ -241,6 +245,9 @@ def my_yieldf(value=None, *, cc): # module level, define my_yield as a magic variable so that accidental uses # outside any make_generator are caught at compile time. The actual template the # make_generator macro needs to splice in is already here in the final example.) + # + # The `@multishot` macro in `unpythonic.syntax.multishot` librarifies this + # pattern; see `test_multishot.py` for canonical usage. if __name__ == '__main__': # pragma: no cover with session(__file__): diff --git a/unpythonic/syntax/tests/test_forall.py b/unpythonic/syntax/tests/test_forall.py index d7e11703..2e858933 100644 --- a/unpythonic/syntax/tests/test_forall.py +++ b/unpythonic/syntax/tests/test_forall.py @@ -10,7 +10,12 @@ def runtests(): # forall: pure AST transformation, with real lexical variables - # - assignment (with List-monadic magic) is ``var << iterable`` + # - assignment (with List-monadic magic) is ``var := iterable``, or ``var << iterable`` in the + # classic syntax of v0.15.0 to v0.15.2, which is still accepted. + # + # Both spellings are exercised below, and deliberately: `forall` does not parse its bindings itself, it + # borrows `letdoutil.isenvassign` from `let` and `do`. So one checker serves three macros, and until + # 2026-08-25 the walrus form was covered where the checker lives and in neither macro that borrows it. with testset("basic usage"): out = forall[y << range(3), # noqa: F821, `forall` defines the name on the LHS of the `<<`. x << range(3), # noqa: F821 @@ -36,6 +41,22 @@ def runtests(): with testset("single item special case"): test[forall[range(3), ] == (range(3),)] + with testset("modern env-assignment syntax"): + out = forall[y := range(3), # noqa: F821, `forall` defines the name on the LHS. + x := range(3), # noqa: F821 + insist(x % 2 == 0), # noqa: F821 + (x, y)] # noqa: F821 + test[out == ((0, 0), (2, 0), (0, 1), (2, 1), (0, 2), (2, 2))] + + # The same triples as above, to show the two spellings agree rather than merely both running. + pt = forall[z := range(1, 21), # noqa: F821 + x := range(1, z + 1), # noqa: F821 + y := range(x, z + 1), # noqa: F821 + insist(x * x + y * y == z * z), # noqa: F821 + (x, y, z)] # noqa: F821 + test[tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20))] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/syntax/tests/test_lambdatools.py b/unpythonic/syntax/tests/test_lambdatools.py index 7349fd36..0f1a4d18 100644 --- a/unpythonic/syntax/tests/test_lambdatools.py +++ b/unpythonic/syntax/tests/test_lambdatools.py @@ -4,7 +4,7 @@ from ...syntax import macros, test, test_raises, warn # noqa: F401 from ...test.fixtures import session, testset -from ...syntax import (macros, multilambda, namedlambda, quicklambda, f, # noqa: F401, F811 +from ...syntax import (macros, multilambda, namedlambda, quicklambda, fn, # noqa: F401, F811 envify, local, let, autocurry, autoreturn) from functools import wraps @@ -57,10 +57,9 @@ def runtests(): foo = let[[f7 << (lambda x: x)] in f7] # let-binding: name as "f7" # noqa: F821 test[foo.__name__ == "f7"] - warn["NamedExpr test currently disabled for syntactic compatibility with Python 3.6 and 3.7."] - # if foo2 := (lambda x: x): # NamedExpr a.k.a. walrus operator (Python 3.8+) - # pass - # test[foo2.__name__ == "foo2"] + if foo2 := (lambda x: x): # NamedExpr a.k.a. walrus operator (Python 3.8+) + pass + test[foo2.__name__ == "foo2"] # function call with named arg def foo(func1, func2): @@ -173,9 +172,9 @@ def decorated(*args, **kwargs): # Outside-in macros. with quicklambda: with multilambda: - func = f[[local[x << _], # noqa: F821, F823, `quicklambda` implicitly defines `f[]` to mean `lambda`. - local[y << _], # noqa: F821 - x + y]] # noqa: F821 + func = fn[[local[x << _], # noqa: F821, F823, `quicklambda` implicitly defines `fn[]` to mean `lambda`. + local[y << _], # noqa: F821 + x + y]] # noqa: F821 test[func(1, 2) == 3] with testset("envify (formal parameters as an unpythonic env)"): diff --git a/unpythonic/syntax/tests/test_lazify.py b/unpythonic/syntax/tests/test_lazify.py index d4f26c7f..2cd6d14f 100644 --- a/unpythonic/syntax/tests/test_lazify.py +++ b/unpythonic/syntax/tests/test_lazify.py @@ -4,6 +4,8 @@ from ...syntax import macros, test, test_raises, error, the # noqa: F401 from ...test.fixtures import session, testset +from mcpyrate.debug import macros, step_expansion # noqa: F811 + from ...syntax import (macros, lazify, lazy, lazyrec, # noqa: F811, F401 let, letseq, letrec, local, tco, @@ -26,6 +28,8 @@ from sys import stderr import gc +# TODO: Add test that `lazify` leaves `type` statements alone once we bump minimum language version to Python 3.12. + def runtests(): # first test the low-level tools with testset("lazyrec (lazify a container literal, recursing into sub-containers)"): @@ -79,7 +83,7 @@ def runtests(): # force1() forces a promise promise = lazy[2 + 3] test[type(promise) is Lazy] - test[type(force1(promise)) == int] + test[type(force1(promise)) is int] test[force1("not a promise") == "not a promise"] # anything else is passed through # force() recurses into containers, forcing any promises found therein @@ -347,6 +351,30 @@ def f14(a, b): return f15(2 * a, 2 * b) test[f14(21, 1 / 0) == 42] + with testset("integration: expand nested inner macro invocations"): + # Here we need to enable expand-once mode to see whether the innermost + # macro expands correctly. This depends on `lazify` expanding inner + # macro invocations in recursive mode, regardless of the mode of the + # expander. + # + # If it doesn't, the innermost macro won't be expanded before `lazify` + # performs its own AST edits (editing also `Subscript` nodes), and in + # the result, it will no longer be a macro invocation, and will hence + # cause a `NameError` at run time. + # + # TODO: This prints a lot of stuff, because that's its primary purpose. + # TODO: Here it would be nicer to use a macro that only enables expand-once mode. + with step_expansion: + with lazify: + # Here we need any macro that expands outside-in. The important thing is + # it doesn't recurse (`expander.visit`) on its own, instead relying on the + # expander's recursive mode to expand any remaining macro invocations inside + # the tree. + # + # Here `with test` is nice, because it asserts the block returns normally at run time. + with test: + lazy[...] # <-- this should get expanded, not raise NameError at run time + # let bindings have a role similar to function arguments, so we auto-lazify there with testset("integration with let, letseq, letrec"): with lazify: @@ -563,7 +591,7 @@ def append_succ(lis): def nextfibo(state): a, b = state fibos.append(a) # store result by side effect - return (b, a + b) # new state, handed to next function in the pipe + return (b, a + b) # new state, handed to the next function in the pipe p = lazy_piped1((1, 1)) # load initial state into a lazy pipe for _ in range(10): # set up pipeline p = p | nextfibo diff --git a/unpythonic/syntax/tests/test_letdo.py b/unpythonic/syntax/tests/test_letdo.py index 08f81d6b..a4c1ee73 100644 --- a/unpythonic/syntax/tests/test_letdo.py +++ b/unpythonic/syntax/tests/test_letdo.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """Let constructs; do (imperative code in expression position).""" -# TODO: Update the @dlet, @dletseq, @dletrec, @blet, @bletseq, @bletrec examples -# TODO: to pass macro arguments using brackets once we bump to minimum Python 3.9. +# NOTE: Decorator macro arguments use parenthesis syntax in some examples below. +# Bracket syntax is preferred for new code; parenthesis syntax is deprecated but kept for backward compatibility. from ...syntax import macros, test, test_raises # noqa: F401 from ...test.fixtures import session, testset @@ -17,22 +17,48 @@ x = "the global x" # for lexical scoping tests def runtests(): - with testset("do (imperative code in an expression)"): + with testset("do (imperative code in an expression) (new env-assignment syntax 0.15.3+)"): # Macro wrapper for unpythonic.seq.do (imperative code in expression position) - # - Declare and initialize a local variable with ``local[var << value]``. + # - Declare and initialize a local variable with ``local[var := value]``. # Is in scope from the next expression onward, for the (lexical) remainder # of the do. - # - Assignment is ``var << value``. Valid from any level inside the ``do`` + # - Assignment is ``var := value``. Valid from any level inside the ``do`` # (including nested ``let`` constructs and similar). # - No need for ``lambda e: ...`` wrappers. Inserted automatically, # so the lines are only evaluated as the underlying seq.do() runs. + # + d1 = do[local[x := 17], + print(x), + (x := 23), + x] + test[d1 == 23] + + # Since we repurposed an existing assignment operator, let's check we didn't accidentally assign to the function scope. + test_raises[NameError, x, "only the `do[]` should have an `x` here"] + + # v0.14.0: do[] now supports deleting previously defined local names with delete[] + a = 5 + d = do[local[a := 17], # noqa: F841, yes, d is unused. + test[a == 17], + delete[a], + test[a == 5], # lexical scoping + True] + + test_raises[KeyError, do[delete[a], ], "should have complained about deleting nonexistent local 'a'"] + + # do0[]: like do[], but return the value of the **first** expression + d2 = do0[local[y := 5], # noqa: F821, `local` defines the name on the LHS of the `<<`. + print("hi there, y =", y), # noqa: F821 + 42] # evaluated but not used + test[d2 == 5] + + with testset("do (imperative code in an expression) (previous modern env-assignment syntax)"): d1 = do[local[x << 17], print(x), x << 23, - x] # do[] returns the value of the last expression + x] # do[] returns the value of the last expression # noqa: F823, it's the `x` from `do[]`, not from the enclosing scope. test[d1 == 23] - # v0.14.0: do[] now supports deleting previously defined local names with delete[] a = 5 d = do[local[a << 17], # noqa: F841, yes, d is unused. test[a == 17], @@ -42,14 +68,41 @@ def runtests(): test_raises[KeyError, do[delete[a], ], "should have complained about deleting nonexistent local 'a'"] - # do0[]: like do[], but return the value of the **first** expression d2 = do0[local[y << 5], # noqa: F821, `local` defines the name on the LHS of the `<<`. print("hi there, y =", y), # noqa: F821 42] # evaluated but not used test[d2 == 5] # Let macros. Lexical scoping supported. - with testset("let, letseq, letrec basic usage"): + with testset("let, letseq, letrec basic usage (new env-assignment syntax 0.15.3+)"): + # parallel binding, i.e. bindings don't see each other + test[let[(x := 17), + (y := 23)][ # noqa: F821, `let` defines `y` here. + (x, y)] == (17, 23)] # noqa: F821 + + # sequential binding, i.e. Scheme/Racket let* + test[letseq[(x := 1), + (y := x + 1)][ # noqa: F821 + (x, y)] == (1, 2)] # noqa: F821 + + test[letseq[(x := 1), + (x := x + 1)][ # in a letseq, rebinding the same name is ok + x] == 2] + + # letrec sugars unpythonic.lispylet.letrec, removing the need for quotes on LHS + # and "lambda e: ..." wrappers on RHS (these are inserted by the macro): + test[letrec[(evenp := (lambda x: (x == 0) or oddp(x - 1))), # noqa: F821, `letrec` defines `evenp` here. + (oddp := (lambda x: (x != 0) and evenp(x - 1)))][ # noqa: F821 + evenp(42)] is True] # noqa: F821 + + # nested letrecs work, too - each environment is internally named by a gensym + # so that outer ones "show through": + test[letrec[(z := 9000)][ # noqa: F821 + letrec[(evenp := (lambda x: (x == 0) or oddp(x - 1))), # noqa: F821 + (oddp := (lambda x: (x != 0) and evenp(x - 1)))][ # noqa: F821 + (evenp(42), z)]] == (True, 9000)] # noqa: F821 + + with testset("let, letseq, letrec basic usage (previous modern env-assignment syntax)"): # parallel binding, i.e. bindings don't see each other test[let[x << 17, y << 23][ # noqa: F821, `let` defines `y` here. @@ -98,6 +151,32 @@ def runtests(): "should not be able to rebind the same name in the same let"] # implicit do: an extra set of brackets denotes a multi-expr body + with testset("implicit do (extra bracket syntax for multi-expr let body) (new env-assignment syntax v0.15.3+)"): + a = let[(x := 1), + (y := 2)][[ # noqa: F821 + y := 1337, # noqa: F821 + (x, y)]] # noqa: F821 + test[a == (1, 1337)] + + # only the outermost extra brackets denote a multi-expr body + a = let[(x, 1), + (y, 2)][[ # noqa: F821 + [1, 2]]] + test[a == [1, 2]] + + # implicit do works also in letseq, letrec + a = letseq[(x := 1), + (y := x + 1)][[ # noqa: F821 + x := 1337, + (x, y)]] # noqa: F821 + test[a == (1337, 2)] + + a = letrec[(x := 1), + (y := x + 1)][[ # noqa: F821 + x := 1337, + (x, y)]] # noqa: F821 + test[a == (1337, 2)] + with testset("implicit do (extra bracket syntax for multi-expr let body)"): a = let[x << 1, y << 2][[ # noqa: F821 @@ -291,7 +370,7 @@ def test1(): @dlet(x << "the env x") def test2(): return x # local var assignment not in effect yet # noqa: F823, `dlet` defines `x` here. - x = "the unused local x" # noqa: F841, this `x` being unused is the point of this test. # pragma: no cover + x = "the unused local x" # noqa: F841 -- unused `x` is the point of this test # pragma: no cover test[test2() == "the env x"] @dlet(x << "the env x") @@ -302,13 +381,13 @@ def test3(): @dlet(x << "the env x") def test4(): - nonlocal x + nonlocal x # noqa: F824, Python 3.12+ complain about this; just testing our let construct; it's correct that there's no local `x` as per Python's normal scoping rules. return x test[test4() == "the nonlocal x"] @dlet(x << "the env x") def test5(): - global x + global x # noqa: F824, Python 3.12+ complain about this; just testing our let construct; it's correct that there's no local `x` as per Python's normal scoping rules. return x test[test5() == "the global x"] @@ -371,7 +450,7 @@ def test9(): def test10(): x = x + " (copied to local)" # noqa: F823 del x # comes into effect for the next statement - return x # so this is env's original x + return x # noqa: F821 -- env's original x, after del of local test[test10() == "the env x"] @dlet(x << "the env x") @@ -393,8 +472,8 @@ def test12(): def test13(): x = "the local x" del x - return x # noqa: F823, this `x` refers to the `x` in the `dlet` env. - x = "the unused local x" # noqa: F841, this `x` being unused is the point of this test. # pragma: no cover + return x # noqa: F821, F823 -- `x` refers to the `dlet` env binding + x = "the unused local x" # noqa: F841 -- unused `x` is the point of this test # pragma: no cover test[test13() == "the env x"] with test_raises[NameError, "should have tried to access the deleted nonlocal x"]: @@ -403,10 +482,37 @@ def test13(): def test14(): nonlocal x del x # ignored by unpythonic's scope analysis, too dynamic - return x # trying to refer to the nonlocal x, which was deleted + return x # noqa: F821 -- trying to refer to the nonlocal x, which was deleted test14() x = "the nonlocal x" # restore the test environment + # v0.15.3+: walrus syntax + @dlet(x := "the env x") + def test15(): + def inner(): + (x := "updated env x") # noqa: F841, this writes to the let env since there is no `x` in an intervening scope, according to Python's standard rules. + inner() + return x + test[test15() == "updated env x"] + + @dlet(x := "the env x") + def test16(): + def inner(): + x = "the inner x" # noqa: F841, unused on purpose, for testing. An assignment *statement* does NOT write to the let env. + inner() + return x + test[test16() == "the env x"] + + @dlet(x := "the env x") + def test17(): + x = "the local x" # This lexical variable shadows the env x. + def inner(): + # The env x is shadowed. Since we don't say `nonlocal x`, this creates a new lexical variable scoped to `inner`. + (x := "the inner x") # noqa: F841, unused on purpose, for testing. + inner() + return x + test[test17() == "the local x"] + # in do[] (also the implicit do), local[] takes effect from the next item test[let[x << "the let x", y << None][ # noqa: F821 diff --git a/unpythonic/syntax/tests/test_letdoutil.py b/unpythonic/syntax/tests/test_letdoutil.py index 45725233..f30d844a 100644 --- a/unpythonic/syntax/tests/test_letdoutil.py +++ b/unpythonic/syntax/tests/test_letdoutil.py @@ -12,11 +12,9 @@ autocurry) from ast import Tuple, Name, Constant, Lambda, BinOp, Attribute, Call -import sys from mcpyrate import unparse -from ...syntax.astcompat import getconstant, Num from ...syntax.letdoutil import (canonize_bindings, isenvassign, islet, isdo, UnexpandedEnvAssignView, @@ -41,7 +39,9 @@ def validate(lst): test[validate(the[canonize_bindings(q[k0, v0].elts)])] # noqa: F821, it's quoted. test[validate(the[canonize_bindings(q[((k0, v0),)].elts)])] # noqa: F821 test[validate(the[canonize_bindings(q[(k0, v0), (k1, v1)].elts)])] # noqa: F821 + test[validate(the[canonize_bindings([q[(k0 := v0)]])])] # noqa: F821, it's quoted. test[validate(the[canonize_bindings([q[k0 << v0]])])] # noqa: F821, it's quoted. + test[validate(the[canonize_bindings(q[(k0 := v0), (k1 := v1)].elts)])] # noqa: F821, it's quoted. test[validate(the[canonize_bindings(q[k0 << v0, k1 << v1].elts)])] # noqa: F821, it's quoted. # -------------------------------------------------------------------------------- @@ -51,13 +51,19 @@ def validate(lst): # need this utility, so we must test it first. with testset("isenvassign"): test[not isenvassign(q[x])] # noqa: F821 + test[isenvassign(q[(x := 42)])] # noqa: F821 test[isenvassign(q[x << 42])] # noqa: F821 with testset("islet"): test[not islet(q[x])] # noqa: F821 test[not islet(q[f()])] # noqa: F821 - # modern notation for bindings + # unpythonic 0.15.3+, Python 3.8+ + test[islet(the[expandrq[let[(x := 21)][2 * x]]]) == ("expanded_expr", "let")] # noqa: F821, `let` defines `x` + test[islet(the[expandrq[let[[x := 21] in 2 * x]]]) == ("expanded_expr", "let")] # noqa: F821 + test[islet(the[expandrq[let[2 * x, where[(x := 21)]]]]) == ("expanded_expr", "let")] # noqa: F821 + + # unpythonic 0.15.0 to 0.15.2, previous modern notation for bindings test[islet(the[expandrq[let[x << 21][2 * x]]]) == ("expanded_expr", "let")] # noqa: F821, `let` defines `x` test[islet(the[expandrq[let[[x << 21] in 2 * x]]]) == ("expanded_expr", "let")] # noqa: F821 test[islet(the[expandrq[let[2 * x, where[x << 21]]]]) == ("expanded_expr", "let")] # noqa: F821 @@ -67,18 +73,30 @@ def validate(lst): test[islet(the[expandrq[let[(x, 21) in 2 * x]]]) == ("expanded_expr", "let")] # noqa: F821 test[islet(the[expandrq[let[2 * x, where(x, 21)]]]) == ("expanded_expr", "let")] # noqa: F821 + # unpythonic 0.15.3+, Python 3.8+ + with expandrq as testdata: + @dlet(x := 21) # noqa: F821 + def f0(): + return 2 * x # noqa: F821 + test[islet(the[testdata[0].decorator_list[0]]) == ("expanded_decorator", "let")] + + # unpythonic 0.15.0 to 0.15.2, previous modern notation for bindings with expandrq as testdata: @dlet(x << 21) # noqa: F821 def f1(): return 2 * x # noqa: F821 test[islet(the[testdata[0].decorator_list[0]]) == ("expanded_decorator", "let")] + # classic notation for bindings with expandrq as testdata: @dlet((x, 21)) # noqa: F821 def f2(): return 2 * x # noqa: F821 test[islet(the[testdata[0].decorator_list[0]]) == ("expanded_decorator", "let")] + testdata = q[let[(x := 21)][2 * x]] # noqa: F821 + test[islet(the[testdata], expanded=False) == ("lispy_expr", "let")] + testdata = q[let[x << 21][2 * x]] # noqa: F821 test[islet(the[testdata], expanded=False) == ("lispy_expr", "let")] @@ -95,6 +113,8 @@ def f2(): testdata = q[let[2 * x, where(x, 21)]] # noqa: F821 test[islet(the[testdata], expanded=False) == ("where_expr", "let")] + testdata = q[let[[x := 21, y := 2] in y * x]] # noqa: F821 + test[islet(the[testdata], expanded=False) == ("in_expr", "let")] testdata = q[let[[x << 21, y << 2] in y * x]] # noqa: F821 test[islet(the[testdata], expanded=False) == ("in_expr", "let")] testdata = q[let[((x, 21), (y, 2)) in y * x]] # noqa: F821 @@ -120,6 +140,12 @@ def f4(): return 2 * x # noqa: F821 test[islet(the[testdata[0].decorator_list[0]], expanded=False) == ("decorator", "dlet")] + with q as testdata: + @dlet(x := 21) # noqa: F821 + def f5(): + return 2 * x # noqa: F821 + test[islet(the[testdata[0].decorator_list[0]], expanded=False) == ("decorator", "dlet")] + with testset("islet integration with autocurry"): # NOTE: We have to be careful with how we set up the test data here. # @@ -167,6 +193,10 @@ def f4(): test[not isdo(q[x])] # noqa: F821 test[not isdo(q[f()])] # noqa: F821 + # unpythonic 0.15.3+, Python 3.8+ + test[isdo(the[expandrq[do[(x := 21), # noqa: F821 + 2 * x]]]) == "expanded"] # noqa: F821 + test[isdo(the[expandrq[do[x << 21, # noqa: F821 2 * x]]]) == "expanded"] # noqa: F821 @@ -177,6 +207,21 @@ def f4(): thedo = testdata[0].value test[isdo(the[thedo]) == "curried"] + # unpythonic 0.15.3+, Python 3.8+ + testdata = q[do[(x := 21), # noqa: F821 + 2 * x]] # noqa: F821 + test[isdo(the[testdata], expanded=False) == "do"] + + testdata = q[do0[23, # noqa: F821 + (x := 21), # noqa: F821 + 2 * x]] # noqa: F821 + test[isdo(the[testdata], expanded=False) == "do0"] + + testdata = q[someothermacro[(x := 21), # noqa: F821 + 2 * x]] # noqa: F821 + test[not isdo(the[testdata], expanded=False)] + + # previous modern notation testdata = q[do[x << 21, # noqa: F821 2 * x]] # noqa: F821 test[isdo(the[testdata], expanded=False) == "do"] @@ -193,22 +238,50 @@ def f4(): # -------------------------------------------------------------------------------- # Destructuring - envassign + with testset("envassign destructuring (new env-assign syntax v0.15.3+)"): + testdata = q[(x := 42)] # noqa: F821 + view = UnexpandedEnvAssignView(testdata) + + # read + test[view.name == "x"] + constant_node = view.value + test[type(the[constant_node]) is Constant and constant_node.value == 42] + + # write + view.name = "y" + view.value = q[23] + test[view.name == "y"] + constant_node = view.value + test[type(the[constant_node]) is Constant and constant_node.value == 23] + + # it's a live view + test[unparse(testdata) == "(y := 23)"] # syntax type `:=` vs. `<<` is preserved + + # error cases + test_raises[TypeError, + UnexpandedEnvAssignView(q[x]), # noqa: F821 + "not an env assignment"] + with test_raises[TypeError, "name must be str"]: + view.name = 1234 + with testset("envassign destructuring"): testdata = q[x << 42] # noqa: F821 view = UnexpandedEnvAssignView(testdata) # read test[view.name == "x"] - test[type(the[view.value]) in (Constant, Num) and getconstant(view.value) == 42] # Python 3.8: ast.Constant + constant_node = view.value + test[type(the[constant_node]) is Constant and constant_node.value == 42] # write view.name = "y" view.value = q[23] test[view.name == "y"] - test[type(the[view.value]) in (Constant, Num) and getconstant(view.value) == 23] # Python 3.8: ast.Constant + constant_node = view.value + test[type(the[constant_node]) is Constant and constant_node.value == 23] # it's a live view - test[unparse(testdata) == "(y << 23)"] + test[unparse(testdata) == "(y << 23)"] # syntax type `:=` vs. `<<` is preserved # error cases test_raises[TypeError, @@ -245,6 +318,8 @@ def testletdestructuring(testdata): test[unparse(view.body) == "(z * t)"] # lispy expr + testdata = q[let[(x := 21), (y := 2)][y * x]] # noqa: F821 + testletdestructuring(testdata) testdata = q[let[x << 21, y << 2][y * x]] # noqa: F821 testletdestructuring(testdata) testdata = q[let[[x, 21], [y, 2]][y * x]] # noqa: F821 @@ -253,6 +328,8 @@ def testletdestructuring(testdata): testletdestructuring(testdata) # haskelly let-in + testdata = q[let[[x := 21, y := 2] in y * x]] # noqa: F821 + testletdestructuring(testdata) testdata = q[let[[x << 21, y << 2] in y * x]] # noqa: F821 testletdestructuring(testdata) testdata = q[let[(x << 21, y << 2) in y * x]] # noqa: F821 @@ -267,6 +344,8 @@ def testletdestructuring(testdata): testletdestructuring(testdata) # haskelly let-where + testdata = q[let[y * x, where[(x := 21), (y := 2)]]] # noqa: F821 + testletdestructuring(testdata) testdata = q[let[y * x, where[x << 21, y << 2]]] # noqa: F821 testletdestructuring(testdata) testdata = q[let[y * x, where(x << 21, y << 2)]] # noqa: F821 @@ -281,6 +360,8 @@ def testletdestructuring(testdata): testletdestructuring(testdata) # disembodied haskelly let-in (just the content, no macro invocation) + testdata = q[[x := 21, y := 2] in y * x] # noqa: F821 + testletdestructuring(testdata) testdata = q[[x << 21, y << 2] in y * x] # noqa: F821 testletdestructuring(testdata) testdata = q[(x << 21, y << 2) in y * x] # noqa: F821 @@ -295,6 +376,8 @@ def testletdestructuring(testdata): testletdestructuring(testdata) # disembodied haskelly let-where (just the content, no macro invocation) + testdata = q[y * x, where[(x := 21), (y := 2)]] # noqa: F821 + testletdestructuring(testdata) testdata = q[y * x, where[x << 21, y << 2]] # noqa: F821 testletdestructuring(testdata) testdata = q[y * x, where(x << 21, y << 2)] # noqa: F821 @@ -311,7 +394,7 @@ def testletdestructuring(testdata): # decorator with q as testdata: @dlet((x, 21), (y, 2)) # noqa: F821 - def f5(): + def f6(): return 2 * x # noqa: F821 # read @@ -392,7 +475,7 @@ def testexpandedletdestructuring(testdata): # decorator with expandrq as testdata: @dlet((x, 21), (y, 2)) # noqa: F821 - def f6(): + def f7(): return 2 * x # noqa: F821 view = ExpandedLetView(testdata[0].decorator_list[0]) test_raises[TypeError, @@ -440,7 +523,7 @@ def testbindings(*expected): test[the[unparse(bk)] == the[f"'{k}'"]] test[type(the[lam]) is Lambda] lambody = lam.body - test[type(the[lambody]) in (Constant, Num) and getconstant(lambody) == the[v]] # Python 3.8: ast.Constant + test[type(the[lambody]) is Constant and lambody.value == the[v]] # read test[len(view.bindings.elts) == 2] @@ -488,7 +571,7 @@ def testbindings(*expected): # decorator, letrec with expandrq as testdata: @dletrec((x, 21), (y, 2)) # noqa: F821 - def f7(): + def f8(): return 2 * x # noqa: F821 view = ExpandedLetView(testdata[0].decorator_list[0]) test_raises[TypeError, @@ -517,16 +600,43 @@ def f7(): # -------------------------------------------------------------------------------- # Destructuring - unexpanded do + with testset("do destructuring (unexpanded) (new env-assign syntax v0.15.3+)"): + testdata = q[do[local[x := 21], # noqa: F821 + 2 * x]] # noqa: F821 + view = UnexpandedDoView(testdata) + # read + thebody = view.body + thing = thebody[0].slice + test[isenvassign(the[thing])] + # write + # This mutates the original, but we have to assign `view.body` to trigger the setter. + thebody[0] = q[local[x := 9001]] # noqa: F821 + view.body = thebody + + # implicit do, a.k.a. extra bracket syntax + testdata = q[let[[local[x := 21], # noqa: F821 + 2 * x]]] # noqa: F821 + theimplicitdo = testdata.slice + view = UnexpandedDoView(theimplicitdo) + # read + thebody = view.body + thing = thebody[0].slice + test[isenvassign(the[thing])] + # write + thebody[0] = q[local[x := 9001]] # noqa: F821 + view.body = thebody + + test_raises[TypeError, + UnexpandedDoView(q[x]), # noqa: F821 + "not a do form"] + with testset("do destructuring (unexpanded)"): testdata = q[do[local[x << 21], # noqa: F821 2 * x]] # noqa: F821 view = UnexpandedDoView(testdata) # read thebody = view.body - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - thing = thebody[0].slice - else: - thing = thebody[0].slice.value + thing = thebody[0].slice test[isenvassign(the[thing])] # write # This mutates the original, but we have to assign `view.body` to trigger the setter. @@ -536,17 +646,11 @@ def f7(): # implicit do, a.k.a. extra bracket syntax testdata = q[let[[local[x << 21], # noqa: F821 2 * x]]] # noqa: F821 - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - theimplicitdo = testdata.slice - else: - theimplicitdo = testdata.slice.value + theimplicitdo = testdata.slice view = UnexpandedDoView(theimplicitdo) # read thebody = view.body - if sys.version_info >= (3, 9, 0): # Python 3.9+: the Index wrapper is gone. - thing = thebody[0].slice - else: - thing = thebody[0].slice.value + thing = thebody[0].slice test[isenvassign(the[thing])] # write thebody[0] = q[local[x << 9001]] # noqa: F821 diff --git a/unpythonic/syntax/tests/test_letsyntax.py b/unpythonic/syntax/tests/test_letsyntax.py index 13940cdd..6e4cca78 100644 --- a/unpythonic/syntax/tests/test_letsyntax.py +++ b/unpythonic/syntax/tests/test_letsyntax.py @@ -59,6 +59,34 @@ class Silly: # This test will either pass, or error out with an AttributeError. test[let_syntax[[alias << realthing] in Silly.alias] == 42] # noqa: F821 + with testset("modern env-assignment syntax"): + # `let_syntax` does not parse its bindings itself; it borrows `letdoutil.isenvassign` in + # `letsyntax_mode`, so it accepts both `name := value` and the classic `name << value`. The classic + # form is covered above; this is the modern one. + evaluations = 0 + def verylongfunctionname(x=1): + nonlocal evaluations + evaluations += 1 + return x + + y = let_syntax[[f := verylongfunctionname] # noqa: F821 + in [f(), # noqa: F821 + f(17)]] # noqa: F821 + test[evaluations == 2] + test[y == 17] + + y = let_syntax[[f(), # noqa: F821 + f(23)], # noqa: F821 + where[f := verylongfunctionname]] # noqa: F821 + test[evaluations == 4] + test[y == 23] + + # **Templates cannot use the modern syntax, and never will**: a template binds `f[a]`, and Python's + # own grammar rejects a walrus with a subscript target - `(f[a] := ...)` is a `SyntaxError` before + # any macro sees it. So `<<` is not merely the older spelling here, it is the only one, and the + # advice to prefer `:=` stops at this one form. + test_raises[SyntaxError, compile("(f[a] := 1)", "", "eval")] + with testset("block variant"): with let_syntax: with block as make123: # capture one or more statements diff --git a/unpythonic/syntax/tests/test_monadic_do.py b/unpythonic/syntax/tests/test_monadic_do.py new file mode 100644 index 00000000..8642224c --- /dev/null +++ b/unpythonic/syntax/tests/test_monadic_do.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +"""Tests for the `with monadic_do[M] as result:` macro.""" + +from ...syntax import macros, test, test_raises, the, monadic_do # noqa: F401 +from ...test.fixtures import session, testset + +from ...llist import nil +from ...monads import Maybe, Either, Left, Right, List, Writer, State, Reader + + +def runtests(): + with testset("basic expansion (Maybe)"): + with monadic_do[Maybe] as a: + [x := Maybe(10), + y := Maybe(x + 1), + Maybe(x + y)] + test[a == Maybe(21)] + + # Short-circuit: Nothing propagates; later bindings never fire. + with monadic_do[Maybe] as b: + [x := Maybe(nil), + y := Maybe(x + 1), + Maybe(x + y)] + test[b == Maybe(nil)] + + # Single-element list: no binds, just the final expression. + with monadic_do[Maybe] as c: + [Maybe(42)] + test[c == Maybe(42)] + + with testset("binding-syntax variants"): + # := is the primary binding syntax + with monadic_do[Maybe] as a: + [x := Maybe(3), + Maybe(x * 2)] + test[a == Maybe(6)] + + # << is the legacy (discordian-deprecated) alternative + with monadic_do[Maybe] as b: + [x << Maybe(3), + Maybe(x * 2)] + test[b == Maybe(6)] + + # Mixed (letdoutil allows both in the same block) + with monadic_do[Maybe] as c: + [x := Maybe(2), + y << Maybe(x + 3), + Maybe(x * y)] + test[c == Maybe(10)] + + with testset("sequencing — bare expressions in bindings"): + # Bare expression on a binding line = Haskell's `do { mx; ... }` + # (sequence, not bind). The macro wraps it synthetically as `_ := mexpr`. + with monadic_do[List] as filtered: + [x := List.from_iterable(range(1, 6)), + List.guard(x % 2 == 0), + List(x)] + test[filtered == List(2, 4)] + + # Mixed bare + binding lines + with monadic_do[List] as mixed: + [x := List(1, 2, 3), + List.guard(x > 1), + y := List(x * 10), + List((x, y))] + test[mixed == List((2, 20), (3, 30))] + + with testset("Either short-circuit"): + with monadic_do[Either] as a: + [x := Right(10), + y := Right(x * 2), + Right(x + y)] + test[a == Right(30)] + + # Left short-circuits; second binding not evaluated + called = [] + def track(v): + called.append(v) + return Right(v * 2) + with monadic_do[Either] as b: + [x := Left("boom"), + y := track(x), + Right(x + y)] + test[b == Left("boom")] + test[called == []] # `track` never invoked + + with testset("List monad — Pythagorean triples"): + def r(lo, hi): + return List.from_iterable(range(lo, hi)) + with monadic_do[List] as pt: + [z := r(1, 21), + x := r(1, z + 1), + y := r(x, z + 1), + List.guard(x * x + y * y == z * z), + List((x, y, z))] + test[tuple(sorted(pt)) == ((3, 4, 5), (5, 12, 13), (6, 8, 10), + (8, 15, 17), (9, 12, 15), (12, 16, 20))] + + with testset("Writer"): + with monadic_do[Writer] as w: + [x := Writer(10, "got 10; "), + y := Writer(x + 1, "added 1; "), + Writer(y * 2, "doubled; ")] + value, log = w.data + test[value == 22] + test[log == "got 10; added 1; doubled; "] + + with testset("State"): + bump = State(lambda s: (s, s + 1)) + with monadic_do[State] as st: + [a := bump, + b := bump, + c := bump, + State.unit((a, b, c))] + vals, final = st.run(10) + test[vals == (10, 11, 12)] + test[final == 13] + + with testset("Reader"): + with monadic_do[Reader] as rd: + [m := Reader.asks(lambda env: env["multiplier"]), + o := Reader.asks(lambda env: env["offset"]), + Reader.unit(m * 5 + o)] + test[rd.run({"multiplier": 3, "offset": 10}) == 25] + + with testset("nested do-blocks"): + # Nested do works because the outer's body is a single statement + # but inside the final expression we can invoke another do. + def maybe_addone(): + with monadic_do[Maybe] as inner: + [x := Maybe(10), + Maybe(x + 1)] + return inner + + with monadic_do[Maybe] as outer: + [y := maybe_addone(), + Maybe(y * 2)] + test[outer == Maybe(22)] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_monadic_do_integration.py b/unpythonic/syntax/tests/test_monadic_do_integration.py new file mode 100644 index 00000000..f60aca05 --- /dev/null +++ b/unpythonic/syntax/tests/test_monadic_do_integration.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +"""Integration tests for `monadic_do` composed with other block macros. + +`monadic_do` is always the innermost `with`; outer block macros expand +inner macros between their passes, so the generated bind chain is +visible to them for further transformation. + +See `briefs/monads-implementation.md` for the full analysis of why each +combination works, and `doc/macros.md` (the xmas-tree section) for the +correct nesting order. +""" + +from ...syntax import (macros, test, the, # noqa: F401 + monadic_do, + autocurry, lazify, tco, continuations, + multilambda, quicklambda, namedlambda, autoreturn, + envify, autoref) +from ...test.fixtures import session, testset + +from ...llist import nil +from ...monads import Maybe, Either, Left, Right + + +def runtests(): + with testset("must: continuations + monadic_do"): + with continuations: + with monadic_do[Maybe] as result: + [x := Maybe(10), + y := Maybe(x + 1), + Maybe(x + y)] + test[result == Maybe(21)] + + with testset("must: autocurry + monadic_do"): + with autocurry: + with monadic_do[Maybe] as result: + [x := Maybe(5), + y := Maybe(x + 1), + Maybe(x * y)] + test[result == Maybe(30)] + + with testset("must: lazify + monadic_do (basic)"): + with lazify: + with monadic_do[Maybe] as result: + [x := Maybe(10), + y := Maybe(x + 1), + Maybe(x + y)] + test[result == Maybe(21)] + + with testset("must: lazify + monadic_do (short-circuit preserves non-forcing)"): + # The key guarantee: on the short-circuit path, later binding RHSs + # must NOT be forced (no observable side effect, no exceptions). + side_effects = [] + def observable_builder(): + side_effects.append("called") + return Maybe(999) + + with lazify: + with monadic_do[Maybe] as result: + [x := Maybe(nil), + y := observable_builder(), + Maybe(x + y)] + test[result == Maybe(nil)] + test[side_effects == []] # observable_builder never invoked + + # Same for Either. + counter = [0] + def bump_and_build(): + counter[0] += 1 + return Right(counter[0]) + + with lazify: + with monadic_do[Either] as result2: + [x := Left("bail"), + y := bump_and_build(), + Right(x + y)] + test[result2 == Left("bail")] + test[counter[0] == 0] + + with testset("must: tco + monadic_do"): + with tco: + with monadic_do[Maybe] as result: + [x := Maybe(7), + y := Maybe(x + 1), + Maybe(x + y)] + test[result == Maybe(15)] + + with testset("smoke: multilambda + monadic_do"): + with multilambda: + with monadic_do[Maybe] as result: + [x := Maybe(3), + y := Maybe(x * 2), + Maybe(x + y)] + test[result == Maybe(9)] + + with testset("smoke: quicklambda + monadic_do"): + with quicklambda: + with monadic_do[Maybe] as result: + [x := Maybe(3), + y := Maybe(x * 2), + Maybe(x + y)] + test[result == Maybe(9)] + + with testset("smoke: namedlambda + monadic_do"): + with namedlambda: + with monadic_do[Maybe] as result: + [x := Maybe(3), + y := Maybe(x * 2), + Maybe(x + y)] + test[result == Maybe(9)] + + with testset("smoke: autoreturn + monadic_do"): + # autoreturn inserts `return` into function bodies; the monadic_do + # body is a single Expr inside a `with`, so autoreturn should + # leave it alone. Verify the `result << expr` exit pattern + # still works. + def compute(): + with autoreturn: + with monadic_do[Maybe] as result: + [x := Maybe(4), + Maybe(x + 6)] + return result + test[compute() == Maybe(10)] + + with testset("smoke: envify + monadic_do"): + with envify: + with monadic_do[Maybe] as result: + [x := Maybe(5), + Maybe(x + 1)] + test[result == Maybe(6)] + + with testset("smoke: autoref + monadic_do"): + from ...env import env as _env + the_env = _env(base=100) + with autoref[the_env]: + with monadic_do[Maybe] as result: + [x := Maybe(5), + Maybe(x + base)] # noqa: F821 -- `base` comes in via autoref + test[result == Maybe(105)] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_multishot.py b/unpythonic/syntax/tests/test_multishot.py new file mode 100644 index 00000000..21faa94e --- /dev/null +++ b/unpythonic/syntax/tests/test_multishot.py @@ -0,0 +1,375 @@ +# -*- coding: utf-8 -*- +"""Tests for `@multishot`, `myield`, and `MultishotIterator`.""" + +import copy + +from ...syntax import macros, test, test_raises # noqa: F401, F811 +from ...test.fixtures import session, testset + +from ...syntax import macros, continuations, multishot, myield, myield_from # noqa: F401, F811 +from ...syntax import MultishotIterator # runtime import (not a macro) + + +def runtests(): + with testset("@multishot: four `myield` forms"): + with continuations: + @multishot + def f(): + myield + myield[42] + k = myield + test[k == 23] + k = myield[42] + test[k == 17] + + k0 = f() # instantiate (returns the initial continuation) + k1 = k0() # run up to the explicit bare `myield` + k2, x2 = k1() # to `myield[42]` + test[x2 == 42] + k3 = k2() # to `k = myield` + k4, x4 = k3(23) # send 23, run to `k = myield[42]` + test[x4 == 42] + test_raises[StopIteration, k4(17)] # send 17, fall off the end + + with testset("@multishot: basic linear consumption"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + try: + out = [] + k = g() + while True: + k, x = k() + out.append(x) + except StopIteration: + pass + test[out == [1, 2, 3]] + + with testset("@multishot: re-invoke an earlier continuation (multi-shot)"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + k0 = g() + k1, x1 = k0() + k2, x2 = k1() + k3, x3 = k2() + k, x = k1() # multi-shot: rewind to k1 + test[x1 == 1] + test[x2 == x == 2] + test[x3 == 3] + test[k.func.__qualname__ == k2.func.__qualname__] # same bookmarked position + test[k.func is not k2.func] # but different closure instance + test_raises[StopIteration, k3()] + + with testset("@multishot: `return value` raises StopIteration(value)"): + with continuations: + @multishot + def h(): + myield[1] + return 42 + + mi = MultishotIterator(h()) + test[next(mi) == 1] + try: + next(mi) + except StopIteration as e: + test[e.value == 42] + else: + test[False] # should have raised + + with testset("MultishotIterator: linear iteration"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + test[[x for x in MultishotIterator(g())] == [1, 2, 3]] + + with testset("MultishotIterator: send"): + with continuations: + @multishot + def f(): + k = myield[10] + test[k == 99] + k = myield[20] + test[k == 100] + + mi = MultishotIterator(f()) + test[next(mi) == 10] + test[mi.send(99) == 20] + test_raises[StopIteration, mi.send(100)] + + with testset("MultishotIterator: close"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + + mi = MultishotIterator(g()) + test[next(mi) == 1] + mi.close() + test_raises[StopIteration, next(mi)] + # close is idempotent + mi.close() + + with testset("MultishotIterator: throw re-enters the continuation"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + + mi = MultishotIterator(g()) + test[next(mi) == 1] + test_raises[ValueError, mi.throw(ValueError("boom"))] + + with testset("MultishotIterator: send(value) on a bare-myield continuation drops the value"): + # Standard-generator parity: gen.send(value) on a bare yield discards + # the value. The `_step` helper detects partial-wrapping (the shape of + # a bare-myield continuation) and routes the advance as a no-arg call. + with continuations: + @multishot + def g(): + myield[1] # bare myield; the captured continuation is partial-wrapped + myield[2] + myield[3] + + mi = MultishotIterator(g()) + test[next(mi) == 1] + # Sending to a bare-myield continuation: value silently dropped, advance proceeds. + test[mi.send("ignored") == 2] + # send(None) on a bare-myield continuation also works (≡ next). + test[mi.send(None) == 3] + + with testset("MultishotIterator: copy.copy forks the iterator (HEADLINE)"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + myield[3] + myield[4] + + # Real generators raise TypeError on copy.copy. Multi-shots fork. + mi = MultishotIterator(g()) + test[next(mi) == 1] # advance original to position 1 + + fork = copy.copy(mi) # snapshot at "after yielding 1" + test[next(mi) == 2] # original advances independently... + test[next(mi) == 3] + test[next(fork) == 2] # ...and so does the fork, from its own snapshot + test[next(fork) == 3] + test[next(fork) == 4] + test_raises[StopIteration, next(fork)] + + # Original is unaffected by fork's exhaustion + test[next(mi) == 4] + test_raises[StopIteration, next(mi)] + + with testset("MultishotIterator: copy.deepcopy raises TypeError"): + with continuations: + @multishot + def g(): + myield[1] + + mi = MultishotIterator(g()) + test_raises[TypeError, copy.deepcopy(mi)] + + with testset("MultishotIterator: gi_running is always False"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + + mi = MultishotIterator(g()) + test[mi.gi_running is False] + next(mi) + test[mi.gi_running is False] + next(mi) + test[mi.gi_running is False] + + with testset("MultishotIterator: gi_frame is always None"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + + mi = MultishotIterator(g()) + test[mi.gi_frame is None] + next(mi) + test[mi.gi_frame is None] + mi.close() + test[mi.gi_frame is None] + + with testset("MultishotIterator: gi_code is the liveness signal"): + with continuations: + @multishot + def g(): + myield[1] + myield[2] + + mi = MultishotIterator(g()) + # While live, gi_code matches the underlying continuation's __code__. + k = mi.k + expected_code = k.func.__code__ if hasattr(k, "func") else k.__code__ + test[mi.gi_code is expected_code] + + next(mi) + # Still live; gi_code reflects the new continuation. + k = mi.k + expected_code = k.func.__code__ if hasattr(k, "func") else k.__code__ + test[mi.gi_code is expected_code] + + mi.close() + # After close, gi_code is None — this is the liveness signal. + test[mi.gi_code is None] + + with testset("myield_from: linear delegation (statement form)"): + with continuations: + @multishot + def inner(): + myield[1] + myield[2] + + @multishot + def outer(): + myield[0] + myield_from[inner()] + myield[3] + + mi = MultishotIterator(outer()) + test[list(mi) == [0, 1, 2, 3]] + + with testset("myield_from: assignment form binds inner's StopIteration value"): + with continuations: + @multishot + def inner(): + myield[1] + return 99 + + @multishot + def outer(): + v = myield_from[inner()] + myield[v] # outer yields whatever inner returned via StopIteration + + mi = MultishotIterator(outer()) + test[list(mi) == [1, 99]] + + with testset("myield_from: gi_yieldfrom tracks the inner iterator while delegating"): + with continuations: + @multishot + def inner(): + myield[1] + myield[2] + + @multishot + def outer(): + myield[0] + myield_from[inner()] + myield[3] + + mi = MultishotIterator(outer()) + test[mi.gi_yieldfrom is None] # not yet delegating + + test[next(mi) == 0] # outer's myield[0] + test[mi.gi_yieldfrom is None] # still not delegating; outer hasn't entered _drive + + test[next(mi) == 1] # entered _drive; first inner value + test[isinstance(mi.gi_yieldfrom, MultishotIterator)] + inner_mi_seen = mi.gi_yieldfrom + + test[next(mi) == 2] # second inner value + test[mi.gi_yieldfrom is inner_mi_seen] # same inner iterator object + + test[next(mi) == 3] # back in outer; inner exhausted + test[mi.gi_yieldfrom is None] # delegation done + + test_raises[StopIteration, next(mi)] + + with testset("myield_from: send forwards value into inner"): + with continuations: + @multishot + def inner(): + v = myield[10] + myield[v] # echo the sent value + + @multishot + def outer(): + myield_from[inner()] + + mi = MultishotIterator(outer()) + test[next(mi) == 10] + test[mi.send(42) == 42] # 42 reached inner's `v` + + with testset("myield_from: throw forwards exception into inner; uncaught propagates out"): + with continuations: + @multishot + def inner(): + myield[1] + myield[2] # throw fires here; inner doesn't catch + + @multishot + def outer(): + myield_from[inner()] + myield["unreached"] + + mi = MultishotIterator(outer()) + test[next(mi) == 1] + test[next(mi) == 2] + test_raises[ValueError, mi.throw(ValueError("boom"))] + # State after a propagated throw: outer's continuation is unchanged + # (no advance happened in `_advance`); next consumer attempt re-enters + # the same continuation. + + with testset("myield_from: multi-shot fork around delegation (copy.copy)"): + # When a fork happens *before* delegation begins, each iterator's + # resume re-runs `MultishotIterator(inner())`, so each gets its own + # fresh inner. Forks then iterate the delegation independently — + # both see inner's full sequence. + with continuations: + @multishot + def inner(): + myield[1] + myield[2] + myield[3] + + @multishot + def outer(): + myield[0] + myield_from[inner()] + + mi = MultishotIterator(outer()) + test[next(mi) == 0] + + fork = copy.copy(mi) # before delegation; forks are independent + + # mi consumes its own delegation + test[next(mi) == 1] + test[next(mi) == 2] + test[next(mi) == 3] + test_raises[StopIteration, next(mi)] + + # fork consumes its own delegation + test[next(fork) == 1] + test[next(fork) == 2] + test[next(fork) == 3] + test_raises[StopIteration, next(fork)] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_nb.py b/unpythonic/syntax/tests/test_nb.py index d4a4a78b..f83d9a0f 100644 --- a/unpythonic/syntax/tests/test_nb.py +++ b/unpythonic/syntax/tests/test_nb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from ...syntax import macros, test, error # noqa: F401 +from ...syntax import macros, test, warn # noqa: F401 from ...test.fixtures import session, testset from ...syntax import macros, nb # noqa: F401, F811 @@ -18,7 +18,7 @@ def runtests(): try: from sympy import symbols, pprint except ImportError: # pragma: no cover - error["SymPy not installed in this Python, cannot test symbolic math in nb."] + warn["SymPy not installed in this Python, skipping symbolic math tests in nb."] else: with nb[pprint]: # you can specify a custom print function (first positional arg) test[_ is None] # noqa: F821 diff --git a/unpythonic/syntax/tests/test_scopeanalyzer.py b/unpythonic/syntax/tests/test_scopeanalyzer.py index 2c2b0927..16cf0995 100644 --- a/unpythonic/syntax/tests/test_scopeanalyzer.py +++ b/unpythonic/syntax/tests/test_scopeanalyzer.py @@ -59,9 +59,8 @@ def sleep(): # Assignment # - # At least up to Python 3.7, all assignments produce Name nodes in - # Store context on their LHS, so we don't need to care what kind of - # assignment it is. + # All assignments produce Name nodes in #tore context on their LHS, + # so we don't need to care what kind of assignment it is. test[get_names_in_store_context(getnames_store_simple) == ["x"]] with q as getnames_tuple: x, y = 1, 2 # noqa: F841 @@ -177,8 +176,8 @@ def f3(): with q as getlexvars_fdef: y = 21 def myfunc(x, *args, kwonlyarg, **kwargs): - nonlocal y # not really needed here, except for exercising the analyzer. - global g + nonlocal y # noqa: F824, for Python 3.12+; just testing our scope analyzer; it's correct that there's no local `y`. Also, not really needed here, except for exercising the analyzer. + global g # noqa: F824, Python 3.12+ complain about this; just testing our scope analyzer; it's correct that there's no local `g`. def inner(blah): abc = 123 # noqa: F841 z = 2 * y # noqa: F841 @@ -270,6 +269,82 @@ def f(): # noqa: F811 n["_apply_test_here_"] scoped_transform(scoped_localvar3, callback=make_checker(["f"])) # x already deleted + # Python 3.10+: `match`/`case` + with testset("match/case: get_names_in_store_context"): + # Simple capture + with q as matchcase_simple: + match x: # noqa: F821, it's only quoted. + case y: # noqa: F841, it's only quoted. + pass + test[get_names_in_store_context(matchcase_simple) == ["y"]] + + # Wildcard `_` — does NOT capture + with q as matchcase_wildcard: + match x: # noqa: F821, it's only quoted. + case _: + pass + test[get_names_in_store_context(matchcase_wildcard) == []] + + # Sequence pattern with star capture + with q as matchcase_sequence: + match x: # noqa: F821, it's only quoted. + case [a, b, *rest]: # noqa: F841, it's only quoted. + pass + test[get_names_in_store_context(matchcase_sequence) == ["a", "b", "rest"]] + + # Class pattern — captures `x` and `y`, but NOT the class reference `Point` + with q as matchcase_class: + match x: # noqa: F821, it's only quoted. + case Point(x, y): # noqa: F821, F841, it's only quoted. + pass + names = get_names_in_store_context(matchcase_class) + test["x" in names] + test["y" in names] + test["Point" not in names] # class reference, not a capture + + # Class pattern with keyword captures + with q as matchcase_class_kw: + match x: # noqa: F821, it's only quoted. + case Point(x=px, y=py): # noqa: F821, F841, it's only quoted. + pass + names = get_names_in_store_context(matchcase_class_kw) + test["px" in names] + test["py" in names] + test["Point" not in names] + + # Mapping pattern with `**rest` + with q as matchcase_mapping: + match x: # noqa: F821, it's only quoted. + case {"key": value, **rest}: # noqa: F841, it's only quoted. + pass + names = get_names_in_store_context(matchcase_mapping) + test["value" in names] + test["rest" in names] + + # Nested: mapping containing a class pattern + with q as matchcase_nested: + match x: # noqa: F821, it's only quoted. + case {"key": Point(px, py)}: # noqa: F821, F841, it's only quoted. + pass + names = get_names_in_store_context(matchcase_nested) + test["px" in names] + test["py" in names] + test["Point" not in names] # class reference, not a capture + + # OR pattern + with q as matchcase_or: + match x: # noqa: F821, it's only quoted. + case 1 | 2 | 3: + pass + test[get_names_in_store_context(matchcase_or) == []] + + # `as` pattern with guard + with q as matchcase_as: + match x: # noqa: F821, it's only quoted. + case (1 | 2) as num: # noqa: F841, it's only quoted. + pass + test[get_names_in_store_context(matchcase_as) == ["num"]] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/syntax/tests/test_scopeanalyzer_3_11.py b/unpythonic/syntax/tests/test_scopeanalyzer_3_11.py new file mode 100644 index 00000000..bd436b58 --- /dev/null +++ b/unpythonic/syntax/tests/test_scopeanalyzer_3_11.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +"""Lexical scope analysis tools — try/except* tests. + +These tests require Python 3.11+ because the ``except*`` syntax +won't parse on earlier versions. + +TODO: Merge into test_scopeanalyzer.py when floor bumps to Python 3.11+. +""" + +from ...syntax import macros, test, test_raises, the # noqa: F401 +from ...test.fixtures import session, testset + +from mcpyrate.quotes import macros, q # noqa: F401, F811 + +from ...syntax.scopeanalyzer import get_names_in_store_context + +def runtests(): + with testset("try/except*: get_names_in_store_context"): + # except* binds names just like except + with q as exceptstar_simple: + try: + pass + except* ValueError as eg: # noqa: F841, it's only quoted. + pass + test[get_names_in_store_context(exceptstar_simple) == ["eg"]] + + with q as exceptstar_multi: + try: + pass + except* ValueError as eg1: # noqa: F841, it's only quoted. + pass + except* TypeError as eg2: # noqa: F841, it's only quoted. + pass + test[get_names_in_store_context(exceptstar_multi) == ["eg1", "eg2"]] + + # Names bound inside the try body are also collected + with q as exceptstar_with_assign: + try: + x = 42 # noqa: F841, it's only quoted. + except* ValueError as eg: # noqa: F841, it's only quoted. + y = 1 # noqa: F841, it's only quoted. + names = get_names_in_store_context(exceptstar_with_assign) + test["x" in the[names]] + test["y" in the[names]] + test["eg" in the[names]] + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_tco.py b/unpythonic/syntax/tests/test_tco.py index 49f95957..87691a5e 100644 --- a/unpythonic/syntax/tests/test_tco.py +++ b/unpythonic/syntax/tests/test_tco.py @@ -5,7 +5,7 @@ from ...test.fixtures import session, testset, returns_normally from ...syntax import (macros, tco, autoreturn, autocurry, do, let, letseq, dletrec, # noqa: F401, F811 - quicklambda, f, continuations, call_cc) + quicklambda, fn, continuations, call_cc) from ...ec import call_ec from ...fploop import looped_over @@ -143,7 +143,7 @@ def result(loop, x, acc): test[looped_over(range(10), acc=0)(lambda loop, x, acc: loop(acc + x)) == 45] with testset("integration with quicklambda"): - # f[] must expand first so that tco sees it as a lambda. + # Use `quicklambda` to force `fn[]` to expand first, so that tco sees it as a lambda. # `quicklambda` is an outside-in macro, so placed on the outside, it expands first. with quicklambda: with tco: @@ -152,10 +152,10 @@ def g(x): # TODO: Improve test to actually detect the tail call. # TODO: Now we just test this runs without errors. - func1 = f[g(3 * _)] # tail call # noqa: F821, _ is magic. + func1 = fn[g(3 * _)] # tail call # noqa: F821, _ is magic. test[func1(10) == 60] - func2 = f[3 * g(_)] # no tail call # noqa: F821, _ is magic. + func2 = fn[3 * g(_)] # no tail call # noqa: F821, _ is magic. test[func2(10) == 60] with testset("integration with continuations"): diff --git a/unpythonic/syntax/tests/test_testingtools.py b/unpythonic/syntax/tests/test_testingtools.py new file mode 100644 index 00000000..13b77a59 --- /dev/null +++ b/unpythonic/syntax/tests/test_testingtools.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +"""Tests for `with test:` block forms — `expect[]`, `return` deprecation, error cases.""" + +from ...syntax import macros, test, test_raises, the, expect # noqa: F401 +from ...test.fixtures import session, testset + +import warnings + + +def _expand(source, filename=""): + """Expand `source` with macros active. Returns the expanded module AST.""" + import mcpyrate.activate # noqa: F401 + from mcpyrate.compiler import expand + return expand(source, filename) + + +_HEADER = """\ +from unpythonic.syntax import macros, test, expect +from unpythonic.test.fixtures import session, testset +""" + + +def runtests(): + with testset("expect[] inside `with test:` block"): + # Basic positive case: a `with test:` block declares its tested + # expression via `expect[expr]`. Failure of the test wouldn't be + # observed inside the block; success is the default contract. + with test: + x = 21 + expect[x + x == 42] + + # Comparison: implicit `the[]` is injected on the LHS, so the failure + # message of an `expect[lhs == rhs]` would report the value of `lhs`. + # Here we just assert the path runs. + with test: + value = "green tea" + expect[value == "green tea"] + + # Explicit `the[]` inside `expect[]` overrides the implicit-LHS rule. + with test: + container = ["a", "b", "c"] + expect["b" in the[container]] + + # No `expect[]` and no `return`: asserts the block completes normally. + with test: + log = [] + log.append("step 1") + log.append("step 2") + # no expect[] — just asserts the statements all completed + + with testset("expect[] error cases (caught at macro expansion)"): + # Two `expect[]` in the same block — SyntaxError. + src = _HEADER + """ +def f(): + with test: + expect[1 == 1] + expect[2 == 2] +""" + try: + _expand(src, "") + except Exception as e: + cur = e + while cur.__cause__ is not None: + cur = cur.__cause__ + test[type(cur) is SyntaxError] + test[the["at most one `expect[]`" in str(cur)]] + else: + test[False, "expected SyntaxError for two expect[]"] + + # `expect[]` and `return` together — SyntaxError. + src = _HEADER + """ +def f(): + with test: + expect[1 == 1] + return 2 == 2 +""" + try: + _expand(src, "") + except Exception as e: + cur = e + while cur.__cause__ is not None: + cur = cur.__cause__ + test[type(cur) is SyntaxError] + test[the["both `expect[]` and `return`" in str(cur)]] + else: + test[False, "expected SyntaxError for expect[] + return together"] + + # `expect[]` outside `with test:` — SyntaxError from the macro itself. + src = _HEADER + """ +expect[1 == 1] +""" + try: + _expand(src, "") + except Exception as e: + cur = e + while cur.__cause__ is not None: + cur = cur.__cause__ + test[type(cur) is SyntaxError] + else: + test[False, "expected SyntaxError for bare expect[]"] + + with testset("`return` form emits DeprecationWarning at expansion time"): + src = _HEADER + """ +def f(): + with test: + return 2 + 2 == 4 +""" + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always") + _expand(src, "") + deprecations = [w for w in captured if issubclass(w.category, DeprecationWarning)] + test[the[len(deprecations)] == 1] + msg = str(deprecations[0].message) + test[the["`return`" in msg and "deprecated" in msg and "expect[]" in msg]] + # The warning carries the user-visible filename and line of the offending `return`. + test[deprecations[0].filename == ""] + # `_HEADER` is two lines, blank line ends it; `def f():` is line 4, + # `with test:` is line 5, `return ...` is line 6. + test[deprecations[0].lineno == 6] + + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/syntax/tests/test_util.py b/unpythonic/syntax/tests/test_util.py index 738f09dc..1ef3f7c8 100644 --- a/unpythonic/syntax/tests/test_util.py +++ b/unpythonic/syntax/tests/test_util.py @@ -7,7 +7,6 @@ from mcpyrate.quotes import macros, q, n, h # noqa: F401, F811 from mcpyrate.metatools import macros, expandrq # noqa: F401, F811 -from ...syntax.astcompat import getconstant, Num, Str from ...syntax.util import (isec, detect_callec, detect_lambda, is_decorator, has_tco, has_curry, has_deco, @@ -156,8 +155,8 @@ def architectural(): test[len(decos) == 3] test[all(type(node) is Call and type(node.func) is Name for node in decos)] test[[node.func.id for node in decos] == ["memoize", "trampolined", "curry"]] - test[type(lam.body) in (Constant, Num)] # Python 3.8+: ast.Constant - test[getconstant(lam.body) == 42] # Python 3.8+: ast.Constant + test[type(lam.body) is Constant] + test[lam.body.value == 42] def test_sort_lambda_decorators(testdata): sort_lambda_decorators(testdata) @@ -185,15 +184,18 @@ def myfunction(x): "finally" collected = [] def collectstrings(tree): - if type(tree) is Expr and type(tree.value) in (Constant, Str): # Python 3.8+: ast.Constant - collected.append(getconstant(tree.value)) + if type(tree) is Expr and type(tree.value) is Constant: + constant_node = tree.value + collected.append(constant_node.value) return [tree] transform_statements(collectstrings, transform_statements_testdata) test[set(collected) == {"function body", "try", "if body", "if else", "finally", "except"}] def ishello(tree): - # Python 3.8+: ast.Constant - return type(tree) is Expr and type(tree.value) in (Constant, Str) and getconstant(tree.value) == "hello" + if type(tree) is Expr and type(tree.value) is Constant: + constant_node = tree.value + return constant_node.value == "hello" + return False # numeric with q as eliminate_ifones_testdata1: diff --git a/unpythonic/syntax/tests/testing_testingtools.py b/unpythonic/syntax/tests/testing_testingtools.py deleted file mode 100644 index 78c799f7..00000000 --- a/unpythonic/syntax/tests/testing_testingtools.py +++ /dev/null @@ -1,297 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utilities for testing. - -The `test[]` macro allows to write unit tests for macro-enabled code, in a -compact assert-like syntax, while letting the rest of the tests run even if -some tests fail. - -This file is not part of the automated test suite of `unpythonic`, hence the -deviation from the common naming scheme. We can hardly use the test framework -to test itself; so this module relies on just asserts. - -There are also not that many automated tests for the test framework - most of -the functionality is visual and it was just eyeballed. See the session example -below to generate lots of colorful output, exercising the different features. -""" - -from ...syntax import macros, test, test_signals, test_raises, fail, error, warn, the # noqa: F401 - -from functools import partial - -from ...test.fixtures import (session, testset, terminate, returns_normally, # noqa: F401 - tests_run, tests_failed, tests_errored, - TestFailure, TestError) - -from ...conditions import invoke, handlers, restarts, cerror # noqa: F401 -from ...excutil import raisef - -def runtests(): - # Low-level machinery. - - # Simple error reporter, just for a demonstration. - # - # If we don't need to configure which restart to invoke after the error has - # been reported, `report` could take just the `err` argument. - def report(the_restart, err): - # print(err, file=sys.stderr) # or log or whatever - invoke(the_restart) - report_and_proceed = partial(report, "proceed") - - # Basic usage. - # - # A `with handlers` block around the tests is mandatory. Without it, - # `test[]` will raise `ControlError` when the condition system detects that - # the cerror (correctable error, signaled by `test[]`) was not handled. - # - # (Only the client code can know what to do with the error, so `test[]` - # cannot automatically write the `with handlers` block for us.) - with handlers((TestFailure, report_and_proceed)): - test[2 + 2 == 5] # fails, but allows further tests to continue - test[2 + 2 == 4] - test[17 + 23 == 40, "my failure message"] - # One wouldn't normally use `assert` in a test module that uses `test[]`, - # but we have to test `test[]` itself somehow. - assert tests_run == 3 # we use the type pun that a box is equal to its content. - assert tests_failed == 1 - assert tests_errored == 0 - - # By setting up our own restart, we can skip the rest of a block of tests. - # - # The handler can be overridden locally. This works, because the - # dynamically most recently bound handler for the same signal type wins - # (see `unpythonic.conditions`). - # - # We can reset the counters by sending a new value into the box. - tests_failed << 0 - tests_errored << 0 - tests_run << 0 - report_and_skip = partial(report, "skip") - with handlers(((TestFailure, TestError), report_and_proceed)): - test[2 + 2 == 5] # fails, but allows further tests to continue - - with restarts(skip=(lambda: None)): # just for control, no return value - with handlers(((TestFailure, TestError), report_and_skip)): - test[2 + 2 == 6] # --> fails, skips the rest of this block - test[2 + 2 == 7] # not reached - - test[2 + 2 == 8] # fails, but allows further tests to continue - test[2 + 2 == 9] - assert tests_run == 4 - assert tests_failed == 4 - assert tests_errored == 0 - - # The test machinery counts an uncaught exception inside a test expr as an error - # (i.e. the test did not run to completion), not a failure. - tests_failed << 0 - tests_errored << 0 - tests_run << 0 - with handlers(((TestFailure, TestError), report_and_proceed)): - test[raisef(RuntimeError)] # errors out, but allows further tests to continue - test[2 + 2 == 4] - test[17 + 23 == 40, "my failure message"] - assert tests_run == 3 - assert tests_failed == 0 - assert tests_errored == 1 - - # Test the `the[]` marker, which changes which subexpression has its value - # captured for test failure message display purposes. - tests_failed << 0 - tests_errored << 0 - tests_run << 0 - with handlers(((TestFailure, TestError), report_and_proceed)): - count = 0 - def counter(): - nonlocal count - count += 1 - return count - test[counter() < counter()] - test[the[counter()] < counter()] - test[counter() < the[counter()]] # evaluation order not affected - assert tests_run == 3 - assert tests_failed == 0 - assert tests_errored == 0 - - # # If you want to proceed after most failures, but there is some particularly - # # critical test which, if it fails, should abort the rest of the whole unit, - # # you can override the handler locally: - # - # def die(err): - # print(err, file=sys.stderr) # or log or whatever - # sys.exit(255) - # - # with handlers(((TestFailure, TestError), report)): - # test[2 + 2 == 5] # fails, but allows further tests to continue - # - # with handlers(((TestFailure, TestError), die)): - # test[2 + 2 == 6] # --> die - # test[17 + 23 == 40, "my failure message"] # not reached - # - # # if this point was ever reached (currently it's not)... - # test[2 + 2 == 7] # ...this fails, but allows further tests to continue - # - # # This works, because the dynamically most recently bound handler for the - # # same signal type wins (see `unpythonic.conditions`). - # # - # # Similarly, if you want to skip the rest of a block of tests upon a failure: - # - # from unpythonic.conditions import restarts, invoker - # - # with handlers(((TestFailure, TestError), report)): - # test[2 + 2 == 5] # fails, but allows further tests to continue - # - # with restarts(skip=(lambda: None)): # just for control, no return value - # with handlers(((TestFailure, TestError), invoker("skip"))): - # test[2 + 2 == 6] # --> fails, skip the rest of this block - # test[17 + 23 == 40, "my failure message"] # not reached - # - # test[2 + 2 == 7] # fails, but allows further tests to continue - - # -------------------------------------------------------------------------------- - # High-level machinery: unpythonic.test.fixtures, a testing framework. - - # - Automatically resume testing upon failure or error, if possible - # - Automatically count passes, fails and errors, summarize totals - # - Print nicely colored ANSI terminal output into `sys.stderr` - # - Don't need to care that it's implemented with conditions and restarts - # - # Example session: - # - # # The session construct provides an exit point for test session - # # termination, and an implicit top-level testset. - # # A session can be started only when not already inside a testset. - # with session("framework demo"): - # # A session may contain bare tests. They are implicitly part of the - # # top-level testset. - # test[2 + 2 == 4] - # # Tests can have a human-readable failure message. - # test[2 + 2 == 5, "should be five, no?"] - # - # # Tests can be further grouped into testsets, if desired. - # with testset(): - # test[2 + 2 == 4] - # test[2 + 2 == 5] - # - # # Testsets can be named. The name is printed in the output. - # with testset("my fancy tests"): - # test[2 + 2 == 4] - # test[raisef(RuntimeError), "augh!"] # exceptions are caught. - # test[cerror(RuntimeError), "owww!"] # signals are caught, too. - # test[2 + 2 == 6] - # - # # A testset reports also any stray signals or exceptions it receives - # # from outside a `test[]` construct. - # # - # # - When a signal arrives via `cerror`, the testset resumes. - # # - When some other signal protocol is used (no "proceed" restart - # # is in scope), the handler returns normally; what then happens - # # depends on which signal protocol it is. - # # - When an exception is caught, the testset terminates, because - # # exceptions do not support resuming. - # cerror(RuntimeError("blargh")) - # raise RuntimeError("gargle") - # - # # Testsets can be nested. - # with testset("outer"): - # with testset("inner 1"): - # test[2 + 2 == 4] - # with testset("inner 2"): - # test[2 + 2 == 4] - # with testset("inner 3"): - # pass - # - # fail["Use fail[] to e.g. signify a line should not be reached."] - # error["Use error[] to e.g. signify optional dependencies failed to load."] - # warn["Use warn[] to e.g. signify that some of your tests are currently disabled."] - # - # # Tests that require statements (e.g. assignments) can be written as a `with test` block. - # # The test block is automatically lifted into a function, so it introduces a local scope. - # # - # # If there is a `return`, the return value will be asserted. - # # If there is no `return`, the test asserts that the block completes normally. - # with testset("test blocks"): - # with test: - # a = 2 - # return a + a == 4 - # - # # A test block can have a failure message: - # with test["should be three, no?"]: - # a = 2 - # return a + a == 3 - # - # # Similarly, there are also `with test_raises` and `with test_signals` blocks, - # # though they don't support `return` - they always assert that the block - # # raises or signals, respectively. - # with test_raises[RuntimeError]: - # raise RuntimeError() - # - # with test_raises[RuntimeError, "should have raised"]: - # raise RuntimeError() - # - # # By default, for test failure reporting, `test[]` captures as "result": - # # - If the test is a comparison: the LHS - # # - Otherwise, the whole expr. - # # To override, tag the interesting part as `the[subexpr]`: - # with testset("the[]"): - # test[5 == 2 + 2] # by default, the framework thinks the LHS "5" is the important part - # test[5 == the[2 + 2]] # override it like this - # test[4 == the[2 + 2]] - # - # # `the[]` also works in `with test` blocks. - # # - # # It doesn't need to be in the `return` expression; it can be on - # # any expression inside the block. - # # - # # Note `with test_raises` and `with test_signals` don't support `the[]`. - # with test: - # a = 2 - # return the[a + a] == 4 - # - # with testset("test_raises"): - # test_raises[RuntimeError, raisef(RuntimeError)] - # test_raises[RuntimeError, 2 + 2 == 4] - # test_raises[RuntimeError, raisef(ValueError)] - # - # with testset("test_signals"): - # test_signals[RuntimeError, cerror(RuntimeError)] - # test_signals[RuntimeError, 2 + 2 == 4] - # test_signals[RuntimeError, cerror(ValueError)] - # - # with testset("nested exceptions"): - # with testset("raise from"): - # try: - # raise ValueError - # except ValueError as e: - # raise RuntimeError from e - # - # with testset("just chain them"): - # try: - # raise ValueError - # except ValueError: - # raise RuntimeError - # - # with testset("normal return, don't care about value"): - # # There's also a block variant that asserts the block completes normally - # # (no exception or signal). - # with test["block variant"]: - # print("hello world") - # - # # To get that effect in the expression variant, call `returns_normally`: - # def f(x): - # return 2 * x - # test[returns_normally(f(21))] - # - # # # The session can be terminated early by calling terminate() - # # # at any point inside the dynamic extent of `with session`. - # # # This causes the `with session` to exit immediately. - # # terminate() - # - # # The session can also be terminated by the first failure in a - # # particular testset by using `terminate` as the `postproc`: - # with testset(postproc=terminate): - # test[2 + 2 == 5] - # test[2 + 2 == 4] # not reached - - print("All tests PASSED") - -if __name__ == '__main__': - runtests() diff --git a/unpythonic/syntax/util.py b/unpythonic/syntax/util.py index 78c697d7..15cfa8ea 100644 --- a/unpythonic/syntax/util.py +++ b/unpythonic/syntax/util.py @@ -16,14 +16,13 @@ from functools import partial -from ast import Call, Lambda, FunctionDef, AsyncFunctionDef, If, stmt +from ast import Call, Constant, Lambda, FunctionDef, AsyncFunctionDef, If, stmt from mcpyrate.core import add_postprocessor from mcpyrate.markers import ASTMarker, delete_markers from mcpyrate.quotes import is_captured_value from mcpyrate.walkers import ASTTransformer, ASTVisitor -from .astcompat import getconstant from .letdoutil import isdo, ExpandedDoView from .nameutil import isx, getname @@ -90,12 +89,12 @@ def detect_lambda(tree): """Find lambdas in tree. Helper for two-pass block macros. A two-pass block macro first performs some processing outside-in, then calls - `expander.visit(tree)` to make any nested macro invocations expand, and then - performs some processing inside-out. + `expander.visit_recursively(tree)` to make any nested macro invocations expand, + and then performs some processing inside-out. Run ``detect_lambda(tree)`` in the outside-in pass, before calling - `expander.visit(tree)`, because nested macro invocations may generate - more lambdas that your block macro is not interested in. + `expander.visit_recursively(tree)`, because nested macro invocations + may generate more lambdas that your block macro is not interested in. The return value is a ``list``of ``id(lam)``, where ``lam`` is a Lambda node that appears in ``tree``. This list is suitable as ``userlambdas`` for the @@ -353,16 +352,12 @@ def eliminate_ifones(body): include a ``call_cc`` (see the example in test_conts_gen.py)... """ def isifone(tree): - if type(tree) is If: - try: - value = getconstant(tree.test) - except TypeError: - pass - else: - if value in (1, True): - return "then" - elif value in (0, False, None): - return "else" + if type(tree) is If and type(tree.test) is Constant: + value = tree.test.value + if value in (1, True): + return "then" + elif value in (0, False, None): + return "else" return False def optimize(tree): # stmt -> list of stmts diff --git a/unpythonic/tco.py b/unpythonic/tco.py index 16eb80da..3eb708ed 100644 --- a/unpythonic/tco.py +++ b/unpythonic/tco.py @@ -127,17 +127,21 @@ def baz(): __all__ = ["jump", "trampolined"] +from collections.abc import Callable from functools import wraps from sys import stderr +from typing import Any, TypeVar from .regutil import register_decorator from .lazyutil import islazy, passthrough_lazy_args, maybe_force_args from .dynassign import dyn +F = TypeVar('F', bound=Callable) + # In principle, jump should have @passthrough_lazy_args, but for performance reasons # it doesn't. "force(target)" is slow, so strict code shouldn't have to do that. # This is handled by a special case in maybe_force_args. -def jump(target, *args, **kwargs): +def jump(target: Callable, *args: Any, **kwargs: Any) -> "_jump": """A jump (noun, not verb). Used in the syntax `return jump(f, ...)` to request the trampoline to @@ -233,7 +237,7 @@ def bar(): # https://stackoverflow.com/questions/6394511/python-functools-wraps-equivalent-for-classes # https://stackoverflow.com/questions/25973376/functools-update-wrapper-doesnt-work-properly#25973438 @register_decorator(priority=40, istco=True) -def trampolined(function): +def trampolined(function: F) -> F: """Decorator to make a function trampolined. Trampolined functions can use ``return jump(f, a, ..., kw=v, ...)`` @@ -243,7 +247,7 @@ def trampolined(function): if not dyn._build_lazy_trampoline: # building a trampoline for regular strict code @wraps(function) - def trampoline(*args, **kwargs): + def trampoline(*args: Any, **kwargs: Any) -> Any: f = function while True: if callable(f): # general case @@ -277,7 +281,7 @@ def trampoline(*args, **kwargs): # This is to avoid a drastic (~10x) performance hit in trampolines # built for regular strict code. @wraps(function) - def trampoline(*args, **kwargs): + def trampoline(*args: Any, **kwargs: Any) -> Any: f = function while True: if callable(f): # the maybe_force_args here causes the performance hit diff --git a/unpythonic/test/fixtures.py b/unpythonic/test/fixtures.py index 76ea87c8..6bce7a67 100644 --- a/unpythonic/test/fixtures.py +++ b/unpythonic/test/fixtures.py @@ -74,17 +74,18 @@ with testset("inner 2"): test[2 + 2 == 4] - # Unconditional errors can be emitted with `error[]`. + # Warnings can be emitted with `warn[]`. # Useful e.g. if an optional dependency is missing: with testset("integration"): try: import blargly except ImportError: - error["blargly not installed, cannot test integration with it."] + warn["blargly not installed, skipping integration tests."] else: ... # blargly integration tests go here - # Similarly, unconditional errors can be emitted with `fail[]`. + # Unconditional errors can be emitted with `error[]`. + # Unconditional failures can be emitted with `fail[]`. # Useful for marking a testing TODO, or for marking a line # that should be unreachable in a code example. with testset("really fancy tests"): @@ -128,13 +129,14 @@ from mcpyrate.bunch import Bunch from mcpyrate.colorizer import Fore, Style, colorize -from ..conditions import handlers, find_restart, invoke +from ..conditions import cerror, handlers, find_restart, invoke from ..collections import box, unbox from ..symbol import sym __all__ = ["session", "testset", "terminate", "returns_normally", "catch_signals", + "emit_warning", "TestConfig", "tests_run", "tests_failed", "tests_errored", "tests_warned", "TestingException", "TestFailure", "TestError", "TestWarning", @@ -182,6 +184,21 @@ def _reset(counter): with _counter_update_lock: counter << 0 +def emit_warning(msg): + """Emit a test warning from infrastructure code (outside a ``test[]`` expression). + + If you are writing tests, use the `warn[]` macro instead. + + Use this function in test runners and other infrastructure that needs to + signal a warning through the test framework without being inside a ``test[]`` + or ``warn[]`` macro. The warning will be displayed and counted by the + nearest enclosing ``testset``. + """ + # Unlike the ``warn[]`` macro, this does not adjust ``tests_run``, + # because no test has been counted for this warning to "replace". + _update(tests_warned, +1) + cerror(TestWarning(msg)) + completed = sym("completed") completed.__doc__ = """TestingException `mode`: the test ran to completion normally. diff --git a/unpythonic/test/runner.py b/unpythonic/test/runner.py new file mode 100644 index 00000000..bea7a21a --- /dev/null +++ b/unpythonic/test/runner.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +"""Generic test runner for projects using ``unpythonic.test.fixtures``. + +Provides test module discovery, version-suffix gating, and a ``run`` +function that wraps the standard session/testset/import_module pattern. + +Usage from a project's top-level ``runtests.py``:: + + import os + from unpythonic.test.runner import discover_testmodules, run + + import mcpyrate.activate # noqa: F401 + + testsets = [("my tests", discover_testmodules(os.path.join("mypackage", "tests")))] + if not run(testsets): + raise SystemExit(1) + +Version-suffixed test modules (e.g. ``test_foo_3_11.py``) are automatically +skipped with a warning on older Pythons. +""" + +import os +import re +import sys +from importlib import import_module + +from .fixtures import session, testset, emit_warning, tests_errored, tests_failed +from ..collections import unbox + +__all__ = ["discover_testmodules", "run"] + +def discover_testmodules(path, prefix="test_", suffix=".py"): + """Discover test modules in a directory. + + Returns a sorted list of dotted module names (e.g. + ``["mypackage.tests.test_foo", "mypackage.tests.test_bar"]``). + + Modules are discovered by filename convention: files matching + ``{prefix}*{suffix}`` in the given directory. + """ + filenames = [fn for fn in os.listdir(path) if fn.startswith(prefix) and fn.endswith(suffix)] + modnames = [_filename_to_modulename(path, fn) for fn in filenames] + return list(sorted(modnames)) + +def _filename_to_modulename(path, filename): + """Convert a path and filename to a dotted module name. + + ``("some/dir", "mod.py")`` → ``"some.dir.mod"`` + """ + # str.replace, not re.sub: on Windows os.path.sep is a lone backslash, + # which as a regex pattern is an incomplete escape and raises re.error. + modpath = path.replace(os.path.sep, ".") + themod = re.sub(r"\.py$", r"", filename) + return ".".join([modpath, themod]) + +def _version_suffix(modulename): + """Parse version suffix from module name. + + E.g. ``"mypackage.tests.test_foo_3_11"`` → ``(3, 11)``, or ``None``. + """ + m = re.search(r"_(\d+)_(\d+)$", modulename) + if m: + return (int(m.group(1)), int(m.group(2))) + return None + +def run(testsets): + """Run test modules, reporting results through ``unpythonic.test.fixtures``. + + ``testsets``: iterable of ``(name, modulenames)`` pairs, where ``name`` + is a human-readable label and ``modulenames`` is a list of dotted module + names. Each module must export a ``runtests()`` function. + + Version-suffixed modules (e.g. ``test_foo_3_11``) are automatically + skipped with a warning on Pythons older than the indicated version. + + Returns ``True`` if all tests passed (no failures or errors). + """ + with session(): + for tsname, modnames in testsets: + with testset(tsname): + for m in modnames: + with testset(m): + ver = _version_suffix(m) + if ver is not None and sys.version_info < ver: + msg = (f"Skipping '{m}' (requires Python {ver[0]}.{ver[1]}+, " + f"running {sys.version_info.major}.{sys.version_info.minor})") + emit_warning(msg) + continue + mod = import_module(m) + mod.runtests() + return (unbox(tests_failed) + unbox(tests_errored)) == 0 diff --git a/unpythonic/tests/test_amb.py b/unpythonic/tests/test_amb.py index 3de66b82..d94e1b3c 100644 --- a/unpythonic/tests/test_amb.py +++ b/unpythonic/tests/test_amb.py @@ -4,7 +4,7 @@ from ..test.fixtures import session, testset from ..amb import (forall, choice, insist, deny, ok, fail, - Assignment, MonadicList, nil) + Choice, MonadicList) def runtests(): with testset("MonadicList (internal utility)"): @@ -13,7 +13,7 @@ def runtests(): test[len(m) == 3] test[m[0] == 1 and m[1] == 2 and m[2] == 3] - m = MonadicList(nil) # special *item* that produces an empty *list* + m = MonadicList() # empty test[tuple(m) == ()] # Monadic bind (for MonadicList, it's flatmap). @@ -25,7 +25,7 @@ def runtests(): # .then(...): discard current value, replace by given value. # The new value must be wrapped in MonadicList. m = MonadicList(1, 2, 3) - const = MonadicList(42) # M b + const = MonadicList(42) # M b (singleton) test[tuple(m.then(const)) == (42, 42, 42)] # one 42 for each element of m test_raises[TypeError, m.then(f)] # expected a MonadicList, got a function @@ -59,7 +59,7 @@ def runtests(): # Usage example for `guard` m = MonadicList(1, 2, 3) test[tuple(m >> (lambda x: MonadicList.guard(x % 2 == 1) - .then(MonadicList(x)))) == (1, 3)] + .then(MonadicList(x)))) == (1, 3)] with testset("basic usage"): test[forall(choice(x=range(5)), @@ -114,11 +114,11 @@ def runtests(): with testset("error cases"): test_raises[ValueError, choice(a=1, b=2)] # choice() takes only one binding - # To trigger this corner case, we must manually create an `Assignment` + # To trigger this corner case, we must manually create a `Choice` # that has an invalid name - in normal use, `choice()` protects against # that by its syntax, since the name of a kwarg must be a valid identifier. invalid_name = "∀δ>0∃ε>0:f(x+δ)-f(x)<ε" - test_raises[ValueError, forall(Assignment(invalid_name, 42))] + test_raises[ValueError, forall(Choice(invalid_name, 42))] test_raises[TypeError, forall(lambda: 42)] # callable body must be able to take in the environment diff --git a/unpythonic/tests/test_arity.py b/unpythonic/tests/test_arity.py index 8716ecd0..59145878 100644 --- a/unpythonic/tests/test_arity.py +++ b/unpythonic/tests/test_arity.py @@ -3,9 +3,7 @@ from ..syntax import macros, test, test_raises, the # noqa: F401 from ..test.fixtures import session, testset -import sys - -from ..arity import (arities, arity_includes, +from ..arity import (arities, arity_includes, # noqa: F401 -- documents API surface required_kwargs, optional_kwargs, kwargs, resolve_bindings, tuplify_bindings, getfunc, UnknownArity) @@ -104,14 +102,6 @@ def instmeth(self): test[arities(target.classmeth) == (1, 1)] test[arities(target.staticmeth) == (1, 1)] - # Methods of builtin types have uninspectable arity up to Python 3.6. - # Python 3.7 seems to fix this at least for `list`, and PyPy3 (7.3.0; Python 3.6.9) - # doesn't have this error either. - if sys.version_info < (3, 7, 0) and sys.implementation.name == "cpython": # pragma: no cover - with testset("uninspectable builtin methods"): - lst = [] - test_raises[UnknownArity, arities(lst.append)] - # resolve_bindings: resolve parameter bindings established by a function # when it is called with the given args and kwargs. # diff --git a/unpythonic/tests/test_assignonce.py b/unpythonic/tests/test_assignonce.py index ccdf9e6f..d3087202 100644 --- a/unpythonic/tests/test_assignonce.py +++ b/unpythonic/tests/test_assignonce.py @@ -20,6 +20,12 @@ def runtests(): with test_raises[AttributeError, "should not be able to rebind an unbound name"]: e.set("c", 3) + with test_raises[AttributeError, "should not be able to delete a defined name (would bypass assign-once)"]: + del e.a + + # `e.a` was 42 from the rebind above; the failed delete must not have removed it. + test[e.a == 42] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_collections.py b/unpythonic/tests/test_collections.py index d318607d..d9156396 100644 --- a/unpythonic/tests/test_collections.py +++ b/unpythonic/tests/test_collections.py @@ -4,6 +4,7 @@ from ..test.fixtures import session, testset from collections.abc import Mapping, MutableMapping, Hashable, Container, Iterable, Sized +from itertools import count, repeat from pickle import dumps, loads import threading @@ -11,6 +12,8 @@ frozendict, view, roview, ShadowedSequence, mogrify, in_slice, index_in_slice) from ..fold import foldr +from ..gmemo import imemoize +from ..symbol import sym from ..llist import cons, ll def runtests(): @@ -87,7 +90,7 @@ def f(b): test[the[b3] == the[b2]] # boxes are considered equal if their contents are # pretty API: unbox(b) is the same as reading b.x - cat = object() + cat = sym("cat") b4 = box(cat) test[b4 is not cat] # the box is not the cat test[unbox(b4) is cat] # but when you look inside the box, you find the cat @@ -96,7 +99,7 @@ def f(b): # b.set(newvalue) is the same as assigning b.x = newvalue # (but like env.set, it's an expression, so you can use it anywhere) - dog = object() + dog = sym("dog") b4.set(dog) test[unbox(b4) is dog] @@ -469,6 +472,16 @@ class Zee: s6 = ShadowedSequence(tpl, slice(2, 4), (23,)) # replacement too short... test_raises[IndexError, s6[3]] # ...which is detected here + # infinite replacements + # Here we must `tuple()` the LHS so that the replacement *iterable*, + # which is not a sequence, is iterated over only once. + test[tuple(ShadowedSequence(tpl, slice(None, None, None), repeat(42))) == (42, 42, 42, 42, 42)] + test[tuple(ShadowedSequence(tpl, slice(None, None, None), count(start=10))) == (10, 11, 12, 13, 14)] + + # reading the start of a memoized infinite replacement backwards + test[tuple(ShadowedSequence(tpl, slice(None, None, -1), imemoize(repeat(42))())) == (42, 42, 42, 42, 42)] + test[tuple(ShadowedSequence(tpl, slice(None, None, -1), imemoize(count(start=10))())) == (14, 13, 12, 11, 10)] + # mogrify: in-place map for various data structures (see docstring for details) with testset("mogrify"): double = lambda x: 2 * x diff --git a/unpythonic/tests/test_conditions.py b/unpythonic/tests/test_conditions.py index e41bf1b4..b8840871 100644 --- a/unpythonic/tests/test_conditions.py +++ b/unpythonic/tests/test_conditions.py @@ -9,10 +9,12 @@ # tests the condition system (up to 0.14.2.1) using plain asserts. # # The really problematic part in a monolithic language extension like -# `unpythonic` is to write tests that test the testing framework. Currently we -# don't do that. The test framework is considered to change at most slowly, so -# for that, manual testing is sufficient (see commented-out example session in -# `unpythonic.syntax.test.testing_testingtools`). +# `unpythonic` is to write tests that test the testing framework. The +# low-level machinery has a bare-`assert` self-test in +# `unpythonic.syntax.tests.selftest_testingtools` (the `selftest_` prefix +# keeps `runtests.py` from picking it up). For a worked example of the +# user-facing framework, see `unpythonic.test.fixtures`'s module +# docstring and the "Test sessions and testsets" chapter in `doc/macros.md`. from ..syntax import macros, test, test_raises, test_signals, fail, the # noqa: F401 from ..test.fixtures import session, testset, catch_signals, returns_normally @@ -27,6 +29,7 @@ from ..excutil import raisef from ..misc import slurp from ..collections import box, unbox +from ..symbol import sym from ..it import subset import threading @@ -44,7 +47,7 @@ def __init__(self, x): # condition, the return value of the restart chosen (by a handler # defined in higher-level code) becomes the result of the block. def lowlevel(): - _drop = object() # gensym/nonce + _drop = sym("_drop") out = [] for k in range(10): with restarts(use_value=(lambda x: x), @@ -408,7 +411,6 @@ def warn_protocol(): # An unhandled `error` or `cerror`, when it **raises** `ControlError`, # sets the cause of that `ControlError` to the original unhandled signal. # In Python 3.7+, this will also produce nice stack traces. - # In up to Python 3.6, it will at least show the chain of causes. with catch_signals(False): try: exc1 = JustTesting("Hullo") @@ -557,7 +559,7 @@ def lowlevel3(): cancel_and_delegate() # Multithreading. Threads behave independently. - with testset("multithreading"): + with testset("thread-safety"): def multithreading(): comm = Queue() def lowlevel4(tag): diff --git a/unpythonic/tests/test_dispatch.py b/unpythonic/tests/test_dispatch.py index fb31df1e..e75e513b 100644 --- a/unpythonic/tests/test_dispatch.py +++ b/unpythonic/tests/test_dispatch.py @@ -3,7 +3,12 @@ from ..syntax import macros, test, test_raises, fail, the # noqa: F401 from ..test.fixtures import session, testset, returns_normally +import collections +import contextlib +import io +import re import typing + from ..fun import curry from ..dispatch import generic, augment, typed, format_methods @@ -272,7 +277,7 @@ def blubnify2(x: float, y: float): with testset("list_methods"): def check_formatted_multimethods(result, expected): - def _remove_space_before_typehint(string): # Python 3.6 doesn't print a space there + def _remove_space_before_typehint(string): # Python 3.6 didn't print a space there, later versions do return string.replace(": ", ":") result_list = result.split("\n") human_readable_header, *multimethod_descriptions = result_list @@ -379,6 +384,113 @@ def flip(traitvalue: IsNotFlippable, x: typing.Any): # noqa: F811 test_raises[TypeError, flip(42), "int should not be flippable"] test_raises[NotImplementedError, flip(2.0), "float should not be registered for the flippable trait"] + # Exercise new typing features (D4 sets 1 and 2) through the dispatch machinery. + # Most-recently-registered multimethod is tried first, so register the + # general case first and the specific ones after (to override). + with testset("@generic with Literal dispatch"): + @generic + def handle_code(code: int): + return "other" + @generic + def handle_code(code: typing.Literal[200, 201]): # noqa: F811 + return "success" + @generic + def handle_code(code: typing.Literal[404]): # noqa: F811 + return "not found" + test[handle_code(200) == "success"] + test[handle_code(201) == "success"] + test[handle_code(404) == "not found"] + test[handle_code(500) == "other"] + + with testset("@generic with Type dispatch"): + @generic + def describe_type(cls: typing.Type[int]): + return "integer type" + @generic + def describe_type(cls: typing.Type[str]): # noqa: F811 + return "string type" + test[describe_type(int) == "integer type"] + test[describe_type(bool) == "integer type"] # bool is a subclass of int + test[describe_type(str) == "string type"] + test_raises[TypeError, describe_type(float)] + + with testset("@generic with mapping variants"): + @generic + def process_mapping(d: typing.Dict[str, int]): + return "dict" + @generic + def process_mapping(d: typing.DefaultDict[str, int]): # noqa: F811 + return "defaultdict" + @generic + def process_mapping(d: typing.Counter[str]): # noqa: F811 + return "counter" + @generic + def process_mapping(d: typing.OrderedDict[str, int]): # noqa: F811 + return "ordereddict" + test[process_mapping(collections.defaultdict(int, a=1)) == "defaultdict"] + test[process_mapping(collections.Counter("abc")) == "counter"] + test[process_mapping(collections.OrderedDict(a=1)) == "ordereddict"] + test[process_mapping({"a": 1}) == "dict"] + + with testset("@generic with IO dispatch"): + @generic + def read_stream(s: typing.TextIO): + return "text" + @generic + def read_stream(s: typing.BinaryIO): # noqa: F811 + return "binary" + test[read_stream(io.StringIO("hello")) == "text"] + test[read_stream(io.BytesIO(b"hello")) == "binary"] + + with testset("@generic with Pattern dispatch"): + @generic + def describe_pattern(p: typing.Pattern[str]): + return "str pattern" + @generic + def describe_pattern(p: typing.Pattern[bytes]): # noqa: F811 + return "bytes pattern" + test[describe_pattern(re.compile(r"\d+")) == "str pattern"] + test[describe_pattern(re.compile(rb"\d+")) == "bytes pattern"] + + with testset("@generic with Generator and ContextManager"): + @generic + def classify(x: typing.Generator): + return "generator" + @generic + def classify(x: typing.ContextManager): # noqa: F811 + return "context manager" + @generic + def classify(x: int): # noqa: F811 + return "int" + def mygen(): + yield 1 + test[classify(mygen()) == "generator"] + test[classify(contextlib.nullcontext()) == "context manager"] + test[classify(42) == "int"] + + with testset("@generic with Iterable dispatch"): + # Best-effort element checking: concrete collections dispatch correctly. + @generic + def process_items(x: typing.Iterable[int]): + return "ints" + @generic + def process_items(x: typing.Iterable[str]): # noqa: F811 + return "strs" + test[process_items([1, 2, 3]) == "ints"] + test[process_items(["a", "b"]) == "strs"] + test[process_items((1, 2)) == "ints"] + test[process_items({"hello", "world"}) == "strs"] + + with testset("@generic with Collection dispatch"): + @generic + def summarize(x: typing.Collection[int]): + return f"collection of {len(list(x))} ints" + @generic + def summarize(x: typing.Collection[str]): # noqa: F811 + return f"collection of {len(list(x))} strs" + test[summarize([1, 2, 3]) == "collection of 3 ints"] + test[summarize(["a", "b"]) == "collection of 2 strs"] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_dynassign.py b/unpythonic/tests/test_dynassign.py index d3457b2a..d19d8ec9 100644 --- a/unpythonic/tests/test_dynassign.py +++ b/unpythonic/tests/test_dynassign.py @@ -38,14 +38,14 @@ def basictests(): test_raises[AttributeError, dyn.b] # no longer exists - with testset("multithreading"): + with testset("thread-safety"): comm = Queue() - def threadtest(q): + def threadtest(que): try: dyn.c # just access dyn.c except AttributeError as err: - q.put(err) - q.put(None) + que.put(err) + que.put(None) with dyn.let(c=42): t1 = threading.Thread(target=threadtest, args=(comm,), kwargs={}) @@ -112,7 +112,7 @@ def threadtest(q): test[noimplicits(dyn.items()) == (("a", 10), ("b", 20))] test[noimplicits(dyn.items()) == ()] - with testset("mass update with multithreading"): + with testset("mass update, thread-safety"): comm = Queue() def worker(): # test[] itself is thread-safe, but the worker threads don't have a diff --git a/unpythonic/tests/test_env.py b/unpythonic/tests/test_env.py index ec0efbd8..1c884987 100644 --- a/unpythonic/tests/test_env.py +++ b/unpythonic/tests/test_env.py @@ -54,7 +54,12 @@ def runtests(): # modify existing binding test[e.set("x", 42) == 42] # returns the new value - test[the[e << ("x", 23) is e]] # instance passthrough for chaining + test[e << ("x", 23) is e] # instance passthrough for chaining + + # `finalize` also passes the instance through, so it can be chained + with env(x=1) as e: + test[e.finalize() is e] + test[env(x=42).finalize().x == 42] # delete a binding with subscript syntax with env(x=1) as e: @@ -131,6 +136,15 @@ def runtests(): with test_raises[AttributeError, "overwriting a reserved name should not be allowed"]: e.set = {1, 2, 3} + # Reserved internal names cannot be clobbered by client code, even + # before finalization. (Previously `_env`/`_finalized` were on a + # `_direct_write` whitelist that bypassed the reserved-name check.) + with env() as e: + with test_raises[AttributeError, "client code must not be able to overwrite the internal _env dict"]: + e._env = {"surprise": 42} + with test_raises[AttributeError, "client code must not be able to overwrite the _finalized flag"]: + e._finalized = True + with env(x=1) as e: e.finalize() with test_raises[TypeError, "deleting binding from finalized environment should not be allowed"]: diff --git a/unpythonic/tests/test_environ.py b/unpythonic/tests/test_environ.py new file mode 100644 index 00000000..7f9f40e6 --- /dev/null +++ b/unpythonic/tests/test_environ.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +from ..syntax import macros, test, test_raises, the # noqa: F401 +from ..test.fixtures import session, testset + +import os + +from ..environ import override + +def runtests(): + with testset("environ.override"): + # Basic override and restore + os.environ["_UNPYTHONIC_TEST_VAR"] = "original" + with override(_UNPYTHONIC_TEST_VAR="overridden"): + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "overridden"] + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "original"] + del os.environ["_UNPYTHONIC_TEST_VAR"] + + # Adding a variable that didn't exist before + key = "_UNPYTHONIC_TEST_NEW" + if key in os.environ: + del os.environ[key] + with override(**{key: "added"}): + test[the[os.environ[key]] == "added"] + test[key not in os.environ] + + # Multiple overrides at once + os.environ["_UNPYTHONIC_TEST_A"] = "a_orig" + os.environ["_UNPYTHONIC_TEST_B"] = "b_orig" + with override(_UNPYTHONIC_TEST_A="a_new", _UNPYTHONIC_TEST_B="b_new"): + test[the[os.environ["_UNPYTHONIC_TEST_A"]] == "a_new"] + test[the[os.environ["_UNPYTHONIC_TEST_B"]] == "b_new"] + test[the[os.environ["_UNPYTHONIC_TEST_A"]] == "a_orig"] + test[the[os.environ["_UNPYTHONIC_TEST_B"]] == "b_orig"] + del os.environ["_UNPYTHONIC_TEST_A"] + del os.environ["_UNPYTHONIC_TEST_B"] + + # Nested overrides (same-thread; RLock allows this) + os.environ["_UNPYTHONIC_TEST_VAR"] = "level0" + with override(_UNPYTHONIC_TEST_VAR="level1"): + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "level1"] + with override(_UNPYTHONIC_TEST_VAR="level2"): + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "level2"] + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "level1"] + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "level0"] + del os.environ["_UNPYTHONIC_TEST_VAR"] + + # Restore on exception + os.environ["_UNPYTHONIC_TEST_VAR"] = "before" + try: + with override(_UNPYTHONIC_TEST_VAR="during"): + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "during"] + raise RuntimeError("boom") + except RuntimeError: + pass + test[the[os.environ["_UNPYTHONIC_TEST_VAR"]] == "before"] + del os.environ["_UNPYTHONIC_TEST_VAR"] + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/tests/test_excutil.py b/unpythonic/tests/test_excutil.py index 858122cf..a8cecf41 100644 --- a/unpythonic/tests/test_excutil.py +++ b/unpythonic/tests/test_excutil.py @@ -7,12 +7,14 @@ from time import sleep import sys -from ..excutil import (raisef, tryf, +from ..excutil import (raisef, tryf, withf, equip_with_traceback, reraise_in, reraise, async_raise) from ..env import env +from contextlib import contextmanager, suppress + def runtests(): # raisef: raise an exception from an expression position with testset("raisef (raise exception from an expression)"): @@ -76,15 +78,67 @@ def runtests(): test_raises[TypeError, tryf(lambda: "hello", ("not a type at all!", lambda: "got a string"))] + # withf: enter context manager(s) in expression position + with testset("withf (with-block in an expression)"): + # A simple value-yielding context manager. + @contextmanager + def producing(value): + yield value + + # A side-effect-only context manager that records enter/exit order. + events = [] + @contextmanager + def tracking(label): + events.append(("enter", label)) + try: + yield label + finally: + events.append(("exit", label)) + + # Bare single CM, body takes the as-value. + test[withf(producing(42), lambda x: x + 1) == 43] + + # Single CM in a 1-tuple — equivalent. + test[withf((producing(42),), lambda x: x + 1) == 43] + + # Multiple CMs: body receives all as-values, in order. + test[withf((producing("a"), producing("b")), lambda x, y: x + y) == "ab"] + + # Thunk body: as-values discarded. Useful for `with lock: ...` style. + events.clear() + test[withf(tracking("L"), lambda: "done") == "done"] + test[events == [("enter", "L"), ("exit", "L")]] + + # Multiple CMs entered left-to-right, exited in reverse. + events.clear() + withf((tracking("A"), tracking("B"), tracking("C")), lambda: None) + test[events == [("enter", "A"), ("enter", "B"), ("enter", "C"), + ("exit", "C"), ("exit", "B"), ("exit", "A")]] + + # Exception inside body propagates after CMs exit. + events.clear() + test_raises[ValueError, withf(tracking("X"), lambda: raisef(ValueError("boom")))] + test[events == [("enter", "X"), ("exit", "X")]] + + # CM may suppress an exception. `withf` returns whatever body returned + # before the raise — i.e. `None` if no return ran, since `suppress` only + # swallows the exception escaping `body`. + test[withf(suppress(ValueError), + lambda: raisef(ValueError("ignored"))) is None] + + # Return value is whatever body returns; supports any object. + test[withf(producing(None), lambda x: (x, "tuple")) == (None, "tuple")] + + # Bad input: not a CM and not iterable. + test_raises[TypeError, withf(42, lambda: None)] + + # Bad input: iterable containing a non-CM. + test_raises[TypeError, withf((producing(1), "not a CM"), lambda x, y: None)] + with testset("equip_with_traceback"): e = Exception("just testing") - try: - e = equip_with_traceback(e) - except NotImplementedError: - warn["equip_with_traceback only supported on Python 3.7+, skipping test."] - else: - # Can't do meaningful testing on the result, so just check it's there. - test[e.__traceback__ is not None] + e = equip_with_traceback(e) + test[e.__traceback__ is not None] # Can't do meaningful testing on the result, so just check it's there. test_raises[TypeError, equip_with_traceback("not an exception")] diff --git a/unpythonic/tests/test_fix.py b/unpythonic/tests/test_fix.py index 0a93b18d..bc17d873 100644 --- a/unpythonic/tests/test_fix.py +++ b/unpythonic/tests/test_fix.py @@ -105,7 +105,7 @@ def iterate1_rec(f, x): f, c = cosser2(1) # f ends up in the return value because it's in the args of iterate1_rec. test[the[c] == the[cos(c)]] - with testset("multithreading"): + with testset("thread-safety"): def threadtest(): a_calls = [] @fix() @@ -119,9 +119,9 @@ def b(tid, k): return a(tid, (k + 1) % 3) comm = Queue() - def worker(q): + def worker(que): r = a(id(threading.current_thread()), 0) - q.put(r is NoReturn) + que.put(r is NoReturn) n = 1000 threads = [threading.Thread(target=worker, args=(comm,), kwargs={}) for _ in range(n)] diff --git a/unpythonic/tests/test_fold.py b/unpythonic/tests/test_fold.py index 7e442960..d12ce6ea 100644 --- a/unpythonic/tests/test_fold.py +++ b/unpythonic/tests/test_fold.py @@ -10,6 +10,7 @@ foldl, foldr, reducel, reducer, rreducel, rfoldl, unfold, unfold1, prod, running_minmax, minmax) from ..fun import curry, composer, composerc, composel, to1st, rotate +from ..funutil import Values from ..llist import cons, nil, ll, lreverse from ..it import take, tail @@ -182,15 +183,18 @@ def step2(k): # x0, x0 + 2, x0 + 4, ... return (k, k + 2) # (value, newstate) def fibo(a, b): - return (a, b, a + b) # (value, *newstates) + # First positional return value is the value to yield. + # Everything else is newstate, to be unpacked to `fibo`'s + # args/kwargs at the next iteration. + return Values(a, a=b, b=a + b) def myiterate(f, x): # x0, f(x0), f(f(x0)), ... - return (x, f, f(x)) + return Values(x, f=f, x=f(x)) def zip_two(As, Bs): if len(As) and len(Bs): (A0, *moreAs), (B0, *moreBs) = As, Bs - return ((A0, B0), moreAs, moreBs) + return Values((A0, B0), As=moreAs, Bs=moreBs) test[tuple(take(10, unfold1(step2, 10))) == (10, 12, 14, 16, 18, 20, 22, 24, 26, 28)] test[tuple(take(10, unfold(fibo, 1, 1))) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55)] diff --git a/unpythonic/tests/test_fpnumerics.py b/unpythonic/tests/test_fpnumerics.py index 4c530a07..e7f8eea3 100644 --- a/unpythonic/tests/test_fpnumerics.py +++ b/unpythonic/tests/test_fpnumerics.py @@ -5,14 +5,16 @@ Based on various sources; links provided in the source code comments. """ -from ..syntax import macros, test # noqa: F401 +from ..syntax import macros, test, warn # noqa: F401 from ..test.fixtures import session, testset, returns_normally from operator import add, mul from itertools import repeat -from math import sin, pi, log2 +from math import sin, cos, pi, log2 +from cmath import sin as complex_sin from ..fun import curry +from ..funutil import Values from ..it import unpack, drop, take, tail, first, second, last, iterate1, within from ..fold import scanl, scanl1, unfold from ..mathseq import gmathify, imathify @@ -132,7 +134,7 @@ def nats(start=0): @gmathify def fibos(): def nextfibo(a, b): - return a, b, a + b + return Values(a, a=b, b=a + b) return unfold(nextfibo, 1, 1) @gmathify def pows(): @@ -192,6 +194,46 @@ def best_differentiate_with_tol(h0, f, x, eps): # Thanks to super_improve, this actually requires taking only three terms. test[abs(best_differentiate_with_tol(0.1, sin, pi / 2, 1e-8)) < 1e-11] + # This is strictly speaking not FP, but it is worth noting that + # numerical derivatives of real-valued functions can also be estimated + # using a not very well known trick based on complex numbers. + # + # Let f be a complex analytic function (or a complex analytic piece of a piecewise defined + # function) that takes on real values for inputs on the real line. Consider the Taylor series + # f(x + iε) = f(x) + i ε f'(x) + O(ε²) + # where x is a real number, i = √-1, and ε is a small real number. We have + # real(f(x + iε)) = f(x) + O(ε²) + # imag(f(x + iε) / ε) = f'(x) + # This gives us both f(x) and f'(x) with one complex-valued computation. + # No cancellation, so we can take a really small ε (e.g. ε = 1e-150). + # + # This comes from + # Goodfellow, Bengio and Courville (2016): Deep Learning, MIT press, p. 434: + # https://www.deeplearningbook.org/contents/guidelines.html + # who cite it to originate from + # William Squire and George Trapp (1998). Using Complex Variables to Estimate Derivatives + # of Real Functions. SIAM Review, 40(1), 110-112. http://doi.org/10.1137/S003614459631241X + # who, in turn, cite it to originate from + # J. N. Lyness and C. B. Moler. 1967. Numerical differentiation of analytic functions, + # SIAM J. Numer. Anal., 4, pp. 202–210. + # and + # J. N. Lyness. 1967. Numerical algorithms based on the theory of complex variables, + # Proc. ACM 22nd Nat. Conf., Thompson Book Co., Washington, DC, pp. 124–134. + # + # See also + # Joaquim J Martins, Peter Sturdza, Juan J Alonso. The complex-step derivative approximation. + # ACM Transactions on Mathematical Software, Association for Computing Machinery, 2003, 29, + # pp.245-262. 10.1145/838250.838251. hal-01483287. + # https://hal.archives-ouvertes.fr/hal-01483287/document + # + # So this technique has been known since the late 1960s, but even as of this writing, + # 55 years later (2022), it has not seen much use. + eps = 1e-150 + def complex_diff(f, x): + return (f(x + eps * 1j) / eps).imag + # This is so accurate in this simple case that we can test for floating point equality. + test[complex_diff(complex_sin, 0.1) == cos(0.1)] + # pi approximation with Euler series acceleration # # See SICP, 2nd ed., sec. 3.5.3. diff --git a/unpythonic/tests/test_fun.py b/unpythonic/tests/test_fun.py index 4eaa7650..cfcd7551 100644 --- a/unpythonic/tests/test_fun.py +++ b/unpythonic/tests/test_fun.py @@ -5,6 +5,9 @@ from collections import Counter import sys +from queue import Queue +import threading +from time import sleep from ..dispatch import generic from ..fun import (memoize, partial, curry, apply, @@ -16,6 +19,8 @@ to1st, to2nd, tokth, tolast, to, withself) from ..funutil import Values +from ..it import allsame +from ..misc import slurp from ..dynassign import dyn @@ -135,6 +140,36 @@ def t(): fail["memoize should not prevent exception propagation."] # pragma: no cover test[evaluations == 1] + with testset("@memoize thread-safety"): + def threadtest(): + @memoize + def f(x): + # Sleep a "long" time to make actual concurrent operation more likely. + sleep(0.001) + + # The trick here is that because only one thread will acquire the lock + # for the memo, then for the same `x`, all the results should be the same. + return (id(threading.current_thread()), x) + + comm = Queue() + def worker(que): + # The value of `x` doesn't matter, as long as it's the same in all workers. + r = f(42) + que.put(r) + + n = 1000 + threads = [threading.Thread(target=worker, args=(comm,), kwargs={}) for _ in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + + # Test that all threads finished, and that the results from each thread are the same. + results = slurp(comm) + test[the[len(results)] == the[n]] + test[allsame(results)] + threadtest() + with testset("partial (type-checking wrapper)"): def nottypedfunc(x): return "ok" @@ -219,16 +254,17 @@ def double(x): with dyn.let(curry_context=["whatever"]): return the[curry(double, 2, nosucharg="foo")] == Values(4, nosucharg="foo") - # This doesn't occur on PyPy3. + # This doesn't occur on PyPy3, or on CPython 3.11+. if sys.implementation.name == "cpython": # pragma: no cover - with testset("uninspectable builtin functions"): - test_raises[ValueError, curry(print)] # builtin function that fails `inspect.signature` - - # Internal feature, used by curry macro. If uninspectables are said to be ok, - # then attempting to curry an uninspectable simply returns the original function. - m1 = print - m2 = curry(print, _curry_allow_uninspectable=True) - test[the[m2] is the[m1]] + if sys.version_info < (3, 11, 0): + with testset("uninspectable builtin functions"): + test_raises[ValueError, curry(print)] # builtin function that fails `inspect.signature` + + # Internal feature, used by curry macro. If uninspectables are said to be ok, + # then attempting to curry an uninspectable simply returns the original function. + m1 = print + m2 = curry(print, _curry_allow_uninspectable=True) + test[the[m2] is the[m1]] with testset("curry kwargs support"): @curry diff --git a/unpythonic/tests/test_funutil.py b/unpythonic/tests/test_funutil.py index 067e622d..6402470a 100644 --- a/unpythonic/tests/test_funutil.py +++ b/unpythonic/tests/test_funutil.py @@ -6,8 +6,8 @@ from operator import add from functools import partial -# `Values` is tested where function composition utilities that use it are; the class itself is trivial. -from ..funutil import call, callwith +# `Values` is also tested where function composition utilities that use it are. +from ..funutil import call, callwith, Values, valuify def runtests(): with testset("@call (def as code block)"): @@ -94,6 +94,98 @@ def mul3(a, b, c): lambda x: x**(1 / 2)]) test[tuple(m) == (6, 9, 3**(1 / 2))] + with testset("Values unpacking in call/callwith"): + # Leading Values: rets become positional args, kwrets become kwargs. + v = Values(1, 2, x=3) + test[call(lambda a, b, x: (a, b, x), v) == (1, 2, 3)] + test[call(add, Values(2, 3)) == 5] + + # Mixed with regular args: Values expands at its position, others stay put. + def f3(a, b, c): + return (a, b, c) + test[call(f3, 1, Values(2, 3)) == (1, 2, 3)] + test[call(f3, Values(1, 2), 3) == (1, 2, 3)] + test[call(f3, 1, Values(2), 3) == (1, 2, 3)] + + # Multiple Values expand in left-to-right order. + def f4(a, b, c, d): + return (a, b, c, d) + test[call(f4, Values(1, 2), Values(3, 4)) == (1, 2, 3, 4)] + test[call(f4, Values(1), 2, Values(3, 4)) == (1, 2, 3, 4)] + + # Trailing positional and keyword args merge after the Values. + def f5(a, b, c, x, y): + return (a, b, c, x, y) + test[call(f5, Values(1, 2, x=10), 3, y=20) == (1, 2, 3, 10, 20)] + + # Rightmost wins per unique keyword name. + # Explicit kwargs override Values.kwrets: + test[call(f5, Values(1, 2, x=10), 3, x=99, y=20) == (1, 2, 3, 99, 20)] + # Among multiple Values, the later one's kwrets override the earlier: + def fkw(a, b, *, x): + return (a, b, x) + test[call(fkw, Values(1, x=10), Values(2, x=20)) == (1, 2, 20)] + + # callwith: same rules, applied at definition time. + test[callwith(Values(2, 3))(add) == 5] + test[callwith(Values(1, 2, x=3))(lambda a, b, x: (a, b, x)) == (1, 2, 3)] + test[callwith(1, Values(2, 3))(f3) == (1, 2, 3)] + test[callwith(Values(1, 2), Values(3, 4))(f4) == (1, 2, 3, 4)] + test[callwith(Values(1, 2, x=10), 3, y=20)(f5) == (1, 2, 3, 10, 20)] + test[callwith(Values(1, 2, x=10), 3, x=99, y=20)(f5) == (1, 2, 3, 99, 20)] + test[callwith(Values(1, x=10), Values(2, x=20))(fkw) == (1, 2, 20)] + + # The `Values` abstraction is used by various parts of `unpythonic` that + # deal with function composition; particularly `curry`, the `compose` and + # `pipe` families, and the `with continuations` macro. + with testset("Values (multiple-return-values, named return values)"): + def f(): + return Values(1, 2, 3) + result = f() + test[isinstance(result, Values)] + test[result.rets == (1, 2, 3)] + test[not result.kwrets] + test[result[0] == 1] + test[result[:-1] == (1, 2)] + a, b, c = result # if no kwrets, can be unpacked like a tuple + a, b, c = f() + + def g(): + return Values(x=3) # named return value + result = g() + test[isinstance(result, Values)] + test[not result.rets] + test[result.kwrets == {"x": 3}] # actually a `frozendict` + test["x" in result] # `in` looks in the named part + test[result["x"] == 3] + test[result.get("x", None) == 3] + test[result.get("y", None) is None] + test[tuple(result.keys()) == ("x",)] # also `values()`, `items()` + + def h(): + return Values(1, 2, x=3) + result = h() + test[isinstance(result, Values)] + test[result.rets == (1, 2)] + test[result.kwrets == {"x": 3}] + a, b = result.rets # positionals can always be unpacked explicitly + test[result[0] == 1] + test["x" in result] + test[result["x"] == 3] + + def silly_but_legal(): + return Values(42) + result = silly_but_legal() + test[result.rets[0] == 42] + test[result.ret == 42] # shorthand for single-value case + + with testset("valuify (convert tuple as multiple-return-values into Values)"): + @valuify + def f(x, y, z): + return x, y, z + test[isinstance(f(1, 2, 3), Values)] + test[f(1, 2, 3) == Values(1, 2, 3)] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_fup.py b/unpythonic/tests/test_fup.py index 335ca0b0..ee77832f 100644 --- a/unpythonic/tests/test_fup.py +++ b/unpythonic/tests/test_fup.py @@ -3,11 +3,12 @@ from ..syntax import macros, test, test_raises, the # noqa: F401 from ..test.fixtures import session, testset -from itertools import repeat +from itertools import count, repeat from collections import namedtuple from ..fup import fupdate from ..collections import frozendict +from ..gmemo import imemoize def runtests(): with testset("mutable sequence"): @@ -77,6 +78,9 @@ def runtests(): test[tup == (1, 2, 3, 4, 5)] test[out == (4, 3, 2, 1, 0)] + out = fupdate(tup, slice(None, None, -1), range(5)) # no tuple() needed + test[out == (4, 3, 2, 1, 0)] + with testset("multiple individual items"): tup = (1, 2, 3, 4, 5) out = fupdate(tup, (1, 2, 3), (17, 23, 42)) @@ -90,6 +94,24 @@ def runtests(): test[tup == tuple(range(10))] test[out == (2, 3, 2, 3, 2, 3, 2, 3, 2, 3)] + with testset("infinite replacement"): + tup = (1, 2, 3, 4, 5) + out = fupdate(tup, slice(None, None, None), repeat(42)) + test[out == (42, 42, 42, 42, 42)] + + tup = (1, 2, 3, 4, 5) + out = fupdate(tup, slice(None, None, None), count(start=10)) + test[out == (10, 11, 12, 13, 14)] + + with testset("memoized infinite replacement, reading its start backwards"): + tup = (1, 2, 3, 4, 5) + out = fupdate(tup, slice(None, None, -1), imemoize(repeat(42))()) + test[out == (42, 42, 42, 42, 42)] + + tup = (1, 2, 3, 4, 5) + out = fupdate(tup, slice(None, None, -1), imemoize(count(start=10))()) + test[out == (14, 13, 12, 11, 10)] + with testset("mix and match"): tup = tuple(range(10)) out = fupdate(tup, (slice(0, 10, 2), slice(1, 10, 2), 6), @@ -105,6 +127,10 @@ def runtests(): # cannot specify both indices and bindings test_raises[ValueError, fupdate(tup, slice(1, None, 2), (10,), somename="some value")] + # not memoized, cannot read a general iterable backwards + tup = (1, 2, 3, 4, 5) + test_raises[IndexError, fupdate(tup, slice(None, None, -1), count(start=10))] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_gmemo.py b/unpythonic/tests/test_gmemo.py index a0e05627..702d4ef6 100644 --- a/unpythonic/tests/test_gmemo.py +++ b/unpythonic/tests/test_gmemo.py @@ -92,6 +92,47 @@ def gen(): fail["Should have raised at the second next() call."] # pragma: no cover test[total_evaluations == 2] + with testset("subscripting to get already computed items"): + @gmemoize + def gen(): + yield from range(5) + g3 = gen() + + # Any item that has entered the memo can be retrieved by subscripting. + # len() is the current length of the memo. + test[len(g3) == 0] + next(g3) + test[len(g3) == 1] + next(g3) + test[len(g3) == 2] + next(g3) + test[len(g3) == 3] + test[g3[0] == 0] + test[g3[1] == 1] + test[g3[2] == 2] + + # Items not yet memoized cannot be retrieved from the memo. + test_raises[IndexError, g3[3]] + + # Negative indices work too, counting from the current end of the memo. + test[g3[-1] == 2] + test[g3[-2] == 1] + test[g3[-3] == 0] + + # Counting back past the start is an error, just like in `list`. + test_raises[IndexError, g3[-4]] + + # Slicing is supported. + test[g3[0:3] == [0, 1, 2]] + test[g3[0:2] == [0, 1]] + test[g3[::-1] == [2, 1, 0]] + test[g3[0::2] == [0, 2]] + test[g3[2::-2] == [2, 0]] + + # Out-of-range slices produce the empty list, like in `list`. + test[g3[3:] == []] + test[g3[-4::-1] == []] + with testset("memoizing a sequence partially"): # To do this, build a chain of generators, then memoize only the last one: evaluations = Counter() @@ -211,7 +252,7 @@ def mprimes3(): while True: nextp = next(theprimes) - lastdigits = [n for n in lastdigits if not n % p == 0] + lastdigits = [n for n in lastdigits if n % p != 0] ns = [k * b + m for k in range(1, nextp) for m in lastdigits] # in ns, we have already eliminated the first np primes as possible factors, so skip checking them @@ -237,7 +278,7 @@ def manual_mprimes3(): while True: nextp = memo[np] - lastdigits = [n for n in lastdigits if not n % p == 0] + lastdigits = [n for n in lastdigits if n % p != 0] ns = [k * b + m for k in range(1, nextp) for m in lastdigits] for n in ns: @@ -264,7 +305,7 @@ def manual_mprimes4(): while True: nextp = memo[np] - lastdigits = [n for n in lastdigits if not n % p == 0] + lastdigits = [n for n in lastdigits if n % p != 0] ns = [k * b + m for k in range(1, nextp) for m in lastdigits] for n in ns: diff --git a/unpythonic/tests/test_it.py b/unpythonic/tests/test_it.py index ffde8b12..50fe546a 100644 --- a/unpythonic/tests/test_it.py +++ b/unpythonic/tests/test_it.py @@ -7,7 +7,7 @@ from itertools import tee, count, takewhile from operator import add, itemgetter from collections import deque -from math import cos, sqrt +from math import cos from ..it import (map, mapr, rmap, zipr, rzip, map_longest, mapr_longest, rmap_longest, @@ -22,10 +22,9 @@ flatten, flatten1, flatten_in, iterate1, iterate, partition, - partition_int, inn, iindex, find, window, chunked, - within, fixpoint, + within, interleave, subset, powerset, allsame) @@ -35,7 +34,6 @@ from ..gmemo import imemoize, gmemoize from ..mathseq import s from ..misc import Popper -from ..numutil import ulp def runtests(): with testset("mapping and zipping"): @@ -343,7 +341,9 @@ def primes(): S = {"cat", "lynx", "lion", "tiger"} # unordered test[all(subset(tuple(s), S) for s in powerset(S))] - # repeated function application + # Repeated function application. + # If you want to compute arithmetic fixpoints (like we do here for testing), + # see `unpythonic.numutil.fixpoint`. with testset("iterate1, iterate"): test[last(take(100, iterate1(cos, 1.0))) == 0.7390851332151607] @@ -351,9 +351,9 @@ def primes(): # it doesn't matter where you start, the fixed point of cosine # remains the same. def cos3(a, b, c): - return cos(a), cos(b), cos(c) + return Values(cos(a), cos(b), cos(c)) fp = 0.7390851332151607 - test[the[last(take(100, iterate(cos3, 1.0, 2.0, 3.0)))] == (the[fp], fp, fp)] + test[the[last(take(100, iterate(cos3, 1.0, 2.0, 3.0)))] == Values(the[fp], fp, fp)] # within() - terminate a Cauchy sequence after a tolerance is reached. # The condition is `abs(a - b) <= tol` **for the last two yielded items**. @@ -373,47 +373,13 @@ def g2(): yield 4 test[tuple(within(0, g2())) == (1, 2, 3, 4, 4)] - # Arithmetic fixed points. - with testset("fixpoint (arithmetic fixed points)"): - c = fixpoint(cos, x0=1) - test[the[c] == the[cos(c)]] # 0.7390851332151607 - - # Actually "Newton's" algorithm for the square root was already known to the - # ancient Babylonians, ca. 2000 BCE. (Carl Boyer: History of mathematics) - def sqrt_newton(n): - def sqrt_iter(x): # has an attractive fixed point at sqrt(n) - return (x + n / x) / 2 - return fixpoint(sqrt_iter, x0=n / 2) - # different algorithm, so not necessarily equal down to the last bit - # (caused by the fixpoint update becoming smaller than the ulp, so it - # stops there, even if the limit is still one ulp away). - test[abs(the[sqrt_newton(2)] - the[sqrt(2)]) <= the[ulp(1.414)]] - # partition: split an iterable according to a predicate with testset("partition"): iseven = lambda item: item % 2 == 0 test[[tuple(it) for it in partition(iseven, range(10))] == [(1, 3, 5, 7, 9), (0, 2, 4, 6, 8)]] - # partition_int: split a small positive integer, in all possible ways, into smaller integers that sum to it - with testset("partition_int"): - test[tuple(partition_int(4)) == ((4,), (3, 1), (2, 2), (2, 1, 1), (1, 3), (1, 2, 1), (1, 1, 2), (1, 1, 1, 1))] - test[tuple(partition_int(5, lower=2)) == ((5,), (3, 2), (2, 3))] - test[tuple(partition_int(5, lower=2, upper=3)) == ((3, 2), (2, 3))] - test[tuple(partition_int(10, lower=3, upper=5)) == ((5, 5), (4, 3, 3), (3, 4, 3), (3, 3, 4))] - test[all(sum(terms) == 10 for terms in partition_int(10))] - test[all(sum(terms) == 10 for terms in partition_int(10, lower=3))] - test[all(sum(terms) == 10 for terms in partition_int(10, lower=3, upper=5))] - - test_raises[TypeError, partition_int("not a number")] - test_raises[TypeError, partition_int(4, lower="not a number")] - test_raises[TypeError, partition_int(4, upper="not a number")] - test_raises[ValueError, partition_int(-3)] - test_raises[ValueError, partition_int(4, lower=-1)] - test_raises[ValueError, partition_int(4, lower=5)] - test_raises[ValueError, partition_int(4, upper=-1)] - test_raises[ValueError, partition_int(4, upper=5)] - test_raises[ValueError, partition_int(4, lower=3, upper=2)] - + # Test whether all items of an iterable are equal. + # (Short-circuits at the first item that is different.) with testset("allsame"): test[allsame(())] test[allsame((1,))] diff --git a/unpythonic/tests/test_llist.py b/unpythonic/tests/test_llist.py index c6798cf1..0b5fe230 100644 --- a/unpythonic/tests/test_llist.py +++ b/unpythonic/tests/test_llist.py @@ -5,7 +5,10 @@ from pickle import dumps, loads -from ..llist import (cons, car, cdr, nil, ll, llist, +from dataclasses import FrozenInstanceError + +from ..llist import (FrozenAttributeError, + cons, car, cdr, nil, ll, llist, caar, cdar, cadr, cddr, caddr, cdddr, member, lreverse, lappend, lzip, BinaryTreeIterator, JackOfAllTradesIterator, @@ -22,8 +25,21 @@ def runtests(): test_raises[TypeError, car("sedan")] test_raises[TypeError, cdr("disc")] - with test_raises[TypeError, "cons cells should be immutable"]: + with test_raises[TypeError, "cons cells should be immutable (no attribute assignment)"]: c.car = 3 + with test_raises[TypeError, "cons cells should be immutable (no attribute deletion)"]: + del c.car + with test_raises[TypeError, "cons cells should be immutable (no new attributes)"]: + c.extra = "nope" + # The exception is `FrozenAttributeError`, a shim that inherits from both + # `TypeError` (legacy unpythonic <= 2.x) and `dataclasses.FrozenInstanceError` + # (stdlib convention). All three `except` clauses catch. + with test_raises[FrozenAttributeError, "should be a FrozenAttributeError"]: + cons(1, 2).car = 3 + with test_raises[FrozenInstanceError, "should also be catchable as the stdlib FrozenInstanceError"]: + cons(1, 2).car = 3 + with test_raises[AttributeError, "should also be catchable as AttributeError (via FrozenInstanceError)"]: + cons(1, 2).car = 3 test[the[c == c]] test[the[cons(1, 2) == cons(1, 2)]] diff --git a/unpythonic/tests/test_mathseq.py b/unpythonic/tests/test_mathseq.py index e04f7305..4dad8134 100644 --- a/unpythonic/tests/test_mathseq.py +++ b/unpythonic/tests/test_mathseq.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- -from ..syntax import macros, test, test_raises, error, the # noqa: F401 +from ..syntax import macros, test, test_raises, warn, the # noqa: F401 from ..test.fixtures import session, testset -from operator import mul +from operator import add, mul from math import exp, trunc, floor, ceil from ..mathseq import (s, imathify, gmathify, sadd, smul, spow, cauchyprod, - primes, fibonacci, + primes, fibonacci, triangular, sign, log) from ..it import take, last from ..fold import scanl @@ -27,7 +27,7 @@ def runtests(): try: from sympy import symbols except ImportError: # pragma: no cover - error["SymPy not installed in this Python, cannot test symbolic input for mathseq."] + warn["SymPy not installed in this Python, skipping symbolic input tests for mathseq."] else: x = symbols("x", positive=True) test[sign(x) == +1] @@ -40,7 +40,7 @@ def runtests(): try: from sympy import symbols, exp as symbolicExp, E as NeperE except ImportError: # pragma: no cover - error["SymPy not installed in this Python, cannot test symbolic input for mathseq."] + warn["SymPy not installed in this Python, skipping symbolic input tests for mathseq."] else: test[log(NeperE**2) == 2] x = symbols("x", positive=True) @@ -172,7 +172,7 @@ def runtests(): test_raises[ValueError, cauchyprod(s(1, 3, 5, ...), s(2, 4, 6, ...), require="invalid_value")] with testset("imathify, gmathify (infix syntax for arithmetic)"): - # Sequences returned by `s` are `imathify`'d implicitly. + # Sequences returned by `s` are imathified implicitly. test[tuple(take(5, s(1, 3, 5, ...) + s(2, 4, 6, ...))) == (3, 7, 11, 15, 19)] test[tuple(take(5, 1 + s(1, 3, ...))) == (2, 4, 6, 8, 10)] test[tuple(take(5, 1 - s(1, 3, ...))) == (0, -2, -4, -6, -8)] @@ -328,7 +328,7 @@ def runtests(): try: from sympy import symbols except ImportError: # pragma: no cover - error["SymPy not installed in this Python, cannot test symbolic input for mathseq."] + warn["SymPy not installed in this Python, skipping symbolic input tests for mathseq."] else: x0 = symbols("x0", real=True) k = symbols("k", positive=True) # important for geometric series @@ -359,10 +359,14 @@ def runtests(): with testset("some special sequences"): test[tuple(take(10, primes())) == (2, 3, 5, 7, 11, 13, 17, 19, 23, 29)] test[tuple(take(10, fibonacci())) == (1, 1, 2, 3, 5, 8, 13, 21, 34, 55)] + test[tuple(take(10, triangular())) == (1, 3, 6, 10, 15, 21, 28, 36, 45, 55)] test[tuple(take(10, primes(optimize="speed"))) == (2, 3, 5, 7, 11, 13, 17, 19, 23, 29)] test[tuple(take(10, primes(optimize="memory"))) == (2, 3, 5, 7, 11, 13, 17, 19, 23, 29)] - test_raises[ValueError, primes(optimize="fun")] # only "speed" and "memory" modes exist + test_raises[ValueError, primes(optimize="fun")] # unfortunately only "speed" and "memory" modes exist + + triangulars = imemoize(scanl(add, 1, s(2, 3, ...))) + test[tuple(take(10, triangulars())) == tuple(take(10, triangular()))] factorials = imemoize(scanl(mul, 1, s(1, 2, ...))) # 0!, 1!, 2!, ... test[last(take(6, factorials())) == 120] diff --git a/unpythonic/tests/test_misc.py b/unpythonic/tests/test_misc.py index 3002c21c..e3a2be51 100644 --- a/unpythonic/tests/test_misc.py +++ b/unpythonic/tests/test_misc.py @@ -4,7 +4,11 @@ from ..test.fixtures import session, testset from collections import deque +import logging +import os from queue import Queue +import sys +import tempfile from ..misc import (pack, namelambda, @@ -13,7 +17,10 @@ Popper, CountingIterator, slurp, callsite_filename, - safeissubclass) + safeissubclass, + maybe_open, redirect_stdin, + UnionFilter, + si_prefix) from ..fun import withself def runtests(): @@ -59,13 +66,13 @@ def __init__(self, x): self.x = x w = Wrapper(Wrapper(42)) - test[type(getattr(w, "x")) == Wrapper] - test[type(getattrrec(w, "x")) == int] + test[type(getattr(w, "x")) is Wrapper] + test[type(getattrrec(w, "x")) is int] test[getattrrec(w, "x") == 42] setattrrec(w, "x", 23) - test[type(getattr(w, "x")) == Wrapper] - test[type(getattrrec(w, "x")) == int] + test[type(getattr(w, "x")) is Wrapper] + test[type(getattrrec(w, "x")) is int] test[getattrrec(w, "x") == 23] # pop-while iterator @@ -118,6 +125,12 @@ def __init__(self, x): with testset("callsite_filename"): test["test_misc.py" in the[callsite_filename()]] + # Skips over our own call-helpers so the *user's* call site is + # reported, not the helper's. Reaching `callsite_filename` via + # `call(...)` should still return this test file. + from ..funutil import call + test["test_misc.py" in the[call(callsite_filename)]] + # Like issubclass, but if `cls` is not a class, swallow the `TypeError` and return `False`. with testset("safeissubclass"): class MetalBox: @@ -131,6 +144,158 @@ class Safe(MetalBox): test[safeissubclass(Safe, (PlasticBox, MetalBox))] test[not safeissubclass("definitely not a class", MetalBox)] + # -------------------------------------------------------------------------- + # maybe_open + + with testset("maybe_open"): + # With an actual file + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as tmp: + tmp.write("hello") + tmpname = tmp.name + try: + with maybe_open(tmpname, "r", sys.stdin) as f: + test[the[f.read()] == "hello"] + finally: + os.unlink(tmpname) + + # With None filename, yields the fallback stream + import io + fallback = io.StringIO("fallback content") + with maybe_open(None, "r", fallback) as f: + test[f is fallback] + test[the[f.read()] == "fallback content"] + + # -------------------------------------------------------------------------- + # redirect_stdin + + with testset("redirect_stdin"): + import io as _io_for_redirect + original_stdin = sys.stdin + + # Basic: redirect sys.stdin, read from input(), restore on exit. + with redirect_stdin(_io_for_redirect.StringIO("42\n")): + test[the[input()] == "42"] + test[sys.stdin is not original_stdin] + test[sys.stdin is original_stdin] + + # Yields the target (matches stdlib redirect_stdout / redirect_stderr). + target = _io_for_redirect.StringIO("hello") + with redirect_stdin(target) as yielded: + test[yielded is target] + test[sys.stdin is original_stdin] + + # Exception inside the block still restores sys.stdin. + try: + with redirect_stdin(_io_for_redirect.StringIO("data")): + raise RuntimeError("boom") + except RuntimeError: + pass + test[sys.stdin is original_stdin] + + # Nested re-entry on the same instance unwinds via the per-instance + # _old_targets stack inherited from contextlib._RedirectStream. + rs = redirect_stdin(_io_for_redirect.StringIO("data")) + with rs: + with rs: + test[sys.stdin is rs._new_target] + # After inner exit, still redirected (outer with is active). + test[sys.stdin is rs._new_target] + test[sys.stdin is original_stdin] + + # -------------------------------------------------------------------------- + # UnionFilter + + with testset("UnionFilter"): + f1 = logging.Filter("myapp.core") + f2 = logging.Filter("myapp.io") + uf = UnionFilter(f1, f2) + + rec_core = logging.LogRecord("myapp.core.engine", logging.INFO, + "", 0, "msg", (), None) + rec_io = logging.LogRecord("myapp.io.disk", logging.INFO, + "", 0, "msg", (), None) + rec_other = logging.LogRecord("otherapp.main", logging.INFO, + "", 0, "msg", (), None) + + test[uf.filter(rec_core)] + test[uf.filter(rec_io)] + test[not uf.filter(rec_other)] + + # Empty UnionFilter matches nothing + empty = UnionFilter() + test[not empty.filter(rec_core)] + + # -------------------------------------------------------------------------- + # si_prefix + + with testset("si_prefix"): + # No prefix (magnitude in [1, 1000)) + test[the[si_prefix(0)] == "0.00"] + test[the[si_prefix(42)] == "42.00"] + test[the[si_prefix(999)] == "999.00"] + + # Large prefixes + test[the[si_prefix(1000)] == "1.00 k"] + test[the[si_prefix(1500)] == "1.50 k"] + test[the[si_prefix(2_500_000)] == "2.50 M"] + test[the[si_prefix(1e9)] == "1.00 G"] + test[the[si_prefix(1e12)] == "1.00 T"] + + # Small prefixes + test[the[si_prefix(0.001)] == "1.00 m"] + test[the[si_prefix(0.0015)] == "1.50 m"] + test[the[si_prefix(0.000001)] == "1.00 \N{MICRO SIGN}"] + test[the[si_prefix(0.0000025)] == "2.50 \N{MICRO SIGN}"] + test[the[si_prefix(1e-9)] == "1.00 n"] + test[the[si_prefix(1e-12)] == "1.00 p"] + + # Negative numbers + test[the[si_prefix(-1500)] == "-1.50 k"] + test[the[si_prefix(-42)] == "-42.00"] + test[the[si_prefix(-0.001)] == "-1.00 m"] + + # Custom precision + test[the[si_prefix(1500, precision=0)] == "2 k"] + test[the[si_prefix(1500, precision=4)] == "1.5000 k"] + test[the[si_prefix(42, precision=1)] == "42.0"] + + # Binary (IEC) mode + test[the[si_prefix(0, binary=True)] == "0.00"] + test[the[si_prefix(500, binary=True)] == "500.00"] + test[the[si_prefix(1024, binary=True)] == "1.00 Ki"] + test[the[si_prefix(1536, binary=True)] == "1.50 Ki"] + test[the[si_prefix(1024**2, binary=True)] == "1.00 Mi"] + test[the[si_prefix(2.5 * 1024**2, binary=True)] == "2.50 Mi"] + test[the[si_prefix(1024**3, binary=True)] == "1.00 Gi"] + test[the[si_prefix(1024**4, binary=True)] == "1.00 Ti"] + test[the[si_prefix(-1536, binary=True)] == "-1.50 Ki"] + test[the[si_prefix(0.5, binary=True)] == "512.00 mi"] + test[the[si_prefix(0.5 / 1024, binary=True)] == "512.00 µi"] + test[the[si_prefix(1 / 1024, binary=True)] == "1.00 mi"] # mibi + test[the[si_prefix(1 / 1024**2, binary=True)] == "1.00 µi"] # microbi + test[the[si_prefix(1 / 1024**3, binary=True)] == "1.00 ni"] # nino + test[the[si_prefix(1 / 1024**4, binary=True)] == "1.00 pi"] # pithon + test[the[si_prefix(1 / 1024**5, binary=True)] == "1.00 fi"] + test[the[si_prefix(1 / 1024**6, binary=True)] == "1.00 ai"] # AI + test[the[si_prefix(1 / 1024**7, binary=True)] == "1.00 zi"] + test[the[si_prefix(1 / 1024**8, binary=True)] == "1.00 yi"] + test[the[si_prefix(1 / 1024**9, binary=True)] == "1.00 ri"] + test[the[si_prefix(1 / 1024**10, binary=True)] == "1.00 qi"] # chi + + # Custom separator + test[the[si_prefix(1500, separator="")] == "1.50k"] + test[the[si_prefix(1500, separator="\N{NO-BREAK SPACE}")] == "1.50\N{NO-BREAK SPACE}k"] + test[the[si_prefix(42, separator="")] == "42.00"] # no prefix -> nothing to separate + + # Separator emitted even without a prefix, so that appending a unit of measurement (W, B, Hz) + # spaces the same way whether or not the magnitude called for a prefix + test[the[si_prefix(42, always_separate=True)] == "42.00 "] + test[the[si_prefix(0, always_separate=True)] == "0.00 "] + test[the[si_prefix(1500, always_separate=True)] == "1.50 k"] # unchanged where a prefix exists + test[the[si_prefix(42, separator="", always_separate=True)] == "42.00"] # empty separator, still empty + test[the[f"{si_prefix(1536, binary=True, precision=1, always_separate=True)}B"] == "1.5 KiB"] + test[the[f"{si_prefix(512, binary=True, precision=0, always_separate=True)}B"] == "512 B"] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_numutil.py b/unpythonic/tests/test_numutil.py index ae2f808d..0c7a09c7 100644 --- a/unpythonic/tests/test_numutil.py +++ b/unpythonic/tests/test_numutil.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- -from ..syntax import macros, test, test_raises, error # noqa: F401 +from ..syntax import macros, test, test_raises, warn, the # noqa: F401 from ..test.fixtures import session, testset +from itertools import count, takewhile +from math import cos, sqrt import sys -from ..numutil import almosteq, ulp +from ..numutil import (almosteq, fixpoint, ulp, + partition_int, partition_int_triangular, partition_int_custom) +from ..it import rev def runtests(): with testset("ulp (unit in the last place; float utility)"): @@ -33,12 +37,75 @@ def runtests(): try: from mpmath import mpf except ImportError: # pragma: no cover - error["mpmath not installed in this Python, cannot test arbitrary precision input for mathseq."] + warn["mpmath not installed in this Python, skipping arbitrary precision input tests."] else: test[almosteq(mpf(1.0), mpf(1.0 + ulp(1.0)))] test[almosteq(1.0, mpf(1.0 + ulp(1.0)))] test[almosteq(mpf(1.0), 1.0 + ulp(1.0))] + # Arithmetic fixed points. + with testset("fixpoint (arithmetic fixed points)"): + c = fixpoint(cos, x0=1) + test[the[c] == the[cos(c)]] # 0.7390851332151607 + + # Actually "Newton's" algorithm for the square root was already known to the + # ancient Babylonians, ca. 2000 BCE. (Carl Boyer: History of mathematics) + # Concerning naming, see also https://en.wikipedia.org/wiki/Stigler's_law_of_eponymy + def sqrt_newton(n): + def sqrt_iter(x): # has an attractive fixed point at sqrt(n) + return (x + n / x) / 2 + return fixpoint(sqrt_iter, x0=n / 2) + # different algorithm, so not necessarily equal down to the last bit + # (caused by the fixpoint update becoming smaller than the ulp, so it + # stops there, even if the limit is still one ulp away). + test[abs(the[sqrt_newton(2)] - the[sqrt(2)]) <= the[ulp(1.414)]] + + # partition_int: split a small positive integer, in all possible ways, into smaller integers that sum to it + with testset("partition_int"): + test[tuple(partition_int(4)) == ((4,), (3, 1), (2, 2), (2, 1, 1), (1, 3), (1, 2, 1), (1, 1, 2), (1, 1, 1, 1))] + test[tuple(partition_int(5, lower=2)) == ((5,), (3, 2), (2, 3))] + test[tuple(partition_int(5, lower=2, upper=3)) == ((3, 2), (2, 3))] + test[tuple(partition_int(10, lower=3, upper=5)) == ((5, 5), (4, 3, 3), (3, 4, 3), (3, 3, 4))] + test[all(sum(terms) == 10 for terms in partition_int(10))] + test[all(sum(terms) == 10 for terms in partition_int(10, lower=3))] + test[all(sum(terms) == 10 for terms in partition_int(10, lower=3, upper=5))] + + test_raises[TypeError, partition_int("not a number")] + test_raises[TypeError, partition_int(4, lower="not a number")] + test_raises[TypeError, partition_int(4, upper="not a number")] + test_raises[ValueError, partition_int(-3)] + test_raises[ValueError, partition_int(4, lower=-1)] + test_raises[ValueError, partition_int(4, lower=5)] + test_raises[ValueError, partition_int(4, upper=-1)] + test_raises[ValueError, partition_int(4, upper=5)] + test_raises[ValueError, partition_int(4, lower=3, upper=2)] + + # partition_int_triangular: like partition_int, but in the output, allow triangular numbers only. + # Triangular numbers are 1, 3, 6, 10, ... + with testset("partition_int_triangular"): + test[frozenset(tuple(sorted(c)) for c in partition_int_triangular(78, lower=10)) == + frozenset({(10, 10, 10, 10, 10, 28), + (10, 10, 15, 15, 28), + (15, 21, 21, 21), + (21, 21, 36), + (78,)})] + + # partition_int_custom: like partition_int, but lets you specify allowed components manually. + # Can be used to build other functions like `partition_int` and `partition_int_triangular`. + with testset("partition_int_custom"): + test[tuple(partition_int_custom(4, [1])) == ((1, 1, 1, 1),)] + test[tuple(partition_int_custom(4, [1, 3])) == ((1, 1, 1, 1), (1, 3), (3, 1))] + + evens_upto_n = lambda n: takewhile(lambda m: m <= n, count(start=2, step=2)) + test[tuple(partition_int_custom(4, rev(evens_upto_n(4)))) == ((4,), (2, 2))] + test[tuple(partition_int_custom(6, rev(evens_upto_n(6)))) == ((6,), (4, 2), (2, 4), (2, 2, 2))] + + test_raises[TypeError, partition_int_custom("not a number", evens_upto_n("blah"))] + test_raises[TypeError, tuple(partition_int_custom(4, [2.0]))] + test_raises[ValueError, partition_int_custom(-3, evens_upto_n(-3))] + test_raises[ValueError, tuple(partition_int_custom(4, [-1]))] + test_raises[ValueError, tuple(partition_int_custom(4, [1, -1]))] + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/tests/test_seq.py b/unpythonic/tests/test_seq.py index 03ea273a..4e127188 100644 --- a/unpythonic/tests/test_seq.py +++ b/unpythonic/tests/test_seq.py @@ -118,7 +118,7 @@ def append_succ(lis): def nextfibo(state): a, b = state fibos.append(a) # store result by side effect - return (b, a + b) # new state, handed to next function in the pipe + return (b, a + b) # new state, handed to the next function in the pipe p = lazy_piped1((1, 1)) # load initial state into a lazy pipe for _ in range(10): # set up pipeline p = p | nextfibo diff --git a/unpythonic/tests/test_slicing.py b/unpythonic/tests/test_slicing.py index 4ae6bcd5..b810abf9 100644 --- a/unpythonic/tests/test_slicing.py +++ b/unpythonic/tests/test_slicing.py @@ -4,9 +4,10 @@ from ..syntax import macros, test, test_raises # noqa: F401 from ..test.fixtures import session, testset -from itertools import repeat +from itertools import count, repeat from ..slicing import fup, islice +from ..gmemo import imemoize from ..mathseq import primes, s def runtests(): @@ -20,6 +21,10 @@ def runtests(): test[fup(tup)[1::2] << tuple(repeat(10, 3)) == (1, 10, 3, 10, 5)] test[fup(tup)[::2] << tuple(repeat(10, 3)) == (10, 2, 10, 4, 10)] test[fup(tup)[::-1] << tuple(range(5)) == (4, 3, 2, 1, 0)] + test[fup(tup)[0::2] << repeat(10) == (10, 2, 10, 4, 10)] # infinite replacement + test[fup(tup)[0::2] << count(start=10) == (10, 2, 11, 4, 12)] + test[fup(tup)[::2] << imemoize(repeat(10))() == (10, 2, 10, 4, 10)] # memoized infinite replacement backwards + test[fup(tup)[::-2] << imemoize(count(start=10))() == (12, 2, 11, 4, 10)] test[tup == (1, 2, 3, 4, 5)] test_raises[TypeError, fup(tup)[2, 3]] # multidimensional indexing not supported diff --git a/unpythonic/tests/test_symbol.py b/unpythonic/tests/test_symbol.py index 476384b1..349271f2 100644 --- a/unpythonic/tests/test_symbol.py +++ b/unpythonic/tests/test_symbol.py @@ -34,7 +34,7 @@ def runtests(): # Symbol interning has nothing to do with string interning. many = 5000 test[the[sym("λ" * many) is sym("λ" * many)]] - # To defeat string interning, used to be that 80 exotic characters + # To defeat string interning, it used to be that 80 exotic characters # would be enough in Python 3.6 to make CPython decide not to intern it, # but Python 3.7 bumped that up. test[the["λ" * many is not "λ" * many]] diff --git a/unpythonic/tests/test_timeutil.py b/unpythonic/tests/test_timeutil.py new file mode 100644 index 00000000..ba7574b0 --- /dev/null +++ b/unpythonic/tests/test_timeutil.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from ..syntax import macros, test # noqa: F401 +from ..test.fixtures import session, testset, returns_normally + +from ..timeutil import seconds_to_human, format_human_time, ETAEstimator + +def runtests(): + with testset("seconds_to_human"): + test[seconds_to_human(30) == (0, 0, 0, 30)] + test[seconds_to_human(30.0) == (0, 0, 0, 30.0)] + test[seconds_to_human(90) == (0, 0, 1, 30)] + test[seconds_to_human(3690) == (0, 1, 1, 30)] + test[seconds_to_human(86400 + 3690) == (1, 1, 1, 30)] + test[seconds_to_human(2 * 86400 + 3690) == (2, 1, 1, 30)] + + with testset("format_human_time"): + test[format_human_time(30) == "30 seconds"] + test[format_human_time(90) == "01:30"] # mm:ss + test[format_human_time(3690) == "01:01:30"] # hh:mm:ss + test[format_human_time(86400 + 3690) == "1 day 01:01:30"] + test[format_human_time(2 * 86400 + 3690) == "2 days 01:01:30"] + + # This is a UI thing so we can't test functionality reliably. Let's just check it doesn't crash. + with testset("ETAEstimator"): + e = ETAEstimator(total=5) + test[returns_normally(e.estimate)] # before the first tick + test[returns_normally(e.elapsed)] + test[returns_normally(e.formatted_eta)] + test[returns_normally(e.tick())] + test[returns_normally(e.estimate)] # after the first tick + test[returns_normally(e.elapsed)] + test[returns_normally(e.formatted_eta)] + +if __name__ == '__main__': # pragma: no cover + with session(__file__): + runtests() diff --git a/unpythonic/tests/test_typecheck.py b/unpythonic/tests/test_typecheck.py index 28d62668..3b8af53f 100644 --- a/unpythonic/tests/test_typecheck.py +++ b/unpythonic/tests/test_typecheck.py @@ -3,7 +3,12 @@ from ..syntax import macros, test, test_raises, warn # noqa: F401 from ..test.fixtures import session, testset +import asyncio import collections +import contextlib +import io +import re +import sys import typing from ..collections import frozendict @@ -32,6 +37,17 @@ def runtests(): test[isoftype("something", typing.Any)] test[isoftype(lambda: ..., typing.Any)] + # NoReturn / Never — the bottom type; no value can match. + with testset("typing.NoReturn"): + test[not isoftype(None, typing.NoReturn)] + test[not isoftype(42, typing.NoReturn)] + test[not isoftype("anything", typing.NoReturn)] + + if sys.version_info >= (3, 11): + with testset("typing.Never"): + test[not isoftype(None, typing.Never)] + test[not isoftype(42, typing.Never)] + # TypeVar, bare; a named type, but behaves like Any. with testset("typing.TypeVar (bare; like a named Any)"): X = typing.TypeVar("X") @@ -67,6 +83,45 @@ def runtests(): test[isoftype(1337, typing.Optional[int])] test[not isoftype(3.14, typing.Optional[int])] + with testset("typing.Literal"): + test[isoftype(1, typing.Literal[1, 2, 3])] + test[isoftype(3, typing.Literal[1, 2, 3])] + test[not isoftype(4, typing.Literal[1, 2, 3])] + test[isoftype("red", typing.Literal["red", "green", "blue"])] + test[not isoftype("yellow", typing.Literal["red", "green", "blue"])] + # Literal values are compared by equality, not identity + test[isoftype(True, typing.Literal[True, False])] + test[not isoftype(None, typing.Literal[True, False])] + + with testset("typing.Type"): + test[isoftype(int, typing.Type[int])] + test[isoftype(bool, typing.Type[int])] # bool is a subclass of int + test[not isoftype(str, typing.Type[int])] + test[not isoftype(42, typing.Type[int])] # an instance, not a class + # bare Type: any class matches + test[isoftype(int, typing.Type)] + test[isoftype(str, typing.Type)] + test[not isoftype(42, typing.Type)] + + with testset("typing.ClassVar"): + test[isoftype(42, typing.ClassVar[int])] + test[not isoftype("hello", typing.ClassVar[int])] + # Compound: ClassVar wrapping a Union + test[isoftype(42, typing.ClassVar[typing.Union[int, str]])] + test[isoftype("hello", typing.ClassVar[typing.Union[int, str]])] + test[not isoftype(3.14, typing.ClassVar[typing.Union[int, str]])] + + with testset("typing.Final"): + test[isoftype(42, typing.Final[int])] + test[not isoftype("hello", typing.Final[int])] + test[isoftype("hello", typing.Final[str])] + + # Empty collections reject parametric type specs (e.g. `Tuple[int, ...]`, + # `List[int]`, `Dict[str, int]`). An empty collection has no elements to + # infer the type from, so matching it against a specific element type would + # be guesswork — which would make multiple dispatch unpredictable. + # Bare (unparametrized) specs like `Tuple` or `Dict` still accept empties. + with testset("typing.Tuple"): test[isoftype((1, 2, 3), typing.Tuple)] test[isoftype((1, 2, 3), typing.Tuple[int, ...])] @@ -101,6 +156,34 @@ def runtests(): # no type arguments: any key/value types ok (consistent with Python 3.7+) test[isoftype({"cat": "animal", "pi": 3.14159, 2.71828: "e"}, typing.Dict)] + with testset("typing.DefaultDict"): + dd = collections.defaultdict(int, {"a": 1, "b": 2}) + test[isoftype(dd, typing.DefaultDict[str, int])] + test[not isoftype(dd, typing.DefaultDict[int, int])] + test[not isoftype({}, typing.DefaultDict[str, int])] # regular dict is not defaultdict + test[not isoftype(collections.defaultdict(int), typing.DefaultDict[str, int])] # empty + + with testset("typing.OrderedDict"): + od = collections.OrderedDict({"x": 1, "y": 2}) + test[isoftype(od, typing.OrderedDict[str, int])] + test[not isoftype(od, typing.OrderedDict[int, int])] + test[not isoftype({}, typing.OrderedDict[str, int])] # regular dict is not OrderedDict + test[not isoftype(collections.OrderedDict(), typing.OrderedDict[str, int])] # empty + + with testset("typing.Counter"): + c = collections.Counter("abracadabra") + test[isoftype(c, typing.Counter[str])] + test[not isoftype(c, typing.Counter[int])] + test[not isoftype({}, typing.Counter[str])] # regular dict is not Counter + test[not isoftype(collections.Counter(), typing.Counter[str])] # empty + + with testset("typing.ChainMap"): + cm = collections.ChainMap({"a": 1}, {"b": 2}) + test[isoftype(cm, typing.ChainMap[str, int])] + test[not isoftype(cm, typing.ChainMap[int, int])] + test[not isoftype({}, typing.ChainMap[str, int])] # regular dict is not ChainMap + test[not isoftype(collections.ChainMap(), typing.ChainMap[str, int])] # empty + # type alias (at run time, this is just an assignment) with testset("type alias"): U = typing.Union[int, str] @@ -179,8 +262,112 @@ def runtests(): test[isoftype([1, 2, 3], typing.Iterable)] test[isoftype([1, 2, 3], typing.Reversible)] test[isoftype([1, 2, 3], typing.Container)] - if hasattr(typing, "Collection"): # Python 3.6+ - test[isoftype([1, 2, 3], typing.Collection)] # Sized Iterable Container + test[isoftype([1, 2, 3], typing.Collection)] # Sized Iterable Container + + with testset("parametric ABCs — uncheckable (type arg ignored)"): + # Iterator: consumed by iteration, can't check elements. + test[isoftype(iter([1, 2, 3]), typing.Iterator[int])] + test[isoftype(iter([1, 2, 3]), typing.Iterator[str])] # type arg ignored + test[not isoftype(42, typing.Iterator[int])] + + # Container: only has __contains__, can't enumerate elements. + test[isoftype([1, 2, 3], typing.Container[int])] + test[isoftype([1, 2, 3], typing.Container[str])] # type arg ignored + test[not isoftype(42, typing.Container[int])] + + with testset("parametric ABCs — best-effort element checking"): + # Iterable[T]: elements checked when value is Sized (concrete collection). + test[isoftype([1, 2, 3], typing.Iterable[int])] + test[not isoftype([1, 2, 3], typing.Iterable[str])] + test[not isoftype([], typing.Iterable[int])] # empty rejects parametric + test[isoftype([], typing.Iterable)] # bare form still accepts empty + test[not isoftype(42, typing.Iterable[int])] + # Opaque iterator (not Sized) — accepts on ABC alone, can't check elements. + test[isoftype(iter([1, 2, 3]), typing.Iterable[int])] + test[isoftype(iter([1, 2, 3]), typing.Iterable[str])] # can't check, accepts + + # Collection[T]: Sized + Iterable + Container. + test[isoftype([1, 2, 3], typing.Collection[int])] + test[not isoftype([1, 2, 3], typing.Collection[str])] + test[not isoftype([], typing.Collection[int])] # empty rejects parametric + test[isoftype([], typing.Collection)] # bare form accepts empty + test[not isoftype(42, typing.Collection[int])] + + # Reversible[T] + test[isoftype([1, 2, 3], typing.Reversible[int])] + test[not isoftype([1, 2, 3], typing.Reversible[str])] + test[not isoftype([], typing.Reversible[int])] # empty rejects parametric + test[isoftype([], typing.Reversible)] # bare form accepts empty + test[not isoftype(42, typing.Reversible[int])] + + # Compound type in element spec + test[isoftype([1, "two", 3], typing.Iterable[typing.Union[int, str]])] + test[not isoftype([1, "two", 3.0], typing.Iterable[typing.Union[int, str]])] + + with testset("typing.TypedDict"): + class Point(typing.TypedDict): + x: float + y: float + + test[isoftype({"x": 1.0, "y": 2.0}, Point)] + test[not isoftype({"x": 1.0}, Point)] # missing required key + test[not isoftype({"x": 1.0, "y": 2.0, "z": 3.0}, Point)] # extra key + test[not isoftype({"x": "hello", "y": 2.0}, Point)] # wrong value type + test[not isoftype(42, Point)] # not a dict + test[not isoftype([], Point)] # not a dict + + # total=False: all keys optional + class Config(typing.TypedDict, total=False): + debug: bool + verbose: bool + + test[isoftype({}, Config)] # all optional, empty is ok + test[isoftype({"debug": True}, Config)] + test[isoftype({"debug": True, "verbose": False}, Config)] + test[not isoftype({"debug": "yes"}, Config)] # wrong type + test[not isoftype({"unknown": True}, Config)] # extra key + + # Inheritance + class Base(typing.TypedDict): + name: str + + class Derived(Base): + age: int + + test[isoftype({"name": "alice", "age": 30}, Derived)] + test[not isoftype({"name": "alice"}, Derived)] # missing age + test[not isoftype({"age": 30}, Derived)] # missing name + + # Compound value types + class Nested(typing.TypedDict): + tags: typing.List[str] + count: typing.Optional[int] + + test[isoftype({"tags": ["a", "b"], "count": 42}, Nested)] + test[isoftype({"tags": ["a"], "count": None}, Nested)] + test[not isoftype({"tags": [1, 2], "count": 42}, Nested)] # wrong list element type + + with testset("typing.Protocol"): + @typing.runtime_checkable + class Drawable(typing.Protocol): + def draw(self) -> None: ... + + class Circle: + def draw(self): + pass + + class Square: + pass + + test[isoftype(Circle(), Drawable)] + test[not isoftype(Square(), Drawable)] + test[not isoftype(42, Drawable)] + + # Non-runtime-checkable Protocol raises TypeError + class NonCheckable(typing.Protocol): + def frobnicate(self) -> int: ... + + test_raises[TypeError, isoftype(Circle(), NonCheckable)] with testset("typing.KeysView, typing.ValuesView, typing.ItemsView"): d = {17: "cat", 23: "fox", 42: "python"} @@ -203,6 +390,94 @@ def runtests(): # https://docs.python.org/3/glossary.html#term-dictionary-view # https://docs.python.org/3/library/stdtypes.html#dict-views + with testset("typing.IO, typing.TextIO, typing.BinaryIO"): + sio = io.StringIO("hello") + bio = io.BytesIO(b"hello") + test[isoftype(sio, typing.IO)] + test[isoftype(bio, typing.IO)] + test[isoftype(sio, typing.TextIO)] + test[not isoftype(bio, typing.TextIO)] + test[isoftype(bio, typing.BinaryIO)] + test[not isoftype(sio, typing.BinaryIO)] + test[not isoftype(42, typing.IO)] + # Parametric IO: IO[str] matches text, IO[bytes] matches binary + test[isoftype(sio, typing.IO[str])] + test[not isoftype(bio, typing.IO[str])] + test[isoftype(bio, typing.IO[bytes])] + test[not isoftype(sio, typing.IO[bytes])] + + with testset("typing.Pattern, typing.Match"): + pstr = re.compile(r"\d+") + pbytes = re.compile(rb"\d+") + mstr = pstr.match("123") + mbytes = pbytes.match(b"123") + # Bare Pattern/Match — any string type + test[isoftype(pstr, typing.Pattern)] + test[isoftype(pbytes, typing.Pattern)] + test[isoftype(mstr, typing.Match)] + test[isoftype(mbytes, typing.Match)] + test[not isoftype("not a pattern", typing.Pattern)] + test[not isoftype(42, typing.Match)] + # Parametric — string type checked + test[isoftype(pstr, typing.Pattern[str])] + test[not isoftype(pstr, typing.Pattern[bytes])] + test[isoftype(pbytes, typing.Pattern[bytes])] + test[not isoftype(pbytes, typing.Pattern[str])] + test[isoftype(mstr, typing.Match[str])] + test[not isoftype(mstr, typing.Match[bytes])] + test[isoftype(mbytes, typing.Match[bytes])] + test[not isoftype(mbytes, typing.Match[str])] + + with testset("typing.ContextManager"): + # contextlib.nullcontext is a context manager + cm = contextlib.nullcontext() + test[isoftype(cm, typing.ContextManager)] + test[isoftype(cm, typing.ContextManager[None])] # type arg ignored (can't check) + test[not isoftype(42, typing.ContextManager)] + + with testset("typing.Generator"): + def mygen(): + yield 1 + yield 2 + g = mygen() + test[isoftype(g, typing.Generator)] + test[isoftype(g, typing.Generator[int, None, None])] # type args ignored + test[not isoftype(42, typing.Generator)] + test[not isoftype([1, 2, 3], typing.Generator)] # iterable, but not a generator + + with testset("typing.Awaitable, typing.Coroutine"): + async def mycoro(): + return 42 + c = mycoro() + test[isoftype(c, typing.Awaitable)] + test[isoftype(c, typing.Coroutine)] + test[isoftype(c, typing.Awaitable[int])] # type arg ignored + test[not isoftype(42, typing.Awaitable)] + test[not isoftype(42, typing.Coroutine)] + c.close() # prevent RuntimeWarning about unawaited coroutine + + with testset("typing.AsyncIterable, typing.AsyncIterator"): + class MyAsyncIter: + def __aiter__(self): + return self + async def __anext__(self): + raise StopAsyncIteration + ai = MyAsyncIter() + test[isoftype(ai, typing.AsyncIterable)] + test[isoftype(ai, typing.AsyncIterator)] + test[isoftype(ai, typing.AsyncIterable[int])] # type arg ignored + test[not isoftype(42, typing.AsyncIterable)] + test[not isoftype([1, 2], typing.AsyncIterator)] # sync iterable, not async + + with testset("typing.AsyncGenerator"): + async def myasyncgen(): + yield 1 + ag = myasyncgen() + test[isoftype(ag, typing.AsyncGenerator)] + test[isoftype(ag, typing.AsyncGenerator[int, None])] # type args ignored + test[not isoftype(42, typing.AsyncGenerator)] + asyncio.run(ag.aclose()) # prevent RuntimeWarning + if __name__ == '__main__': # pragma: no cover with session(__file__): runtests() diff --git a/unpythonic/timeutil.py b/unpythonic/timeutil.py new file mode 100644 index 00000000..e05588aa --- /dev/null +++ b/unpythonic/timeutil.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +"""Some additional batteries for time handling.""" + +__all__ = ["seconds_to_human", "format_human_time", + "ETAEstimator"] + +from collections import deque +import time +import typing + +def seconds_to_human(s: typing.Union[float, int]) -> typing.Tuple[int, int, int, float]: + """Convert a number of seconds into (days, hours, minutes, seconds).""" + d = int(s // 86400) + s -= d * 86400 + h = int(s // 3600) + s -= h * 3600 + m = int(s // 60) + s -= m * 60 + return d, h, m, s + + +def format_human_time(s: typing.Union[float, int]) -> str: + """Convert a number of seconds to a human-readable string. + + The representation format switches automatically depending on + how large `s` is. Examples: + + assert format_human_time(30) == "30 seconds" + assert format_human_time(90) == "01:30" # mm:ss + assert format_human_time(3690) == "01:01:30" # hh:mm:ss + assert format_human_time(86400 + 3690) == "1 day 01:01:30" + assert format_human_time(2 * 86400 + 3690) == "2 days 01:01:30" + """ + d, h, m, s = seconds_to_human(s) + + if all(x == 0 for x in (d, h, m)): # under one minute + plural = "s" if int(s) != 1.0 else "" + return f"{int(s):d} second{plural}" + + if d > 0: + plural = "s" if d > 1 else "" + days = f"{d:d} day{plural} " + else: + days = "" + hours = f"{h:02d}:" if (d > 0 or h > 0) else "" + minutes = f"{m:02d}:" + seconds = f"{int(s):02d}" + return f"{days}{hours}{minutes}{seconds}" + + +class ETAEstimator: + """Estimate the time of completion. + + `total`: number of tasks in the whole job, used for estimating + how much work is still needed. + + Stored in `self.total`, which is writable; but note that + if you move the goalposts, the ETA cannot be accurate. + Changing `self.total` is mostly useful if you suddenly + discover that the workload is actually larger or smaller + than what was initially expected, and want the estimate + to reflect this sudden new information. + + `keep_last`: use the timings from at most this many most recently + completed tasks when computing the estimate. + + If not given, keep all. + + If you need it, the number of tasks that have been marked completed + is available in `self.completed`. + """ + def __init__(self, total: int, keep_last: typing.Optional[int] = None): + self.t1 = time.perf_counter() # time since last tick + self.t0 = self.t1 # time since beginning + self.total = total # total number of work items + self.completed = 0 # number of completed work items + self.que = deque([], maxlen=keep_last) + + def tick(self) -> None: + """Mark one more task as completed, automatically updating the internal timings cache.""" + self.completed += 1 + t = time.perf_counter() + dt = t - self.t1 + self.t1 = t + self.que.append(dt) + + def _estimate(self) -> typing.Optional[float]: + if self.completed == 0: + return None + # TODO: Smoother ETA? + # + # Let us consider the ETA estimation process as downsampling the data + # vector (deque) into an extremely low-resolution version that has just + # one sample. + # + # As we know from signal processing, as a downsampling filter, the + # running average has an abysmal frequency response; so we should + # expect the ETA to fluctuate wildly depending on the smoothness of + # the input data (i.e. the time taken by each task)... which actually + # matches observation. + # + # Maybe we could use a Lanczos downsampling filter to make the ETA + # behave more smoothly? + remaining = self.total - self.completed + if remaining <= 0: + return 0.0 + dt_avg = sum(self.que) / len(self.que) + return remaining * dt_avg + estimate = property(fget=_estimate, doc="Estimate of time remaining, in seconds. Computed when read; read-only. If no tasks have been marked completed yet, the estimate is `None`.") + + def _elapsed(self) -> float: + return time.perf_counter() - self.t0 + elapsed = property(fget=_elapsed, doc="Total elapsed time, in seconds. Computed when read; read-only.") + + def _formatted_eta(self) -> str: + elapsed = self.elapsed + estimate = self.estimate + if estimate is not None: + total = elapsed + estimate + formatted_estimate = format_human_time(estimate) + formatted_total = format_human_time(total) + else: + formatted_estimate = "unknown" + formatted_total = "unknown" + formatted_elapsed = format_human_time(elapsed) + return f"elapsed {formatted_elapsed}, ETA {formatted_estimate}, total {formatted_total}" + formatted_eta = property(fget=_formatted_eta, doc="Human-readable estimate, with elapsed, ETA and remaining time. See `format_human_time` for details of the format used.") diff --git a/unpythonic/typecheck.py b/unpythonic/typecheck.py index 089cfd63..f0ce37a0 100644 --- a/unpythonic/typecheck.py +++ b/unpythonic/typecheck.py @@ -1,33 +1,26 @@ # -*- coding: utf-8; -*- -"""Simplistic run-time type checker. +"""Lightweight run-time type checker. -This implements just a minimal feature set needed for checking function -arguments in typical uses of multiple dispatch (see `unpythonic.dispatch`). -That said, this DOES support many (but not all) features of the `typing` stdlib -module. +Originally built for the minimal feature set needed by multiple dispatch +(see `unpythonic.dispatch`), but designed as a general-purpose utility. +Supports many (but not all) features of the `typing` stdlib module. We currently provide `isoftype` (cf. `isinstance`), but no `issubtype` (cf. `issubclass`). -If you need a run-time type checker for serious general use, consider `typeguard`: +If you need a run-time type checker, but not the other features of `unpythonic`, +see `typeguard`: https://github.com/agronholm/typeguard """ import collections +import contextlib +import io +import re +import sys +import types import typing -try: - _MyGenericAlias = typing._GenericAlias # Python 3.7+ -except AttributeError: # Python 3.6 and earlier # pragma: no cover - class _MyGenericAlias: # unused, but must be a class to support isinstance() check. - pass - -try: - _MySupportsIndex = typing.SupportsIndex # Python 3.8+ -except AttributeError: # Python 3.7 and earlier # pragma: no cover - class _MySupportsIndex: # unused, but must be a class to support isinstance() check. - pass - from .misc import safeissubclass __all__ = ["isoftype"] @@ -50,14 +43,33 @@ def isoftype(value, T): - `TypeVar` - `NewType` (any instance of the underlying actual type will match) - `Union[T1, T2, ..., TN]` + - `NoReturn`, `Never` (no value matches; `Never` requires Python 3.11+) + - `Literal[v1, v2, ...]` + - `Type[X]` (value must be a class that is `X` or a subclass of `X`) + - `ClassVar[T]`, `Final[T]` (wrapper stripped, inner type checked) - `Tuple`, `Tuple[T, ...]`, `Tuple[T1, T2, ..., TN]`, `Sequence[T]` - `List[T]`, `MutableSequence[T]` - `FrozenSet[T]`, `AbstractSet[T]` - `Set[T]`, `MutableSet[T]` - - `Dict[K, V]`, `MutableMapping[K, V]`, `Mapping[K, V]` + - `Dict[K, V]`, `DefaultDict[K, V]`, `OrderedDict[K, V]` + - `Counter[T]` (element type checked; value type is always `int`) + - `ChainMap[K, V]` + - `MutableMapping[K, V]`, `Mapping[K, V]` - `ItemsView[K, V]`, `KeysView[K]`, `ValuesView[V]` - `Callable` (argument and return value types currently NOT checked) - - `Text` + - `IO`, `TextIO`, `BinaryIO` (mapped to ``io`` module ABCs) + - `Pattern[T]`, `Match[T]` (string type checked when parametric) + - `ContextManager[T]`, `AsyncContextManager[T]` + - `Awaitable[T]`, `Coroutine[T1, T2, T3]` + - `AsyncIterable[T]`, `AsyncIterator[T]` + - `Generator[Y, S, R]`, `AsyncGenerator[Y, S]` + - `Iterable[T]`, `Collection[T]`, `Reversible[T]` (best-effort element + checking: elements checked when value is ``Sized``; ABC-only when not) + - `Iterator[T]`, `Container[T]` (parametric form accepted; type arg ignored) + - `Hashable`, `Sized` (non-generic; bare form only) + - `TypedDict` (structural check: required/optional keys, value types) + - ``@runtime_checkable`` ``Protocol`` subclasses + - `Text` (deprecated since Python 3.11; will be removed at floor Python 3.12) Any checks on the type arguments of the meta-utilities are performed recursively using `isoftype`, in order to allow compound specifications. @@ -70,122 +82,43 @@ def isoftype(value, T): Returns `True` if `value` matches the type specification; `False` if not. """ - # TODO: This function is one big hack. - # - # As of Python 3.6, there seems to be no consistent way to identify a type - # specification at run time. So what we have is a mess. - # - # - Many `typing` meta-utilities explicitly `raise TypeError` when one - # attempts The One Obvious Way To Do It (`isinstance`, `issubclass`). - # - # - Their `type` can be something like `typing.TypeVar`, `typing.Union`, - # ``, ``... the - # format is case-dependent. A check like `type(T) is typing.TypeVar` - # doesn't work. + # Many `typing` meta-utilities explicitly raise TypeError from isinstance/issubclass, + # so we identify them via typing.get_origin, isinstance checks, or identity comparisons. + # We also access some internal fields (__args__, __constraints__, __supertype__) where + # Python provides no official public API for run-time type introspection. # - # So, we inspect `repr(T.__class__)` to match on the names of the prickly types, - # and call `issubclass` on those that don't hate us for doing so (catching - # `TypeError`, just in case `T` is an unsupported yet prickly type). - # - # Obviously, this won't work if someone subclasses one of the prickly types. - # `issubclass` would be The Right Thing, but since it's explicitly blocked, - # there's not much we can do. - - # TODO: Right now we're accessing internal fields to get what we need. - # TODO: Would be nice to update this if Python, at some point, adds an - # TODO: official API to access the static type information at run time. + # Unsupported typing features: + # NamedTuple (specific NamedTuple subclasses work via isinstance fallback), + # Generic, ForwardRef if T is typing.Any: return True + # NoReturn means a function never returns — no value has this type. + # Never (3.11+) is the bottom type; semantically the same for our purposes. + if T is typing.NoReturn: + return False + if sys.version_info >= (3, 11) and T is typing.Never: + return False + # AnyStr normalizes to TypeVar("AnyStr", str, bytes) - # Python 3.6 has "typing.TypeVar" as the repr, but Python 3.7+ adds the "" around it. - if repr(T.__class__) == "typing.TypeVar" or repr(T.__class__) == "": + if isinstance(T, typing.TypeVar): if not T.__constraints__: # just an abstract type name return True return any(isoftype(value, U) for U in T.__constraints__) - # TODO: Here is THE FULL LIST of `typing` features we **don't** currently support, - # TODO: as of Python 3.8 (March 2020). https://docs.python.org/3/library/typing.html - # TODO: If you add a feature to the type checker, please update this list. - # - # Python 3.6+: - # NamedTuple, DefaultDict, Counter, ChainMap, - # IO, TextIO, BinaryIO, - # Pattern, Match, (regular expressions) - # Generic, Type, - # Awaitable, Coroutine, AsyncIterable, AsyncIterator, - # ContextManager, AsyncContextManager, - # Generator, AsyncGenerator, - # NoReturn (callable return value only), - # ClassVar, Final - # - # Python 3.7+: OrderedDict - # Python 3.8+: Protocol, TypedDict, Literal - # - # TODO: Do we need to support `typing.ForwardRef`? - # No, if `get_type_hints` already resolves that. Consider our main use case, - # in `unpythonic.dispatch`. And see: - # https://docs.python.org/3/library/typing.html#typing.get_type_hints - - # TODO: Python 3.8 adds `typing.get_origin` and `typing.get_args`: - # https://docs.python.org/3/library/typing.html#typing.get_origin - # TODO: We replicate them here so that we can use them in 3.7. - # TODO: Delete the local copies once we start requiring Python 3.8. - # - # Used under the PSF license. Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, - # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; All Rights Reserved - # https://github.com/python/cpython/blob/3.8/LICENSE - def get_origin(tp): - """Get the unsubscripted version of a type. - This supports generic types, Callable, Tuple, Union, Literal, Final and ClassVar. - Return None for unsupported types. Examples:: - get_origin(Literal[42]) is Literal - get_origin(int) is None - get_origin(ClassVar[int]) is ClassVar - get_origin(Generic) is Generic - get_origin(Generic[T]) is Generic - get_origin(Union[T, int]) is Union - get_origin(List[Tuple[T, T]][int]) == list - """ - if isinstance(tp, _MyGenericAlias): - return tp.__origin__ - if tp is typing.Generic: - return typing.Generic - return None - # def get_args(tp): - # """Get type arguments with all substitutions performed. - # For unions, basic simplifications used by Union constructor are performed. - # Examples:: - # get_args(Dict[str, int]) == (str, int) - # get_args(int) == () - # get_args(Union[int, Union[T, int], str][int]) == (int, str) - # get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - # get_args(Callable[[], T][int]) == ([], int) - # """ - # if isinstance(tp, _MyGenericAlias) and not tp._special: - # res = tp.__args__ - # if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis: - # res = (list(res[:-1]), res[-1]) - # return res - # return () - # <--- end of local copies of get_origin and get_args. The rest is our code. - - # Optional normalizes to Union[argtype, NoneType]. - # Python 3.6 has the repr, 3.7+ use typing._GenericAlias. - if repr(T.__class__) == "typing.Union" or get_origin(T) is typing.Union: - if T.__args__ is None: # Python 3.6 bare `typing.Union`; empty, has no types in it, so no value can match. - return False - if not any(isoftype(value, U) for U in T.__args__): - return False - return True + # typing.Union[X, Y] and the builtin X | Y syntax (types.UnionType, Python 3.10+). + # Optional[X] normalizes to Union[X, NoneType]. + if typing.get_origin(T) is typing.Union or isinstance(T, types.UnionType): + return any(isoftype(value, U) for U in T.__args__) - # Python 3.7+ bare typing.Union; empty, has no types in it, so no value can match. - if T is typing.Union: # isinstance(T, typing._SpecialForm) and T._name == "Union": - return False # pragma: no cover, Python 3.7+ only. + # Bare typing.Union; empty, has no types in it, so no value can match. + if T is typing.Union: + return False # pragma: no cover - # TODO: in Python 3.7+, what is the mysterious callable that doesn't have `__qualname__`? - if callable(T) and hasattr(T, "__qualname__") and T.__qualname__ == "NewType..new_type": + def isNewType(T): + return isinstance(T, typing.NewType) + if isNewType(T): # This is the best we can do, because the static types created by `typing.NewType` # have a constructor that discards the type information at runtime: # UserId = typing.NewType("UserId", int) @@ -193,45 +126,146 @@ def get_origin(tp): # print(type(i)) # int return isinstance(value, T.__supertype__) - # Some one-trick ponies. - for U in (typing.Iterator, # can't non-destructively check element type - typing.Iterable, # can't non-destructively check element type - typing.Container, # can't check element type - typing.Collection, # Sized Iterable Container; can't check element type - typing.Hashable, - typing.Sized): - if U is T: - return isinstance(value, U) + # Literal[v1, v2, ...] — value must be one of the listed constants. + if typing.get_origin(T) is typing.Literal: + return value in T.__args__ - if T is typing.Reversible: # can't non-destructively check element type - # We don't isinstance(), because in Python 3.5, typing.Reversible used to be just a protocol, - # and ": Protocols cannot be used with isinstance()." - # https://docs.python.org/3/library/collections.abc.html#module-collections.abc - return hasattr(value, "__reversed__") + # Type[X] — value must be a class that is X or a subclass of X. + if typing.get_origin(T) is type: + if not isinstance(value, type): + return False + args = getattr(T, "__args__", None) + if args is None: + return True # bare Type, any class matches + return issubclass(value, args[0]) + + # ClassVar[T] and Final[T] — these are declaration wrappers. At runtime, + # we just strip the wrapper and check the inner type. + for wrapper_origin in (typing.ClassVar, typing.Final): + if typing.get_origin(T) is wrapper_origin: + args = getattr(T, "__args__", None) + if args is None: + return True # bare ClassVar or Final, no inner type constraint + return isoftype(value, args[0]) + + # Non-generic ABCs, and parametric ABCs where element type can't be checked. + # Iterator: consumed by iteration. Container: only has __contains__, can't enumerate. + # Hashable, Sized: not generic (can't be parameterized). + for abc in (collections.abc.Hashable, + collections.abc.Sized, + collections.abc.Iterator, + collections.abc.Container): + if typing.get_origin(T) is abc: + return isinstance(value, abc) + + # Parametric ABCs with best-effort element checking. + # If the value is Sized (a concrete collection), we can safely iterate + # and check elements. Otherwise (opaque iterator), accept on ABC alone. + for abc in (collections.abc.Iterable, + collections.abc.Collection, + collections.abc.Reversible): + if typing.get_origin(T) is abc: + if not isinstance(value, abc): + return False + args = getattr(T, "__args__", None) + if args is None: + return True # bare form, no element type constraint + assert len(args) == 1 + if not isinstance(value, collections.abc.Sized): + return True # opaque iterator — can't check elements non-destructively + if not value: # empty sized collection has no element type + return False + U = args[0] + return all(isoftype(elt, U) for elt in value) # "Protocols cannot be used with isinstance()", so: for U in (typing.SupportsInt, typing.SupportsFloat, typing.SupportsComplex, typing.SupportsBytes, - _MySupportsIndex, + typing.SupportsIndex, typing.SupportsAbs, typing.SupportsRound): if U is T: return safeissubclass(type(value), U) + # TypedDict — structural check on dict contents. + # isinstance doesn't work with TypedDict, so we check keys and value types. + if typing.is_typeddict(T): + if not isinstance(value, dict): + return False + hints = typing.get_type_hints(T) + required = T.__required_keys__ + optional = T.__optional_keys__ + allowed = required | optional + if not required.issubset(value.keys()): + return False + if not set(value.keys()).issubset(allowed): + return False + return all(isoftype(v, hints[k]) for k, v in value.items()) + # We don't have a match yet, so T might still be one of those meta-utilities # that hate `issubclass` with a passion. - if safeissubclass(T, typing.Text): # https://docs.python.org/3/library/typing.html#typing.Text - return isinstance(value, str) # alias for str + # DEPRECATED: typing.Text is deprecated since Python 3.11 (it's just an alias for str). + # TODO: Remove this branch when the floor bumps to Python 3.12. + if safeissubclass(T, typing.Text): + return isinstance(value, str) + + # IO, TextIO, BinaryIO — typing module stubs that don't participate in the + # MRO of real IO classes. Map to the io module ABCs instead. + # IO[str] → TextIO, IO[bytes] → BinaryIO when parametric. + if T is typing.IO or typing.get_origin(T) is typing.IO: + args = getattr(T, "__args__", None) + if args is not None: + if args[0] is str: + return isinstance(value, io.TextIOBase) + if args[0] is bytes: + return isinstance(value, (io.RawIOBase, io.BufferedIOBase)) + return isinstance(value, io.IOBase) + if T is typing.TextIO: + return isinstance(value, io.TextIOBase) + if T is typing.BinaryIO: + return isinstance(value, (io.RawIOBase, io.BufferedIOBase)) + + # Pattern[T] and Match[T] — the type arg (str or bytes) can be checked. + if typing.get_origin(T) is re.Pattern: + if not isinstance(value, re.Pattern): + return False + args = getattr(T, "__args__", None) + if args is not None: + return isinstance(value.pattern, args[0]) + return True + if typing.get_origin(T) is re.Match: + if not isinstance(value, re.Match): + return False + args = getattr(T, "__args__", None) + if args is not None: + return isinstance(value.string, args[0]) + return True - # Subclass test for Python 3.6 only. Python 3.7+ have typing._GenericAlias for the generics. - if safeissubclass(T, typing.Tuple) or get_origin(T) is tuple: + # ContextManager and AsyncContextManager — can't check the return type + # of __enter__ non-destructively, so just check the ABC. + if typing.get_origin(T) is contextlib.AbstractContextManager: + return isinstance(value, contextlib.AbstractContextManager) + if typing.get_origin(T) is contextlib.AbstractAsyncContextManager: + return isinstance(value, contextlib.AbstractAsyncContextManager) + + # Async ABCs and generator types — type parameters (yield, send, return) + # can't be checked non-destructively, so just check the ABC. + for runtimetype in (collections.abc.Awaitable, + collections.abc.Coroutine, + collections.abc.AsyncIterable, + collections.abc.AsyncIterator, + collections.abc.Generator, + collections.abc.AsyncGenerator): + if typing.get_origin(T) is runtimetype: + return isinstance(value, runtimetype) + + if typing.get_origin(T) is tuple: if not isinstance(value, tuple): return False # bare `typing.Tuple`, no restrictions on length or element type. - # Python 3.9: if a generic has no args, it has no `__args__` attribute. - if not hasattr(T, "__args__") or not T.__args__: + if not getattr(T, "__args__", None): return True # homogeneous element type, arbitrary length if len(T.__args__) == 2 and T.__args__[1] is Ellipsis: @@ -247,35 +281,47 @@ def get_origin(tp): return all(isoftype(elt, U) for elt, U in zip(value, T.__args__)) # Check mapping types that allow non-destructive iteration. - def ismapping(statictype, runtimetype): + def ismapping(runtimetype): if not isinstance(value, runtimetype): return False - # Python 3.9: if a generic has no args, it has no `__args__` attribute. - if not hasattr(T, "__args__") or T.__args__ is None: # Python 3.6: consistent behavior with 3.7+, which use unconstrained TypeVar KT, VT. + args = getattr(T, "__args__", None) + if args is None: args = (typing.TypeVar("KT"), typing.TypeVar("VT")) - else: - args = T.__args__ assert len(args) == 2 if not value: # An empty dict has no key and value types. return False K, V = args return all(isoftype(k, K) and isoftype(v, V) for k, v in value.items()) - for statictype, runtimetype in ((typing.Dict, dict), - (typing.MutableMapping, collections.abc.MutableMapping), - (typing.Mapping, collections.abc.Mapping)): - if safeissubclass(T, statictype) or get_origin(T) is runtimetype: - return ismapping(statictype, runtimetype) + # Counter[T] is a mapping (keys: T, values: int), but has only one type arg. + if typing.get_origin(T) is collections.Counter: + if not isinstance(value, collections.Counter): + return False + args = getattr(T, "__args__", None) + if args is None: + args = (typing.TypeVar("T"),) + assert len(args) == 1 + if not value: + return False + U = args[0] + return all(isoftype(k, U) and isinstance(v, int) for k, v in value.items()) + + for runtimetype in (dict, + collections.defaultdict, + collections.OrderedDict, + collections.ChainMap, + collections.abc.MutableMapping, + collections.abc.Mapping): + if typing.get_origin(T) is runtimetype: + return ismapping(runtimetype) # ItemsView is a special-case mapping in that we must not call # `.items()` on `value`. - if safeissubclass(T, typing.ItemsView) or get_origin(T) is collections.abc.ItemsView: + if typing.get_origin(T) is collections.abc.ItemsView: if not isinstance(value, collections.abc.ItemsView): return False - # Python 3.9: if a generic has no args, it has no `__args__` attribute. - if not hasattr(T, "__args__") or T.__args__ is None: # Python 3.6: consistent behavior with 3.7+, which use unconstrained TypeVar KT, VT. + args = getattr(T, "__args__", None) + if args is None: args = (typing.TypeVar("KT"), typing.TypeVar("VT")) - else: - args = T.__args__ assert len(args) == 2 if not value: # An empty dict has no key and value types. return False @@ -291,17 +337,17 @@ def ismapping(statictype, runtimetype): def iscollection(statictype, runtimetype): if not isinstance(value, runtimetype): return False - if safeissubclass(statictype, typing.ByteString) or get_origin(statictype) is collections.abc.ByteString: + if typing.get_origin(statictype) is collections.abc.ByteString: + # DEPRECATED: typing.ByteString is deprecated since Python 3.12. + # TODO: Remove this branch and the ByteString entry in the loop below + # when the floor bumps to Python 3.12. + # # WTF? A ByteString is a Sequence[int], but only statically. - # At run time, the `__args__` are actually empty - it looks + # At run time, the `__args__` are actually empty — it looks # like a bare Sequence, which is invalid. HACK the special case. typeargs = (int,) - # Python 3.9: if a generic has no args, it has no `__args__` attribute. - elif hasattr(T, "__args__"): - typeargs = T.__args__ else: - typeargs = None - # Python 3.6: consistent behavior with 3.7+, which use an unconstrained TypeVar T. + typeargs = getattr(T, "__args__", None) if typeargs is None: typeargs = (typing.TypeVar("T"),) # Judging by the docs, List takes one type argument. The rest are similar. @@ -315,7 +361,7 @@ def iscollection(statictype, runtimetype): (typing.FrozenSet, frozenset), (typing.Set, set), (typing.Deque, collections.deque), - (typing.ByteString, collections.abc.ByteString), # must check before Sequence + (typing.ByteString, collections.abc.ByteString), # DEPRECATED; must check before Sequence (typing.MutableSet, collections.abc.MutableSet), # must check mutable first # because a mutable value has *also* the interface of the immutable variant # (e.g. MutableSet is a subtype of AbstractSet) @@ -326,13 +372,11 @@ def iscollection(statictype, runtimetype): (typing.MutableSequence, collections.abc.MutableSequence), (typing.MappingView, collections.abc.MappingView), (typing.Sequence, collections.abc.Sequence)): - if safeissubclass(T, statictype) or get_origin(T) is runtimetype: + if typing.get_origin(T) is runtimetype: return iscollection(statictype, runtimetype) - if safeissubclass(T, typing.Callable) or get_origin(T) is collections.abc.Callable: - if not callable(value): - return False - return True + if typing.get_origin(T) is collections.abc.Callable: + return callable(value) # # TODO: analyze Callable[[a0, a1, ...], ret], Callable[..., ret]. # if T.__args__ is None: # bare `typing.Callable`, no restrictions on arg/return types. # return True @@ -359,6 +403,18 @@ def iscollection(statictype, runtimetype): # return False # return True + # Protocol — support @runtime_checkable Protocols; clear error for others. + # Specific Protocols (Supports* ABCs) are already handled above by identity check. + # We use _is_protocol (not issubclass) because issubclass(X, Protocol) returns + # True for some non-Protocol types (e.g. int) on Python 3.10. + if isinstance(T, type) and T is not typing.Protocol and getattr(T, '_is_protocol', False): + if getattr(T, '_is_runtime_protocol', False): + return isinstance(value, T) + raise TypeError( + f"isoftype: {T.__qualname__} is a Protocol but not @typing.runtime_checkable, " + f"so runtime structural checks are not possible. " + f"Add @typing.runtime_checkable to enable isinstance checks.") + # Catch any `typing` meta-utilities we don't currently support. if hasattr(T, "__module__") and T.__module__ == "typing": # pragma: no cover, only happens when something goes wrong. fullname = repr(T.__class__) diff --git a/uploaddist.sh b/uploaddist.sh deleted file mode 100755 index 0b2ac6d9..00000000 --- a/uploaddist.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -VERSION="$1" -twine upload dist/unpythonic-${VERSION}.tar.gz dist/unpythonic-${VERSION}-py3-none-any.whl