"""The substrate the scene round-trip reads and edits Python source through.
Everything above this package deals in *statements*, *leading trivia*, *trailing
comments*, *statement groups* and *arguments*. No parser type crosses the seam
in either direction, so the tier above can be read without knowing which parser
is underneath, and the parser can be replaced without the tier above noticing.
A :class:`SourceDocument` is parsed from text and dumps back to text, byte for
byte where nothing was edited. What it hands out are :class:`Statement`
handles, which are markers rather than parse-tree nodes: an edit that takes a
statement out of the document leaves any handle to it stale, and using a stale
handle raises :class:`StaleHandleError` rather than answering from something the
document no longer holds. :attr:`SourceDocument.epoch` counts the mutations, so
a caller holding handles across an edit can tell that one happened.
Backends register under a name and the default one is resolved lazily, so
importing this package costs nothing until a document is parsed. One is
registered: ``ast``, which is :data:`DEFAULT_BACKEND`, reads through CPython's
own parser and so accepts exactly the Python that runs. The registry stays
because a second backend is a legitimate thing to add, not because one is
expected.
Trivia is defined so the two queries never overlap and neither depends on how a
parser stores whitespace:
* **leading trivia** is everything between the previous statement's line
terminator and this statement's first character (blank lines, whole-line
comments, the indent run), minus a comment that was written on the previous
statement's own line;
* **trailing comment** is the comment written after the code on the statement's
first line, and only the last statement on a line can have one.
The editing rules the tier above depends on, stated once here because they are
the contract rather than an implementation detail:
* a removed statement takes its own leading trivia with it, and surplus blank
lines collapse to at most one so repeated removals do not open a hole;
* a moved statement carries its leading trivia, except that the block's head
trivia stays with whatever ends up first;
* an inserted statement copies the indent of its anchor and nothing else.
"""
from __future__ import annotations
import abc
from collections.abc import Iterable
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Sequence
__all__ = [
"Anchor",
"Argument",
"CallView",
"ClassDecl",
"DEFAULT_BACKEND",
"ImportDecl",
"ImportedName",
"SceneSyntaxError",
"SourceBackend",
"SourceDocument",
"StaleHandleError",
"Statement",
"StatementKind",
"backend_names",
"get_backend",
"names_bound_in",
"names_mentioned_in",
"parse",
"parse_expression",
"rename_name",
]
#: The backend :func:`parse` uses when no other is named.
DEFAULT_BACKEND = "ast"
[docs]
class SceneSyntaxError(ValueError):
"""The source cannot be parsed, presented as the parser reported it.
Carries the parser's own message and the position it stopped at, so a
refusal can quote both. ``line`` is 1-indexed and ``column`` 0-indexed,
matching the editor's cursor convention.
"""
def __init__(self, message: str, *, line: int | None = None, column: int | None = None) -> None:
where = "" if line is None else f" (line {line})"
super().__init__(f"{message}{where}")
self.message = message
self.line = line
self.column = column
[docs]
@classmethod
def from_syntax_error(cls, error: SyntaxError) -> SceneSyntaxError:
"""Wrap a :class:`SyntaxError` without editing what it says.
CPython's message, line and caret column are what a user should read;
this only moves them across the seam. The column is converted to the
0-indexed convention :class:`SceneSyntaxError` documents.
"""
column = None if error.offset is None else max(error.offset - 1, 0)
wrapped = cls(error.msg, line=error.lineno, column=column)
wrapped.__cause__ = error
return wrapped
[docs]
class StaleHandleError(RuntimeError):
"""A handle was used after the document stopped holding its statement.
Raised rather than answering from a statement the document has removed or
rewritten, so a caller keeping handles across an edit finds out at the point
of use instead of writing the answer somewhere.
"""
[docs]
class StatementKind(StrEnum):
"""What a statement is, at the coarseness the tier above distinguishes."""
#: Binds something: ``x = f()``, ``self.x: int = 1``, ``x += 1``.
ASSIGNMENT = "assignment"
#: An expression evaluated for its effect: ``self.add_child(hero)``.
EXPRESSION = "expression"
#: ``import x`` or ``from x import y``.
IMPORT = "import"
#: ``return`` or ``return <expression>``.
RETURN = "return"
#: A statement carrying a block of its own: ``if``, ``for``, ``with``,
#: ``try``, ``while``, ``def``, ``class``, ``match``.
COMPOUND = "compound"
#: Everything else a statement can be: ``pass``, ``raise``, ``del``,
#: ``global``, ``assert``.
OTHER = "other"
[docs]
class Anchor(StrEnum):
"""Where an insertion lands when no statement is there to anchor it."""
#: The top of ``__init__``'s body, above every statement it holds.
INIT_HEAD = "init_head"
#: The bottom of ``__init__``'s body, below every statement it holds.
INIT_TAIL = "init_tail"
#: The top of the class body, above the first thing the class declares.
BODY_HEAD = "body_head"
#: The bottom of the class body, below everything the class declares.
BODY_TAIL = "body_tail"
[docs]
@dataclass(frozen=True, slots=True)
class Argument:
"""One thing a call passes, as it is written.
``name`` is the keyword it is passed under, and ``None`` for one passed by
position or by unpacking. ``index`` counts every argument the call passes,
named and not, so it matches the order the interpreter reads them in;
:meth:`CallView.set_positional` counts only the unnamed ones and takes its
own index.
"""
#: The keyword, or ``None`` when the argument names none.
name: str | None
#: The source of the value, exactly as written.
value: str
#: Where it sits among everything the call passes, from 0.
index: int
#: True for ``*args`` / ``**kwargs``, which name no parameter.
is_unpacking: bool = False
[docs]
@dataclass(frozen=True, slots=True)
class ImportedName:
"""One name an import binds, with the alias it binds it under."""
#: The name as written in the module being imported from.
name: str
#: The name it is bound to locally, or ``None`` when it is bound as itself.
alias: str | None = None
[docs]
@property
def binding(self) -> str:
"""The name the importing module can use."""
return self.alias or self.name
[docs]
class Statement(abc.ABC):
"""One statement of a document, held as a marker rather than a tree node.
A semicolon writes several statements on one line and each of them is one
of these; :meth:`line_group` is how a caller asks for the rest of the line,
and :meth:`shares_line` how it asks whether there is any.
Two handles to the same statement compare equal and hash alike, so a caller
can hold them in sets and dicts across the edits that do not disturb them.
"""
__slots__ = ()
[docs]
@abc.abstractmethod
def text(self) -> str:
"""The statement's own source, with no leading trivia and no line terminator.
A comment written after the code is not part of it: that is
:meth:`trailing_comment`. A statement carrying a block of its own comes
back whole, header and body together, since that is what the statement
is, and the comments written inside the block come with it.
"""
[docs]
@abc.abstractmethod
def kind(self) -> StatementKind:
"""What kind of statement this is."""
[docs]
@abc.abstractmethod
def leading_trivia(self) -> str:
"""The blank lines, whole-line comments and indent written above this.
Empty for a statement written directly under the previous one at column
zero. A comment that sat on the previous statement's line is not part of
this: it is that statement's :meth:`trailing_comment`.
"""
[docs]
@abc.abstractmethod
def shares_line(self) -> bool:
"""Is another statement written on this one's line, behind a semicolon?"""
[docs]
@abc.abstractmethod
def line_group(self) -> list[Statement]:
"""Every statement written on this one's line, including this one, in order."""
[docs]
@abc.abstractmethod
def line(self) -> int:
"""The 1-indexed line the statement starts on, as the document stands now."""
[docs]
@abc.abstractmethod
def column(self) -> int:
"""The 0-indexed column the statement starts at, as the document stands now."""
[docs]
@abc.abstractmethod
def is_live(self) -> bool:
"""Does the document still hold this statement?
The question :class:`StaleHandleError` is the answer to everywhere else:
asking it costs nothing and never raises, so a caller sweeping a list of
handles after an edit can drop the dead ones itself.
"""
[docs]
@abc.abstractmethod
def call(self) -> CallView | None:
"""The call this statement's value is, or ``None`` when it is not one.
The value of an assignment is what it assigns (``hero = Sprite2D(...)``
gives the construction) and the value of an expression statement is the
expression itself (``self.add_child(hero)`` gives the ``add_child``
call). A call passed as an argument to that one is reached through
:meth:`CallView.argument_call`.
"""
[docs]
@abc.abstractmethod
def set_value(self, text: str) -> None:
"""Write ``text`` where this statement's value is, keeping the rest of it.
The same value :meth:`call` reads: what an assignment assigns, or the
expression an expression statement is. Everything written around it
stays as the author wrote it -- the targets, the type an annotation
gives them, the comment written after the code, and the lines the
statement is spread over, which are not laid out again because the
author's layout is not this operation's to revise.
``text`` must be one expression, and is refused with
:class:`SceneSyntaxError` when it is not. A statement that has no value
to replace -- an annotation standing on its own, a ``pass``, a block --
raises :class:`ValueError`.
An expression statement *is* its value, so writing over it writes over
the statement: handles to it go stale, as they do for any statement the
document stops holding. An assignment keeps its handle, since what was
replaced was written inside it.
"""
[docs]
@abc.abstractmethod
def bound_names(self) -> set[str]:
"""Every name this statement binds, attributes spelled ``self.<name>``.
Assignment targets, ``for`` and ``with`` and ``except`` targets, and the
names an ``import``, a ``def`` or a ``class`` binds. A statement
carrying a block is asked about its whole block, since a name bound
inside one is bound by the statement as far as anything outside it can
tell.
"""
[docs]
@abc.abstractmethod
def mentioned_names(self) -> set[str]:
"""Every name this statement writes, bound here or not, ``self.<name>`` for attributes.
Both directions of a dependency are this question: the statement
establishing a name and the statement reading it both mention it. The
name after a dot belongs to whatever precedes it rather than to any
binding, and the name before the ``=`` of a keyword argument is a
parameter of the callee, so neither counts.
"""
[docs]
@abc.abstractmethod
def rename_local(self, old: str, new: str) -> int:
"""Rewrite every mention of the local ``old`` as ``new``; return how many.
Only names standing on their own are touched, which is what a local is:
an attribute after a dot and a keyword argument's parameter name are
left as they are written.
"""
[docs]
class CallView(abc.ABC):
"""Argument-level operations on one call expression.
A view, not a copy: every method reads or writes the document the statement
it came from belongs to, and using one after that statement is gone raises
:class:`StaleHandleError`.
"""
__slots__ = ()
[docs]
@abc.abstractmethod
def callee(self) -> str:
"""The source of what is being called, without the arguments."""
[docs]
@abc.abstractmethod
def text(self) -> str:
"""The source of the whole call expression, arguments included."""
[docs]
@abc.abstractmethod
def arguments(self) -> list[Argument]:
"""Everything the call passes, named and not, in the order it is written."""
[docs]
@abc.abstractmethod
def kwarg(self, name: str) -> str | None:
"""The source of the value passed under ``name``, or ``None`` for none."""
[docs]
@abc.abstractmethod
def set_kwarg(self, name: str, value_expr: str) -> None:
"""Pass ``value_expr`` under ``name``, replacing any value already there.
Appends when the call does not pass ``name`` yet, keeping the order of
the arguments it does pass. The statement is laid out again afterwards,
so a call that no longer fits its line breaks the way the emitter would
have broken it, and one that fits again comes back together.
"""
[docs]
@abc.abstractmethod
def remove_kwarg(self, name: str) -> None:
"""Stop passing ``name``. Raises :class:`ValueError` when it is not passed."""
[docs]
@abc.abstractmethod
def set_positional(self, index: int, value_expr: str) -> None:
"""Replace the ``index``-th argument the call passes by position.
Which parameter a position fills is a fact about the callable rather
than about the text, so establishing that ``index`` is the caller's
business. Raises :class:`ValueError` when the call passes no such
position.
"""
[docs]
@abc.abstractmethod
def argument_call(self, index: int) -> CallView | None:
"""The call passed at ``index``, or ``None`` when what is passed is not one.
``index`` counts every argument, as :attr:`Argument.index` does, so a
call written inside another (``self.add_child(Panel(name="A"))``) is
reached without the caller taking the text apart.
"""
[docs]
class ImportDecl(abc.ABC):
"""One import statement, read as what it binds rather than as text."""
__slots__ = ()
[docs]
@property
@abc.abstractmethod
def statement(self) -> Statement:
"""The statement this import is written as, for editing and removal."""
[docs]
@property
@abc.abstractmethod
def module(self) -> str | None:
"""The module named after ``from``, or ``None`` for a plain ``import``."""
[docs]
@property
@abc.abstractmethod
def names(self) -> list[ImportedName]:
"""The names this import binds, in the order they are written.
``from x import *`` binds nothing this can name, and comes back empty.
"""
[docs]
class ClassDecl(abc.ABC):
"""One top-level class of a document, and the blocks a scene edits.
Two blocks matter: the class body, which is where a class declares things
about itself, and ``__init__``'s body, which is where a scene builds its
tree.
"""
__slots__ = ()
[docs]
@property
@abc.abstractmethod
def name(self) -> str:
"""The class's name."""
[docs]
@property
@abc.abstractmethod
def statement(self) -> Statement:
"""The statement this class is written as, for placing things around it.
The whole of it, decorators included: that is what the module holds in
its order, and inserting above the class means inserting above the
decorators it carries.
"""
[docs]
@abc.abstractmethod
def line(self) -> int:
"""The 1-indexed line the class statement starts on."""
[docs]
@abc.abstractmethod
def body_statements(self) -> list[Statement]:
"""Every statement of the class body, in order, ``__init__`` among them.
A semicolon-joined line counts as the several statements it holds.
"""
[docs]
@abc.abstractmethod
def has_init(self) -> bool:
"""Does the class define ``__init__``?"""
[docs]
@abc.abstractmethod
def init_statements(self) -> list[Statement]:
"""Every statement of ``__init__``'s body, in order, blocks among them.
A semicolon-joined line counts as the several statements it holds, and a
statement carrying a block of its own counts as one: what is inside it
is that statement's business. Empty when the class defines no
``__init__``.
"""
[docs]
@abc.abstractmethod
def bases(self) -> list[str]:
"""The classes this one derives from, each as the header spells it.
Empty for a class written with no bases and for one written with empty
parentheses. A keyword the header passes to the metaclass machinery
(``metaclass=Meta``) comes back too, since it is written where a base
is; it simply does not answer to a base's name. What :meth:`set_base`
matches ``old`` against, read rather than written.
"""
[docs]
@abc.abstractmethod
def set_base(self, old: str, new: str) -> None:
"""Write ``new`` where the class lists ``old`` among the classes it derives from.
``old`` is one base as the header spells it, dots and all. ``new`` is
what takes its place in the header, and is a base *list* rather than one
base: ``"Mixin, Node3D"`` puts two where one was, which is how a class
gains a behaviour without losing the base it already had. It has to
satisfy the grammar of what a class header holds and nothing looser, so
a keyword (``metaclass=Meta``) is accepted and text that is not a base
list is refused with :class:`SceneSyntaxError`.
Only the matched slot is rewritten: the other bases, the punctuation
between them and the spacing around them stay as the author wrote them,
and so does every other byte of the file. A class that does not list
``old`` raises :class:`ValueError` rather than gaining a base it was
not asked for.
"""
[docs]
@abc.abstractmethod
def insert(self, text: str, *, after: Statement | Anchor) -> Statement:
"""Write ``text`` as a new statement below ``after``; return a handle to it.
``text`` is one statement's source at column zero: the indent comes from
the anchor's line and nothing else is added, so statements inserted in
sequence pack tightly. A :class:`Statement` anchor a semicolon put in
the middle of a line is taken to mean that line.
"""
[docs]
@abc.abstractmethod
def insert_before(self, text: str, *, before: Statement) -> Statement:
"""Write ``text`` as a new statement above ``before``; return a handle to it.
The leading trivia of ``before`` stays on ``before``: a comment written
above a statement describes that statement, not whatever is inserted in
front of it.
"""
[docs]
@abc.abstractmethod
def remove(self, stmt: Statement) -> None:
"""Take ``stmt`` out, with its leading trivia, collapsing surplus blank lines.
A statement sharing its line goes without taking the line: the others
written on it stay, and the line keeps its indent whichever end it lost.
"""
[docs]
@abc.abstractmethod
def reorder(self, groups: list[list[Statement]]) -> None:
"""Rewrite the statements in ``groups`` to run in the order given.
Each group is the statements that move together, and every statement in
every group must be one this class holds, in one block, once. The lines
they are written on are re-inserted contiguously where the first of them
was, so a statement of the author's that was between two groups ends up
after all of them.
The trivia above the first line stays where it is, on whatever ends up
first; every other line carries its own. A statement sharing its line
with one outside its group is refused with :class:`ValueError`, since
the line cannot follow the group without taking that statement along.
"""
[docs]
class SourceDocument(abc.ABC):
"""A parsed Python source file that dumps back byte for byte.
Handed out by :func:`parse`. Everything an edit needs is reached from here:
the classes, the imports, and the statements they hold.
"""
__slots__ = ()
[docs]
@classmethod
def parse(cls, text: str, *, backend: str | None = None) -> SourceDocument:
"""Parse ``text``, raising :class:`SceneSyntaxError` when it will not.
The spelling most call sites use; :func:`parse` is the same call.
"""
return get_backend(backend).parse(text)
[docs]
@property
@abc.abstractmethod
def backend_name(self) -> str:
"""Which backend read this document."""
[docs]
@property
@abc.abstractmethod
def epoch(self) -> int:
"""How many mutations this document has taken, from 0.
Rises by one per operation that changes the text and never falls, so a
caller holding state derived from the document can tell it is stale
without diffing anything.
"""
[docs]
@property
@abc.abstractmethod
def original_text(self) -> str:
"""The text this document was parsed from."""
[docs]
@abc.abstractmethod
def dump(self) -> str:
"""The document's current text, byte for byte where nothing was edited."""
[docs]
@abc.abstractmethod
def is_unchanged(self) -> bool:
"""Is :meth:`dump` still byte-identical to what was parsed?"""
[docs]
@abc.abstractmethod
def top_level_classes(self) -> list[ClassDecl]:
"""Every class the module declares at the top level, in source order."""
[docs]
@abc.abstractmethod
def find_class(self, name: str) -> ClassDecl | None:
"""The first top-level class called ``name``, or ``None`` for none."""
[docs]
@abc.abstractmethod
def imports(self) -> list[ImportDecl]:
"""Every top-level import, in source order."""
[docs]
@abc.abstractmethod
def top_level_statements(self) -> list[Statement]:
"""Every statement the module holds at the top level, in source order."""
[docs]
@abc.abstractmethod
def insert_top_level(self, text: str, *, after: Statement | None = None) -> Statement:
"""Write ``text`` at module scope below ``after``; return a handle to it.
``text`` is written as given, at column zero, with a line terminator
added when it ends without one: what separates it from its neighbours is
the caller's to write, because a blank line between two classes and no
blank line between two imports are both correct and this cannot tell
which is being inserted. A handle to the first statement it holds comes
back.
``after`` is ``None`` for the top of the module, above everything it
holds but below the comments and blank lines written above the first
statement, which describe that statement rather than the file. The end
of the module is the last of :meth:`top_level_statements`.
"""
[docs]
@abc.abstractmethod
def ensure_import(self, name: str, *, from_: str | None = None) -> None:
"""Import ``name``, from ``from_`` when one is named, unless it already is.
A ``from <from_> import ...`` line already in the file takes the new
name rather than a second line being written: the names are sorted,
deduplicated and the line is laid out again to the width the rest of the
file is written at (:func:`~simvx.core.scene_io.layout.wrap_import`),
which keeps a hand-broken line broken and takes the brackets off one
that fits. Everything written around the names -- the module, the
comment after the code -- stays as the author wrote it, and each name
already there is carried across as it was written, alias and all.
A line of its own is written when there is nothing to merge into: no
such ``from`` line, a plain ``import``, or ``from x import *``, which
names nothing that can be added to. It goes below the last top-level
import, or at the top of the module when there is none.
"""
[docs]
@abc.abstractmethod
def remove_import(self, name: str, *, from_: str | None = None) -> None:
"""Stop importing ``name`` from ``from_``; do nothing when it is not imported.
``name`` is the name as the module being imported from spells it, which
is what :attr:`ImportedName.name` reads and what an alias renames rather
than replaces. A line that imported nothing else goes with it; a line
that imported more keeps the rest exactly as they are written.
"""
[docs]
class SourceBackend(abc.ABC):
"""One implementation of the seam, registered under a name."""
__slots__ = ()
[docs]
@property
@abc.abstractmethod
def name(self) -> str:
"""The name :func:`parse` selects this backend by."""
[docs]
@abc.abstractmethod
def parse(self, text: str) -> SourceDocument:
"""Parse ``text`` into a document, or raise :class:`SceneSyntaxError`."""
[docs]
@abc.abstractmethod
def parse_expression(self, text: str) -> None:
"""Raise :class:`SceneSyntaxError` unless ``text`` is one whole expression."""
#: Backend name -> the module holding it, imported on first use. Nothing here
#: is imported at import time: this package sits under ``simvx.core.scene_io``,
#: which the engine imports eagerly and the browser runtime cannot give a
#: parser to.
_BACKEND_MODULES = {"ast": "._source_ast"}
_BACKENDS: dict[str, SourceBackend] = {}
[docs]
def backend_names() -> tuple[str, ...]:
"""Every backend name :func:`parse` will accept, in preference order."""
return tuple(_BACKEND_MODULES)
[docs]
def get_backend(name: str | None = None) -> SourceBackend:
"""The backend called ``name``, or the default one when unnamed."""
resolved = name or DEFAULT_BACKEND
backend = _BACKENDS.get(resolved)
if backend is not None:
return backend
module_name = _BACKEND_MODULES.get(resolved)
if module_name is None:
known = ", ".join(backend_names())
raise ValueError(f"unknown source backend {resolved!r}; known backends are {known}")
from importlib import import_module
module = import_module(module_name, __name__)
backend = module.BACKEND
if not isinstance(backend, SourceBackend): # pragma: no cover - a backend module is written once
raise TypeError(f"source backend {resolved!r} does not implement SourceBackend")
_BACKENDS[resolved] = backend
return backend
[docs]
def parse(text: str, *, backend: str | None = None) -> SourceDocument:
"""Parse ``text`` into a document, raising :class:`SceneSyntaxError` when it will not."""
return get_backend(backend).parse(text)
[docs]
def parse_expression(text: str, *, backend: str | None = None) -> None:
"""Check that ``text`` is one whole Python expression, and nothing else.
What an emitted value is put through before it is spliced into a document,
so a value that would not parse is refused where it was written rather than
where it lands. Returns nothing: the answer is whether it raises.
"""
get_backend(backend).parse_expression(text)
[docs]
def names_bound_in(statements: Iterable[Statement]) -> set[str]:
"""Every name these statements bind, attributes spelled ``self.<name>``."""
bound: set[str] = set()
for statement in statements:
bound |= statement.bound_names()
return bound
[docs]
def names_mentioned_in(statements: Iterable[Statement]) -> set[str]:
"""Every name these statements write, bound there or not, ``self.<name>`` for attributes."""
mentioned: set[str] = set()
for statement in statements:
mentioned |= statement.mentioned_names()
return mentioned
[docs]
def rename_name(statements: Sequence[Statement], old: str, new: str) -> int:
"""Rename the local ``old`` to ``new`` across ``statements``; return how many mentions moved.
A local is one name for the whole function it is written in, so the
statements passed are that function's: every mention of the name in them is
the same variable and follows. Attributes and keyword-argument names are not
locals and are left alone.
"""
return sum(statement.rename_local(old, new) for statement in statements)