Mouse Picking

click cubes to change their colour.

▶ Run in browser

Tags: 3d

Uses the engine’s built-in object picking: give a node a CollisionShape3D with pickable=True and override on_picked. Each click casts a ray (SceneTree.input_cast) and the nearest pickable collider’s owner receives on_picked. For the manual approach (casting a world-space ray with screen_to_ray and querying self.physics.raycast yourself) see raycast.py.

Usage: uv run python examples/features/3d/picking.py uv run python examples/features/3d/picking.py –test

Source

"""Mouse Picking: click cubes to change their colour.

Uses the engine's built-in object picking: give a node a ``CollisionShape3D``
with ``pickable=True`` and override ``on_picked``. Each click casts a ray
(``SceneTree.input_cast``) and the nearest pickable collider's owner receives
``on_picked``. For the manual approach (casting a world-space ray with
``screen_to_ray`` and querying ``self.physics.raycast`` yourself) see ``raycast.py``.

Usage:
    uv run python examples/features/3d/picking.py
    uv run python examples/features/3d/picking.py --test
"""

from simvx.core import (
    Camera3D,
    CollisionShape3D,
    Material,
    Mesh,
    MeshInstance3D,
    Node,
    Signal,
    SphereShape3D,
    Text2D,
)
from simvx.graphics import App


class PickableCube(MeshInstance3D):
    """A cube that responds to mouse clicks."""

    picked = Signal(str)  # emits this cube's label whenever it is clicked

    _COLOURS = [
        (0.9, 0.1, 0.1, 1),
        (0.1, 0.9, 0.1, 1),
        (0.1, 0.1, 0.9, 1),
        (0.9, 0.9, 0.1, 1),
        (0.9, 0.1, 0.9, 1),
        (0.1, 0.9, 0.9, 1),
    ]

    def __init__(self, label="Cube", colour_idx=0, **kwargs):
        mat = Material(colour=self._COLOURS[colour_idx % len(self._COLOURS)])
        super().__init__(material=mat, **kwargs)
        self.label = label
        self._colour_idx = colour_idx

    def on_ready(self):
        # Inscribed sphere (touches face centres) for tight picking
        shape = CollisionShape3D(shape=SphereShape3D(radius=0.5))
        shape.pickable = True
        self.add_child(shape)

    def on_picked(self, event):
        """Called when this node is picked by mouse ray."""
        self._colour_idx = (self._colour_idx + 1) % len(self._COLOURS)
        self.material.colour = self._COLOURS[self._colour_idx]
        self.picked.emit(self.label)


class PickingScene(Node):
    def on_ready(self):
        # Camera: looking down Y axis
        cam = Camera3D(position=(0, -15, 0))
        cam.look_at((0, 0, 0), up=(0, 0, 1))
        self.add_child(cam)

        # Shared mesh
        cube_mesh = Mesh.cube()

        # Grid of pickable cubes
        positions = [
            (-3, 0, -2),
            (0, 0, -2),
            (3, 0, -2),
            (-3, 0, 2),
            (0, 0, 2),
            (3, 0, 2),
        ]
        for i, pos in enumerate(positions):
            cube = PickableCube(
                label=f"Cube_{i}",
                colour_idx=i,
                mesh=cube_mesh,
                position=pos,
            )
            cube.picked.connect(self._on_cube_picked)
            self.add_child(cube)

        # HUD
        self.add_child(Text2D(text="Click cubes to change colour", position=(10, 10), font_scale=1.5))
        self._status = self.add_child(Text2D(text="Nothing picked yet", position=(10, 40), font_scale=1.2))

    def _on_cube_picked(self, label):
        self._status.text = f"Picked {label}"


def _selftest() -> bool:
    """Headless: click each cube where it appears on screen and check who answers.

    Nothing casts a ray here. The click is a simulated mouse press, and the frame
    loop turns it into the cast exactly as it does on the desktop, so what runs is
    the whole path: the press, the ray, the pickable collider it hits first, and
    ``on_picked`` on the node that owns it. Picking the RIGHT cube is the claim,
    so each one is clicked in turn and the label it reports back is compared to
    the one that was aimed at.
    """
    import numpy as np

    from simvx.core.testing import InputSimulator
    from simvx.graphics.testing import assert_not_blank, save_png

    WIDTH, HEIGHT = 1280, 720
    app = App(title="Picking Demo", width=WIDTH, height=HEIGHT, visible=False)
    scene = PickingScene(name="PickingScene")
    sim = InputSimulator()
    seen: dict[str, object] = {}

    def to_screen(camera, world) -> tuple[float, float]:
        """Where a world point lands on screen: the inverse of ``screen_to_ray``."""
        proj = camera.projection_matrix(WIDTH / HEIGHT)
        clip = proj @ camera.view_matrix @ np.array([*world, 1.0], dtype=np.float32)
        ndc = clip[:3] / clip[3]
        ndc_y = -ndc[1] if proj[1, 1] < 0 else ndc[1]
        return ((ndc[0] + 1.0) * 0.5 * WIDTH, (1.0 - ndc_y) * 0.5 * HEIGHT)

    cubes: list[PickableCube] = []
    plan: list[tuple[int, PickableCube]] = []  # (frame, cube) -- one click each, spaced out

    def on_frame(idx: int, _t: float) -> bool:
        if idx == 0:
            cubes.extend(c for c in scene.children if isinstance(c, PickableCube))
            camera = next(c for c in scene.children if isinstance(c, Camera3D))
            seen["camera"] = camera
            seen["start_colours"] = [tuple(c.material.colour) for c in cubes]
            plan.extend((4 + i * 4, cube) for i, cube in enumerate(cubes))
            seen["last_frame"] = plan[-1][0] + 8
        for frame, cube in plan:
            if idx == frame:
                sim.click(to_screen(seen["camera"], tuple(cube.position)))
            elif idx == frame + 1:
                seen.setdefault("answers", []).append((cube.label, scene._status.text))
        if idx == seen["last_frame"]:
            seen["after_colours"] = [tuple(c.material.colour) for c in cubes]
            # A click on the empty corner must reach nothing at all.
            seen["before_miss"] = scene._status.text
            sim.click((8.0, 8.0))
        elif idx == seen["last_frame"] + 1:
            seen["after_miss"] = scene._status.text
        return True

    frames = app.run_headless(scene, frames=40, on_frame=on_frame, capture_frames=[39])
    assert_not_blank(frames[0])
    save_png(frames[0], "/tmp/picking_test.png")

    ok = True

    def check(label: str, passed: bool, detail: str) -> None:
        nonlocal ok
        ok = ok and passed
        print(f"{'ok  ' if passed else 'FAIL'} {label}: {detail}")

    answers = seen["answers"]
    wrong = [(aimed, got) for aimed, got in answers if got != f"Picked {aimed}"]
    check(
        "each cube is picked by clicking where it appears, and no other",
        len(answers) == len(cubes) and not wrong,
        f"{len(answers)} clicks, {len(answers) - len(wrong)} answered by the cube aimed at",
    )

    # on_picked advances the cube's colour, so every cube must have moved on by
    # exactly one step of its own palette after exactly one pick each.
    palette = PickableCube._COLOURS

    def index_of(colour) -> int:
        """Which palette entry this colour is (the material stores it as float32)."""
        return min(range(len(palette)), key=lambda i: max(abs(a - b) for a, b in zip(palette[i], colour, strict=True)))

    stepped = [
        index_of(after) == (index_of(before) + 1) % len(palette)
        for before, after in zip(seen["start_colours"], seen["after_colours"], strict=True)
    ]
    check(
        "picking a cube steps it to the next colour in its palette",
        all(stepped),
        f"{sum(stepped)}/{len(stepped)} cubes advanced one step",
    )

    check(
        "a click on empty space picks nothing",
        seen["after_miss"] == seen["before_miss"],
        f"status stayed at {seen['after_miss']!r}",
    )

    print("screenshot: /tmp/picking_test.png")
    print("SELFTEST:", "PASS" if ok else "FAIL")
    return ok


if __name__ == "__main__":
    import sys

    if "--test" in sys.argv:
        sys.exit(0 if _selftest() else 1)
    app = App(title="Picking Demo", width=1280, height=720)
    app.run(PickingScene())