Screen-space refraction

a glass slab reads the scene behind it and bends it.

▶ Run in browser

Tags: 3d

A screen-reading material (Material(needs_scene_colour=True)) makes the renderer split the opaque and transparent phases: after the opaque family draws, the HDR colour + depth are copied into sampleable textures, then the transparent slab samples that scene colour (offset by its surface normal) so the colourful grid behind it appears refracted through the glass. This is the desktop scene colour/depth copy plumbing (design D1); the offset here is a minimal proof, full water shading lands with the WaterMaterial pass.

The split is zero-cost when unused: a frame with no screen-reading material records exactly as before (no pass break, no copy).

Run: uv run python examples/features/3d/refraction_probe.py

Controls: A / D - Orbit camera left / right W / S - Zoom in / out Space / Click - Toggle refraction (screen read) on/off

Source

#!/usr/bin/env python3
"""Screen-space refraction: a glass slab reads the scene behind it and bends it.

# /// simvx
# screenshot_frame = 30
# web = { width = 960, height = 720, root = "RefractionProbe", responsive = true }
# ///

A screen-reading material (``Material(needs_scene_colour=True)``) makes the
renderer split the opaque and transparent phases: after the opaque family draws,
the HDR colour + depth are copied into sampleable textures, then the transparent
slab samples that scene colour (offset by its surface normal) so the colourful
grid behind it appears refracted through the glass. This is the desktop scene
colour/depth copy plumbing (design D1); the offset here is a minimal proof, full
water shading lands with the WaterMaterial pass.

The split is zero-cost when unused: a frame with no screen-reading material
records exactly as before (no pass break, no copy).

Run: uv run python examples/features/3d/refraction_probe.py

Controls:
    A / D          - Orbit camera left / right
    W / S          - Zoom in / out
    Space / Click  - Toggle refraction (screen read) on/off
"""

import math
import sys

from simvx.core import (
    Camera3D,
    DirectionalLight3D,
    Input,
    InputMap,
    Key,
    Material,
    Mesh,
    MeshInstance3D,
    MouseButton,
    Node3D,
    PointLight3D,
    Quat,
    Text2D,
    Vec3,
    WorldEnvironment,
)
from simvx.graphics import App

WIDTH, HEIGHT = 960, 720

_GRID_COLOURS = [
    (0.90, 0.20, 0.20),
    (0.20, 0.85, 0.30),
    (0.20, 0.45, 0.95),
    (0.95, 0.80, 0.20),
    (0.85, 0.30, 0.85),
    (0.25, 0.85, 0.85),
]


class RefractionProbe(Node3D):
    def __init__(self, **kwargs):
        super().__init__(name="RefractionProbe", **kwargs)

        self._cam_angle = 90.0
        self._cam_height = 0.5
        self._cam_dist = 7.0
        self.camera = self.add_child(Camera3D(name="Camera", fov=55, near=0.1, far=100.0))
        self._update_camera()

        sun = self.add_child(DirectionalLight3D(name="Sun"))
        sun.colour = (1.0, 0.96, 0.9)
        sun.intensity = 1.6
        sun.rotation = Quat.from_euler(math.radians(-45), math.radians(-25), 0)
        fill = self.add_child(PointLight3D(name="Fill", position=Vec3(-4, 3, 5)))
        fill.colour = (0.5, 0.6, 0.9)
        fill.intensity = 0.6
        fill.range = 20.0

        # Opaque colourful grid behind the glass: the refraction target.
        k = 0
        for gy in (1.4, 0.0, -1.4):
            for gx in (-2.0, 0.0, 2.0):
                self.add_child(
                    MeshInstance3D(
                        name=f"Tile{k}",
                        mesh=Mesh.cube(1.2),
                        material=Material(colour=(*_GRID_COLOURS[k % len(_GRID_COLOURS)], 1.0), roughness=0.55),
                        position=Vec3(gx, gy, -1.2),
                    )
                )
                k += 1

        # Screen-reading glass slab in front of the grid.
        self._glass_mat = Material(
            colour=(0.75, 0.87, 1.0, 0.55),
            blend="alpha",
            roughness=0.05,
            metallic=0.0,
            needs_scene_colour=True,
        )
        self.add_child(
            MeshInstance3D(
                name="Glass",
                mesh=Mesh.cube(1.0),
                material=self._glass_mat,
                position=Vec3(0.0, 0.0, 1.6),
                scale=Vec3(4.2, 3.4, 0.08),
            )
        )

        env = self.add_child(WorldEnvironment(name="Env"))
        env.bloom_enabled = True

        self._refract_on = True
        self._toggle_cd = 0.0
        self.add_child(Text2D(text="SCREEN-SPACE REFRACTION", position=(10, 8), font_scale=1.5))
        self._status = self.add_child(Text2D(text="Refraction: ON", position=(10, 40), font_scale=1.2))
        self._hint = self.add_child(
            Text2D(text="SPACE/CLICK:Toggle  A/D:Orbit  W/S:Zoom", position=(10, HEIGHT - 30), font_scale=1.0)
        )

    def on_ready(self):
        InputMap.add_action("cam_left", [Key.A, Key.LEFT])
        InputMap.add_action("cam_right", [Key.D, Key.RIGHT])
        InputMap.add_action("cam_fwd", [Key.W, Key.UP])
        InputMap.add_action("cam_back", [Key.S, Key.DOWN])
        # Mouse/touch fallback: touch arrives as MouseButton.LEFT on web.
        InputMap.add_action("toggle_refract", [Key.SPACE, MouseButton.LEFT])

    def _update_camera(self):
        rad = math.radians(self._cam_angle)
        x = math.cos(rad) * self._cam_dist
        z = math.sin(rad) * self._cam_dist
        self.camera.position = Vec3(x, self._cam_height, z)
        self.camera.look_at(Vec3(0, 0, 0))

    def on_fixed_update(self, dt: float):
        speed = 45.0
        if Input.is_action_pressed("cam_left"):
            self._cam_angle += speed * dt
        if Input.is_action_pressed("cam_right"):
            self._cam_angle -= speed * dt
        if Input.is_action_pressed("cam_fwd"):
            self._cam_dist = max(4.0, self._cam_dist - 8 * dt)
        if Input.is_action_pressed("cam_back"):
            self._cam_dist = min(14.0, self._cam_dist + 8 * dt)
        self._update_camera()

    def on_update(self, dt: float):
        # Keep the controls hint pinned to the live window's bottom edge.
        self._hint.position = (10, self.app.height - 30)
        self._toggle_cd = max(0.0, self._toggle_cd - dt)
        if Input.is_action_just_pressed("toggle_refract") and self._toggle_cd <= 0:
            self._refract_on = not self._refract_on
            self._toggle_cd = 0.3
            # Flipping needs_scene_colour on the live material sets/clears the
            # screen-read bit, so the next frame records with/without the split.
            self._glass_mat.needs_scene_colour = self._refract_on
        self._status.text = f"Refraction: {'ON' if self._refract_on else 'OFF'}"


def _selftest() -> bool:
    """Headless self-check: the screen-read split renders a non-blank frame and
    the refraction actually bends pixels vs the same scene with it off."""
    import numpy as np

    from simvx.graphics.testing import assert_not_blank, save_png

    def render(screen_read: bool):
        scene = RefractionProbe()
        scene._glass_mat.needs_scene_colour = screen_read
        app = App(width=WIDTH // 2, height=HEIGHT // 2, title="refraction", visible=False)
        frame = app.run_headless(scene, frames=6, capture_frames=[5])[-1]
        return np.asarray(frame)[..., :3].astype(np.int32)

    on = render(True)
    off = render(False)
    assert_not_blank(on)
    save_png("/tmp/refraction_probe_test.png", on.astype("uint8"))
    diff = np.abs(on - off).sum(-1)
    changed = float((diff > 4).mean())
    print(f"refraction changed-pixel fraction = {changed:.3f} (max diff {int(np.abs(on - off).max())})")
    print("screenshot: /tmp/refraction_probe_test.png")
    return changed > 0.02


def main():
    scene = RefractionProbe()
    app = App(title="SimVX Refraction Probe", width=WIDTH, height=HEIGHT, physics_fps=60)
    app.run(scene)


if __name__ == "__main__":
    if "--test" in sys.argv:
        sys.exit(0 if _selftest() else 1)
    main()