unlit3dx_examples/
interactive.rs

1//! The interactive run: the whole scene, stepped inline and driven by input.
2//!
3//! The windowed mode runs the same scene across threads, paced by the display.
4//! This run keeps every world stepped on one thread and advances it one frame
5//! at a time, so a caller reproduces exactly what it drove: the wasm test that
6//! clicks buttons and reads frames back, or a native one that does the same
7//! without a process boundary. The GPU is injected, the way the headless run
8//! takes one, so a test asks for a headless device and a windowed caller
9//! builds its own from the surface.
10
11use std::sync::Arc;
12
13use unlit_ecs::{Entity, World, bundle};
14use unlit_wgpu::resources::ResHandle;
15use unlit3d::frame::{Frame, dispatch_frame};
16use unlit3d::input::{InputEvent, InputWriter, dispatch_input};
17use unlit3d::prelude::*;
18use unlit3dx::prelude::*;
19
20use crate::headless::{AUDIO_CHANNELS, AUDIO_SAMPLE_RATE};
21use crate::scene::{
22    AudioSetup, Ball, HEADLESS_HZ, PanelLayout, PanelSlots, SceneHandles, SceneStepper,
23    SceneTracks, Track, WINDOW_SIZE, mount_scene_content, setup_render_world, spawn_scene,
24};
25
26/// An interactive scene stepping inline: one frame per call, no pacing of its own.
27pub struct Interactive {
28    worlds: SceneHandles,
29    render: Entity,
30    renderer: Entity,
31    /// The entity the render sub-world is spawned under.
32    pub render_world: Entity,
33    /// The texture the offscreen pass draws into. Its resource id is what a
34    /// caller reads frames back through.
35    pub target: ResHandle<wgpu::Texture>,
36    stepper: SceneStepper,
37    frame: Frame,
38    samples: Vec<f32>,
39    recording: Option<unlit3dx::record::RecordingHandle>,
40    device: wgpu::Device,
41    queue: wgpu::Queue,
42}
43
44impl Interactive {
45    /// Step `frames` frames: dispatch the queued input, advance every world and
46    /// pull the mix the frames would have played.
47    ///
48    /// Nothing is drawn here, so a run that only reads the state back rather
49    /// than the picture costs no draw calls. [`Interactive::draw`] draws the
50    /// frame where the scene then stands, which is the same frame a step would
51    /// have drawn: a draw does not advance the scene, and what it does write is
52    /// only what a frame publishes for its readers, which the next frame
53    /// replaces anyway.
54    pub fn step(&mut self, host: &World, frames: u32) {
55        for _ in 0..frames {
56            dispatch_input(host);
57            self.stepper.step(host, &self.worlds, 1.0 / HEADLESS_HZ);
58            let frames = self.stepper.audio_frames(1.0 / HEADLESS_HZ);
59            if frames > 0 {
60                self.samples.clear();
61                self.samples.resize(frames * AUDIO_CHANNELS, 0.0);
62                let filled = with_stepped_world(host, self.worlds.audio.entity, |audio| {
63                    dispatch_audio_fill(
64                        audio,
65                        &mut self.samples,
66                        AUDIO_SAMPLE_RATE,
67                        AUDIO_CHANNELS as u16,
68                    );
69                });
70                assert!(filled.is_ok(), "the audio world is a stepped world");
71            }
72            let mut nested = host
73                .root_scope()
74                .query_one::<&mut World>(self.render)
75                .expect("the render sub-world exists");
76            let world = nested.get();
77            drain_sub_world(world);
78            dispatch_frame(world, &mut self.frame);
79            self.frame.index = self.frame.index.wrapping_add(1);
80        }
81    }
82
83    /// The frame the next step draws.
84    pub fn frame(&self) -> u32 {
85        self.frame.index
86    }
87
88    /// The handles the scene's worlds were spawned under.
89    pub fn worlds_handle(&self) -> &SceneHandles {
90        &self.worlds
91    }
92
93    /// The panel buttons the last drawn frame laid out, keyed by label.
94    ///
95    /// The layout lives on the gameplay world, refreshed on every tick.
96    pub fn panel_buttons(
97        host: &World,
98        worlds: &SceneHandles,
99    ) -> Vec<(&'static str, unlit3d::ui::egui::Rect)> {
100        with_stepped_world(host, worlds.gameplay.entity, |world| {
101            world
102                .root_scope()
103                .query::<&PanelLayout>()
104                .iter()
105                .next()
106                .map(|layout| {
107                    layout
108                        .buttons
109                        .iter()
110                        .map(|(label, rect)| (*label, *rect))
111                        .collect()
112                })
113                .unwrap_or_default()
114        })
115        .expect("the gameplay world is a stepped world")
116    }
117
118    /// Draw the frame where the scene stands and read its pixels back.
119    ///
120    /// The readback blocks until the copy lands. Drawing is what publishes the
121    /// target the panels lay out against, so nothing is laid out until one
122    /// frame has been drawn.
123    pub fn render_frame(&self, host: &World) -> Vec<u8> {
124        self.draw(host);
125        let texture = {
126            let mut nested = host
127                .root_scope()
128                .query_one::<&World>(self.render)
129                .expect("the render sub-world exists");
130            let world = nested.get();
131            let context = world
132                .root_scope()
133                .query_one::<&Renderer>(self.renderer)
134                .expect("the renderer is a resource entity")
135                .get()
136                .context();
137            let mut graph = context
138                .try_graph(&world.root_scope())
139                .expect("the frame's resource graph exists");
140            graph
141                .get()
142                .get(&self.target)
143                .cloned()
144                .expect("the target texture is a resource node")
145        };
146        unlit_wgpu::readback::readback_texture(&self.device, &self.queue, &texture)
147    }
148
149    /// Draw the frame where the scene stands, into the offscreen texture.
150    ///
151    /// The render sub-world catches up on the jobs queued for it first, so the
152    /// frame draws the scene as the steps left it rather than as the jobs
153    /// found it.
154    pub fn draw(&self, host: &World) {
155        let mut nested = host
156            .root_scope()
157            .query_one::<&mut World>(self.render)
158            .expect("the render sub-world exists");
159        let world = nested.get();
160        drain_sub_world(world);
161        let _ = world
162            .root_scope()
163            .query_one::<&mut Renderer>(self.renderer)
164            .map(|mut renderer| renderer.get().render(&*world));
165    }
166
167    /// The samples the last step pulled from the audio world.
168    pub fn samples(&self) -> &[f32] {
169        &self.samples
170    }
171
172    /// Click at `position`: move the pointer there, then press and release
173    /// before the next frame, the sequence the MCP input tool sends.
174    pub fn click(&mut self, host: &World, position: [f32; 2]) {
175        let button = |pressed| {
176            InputEvent::Mouse(unlit3d::input::MouseEvent::Button {
177                position,
178                button: unlit3d::input::MouseButton::Primary,
179                pressed,
180                modifiers: Default::default(),
181            })
182        };
183        host.root_scope()
184            .query::<&InputWriter>()
185            .iter()
186            .next()
187            .expect("the interactive run carries an input writer")
188            .push_all([
189                InputEvent::Mouse(unlit3d::input::MouseEvent::Moved { position }),
190                button(true),
191                button(false),
192            ]);
193    }
194
195    /// Start recording the mix the audio world fills.
196    ///
197    /// The recording lives on the audio world, so its fills are counted even
198    /// though the world is stepped from here.
199    pub fn start_recording(&mut self, host: &World, max_seconds: f32) {
200        let mut handle = None;
201        with_stepped_world_mut(host, self.worlds.audio.entity, |audio| {
202            handle = Some(unlit3dx::record::start_recording(audio, max_seconds));
203        })
204        .expect("the audio world is a stepped world");
205        self.recording = handle;
206    }
207
208    /// Stop the recording and take the samples it captured.
209    pub fn stop_recording(&mut self, host: &World) -> Option<Recording> {
210        let recording = self.recording.take()?;
211        let mut audio = None;
212        with_stepped_world_mut(host, self.worlds.audio.entity, |world| {
213            audio = recording.stop(world);
214        })
215        .expect("the audio world is a stepped world");
216        audio.map(Recording::from)
217    }
218}
219
220/// A finished recording: the format it captured at, and the samples it held.
221///
222/// The recorder answers [`unlit3dx::record::RecordedAudio`]; this type carries
223/// the same samples across the module boundary that hides the crate's internals.
224pub struct Recording {
225    /// The rate the mix was pulled at, and its channel count.
226    pub format: AudioFormat,
227    /// The interleaved samples, in the format's channel order.
228    pub samples: Vec<f32>,
229}
230
231impl From<unlit3dx::record::RecordedAudio> for Recording {
232    fn from(audio: unlit3dx::record::RecordedAudio) -> Self {
233        Self {
234            format: audio.format,
235            samples: audio.samples,
236        }
237    }
238}
239
240/// The scene state a caller asserts on: the balls and the sequence.
241#[derive(Clone, Copy)]
242pub struct SceneStatus {
243    /// How many balls the gameplay world holds.
244    pub balls: usize,
245    /// Whether the MIDI sequence is paused, and where it has got to.
246    pub sequence: Option<(bool, f64, bool)>,
247}
248
249/// Report the scene state: the balls the gameplay world holds and where its
250/// MIDI sequence stands.
251pub fn scene_status(host: &World, worlds: &SceneHandles) -> SceneStatus {
252    with_stepped_world(host, worlds.gameplay.entity, |world| {
253        let scope = world.root_scope();
254        let balls = scope.query::<&Ball>().iter().count();
255        let sequence = scope
256            .query::<&SceneTracks>()
257            .iter()
258            .next()
259            .and_then(|tracks| {
260                scope
261                    .query_one::<&Track>(tracks.sequence)
262                    .ok()
263                    .map(|mut track| {
264                        let track = track.get();
265                        (track.midi.paused, track.midi.time, track.midi.finished)
266                    })
267            });
268        SceneStatus { balls, sequence }
269    })
270    .expect("the gameplay world is a stepped world")
271}
272
273/// The settings one interactive run builds with.
274pub struct InteractiveConfig {
275    /// The GPU the scene renders with.
276    pub device: wgpu::Device,
277    /// The queue the scene's commands submit through.
278    pub queue: wgpu::Queue,
279    /// The limits and features the device was requested with.
280    pub capabilities: DeviceCapabilities,
281    /// The soundfont the synthesizer plays, as raw bytes; an empty slice
282    /// silences the audio world the way the headless run's does.
283    pub soundfont: Arc<[u8]>,
284}
285
286/// Build the interactive scene: an input writer on the host, the scene's
287/// worlds, an offscreen render target and the stepping interactive.
288///
289/// The host owns everything; the caller steps it through the returned
290/// [`Interactive`] and draws nothing to any window.
291pub fn run(config: InteractiveConfig) -> (World, InputWriter, Interactive) {
292    let mut host = World::new();
293    let input = InputWriter::new();
294    host.root_scope_mut()
295        .spawn(bundle![input.clone(), input.subscribe()]);
296    let render = spawn_sub_world(&mut host, "render", 64);
297    let panels = PanelSlots::default();
298    let audio_setup = if config.soundfont.is_empty() {
299        AudioSetup::Silent
300    } else {
301        AudioSetup::Pulled {
302            soundfont: config.soundfont,
303            autoplay_background: false,
304        }
305    };
306    let worlds = spawn_scene::<Stepped>(
307        &mut host,
308        render.clone(),
309        audio_setup,
310        panels.clone(),
311        &input,
312        false,
313    );
314    let (renderer, pass) = setup_render_world(
315        &mut host,
316        render.entity,
317        &config.device,
318        &config.queue,
319        config.capabilities,
320    );
321    mount_scene_content(&render, pass, Some(panels));
322    let target = {
323        let mut nested = host
324            .root_scope()
325            .query_one::<&mut World>(render.entity)
326            .expect("the render sub-world exists");
327        let world = nested.get();
328        let context = world
329            .root_scope()
330            .query_one::<&Renderer>(renderer.0)
331            .expect("the renderer is a resource entity")
332            .get()
333            .context();
334        PersistentTarget::bind(
335            world,
336            context,
337            pass.0,
338            &config.device,
339            WINDOW_SIZE,
340            TargetConfig::default(),
341        )
342        .expect("the offscreen target binds")
343    };
344    let interactive = Interactive {
345        worlds,
346        render: render.entity,
347        renderer: renderer.0,
348        render_world: render.entity,
349        target: target.texture,
350        stepper: SceneStepper::new(PHYSICS_HZ, Some(AUDIO_SAMPLE_RATE)),
351        frame: Frame {
352            delta_time: (1.0 / HEADLESS_HZ) as f32,
353            index: 0,
354            size: WINDOW_SIZE,
355        },
356        samples: Vec::new(),
357        recording: None,
358        device: config.device,
359        queue: config.queue,
360    };
361    // One frame is drawn before the caller takes over: a drawn frame is what
362    // publishes the target the panels lay out against, so a run that never
363    // draws lays out no panels and has no buttons to click.
364    interactive.draw(&host);
365    (host, input, interactive)
366}
unlit3dx_examples/interactive.rs

unlit3dx_examples/
interactive.rs
