diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index c3c44bec553..08739f441a3 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -7,15 +7,16 @@ use cap_project::{ TimelineFrameMapping, TimelineSegment, XY, }; use cap_rendering::{ - PrecomputedCursorTimeline, ProjectRecordingsMeta, ProjectUniforms, RecordingSegmentDecoders, - RenderVideoConstants, RendererLayers, SegmentVideoPaths, SharedWgpuDevice, Video, - ZoomTransformTimeline, get_duration, spring_mass_damper::SpringMassDamperSimulationConfig, + CursorEventOrder, PrecomputedCursorTimeline, ProjectRecordingsMeta, ProjectUniforms, + RecordingSegmentDecoders, RenderVideoConstants, RendererLayers, SegmentVideoPaths, + SharedWgpuDevice, Video, ZoomTransformTimeline, get_duration, + spring_mass_damper::SpringMassDamperSimulationConfig, }; use std::{ collections::HashMap, path::{Path, PathBuf}, sync::{ - Arc, + Arc, Weak, atomic::{AtomicBool, AtomicU64, Ordering}, }, time::{Duration, Instant}, @@ -70,6 +71,12 @@ pub(super) fn cursor_timeline_settings(cursor: &cap_project::CursorConfiguration #[derive(Default)] pub(super) struct PreviewCursorCache { entries: Vec, + event_orders: Vec, +} + +struct PreviewCursorOrderEntry { + cursor: Weak, + order: CursorEventOrder, } struct PreviewCursorCacheEntry { @@ -80,6 +87,29 @@ struct PreviewCursorCacheEntry { } impl PreviewCursorCache { + pub(super) fn event_order(&mut self, cursor: &Arc) -> CursorEventOrder { + if let Some(index) = self + .event_orders + .iter() + .position(|entry| std::ptr::eq(entry.cursor.as_ptr(), Arc::as_ptr(cursor))) + { + let entry = self.event_orders.remove(index); + let order = entry.order; + self.event_orders.push(entry); + return order; + } + + if self.event_orders.len() == PREVIEW_CURSOR_CACHE_CAPACITY { + self.event_orders.remove(0); + } + let order = CursorEventOrder::new(cursor); + self.event_orders.push(PreviewCursorOrderEntry { + cursor: Arc::downgrade(cursor), + order, + }); + order + } + pub(super) fn get( &mut self, recording_clip: u32, @@ -1254,7 +1284,7 @@ impl EditorInstance { cursor_timeline, ) } else { - ProjectUniforms::new( + ProjectUniforms::new_with_cursor_event_order( &self.render_constants, &project, frame_number, @@ -1264,6 +1294,7 @@ impl EditorInstance { &outgoing_frames, total_duration, outgoing_zoom, + cursor_cache.event_order(&outgoing_media.cursor), ) }; Some(( @@ -1315,7 +1346,7 @@ impl EditorInstance { cursor_timeline, ) } else { - ProjectUniforms::new( + ProjectUniforms::new_with_cursor_event_order( &self.render_constants, &project, frame_number, @@ -1325,6 +1356,7 @@ impl EditorInstance { &segment_frames, total_duration, zoom_timeline, + cursor_cache.event_order(&segment_medias.cursor), ) }; @@ -2175,6 +2207,20 @@ mod tests { assert!(cache.entries.is_empty()); assert!(smoothed_weak.upgrade().is_none()); assert_eq!(Arc::strong_count(&cursor), 1); + let order = cache.event_order(&cursor); + assert_eq!(order, CursorEventOrder::new(&cursor)); + assert_ne!(order, CursorEventOrder::default()); + assert_eq!(cache.event_order(&cursor), order); + assert_eq!(cache.event_orders.len(), 1); + assert_eq!(Arc::strong_count(&cursor), 1); + + let mut reversed = CursorEvents::clone(&cursor); + reversed.moves.reverse(); + reversed.clicks.reverse(); + let replaced = Arc::new(reversed); + assert_eq!(cache.event_order(&replaced), CursorEventOrder::default()); + assert_eq!(cache.event_orders.len(), 2); + assert_eq!(Arc::strong_count(&replaced), 1); project.cursor.tension += 1.0; project.cursor.click_spring = Some(cap_project::ClickSpringConfig { diff --git a/crates/editor/src/preparing_preview.rs b/crates/editor/src/preparing_preview.rs index f13bc342481..f44b5a14d53 100644 --- a/crates/editor/src/preparing_preview.rs +++ b/crates/editor/src/preparing_preview.rs @@ -672,7 +672,7 @@ async fn run_preview( cursor_timeline, ) } else { - ProjectUniforms::new( + ProjectUniforms::new_with_cursor_event_order( &constants, &project, request.frame_number, @@ -682,6 +682,7 @@ async fn run_preview( &frames, duration, zoom, + cursor_cache.event_order(&cursor), ) }; let layout = uniforms.frame_layout(); diff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rs index 5696c2d47b7..ac6723ed55e 100644 --- a/crates/rendering/src/cursor_interpolation.rs +++ b/crates/rendering/src/cursor_interpolation.rs @@ -343,17 +343,39 @@ struct CursorTimelineVariant { } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct CursorEventOrder { - pub moves: bool, - pub clicks: bool, +pub struct CursorEventOrder { + pub(crate) moves: bool, + pub(crate) clicks: bool, +} + +impl CursorEventOrder { + pub fn new(cursor: &CursorEvents) -> Self { + Self { + moves: cursor.moves.is_sorted_by(|a, b| a.time_ms <= b.time_ms), + clicks: cursor + .clicks + .iter() + .try_fold(f64::NEG_INFINITY, |previous, click| { + (previous <= click.time_ms).then_some(click.time_ms) + }) + .is_some(), + } + } + + pub(crate) fn interpolate_raw( + self, + cursor: &CursorEvents, + time_secs: f32, + ) -> Option { + interpolate_raw_cursor(cursor, (time_secs * 1000.0) as f64, self.moves) + } } pub struct PrecomputedCursorTimeline { timeline: Vec, cursor_ids: Vec, raw_cursor: CursorEvents, - raw_moves_are_ordered: bool, - clicks_are_ordered: bool, + event_order: CursorEventOrder, has_smoothing: bool, variants: Mutex>, } @@ -364,21 +386,13 @@ impl PrecomputedCursorTimeline { smoothing: Option, click_spring: Option, ) -> Self { - let raw_moves_are_ordered = cursor.moves.is_sorted_by(|a, b| a.time_ms <= b.time_ms); - let clicks_are_ordered = cursor - .clicks - .iter() - .try_fold(f64::NEG_INFINITY, |previous, click| { - (previous <= click.time_ms).then_some(click.time_ms) - }) - .is_some(); + let event_order = CursorEventOrder::new(cursor); if cursor.moves.is_empty() || smoothing.is_none() { return Self { timeline: vec![], cursor_ids: Vec::new(), raw_cursor: cursor.clone(), - raw_moves_are_ordered, - clicks_are_ordered, + event_order, has_smoothing: false, variants: Mutex::new(Vec::new()), }; @@ -398,8 +412,7 @@ impl PrecomputedCursorTimeline { timeline: built.events, cursor_ids: built.cursor_ids, raw_cursor: CursorEvents::default(), - raw_moves_are_ordered, - clicks_are_ordered, + event_order, has_smoothing: true, variants: Mutex::new(Vec::new()), } @@ -454,10 +467,7 @@ impl PrecomputedCursorTimeline { } pub(crate) fn event_order(&self) -> CursorEventOrder { - CursorEventOrder { - moves: self.raw_moves_are_ordered, - clicks: self.clicks_are_ordered, - } + self.event_order } pub fn interpolate(&self, time_secs: f32) -> Option { @@ -465,7 +475,7 @@ impl PrecomputedCursorTimeline { if self.has_smoothing { interpolate_timeline(&self.timeline, &self.cursor_ids, time_ms) } else { - interpolate_raw_cursor(&self.raw_cursor, time_ms, self.raw_moves_are_ordered) + interpolate_raw_cursor(&self.raw_cursor, time_ms, self.event_order.moves) } } @@ -475,11 +485,7 @@ impl PrecomputedCursorTimeline { cursor: &CursorEvents, time_secs: f32, ) -> Option { - interpolate_raw_cursor( - cursor, - (time_secs * 1000.0) as f64, - self.raw_moves_are_ordered, - ) + self.event_order.interpolate_raw(cursor, time_secs) } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index da01e6295eb..b4db5d9d14d 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -8,8 +8,7 @@ use cap_project::{ use composite_frame::{ColorGradeUniformParams, CompositeVideoFrameUniforms}; use core::f64; use cursor_interpolation::{ - CursorEventOrder, InterpolatedCursorPosition, interpolate_cursor, - interpolate_cursor_with_click_spring, + InterpolatedCursorPosition, interpolate_cursor, interpolate_cursor_with_click_spring, }; use decoder::{AsyncVideoDecoderHandle, spawn_decoder}; #[cfg(target_os = "macos")] @@ -216,7 +215,7 @@ mod font_family_tests { } use camera3d::{Camera3DFrame, interpolate_camera3d}; -pub use cursor_interpolation::PrecomputedCursorTimeline; +pub use cursor_interpolation::{CursorEventOrder, PrecomputedCursorTimeline}; use mask::interpolate_masks; use scene::*; use takeover::InterpolatedTakeover; @@ -4193,6 +4192,33 @@ impl ProjectUniforms { segment_frames: &DecodedSegmentFrames, total_duration: f64, zoom_timeline: &ZoomTransformTimeline, + ) -> Self { + Self::new_with_cursor_event_order( + constants, + project, + frame_number, + fps, + resolution_base, + cursor_events, + segment_frames, + total_duration, + zoom_timeline, + CursorEventOrder::default(), + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn new_with_cursor_event_order( + constants: &RenderVideoConstants, + project: &Arc, + frame_number: u32, + fps: u32, + resolution_base: XY, + cursor_events: &CursorEvents, + segment_frames: &DecodedSegmentFrames, + total_duration: f64, + zoom_timeline: &ZoomTransformTimeline, + cursor_event_order: CursorEventOrder, ) -> Self { let cursor = project.cursor_at(f64::from(frame_number) / f64::from(fps.max(1))); let cursor_smoothing = (!cursor.raw).then_some(SpringMassDamperSimulationConfig { @@ -4210,7 +4236,7 @@ impl ProjectUniforms { Some(cfg), Some(click_spring_cfg), ), - None => interpolate_cursor(cursor_events, time, None), + None => cursor_event_order.interpolate_raw(cursor_events, time), } }; @@ -4225,7 +4251,7 @@ impl ProjectUniforms { total_duration, zoom_timeline, &cursor_interp_fn, - CursorEventOrder::default(), + cursor_event_order, ) } @@ -9820,6 +9846,308 @@ mod project_uniforms_tests { } } + #[tokio::test] + async fn raw_cursor_event_order_preserves_preview_uniforms_and_pixels() { + let mut constants = cursor_test_constants().await; + let size = XY::new(320, 180); + constants.options.screen_size = size; + let click = |time_ms: f64, down: bool| cap_project::CursorClickEvent { + active_modifiers: Vec::new(), + cursor_num: 0, + cursor_id: "primary".to_string(), + time_ms, + down, + }; + let mut seed = 0x9e37_79b9_u32; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + let mut random_time = 0.0; + let random_moves = (0..400) + .map(|index| { + if next() % 4 != 0 { + random_time += f64::from(next() % 40); + } + let mut event = cursor_move( + random_time, + f64::from(next() % 1000) / 1000.0, + f64::from(next() % 1000) / 1000.0, + ); + event.cursor_id = (index / 37 % 3).to_string(); + event + }) + .collect::>(); + let mut click_time = 0.0; + let random_clicks = (0..40) + .map(|index| { + click_time += f64::from(next() % 400); + click(click_time, index % 2 == 0) + }) + .collect::>(); + let mut unordered_moves = random_moves.clone(); + unordered_moves.swap(120, 121); + unordered_moves.swap(10, 300); + let mut unordered_clicks = random_clicks.clone(); + unordered_clicks.swap(3, 30); + let cases = [ + CursorEvents { + moves: random_moves.clone(), + clicks: random_clicks.clone(), + }, + CursorEvents { + moves: unordered_moves, + clicks: unordered_clicks, + }, + CursorEvents { + moves: vec![ + cursor_move(0.0, 0.1, 0.5), + cursor_move(0.0, 0.2, 0.5), + cursor_move(250.0, 0.3, 0.4), + cursor_move(250.0, 0.35, 0.45), + cursor_move(800.0, 0.8, 0.6), + ], + clicks: vec![click(250.0, true), click(250.0, false), click(260.0, true)], + }, + CursorEvents { + moves: Vec::new(), + clicks: random_clicks, + }, + ]; + let mut project = ProjectConfiguration::default(); + project.cursor.raw = true; + project.cursor.set_cursor_type(CursorType::Circle); + project.cursor.motion_blur = 0.6; + project.cursor.rotation_amount = 1.0; + project.cursor.hide_when_idle = true; + project.cursor.hide_when_idle_delay = 0.5; + project.cursor.stop_movement_in_last_seconds = Some(0.5); + project.cursor.ripple.enabled = true; + project.timeline = Some( + serde_json::from_value(serde_json::json!({ + "segments": [{ "start": 0.0, "end": 10.0, "timescale": 1.0 }], + "zoomSegments": [] + })) + .unwrap(), + ); + let project = Arc::new(project); + let source = DecodedFrame::new(vec![160; 320 * 180 * 4], size.x, size.y); + let mut linear_renderer = FrameRenderer::new(&constants); + let mut ordered_renderer = FrameRenderer::new(&constants); + let mut linear_layers = RendererLayers::new(&constants.device, &constants.queue); + let mut ordered_layers = RendererLayers::new(&constants.device, &constants.queue); + let mut ordered_cases = 0; + for events in &cases { + let order = CursorEventOrder::new(events); + if order != CursorEventOrder::default() { + ordered_cases += 1; + } + let zoom = ZoomTransformTimeline::from_project(&project, events, 10.0, size); + let event_frames = events + .moves + .iter() + .map(|event| event.time_ms) + .chain(events.clicks.iter().map(|click| click.time_ms)) + .step_by(7) + .map(|time_ms| (time_ms * 60.0 / 1000.0) as u32); + for frame_number in [0, 1, 15, 16, 48, 120, 300, 599] + .into_iter() + .chain(event_frames) + { + let time = frame_number as f32 / 60.0; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: Some(source.clone()), + camera_frame: None, + segment_time: time, + recording_time: time, + segment_has_camera: false, + }; + let linear = ProjectUniforms::new( + &constants, + &project, + frame_number, + 60, + size, + events, + &frames, + 10.0, + &zoom, + ); + let ordered = ProjectUniforms::new_with_cursor_event_order( + &constants, + &project, + frame_number, + 60, + size, + events, + &frames, + 10.0, + &zoom, + order, + ); + assert_eq!(ordered.cursor_event_order, order); + let mut neutral = ordered.clone(); + neutral.cursor_event_order = CursorEventOrder::default(); + assert_eq!( + format!("{linear:?}"), + format!("{neutral:?}"), + "frame={frame_number}: uniforms changed" + ); + let linear = linear_renderer + .render_immediate(frames.clone(), linear, events, true, &mut linear_layers) + .await + .unwrap(); + let ordered = ordered_renderer + .render_immediate(frames, ordered, events, true, &mut ordered_layers) + .await + .unwrap(); + assert!( + linear.data == ordered.data, + "frame={frame_number}: rendered cursor pixels changed" + ); + } + } + assert_eq!(ordered_cases, 3); + } + + #[tokio::test] + #[ignore] + async fn benchmark_raw_cursor_preview_uniforms() { + use std::hint::black_box; + + let mut constants = cursor_test_constants().await; + let size = XY::new(320, 180); + constants.options.screen_size = size; + let source = DecodedFrame::new(vec![160; 320 * 180 * 4], size.x, size.y); + for duration_secs in [60, 600, 3600] { + let mut project = ProjectConfiguration::default(); + project.cursor.raw = true; + project.cursor.ripple.enabled = true; + project.timeline = Some( + serde_json::from_value(serde_json::json!({ + "segments": [{ "start": 0.0, "end": duration_secs, "timescale": 1.0 }], + "zoomSegments": [] + })) + .unwrap(), + ); + let project = Arc::new(project); + let events = CursorEvents { + moves: (0..=duration_secs * 60) + .map(|index| { + cursor_move( + index as f64 * 1000.0 / 60.0, + (index % 479) as f64 / 479.0, + (index % 283) as f64 / 283.0, + ) + }) + .collect(), + clicks: (0..duration_secs * 2) + .map(|index| cap_project::CursorClickEvent { + active_modifiers: Vec::new(), + cursor_num: 0, + cursor_id: "primary".to_string(), + time_ms: index as f64 * 500.0, + down: index % 2 == 0, + }) + .collect(), + }; + let started = Instant::now(); + let order = CursorEventOrder::new(black_box(&events)); + let order_us = started.elapsed().as_secs_f64() * 1e6; + assert_eq!( + order, + CursorEventOrder { + moves: true, + clicks: true + } + ); + let zoom = + ZoomTransformTimeline::from_project(&project, &events, duration_secs as f64, size); + for workload in ["middle", "end"] { + let mut baseline = Vec::new(); + let mut optimized = Vec::new(); + for index in 0..544u32 { + let frame_number = match workload { + "middle" => duration_secs * 30, + _ => (duration_secs - 3) * 60, + } + index % 120; + let time = frame_number as f32 / 60.0; + let frames = DecodedSegmentFrames { + screen_size: size, + screen_frame: Some(source.clone()), + camera_frame: None, + segment_time: time, + recording_time: time, + segment_has_camera: false, + }; + let old = || { + ProjectUniforms::new( + black_box(&constants), + black_box(&project), + black_box(frame_number), + 60, + size, + black_box(&events), + black_box(&frames), + duration_secs as f64, + black_box(&zoom), + ) + }; + let new = || { + ProjectUniforms::new_with_cursor_event_order( + black_box(&constants), + black_box(&project), + black_box(frame_number), + 60, + size, + black_box(&events), + black_box(&frames), + duration_secs as f64, + black_box(&zoom), + black_box(order), + ) + }; + let measure = |build: &dyn Fn() -> ProjectUniforms| { + let started = Instant::now(); + let uniforms = build(); + (started.elapsed().as_secs_f64() * 1e6, uniforms) + }; + let ((old_us, old_uniforms), (new_us, new_uniforms)) = if index % 2 == 0 { + (measure(&old), measure(&new)) + } else { + let new = measure(&new); + (measure(&old), new) + }; + let mut neutral = new_uniforms; + neutral.cursor_event_order = CursorEventOrder::default(); + assert!(old_uniforms.interpolated_cursor.is_some()); + assert_eq!(format!("{old_uniforms:?}"), format!("{neutral:?}")); + if index >= 32 { + baseline.push(old_us); + optimized.push(new_us); + } + } + baseline.sort_by(f64::total_cmp); + optimized.sort_by(f64::total_cmp); + let p50 = baseline.len() / 2; + let p95 = baseline.len() * 95 / 100; + println!( + "raw_preview_uniforms duration_secs={duration_secs} workload={workload} moves={} clicks={} frames={} order_setup_us={order_us:.3} baseline_p50_us={:.3} baseline_p95_us={:.3} optimized_p50_us={:.3} optimized_p95_us={:.3}", + events.moves.len(), + events.clicks.len(), + baseline.len(), + baseline[p50], + baseline[p95], + optimized[p50], + optimized[p95], + ); + } + } + } + #[tokio::test] async fn ordered_cursor_event_lookups_preserve_rendered_pixels() { let mut constants = cursor_test_constants().await;