diff --git a/crates/cursor-capture/src/position.rs b/crates/cursor-capture/src/position.rs index d26d10c09d8..915eb42557c 100644 --- a/crates/cursor-capture/src/position.rs +++ b/crates/cursor-capture/src/position.rs @@ -136,7 +136,7 @@ impl std::fmt::Debug for RelativeCursorPosition { } } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] /// Needs to be logical coordinates on macOS and physical on Windows /// This type is opqaue on purpose as the logical/physical invariants need to hold pub struct CursorCropBounds { diff --git a/crates/recording/examples/cursor-poll-benchmark.rs b/crates/recording/examples/cursor-poll-benchmark.rs index b7a6acd703d..7786b727d2f 100644 --- a/crates/recording/examples/cursor-poll-benchmark.rs +++ b/crates/recording/examples/cursor-poll-benchmark.rs @@ -52,7 +52,6 @@ fn main() -> anyhow::Result<()> { CursorCaptureTarget { crop_bounds, display, - #[cfg(target_os = "linux")] window: None, }, dir.path().join("cursors"), diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index 45a8ee07225..a3a71e0ac44 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -51,10 +51,61 @@ pub struct IncrementalCaptureOutputs { pub struct CursorCaptureTarget { pub crop_bounds: CursorCropBounds, pub display: scap_targets::Display, - #[cfg(target_os = "linux")] pub window: Option, } +#[cfg(target_os = "macos")] +fn macos_window_crop( + window: &scap_targets::WindowId, + display: &scap_targets::Display, +) -> Option { + use scap_targets::bounds::{LogicalBounds, LogicalPosition}; + + let window_bounds = scap_targets::Window::from_id_direct(window)? + .raw_handle() + .logical_bounds()?; + let display_position = display.raw_handle().logical_position(); + // The video reads the window inside its rounded corners; the cursor + // is placed against the same trimmed frame. + Some(CursorCropBounds::new_macos( + crate::sources::screen_capture::trim_window_corners(LogicalBounds::new( + LogicalPosition::new( + window_bounds.position().x() - display_position.x(), + window_bounds.position().y() - display_position.y(), + ), + window_bounds.size(), + )), + )) +} + +#[cfg(windows)] +fn windows_window_crop( + window: &scap_targets::WindowId, + display: &scap_targets::Display, +) -> Option { + let window_bounds = scap_targets::Window::from_id_direct(window)? + .raw_handle() + .physical_bounds()?; + let display_bounds = display.raw_handle().physical_bounds()?; + Some(window_crop_on_display(window_bounds, display_bounds)) +} + +#[cfg(windows)] +fn window_crop_on_display( + window_bounds: scap_targets::bounds::PhysicalBounds, + display_bounds: scap_targets::bounds::PhysicalBounds, +) -> CursorCropBounds { + use scap_targets::bounds::{PhysicalBounds, PhysicalPosition}; + + CursorCropBounds::new_windows(PhysicalBounds::new( + PhysicalPosition::new( + window_bounds.position().x() - display_bounds.position().x(), + window_bounds.position().y() - display_bounds.position().y(), + ), + window_bounds.size(), + )) +} + #[cfg(target_os = "linux")] struct X11WindowCursor { connection: x11rb::rust_connection::RustConnection, @@ -108,6 +159,32 @@ fn normalized_window_cursor(x: i16, y: i16, width: u16, height: u16) -> Option<( }) } +#[cfg(all(test, windows))] +mod windows_window_cursor_tests { + use super::window_crop_on_display; + use scap_targets::bounds::{PhysicalBounds, PhysicalPosition, PhysicalSize}; + + fn bounds(x: f64, y: f64, width: f64, height: f64) -> PhysicalBounds { + PhysicalBounds::new( + PhysicalPosition::new(x, y), + PhysicalSize::new(width, height), + ) + } + + #[test] + fn the_cursor_crop_follows_the_window_when_it_moves() { + let display = bounds(-1920.0, 0.0, 1920.0, 1080.0); + let started = window_crop_on_display(bounds(-1720.0, 200.0, 800.0, 600.0), display); + let moved = window_crop_on_display(bounds(-1320.0, 350.0, 800.0, 600.0), display); + + assert_ne!(started, moved); + assert_eq!( + moved, + cap_cursor_capture::CursorCropBounds::new_windows(bounds(600.0, 350.0, 800.0, 600.0)) + ); + } +} + #[cfg(all(test, target_os = "linux"))] mod window_cursor_tests { use super::normalized_window_cursor; @@ -634,6 +711,8 @@ pub fn spawn_cursor_recorder( }); #[cfg(target_os = "linux")] let mut last_window_position = None; + #[cfg(any(windows, target_os = "macos"))] + let mut last_window_crop: Option = None; let device_state = DeviceState::new(); // Reading the held keys asks the OS about every key it knows (two // CoreGraphics queries per key on macOS, 256 GetAsyncKeyState calls @@ -714,6 +793,29 @@ pub fn spawn_cursor_recorder( { last_window_position = window_position; } + // A window recording follows the window, so the cursor has to be + // placed against where the window is now, not where it started. + #[cfg(windows)] + let window_crop = target + .window + .as_ref() + .and_then(|window| windows_window_crop(window, &display)); + #[cfg(target_os = "macos")] + let window_crop = target + .window + .as_ref() + .and_then(|window| macos_window_crop(window, &display)); + #[cfg(any(windows, target_os = "macos"))] + let position_changed = + position_changed || (window_crop.is_some() && window_crop != last_window_crop); + #[cfg(any(windows, target_os = "macos"))] + { + if window_crop.is_some() { + last_window_crop = window_crop; + } + } + #[cfg(any(windows, target_os = "macos"))] + let crop_bounds = last_window_crop.unwrap_or(crop_bounds); // The cursor image only labels this tick's move and click events, // and reading it builds a full TIFF of every representation diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index 84a51044053..6c0102a11fc 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -859,6 +859,7 @@ impl Actor { pub struct ActorBuilder { output_path: PathBuf, capture_target: ScreenCaptureTarget, + window_only: bool, system_audio: bool, mic_feed: Option>, camera_feed: Option>, @@ -884,6 +885,7 @@ impl ActorBuilder { Self { output_path: output, capture_target, + window_only: false, system_audio: false, mic_feed: None, camera_feed: None, @@ -922,6 +924,15 @@ impl ActorBuilder { self } + /// Record only the target window, following it, on macOS and Windows. + /// Ignored while a camera is attached there: Instant has no camera + /// compositing on those platforms, so the camera bubble only reaches + /// the video because the screen capture records it. + pub fn with_window_only(mut self, window_only: bool) -> Self { + self.window_only = window_only; + self + } + pub fn with_mic_feed(mut self, mic_feed: Arc) -> Self { self.mic_feed = Some(mic_feed); self @@ -986,6 +997,7 @@ impl ActorBuilder { self.output_path, RecordingBaseInputs { capture_target: self.capture_target, + window_only: self.window_only, capture_system_audio: self.system_audio, mic_feed: self.mic_feed, camera_feed: self.camera_feed, @@ -1337,6 +1349,13 @@ async fn build_instant_recording_actor( #[cfg(not(target_os = "macos"))] let max_capture_size = None; + #[cfg(any(target_os = "macos", windows))] + let capture_window = instant_capture_window( + &inputs.capture_target, + inputs.window_only, + inputs.camera_feed.is_some(), + ); + let screen_source = ScreenCaptureConfig::::init( display, crop_bounds, @@ -1345,6 +1364,8 @@ async fn build_instant_recording_actor( max_capture_size, timestamps.system_time(), inputs.capture_system_audio, + #[cfg(any(target_os = "macos", windows))] + capture_window, #[cfg(target_os = "linux")] crate::sources::screen_capture::LinuxCaptureSource::from_target( &inputs.capture_target, @@ -1567,10 +1588,37 @@ fn clamp_size(input: (u32, u32), max: (u32, u32)) -> (u32, u32) { } } +// Instant mode on macOS and Windows has no camera compositing: the camera +// preview only reaches the video because the display capture records it, so a +// window-only capture would drop it. +#[cfg(any(target_os = "macos", windows))] +fn instant_capture_window( + target: &ScreenCaptureTarget, + window_only: bool, + camera_attached: bool, +) -> Option { + crate::window_only_target(target, window_only).filter(|_| !camera_attached) +} + #[cfg(test)] mod tests { use super::*; + #[cfg(any(target_os = "macos", windows))] + #[test] + fn instant_captures_the_window_only_when_asked_and_without_a_camera() { + let id: scap_targets::WindowId = "247".parse().unwrap(); + let window = ScreenCaptureTarget::Window { id: id.clone() }; + + assert_eq!(instant_capture_window(&window, true, false), Some(id)); + assert_eq!(instant_capture_window(&window, true, true), None); + assert_eq!(instant_capture_window(&window, false, false), None); + assert_eq!( + instant_capture_window(&ScreenCaptureTarget::CameraOnly, true, false), + None + ); + } + #[cfg(target_os = "linux")] #[test] fn linux_presentation_builder_preserves_explicit_config_and_cli_default() { diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index a7dcbbcc4c9..23e2f98dd81 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -79,9 +79,21 @@ pub struct RecordingOptions { pub mode: RecordingMode, } +/// The window a capture should record by itself: only for a window target +/// with window-only capture on. +pub fn window_only_target( + target: &ScreenCaptureTarget, + window_only: bool, +) -> Option { + target.window().filter(|_| window_only) +} + #[derive(Clone)] pub struct RecordingBaseInputs { pub capture_target: ScreenCaptureTarget, + /// For a window target on macOS and Windows: record only that window, + /// following it, instead of the region of the screen it starts in. + pub window_only: bool, pub capture_system_audio: bool, pub mic_feed: Option>, pub camera_feed: Option>, diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 66972d65eb4..e5dfa90f14f 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -28,6 +28,82 @@ use tracing::{debug, error, info, warn}; const MAX_CAPTURE_RESTARTS: u32 = 3; const RESTART_DELAY: Duration = Duration::from_secs(2); +// The display filter is what hides excluded windows, so an excluded selection must stay on +// the display-crop path instead of being captured on its own. +fn capturable_window<'a>( + config: &'a Config, + excluded_windows: &[WindowId], + capture_phase: &'static str, +) -> Option<&'a WindowId> { + let window_id = config.window.as_ref()?; + + if excluded_windows.contains(window_id) { + warn!( + %window_id, + capture_phase, + "Capture window is excluded from recordings, using display crop with exclusions" + ); + return None; + } + + Some(window_id) +} + +fn window_content_filter( + window_id: &WindowId, + content: &sc::ShareableContent, + capture_phase: &'static str, +) -> Option> { + let Ok(native_id) = window_id.to_string().parse::() else { + warn!(%window_id, capture_phase, "Capture window has an invalid native id"); + return None; + }; + + let windows = content.windows(); + let Some(window) = windows.iter().find(|window| window.id() == native_id) else { + warn!( + %window_id, + capture_phase, + "Capture window missing from ScreenCaptureKit shareable content" + ); + return None; + }; + + Some(sc::ContentFilter::with_desktop_independent_window(window)) +} + +/// A single window at the display's full pixel density, filling the frame. +/// Left to themselves, window captures pick their resolution automatically, +/// which can read a Retina window at 1x and upscale it, and do not scale the +/// window to the output, leaving it in a corner of a larger frame. +fn configure_window_capture(settings: &mut sc::StreamCfg) { + settings.set_scales_to_fit(true); + // Both arrived in macOS 14; Cap runs from 12.3. + if cidre::api::macos_available("14.0") { + unsafe { + settings.set_capture_resolution(sc::CaptureResolution::Best); + settings.set_preserves_aspect_ratio(true); + } + } +} + +/// What the stream reads. A display capture crops to `crop_bounds`; a window +/// capture reads the window itself, inside its rounded corners, where +/// `crop_bounds` is the window already trimmed by `WINDOW_CORNER_TRIM`. +fn source_rect(captures_window: bool, crop_bounds: super::CropBounds) -> cg::Rect { + let (x, y) = if captures_window { + (super::WINDOW_CORNER_TRIM, super::WINDOW_CORNER_TRIM) + } else { + (crop_bounds.position().x(), crop_bounds.position().y()) + }; + cg::Rect::new( + x, + y, + crop_bounds.size().width(), + crop_bounds.size().height(), + ) +} + fn excluded_shareable_windows( excluded_windows: &[WindowId], content: &sc::ShareableContent, @@ -178,6 +254,10 @@ pub enum SourceError { NoDisplay(DisplayId), #[error("AsContentFilter")] AsContentFilter, + /// A window-only capture whose window is gone. A display crop would + /// record whatever now covers the window's rectangle. + #[error("WindowGone: Id '{0}'")] + WindowGone(WindowId), } pub struct VideoFrame { @@ -219,16 +299,32 @@ impl ScreenCaptureConfig { let display = Display::from_id(&self.config.display) .ok_or_else(|| SourceError::NoDisplay(self.config.display.clone()))?; - let excluded_sc_windows = - excluded_shareable_windows(&self.excluded_windows, &self.shareable_content, "initial"); + let window_filter = capturable_window(&self.config, &self.excluded_windows, "initial") + .map(|id| { + window_content_filter(id, &self.shareable_content, "initial") + .ok_or_else(|| SourceError::WindowGone(id.clone())) + }) + .transpose()?; + let captures_window = window_filter.is_some(); + + let content_filter = match window_filter { + Some(filter) => filter, + None => { + let excluded_sc_windows = excluded_shareable_windows( + &self.excluded_windows, + &self.shareable_content, + "initial", + ); - let content_filter = display - .raw_handle() - .as_content_filter_excluding_windows( - self.shareable_content.clone(), - excluded_sc_windows, - ) - .ok_or(SourceError::AsContentFilter)?; + display + .raw_handle() + .as_content_filter_excluding_windows( + self.shareable_content.clone(), + excluded_sc_windows, + ) + .ok_or(SourceError::AsContentFilter)? + } + }; debug!("SCK content filter: {:?}", content_filter); @@ -256,15 +352,13 @@ impl ScreenCaptureConfig { settings.set_pixel_format(cv::PixelFormat::_420V); settings.set_color_space_name(cg::color_space::names::srgb()); + if captures_window { + configure_window_capture(&mut settings); + } if let Some(crop_bounds) = self.config.crop_bounds { debug!("crop bounds: {:?}", crop_bounds); - settings.set_src_rect(cg::Rect::new( - crop_bounds.position().x(), - crop_bounds.position().y(), - crop_bounds.size().width(), - crop_bounds.size().height(), - )); + settings.set_src_rect(source_rect(captures_window, crop_bounds)); } cap_fail::fail_err!( "macos::ScreenCaptureActor::new", @@ -948,13 +1042,27 @@ async fn rebuild_capturer(params: &CapturerRebuildParams) -> anyhow::Result filter, + None => { + let excluded_sc_windows = + excluded_shareable_windows(¶ms.excluded_windows, &shareable_content, "restart"); + + display + .raw_handle() + .as_content_filter_excluding_windows(shareable_content, excluded_sc_windows) + .ok_or_else(|| anyhow!("Failed to create content filter during restart"))? + } + }; let size = PhysicalSize::new( params.video_info.width as f64, @@ -976,14 +1084,12 @@ async fn rebuild_capturer(params: &CapturerRebuildParams) -> anyhow::Result WindowId { + id.parse().unwrap() + } + + fn window_config(window: Option) -> Config { + Config { + display: "1".parse().unwrap(), + crop_bounds: None, + fps: 60, + show_cursor: true, + window, + } + } + + #[test] + fn initial_capture_uses_window_filter_for_non_excluded_window() { + let config = window_config(Some(window_id("247"))); + let excluded = vec![window_id("12"), window_id("99")]; + + assert_eq!( + capturable_window(&config, &excluded, "initial"), + Some(&window_id("247")) + ); + } + + #[test] + fn initial_capture_keeps_excluded_window_on_display_path() { + let config = window_config(Some(window_id("247"))); + let excluded = vec![window_id("12"), window_id("247")]; + + assert_eq!(capturable_window(&config, &excluded, "initial"), None); + } + + #[test] + fn restart_capture_keeps_excluded_window_on_display_path() { + let config = window_config(Some(window_id("247"))); + let excluded = vec![window_id("247")]; + let (restart_config, restart_excluded) = (config.clone(), excluded.clone()); + + assert_eq!( + capturable_window(&restart_config, &restart_excluded, "restart"), + None + ); + } + + #[test] + fn restart_capture_uses_window_filter_for_non_excluded_window() { + let config = window_config(Some(window_id("247"))); + let excluded = vec![window_id("12")]; + let (restart_config, restart_excluded) = (config.clone(), excluded.clone()); + + assert_eq!( + capturable_window(&restart_config, &restart_excluded, "restart"), + Some(&window_id("247")) + ); + } + + #[test] + fn non_window_targets_never_use_window_filter() { + let config = window_config(None); + + assert_eq!(capturable_window(&config, &[], "initial"), None); + assert_eq!(capturable_window(&config, &[], "restart"), None); + } +} diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index 688cee38b62..6e4595edf95 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -382,11 +382,51 @@ pub struct Config { show_cursor: bool, #[cfg(target_os = "linux")] linux_source: LinuxCaptureSource, + /// Capture this window itself rather than a crop of its display. + #[cfg(any(target_os = "macos", windows))] + window: Option, } #[cfg(target_os = "macos")] pub type CropBounds = LogicalBounds; +/// macOS hands a single window over with its rounded corners transparent, +/// which the encoder fills with black. Window-only capture reads this many +/// points in from every edge: a corner's curve clears 0.29 of its radius in, +/// and this covers radii up to 27 pt. +#[cfg(target_os = "macos")] +pub const WINDOW_CORNER_TRIM: f64 = 8.0; + +/// `bounds` shrunk to whole, even pixel dimensions at `scale`. The encoder +/// needs even frames; a region an odd number of pixels across would be read +/// whole and squeezed a pixel smaller, resampling, and so softening, every +/// line of text. Reading the even region instead copies pixels one to one. +#[cfg(target_os = "macos")] +pub fn even_pixel_crop(bounds: CropBounds, scale: f64) -> CropBounds { + let size = bounds.size(); + let pixels = |points: f64| ensure_even((points * scale).floor() as u32) as f64 / scale; + LogicalBounds::new( + bounds.position(), + LogicalSize::new(pixels(size.width()), pixels(size.height())), + ) +} + +/// `bounds` with [`WINDOW_CORNER_TRIM`] taken off every side, or as it was +/// when the window is too small to trim. +#[cfg(target_os = "macos")] +pub fn trim_window_corners(bounds: CropBounds) -> CropBounds { + let size = bounds.size(); + let trim = WINDOW_CORNER_TRIM; + if size.width() <= trim * 4. || size.height() <= trim * 4. { + return bounds; + } + let position = bounds.position(); + LogicalBounds::new( + LogicalPosition::new(position.x() + trim, position.y() + trim), + LogicalSize::new(size.width() - 2. * trim, size.height() - 2. * trim), + ) +} + #[cfg(windows)] pub type CropBounds = PhysicalBounds; @@ -473,6 +513,7 @@ impl ScreenCaptureConfig { max_capture_size: Option<(u32, u32)>, start_time: SystemTime, system_audio: bool, + #[cfg(any(target_os = "macos", windows))] window: Option, #[cfg(target_os = "linux")] linux_source: LinuxCaptureSource, #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, #[cfg(target_os = "macos")] shareable_content: SendableShareableContent, @@ -483,21 +524,36 @@ impl ScreenCaptureConfig { let target_refresh = validated_refresh_rate(display.refresh_rate()); let fps = std::cmp::max(1, std::cmp::min(max_fps, target_refresh)); + // The capture reads the window inside its corners (see + // `WINDOW_CORNER_TRIM`), so the output and the cursor crop match that. + #[cfg(target_os = "macos")] + let crop_bounds = match window { + Some(_) => crop_bounds.map(trim_window_corners), + None => crop_bounds, + }; + // Read exactly as many pixels as the frame holds (see `even_pixel_crop`). + #[cfg(target_os = "macos")] + let crop_bounds = crop_bounds + .map(|bounds| even_pixel_crop(bounds, display.raw_handle().scale().unwrap_or(1.0))); + let native_output_size: PhysicalSize = { #[cfg(target_os = "macos")] { crop_bounds.and_then(|b| { let logical_size = b.size(); let scale = display.raw_handle().scale()?; - let width = ensure_even((logical_size.width() * scale) as u32) as f64; - let height = ensure_even((logical_size.height() * scale) as u32) as f64; + let width = ensure_even((logical_size.width() * scale).round() as u32) as f64; + let height = ensure_even((logical_size.height() * scale).round() as u32) as f64; Some(PhysicalSize::new(width, height)) }) } #[cfg(target_os = "windows")] { - crop_bounds.map(|b| b.size().map(|v| (v / 2.0).floor() * 2.0)) + window + .as_ref() + .and_then(window_capture_size) + .or_else(|| crop_bounds.map(|b| b.size().map(|v| (v / 2.0).floor() * 2.0))) } #[cfg(target_os = "linux")] @@ -535,6 +591,8 @@ impl ScreenCaptureConfig { show_cursor, #[cfg(target_os = "linux")] linux_source, + #[cfg(any(target_os = "macos", windows))] + window, }, video_info: VideoInfo::from_raw_ffmpeg( TCaptureFormat::pixel_format(), @@ -1107,3 +1165,46 @@ mod tests { assert_eq!(crop.size().height(), 450.0); } } + +#[cfg(all(test, target_os = "macos"))] +mod window_corner_tests { + use super::*; + + #[test] + fn window_capture_reads_inside_the_rounded_corners() { + let window = LogicalBounds::new( + LogicalPosition::new(100., 50.), + LogicalSize::new(920., 436.), + ); + let trimmed = trim_window_corners(window); + assert_eq!(trimmed.position().x(), 108.); + assert_eq!(trimmed.position().y(), 58.); + assert_eq!(trimmed.size().width(), 904.); + assert_eq!(trimmed.size().height(), 420.); + } + + #[test] + fn crops_read_whole_even_pixels() { + let odd = LogicalBounds::new(LogicalPosition::new(8., 8.), LogicalSize::new(1690., 753.)); + let even = even_pixel_crop(odd, 1.0); + assert_eq!(even.size().width(), 1690.); + assert_eq!(even.size().height(), 752.); + assert_eq!(even.position().x(), 8.); + // At 2x, a half point is a whole pixel: 753.5 pt is 1507 px, read as 1506. + let retina = even_pixel_crop( + LogicalBounds::new(LogicalPosition::new(0., 0.), LogicalSize::new(845., 753.5)), + 2.0, + ); + assert_eq!(retina.size().width() * 2.0, 1690.); + assert_eq!(retina.size().height() * 2.0, 1506.); + } + + #[test] + fn a_tiny_window_is_left_whole() { + let window = LogicalBounds::new(LogicalPosition::new(0., 0.), LogicalSize::new(30., 200.)); + let kept = trim_window_corners(window); + assert_eq!(kept.position().x(), 0.); + assert_eq!(kept.size().width(), 30.); + assert_eq!(kept.size().height(), 200.); + } +} diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 36d5d448386..bf99efb640f 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1,6 +1,8 @@ use crate::{ AudioFrame, SetupCtx, output_pipeline, - screen_capture::{ScreenCaptureConfig, ScreenCaptureFormat, cadence::FrameCadenceGate}, + screen_capture::{ + CropBounds, ScreenCaptureConfig, ScreenCaptureFormat, cadence::FrameCadenceGate, + }, }; use ::windows::Win32::Graphics::Direct3D11::{ D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, @@ -16,7 +18,7 @@ use futures::{ channel::{mpsc, oneshot}, }; use scap_ffmpeg::*; -use scap_targets::{Display, DisplayId}; +use scap_targets::{Display, DisplayId, Window, WindowId, bounds::PhysicalSize}; use std::{ sync::{ Arc, Mutex, @@ -351,13 +353,51 @@ impl output_pipeline::VideoFrame for VideoFrame { } } +/// The size a window recording is encoded at: the window's own capture item, +/// floored to even. WGC delivers the window's frames at the item's size, which +/// differs from the window rect (invisible resize borders, odd dimensions), so +/// sizing the output from the rect sends every frame through the CPU scaler. +/// The odd row or column is trimmed on the GPU (`Settings::even_dimensions`), +/// which keeps the frames exactly this size. +pub fn window_capture_size(window: &WindowId) -> Option { + let size = Window::from_id_direct(window)? + .raw_handle() + .try_as_capture_item() + .ok()? + .Size() + .ok()?; + even_capture_size(size.Width, size.Height) +} + +fn even_capture_size(width: i32, height: i32) -> Option { + let width = (width.max(0) as u32) & !1; + let height = (height.max(0) as u32) & !1; + (width > 0 && height > 0).then(|| PhysicalSize::new(width as f64, height as f64)) +} + +/// A window target captures the window's own GraphicsCaptureItem, so the crop +/// that carves the window out of the display is neither needed nor correct: +/// WGC already hands back just that window, and the crop would additionally +/// pin the recording to wherever the window started out. +fn crop_for_target( + window: Option<&WindowId>, + crop_bounds: Option, +) -> Option { + match window { + Some(_) => None, + None => crop_bounds, + } +} + impl ScreenCaptureConfig { pub async fn to_sources( &self, ) -> anyhow::Result<(VideoSourceConfig, Option)> { + let crop_bounds = crop_for_target(self.config.window.as_ref(), self.config.crop_bounds); + let mut settings = scap_direct3d::Settings { pixel_format: Direct3DCapture::PIXEL_FORMAT, - crop: self.config.crop_bounds.map(|b| { + crop: crop_bounds.map(|b| { let position = b.position(); let size = b.size().map(|v| (v / 2.0).floor() * 2.0); @@ -375,6 +415,11 @@ impl ScreenCaptureConfig { back: 1, } }), + // Matches `window_capture_size`, which the output is sized from. + even_dimensions: self.config.window.is_some(), + // Only a window target follows its item's size; display and area + // recordings keep the pool they were created with. + follow_item_size: self.config.window.is_some(), ..Default::default() }; @@ -393,12 +438,13 @@ impl ScreenCaptureConfig { settings.fps = Some(self.config.fps); - // Store the display ID instead of GraphicsCaptureItem to avoid COM threading issues + // Store the target's ID instead of GraphicsCaptureItem to avoid COM threading issues // The GraphicsCaptureItem will be created on the capture thread Ok(( VideoSourceConfig { video_info: self.video_info, display_id: self.config.display.clone(), + window_id: self.config.window.clone(), settings, d3d_device: self.d3d_device.clone(), }, @@ -416,6 +462,7 @@ pub enum VideoSourceError { pub struct VideoSourceConfig { video_info: VideoInfo, display_id: DisplayId, + window_id: Option, settings: scap_direct3d::Settings, pub d3d_device: ID3D11Device, } @@ -487,6 +534,7 @@ struct CaptureClosureEvent { struct CreateCapturerParams<'a> { display_id: &'a DisplayId, + window_id: Option<&'a WindowId>, settings: &'a scap_direct3d::Settings, d3d_device: &'a ID3D11Device, video_tx: &'a mpsc::Sender, @@ -511,11 +559,21 @@ fn create_d3d_capturer( params: &CreateCapturerParams, error_tx: &mpsc::Sender, ) -> anyhow::Result { - let capture_item = Display::from_id(params.display_id) - .ok_or_else(|| anyhow!("Display not found for ID: {:?}", params.display_id))? - .raw_handle() - .try_as_capture_item() - .map_err(|e| anyhow!("Failed to create GraphicsCaptureItem: {}", e))?; + let capture_item = match params.window_id { + // Resolve the handle directly: `from_id` searches `list()`, which + // skips minimized windows, so a target minimized between selection + // and capture setup would fail to start despite still existing. + Some(window_id) => Window::from_id_direct(window_id) + .ok_or_else(|| anyhow!("Window not found for ID: {:?}", window_id))? + .raw_handle() + .try_as_capture_item() + .map_err(|e| anyhow!("Failed to create window GraphicsCaptureItem: {}", e))?, + None => Display::from_id(params.display_id) + .ok_or_else(|| anyhow!("Display not found for ID: {:?}", params.display_id))? + .raw_handle() + .try_as_capture_item() + .map_err(|e| anyhow!("Failed to create GraphicsCaptureItem: {}", e))?, + }; scap_direct3d::Capturer::new( capture_item, @@ -661,6 +719,7 @@ impl output_pipeline::VideoSource for VideoSource { VideoSourceConfig { video_info, display_id, + window_id, settings, d3d_device, }: Self::Config, @@ -723,6 +782,7 @@ impl output_pipeline::VideoSource for VideoSource { ($device:expr) => { CreateCapturerParams { display_id: &display_id, + window_id: window_id.as_ref(), settings: &settings, d3d_device: $device, video_tx: &video_tx, @@ -1598,3 +1658,40 @@ mod first_screen_frame_tests { assert!(error.to_string().contains("closed before its first frame")); } } + +#[cfg(test)] +mod capture_target_tests { + use super::*; + use scap_targets::bounds::{PhysicalPosition, PhysicalSize}; + + fn bounds() -> CropBounds { + CropBounds::new( + PhysicalPosition::new(120.0, 80.0), + PhysicalSize::new(640.0, 480.0), + ) + } + + #[test] + fn a_display_target_keeps_its_crop() { + let kept = crop_for_target(None, Some(bounds())).expect("crop kept"); + assert_eq!(kept.position().x(), 120.0); + assert_eq!(kept.size().width(), 640.0); + assert!(crop_for_target(None, None).is_none()); + } + + #[test] + fn a_window_recording_is_sized_to_its_capture_item_floored_to_even() { + let size = even_capture_size(801, 601).expect("size"); + assert_eq!((size.width(), size.height()), (800.0, 600.0)); + let size = even_capture_size(786, 593).expect("size"); + assert_eq!((size.width(), size.height()), (786.0, 592.0)); + assert!(even_capture_size(1, 600).is_none()); + assert!(even_capture_size(-4, 600).is_none()); + } + + #[test] + fn a_window_target_captures_the_window_rather_than_a_crop_of_the_display() { + let window: WindowId = "1234".parse().unwrap(); + assert!(crop_for_target(Some(&window), Some(bounds())).is_none()); + } +} diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 05f56a6de45..c868f574c63 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -2144,6 +2144,7 @@ pub struct ActorBuilder { lifetime: StudioLifetimeOwner, output_path: PathBuf, capture_target: screen_capture::ScreenCaptureTarget, + window_only: bool, system_audio: bool, mic_feed: Option>, camera_feed: Option>, @@ -2168,6 +2169,7 @@ impl ActorBuilder { }, output_path: output, capture_target, + window_only: false, system_audio: false, mic_feed: None, camera_feed: None, @@ -2188,6 +2190,12 @@ impl ActorBuilder { self } + /// Record only the target window, following it, on macOS and Windows. + pub fn with_window_only(mut self, window_only: bool) -> Self { + self.window_only = window_only; + self + } + /// Prime the capture pipeline ahead of the start cue: every source, encoder /// and muxer is live once `build` returns, but nothing is recorded until the /// gate is armed. See [`RecordingStartGate`]. @@ -2265,6 +2273,7 @@ impl ActorBuilder { self.output_path, RecordingBaseInputs { capture_target: self.capture_target, + window_only: self.window_only, capture_system_audio: self.system_audio, mic_feed: self.mic_feed, camera_feed: self.camera_feed, @@ -3413,6 +3422,8 @@ async fn create_segment_pipeline( max_capture_size, start_time.system_time(), base_inputs.capture_system_audio, + #[cfg(any(target_os = "macos", windows))] + crate::window_only_target(&capture_target, base_inputs.window_only), #[cfg(target_os = "linux")] sources::screen_capture::LinuxCaptureSource::from_target(&capture_target), #[cfg(windows)] @@ -3656,6 +3667,11 @@ async fn create_segment_pipeline( display: cursor_display, #[cfg(target_os = "linux")] window: base_inputs.capture_target.window(), + #[cfg(not(target_os = "linux"))] + window: crate::window_only_target( + &base_inputs.capture_target, + base_inputs.window_only, + ), }, cursors_dir.to_path_buf(), prev_cursors, @@ -3987,6 +4003,7 @@ mod tests { root.join("content/cursors"), RecordingBaseInputs { capture_target: screen_capture::ScreenCaptureTarget::CameraOnly, + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -4097,6 +4114,7 @@ mod tests { path.join("content/cursors"), RecordingBaseInputs { capture_target: target.clone(), + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -4195,6 +4213,7 @@ mod tests { path.join("content/cursors"), RecordingBaseInputs { capture_target: target.clone(), + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -4332,6 +4351,7 @@ mod tests { path.join("content/cursors"), RecordingBaseInputs { capture_target: target.clone(), + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -4576,6 +4596,7 @@ mod tests { path.join("content/cursors"), RecordingBaseInputs { capture_target: target.clone(), + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -6255,6 +6276,7 @@ mod tests { recording_dir.join("content/cursors"), RecordingBaseInputs { capture_target: screen_capture::ScreenCaptureTarget::CameraOnly, + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, @@ -6502,6 +6524,7 @@ mod windows_cancel_tests { PathBuf::new(), RecordingBaseInputs { capture_target: screen_capture::ScreenCaptureTarget::CameraOnly, + window_only: false, capture_system_audio: false, mic_feed: None, camera_feed: None, diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index dbe42566731..92e37f9d342 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -328,6 +328,77 @@ fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device } } +fn create_copy_texture( + d3d_device: &ID3D11Device, + width: u32, + height: u32, + pixel_format: PixelFormat, +) -> windows::core::Result { + let desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { d3d_device.CreateTexture2D(&desc, None, Some(&mut texture)) }?; + texture.ok_or_else(windows::core::Error::from_win32) +} + +/// Textures for frames trimmed down from their surface. A frame hands its +/// texture back when it is dropped, so a texture is only written again once +/// no queued frame still reads it. +#[derive(Default)] +struct TrimPool { + free: Mutex>, +} + +/// Enough for frames waiting on the encoder; more are dropped when returned. +const TRIM_POOL_SIZE: usize = 8; + +impl TrimPool { + fn take( + &self, + d3d_device: &ID3D11Device, + width: u32, + height: u32, + pixel_format: PixelFormat, + ) -> windows::core::Result { + let reused = { + let mut free = self + .free + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + free.retain(|(_, w, h)| (*w, *h) == (width, height)); + free.pop() + }; + match reused { + Some((texture, _, _)) => Ok(texture), + None => create_copy_texture(d3d_device, width, height, pixel_format), + } + } + + fn give(&self, texture: ID3D11Texture2D, width: u32, height: u32) { + let mut free = self + .free + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if free.len() < TRIM_POOL_SIZE { + free.push((texture, width, height)); + } + } +} + #[derive(Clone, Default, Debug)] pub struct Settings { pub is_border_required: Option, @@ -336,6 +407,14 @@ pub struct Settings { pub pixel_format: PixelFormat, pub crop: Option, pub fps: Option, + /// Trim an odd-sized frame to the even size below it on the GPU, so a + /// consumer that encodes at even dimensions receives frames it can take as + /// they are. Ignored when `crop` is set. + pub even_dimensions: bool, + /// Follow the capture item when it changes size: recreate the frame pool + /// at the new size, and drop frames a `crop` box no longer fits inside. + /// Off by default so display and area captures behave as they always have. + pub follow_item_size: bool, } impl Settings { @@ -506,14 +585,18 @@ impl Capturer { .map(|fps| ((fps as f32 / 30.0 * 2.0).ceil() as i32).clamp(2, 4)) .unwrap_or(2); + let item_size = item.Size().map_err(NewCapturerError::ItemSize)?; + let frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( &direct3d_device, settings.pixel_format.as_directx(), frame_pool_size, - item.Size().map_err(NewCapturerError::ItemSize)?, + item_size, ) .map_err(NewCapturerError::FramePool)?; + let pool_size = Arc::new(Mutex::new((item_size.Width, item_size.Height))); + let session = frame_pool .CreateCaptureSession(&item) .map_err(NewCapturerError::CaptureSession)?; @@ -539,34 +622,23 @@ impl Capturer { let crop_data = settings .crop .map(|crop| { - let desc = D3D11_TEXTURE2D_DESC { - Width: (crop.right - crop.left), - Height: (crop.bottom - crop.top), - MipLevels: 1, - ArraySize: 1, - Format: settings.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let mut texture = None; - unsafe { d3d_device.CreateTexture2D(&desc, None, Some(&mut texture)) } - .map_err(NewCapturerError::CropTexture)?; - - let texture = texture.ok_or_else(|| { - NewCapturerError::CropTexture(windows::core::Error::from_win32()) - })?; + let texture = create_copy_texture( + &d3d_device, + crop.right - crop.left, + crop.bottom - crop.top, + settings.pixel_format, + ) + .map_err(NewCapturerError::CropTexture)?; Ok::<_, NewCapturerError>((texture, crop)) }) .transpose()?; + // Only used by frames whose size differs from their surface (odd-sized, + // or smaller than the pool's surface), so a capture that matches its + // surface never pays for the copy. + let trim = Arc::new(TrimPool::default()); + let frame_arrived_token = frame_pool .FrameArrived( &TypedEventHandler::::new({ @@ -575,6 +647,10 @@ impl Capturer { let stop_flag = stop_flag.clone(); let staging_pool = staging_pool.clone(); let callback_activity = callback_activity.clone(); + let pixel_format = settings.pixel_format; + let pool_size = pool_size.clone(); + let even_dimensions = settings.even_dimensions && settings.crop.is_none(); + let follow_item_size = settings.follow_item_size; move |frame_pool, _| { let _activity_guard = callback_activity.enter(); @@ -591,11 +667,89 @@ impl Capturer { let size = frame.ContentSize()?; + // The item can change size mid-capture (a recorded + // window is resized, or the display's resolution + // changes). The pool keeps handing out surfaces at its + // creation size, so the frame's own ContentSize stops + // describing its texture and every later frame reads + // back as black. Recreate the pool for the new size and + // skip this frame; the next one arrives correctly sized. + if follow_item_size { + let mut pool_size = pool_size + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if (size.Width, size.Height) != *pool_size { + drop(frame); + // Windows.Graphics.Capture fails fast, on a + // thread nothing here can catch, once its device + // is removed. Do not hand it the removed device + // again; report the removal instead. + if let Err(error) = unsafe { d3d_device.GetDeviceRemovedReason() } { + tracing::error!( + %error, + "scap-direct3d: capture device removed, not recreating the frame pool" + ); + return Err(error); + } + // IDirect3DDevice is not Send, so it is rebuilt + // here from the capture device rather than held + // across the event handler. + let dxgi_device = d3d_device.cast::()?; + let direct3d_device = + unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi_device) }? + .cast::()?; + frame_pool.Recreate( + &direct3d_device, + pixel_format.as_directx(), + frame_pool_size, + size, + )?; + *pool_size = (size.Width, size.Height); + return Ok(()); + } + } + let surface = frame.Surface()?; let dxgi_interface = surface.cast::()?; let texture = unsafe { dxgi_interface.GetInterface::() }?; + // ContentSize is the item's size when the frame was + // captured, not the size of the surface it was written + // to: it runs ahead of the surface whenever the item + // grows before the pool is recreated, including frames + // the old pool had already queued when `Recreate` ran. + // Copies are bounded by the surface itself, because a + // source box past its edge is undefined behaviour that + // a hardware driver ignores but WARP answers by removing + // the device, and Windows.Graphics.Capture then fails + // fast on its next use of it. + let mut surface_desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { texture.GetDesc(&mut surface_desc) }; + let content_width = size.Width.max(0) as u32; + let content_height = size.Height.max(0) as u32; + + // A frame queued before the pool was recreated: its + // surface is the old size, and the next frame is not. + if follow_item_size + && (surface_desc.Width, surface_desc.Height) + != (content_width, content_height) + { + return Ok(()); + } + let frame = if let Some((cropped_texture, crop)) = crop_data.clone() { + // A shrunk item leaves the crop box hanging off the + // edge of the source texture, which copies nothing + // and would vend a stale surface as if it were live. + if crop.right > surface_desc.Width + || crop.bottom > surface_desc.Height + || (follow_item_size + && (crop.right > content_width + || crop.bottom > content_height)) + { + return Ok(()); + } + unsafe { d3d_context.CopySubresourceRegion( &cropped_texture, @@ -618,17 +772,74 @@ impl Capturer { d3d_context: d3d_context.clone(), d3d_device: d3d_device.clone(), staging_pool: staging_pool.clone(), + recycle: None, } } else { - Frame { - width: size.Width as u32, - height: size.Height as u32, - pixel_format: settings.pixel_format, - inner: frame, - texture, - d3d_context: d3d_context.clone(), - d3d_device: d3d_device.clone(), - staging_pool: staging_pool.clone(), + // The frame is the content bounded by its surface, + // floored to even when asked. A surface larger than + // the content (a window item whose pool surface + // includes the invisible resize borders) is copied + // down to it: handing the whole surface over under + // the content's size makes `as_buffer`'s + // `CopyResource` a copy between different sizes, + // which D3D11 drops, so the readback is all zeros. + let mut width = content_width.min(surface_desc.Width); + let mut height = content_height.min(surface_desc.Height); + if even_dimensions { + width &= !1; + height &= !1; + } + if width == 0 || height == 0 { + return Ok(()); + } + + if (width, height) == (surface_desc.Width, surface_desc.Height) { + Frame { + width, + height, + pixel_format: settings.pixel_format, + inner: frame, + texture, + d3d_context: d3d_context.clone(), + d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), + recycle: None, + } + } else { + let trimmed_texture = + trim.take(&d3d_device, width, height, pixel_format)?; + + unsafe { + d3d_context.CopySubresourceRegion( + &trimmed_texture, + 0, + 0, + 0, + 0, + &texture, + 0, + Some(&D3D11_BOX { + left: 0, + top: 0, + front: 0, + right: width, + bottom: height, + back: 1, + }), + ); + } + + Frame { + width, + height, + pixel_format: settings.pixel_format, + inner: frame, + texture: trimmed_texture, + d3d_context: d3d_context.clone(), + d3d_device: d3d_device.clone(), + staging_pool: staging_pool.clone(), + recycle: Some(trim.clone()), + } } }; @@ -728,6 +939,16 @@ pub struct Frame { d3d_device: ID3D11Device, d3d_context: ID3D11DeviceContext, staging_pool: Arc, + /// Where a trimmed frame's texture goes back to once the frame is done. + recycle: Option>, +} + +impl Drop for Frame { + fn drop(&mut self) { + if let Some(pool) = self.recycle.take() { + pool.give(self.texture.clone(), self.width, self.height); + } + } } impl std::fmt::Debug for Frame { diff --git a/crates/scap-direct3d/tests/window_capture.rs b/crates/scap-direct3d/tests/window_capture.rs new file mode 100644 index 00000000000..9e2875a6d3d --- /dev/null +++ b/crates/scap-direct3d/tests/window_capture.rs @@ -0,0 +1,774 @@ +#![cfg(target_os = "windows")] + +//! Window capture regressions for CapSoftware/Cap#2151. +//! +//! The fixture process owns the windows because `Window::list()` skips the +//! calling process, the same arrangement `scap-targets`' window discovery test +//! uses. + +use scap_direct3d::{Capturer, PixelFormat, Settings}; +use scap_targets::{Window, WindowId}; +use std::{ + io::{BufRead, BufReader, Write}, + process::{Child, Command, Stdio}, + sync::{Arc, Mutex, Once, mpsc}, + time::{Duration, Instant}, +}; +use windows::{ + Win32::{ + Foundation::{COLORREF, HMODULE, HWND, LPARAM, LRESULT, RECT, WPARAM}, + Graphics::{ + Direct3D::D3D_DRIVER_TYPE_WARP, + Direct3D11::{ + D3D11_BOX, D3D11_CREATE_DEVICE_FLAG, D3D11_SDK_VERSION, D3D11CreateDevice, + ID3D11Device, + }, + Gdi::{CreateSolidBrush, InvalidateRect}, + }, + UI::{ + HiDpi::{PROCESS_PER_MONITOR_DPI_AWARE, SetProcessDpiAwareness}, + WindowsAndMessaging::{ + CS_HREDRAW, CS_VREDRAW, CreateWindowExW, DefWindowProcW, DispatchMessageW, + HWND_TOPMOST, MSG, PM_REMOVE, PeekMessageW, RegisterClassW, SPI_GETWORKAREA, + SWP_NOACTIVATE, SWP_SHOWWINDOW, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS, SetWindowPos, + SystemParametersInfoW, TranslateMessage, WINDOW_EX_STYLE, WNDCLASSW, + WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_OVERLAPPEDWINDOW, WS_POPUP, WS_VISIBLE, + }, + }, + }, + core::{PCWSTR, w}, +}; + +/// Where the fixture puts the target window: a fixed fraction of the primary +/// display's work area, so the windows, the crop taken around them and every +/// resize stay on screen from a 1024x768 runner up to a 4K desktop. +#[derive(Clone, Copy, Debug)] +struct Layout { + x: i32, + y: i32, + width: i32, + height: i32, +} + +impl Layout { + /// Physical pixels once the process is per-monitor DPI aware, which the + /// fixture and every test arrange before calling this. + fn of_primary_work_area() -> Self { + let mut area = RECT::default(); + unsafe { + SystemParametersInfoW( + SPI_GETWORKAREA, + 0, + Some(&mut area as *mut RECT as *mut _), + SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0), + ) + } + .expect("primary display work area"); + let (width, height) = (area.right - area.left, area.bottom - area.top); + Self { + x: area.left + width / 8, + y: area.top + height / 8, + width: width / 2, + height: height / 2, + } + } + + /// Inside the target's client area, clear of its title bar. + fn overlay(&self) -> (i32, i32, i32, i32) { + ( + self.x + self.width / 4, + self.y + self.height / 4, + self.width / 3, + self.height / 3, + ) + } + + /// Still inside the work area: x + 3w/2 and y + 5h/4 end at 7/8 and 6/8. + fn grown(&self) -> (i32, i32) { + (self.width * 3 / 2, self.height * 5 / 4) + } +} + +struct FixtureProcess(Child); + +impl Drop for FixtureProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// Process-wide and only settable once, so both tests here share one call. +fn match_cli_dpi_awareness() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + unsafe { SetProcessDpiAwareness(PROCESS_PER_MONITOR_DPI_AWARE) } + .expect("match CLI per-monitor DPI awareness"); + }); +} + +unsafe extern "system" fn window_proc( + window: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + unsafe { DefWindowProcW(window, message, wparam, lparam) } +} + +fn register_class(class: PCWSTR, colour: u32) { + let class = WNDCLASSW { + style: CS_HREDRAW | CS_VREDRAW, + lpfnWndProc: Some(window_proc), + hbrBackground: unsafe { CreateSolidBrush(COLORREF(colour)) }, + lpszClassName: class, + ..Default::default() + }; + assert!(unsafe { RegisterClassW(&class) } != 0, "register class"); +} + +/// Runs in the fixture process: a blue window with a red always-on-top window +/// covering part of it, resized once `resize` is read on stdin. +#[test] +fn window_capture_fixture_process() { + if std::env::var_os("CAP_WINDOW_CAPTURE_FIXTURE").is_none() { + return; + } + + match_cli_dpi_awareness(); + let layout = Layout::of_primary_work_area(); + let (overlay_x, overlay_y, overlay_w, overlay_h) = layout.overlay(); + + // COLORREF is 0x00BBGGRR. + register_class(w!("Cap2151Target"), 0x00FF_0000); + register_class(w!("Cap2151Overlay"), 0x0000_00FF); + + let target = unsafe { + CreateWindowExW( + WINDOW_EX_STYLE(0), + w!("Cap2151Target"), + w!("Cap2151 target"), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + layout.x, + layout.y, + layout.width, + layout.height, + None, + None, + None, + None, + ) + } + .expect("create target window"); + let overlay = unsafe { + CreateWindowExW( + WS_EX_TOPMOST | WS_EX_TOOLWINDOW, + w!("Cap2151Overlay"), + w!("Cap2151 overlay"), + WS_POPUP | WS_VISIBLE, + overlay_x, + overlay_y, + overlay_w, + overlay_h, + None, + None, + None, + None, + ) + } + .expect("create overlay window"); + unsafe { + SetWindowPos( + overlay, + Some(HWND_TOPMOST), + overlay_x, + overlay_y, + overlay_w, + overlay_h, + SWP_SHOWWINDOW | SWP_NOACTIVATE, + ) + } + .expect("raise overlay"); + + println!("WINDOW_CAPTURE_FIXTURE {}", target.0 as u64); + std::io::stdout().flush().unwrap(); + + let (command_tx, command_rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(std::io::stdin()).lines() { + let Ok(line) = line else { break }; + if command_tx.send(line).is_err() { + break; + } + } + }); + + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(60) { + match command_rx.try_recv() { + Ok(line) if line.trim() == "resize" => unsafe { + let (width, height) = layout.grown(); + SetWindowPos( + target, + None, + layout.x, + layout.y, + width, + height, + SWP_SHOWWINDOW | SWP_NOACTIVATE, + ) + .expect("resize target"); + }, + // `size W H` resizes in place; `at X Y W H` also moves the window. + Ok(line) if line.trim().starts_with("size ") || line.trim().starts_with("at ") => { + let mut words = line.trim().split(' '); + let moves = words.next() == Some("at"); + let mut dims = words.map(|v| v.parse::().expect("fixture command number")); + let (x, y) = if moves { + (dims.next().unwrap(), dims.next().unwrap()) + } else { + (layout.x, layout.y) + }; + let (width, height) = (dims.next().unwrap(), dims.next().unwrap()); + unsafe { + SetWindowPos( + target, + None, + x, + y, + width, + height, + SWP_SHOWWINDOW | SWP_NOACTIVATE, + ) + .expect("size target"); + } + } + Ok(line) if line.trim() == "stop" => return, + Ok(_) | Err(mpsc::TryRecvError::Empty) => {} + Err(mpsc::TryRecvError::Disconnected) => return, + } + + // WGC only delivers a frame when the content changes, so keep the + // window repainting for the duration of the capture. + let _ = unsafe { InvalidateRect(Some(target), None, true) }; + let mut message = MSG::default(); + while unsafe { PeekMessageW(&mut message, None, 0, 0, PM_REMOVE) }.as_bool() { + let _ = unsafe { TranslateMessage(&message) }; + unsafe { DispatchMessageW(&message) }; + } + std::thread::sleep(Duration::from_millis(16)); + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Sample { + width: u32, + height: u32, + /// The frame's ContentSize, which on some systems (Server 2022) is + /// smaller than `item.Size()` because the latter counts the window's + /// invisible resize borders. + content: (u32, u32), + red: usize, + blue: usize, +} + +fn capture_samples( + item: windows::Graphics::Capture::GraphicsCaptureItem, + crop: Option, + duration: Duration, + during: impl FnOnce(), +) -> Vec { + capture_samples_with( + item, + Settings { + crop, + ..Default::default() + }, + duration, + during, + ) +} + +/// The WARP software rasterizer, which Cap falls back to without a usable GPU. +/// It removes its device over copies a hardware driver lets pass, so it is +/// what catches an out-of-bounds copy on a machine that has a GPU. +fn warp_device() -> ID3D11Device { + let mut device = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_WARP, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG::default(), + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + } + .expect("create WARP device"); + device.expect("WARP device") +} + +/// `CAP_SCAP_TEST_WARP=1` runs every capture here on WARP; otherwise the +/// capturer picks its device the way the recorder does. +fn device_from_env() -> Option { + std::env::var_os("CAP_SCAP_TEST_WARP").map(|_| warp_device()) +} + +fn capture_samples_with( + item: windows::Graphics::Capture::GraphicsCaptureItem, + settings: Settings, + duration: Duration, + during: impl FnOnce(), +) -> Vec { + capture_samples_on( + item, + settings, + device_from_env(), + Duration::ZERO, + duration, + during, + ) +} + +/// `frame_delay` holds each frame in the callback, standing in for a consumer +/// slower than the capture, so frames queue up in the pool behind it. +fn capture_samples_on( + item: windows::Graphics::Capture::GraphicsCaptureItem, + settings: Settings, + device: Option, + frame_delay: Duration, + duration: Duration, + during: impl FnOnce(), +) -> Vec { + let samples: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink = samples.clone(); + let mut capturer = Capturer::new( + item, + Settings { + is_border_required: Some(false), + is_cursor_capture_enabled: Some(false), + pixel_format: PixelFormat::R8G8B8A8Unorm, + fps: Some(30), + ..settings + }, + move |frame| { + if !frame_delay.is_zero() { + std::thread::sleep(frame_delay); + } + let content = frame.inner().ContentSize()?; + let content = (content.Width.max(0) as u32, content.Height.max(0) as u32); + let buffer = frame.as_buffer()?; + let (width, height, stride) = ( + buffer.width() as usize, + buffer.height() as usize, + buffer.stride() as usize, + ); + let data = buffer.data(); + let (mut red, mut blue) = (0usize, 0usize); + for y in 0..height { + for x in 0..width { + let pixel = &data[y * stride + x * 4..y * stride + x * 4 + 4]; + let (r, g, b) = (pixel[0], pixel[1], pixel[2]); + if r > 200 && g < 60 && b < 60 { + red += 1; + } else if b > 200 && r < 60 && g < 60 { + blue += 1; + } + } + } + sink.lock().unwrap().push(Sample { + width: width as u32, + height: height as u32, + content, + red, + blue, + }); + Ok(()) + }, + || Ok(()), + device, + ) + .expect("create capturer"); + + capturer.start().expect("start capture"); + during(); + std::thread::sleep(duration); + capturer.stop().expect("stop capture"); + // A removed device stops the frames without failing anything else, and + // on WARP it is how an invalid copy shows up. + unsafe { capturer.d3d_device().GetDeviceRemovedReason() } + .expect("the capture device must not be removed during the capture"); + let samples = samples.lock().unwrap(); + samples.clone() +} + +struct Fixture { + process: FixtureProcess, + target: WindowId, + layout: Layout, +} + +impl Fixture { + fn start() -> Self { + let mut process = FixtureProcess( + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "window_capture_fixture_process", "--nocapture"]) + .env("CAP_WINDOW_CAPTURE_FIXTURE", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("start window capture fixture"), + ); + let stdout = process.0.stdout.take().unwrap(); + let (ready_tx, ready_rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + let Ok(line) = line else { break }; + if let Some((_, id)) = line.split_once("WINDOW_CAPTURE_FIXTURE ") { + let _ = ready_tx.send(id.to_string()); + } + } + }); + let target: WindowId = ready_rx + .recv_timeout(Duration::from_secs(15)) + .expect("fixture must become ready") + .trim() + .parse() + .expect("parse fixture window id"); + // Give the windows a moment to paint before capturing them. + std::thread::sleep(Duration::from_millis(800)); + Self { + process, + target, + layout: Layout::of_primary_work_area(), + } + } + + fn window(&self) -> Window { + Window::from_id(&self.target).expect("fixture window must be discoverable") + } + + /// Resizes the target in place to its starting size plus `pixels`. + fn grow_by(&mut self, pixels: i32) { + let (width, height) = (self.layout.width + pixels, self.layout.height + pixels); + self.send(&format!("size {width} {height}")); + } + + fn send(&mut self, command: &str) { + let stdin = self.process.0.stdin.as_mut().expect("fixture stdin"); + stdin.write_all(command.as_bytes()).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); + } +} + +/// Capturing a window through the display's capture item sees whatever is +/// stacked on top of it; capturing the window's own item does not. +#[test] +fn window_item_excludes_windows_stacked_on_top() { + match_cli_dpi_awareness(); + + let fixture = Fixture::start(); + let window = fixture.window(); + let display = window.display().expect("window display"); + let window_bounds = window + .raw_handle() + .physical_bounds() + .expect("window bounds"); + let display_position = display + .raw_handle() + .physical_position() + .expect("display position"); + + let crop = D3D11_BOX { + left: (window_bounds.position().x() - display_position.x()).max(0.0) as u32, + top: (window_bounds.position().y() - display_position.y()).max(0.0) as u32, + right: (window_bounds.position().x() - display_position.x() + window_bounds.size().width()) + .max(0.0) as u32, + bottom: (window_bounds.position().y() - display_position.y() + + window_bounds.size().height()) + .max(0.0) as u32, + front: 0, + back: 1, + }; + + let cropped = capture_samples( + display + .raw_handle() + .try_as_capture_item() + .expect("display capture item"), + Some(crop), + Duration::from_secs(2), + || {}, + ); + let cropped = cropped.last().copied().expect("cropped display frames"); + assert!( + cropped.red > 0, + "a display capture cropped to the window must show the window stacked on top of it, \ + which is the bug being fixed: {cropped:?}" + ); + + let window_item = capture_samples( + window + .raw_handle() + .try_as_capture_item() + .expect("window capture item"), + None, + Duration::from_secs(2), + || {}, + ); + let window_item = window_item.last().copied().expect("window frames"); + assert_eq!( + window_item.red, 0, + "the window's own capture item must not contain the window stacked on top of it: \ + {window_item:?}" + ); + assert!( + window_item.blue > 0, + "the window's own capture item must contain the window: {window_item:?}" + ); +} + +/// The frame pool is created at the item's size; when the item resizes, frames +/// keep arriving with the new ContentSize but the pool's surfaces are still the +/// old size, so every frame reads back empty until the pool is recreated. +#[test] +fn frames_survive_the_capture_item_resizing() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let window = fixture.window(); + let before = window.physical_size().expect("window size"); + + let samples = capture_samples_with( + window + .raw_handle() + .try_as_capture_item() + .expect("window capture item"), + Settings { + follow_item_size: true, + ..Default::default() + }, + Duration::from_secs(4), + || { + std::thread::sleep(Duration::from_millis(500)); + fixture.send("resize"); + }, + ); + + let grown: Vec<_> = samples + .iter() + .filter(|sample| sample.width > before.width() as u32) + .collect(); + assert!( + !grown.is_empty(), + "the capture must follow the window's new size, saw {:?}", + samples + .iter() + .map(|s| (s.width, s.height)) + .collect::>() + ); + let last = grown.last().expect("a frame after the resize"); + assert!( + last.blue > 0, + "frames after a resize must still carry the window's contents: {last:?}" + ); +} + +/// A window's capture item is often odd-sized (the window's own dimensions, or +/// the invisible resize borders taken off them), while the recording encodes +/// at even dimensions. With `even_dimensions` the odd row or column is trimmed +/// on the GPU, so every frame arrives at exactly the even size the recording +/// sizes its output from, and none fall through to the CPU scaler. Two sizes +/// one pixel apart guarantee each axis is odd for one of them. +#[test] +fn window_frames_are_trimmed_to_even_dimensions_on_the_gpu() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let item = fixture + .window() + .raw_handle() + .try_as_capture_item() + .expect("window capture item"); + let samples = capture_samples_with( + item, + Settings { + even_dimensions: true, + follow_item_size: true, + ..Default::default() + }, + Duration::from_secs(2), + || { + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(1); + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(2); + }, + ); + + let sizes: Vec<_> = samples.iter().map(|s| (s.width, s.height)).collect(); + let first = samples.first().expect("frames"); + assert_eq!( + (first.width, first.height), + (first.content.0 & !1, first.content.1 & !1), + "the first frame must be its ContentSize floored to even: {samples:?}" + ); + assert!( + sizes.iter().all(|(w, h)| w % 2 == 0 && h % 2 == 0), + "every frame must have even dimensions: {sizes:?}" + ); + let mut distinct = sizes.clone(); + distinct.dedup(); + assert!( + distinct.len() >= 2, + "the capture must follow both resizes, saw {distinct:?}" + ); + let last = samples.last().expect("frames"); + assert!( + last.blue > 0, + "trimmed frames must still carry the window's contents: {last:?}" + ); +} + +/// A consumer slower than the capture leaves frames queued in the pool when it +/// is recreated for a new size. Those frames report the new ContentSize on a +/// surface still at the old size, so trimming them to their ContentSize reads +/// past the surface's edge. WARP removes the device over that copy and +/// Windows.Graphics.Capture then fails fast on its next use of it, taking the +/// whole process down with 0xc0000409. Runs on WARP whatever the machine has. +/// Which frames are queued is up to the capture's timing, so before the fix +/// this crashed on most runs rather than every one. +#[test] +fn frames_queued_before_the_pool_is_recreated_are_not_copied() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let item = fixture + .window() + .raw_handle() + .try_as_capture_item() + .expect("window capture item"); + + let samples = capture_samples_on( + item, + Settings { + even_dimensions: true, + follow_item_size: true, + ..Default::default() + }, + Some(warp_device()), + Duration::from_millis(150), + Duration::from_secs(2), + || { + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(1); + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(2); + }, + ); + + let sizes: Vec<_> = samples.iter().map(|s| (s.width, s.height)).collect(); + assert!( + sizes.iter().all(|(w, h)| w % 2 == 0 && h % 2 == 0), + "every frame must have even dimensions: {sizes:?}" + ); + let mut distinct = sizes.clone(); + distinct.dedup(); + assert!( + distinct.len() >= 2, + "the capture must follow the resizes, saw {distinct:?}" + ); +} + +/// Without `follow_item_size` the pool keeps its creation size while the item +/// grows past it, so the trim has to stay inside the surface rather than +/// follow ContentSize off its edge. Runs on WARP, where that copy removes the +/// device and the capture stops delivering frames. +#[test] +fn even_dimension_trim_stays_inside_the_surface() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let item = fixture + .window() + .raw_handle() + .try_as_capture_item() + .expect("window capture item"); + // Without `follow_item_size` the pool keeps the surface size it was created + // with, which is the item's size at the start. + let start = item.Size().expect("item size"); + let pool = (start.Width.max(0) as u32, start.Height.max(0) as u32); + + let samples = capture_samples_on( + item, + Settings { + even_dimensions: true, + ..Default::default() + }, + Some(warp_device()), + Duration::ZERO, + Duration::from_secs(2), + || { + // WARP can be slow to deliver its first frame while the other + // tests capture alongside it. + std::thread::sleep(Duration::from_millis(1000)); + fixture.grow_by(1); + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(2); + }, + ); + + let sizes: Vec<_> = samples.iter().map(|s| (s.width, s.height)).collect(); + assert!(!sizes.is_empty(), "the capture must deliver frames"); + assert!( + samples + .iter() + .all(|s| (s.width, s.height) + == (s.content.0.min(pool.0) & !1, s.content.1.min(pool.1) & !1)), + "every frame must be its ContentSize, bounded by the pool's surface, floored to even: {samples:?}" + ); +} + +/// With no crop, trim or pool recreation, a frame whose ContentSize is smaller +/// than its surface must still read back. Handing the whole surface over under +/// the smaller size made `as_buffer`'s `CopyResource` a copy between different +/// sizes, which D3D11 drops, so the frame read back blank. Shrinking the window +/// under a fixed pool reaches this anywhere; on Server 2022 every window item +/// does, because its surface includes the invisible resize borders. +#[test] +fn frames_smaller_than_their_surface_still_read_back() { + match_cli_dpi_awareness(); + + let mut fixture = Fixture::start(); + let item = fixture + .window() + .raw_handle() + .try_as_capture_item() + .expect("window capture item"); + let start = item.Size().expect("item size"); + let pool = (start.Width.max(0) as u32, start.Height.max(0) as u32); + + let samples = capture_samples(item, None, Duration::from_secs(1), || { + std::thread::sleep(Duration::from_millis(700)); + fixture.grow_by(-41); + }); + + let shrunk: Vec<_> = samples + .iter() + .filter(|s| s.content.0 < pool.0 && s.content.1 < pool.1) + .collect(); + assert!( + !shrunk.is_empty(), + "the capture must deliver frames after the shrink: {samples:?}" + ); + assert!( + shrunk + .iter() + .all(|s| (s.width, s.height) == s.content && s.blue > 0), + "a frame smaller than its surface must be its ContentSize and carry the window: {shrunk:?}" + ); +} diff --git a/crates/scap-targets/src/lib.rs b/crates/scap-targets/src/lib.rs index 34a085f186d..1d30106bec8 100644 --- a/crates/scap-targets/src/lib.rs +++ b/crates/scap-targets/src/lib.rs @@ -147,6 +147,15 @@ impl Window { Self::list().into_iter().find(|d| &d.id() == id) } + /// Like [`Window::from_id`], but resolves the handle directly instead of + /// searching `list()`. Prefer it when the window was already chosen and + /// only needs to still exist: enumeration drops minimized windows, so + /// `from_id` reports a minimized target as gone. + #[cfg(any(windows, target_os = "macos"))] + pub fn from_id_direct(id: &WindowId) -> Option { + WindowImpl::from_id_direct(&id.0).map(Self) + } + pub fn physical_size(&self) -> Option { self.0.physical_size() } diff --git a/crates/scap-targets/src/platform/macos.rs b/crates/scap-targets/src/platform/macos.rs index a31e5a9924d..77e2202a25c 100644 --- a/crates/scap-targets/src/platform/macos.rs +++ b/crates/scap-targets/src/platform/macos.rs @@ -349,6 +349,12 @@ impl WindowImpl { windows_with_level.first().map(|(window, _)| *window) } + /// The window `id`, without enumerating every window. Whether it still + /// exists shows in what its getters return. + pub fn from_id_direct(id: &WindowIdImpl) -> Option { + Some(Self(id.0)) + } + pub fn id(&self) -> WindowIdImpl { WindowIdImpl(self.0) } diff --git a/crates/scap-targets/src/platform/win.rs b/crates/scap-targets/src/platform/win.rs index 03d42f5f052..8245222a478 100644 --- a/crates/scap-targets/src/platform/win.rs +++ b/crates/scap-targets/src/platform/win.rs @@ -52,7 +52,7 @@ use windows::{ GWL_EXSTYLE, GWL_STYLE, GetClassLongPtrW, GetClassNameW, GetClientRect, GetCursorPos, GetIconInfo, GetLayeredWindowAttributes, GetWindow, GetWindowLongPtrW, GetWindowLongW, GetWindowRect, GetWindowTextLengthW, - GetWindowTextW, GetWindowThreadProcessId, HICON, ICONINFO, IsIconic, + GetWindowTextW, GetWindowThreadProcessId, HICON, ICONINFO, IsIconic, IsWindow, IsWindowVisible, PrivateExtractIconsW, SendMessageW, WM_GETICON, WS_CHILD, WS_EX_LAYERED, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, WS_EX_TRANSPARENT, WindowFromPoint, }, @@ -420,6 +420,18 @@ impl WindowImpl { context.list } + /// Resolve an id straight back to its HWND, without going through + /// `list()`. Enumeration deliberately skips minimized and cloaked + /// windows, so a window the user picked and then minimized is absent + /// from `list()` while still being a perfectly capturable target. + /// `IsWindow` is the only liveness question that matters here. + pub fn from_id_direct(id: &WindowIdImpl) -> Option { + let hwnd = HWND(id.0 as *mut std::ffi::c_void); + unsafe { IsWindow(Some(hwnd)) } + .as_bool() + .then_some(Self(hwnd)) + } + pub fn inner(&self) -> HWND { self.0 } diff --git a/crates/verify-stimulus/src/main.rs b/crates/verify-stimulus/src/main.rs index 44ff9124da5..055e40ec049 100644 --- a/crates/verify-stimulus/src/main.rs +++ b/crates/verify-stimulus/src/main.rs @@ -114,6 +114,7 @@ mod app { go_at: None, window: None, surface: None, + occluder: None, size: (0, 0), shown: None, ready: false, @@ -340,6 +341,8 @@ mod app { go_at: Option<(Instant, u64)>, window: Option>, surface: Option, + /// The magenta window in front of the stimulus, when the spec asks. + occluder: Option<(Arc, canvas::Canvas)>, size: (u32, u32), shown: Option, ready: bool, @@ -663,6 +666,31 @@ mod app { } window.request_redraw(); self.window = Some(window); + if let Some((x, y, w, h)) = self.spec.occluder { + let attrs = Window::default_attributes() + .with_title(cap_verify::stimulus::OCCLUDER_TITLE) + .with_window_level(WindowLevel::AlwaysOnTop) + .with_decorations(false) + .with_resizable(false) + .with_position(LogicalPosition::new(x, y)) + .with_inner_size(LogicalSize::new(w, h)); + match event_loop + .create_window(attrs) + .map_err(|e| e.to_string()) + .and_then(|window| { + let window = Arc::new(window); + canvas::new(&window).map(|canvas| (window, canvas)) + }) { + Ok((window, canvas)) => { + window.request_redraw(); + self.occluder = Some((window, canvas)); + } + Err(e) => { + self.error = Some(format!("occluder: {e}")); + event_loop.exit(); + } + } + } } fn user_event(&mut self, event_loop: &ActiveEventLoop, command: Command) { @@ -681,12 +709,23 @@ mod app { } } - fn window_event( - &mut self, - event_loop: &ActiveEventLoop, - _id: WindowId, - event: WindowEvent, - ) { + fn window_event(&mut self, event_loop: &ActiveEventLoop, id: WindowId, event: WindowEvent) { + if let Some((window, canvas)) = self + .occluder + .as_mut() + .filter(|(window, _)| window.id() == id) + { + if let WindowEvent::RedrawRequested = event { + let size = window.inner_size(); + // BGRA magenta: nothing the stimulus draws has colour. + canvas::present(canvas, size.width.max(1), size.height.max(1), |bytes| { + for pixel in bytes.chunks_exact_mut(4) { + pixel.copy_from_slice(&[255, 0, 255, 255]); + } + }); + } + return; + } match event { WindowEvent::CloseRequested => event_loop.exit(), WindowEvent::Resized(size) => { diff --git a/crates/verify/src/stimulus.rs b/crates/verify/src/stimulus.rs index d43e7eef6d9..2bcb90d3899 100644 --- a/crates/verify/src/stimulus.rs +++ b/crates/verify/src/stimulus.rs @@ -33,8 +33,15 @@ pub struct StimulusSpec { /// `None` = borderless fullscreen on the primary display; otherwise a /// window at (x, y, width, height) in points. pub window: Option<(i32, i32, u32, u32)>, + /// A solid magenta window at (x, y, width, height) in points, in front + /// of the stimulus window: what a window-only capture must leave out. + #[serde(default)] + pub occluder: Option<(i32, i32, u32, u32)>, } +/// The occluder's title, for the oracle to find its window. +pub const OCCLUDER_TITLE: &str = "Cap Verify Occluder"; + impl StimulusSpec { pub fn frame_count(&self) -> u32 { (self.seconds * self.fps).round() as u32 diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs index b88caa19500..aad2d2af535 100644 --- a/crates/verify/tests/recording.rs +++ b/crates/verify/tests/recording.rs @@ -52,7 +52,7 @@ use cap_verify::{ SENTINEL_FILE, SentinelLog, StimulusEvent, StimulusProcess, StimulusSpec, host_add_secs, host_now, host_secs, }, - video_code::{self, FrameMarker, Readout, StreamKind}, + video_code::{self, FrameMarker, PixelSource as _, Readout, StreamKind}, }; use platform::{Shareable, host_timestamp}; @@ -136,6 +136,18 @@ mod platform { use cpal::traits::HostTrait; cpal::default_host().default_output_device().is_some() } + + /// What Cap's manifest declares: window bounds are read in physical + /// pixels only by a per-monitor DPI aware process, and a window target + /// has none otherwise. + pub fn declare_dpi_awareness() { + const PROCESS_PER_MONITOR_DPI_AWARE: i32 = 2; + #[link(name = "shcore")] + unsafe extern "system" { + fn SetProcessDpiAwareness(value: i32) -> i32; + } + unsafe { SetProcessDpiAwareness(PROCESS_PER_MONITOR_DPI_AWARE) }; + } } static ATTEMPT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -198,10 +210,30 @@ struct Spec { mic_delay: f64, mic: bool, system_audio: bool, + /// Focus: record the stimulus window by itself (window-only capture) + /// with a magenta window in front of it, which must stay out. + focus: bool, } fn specs() -> Vec { let mut specs = vec![ + Spec { + id: "recording/studio-window/focus", + tier: Tier::Fast, + mode: Mode::Studio { + fragmented: true, + camera: false, + }, + seconds: 6.0, + stimulus_fps: 20.0, + holds: vec![], + camera_drops_at: vec![], + gated: false, + mic_delay: 0.0, + mic: true, + system_audio: false, + focus: true, + }, Spec { id: "recording/studio-screen/fragmented", tier: Tier::Fast, @@ -217,6 +249,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/studio-camera/startup-drops", @@ -233,6 +266,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/instant/screen-mic-system", @@ -246,6 +280,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/studio-screen/gated", @@ -262,6 +297,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/instant/gated", @@ -275,6 +311,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/instant/late-mic", @@ -288,6 +325,7 @@ fn specs() -> Vec { mic_delay: 1.0, mic: true, system_audio: false, + focus: false, }, Spec { id: "recording/instant/system-only", @@ -301,6 +339,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: false, system_audio: true, + focus: false, }, Spec { id: "recording/studio-screen/late-mic", @@ -317,6 +356,7 @@ fn specs() -> Vec { mic_delay: 0.3, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/studio-recovery/crash", @@ -330,6 +370,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/studio-screen/mp4-holds", @@ -346,6 +387,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/studio-camera/long", @@ -362,6 +404,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/instant/long-holds", @@ -375,6 +418,7 @@ fn specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, ]; specs.extend(soak_specs()); @@ -420,6 +464,7 @@ fn soak_specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, Spec { id: "recording/soak/instant", @@ -433,6 +478,7 @@ fn soak_specs() -> Vec { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }, ] } @@ -503,6 +549,8 @@ fn runtime() -> tokio::runtime::Runtime { } fn init() { + #[cfg(windows)] + platform::declare_dpi_awareness(); media::init(); if std::env::var_os("RUST_LOG").is_some() { let _ = tracing_subscriber::fmt() @@ -558,6 +606,7 @@ fn real_capture() { mic_delay: 0.0, mic: true, system_audio: true, + focus: false, }; let mut case = Case::new(warmup.id); let started = std::time::Instant::now(); @@ -1300,6 +1349,8 @@ struct Recorded { /// Recovery after a crash may lose at most this many seconds before the /// crash (the in-flight fragment). loss_budget: Option, + /// A Focus case: the screen track must be the stimulus window alone. + focus: bool, } async fn run_case( @@ -2180,6 +2231,12 @@ async fn record( reference: u64, case: &mut Case, ) -> anyhow::Result { + // Focus records one window, whatever the host's default capture is. + let capture = if spec.focus { + CaptureMode::Window + } else { + capture + }; let system_output = platform::system_audio_output(); if spec.system_audio && require_system_audio() && !system_output { case.problem( @@ -2203,6 +2260,9 @@ async fn record( CaptureMode::Display => None, CaptureMode::Window => Some((80, 120, 640, 400)), }, + // Over the stimulus window's lower right, where a region capture + // would record it. + occluder: spec.focus.then_some((480, 320, 360, 260)), }; let mut stimulus = tokio::task::block_in_place(|| { StimulusProcess::spawn( @@ -2212,7 +2272,16 @@ async fn record( ) })?; tokio::task::block_in_place(|| stimulus.wait_for("ready", Duration::from_secs(30)))?; - let (target, excluded, shareable) = capture_setup(capture).await?; + let (target, mut excluded, shareable) = capture_setup(capture).await?; + if spec.focus { + // Left in the picture on purpose: the window-only capture itself must + // keep it out, not the exclusion list a region capture relies on. + if let Some(occluder) = find_window(cap_verify::stimulus::OCCLUDER_TITLE) { + excluded.retain(|id| *id != occluder); + } else { + case.problem("missing", "focus: the occluder window never appeared"); + } + } let camera = matches!(spec.mode, Mode::Studio { camera: true, .. }); let feeds = start_feeds( @@ -2250,7 +2319,11 @@ async fn record( camera, system_audio, ); - let mut builder = builder; + // `CAP_VERIFY_FOCUS_OFF=1` records a Focus case as a region + // instead, which the Focus checks must catch: the control that + // shows they can fail. + let mut builder = + builder.with_window_only(spec.focus && !env_flag("CAP_VERIFY_FOCUS_OFF")); if spec.mic { builder = builder.with_mic_feed(mic_lock.clone()); } @@ -2414,6 +2487,7 @@ async fn record( sources, sentinel, loss_budget, + focus: spec.focus, }) } @@ -3126,6 +3200,105 @@ impl VideoCheck<'_> { /// recorded picture is the first one presented, 9 to 69 ms after the start. const START_COVERAGE_SECS: f64 = 0.3; +/// A Focus recording's screen track is the stimulus window and nothing else: +/// no magenta from the occluder in front of it (the screen key is blue, the +/// noise grey, so strong positive Cr is the occluder alone), no black column +/// or row along an edge (the window drawn smaller than its frame), and the +/// window's own size, less the rounded corners macOS trims off. +fn check_focus(case: &mut Case, track: &Track, stimulus: &[StimulusEvent]) -> anyhow::Result<()> { + let Some((width, height, scale)) = stimulus.iter().find_map(|event| match event { + StimulusEvent::Ready { + width, + height, + scale, + .. + } => Some((*width, *height, *scale)), + _ => None, + }) else { + case.problem("missing", "focus: the stimulus never reported its size"); + return Ok(()); + }; + #[cfg(target_os = "macos")] + let trim = + 2 * (cap_recording::sources::screen_capture::WINDOW_CORNER_TRIM * scale).round() as u32; + #[cfg(not(target_os = "macos"))] + let trim = { + let _ = scale; + 0 + }; + let expected = (width.saturating_sub(trim), height.saturating_sub(trim)); + + let mut frames = 0usize; + let mut sizes = BTreeMap::new(); + let mut leaks = Vec::new(); + let mut edges = Vec::new(); + media::decode_video(&track.path, |picture| { + let view = picture.view(); + let (w, h) = (view.width(), view.height()); + *sizes.entry((w, h)).or_insert(0usize) += 1; + frames += 1; + let mut magenta = 0usize; + for y in (0..h).step_by(2) { + for x in (0..w).step_by(2) { + let (cb, cr) = view.chroma(x, y); + if cr > 60.0 && cb > 30.0 { + magenta += 1; + } + } + } + if magenta > 0 && leaks.len() < 3 { + leaks.push(format!("{:.3}s: {magenta} pixels", picture.pts_secs)); + } + let black_column = |x: usize| (0..h).all(|y| view.luma(x, y) < 20.0); + let black_row = |y: usize| (0..w).all(|x| view.luma(x, y) < 20.0); + let empty = [ + ("left", black_column(0)), + ("right", black_column(w - 1)), + ("top", black_row(0)), + ("bottom", black_row(h - 1)), + ]; + if let Some((side, _)) = empty.iter().find(|(_, black)| *black) + && edges.len() < 3 + { + edges.push(format!("{:.3}s: {side} edge", picture.pts_secs)); + } + })?; + + if frames == 0 { + case.problem("missing", "focus: no screen frames"); + return Ok(()); + } + if !leaks.is_empty() { + case.problem( + "focus-leak", + format!("the window in front was recorded: {}", leaks.join("; ")), + ); + } + if !edges.is_empty() { + case.problem( + "focus-edge", + format!("an empty strip along the frame: {}", edges.join("; ")), + ); + } + // Frame sizes are rounded to even dimensions for the encoder. + let near = |(w, h): (usize, usize)| { + (w as i64 - expected.0 as i64).abs() <= 2 && (h as i64 - expected.1 as i64).abs() <= 2 + }; + if !sizes.keys().all(|size| near(*size)) { + case.problem( + "focus-size", + format!( + "expected {}x{} (the window less {trim} px of corners), recorded {sizes:?}", + expected.0, expected.1 + ), + ); + } + case.note(format!( + "focus: {frames} frames at {sizes:?}, window {width}x{height} @{scale}x" + )); + Ok(()) +} + fn check_video(case: &mut Case, track: &Track, mut check: VideoCheck) -> anyhow::Result { let mut frames: Vec<(f64, Readout)> = Vec::new(); media::decode_video(&track.path, |picture| { @@ -3733,6 +3906,9 @@ fn analyze(rec: &Recorded, reference: u64, case: &mut Case) -> anyhow::Result<() observed, }, )?; + if rec.focus { + check_focus(case, &layout.display, &rec.stimulus)?; + } let camera = match (&layout.camera, &rec.camera) { (Some(track), Some((truth, dropped))) if track.path.is_file() => { case.note(format!("camera: dropped at source {dropped:?}"));