diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aae1162..7bed8747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## [Unreleased](https://github.com/rubycdp/ferrum/compare/v0.19.1...main) ## ### Added +- `:keepalive_interval` sends a CDP command at least every given number of seconds while waiting for a navigation, + so a proxy with an idle timeout doesn't drop the connection during a long `:timeout` [#671] - `keyboard.hold(:shift) { }` holds modifiers for every keyboard and mouse event in the block, e.g. a shift+drag, and releases them afterwards. `mouse.move` accepts `modifiers:` like `down` and `up` [#581] - `page.console` collects `console.*` calls, browser log entries and uncaught exceptions as `Console::Message`s, diff --git a/docs/2-customization.md b/docs/2-customization.md index e21b5e07..eda64ca1 100644 --- a/docs/2-customization.md +++ b/docs/2-customization.md @@ -29,6 +29,9 @@ Ferrum::Browser.new(options) Useful companion of headless option, so that you have time to see changes. * `:timeout` (Numeric) - The number of seconds we'll wait for a response when communicating with browser. Default is 5. + * `:keepalive_interval` (Numeric) - While waiting for a navigation, send a CDP command at least this often + (in seconds) so a proxy between Ruby and Chrome doesn't drop the idle connection, e.g. Heroku's router + closes connections idle for 55 seconds. The total wait is still bounded by `:timeout`. Default is nil (off). * `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby. * `:pending_connection_errors` (Boolean) - Raise `PendingConnectionsError` when main frame is still waiting for slow responses and timeout is reached. Default is false. diff --git a/lib/ferrum/browser/options.rb b/lib/ferrum/browser/options.rb index 3a1c0fff..ff1c0458 100644 --- a/lib/ferrum/browser/options.rb +++ b/lib/ferrum/browser/options.rb @@ -22,7 +22,7 @@ class Options warn "Ferrum: FERRUM_DEBUG is deprecated, use FERRUM_LOG=* instead" if ENV.key?("FERRUM_DEBUG") attr_reader :window_size, :logger, :log, :ws_max_receive_size, - :js_errors, :base_url, :slowmo, :pending_connection_errors, + :js_errors, :base_url, :slowmo, :keepalive_interval, :pending_connection_errors, :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path, :save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options, :ignore_default_browser_options, :xvfb, :flatten, :extensions @@ -44,6 +44,7 @@ def initialize(options = nil) @pending_connection_errors = @options.fetch(:pending_connection_errors, false) @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT) @slowmo = @options[:slowmo].to_f + @keepalive_interval = @options[:keepalive_interval] @env = @options[:env] @xvfb = @options[:xvfb] diff --git a/lib/ferrum/client.rb b/lib/ferrum/client.rb index 48af6f63..b3f39ff2 100644 --- a/lib/ferrum/client.rb +++ b/lib/ferrum/client.rb @@ -228,7 +228,8 @@ def send_message(message, async:, timeout: nil) @pendings[message[:id]] = pending begin @ws.send_message(message, timeout) - data = pending.value!(timeout) + wait_with_keepalive(timeout) { |slice| pending.wait(slice).complete? } + data = pending.value!(0) ensure @pendings.delete(message[:id]) end @@ -335,8 +336,39 @@ def build_message(method, params) { method: method, params: params }.merge(id: next_command_id) end + # + # Waits up to `timeout` seconds for the block to return truthy, calling it + # with how long it may block. With {Browser::Options#keepalive_interval} + # set, the wait is split into slices of at most that length with a CDP + # command sent between them, so the connection never sits idle. + # + # @param [Numeric] timeout + # + # @yieldparam [Numeric] slice + # The longest the block may block for. + # + # @return [Boolean] + # Whether the block returned truthy before the timeout. + # + def wait_with_keepalive(timeout) + return !!yield(timeout) unless (interval = options.keepalive_interval) + + deadline = Utils::ElapsedTime.monotonic_time + timeout + loop do + remaining = deadline - Utils::ElapsedTime.monotonic_time + return false unless remaining.positive? + return true if yield([interval, remaining].min) + + keepalive + end + end + private + def keepalive + @ws.send_message(build_message("Browser.getVersion", {}), protocol_timeout) + end + def start @thread = Utils::Thread.spawn do loop do diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index 577ae345..634610f8 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -506,7 +506,7 @@ def command(method, wait: 0, slowmoable: false, timeout: nil, **params) # the main frame, and it started to load new content. @event.wait(wait) if iteration != @event.iteration - set = @event.wait(self.timeout) + set = client.wait_with_keepalive(self.timeout) { |slice| @event.wait(slice) } raise TimeoutError unless set end end diff --git a/sig/ferrum/browser/options.rbs b/sig/ferrum/browser/options.rbs index 1775c2f7..829adb13 100644 --- a/sig/ferrum/browser/options.rbs +++ b/sig/ferrum/browser/options.rbs @@ -34,6 +34,8 @@ module Ferrum attr_reader slowmo: ::Float + attr_reader keepalive_interval: ::Numeric? + attr_reader pending_connection_errors: bool attr_reader url: String? @@ -81,6 +83,7 @@ module Ferrum @pending_connection_errors: bool @process_timeout: ::Integer @slowmo: ::Float + @keepalive_interval: ::Numeric? @env: Hash[String, String]? @xvfb: bool? @save_path: String? diff --git a/sig/ferrum/client.rbs b/sig/ferrum/client.rbs index bc90a592..3525a740 100644 --- a/sig/ferrum/client.rbs +++ b/sig/ferrum/client.rbs @@ -70,8 +70,12 @@ module Ferrum def protocol_timeout=: (::Numeric value) -> ::Numeric + def wait_with_keepalive: (::Numeric timeout) { (::Numeric slice) -> boolish } -> bool + private + def keepalive: () -> void + def start: () -> void def release_pendings: () -> void diff --git a/spec/browser_spec.rb b/spec/browser_spec.rb index 8f4c046a..708c9126 100644 --- a/spec/browser_spec.rb +++ b/spec/browser_spec.rb @@ -673,6 +673,57 @@ def proxy_tunnel(socket, request, browser_url) end end + context ":keepalive_interval argument" do + it "sends a CDP command while waiting for a slow navigation", :launch do + browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5, keepalive_interval: 0.5) + + browser.go_to("/really_slow") + + expect(browser.body).to include("really slow page") + expect(logger.string.scan("Browser.getVersion").size).to be >= 4 + ensure + browser&.quit + end + + it "sends a CDP command while waiting for a navigation started by a click", :launch do + browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5, keepalive_interval: 0.5) + browser.go_to("/") + browser.execute(<<~JS) + document.body.innerHTML = 'Slow' + JS + calls_before_click = logger.string.scan("Browser.getVersion").size + + browser.at_css("#slow").click + + expect(browser.body).to include("really slow page") + expect(logger.string.scan("Browser.getVersion").size - calls_before_click).to be >= 3 + ensure + browser&.quit + end + + it "still times out after :timeout", :launch do + browser = Ferrum::Browser.new(base_url: base_url, logger: logger, pending_connection_errors: true, + timeout: 1, keepalive_interval: 0.3) + startup_calls = logger.string.scan("Browser.getVersion").size + + expect { browser.go_to("/really_slow") }.to raise_error(Ferrum::PendingConnectionsError) + expect(logger.string.scan("Browser.getVersion").size).to be > startup_calls + ensure + browser&.quit + end + + it "doesn't send keepalive commands when not set", :launch do + browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5) + startup_calls = logger.string.scan("Browser.getVersion").size + + browser.go_to("/really_slow") + + expect(logger.string.scan("Browser.getVersion").size).to eq(startup_calls) + ensure + browser&.quit + end + end + it "supports :pending_connection_errors argument", :launch do browser = Ferrum::Browser.new(base_url: base_url, pending_connection_errors: true, timeout: 0.5)