feat(client): add :keepalive_interval to keep the connection active during long waits

CHANGELOG.md

@@ -1,6 +1,8 @@
 ## [Unreleased](https://github.com/rubycdp/ferrum/compare/v0.19.1...main) ##
 
 ### Added
+- `:keepalive_interval` sends a CDP command at least every given number of seconds while waiting for a navigation,
+  so a proxy with an idle timeout doesn't drop the connection during a long `:timeout` [#671]
 - `keyboard.hold(:shift) { }` holds modifiers for every keyboard and mouse event in the block, e.g. a shift+drag, and
   releases them afterwards. `mouse.move` accepts `modifiers:` like `down` and `up` [#581]
 - `page.console` collects `console.*` calls, browser log entries and uncaught exceptions as `Console::Message`s,

docs/2-customization.md

@@ -29,6 +29,9 @@ Ferrum::Browser.new(options)
       Useful companion of headless option, so that you have time to see changes.
     * `:timeout` (Numeric) - The number of seconds we'll wait for a response when
       communicating with browser. Default is 5.
+    * `:keepalive_interval` (Numeric) - While waiting for a navigation, send a CDP command at least this often
+      (in seconds) so a proxy between Ruby and Chrome doesn't drop the idle connection, e.g. Heroku's router
+      closes connections idle for 55 seconds. The total wait is still bounded by `:timeout`. Default is nil (off).
     * `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby.
     * `:pending_connection_errors` (Boolean) - Raise `PendingConnectionsError` when main frame is still waiting
       for slow responses and timeout is reached. Default is false.

lib/ferrum/browser/options.rb

@@ -22,7 +22,7 @@ class Options
       warn "Ferrum: FERRUM_DEBUG is deprecated, use FERRUM_LOG=* instead" if ENV.key?("FERRUM_DEBUG")
 
       attr_reader :window_size, :logger, :log, :ws_max_receive_size,
-                  :js_errors, :base_url, :slowmo, :pending_connection_errors,
+                  :js_errors, :base_url, :slowmo, :keepalive_interval, :pending_connection_errors,
                   :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
                   :save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
                   :ignore_default_browser_options, :xvfb, :flatten, :extensions
@@ -44,6 +44,7 @@ def initialize(options = nil)
         @pending_connection_errors = @options.fetch(:pending_connection_errors, false)
         @process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
         @slowmo = @options[:slowmo].to_f
+        @keepalive_interval = @options[:keepalive_interval]
 
         @env = @options[:env]
         @xvfb = @options[:xvfb]

lib/ferrum/client.rb

@@ -228,7 +228,8 @@ def send_message(message, async:, timeout: nil)
         @pendings[message[:id]] = pending
         begin
           @ws.send_message(message, timeout)
-          data = pending.value!(timeout)
+          wait_with_keepalive(timeout) { |slice| pending.wait(slice).complete? }
+          data = pending.value!(0)
         ensure
           @pendings.delete(message[:id])
         end
@@ -335,8 +336,39 @@ def build_message(method, params)
       { method: method, params: params }.merge(id: next_command_id)
     end
 
+    #
+    # Waits up to `timeout` seconds for the block to return truthy, calling it
+    # with how long it may block. With {Browser::Options#keepalive_interval}
+    # set, the wait is split into slices of at most that length with a CDP
+    # command sent between them, so the connection never sits idle.
+    #
+    # @param [Numeric] timeout
+    #
+    # @yieldparam [Numeric] slice
+    #   The longest the block may block for.
+    #
+    # @return [Boolean]
+    #   Whether the block returned truthy before the timeout.
+    #
+    def wait_with_keepalive(timeout)
+      return !!yield(timeout) unless (interval = options.keepalive_interval)
+
+      deadline = Utils::ElapsedTime.monotonic_time + timeout
+      loop do
+        remaining = deadline - Utils::ElapsedTime.monotonic_time
+        return false unless remaining.positive?
+        return true if yield([interval, remaining].min)
+
+        keepalive
+      end
+    end
+
     private
 
+    def keepalive
+      @ws.send_message(build_message("Browser.getVersion", {}), protocol_timeout)
+    end
+
     def start
       @thread = Utils::Thread.spawn do
         loop do

lib/ferrum/page.rb

@@ -506,7 +506,7 @@ def command(method, wait: 0, slowmoable: false, timeout: nil, **params)
         # the main frame, and it started to load new content.
         @event.wait(wait)
         if iteration != @event.iteration
-          set = @event.wait(self.timeout)
+          set = client.wait_with_keepalive(self.timeout) { |slice| @event.wait(slice) }
           raise TimeoutError unless set
         end
       end

sig/ferrum/browser/options.rbs

@@ -34,6 +34,8 @@ module Ferrum
 
       attr_reader slowmo: ::Float
 
+      attr_reader keepalive_interval: ::Numeric?
+
       attr_reader pending_connection_errors: bool
 
       attr_reader url: String?
@@ -81,6 +83,7 @@ module Ferrum
       @pending_connection_errors: bool
       @process_timeout: ::Integer
       @slowmo: ::Float
+      @keepalive_interval: ::Numeric?
       @env: Hash[String, String]?
       @xvfb: bool?
       @save_path: String?

sig/ferrum/client.rbs

@@ -70,8 +70,12 @@ module Ferrum
 
     def protocol_timeout=: (::Numeric value) -> ::Numeric
 
+    def wait_with_keepalive: (::Numeric timeout) { (::Numeric slice) -> boolish } -> bool
+
     private
 
+    def keepalive: () -> void
+
     def start: () -> void
 
     def release_pendings: () -> void

spec/browser_spec.rb

@@ -673,6 +673,57 @@ def proxy_tunnel(socket, request, browser_url)
       end
     end
 
+    context ":keepalive_interval argument" do
+      it "sends a CDP command while waiting for a slow navigation", :launch do
+        browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5, keepalive_interval: 0.5)
+
+        browser.go_to("/really_slow")
+
+        expect(browser.body).to include("really slow page")
+        expect(logger.string.scan("Browser.getVersion").size).to be >= 4
+      ensure
+        browser&.quit
+      end
+
+      it "sends a CDP command while waiting for a navigation started by a click", :launch do
+        browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5, keepalive_interval: 0.5)
+        browser.go_to("/")
+        browser.execute(<<~JS)
+          document.body.innerHTML = '<a id="slow" href="/really_slow">Slow</a>'
+        JS
+        calls_before_click = logger.string.scan("Browser.getVersion").size
+
+        browser.at_css("#slow").click
+
+        expect(browser.body).to include("really slow page")
+        expect(logger.string.scan("Browser.getVersion").size - calls_before_click).to be >= 3
+      ensure
+        browser&.quit
+      end
+
+      it "still times out after :timeout", :launch do
+        browser = Ferrum::Browser.new(base_url: base_url, logger: logger, pending_connection_errors: true,
+                                      timeout: 1, keepalive_interval: 0.3)
+        startup_calls = logger.string.scan("Browser.getVersion").size
+
+        expect { browser.go_to("/really_slow") }.to raise_error(Ferrum::PendingConnectionsError)
+        expect(logger.string.scan("Browser.getVersion").size).to be > startup_calls
+      ensure
+        browser&.quit
+      end
+
+      it "doesn't send keepalive commands when not set", :launch do
+        browser = Ferrum::Browser.new(base_url: base_url, logger: logger, timeout: 5)
+        startup_calls = logger.string.scan("Browser.getVersion").size
+
+        browser.go_to("/really_slow")
+
+        expect(logger.string.scan("Browser.getVersion").size).to eq(startup_calls)
+      ensure
+        browser&.quit
+      end
+    end
+
     it "supports :pending_connection_errors argument", :launch do
       browser = Ferrum::Browser.new(base_url: base_url, pending_connection_errors: true, timeout: 0.5)