From 99d2f8570f23380d56f7186338b9b46491fcb2db Mon Sep 17 00:00:00 2001 From: Negar Bayati Date: Fri, 30 Oct 2026 02:00:50 +0000 Subject: [PATCH 1/4] fix(auth): add startup polling for Cloud Run SPIFFE mTLS certificates A startup latency can occur where the environment variable pointing to the certificate configuration is set before the files are actually mounted on the filesystem. While token refresh already polls for the certificate file, mTLS transport discovery did not, causing transports initialized at container startup to fall back to standard TLS and fail with 401 once a bound token was minted. * Return True from check_use_client_cert() when GOOGLE_API_CERTIFICATE_CONFIG points inside the well-known SPIFFE directory so mTLS endpoint selection is enabled at startup without blocking on disk I/O. * Poll up to 30 seconds (50 retries at 100ms followed by 50 retries at 500ms) in _get_workload_cert_and_key() and _get_workload_cert_and_key_paths() for the config, certificate, and private key files to be written, failing fast on invalid config schemas, or PermissionError. * Add unit tests in TestSpiffeWorkloadPolling covering cold-start polling recovery, timeout, and fail-fast conditions. --- .../google/auth/transport/_mtls_helper.py | 137 ++++++-- .../tests/transport/test__mtls_helper.py | 310 +++++++++++++++++- 2 files changed, 417 insertions(+), 30 deletions(-) diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index 7ff5b2a81d81..4c8e35f52588 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -22,6 +22,7 @@ import subprocess import sys import tempfile +import time from os import environ, getenv, path from typing import Generator, List, Optional, Tuple, Union, cast from urllib.parse import urlsplit @@ -415,6 +416,13 @@ def _get_workload_cert_and_key( the certificate or key information. """ + spiffe_config_path = _get_spiffe_polling_config_path(certificate_config_path) + if spiffe_config_path is not None: + _, _, _, cert_data, key_data = _poll_spiffe_workload_credentials( + spiffe_config_path + ) + return cert_data, key_data + cert_path, key_path, config_file_path = _get_workload_cert_and_key_paths( certificate_config_path, include_context_aware ) @@ -484,32 +492,26 @@ def _get_cert_config_path(certificate_config_path=None, include_context_aware=Tr def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=True): + spiffe_config_path = _get_spiffe_polling_config_path(config_path) + if spiffe_config_path is not None: + cert_path, key_path, config_file_path, _, _ = _poll_spiffe_workload_credentials( + spiffe_config_path + ) + return cert_path, key_path, config_file_path + absolute_path = _get_cert_config_path(config_path, include_context_aware) if absolute_path is None: return None, None, None data = _load_json_file(absolute_path) - - if ( - not isinstance(data, dict) - or "cert_configs" not in data - or not isinstance(data["cert_configs"], dict) - ): - raise exceptions.ClientCertError( - 'Certificate config file {} is in an invalid format, a "cert configs" object is expected'.format( - absolute_path - ) - ) - cert_configs = data["cert_configs"] + cert_path, key_path = _extract_workload_paths_from_config(data, absolute_path) # We return None, None if the expected workload fields are not present. # The certificate config might be present for other types of connections (e.g. gECC), # and we want to gracefully fallback to testing other mTLS configurations # like SecureConnect instead of throwing an exception. - if ( - not isinstance(cert_configs, dict) or "workload" not in cert_configs - ) and config_path is None: + if cert_path is None and key_path is None and config_path is None: default_home_path = path.expanduser( os.path.join( _cloud_sdk.get_config_path(), @@ -521,19 +523,53 @@ def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=Tru ) != os.path.normpath(absolute_path): try: home_data = _load_json_file(default_home_path) - if isinstance(home_data, dict): - home_cert_configs = home_data.get("cert_configs") - if ( - isinstance(home_cert_configs, dict) - and "workload" in home_cert_configs - ): - cert_configs = home_cert_configs - absolute_path = default_home_path except (exceptions.ClientCertError, OSError): - pass + home_data = None + if ( + isinstance(home_data, dict) + and isinstance(home_data.get("cert_configs"), dict) + and "workload" in home_data["cert_configs"] + ): + cert_path, key_path = _extract_workload_paths_from_config( + home_data, default_home_path + ) + return cert_path, key_path, default_home_path + + return cert_path, key_path, absolute_path + + +def _get_spiffe_polling_config_path(config_path=None): + """Returns the expanded SPIFFE config path if startup polling applies, else None.""" + if _check_use_client_cert_env() is False: + return None + candidate_path = ( + config_path + if config_path is not None + else environ.get(environment_vars.GOOGLE_API_CERTIFICATE_CONFIG) + ) + if not candidate_path: + return None + expanded_path = path.expanduser(candidate_path) + if _agent_identity_utils._is_in_well_known_dir(expanded_path): + return expanded_path + return None + - if not isinstance(cert_configs, dict) or "workload" not in cert_configs: - return None, None, absolute_path +def _extract_workload_paths_from_config(data, absolute_path): + """Validates certificate_config.json schema and returns (cert_path, key_path).""" + if ( + not isinstance(data, dict) + or "cert_configs" not in data + or not isinstance(data["cert_configs"], dict) + ): + raise exceptions.ClientCertError( + 'Certificate config file {} is in an invalid format, a "cert configs" object is expected'.format( + absolute_path + ) + ) + cert_configs = data["cert_configs"] + if "workload" not in cert_configs: + return None, None workload = cert_configs["workload"] if ( @@ -548,10 +584,51 @@ def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=Tru absolute_path ) ) - cert_path = workload["cert_path"] - key_path = workload["key_path"] + return workload["cert_path"], workload["key_path"] - return cert_path, key_path, absolute_path + +def _poll_spiffe_workload_credentials(spiffe_config_path): + """Polls up to 30s for SPIFFE workload config, cert, and key files to be ready.""" + for attempt, interval in enumerate(_agent_identity_utils._POLLING_INTERVALS): + try: + data = _load_json_file(spiffe_config_path) + except PermissionError: + raise + except (OSError, exceptions.ClientCertError): + pass + else: + cert_path, key_path = _extract_workload_paths_from_config( + data, spiffe_config_path + ) + if cert_path is None and key_path is None: + return None, None, spiffe_config_path, None, None + + cert_path = path.expanduser(cert_path) + key_path = path.expanduser(key_path) + try: + cert_data, key_data = _read_cert_and_key_files(cert_path, key_path) + return cert_path, key_path, spiffe_config_path, cert_data, key_data + except PermissionError: + raise + except (OSError, exceptions.ClientCertError): + if not ( + _agent_identity_utils._is_in_well_known_dir(cert_path) + and _agent_identity_utils._is_in_well_known_dir(key_path) + ): + raise + + if attempt + 1 == _agent_identity_utils._FAST_POLL_CYCLES: + _LOGGER.debug( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + spiffe_config_path, + ) + time.sleep(interval) + + raise exceptions.ClientCertError( + "Identity certificate configuration or certificate/key files at {} were not ready after {}s".format( + spiffe_config_path, _agent_identity_utils._TOTAL_TIMEOUT + ) + ) def _read_credential_bundle_file(bundle_path): @@ -849,6 +926,8 @@ def check_use_client_cert(): return env_override # Auto-enablement checks (when GOOGLE_API_USE_CLIENT_CERTIFICATE is not set) + if _get_spiffe_polling_config_path() is not None: + return True # Check if a workload config file exists. cert_path = _get_cert_config_path(include_context_aware=True) diff --git a/packages/google-auth/tests/transport/test__mtls_helper.py b/packages/google-auth/tests/transport/test__mtls_helper.py index c4e5b52c5973..add16d73feb0 100644 --- a/packages/google-auth/tests/transport/test__mtls_helper.py +++ b/packages/google-auth/tests/transport/test__mtls_helper.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re import sys @@ -23,7 +24,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec -from google.auth import environment_vars, exceptions +from google.auth import _agent_identity_utils, environment_vars, exceptions from google.auth.transport import _mtls_helper if not hasattr(os, "MFD_CLOEXEC"): @@ -683,6 +684,21 @@ def load_json_side_effect(path): ) mock_read_cert_and_key_files.assert_not_called() + def test_no_workload_fallback_to_home_without_workload(self, tmp_path, monkeypatch): + ecp_config = tmp_path / "ecp_config.json" + ecp_config.write_text('{"cert_configs": {"pkcs11": {}}}') + home_dir = tmp_path / "gcloud" + home_dir.mkdir() + (home_dir / "certificate_config.json").write_text('{"cert_configs": {}}') + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(ecp_config) + ) + monkeypatch.setattr( + _mtls_helper._cloud_sdk, "get_config_path", lambda: str(home_dir) + ) + + assert _mtls_helper._get_workload_cert_and_key() == (None, None) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) # noqa: E501 @mock.patch( "google.auth.transport._mtls_helper._get_cert_config_path", @@ -2313,3 +2329,295 @@ def test_no_gke_fallback_when_implicit_config_has_no_workload( assert _mtls_helper._get_workload_cert_and_key() == (None, None) assert _mtls_helper.check_use_client_cert() is False mock_is_ready.assert_not_called() + + +class TestSpiffeWorkloadPolling(object): + @pytest.fixture + def spiffe_dir(self, tmp_path, monkeypatch): + well_known_dir = tmp_path / "workload-spiffe-credentials" + well_known_dir.mkdir() + monkeypatch.setattr( + _agent_identity_utils, + "_WELL_KNOWN_CERT_PATH", + str(well_known_dir / "certificates.pem"), + ) + return well_known_dir + + def _write_workload_config(self, config_file, cert_path, key_path): + config_file.write_text( + json.dumps( + { + "cert_configs": { + "workload": { + "cert_path": str(cert_path), + "key_path": str(key_path), + } + } + } + ) + ) + + def test_get_spiffe_polling_config_path_disabled_by_env( + self, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "credentials.json"), + ) + monkeypatch.setenv(environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false") + assert _mtls_helper._get_spiffe_polling_config_path() is None + + def test_get_spiffe_polling_config_path_ignores_non_spiffe_and_context_aware( + self, spiffe_dir, tmp_path, monkeypatch + ): + monkeypatch.setenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, + str(spiffe_dir / "credentials.json"), + ) + assert _mtls_helper._get_spiffe_polling_config_path() is None + assert ( + _mtls_helper._get_spiffe_polling_config_path( + str(tmp_path / "certificate_config.json") + ) + is None + ) + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_check_use_client_cert_spiffe_returns_true_without_polling( + self, mock_sleep, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "missing_credentials.json"), + ) + + assert _mtls_helper.check_use_client_cert() is True + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_immediate_readiness_zero_sleep(self, mock_sleep, spiffe_dir, monkeypatch): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + self._write_workload_config(config_file, cert_file, key_file) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + assert _mtls_helper._get_workload_cert_and_key_paths() == ( + str(cert_file), + str(key_file), + str(config_file), + ) + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + mock_sleep.assert_not_called() + + @pytest.mark.parametrize("initial_config", ["", "{truncated"]) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_config_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch, initial_config + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + config_file.write_text(initial_config) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_config(*args, **kwargs): + self._write_workload_config(config_file, cert_file, key_file) + + mock_sleep.side_effect = write_config + + actual_cert, actual_key = _mtls_helper._get_workload_cert_and_key() + assert actual_cert == pytest.public_cert_bytes + assert actual_key == pytest.private_key_bytes + assert mock_sleep.call_count == 1 + + @pytest.mark.parametrize( + "initial_cert,initial_key", + [ + (b"", b""), + (b"-----BEGIN CERTIFICATE-----\ntruncated", pytest.private_key_bytes), + (pytest.public_cert_bytes, b"-----BEGIN PRIVATE KEY-----\ntruncated"), + ], + ) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_cert_and_key_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch, initial_cert, initial_key + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + self._write_workload_config(config_file, cert_file, key_file) + cert_file.write_bytes(initial_cert) + key_file.write_bytes(initial_key) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_cert_and_key(*args, **kwargs): + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + + mock_sleep.side_effect = write_cert_and_key + + actual_cert, actual_key = _mtls_helper._get_workload_cert_and_key() + assert actual_cert == pytest.public_cert_bytes + assert actual_key == pytest.private_key_bytes + assert mock_sleep.call_count == 1 + + @mock.patch("google.auth.transport._mtls_helper._LOGGER.debug") + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_timeout_waiting_for_config_logs_debug_once( + self, mock_sleep, mock_debug, spiffe_dir + ): + config_file = spiffe_dir / "credentials.json" + + with pytest.raises(exceptions.ClientCertError, match="not ready after 30s"): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + + assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) + mock_debug.assert_called_once_with( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + str(config_file), + ) + + @mock.patch("google.auth.transport._mtls_helper._LOGGER.debug") + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_timeout_waiting_for_cert_and_key_fails_closed( + self, mock_sleep, mock_debug, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + self._write_workload_config( + config_file, + spiffe_dir / "certificates.pem", + spiffe_dir / "private_key.pem", + ) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with pytest.raises(exceptions.ClientCertError, match="not ready after 30s"): + _mtls_helper._get_workload_cert_and_key() + + assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) + mock_debug.assert_called_once_with( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + str(config_file), + ) + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_config_without_workload_section_does_not_poll( + self, mock_sleep, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + config_file.write_text('{"cert_configs": {}}') + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + assert _mtls_helper._get_workload_cert_and_key() == (None, None) + mock_sleep.assert_not_called() + + @pytest.mark.parametrize( + "invalid_config", + [ + "[]", + "{}", + '{"cert_configs": []}', + '{"cert_configs": {"workload": []}}', + '{"cert_configs": {"workload": {"cert_path": "", "key_path": "/key"}}}', + '{"cert_configs": {"workload": {"cert_path": "/cert", "key_path": ""}}}', + ], + ) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_invalid_config_schema_fails_immediately_without_polling( + self, mock_sleep, spiffe_dir, monkeypatch, invalid_config + ): + config_file = spiffe_dir / "credentials.json" + config_file.write_text(invalid_config) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with pytest.raises(exceptions.ClientCertError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_external_cert_and_key_paths_not_polled( + self, mock_sleep, spiffe_dir, tmp_path + ): + external_cert = tmp_path / "external_cert.pem" + external_key = tmp_path / "external_key.pem" + config_file = spiffe_dir / "credentials.json" + self._write_workload_config(config_file, external_cert, external_key) + + with pytest.raises(FileNotFoundError): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + external_cert.write_bytes(b"invalid pem") + with pytest.raises( + exceptions.ClientCertError, + match="Certificate file .* is in an invalid format", + ): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + external_cert.write_bytes(pytest.public_cert_bytes) + external_key.write_bytes(pytest.private_key_bytes) + assert _mtls_helper._get_workload_cert_and_key(str(config_file)) == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + mock_sleep.assert_not_called() + + @mock.patch("builtins.open", side_effect=PermissionError("denied")) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_config_permission_error_propagates( + self, mock_sleep, mock_open, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "credentials.json"), + ) + + with pytest.raises(PermissionError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_cert_and_key_permission_error_propagates( + self, mock_sleep, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + self._write_workload_config( + config_file, + spiffe_dir / "certificates.pem", + spiffe_dir / "private_key.pem", + ) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with open(config_file) as config_handle: + with mock.patch( + "builtins.open", + side_effect=[config_handle, PermissionError("denied")], + ): + with pytest.raises(PermissionError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() From ec909ed09e9f5ca46812c258848fd4c85692e714 Mon Sep 17 00:00:00 2001 From: Negar Bayati Date: Fri, 30 Oct 2026 19:29:52 +0000 Subject: [PATCH 2/4] fix: avoid duplicate SPIFFE polling in has_default_client_cert_source --- .../google/auth/transport/_mtls_helper.py | 10 +++++++++- .../google-auth/google/auth/transport/mtls.py | 5 ++++- .../google-auth/tests/transport/test_mtls.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index 4c8e35f52588..e6e814d8a974 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -423,7 +423,7 @@ def _get_workload_cert_and_key( ) return cert_data, key_data - cert_path, key_path, config_file_path = _get_workload_cert_and_key_paths( + cert_path, key_path, config_file_path = _get_non_spiffe_workload_cert_and_key_paths( certificate_config_path, include_context_aware ) @@ -499,6 +499,14 @@ def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=Tru ) return cert_path, key_path, config_file_path + return _get_non_spiffe_workload_cert_and_key_paths( + config_path, include_context_aware + ) + + +def _get_non_spiffe_workload_cert_and_key_paths( + config_path=None, include_context_aware=True +): absolute_path = _get_cert_config_path(config_path, include_context_aware) if absolute_path is None: return None, None, None diff --git a/packages/google-auth/google/auth/transport/mtls.py b/packages/google-auth/google/auth/transport/mtls.py index db6d47daf688..a60603e36fb4 100644 --- a/packages/google-auth/google/auth/transport/mtls.py +++ b/packages/google-auth/google/auth/transport/mtls.py @@ -42,12 +42,15 @@ def has_default_client_cert_source(include_context_aware=True): Returns: bool: indicating if the default client cert source exists. """ + if _mtls_helper._get_spiffe_polling_config_path() is not None: + return True + try: ( cert_path, key_path, config_file_path, - ) = _mtls_helper._get_workload_cert_and_key_paths( + ) = _mtls_helper._get_non_spiffe_workload_cert_and_key_paths( None, include_context_aware=include_context_aware ) except (exceptions.ClientCertError, OSError): diff --git a/packages/google-auth/tests/transport/test_mtls.py b/packages/google-auth/tests/transport/test_mtls.py index 5899a1b1c73b..1bee2fbbe499 100644 --- a/packages/google-auth/tests/transport/test_mtls.py +++ b/packages/google-auth/tests/transport/test_mtls.py @@ -558,3 +558,20 @@ def test_has_default_client_cert_source_malformed_config_no_gke_fallback( assert mtls.has_default_client_cert_source(include_context_aware=False) is True mock_is_ready.assert_not_called() + + +@mock.patch( + "google.auth.transport._mtls_helper._get_non_spiffe_workload_cert_and_key_paths", + autospec=True, +) +@mock.patch( + "google.auth.transport._mtls_helper._get_spiffe_polling_config_path", + return_value="/var/run/secrets/workload-spiffe-credentials/credentials.json", + autospec=True, +) +def test_has_default_client_cert_source_spiffe_returns_true_without_polling( + mock_spiffe_path, mock_non_spiffe_paths +): + assert mtls.has_default_client_cert_source() is True + mock_spiffe_path.assert_called_once_with() + mock_non_spiffe_paths.assert_not_called() From 60dc2fdf7f480d1020d6bcb7aeeb51fe8d609c29 Mon Sep 17 00:00:00 2001 From: Negar Bayati Date: Fri, 30 Oct 2026 20:56:40 +0000 Subject: [PATCH 3/4] fix: chain underlying exception on SPIFFE polling timeout --- .../google-auth/google/auth/transport/_mtls_helper.py | 10 ++++++---- .../google-auth/tests/transport/test__mtls_helper.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index e6e814d8a974..9ec199926d71 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -597,13 +597,14 @@ def _extract_workload_paths_from_config(data, absolute_path): def _poll_spiffe_workload_credentials(spiffe_config_path): """Polls up to 30s for SPIFFE workload config, cert, and key files to be ready.""" + last_error = None for attempt, interval in enumerate(_agent_identity_utils._POLLING_INTERVALS): try: data = _load_json_file(spiffe_config_path) except PermissionError: raise - except (OSError, exceptions.ClientCertError): - pass + except (OSError, exceptions.ClientCertError) as caught_exc: + last_error = caught_exc else: cert_path, key_path = _extract_workload_paths_from_config( data, spiffe_config_path @@ -618,12 +619,13 @@ def _poll_spiffe_workload_credentials(spiffe_config_path): return cert_path, key_path, spiffe_config_path, cert_data, key_data except PermissionError: raise - except (OSError, exceptions.ClientCertError): + except (OSError, exceptions.ClientCertError) as caught_exc: if not ( _agent_identity_utils._is_in_well_known_dir(cert_path) and _agent_identity_utils._is_in_well_known_dir(key_path) ): raise + last_error = caught_exc if attempt + 1 == _agent_identity_utils._FAST_POLL_CYCLES: _LOGGER.debug( @@ -636,7 +638,7 @@ def _poll_spiffe_workload_credentials(spiffe_config_path): "Identity certificate configuration or certificate/key files at {} were not ready after {}s".format( spiffe_config_path, _agent_identity_utils._TOTAL_TIMEOUT ) - ) + ) from last_error def _read_credential_bundle_file(bundle_path): diff --git a/packages/google-auth/tests/transport/test__mtls_helper.py b/packages/google-auth/tests/transport/test__mtls_helper.py index add16d73feb0..6dbfe2b65cad 100644 --- a/packages/google-auth/tests/transport/test__mtls_helper.py +++ b/packages/google-auth/tests/transport/test__mtls_helper.py @@ -2482,9 +2482,12 @@ def test_polling_timeout_waiting_for_config_logs_debug_once( ): config_file = spiffe_dir / "credentials.json" - with pytest.raises(exceptions.ClientCertError, match="not ready after 30s"): + with pytest.raises( + exceptions.ClientCertError, match="not ready after 30s" + ) as exc_info: _mtls_helper._get_workload_cert_and_key(str(config_file)) + assert isinstance(exc_info.value.__cause__, FileNotFoundError) assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) mock_debug.assert_called_once_with( "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", @@ -2506,9 +2509,13 @@ def test_polling_timeout_waiting_for_cert_and_key_fails_closed( environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) ) - with pytest.raises(exceptions.ClientCertError, match="not ready after 30s"): + with pytest.raises( + exceptions.ClientCertError, match="not ready after 30s" + ) as exc_info: _mtls_helper._get_workload_cert_and_key() + assert isinstance(exc_info.value.__cause__, FileNotFoundError) + assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) mock_debug.assert_called_once_with( "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", From e514435b7cbd17c46ccfe439e88d0708113d34b3 Mon Sep 17 00:00:00 2001 From: Negar Bayati Date: Fri, 30 Oct 2026 22:00:26 +0000 Subject: [PATCH 4/4] test: expand SPIFFE polling tests and update docstring --- .../google/auth/transport/_mtls_helper.py | 4 ++ .../tests/transport/test__mtls_helper.py | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index 9ec199926d71..d3cc25f4ccb0 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -926,6 +926,10 @@ def check_use_client_cert(): containing a "workload" section, or if no certificate config file is configured/present and the GKE credential bundle exists at /var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem. + When GOOGLE_API_CERTIFICATE_CONFIG points to the well-known SPIFFE + directory (/var/run/secrets/workload-spiffe-credentials), this returns True + without waiting for the file to exist on disk so startup polling can run + when credentials are loaded. Otherwise, it returns False. Returns: diff --git a/packages/google-auth/tests/transport/test__mtls_helper.py b/packages/google-auth/tests/transport/test__mtls_helper.py index 6dbfe2b65cad..c90bbd40ab99 100644 --- a/packages/google-auth/tests/transport/test__mtls_helper.py +++ b/packages/google-auth/tests/transport/test__mtls_helper.py @@ -2417,6 +2417,46 @@ def test_immediate_readiness_zero_sleep(self, mock_sleep, spiffe_dir, monkeypatc ) mock_sleep.assert_not_called() + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_missing_files_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_config(*args, **kwargs): + self._write_workload_config(config_file, cert_file, key_file) + + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + mock_sleep.side_effect = write_config + + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + assert mock_sleep.call_count == 1 + + cert_file.unlink() + key_file.unlink() + mock_sleep.reset_mock() + + def write_cert_and_key(*args, **kwargs): + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + + mock_sleep.side_effect = write_cert_and_key + + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + assert mock_sleep.call_count == 1 + @pytest.mark.parametrize("initial_config", ["", "{truncated"]) @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) def test_polling_config_retry_success( @@ -2583,6 +2623,14 @@ def test_external_cert_and_key_paths_not_polled( mock_sleep.assert_not_called() external_cert.write_bytes(pytest.public_cert_bytes) + spiffe_cert = spiffe_dir / "certificates.pem" + spiffe_cert.write_bytes(pytest.public_cert_bytes) + self._write_workload_config(config_file, spiffe_cert, external_key) + with pytest.raises(FileNotFoundError): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + self._write_workload_config(config_file, external_cert, external_key) external_key.write_bytes(pytest.private_key_bytes) assert _mtls_helper._get_workload_cert_and_key(str(config_file)) == ( pytest.public_cert_bytes,