diff --git a/packages/google-auth/google/auth/transport/_mtls_helper.py b/packages/google-auth/google/auth/transport/_mtls_helper.py index 7ff5b2a81d81..d3cc25f4ccb0 100644 --- a/packages/google-auth/google/auth/transport/_mtls_helper.py +++ b/packages/google-auth/google/auth/transport/_mtls_helper.py @@ -22,6 +22,7 @@ import subprocess import sys import tempfile +import time from os import environ, getenv, path from typing import Generator, List, Optional, Tuple, Union, cast from urllib.parse import urlsplit @@ -415,7 +416,14 @@ def _get_workload_cert_and_key( the certificate or key information. """ - cert_path, key_path, config_file_path = _get_workload_cert_and_key_paths( + spiffe_config_path = _get_spiffe_polling_config_path(certificate_config_path) + if spiffe_config_path is not None: + _, _, _, cert_data, key_data = _poll_spiffe_workload_credentials( + spiffe_config_path + ) + return cert_data, key_data + + cert_path, key_path, config_file_path = _get_non_spiffe_workload_cert_and_key_paths( certificate_config_path, include_context_aware ) @@ -484,32 +492,34 @@ def _get_cert_config_path(certificate_config_path=None, include_context_aware=Tr def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=True): + spiffe_config_path = _get_spiffe_polling_config_path(config_path) + if spiffe_config_path is not None: + cert_path, key_path, config_file_path, _, _ = _poll_spiffe_workload_credentials( + spiffe_config_path + ) + return cert_path, key_path, config_file_path + + return _get_non_spiffe_workload_cert_and_key_paths( + config_path, include_context_aware + ) + + +def _get_non_spiffe_workload_cert_and_key_paths( + config_path=None, include_context_aware=True +): absolute_path = _get_cert_config_path(config_path, include_context_aware) if absolute_path is None: return None, None, None data = _load_json_file(absolute_path) - - if ( - not isinstance(data, dict) - or "cert_configs" not in data - or not isinstance(data["cert_configs"], dict) - ): - raise exceptions.ClientCertError( - 'Certificate config file {} is in an invalid format, a "cert configs" object is expected'.format( - absolute_path - ) - ) - cert_configs = data["cert_configs"] + cert_path, key_path = _extract_workload_paths_from_config(data, absolute_path) # We return None, None if the expected workload fields are not present. # The certificate config might be present for other types of connections (e.g. gECC), # and we want to gracefully fallback to testing other mTLS configurations # like SecureConnect instead of throwing an exception. - if ( - not isinstance(cert_configs, dict) or "workload" not in cert_configs - ) and config_path is None: + if cert_path is None and key_path is None and config_path is None: default_home_path = path.expanduser( os.path.join( _cloud_sdk.get_config_path(), @@ -521,19 +531,53 @@ def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=Tru ) != os.path.normpath(absolute_path): try: home_data = _load_json_file(default_home_path) - if isinstance(home_data, dict): - home_cert_configs = home_data.get("cert_configs") - if ( - isinstance(home_cert_configs, dict) - and "workload" in home_cert_configs - ): - cert_configs = home_cert_configs - absolute_path = default_home_path except (exceptions.ClientCertError, OSError): - pass + home_data = None + if ( + isinstance(home_data, dict) + and isinstance(home_data.get("cert_configs"), dict) + and "workload" in home_data["cert_configs"] + ): + cert_path, key_path = _extract_workload_paths_from_config( + home_data, default_home_path + ) + return cert_path, key_path, default_home_path + + return cert_path, key_path, absolute_path + + +def _get_spiffe_polling_config_path(config_path=None): + """Returns the expanded SPIFFE config path if startup polling applies, else None.""" + if _check_use_client_cert_env() is False: + return None + candidate_path = ( + config_path + if config_path is not None + else environ.get(environment_vars.GOOGLE_API_CERTIFICATE_CONFIG) + ) + if not candidate_path: + return None + expanded_path = path.expanduser(candidate_path) + if _agent_identity_utils._is_in_well_known_dir(expanded_path): + return expanded_path + return None - if not isinstance(cert_configs, dict) or "workload" not in cert_configs: - return None, None, absolute_path + +def _extract_workload_paths_from_config(data, absolute_path): + """Validates certificate_config.json schema and returns (cert_path, key_path).""" + if ( + not isinstance(data, dict) + or "cert_configs" not in data + or not isinstance(data["cert_configs"], dict) + ): + raise exceptions.ClientCertError( + 'Certificate config file {} is in an invalid format, a "cert configs" object is expected'.format( + absolute_path + ) + ) + cert_configs = data["cert_configs"] + if "workload" not in cert_configs: + return None, None workload = cert_configs["workload"] if ( @@ -548,10 +592,53 @@ def _get_workload_cert_and_key_paths(config_path=None, include_context_aware=Tru absolute_path ) ) - cert_path = workload["cert_path"] - key_path = workload["key_path"] + return workload["cert_path"], workload["key_path"] - return cert_path, key_path, absolute_path + +def _poll_spiffe_workload_credentials(spiffe_config_path): + """Polls up to 30s for SPIFFE workload config, cert, and key files to be ready.""" + last_error = None + for attempt, interval in enumerate(_agent_identity_utils._POLLING_INTERVALS): + try: + data = _load_json_file(spiffe_config_path) + except PermissionError: + raise + except (OSError, exceptions.ClientCertError) as caught_exc: + last_error = caught_exc + else: + cert_path, key_path = _extract_workload_paths_from_config( + data, spiffe_config_path + ) + if cert_path is None and key_path is None: + return None, None, spiffe_config_path, None, None + + cert_path = path.expanduser(cert_path) + key_path = path.expanduser(key_path) + try: + cert_data, key_data = _read_cert_and_key_files(cert_path, key_path) + return cert_path, key_path, spiffe_config_path, cert_data, key_data + except PermissionError: + raise + except (OSError, exceptions.ClientCertError) as caught_exc: + if not ( + _agent_identity_utils._is_in_well_known_dir(cert_path) + and _agent_identity_utils._is_in_well_known_dir(key_path) + ): + raise + last_error = caught_exc + + if attempt + 1 == _agent_identity_utils._FAST_POLL_CYCLES: + _LOGGER.debug( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + spiffe_config_path, + ) + time.sleep(interval) + + raise exceptions.ClientCertError( + "Identity certificate configuration or certificate/key files at {} were not ready after {}s".format( + spiffe_config_path, _agent_identity_utils._TOTAL_TIMEOUT + ) + ) from last_error def _read_credential_bundle_file(bundle_path): @@ -839,6 +926,10 @@ def check_use_client_cert(): containing a "workload" section, or if no certificate config file is configured/present and the GKE credential bundle exists at /var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem. + When GOOGLE_API_CERTIFICATE_CONFIG points to the well-known SPIFFE + directory (/var/run/secrets/workload-spiffe-credentials), this returns True + without waiting for the file to exist on disk so startup polling can run + when credentials are loaded. Otherwise, it returns False. Returns: @@ -849,6 +940,8 @@ def check_use_client_cert(): return env_override # Auto-enablement checks (when GOOGLE_API_USE_CLIENT_CERTIFICATE is not set) + if _get_spiffe_polling_config_path() is not None: + return True # Check if a workload config file exists. cert_path = _get_cert_config_path(include_context_aware=True) diff --git a/packages/google-auth/google/auth/transport/mtls.py b/packages/google-auth/google/auth/transport/mtls.py index db6d47daf688..a60603e36fb4 100644 --- a/packages/google-auth/google/auth/transport/mtls.py +++ b/packages/google-auth/google/auth/transport/mtls.py @@ -42,12 +42,15 @@ def has_default_client_cert_source(include_context_aware=True): Returns: bool: indicating if the default client cert source exists. """ + if _mtls_helper._get_spiffe_polling_config_path() is not None: + return True + try: ( cert_path, key_path, config_file_path, - ) = _mtls_helper._get_workload_cert_and_key_paths( + ) = _mtls_helper._get_non_spiffe_workload_cert_and_key_paths( None, include_context_aware=include_context_aware ) except (exceptions.ClientCertError, OSError): diff --git a/packages/google-auth/tests/transport/test__mtls_helper.py b/packages/google-auth/tests/transport/test__mtls_helper.py index c4e5b52c5973..c90bbd40ab99 100644 --- a/packages/google-auth/tests/transport/test__mtls_helper.py +++ b/packages/google-auth/tests/transport/test__mtls_helper.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import re import sys @@ -23,7 +24,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec -from google.auth import environment_vars, exceptions +from google.auth import _agent_identity_utils, environment_vars, exceptions from google.auth.transport import _mtls_helper if not hasattr(os, "MFD_CLOEXEC"): @@ -683,6 +684,21 @@ def load_json_side_effect(path): ) mock_read_cert_and_key_files.assert_not_called() + def test_no_workload_fallback_to_home_without_workload(self, tmp_path, monkeypatch): + ecp_config = tmp_path / "ecp_config.json" + ecp_config.write_text('{"cert_configs": {"pkcs11": {}}}') + home_dir = tmp_path / "gcloud" + home_dir.mkdir() + (home_dir / "certificate_config.json").write_text('{"cert_configs": {}}') + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(ecp_config) + ) + monkeypatch.setattr( + _mtls_helper._cloud_sdk, "get_config_path", lambda: str(home_dir) + ) + + assert _mtls_helper._get_workload_cert_and_key() == (None, None) + @mock.patch("google.auth.transport._mtls_helper._load_json_file", autospec=True) # noqa: E501 @mock.patch( "google.auth.transport._mtls_helper._get_cert_config_path", @@ -2313,3 +2329,350 @@ def test_no_gke_fallback_when_implicit_config_has_no_workload( assert _mtls_helper._get_workload_cert_and_key() == (None, None) assert _mtls_helper.check_use_client_cert() is False mock_is_ready.assert_not_called() + + +class TestSpiffeWorkloadPolling(object): + @pytest.fixture + def spiffe_dir(self, tmp_path, monkeypatch): + well_known_dir = tmp_path / "workload-spiffe-credentials" + well_known_dir.mkdir() + monkeypatch.setattr( + _agent_identity_utils, + "_WELL_KNOWN_CERT_PATH", + str(well_known_dir / "certificates.pem"), + ) + return well_known_dir + + def _write_workload_config(self, config_file, cert_path, key_path): + config_file.write_text( + json.dumps( + { + "cert_configs": { + "workload": { + "cert_path": str(cert_path), + "key_path": str(key_path), + } + } + } + ) + ) + + def test_get_spiffe_polling_config_path_disabled_by_env( + self, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "credentials.json"), + ) + monkeypatch.setenv(environment_vars.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false") + assert _mtls_helper._get_spiffe_polling_config_path() is None + + def test_get_spiffe_polling_config_path_ignores_non_spiffe_and_context_aware( + self, spiffe_dir, tmp_path, monkeypatch + ): + monkeypatch.setenv( + environment_vars.CLOUDSDK_CONTEXT_AWARE_CERTIFICATE_CONFIG_FILE_PATH, + str(spiffe_dir / "credentials.json"), + ) + assert _mtls_helper._get_spiffe_polling_config_path() is None + assert ( + _mtls_helper._get_spiffe_polling_config_path( + str(tmp_path / "certificate_config.json") + ) + is None + ) + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_check_use_client_cert_spiffe_returns_true_without_polling( + self, mock_sleep, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "missing_credentials.json"), + ) + + assert _mtls_helper.check_use_client_cert() is True + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_immediate_readiness_zero_sleep(self, mock_sleep, spiffe_dir, monkeypatch): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + self._write_workload_config(config_file, cert_file, key_file) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + assert _mtls_helper._get_workload_cert_and_key_paths() == ( + str(cert_file), + str(key_file), + str(config_file), + ) + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_missing_files_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_config(*args, **kwargs): + self._write_workload_config(config_file, cert_file, key_file) + + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + mock_sleep.side_effect = write_config + + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + assert mock_sleep.call_count == 1 + + cert_file.unlink() + key_file.unlink() + mock_sleep.reset_mock() + + def write_cert_and_key(*args, **kwargs): + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + + mock_sleep.side_effect = write_cert_and_key + + assert _mtls_helper._get_workload_cert_and_key() == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + assert mock_sleep.call_count == 1 + + @pytest.mark.parametrize("initial_config", ["", "{truncated"]) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_config_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch, initial_config + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + config_file.write_text(initial_config) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_config(*args, **kwargs): + self._write_workload_config(config_file, cert_file, key_file) + + mock_sleep.side_effect = write_config + + actual_cert, actual_key = _mtls_helper._get_workload_cert_and_key() + assert actual_cert == pytest.public_cert_bytes + assert actual_key == pytest.private_key_bytes + assert mock_sleep.call_count == 1 + + @pytest.mark.parametrize( + "initial_cert,initial_key", + [ + (b"", b""), + (b"-----BEGIN CERTIFICATE-----\ntruncated", pytest.private_key_bytes), + (pytest.public_cert_bytes, b"-----BEGIN PRIVATE KEY-----\ntruncated"), + ], + ) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_cert_and_key_retry_success( + self, mock_sleep, spiffe_dir, monkeypatch, initial_cert, initial_key + ): + cert_file = spiffe_dir / "certificates.pem" + key_file = spiffe_dir / "private_key.pem" + config_file = spiffe_dir / "credentials.json" + self._write_workload_config(config_file, cert_file, key_file) + cert_file.write_bytes(initial_cert) + key_file.write_bytes(initial_key) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + def write_cert_and_key(*args, **kwargs): + cert_file.write_bytes(pytest.public_cert_bytes) + key_file.write_bytes(pytest.private_key_bytes) + + mock_sleep.side_effect = write_cert_and_key + + actual_cert, actual_key = _mtls_helper._get_workload_cert_and_key() + assert actual_cert == pytest.public_cert_bytes + assert actual_key == pytest.private_key_bytes + assert mock_sleep.call_count == 1 + + @mock.patch("google.auth.transport._mtls_helper._LOGGER.debug") + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_timeout_waiting_for_config_logs_debug_once( + self, mock_sleep, mock_debug, spiffe_dir + ): + config_file = spiffe_dir / "credentials.json" + + with pytest.raises( + exceptions.ClientCertError, match="not ready after 30s" + ) as exc_info: + _mtls_helper._get_workload_cert_and_key(str(config_file)) + + assert isinstance(exc_info.value.__cause__, FileNotFoundError) + assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) + mock_debug.assert_called_once_with( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + str(config_file), + ) + + @mock.patch("google.auth.transport._mtls_helper._LOGGER.debug") + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_polling_timeout_waiting_for_cert_and_key_fails_closed( + self, mock_sleep, mock_debug, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + self._write_workload_config( + config_file, + spiffe_dir / "certificates.pem", + spiffe_dir / "private_key.pem", + ) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with pytest.raises( + exceptions.ClientCertError, match="not ready after 30s" + ) as exc_info: + _mtls_helper._get_workload_cert_and_key() + + assert isinstance(exc_info.value.__cause__, FileNotFoundError) + + assert mock_sleep.call_count == len(_agent_identity_utils._POLLING_INTERVALS) + mock_debug.assert_called_once_with( + "Identity certificate files are taking longer than 5 seconds to mount at %s. Retrying...", + str(config_file), + ) + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_config_without_workload_section_does_not_poll( + self, mock_sleep, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + config_file.write_text('{"cert_configs": {}}') + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + assert _mtls_helper._get_workload_cert_and_key() == (None, None) + mock_sleep.assert_not_called() + + @pytest.mark.parametrize( + "invalid_config", + [ + "[]", + "{}", + '{"cert_configs": []}', + '{"cert_configs": {"workload": []}}', + '{"cert_configs": {"workload": {"cert_path": "", "key_path": "/key"}}}', + '{"cert_configs": {"workload": {"cert_path": "/cert", "key_path": ""}}}', + ], + ) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_invalid_config_schema_fails_immediately_without_polling( + self, mock_sleep, spiffe_dir, monkeypatch, invalid_config + ): + config_file = spiffe_dir / "credentials.json" + config_file.write_text(invalid_config) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with pytest.raises(exceptions.ClientCertError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_external_cert_and_key_paths_not_polled( + self, mock_sleep, spiffe_dir, tmp_path + ): + external_cert = tmp_path / "external_cert.pem" + external_key = tmp_path / "external_key.pem" + config_file = spiffe_dir / "credentials.json" + self._write_workload_config(config_file, external_cert, external_key) + + with pytest.raises(FileNotFoundError): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + external_cert.write_bytes(b"invalid pem") + with pytest.raises( + exceptions.ClientCertError, + match="Certificate file .* is in an invalid format", + ): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + external_cert.write_bytes(pytest.public_cert_bytes) + spiffe_cert = spiffe_dir / "certificates.pem" + spiffe_cert.write_bytes(pytest.public_cert_bytes) + self._write_workload_config(config_file, spiffe_cert, external_key) + with pytest.raises(FileNotFoundError): + _mtls_helper._get_workload_cert_and_key(str(config_file)) + mock_sleep.assert_not_called() + + self._write_workload_config(config_file, external_cert, external_key) + external_key.write_bytes(pytest.private_key_bytes) + assert _mtls_helper._get_workload_cert_and_key(str(config_file)) == ( + pytest.public_cert_bytes, + pytest.private_key_bytes, + ) + mock_sleep.assert_not_called() + + @mock.patch("builtins.open", side_effect=PermissionError("denied")) + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_config_permission_error_propagates( + self, mock_sleep, mock_open, spiffe_dir, monkeypatch + ): + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, + str(spiffe_dir / "credentials.json"), + ) + + with pytest.raises(PermissionError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() + + @mock.patch("google.auth.transport._mtls_helper.time.sleep", autospec=True) + def test_cert_and_key_permission_error_propagates( + self, mock_sleep, spiffe_dir, monkeypatch + ): + config_file = spiffe_dir / "credentials.json" + self._write_workload_config( + config_file, + spiffe_dir / "certificates.pem", + spiffe_dir / "private_key.pem", + ) + monkeypatch.setenv( + environment_vars.GOOGLE_API_CERTIFICATE_CONFIG, str(config_file) + ) + + with open(config_file) as config_handle: + with mock.patch( + "builtins.open", + side_effect=[config_handle, PermissionError("denied")], + ): + with pytest.raises(PermissionError): + _mtls_helper._get_workload_cert_and_key() + + mock_sleep.assert_not_called() diff --git a/packages/google-auth/tests/transport/test_mtls.py b/packages/google-auth/tests/transport/test_mtls.py index 5899a1b1c73b..1bee2fbbe499 100644 --- a/packages/google-auth/tests/transport/test_mtls.py +++ b/packages/google-auth/tests/transport/test_mtls.py @@ -558,3 +558,20 @@ def test_has_default_client_cert_source_malformed_config_no_gke_fallback( assert mtls.has_default_client_cert_source(include_context_aware=False) is True mock_is_ready.assert_not_called() + + +@mock.patch( + "google.auth.transport._mtls_helper._get_non_spiffe_workload_cert_and_key_paths", + autospec=True, +) +@mock.patch( + "google.auth.transport._mtls_helper._get_spiffe_polling_config_path", + return_value="/var/run/secrets/workload-spiffe-credentials/credentials.json", + autospec=True, +) +def test_has_default_client_cert_source_spiffe_returns_true_without_polling( + mock_spiffe_path, mock_non_spiffe_paths +): + assert mtls.has_default_client_cert_source() is True + mock_spiffe_path.assert_called_once_with() + mock_non_spiffe_paths.assert_not_called()