From c399f7aa80e0bb7bdc95e9003a7ed152288265da Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Thu, 22 Oct 2026 18:16:11 +0200 Subject: [PATCH 01/10] fix: ai privacy mode leaks --- .sampo/changesets/ai-privacy-mode-leaks.md | 12 + posthog/ai/anthropic/anthropic_converter.py | 4 +- posthog/ai/gemini/_shared.py | 12 +- posthog/ai/gemini/gemini.py | 6 +- posthog/ai/gemini/gemini_async.py | 6 +- posthog/ai/gemini/gemini_converter.py | 4 +- posthog/ai/langchain/callbacks.py | 69 ++- posthog/ai/langchain/middleware.py | 12 +- posthog/ai/openai/openai_converter.py | 8 +- posthog/ai/openai_agents/processor.py | 36 +- posthog/ai/otel/_privacy.py | 120 ++++ posthog/ai/otel/exporter.py | 17 +- posthog/ai/otel/processor.py | 20 +- posthog/ai/utils.py | 216 ++++++- posthog/test/ai/gemini/test_gemini_parity.py | 20 +- posthog/test/ai/langchain/test_callbacks.py | 22 +- posthog/test/ai/langchain/test_middleware.py | 2 +- posthog/test/ai/test_privacy_mode.py | 584 +++++++++++++++++++ references/public_api_snapshot.txt | 6 +- 19 files changed, 1093 insertions(+), 83 deletions(-) create mode 100644 .sampo/changesets/ai-privacy-mode-leaks.md create mode 100644 posthog/ai/otel/_privacy.py create mode 100644 posthog/test/ai/test_privacy_mode.py diff --git a/.sampo/changesets/ai-privacy-mode-leaks.md b/.sampo/changesets/ai-privacy-mode-leaks.md new file mode 100644 index 000000000..8eb4ececd --- /dev/null +++ b/.sampo/changesets/ai-privacy-mode-leaks.md @@ -0,0 +1,12 @@ +--- +pypi/posthog: patch +--- + +Privacy mode now covers error text and the remaining content paths in the AI integrations. + +- `$ai_error` no longer carries the exception message when privacy mode is on. Provider errors and validation errors often quote the prompt or the model output, so the SDK now sends a summary built only from structured fields: the class name, HTTP status, provider error code, `param`, request ID and the class names of chained causes (for example `BadRequestError: 400 invalid_request_error param=messages[3].content request_id=req_123`). This applies to the OpenAI, Anthropic and Gemini wrappers (including Gemini embeddings), LangChain generations, spans and traces, and OpenAI Agents spans. +- The LangChain callback handler no longer sends an `$exception` event for a failed run when privacy mode is on. +- `$ai_tools` masks the `headers` and `authorization` of MCP tool definitions (`type: "mcp"`) with `[REDACTED]`, with or without privacy mode. The tools you pass in are not modified. +- Gemini: a per-call `posthog_privacy_mode=False` no longer turns off privacy mode set on the client, matching the OpenAI and Anthropic wrappers. +- OpenAI Agents: privacy mode now redacts `$ai_trace_metadata` and the `prompt` and `instructions` in transcription and speech span `model_config`. +- OpenTelemetry: `PostHogSpanProcessor` and `PostHogTraceExporter` accept `privacy_mode` and `client`, and also honor the client's (or the module-level) `privacy_mode`. When on, exported spans drop prompt, completion, tool argument and tool result attributes, non-exception span events, exception messages and stack traces, and the span status description. diff --git a/posthog/ai/anthropic/anthropic_converter.py b/posthog/ai/anthropic/anthropic_converter.py index 650c5587f..1e476c5c5 100644 --- a/posthog/ai/anthropic/anthropic_converter.py +++ b/posthog/ai/anthropic/anthropic_converter.py @@ -18,7 +18,7 @@ TokenUsage, ToolInProgress, ) -from posthog.ai.utils import serialize_raw_usage +from posthog.ai.utils import _redact_mcp_tool_credentials, serialize_raw_usage def format_anthropic_response(response: Any) -> List[FormattedMessage]: @@ -153,7 +153,7 @@ def extract_anthropic_tools(kwargs: Dict[str, Any]) -> Optional[Any]: Tool definitions if present, None otherwise """ - return kwargs.get("tools", None) + return _redact_mcp_tool_credentials(kwargs.get("tools", None)) def format_anthropic_streaming_content( diff --git a/posthog/ai/gemini/_shared.py b/posthog/ai/gemini/_shared.py index e1036929d..0886ea5fd 100644 --- a/posthog/ai/gemini/_shared.py +++ b/posthog/ai/gemini/_shared.py @@ -16,6 +16,7 @@ _capture_ai_event, capture_streaming_event, finalize_ai_content, + _format_ai_error, merge_system_prompt, with_privacy_mode, ) @@ -175,11 +176,8 @@ def _merge_posthog_params( if call_distinct_id is not None else self._default_distinct_id ) - privacy_mode = ( - call_privacy_mode - if call_privacy_mode is not None - else self._default_privacy_mode - ) + # A call can add redaction but never lift the client default. + privacy_mode = self._default_privacy_mode or call_privacy_mode is True groups = call_groups if call_groups is not None else self._default_groups properties = dict(self._default_properties) @@ -266,7 +264,9 @@ def _capture_embedding_outcome( if error: event_properties["$ai_is_error"] = True - event_properties["$ai_error"] = str(error) + event_properties["$ai_error"] = _format_ai_error( + self._ph_client, privacy_mode, error + ) if distinct_id is None: event_properties["$process_person_profile"] = False diff --git a/posthog/ai/gemini/gemini.py b/posthog/ai/gemini/gemini.py index c5006544a..dad87da9a 100644 --- a/posthog/ai/gemini/gemini.py +++ b/posthog/ai/gemini/gemini.py @@ -87,7 +87,7 @@ def __init__( posthog_client: PostHog client for tracking usage posthog_distinct_id: Default distinct ID for all calls (can be overridden per call) posthog_properties: Default properties for all calls (can be overridden per call) - posthog_privacy_mode: Default privacy mode for all calls (can be overridden per call) + posthog_privacy_mode: Default privacy mode for all calls (a call can enable it, not disable it) posthog_groups: Default groups for all calls (can be overridden per call) **kwargs: Additional arguments (for future compatibility) """ @@ -212,7 +212,7 @@ def generate_content( posthog_distinct_id: ID to associate with the usage event (overrides client default) posthog_trace_id: Trace UUID for linking events (auto-generated if not provided) posthog_properties: Extra properties to include in the event (merged with client defaults) - posthog_privacy_mode: Whether to redact sensitive information (overrides client default) + posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True) posthog_groups: Group analytics properties (overrides client default) **kwargs: Arguments passed to Gemini's generate_content """ @@ -384,7 +384,7 @@ def embed_content( posthog_distinct_id: ID to associate with the usage event (overrides client default) posthog_trace_id: Trace UUID for linking events (auto-generated if not provided) posthog_properties: Extra properties to include in the event (merged with client defaults) - posthog_privacy_mode: Whether to redact sensitive information (overrides client default) + posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True) posthog_groups: Group analytics properties (overrides client default) **kwargs: Arguments passed to Gemini's embed_content (e.g., config) """ diff --git a/posthog/ai/gemini/gemini_async.py b/posthog/ai/gemini/gemini_async.py index 0580b285b..6c70efd9b 100644 --- a/posthog/ai/gemini/gemini_async.py +++ b/posthog/ai/gemini/gemini_async.py @@ -82,7 +82,7 @@ def __init__( posthog_client: PostHog client for tracking usage posthog_distinct_id: Default distinct ID for all calls (can be overridden per call) posthog_properties: Default properties for all calls (can be overridden per call) - posthog_privacy_mode: Default privacy mode for all calls (can be overridden per call) + posthog_privacy_mode: Default privacy mode for all calls (a call can enable it, not disable it) posthog_groups: Default groups for all calls (can be overridden per call) **kwargs: Additional arguments (for future compatibility) """ @@ -196,7 +196,7 @@ async def generate_content( posthog_distinct_id: ID to associate with the usage event (overrides client default) posthog_trace_id: Trace UUID for linking events (auto-generated if not provided) posthog_properties: Extra properties to include in the event (merged with client defaults) - posthog_privacy_mode: Whether to redact sensitive information (overrides client default) + posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True) posthog_groups: Group analytics properties (overrides client default) **kwargs: Arguments passed to Gemini's generate_content """ @@ -371,7 +371,7 @@ async def embed_content( posthog_distinct_id: ID to associate with the usage event (overrides client default) posthog_trace_id: Trace UUID for linking events (auto-generated if not provided) posthog_properties: Extra properties to include in the event (merged with client defaults) - posthog_privacy_mode: Whether to redact sensitive information (overrides client default) + posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True) posthog_groups: Group analytics properties (overrides client default) **kwargs: Arguments passed to Gemini's embed_content (e.g., config) """ diff --git a/posthog/ai/gemini/gemini_converter.py b/posthog/ai/gemini/gemini_converter.py index 669f63295..ae4522746 100644 --- a/posthog/ai/gemini/gemini_converter.py +++ b/posthog/ai/gemini/gemini_converter.py @@ -13,7 +13,7 @@ FormattedMessage, TokenUsage, ) -from posthog.ai.utils import serialize_raw_usage +from posthog.ai.utils import _redact_mcp_tool_credentials, serialize_raw_usage _MEDIA_KINDS = {"image": "image", "video": "video", "audio": "audio"} _BARE_PART_KEYS = { @@ -364,7 +364,7 @@ def extract_gemini_tools(kwargs: Dict[str, Any]) -> Optional[Any]: """ if "config" in kwargs and hasattr(kwargs["config"], "tools"): - return kwargs["config"].tools + return _redact_mcp_tool_credentials(kwargs["config"].tools) return None diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 5f996fb6c..7c07d9314 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -47,7 +47,10 @@ _capture_ai_event, _extract_cache_creation_ttl_breakdown, finalize_ai_content, + _format_ai_error, get_model_params, + _privacy_mode_enabled, + _redact_mcp_tool_credentials, _responses_stop_reason, with_privacy_mode, ) @@ -560,14 +563,12 @@ def _capture_trace_or_span( event_properties.update(self._properties) if isinstance(outputs, BaseException): - event_properties["$ai_error"] = _stringify_exception(outputs) + event_properties["$ai_error"] = _format_ai_error( + self._ph_client, self._privacy_mode, outputs, _stringify_exception + ) event_properties["$ai_is_error"] = True - event_properties = _capture_exception_and_update_properties( - self._ph_client, - outputs, - self._distinct_id, - self._groups, - event_properties, + event_properties = self._capture_exception_and_update_properties( + outputs, event_properties ) elif outputs is not None: @@ -588,6 +589,25 @@ def _capture_trace_or_span( groups=self._groups, ) + def _capture_exception_and_update_properties( + self, exception: BaseException, event_properties: Dict[str, Any] + ) -> Dict[str, Any]: + # An `$exception` event carries the message and stack, which echo prompts and output. + if not self._ph_client.enable_exception_autocapture or _privacy_mode_enabled( + self._ph_client, self._privacy_mode + ): + return event_properties + + exception_id = self._ph_client.capture_exception( + exception, + distinct_id=self._distinct_id, + groups=self._groups, + properties=event_properties, + ) + if exception_id: + event_properties["$exception_event_id"] = exception_id + return event_properties + def _pop_run_and_capture_generation( self, run_id: UUID, @@ -657,7 +677,7 @@ def _capture_generation( event_properties.update(run.posthog_properties) if run.tools: - event_properties["$ai_tools"] = run.tools + event_properties["$ai_tools"] = _redact_mcp_tool_credentials(run.tools) if self._properties: event_properties.update(self._properties) @@ -667,15 +687,13 @@ def _capture_generation( if isinstance(output, BaseException): event_properties["$ai_http_status"] = _get_http_status(output) - event_properties["$ai_error"] = _stringify_exception(output) + event_properties["$ai_error"] = _format_ai_error( + self._ph_client, self._privacy_mode, output, _stringify_exception + ) event_properties["$ai_is_error"] = True - event_properties = _capture_exception_and_update_properties( - self._ph_client, - output, - self._distinct_id, - self._groups, - event_properties, + event_properties = self._capture_exception_and_update_properties( + output, event_properties ) else: # Add usage @@ -1011,27 +1029,6 @@ def _parse_usage( return llm_usage -def _capture_exception_and_update_properties( - client: Client, - exception: BaseException, - distinct_id: Optional[Union[str, int, UUID]], - groups: Optional[Dict[str, Any]], - event_properties: Dict[str, Any], -): - if client.enable_exception_autocapture: - exception_id = client.capture_exception( - exception, - distinct_id=distinct_id, - groups=groups, - properties=event_properties, - ) - - if exception_id: - event_properties["$exception_event_id"] = exception_id - - return event_properties - - def _get_http_status(error: BaseException) -> int: # OpenAI: https://github.com/openai/openai-python/blob/main/src/openai/_exceptions.py # Anthropic: https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/_exceptions.py diff --git a/posthog/ai/langchain/middleware.py b/posthog/ai/langchain/middleware.py index 34783747e..8cf282a87 100644 --- a/posthog/ai/langchain/middleware.py +++ b/posthog/ai/langchain/middleware.py @@ -21,6 +21,7 @@ from langgraph.errors import GraphBubbleUp from langgraph.types import Command from ...client import Client +from ..utils import _with_privacy_safe_message from .callbacks import ( CallbackHandler, GenerationMetadata, @@ -329,13 +330,10 @@ def _privacy_mode_enabled(self) -> bool: or getattr(self._callback._ph_client, "privacy_mode", False) ) - def _returned_tool_error(self, response: ToolMessage) -> RuntimeError: - message = ( - "Tool returned an error" - if self._privacy_mode_enabled() - else str(response.content) - ) - return RuntimeError(message) + def _returned_tool_error(self, response: ToolMessage) -> BaseException: + if self._privacy_mode_enabled(): + return _with_privacy_safe_message(RuntimeError("Tool returned an error")) + return RuntimeError(str(response.content)) @staticmethod def _normalize_tool(tool: Any) -> dict[str, Any]: diff --git a/posthog/ai/openai/openai_converter.py b/posthog/ai/openai/openai_converter.py index e0f397ff6..d127badfe 100644 --- a/posthog/ai/openai/openai_converter.py +++ b/posthog/ai/openai/openai_converter.py @@ -17,7 +17,11 @@ FormattedTextContent, TokenUsage, ) -from posthog.ai.utils import _responses_stop_reason, serialize_raw_usage +from posthog.ai.utils import ( + _responses_stop_reason, + _redact_mcp_tool_credentials, + serialize_raw_usage, +) def _item_attr(item: Any, name: str, default: Any = None) -> Any: @@ -299,7 +303,7 @@ def extract_openai_tools(kwargs: Dict[str, Any]) -> Optional[Any]: # Check for tools parameter (newer API) if "tools" in kwargs: - return kwargs["tools"] + return _redact_mcp_tool_credentials(kwargs["tools"]) # Check for functions parameter (older API) if "functions" in kwargs: diff --git a/posthog/ai/openai_agents/processor.py b/posthog/ai/openai_agents/processor.py index 8bdea6c7e..e3e8450e2 100644 --- a/posthog/ai/openai_agents/processor.py +++ b/posthog/ai/openai_agents/processor.py @@ -25,12 +25,16 @@ from posthog.ai.utils import ( _capture_processor_event, finalize_ai_content, + _format_error_summary, + _privacy_mode_enabled, with_privacy_mode as _with_privacy_mode, ) from posthog.client import Client log = logging.getLogger("posthog") +_AUDIO_MODEL_CONFIG_CONTENT_KEYS = frozenset({"prompt", "instructions"}) + def _parse_iso_timestamp(iso_str: Optional[str]) -> Optional[float]: """Parse ISO timestamp to Unix timestamp.""" @@ -125,6 +129,15 @@ def _with_privacy_mode(self, value: Any) -> Any: """Apply privacy mode redaction if enabled.""" return _with_privacy_mode(self._client, self._privacy_mode, value) + def _privacy_enabled(self) -> bool: + return _privacy_mode_enabled(self._client, self._privacy_mode) + + @staticmethod + def _summarize_span_error(error_info: Any) -> str: + # Agents SDK span errors are plain dicts, not exceptions: only a type name is structured. + error_type = error_info.get("type") if isinstance(error_info, dict) else None + return _format_error_summary(error_type) + def _evict_stale_entries(self) -> None: """Evict oldest entries if dicts exceed max size to prevent unbounded growth.""" if len(self._span_start_times) > self._max_tracked_entries: @@ -226,7 +239,9 @@ def on_trace_end(self, trace: Trace) -> None: # Include trace metadata if present if metadata: - properties["$ai_trace_metadata"] = _ensure_serializable(metadata) + properties["$ai_trace_metadata"] = self._with_privacy_mode( + _ensure_serializable(metadata) + ) if distinct_id is None: properties["$process_person_profile"] = False @@ -313,7 +328,11 @@ def on_span_end(self, span: Span[Any]) -> None: error_properties = { "$ai_is_error": True, - "$ai_error": error_message, + "$ai_error": ( + self._summarize_span_error(error_info) + if self._privacy_enabled() + else error_message + ), "$ai_error_type": error_type, } @@ -778,7 +797,9 @@ def _handle_audio_span( # Add model config if available (pass-through property) if hasattr(span_data, "model_config") and span_data.model_config: - properties["model_config"] = _ensure_serializable(span_data.model_config) + properties["model_config"] = self._redact_audio_model_config( + _ensure_serializable(span_data.model_config) + ) # Add time to first audio byte for speech spans (pass-through property) if hasattr(span_data, "first_content_at") and span_data.first_content_at: @@ -825,6 +846,15 @@ def _handle_audio_span( self._capture_event("$ai_span", properties, distinct_id) + def _redact_audio_model_config(self, model_config: Any) -> Any: + # The STT prompt and TTS instructions are user-authored text. + if not isinstance(model_config, dict) or not self._privacy_enabled(): + return model_config + return { + key: (None if key in _AUDIO_MODEL_CONFIG_CONTENT_KEYS else value) + for key, value in model_config.items() + } + def _handle_mcp_span( self, span_data: MCPListToolsSpanData, diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py new file mode 100644 index 000000000..af6c5f264 --- /dev/null +++ b/posthog/ai/otel/_privacy.py @@ -0,0 +1,120 @@ +"""Privacy mode for OpenTelemetry spans exported to PostHog.""" + +from typing import TYPE_CHECKING, Optional + +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace.status import Status + +if TYPE_CHECKING: + from ...client import Client + +# Attributes PostHog ingestion maps into $ai_input, $ai_output_choices, $ai_input_state, +# $ai_output_state or $ai_error, plus the raw payloads it strips. Tool definitions +# ($ai_tools) are not content and stay, as in the SDK wrappers. +_CONTENT_ATTRIBUTES = frozenset( + { + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "anthropic.tool_result.content", + "events", + "pydantic_ai.all_messages", + "final_result", + "tool_arguments", + "tool_response", + "traceloop.entity.input", + "traceloop.entity.output", + "ai.prompt", + "ai.prompt.messages", + "ai.response.text", + "ai.response.object", + "ai.response.toolCalls", + "ai.response.reasoning", + "ai.toolCall.args", + "ai.toolCall.result", + "ai.value", + "ai.values", + "input.value", + "output.value", + "exception.message", + "exception.stacktrace", + "error.message", + } +) + +# Matched as the attribute itself or as a dotted prefix, e.g. `gen_ai.prompt.0.content`. +_CONTENT_ATTRIBUTE_ROOTS = ( + "gen_ai.prompt", + "gen_ai.completion", + "llm.prompts", + "llm.input_messages", + "llm.output_messages", +) + +_EXCEPTION_EVENT_NAME = "exception" + + +def _is_content_attribute(key: str) -> bool: + """Whether a span or event attribute can carry prompts, completions, tool data or error text.""" + if key in _CONTENT_ATTRIBUTES: + return True + return any( + key == root or key.startswith(f"{root}.") for root in _CONTENT_ATTRIBUTE_ROOTS + ) + + +def _otel_privacy_mode_enabled(privacy_mode: bool, client: Optional["Client"]) -> bool: + """Explicit flag, else the given client's `privacy_mode`, else the module-level configuration.""" + if privacy_mode: + return True + if client is not None: + return getattr(client, "privacy_mode", False) is True + # Imported here so later changes to the module-level settings still apply. + from ... import default_client, privacy_mode as global_privacy_mode + + return bool(global_privacy_mode) or ( + getattr(default_client, "privacy_mode", False) is True + ) + + +def _redact_span(span: ReadableSpan) -> ReadableSpan: + """ + Return a copy of ``span`` without content attributes, non-exception events, + exception messages and stack traces, or the status description. + """ + attributes = { + key: value + for key, value in (span.attributes or {}).items() + if not _is_content_attribute(key) + } + # Other span events (older GenAI semconv messages, framework logs) carry + # content and PostHog ingestion only reads exception events. + events = [ + Event( + name=event.name, + attributes={ + key: value + for key, value in (event.attributes or {}).items() + if not _is_content_attribute(key) + }, + timestamp=event.timestamp, + ) + for event in span.events + if event.name == _EXCEPTION_EVENT_NAME + ] + return ReadableSpan( + name=span.name, + context=span.get_span_context(), + parent=span.parent, + resource=span.resource, + attributes=attributes, + events=events, + links=span.links, + kind=span.kind, + status=Status(span.status.status_code), + start_time=span.start_time, + end_time=span.end_time, + instrumentation_scope=span.instrumentation_scope, + ) diff --git a/posthog/ai/otel/exporter.py b/posthog/ai/otel/exporter.py index ec6e387ff..b6cfbe83b 100644 --- a/posthog/ai/otel/exporter.py +++ b/posthog/ai/otel/exporter.py @@ -5,15 +5,19 @@ SpanExporter (e.g. as an argument to BatchSpanProcessor). """ -from typing import Optional, Sequence +from typing import TYPE_CHECKING, Optional, Sequence from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from ..gateway import warn_if_posthog_ai_gateway_otel_attributes +from ._privacy import _otel_privacy_mode_enabled, _redact_span from .spans import DEFAULT_HOST, is_ai_span +if TYPE_CHECKING: + from ...client import Client + class PostHogTraceExporter(SpanExporter): """Span exporter that filters AI spans and forwards them to PostHog. @@ -41,14 +45,23 @@ def __init__( self, api_key: str, host: str = DEFAULT_HOST, + *, + privacy_mode: bool = False, + client: Optional["Client"] = None, ): """ Args: api_key: PostHog project API key. host: PostHog host URL. Defaults to US cloud. + privacy_mode: Drop prompts, completions, tool arguments and results, + and error messages from exported spans. + client: PostHog client whose ``privacy_mode`` also applies. Without + one, the module-level ``posthog.privacy_mode`` setting applies. """ self._api_key = api_key self._host = host.rstrip("/") + self._privacy_mode = privacy_mode + self._client = client self._exporter = OTLPSpanExporter( endpoint=f"{self._host}/i/v0/ai/otel", @@ -70,6 +83,8 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: return SpanExportResult.SUCCESS for span in ai_spans: warn_if_posthog_ai_gateway_otel_attributes(span.attributes) + if _otel_privacy_mode_enabled(self._privacy_mode, self._client): + ai_spans = [_redact_span(span) for span in ai_spans] return self._exporter.export(ai_spans) def shutdown(self) -> None: diff --git a/posthog/ai/otel/processor.py b/posthog/ai/otel/processor.py index 31073c54b..3024b8efe 100644 --- a/posthog/ai/otel/processor.py +++ b/posthog/ai/otel/processor.py @@ -9,7 +9,7 @@ import os import threading import weakref -from typing import Dict, Optional, Tuple +from typing import TYPE_CHECKING, Dict, Optional, Tuple from opentelemetry.context import Context from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor @@ -22,8 +22,11 @@ capture_request_context, with_request_context, ) +from ._privacy import _otel_privacy_mode_enabled, _redact_span from .spans import DEFAULT_HOST, is_ai_span +if TYPE_CHECKING: + from ...client import Client log = logging.getLogger("posthog") @@ -89,14 +92,23 @@ def __init__( self, api_key: str, host: str = DEFAULT_HOST, + *, + privacy_mode: bool = False, + client: Optional["Client"] = None, ): """ Args: api_key: PostHog project API key. host: PostHog host URL. Defaults to US cloud. + privacy_mode: Drop prompts, completions, tool arguments and results, + and error messages from exported spans. + client: PostHog client whose ``privacy_mode`` also applies. Without + one, the module-level ``posthog.privacy_mode`` setting applies. """ self._api_key = api_key self._host = host.rstrip("/") + self._privacy_mode = privacy_mode + self._client = client exporter = OTLPSpanExporter( endpoint=f"{self._host}/i/v0/ai/otel", @@ -164,6 +176,12 @@ def on_end(self, span: ReadableSpan) -> None: span = with_request_context(span, captured) except Exception: log.debug("Failed to add PostHog context to span", exc_info=True) + if _otel_privacy_mode_enabled(self._privacy_mode, self._client): + try: + span = _redact_span(span) + except Exception: + log.debug("Failed to redact span, dropping it", exc_info=True) + return self._processor.on_end(span) def shutdown(self) -> None: diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 5f2b956e0..b4a85bd80 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1,5 +1,7 @@ import asyncio +import copy import logging +import re import time import uuid from typing import Any, Callable, Dict, List, Optional, Tuple, cast @@ -489,7 +491,7 @@ def call_llm_and_track_usage( ) # default to 0 becuase its likely an SDK error error_params = { "$ai_is_error": True, - "$ai_error": exc.__str__(), + "$ai_error": _format_ai_error(ph_client, posthog_privacy_mode, exc), } # TODO: Add exception capture for OpenAI/Anthropic/Gemini wrappers when # enable_exception_autocapture is True, similar to LangChain callbacks. @@ -663,7 +665,7 @@ async def call_llm_and_track_usage_async( ) # default to 0 because its likely an SDK error error_params = { "$ai_is_error": True, - "$ai_error": exc.__str__(), + "$ai_error": _format_ai_error(ph_client, posthog_privacy_mode, exc), } # TODO: Add exception capture for OpenAI/Anthropic/Gemini wrappers when # enable_exception_autocapture is True, similar to LangChain callbacks. @@ -808,12 +810,211 @@ def finalize_ai_content(value: Any, ph_client: Any = None) -> Any: return redact_media(value, ph_client=ph_client) +def _privacy_mode_enabled(ph_client: Any, privacy_mode: Optional[bool]) -> bool: + return bool(getattr(ph_client, "privacy_mode", False) or privacy_mode) + + def with_privacy_mode(ph_client: PostHogClient, privacy_mode: bool, value: Any): - if getattr(ph_client, "privacy_mode", False) or privacy_mode: + if _privacy_mode_enabled(ph_client, privacy_mode): return None return value +_ERROR_CLASS_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]{0,127}$") +_ERROR_PROVIDER_CODE_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$") +_ERROR_PARAM_RE = re.compile(r"^[A-Za-z0-9_.\[\]-]{1,128}$") +_ERROR_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") +_ERROR_CODE_FIELDS = ("code", "type", "status") +_ERROR_STATUS_FIELDS = ("status_code", "status", "code") +_MAX_ERROR_CAUSES = 3 +_FALLBACK_ERROR_CLASS_NAME = "Exception" + + +def _read_error_field(source: Any, key: str) -> Any: + if source is None: + return None + try: + if isinstance(source, dict): + return source.get(key) + return getattr(source, key, None) + except Exception: + return None + + +def _matching(value: object, grammar: "re.Pattern[str]") -> Optional[str]: + if isinstance(value, str) and grammar.fullmatch(value): + return value + return None + + +def _first_matching(values: List[Any], grammar: "re.Pattern[str]") -> Optional[str]: + for value in values: + match = _matching(value, grammar) + if match is not None: + return match + return None + + +def _error_class_name(name: object) -> str: + return _matching(name, _ERROR_CLASS_NAME_RE) or _FALLBACK_ERROR_CLASS_NAME + + +def _error_http_status(exc: BaseException) -> Optional[int]: + for key in _ERROR_STATUS_FIELDS: + value = _read_error_field(exc, key) + if ( + isinstance(value, int) + and not isinstance(value, bool) + and 100 <= value <= 599 + ): + return value + return None + + +def _error_body(exc: BaseException) -> Any: + # Anthropic nests the detail as `{"type": "error", "error": {"type": ...}}`. + body = _read_error_field(exc, "body") + inner = _read_error_field(body, "error") + return inner if isinstance(inner, dict) else body + + +def _error_cause_names(exc: BaseException) -> List[str]: + names: List[str] = [] + cause = exc.__cause__ or exc.__context__ + while cause is not None and len(names) < _MAX_ERROR_CAUSES: + names.append(_error_class_name(type(cause).__name__)) + cause = cause.__cause__ or cause.__context__ + return names + + +def _format_error_summary( + name: object, + *, + status: Optional[int] = None, + codes: Optional[List[str]] = None, + param: Optional[str] = None, + request_id: Optional[str] = None, + causes: Optional[List[str]] = None, +) -> str: + """Render the privacy-safe `$ai_error` format shared with the JS SDK.""" + summary = _error_class_name(name) + if status is not None: + summary += f": {status}" + for code in dict.fromkeys(codes or []): + summary += f" {code}" + if param is not None: + summary += f" param={param}" + if request_id is not None: + summary += f" request_id={request_id}" + if causes: + summary += f" (caused by {', '.join(causes)})" + return summary + + +def _summarize_error_for_privacy(exc: BaseException) -> str: + """ + Describe an exception for `$ai_error` without its message, traceback or + arbitrary attributes, which routinely echo prompts and model output. + Only structured fields that pass a strict grammar are kept. + """ + try: + sources = [exc, _error_body(exc)] + codes = [ + code + for source in sources + for key in _ERROR_CODE_FIELDS + if ( + code := _matching( + _read_error_field(source, key), _ERROR_PROVIDER_CODE_RE + ) + ) + is not None + ] + return _format_error_summary( + type(exc).__name__, + status=_error_http_status(exc), + codes=codes, + param=_first_matching( + [_read_error_field(source, "param") for source in sources], + _ERROR_PARAM_RE, + ), + request_id=_matching( + _read_error_field(exc, "request_id"), _ERROR_REQUEST_ID_RE + ), + causes=_error_cause_names(exc), + ) + except Exception: + return _FALLBACK_ERROR_CLASS_NAME + + +_PRIVACY_SAFE_MESSAGE_ATTR = "_posthog_privacy_safe_message" + + +def _with_privacy_safe_message(exc: BaseException) -> BaseException: + """Mark an SDK-built exception whose message is fixed text, so privacy mode keeps it.""" + setattr(exc, _PRIVACY_SAFE_MESSAGE_ATTR, True) + return exc + + +def _format_ai_error( + ph_client: Any, + privacy_mode: Optional[bool], + exc: BaseException, + describe: Callable[[BaseException], str] = str, +) -> str: + """The `$ai_error` value for `exc`, summarized when privacy mode is on.""" + if _privacy_mode_enabled(ph_client, privacy_mode) and ( + getattr(exc, _PRIVACY_SAFE_MESSAGE_ATTR, False) is not True + ): + return _summarize_error_for_privacy(exc) + return describe(exc) + + +_MCP_TOOL_CREDENTIAL_FIELDS = ("headers", "authorization") +_REDACTED_MCP_CREDENTIAL = "[REDACTED]" + + +def _redact_mcp_tool(tool: Any) -> Any: + if not isinstance(tool, dict): + if getattr(tool, "type", None) != "mcp": + return tool + model_dump = getattr(tool, "model_dump", None) + if callable(model_dump): + return _redact_mcp_tool(model_dump()) + try: + redacted = copy.copy(tool) + for field in _MCP_TOOL_CREDENTIAL_FIELDS: + if getattr(tool, field, None) is not None: + setattr(redacted, field, _REDACTED_MCP_CREDENTIAL) + return redacted + except Exception: + return {"type": "mcp"} + if tool.get("type") != "mcp": + return tool + return { + key: ( + _REDACTED_MCP_CREDENTIAL + if key in _MCP_TOOL_CREDENTIAL_FIELDS and value is not None + else value + ) + for key, value in tool.items() + } + + +def _redact_mcp_tool_credentials(tools: Any) -> Any: + """ + Mask the `headers` and `authorization` of MCP tool definitions, which carry + credentials for the remote server. Returns copies; the caller's tools are + left untouched. + """ + if not isinstance(tools, (list, tuple)): + return tools + redacted = [_redact_mcp_tool(tool) for tool in tools] + if all(new is old for new, old in zip(redacted, tools)): + return tools + return redacted + + def capture_streaming_event( ph_client: PostHogClient, event_data: StreamingEventData, @@ -876,7 +1077,14 @@ def capture_streaming_event( "$ai_base_url": str(event_data["base_url"]), **(event_data.get("properties") or {}), **( - {"$ai_is_error": True, "$ai_error": str(error)} if error is not None else {} + { + "$ai_is_error": True, + "$ai_error": _format_ai_error( + ph_client, event_data["privacy_mode"], error + ), + } + if error is not None + else {} ), } diff --git a/posthog/test/ai/gemini/test_gemini_parity.py b/posthog/test/ai/gemini/test_gemini_parity.py index de9f9b82f..e4605e49c 100644 --- a/posthog/test/ai/gemini/test_gemini_parity.py +++ b/posthog/test/ai/gemini/test_gemini_parity.py @@ -36,7 +36,19 @@ def test_sync_and_async_clients_preserve_explicit_empty_api_key( @pytest.mark.parametrize("client_class", [Client, AsyncClient]) -def test_sync_and_async_clients_merge_posthog_defaults_without_mutation(client_class): +@pytest.mark.parametrize( + "default_privacy_mode, call_privacy_mode, expected_privacy_mode", + [ + (True, False, True), + (True, None, True), + (False, True, True), + (False, None, False), + (False, False, False), + ], +) +def test_sync_and_async_clients_merge_posthog_defaults_without_mutation( + client_class, default_privacy_mode, call_privacy_mode, expected_privacy_mode +): default_properties = {"shared": "default", "default-only": True} with patch.object(google_genai, "Client"): @@ -45,7 +57,7 @@ def test_sync_and_async_clients_merge_posthog_defaults_without_mutation(client_c posthog_client=MagicMock(), posthog_distinct_id="default-id", posthog_properties=default_properties, - posthog_privacy_mode=True, + posthog_privacy_mode=default_privacy_mode, posthog_groups={"organization": "default-org"}, ) @@ -53,7 +65,7 @@ def test_sync_and_async_clients_merge_posthog_defaults_without_mutation(client_c None, "", {"shared": "call", "call-only": True}, - False, + call_privacy_mode, None, ) @@ -61,7 +73,7 @@ def test_sync_and_async_clients_merge_posthog_defaults_without_mutation(client_c "default-id", "", {"shared": "call", "default-only": True, "call-only": True}, - False, + expected_privacy_mode, {"organization": "default-org"}, ) assert default_properties == {"shared": "default", "default-only": True} diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index b33e154fc..4c4de1be2 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2716,16 +2716,26 @@ def failing_span(_): assert span_props["$ai_error"] == "ValueError: test error" -def test_exception_autocapture_disabled_does_not_capture(): - """Test that capture_exception is NOT called when autocapture is disabled.""" +@pytest.mark.parametrize( + "autocapture, client_privacy_mode, handler_privacy_mode, expected_error", + [ + (False, False, False, "ValueError: test error"), + (True, True, False, "ValueError"), + (True, False, True, "ValueError"), + ], +) +def test_exception_autocapture_disabled_does_not_capture( + autocapture, client_privacy_mode, handler_privacy_mode, expected_error +): + """capture_exception is skipped when autocapture is off or privacy mode is on.""" mock_client = MagicMock() - mock_client.privacy_mode = False - mock_client.enable_exception_autocapture = False + mock_client.privacy_mode = client_privacy_mode + mock_client.enable_exception_autocapture = autocapture def failing_span(_): raise ValueError("test error") - callbacks = [CallbackHandler(mock_client)] + callbacks = [CallbackHandler(mock_client, privacy_mode=handler_privacy_mode)] chain = RunnableLambda(failing_span) try: @@ -2746,7 +2756,7 @@ def failing_span(_): span_props = span_calls[0][1]["properties"] assert "$exception_event_id" not in span_props - assert span_props["$ai_error"] == "ValueError: test error" + assert span_props["$ai_error"] == expected_error def test_exception_autocapture_on_llm_generation_error(mock_client): diff --git a/posthog/test/ai/langchain/test_middleware.py b/posthog/test/ai/langchain/test_middleware.py index ed68bca85..f01dd91c2 100644 --- a/posthog/test/ai/langchain/test_middleware.py +++ b/posthog/test/ai/langchain/test_middleware.py @@ -641,7 +641,7 @@ def handler(inner_request: ToolCallRequest) -> ToolMessage: assert actual is response span = _event(client, "$ai_span")["properties"] assert span["$ai_is_error"] is True - assert secret not in str(span.get("$ai_error", "")) + assert span["$ai_error"] == "RuntimeError: Tool returned an error" @pytest.mark.parametrize("asynchronous", [False, True]) diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py new file mode 100644 index 000000000..2e2df5cf4 --- /dev/null +++ b/posthog/test/ai/test_privacy_mode.py @@ -0,0 +1,584 @@ +"""Privacy mode across the AI integrations: nothing user-authored may leave the process.""" + +import json +from types import SimpleNamespace +from typing import Any, Callable, Dict, Iterator, List +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import BaseModel, ValidationError + +from posthog.ai.utils import ( + _REDACTED_MCP_CREDENTIAL, + extract_available_tool_calls, + _summarize_error_for_privacy, +) +from posthog.client import Client +from posthog.test.test_utils import FAKE_TEST_API_KEY + +CANARY = "CANARY_7f3a" + + +class _Parsed(BaseModel): + answer: int + + +def _validation_error() -> ValidationError: + try: + _Parsed.model_validate_json(json.dumps({"answer": CANARY})) + except ValidationError as exc: + return exc + raise AssertionError("validation unexpectedly passed") + + +def _httpx_response(status: int) -> Any: + import httpx + + return httpx.Response( + status, + request=httpx.Request("POST", "https://api.example.com/v1/run"), + headers={"x-request-id": "req_123", "request-id": "req_123"}, + ) + + +def _openai_bad_request() -> Exception: + import openai + + return openai.BadRequestError( + f"Error code: 400 - {CANARY}", + response=_httpx_response(400), + body={ + "message": f"echoed request: {CANARY}", + "type": "invalid_request_error", + "param": "messages[3].content", + "code": None, + }, + ) + + +def _anthropic_bad_request() -> Exception: + import anthropic + + return anthropic.BadRequestError( + f"echoed request: {CANARY}", + response=_httpx_response(400), + body={ + "type": "error", + "error": {"type": "invalid_request_error", "message": CANARY}, + }, + ) + + +def _gemini_client_error() -> Exception: + from google.genai import errors + + return errors.ClientError( + 400, + {"error": {"code": 400, "message": CANARY, "status": "INVALID_ARGUMENT"}}, + ) + + +class _ProviderShapedError(Exception): + def __init__(self, **fields: Any) -> None: + super().__init__(CANARY) + self.__dict__.update(fields) + + +class _UnreadableError(Exception): + @property + def status_code(self) -> int: + raise RuntimeError(CANARY) + + +def _chained_error(depth: int) -> BaseException: + error_types = [KeyError, ValueError, TypeError, LookupError, RuntimeError] + error: BaseException = error_types[0](CANARY) + for error_type in error_types[1 : depth + 1]: + try: + try: + raise error + except BaseException as cause: + raise error_type(CANARY) from cause + except BaseException as chained: + error = chained + return error + + +def _implicit_context_error() -> BaseException: + try: + try: + raise KeyError(CANARY) + except KeyError: + raise ValueError(CANARY) + except ValueError as exc: + return exc + + +@pytest.mark.parametrize( + "make_error, expected", + [ + pytest.param( + _openai_bad_request, + "BadRequestError: 400 invalid_request_error param=messages[3].content request_id=req_123", + id="openai", + ), + pytest.param( + _anthropic_bad_request, + "BadRequestError: 400 invalid_request_error request_id=req_123", + id="anthropic", + ), + pytest.param( + _gemini_client_error, "ClientError: 400 INVALID_ARGUMENT", id="gemini" + ), + pytest.param(_validation_error, "ValidationError", id="pydantic"), + pytest.param( + lambda: _ProviderShapedError( + code="invalid_request_error", + type="invalid_request_error", + param="x", + ), + "_ProviderShapedError invalid_request_error param=x", + id="no-status-and-duplicate-codes", + ), + pytest.param( + lambda: _ProviderShapedError(status_code=503, code="overloaded"), + "_ProviderShapedError: 503 overloaded", + id="generic-status-and-code", + ), + pytest.param( + lambda: _ProviderShapedError( + status_code=700, + code=f"{CANARY} with spaces", + type="x" * 65, + param="messages[0] content", + request_id="req/123", + ), + "_ProviderShapedError", + id="grammar-rejects-every-field", + ), + pytest.param( + lambda: _ProviderShapedError(status_code=True, status="429"), + "_ProviderShapedError 429", + id="bool-and-string-status-are-not-http-status", + ), + pytest.param( + lambda: type("Bad Name", (Exception,), {})(CANARY), + "Exception", + id="class-name-rejected", + ), + pytest.param( + lambda: _UnreadableError(CANARY), "_UnreadableError", id="raising-property" + ), + pytest.param( + lambda: _chained_error(2), + "TypeError (caused by ValueError, KeyError)", + id="explicit-cause", + ), + pytest.param( + _implicit_context_error, + "ValueError (caused by KeyError)", + id="implicit-context", + ), + pytest.param( + lambda: _chained_error(4), + "RuntimeError (caused by LookupError, TypeError, ValueError)", + id="cause-chain-capped-at-three", + ), + ], +) +def test_summarize_error_for_privacy( + make_error: Callable[[], BaseException], expected: str +) -> None: + summary = _summarize_error_for_privacy(make_error()) + + assert summary == expected + assert CANARY not in summary + + +@pytest.mark.parametrize("provider", ["openai", "anthropic", "gemini"]) +def test_mcp_tool_credentials_are_masked_without_mutating_caller_tools( + provider: str, +) -> None: + from openai.types.responses.tool import Mcp + + mcp_dict = { + "type": "mcp", + "server_label": "docs", + "server_url": "https://mcp.example.com", + "headers": {"Authorization": f"Bearer {CANARY}"}, + "authorization": CANARY, + } + mcp_model = Mcp( + type="mcp", + server_label="docs", + server_url="https://mcp.example.com", + headers={"X-Api-Key": CANARY}, + ) + mcp_object = SimpleNamespace(type="mcp", headers={"X-Api-Key": CANARY}) + function_tool = {"type": "function", "name": "lookup", "headers": "kept"} + tools = [mcp_dict, mcp_model, mcp_object, function_tool] + kwargs: Dict[str, Any] = ( + {"config": SimpleNamespace(tools=tools)} + if provider == "gemini" + else {"tools": tools} + ) + + captured = extract_available_tool_calls(provider, kwargs) + + assert captured[0] == { + **mcp_dict, + "headers": _REDACTED_MCP_CREDENTIAL, + "authorization": _REDACTED_MCP_CREDENTIAL, + } + assert captured[1]["headers"] == _REDACTED_MCP_CREDENTIAL + assert captured[1]["authorization"] is None + assert captured[1]["server_url"] == "https://mcp.example.com" + assert captured[2].headers == _REDACTED_MCP_CREDENTIAL + assert captured[3] is function_tool + assert CANARY not in json.dumps(captured, default=str) + assert tools == [mcp_dict, mcp_model, mcp_object, function_tool] + assert mcp_dict["headers"] == {"Authorization": f"Bearer {CANARY}"} + assert mcp_model.headers == {"X-Api-Key": CANARY} + assert mcp_object.headers == {"X-Api-Key": CANARY} + + +@pytest.fixture +def private_client() -> Iterator[Client]: + client = Client( + FAKE_TEST_API_KEY, privacy_mode=True, enable_exception_autocapture=True + ) + client._enqueue = MagicMock() # type: ignore[method-assign] + yield client + if client.exception_capture: + client.exception_capture.close() + + +def _sent_events(client: Client) -> List[Dict[str, Any]]: + return [call.args[0] for call in client._enqueue.call_args_list] # type: ignore[attr-defined] + + +def _assert_no_canary(client: Client) -> List[Dict[str, Any]]: + events = _sent_events(client) + assert events + assert any(event["properties"].get("$ai_is_error") for event in events) + assert CANARY not in json.dumps(events, default=str) + return events + + +def _function_tool() -> Dict[str, Any]: + return { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + } + + +@pytest.mark.parametrize( + "resource, method", + [("chat_create", "create"), ("chat_parse", "parse"), ("responses", "create")], +) +def test_openai_privacy_mode_sends_no_content( + private_client: Client, resource: str, method: str +) -> None: + import openai + + from posthog.ai.openai import OpenAI + + error = _validation_error() if resource == "chat_parse" else _openai_bad_request() + target = ( + openai.resources.responses.Responses + if resource == "responses" + else openai.resources.chat.completions.Completions + ) + client = OpenAI(posthog_client=private_client, api_key="test-key") + call = ( + getattr(client.responses, method) + if resource == "responses" + else getattr(client.chat.completions, method) + ) + request: Dict[str, Any] = ( + { + "input": f"user prompt {CANARY}", + "instructions": CANARY, + "tools": [ + { + "type": "mcp", + "server_label": "docs", + "headers": {"Authorization": CANARY}, + } + ], + } + if resource == "responses" + else { + "messages": [{"role": "user", "content": f"user prompt {CANARY}"}], + "tools": [_function_tool()], + } + ) + + with patch.object(target, method, side_effect=error): + with pytest.raises(type(error)): + call(model="gpt-4o", **request) + + _assert_no_canary(private_client) + + +def test_anthropic_privacy_mode_sends_no_content(private_client: Client) -> None: + from posthog.ai.anthropic import Anthropic + + client = Anthropic(api_key="test-key", posthog_client=private_client) + + with patch( + "anthropic.resources.Messages.create", side_effect=_anthropic_bad_request() + ): + with pytest.raises(Exception): + client.messages.create( + model="claude-sonnet-4-5", + max_tokens=16, + system=CANARY, + messages=[{"role": "user", "content": f"user prompt {CANARY}"}], + ) + + events = _assert_no_canary(private_client) + assert ( + events[0]["properties"]["$ai_error"] + == "BadRequestError: 400 invalid_request_error request_id=req_123" + ) + + +@pytest.mark.parametrize("method", ["generate_content", "embed_content"]) +def test_gemini_privacy_mode_sends_no_content( + private_client: Client, method: str +) -> None: + google_genai = pytest.importorskip("google.genai") + from posthog.ai.gemini import Client as GeminiClient + + error = ( + _validation_error() if method == "generate_content" else _gemini_client_error() + ) + client = GeminiClient(api_key="test-key", posthog_client=private_client) + + with patch.object(google_genai.models.Models, method, side_effect=error): + with pytest.raises(type(error)): + getattr(client.models, method)( + model="gemini-2.5-flash", + contents=[f"user prompt {CANARY}"], + posthog_privacy_mode=False, + ) + + _assert_no_canary(private_client) + + +def test_langchain_privacy_mode_sends_no_content_or_exception_event( + private_client: Client, +) -> None: + import uuid + + from langchain_core.exceptions import OutputParserException + from langchain_core.messages import HumanMessage + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.runnables import RunnableLambda + + from posthog.ai.langchain import CallbackHandler + + callbacks = CallbackHandler(private_client) + + def failing_parser(_: Any) -> None: + raise OutputParserException( + f"Could not parse output: {CANARY}", llm_output=CANARY + ) + + chain = ChatPromptTemplate.from_messages([("user", CANARY)]) | RunnableLambda( + failing_parser + ) + with pytest.raises(OutputParserException): + chain.invoke({}, config={"callbacks": [callbacks]}) + + run_id = uuid.uuid4() + callbacks.on_chat_model_start( + {"kwargs": {"model_name": "gpt-4o"}}, + [[HumanMessage(content=CANARY)]], + run_id=run_id, + invocation_params={ + "tools": [ + _function_tool(), + {"type": "mcp", "server_label": "docs", "headers": {"k": CANARY}}, + ] + }, + ) + callbacks.on_llm_error(_openai_bad_request(), run_id=run_id) + + events = _assert_no_canary(private_client) + assert "$exception" not in {event["event"] for event in events} + errors = {event["properties"].get("$ai_error") for event in events} + assert "OutputParserException" in errors + + +def test_openai_agents_privacy_mode_sends_no_content(private_client: Client) -> None: + pytest.importorskip("agents") + from agents.tracing.span_data import ( + FunctionSpanData, + GenerationSpanData, + SpeechSpanData, + TranscriptionSpanData, + ) + + from posthog.ai.openai_agents import PostHogTracingProcessor + + processor = PostHogTracingProcessor(client=private_client, distinct_id="user") + trace = MagicMock( + trace_id="trace_1", group_id=None, metadata={"customer_note": CANARY} + ) + trace.name = "workflow" + processor.on_trace_start(trace) + + span_datas = [ + GenerationSpanData( + input=[{"role": "user", "content": CANARY}], + output=[{"role": "assistant", "content": CANARY}], + model="gpt-4o", + ), + FunctionSpanData(name="lookup", input=CANARY, output=CANARY), + TranscriptionSpanData( + input="AAAA", + input_format="pcm", + output=CANARY, + model="gpt-4o-transcribe", + model_config={"prompt": CANARY, "language": "en"}, + ), + SpeechSpanData( + input=CANARY, + output=None, + output_format="pcm", + model="gpt-4o-mini-tts", + model_config={"instructions": CANARY, "voice": "alloy"}, + ), + ] + for index, span_data in enumerate(span_datas): + span = MagicMock( + trace_id="trace_1", + span_id=f"span_{index}", + parent_id=None, + started_at=None, + ended_at=None, + span_data=span_data, + error={"message": f"Tool failed: {CANARY}", "data": {"detail": CANARY}}, + ) + processor.on_span_start(span) + processor.on_span_end(span) + processor.on_trace_end(trace) + + events = _assert_no_canary(private_client) + audio_configs = [ + event["properties"]["model_config"] + for event in events + if "model_config" in event["properties"] + ] + assert audio_configs == [ + {"prompt": None, "language": "en"}, + {"instructions": None, "voice": "alloy"}, + ] + + +_KEPT_OTEL_ATTRIBUTES: Dict[str, Any] = { + "gen_ai.request.model": "gpt-4o", + "gen_ai.usage.input_tokens": 3, + "gen_ai.tool.definitions": '[{"name": "lookup"}]', + "llm.request.functions.0.name": "lookup", + "ai.prompt.tools": ('{"name": "lookup"}',), + "ai.prompt.toolChoice": '{"type": "auto"}', +} + + +def _emit_canary_span(provider: Any) -> None: + from opentelemetry.trace import Status, StatusCode + + tracer = provider.get_tracer("test") + content_attributes = { + "gen_ai.input.messages": json.dumps([{"role": "user", "content": CANARY}]), + "gen_ai.output.messages": CANARY, + "gen_ai.system_instructions": CANARY, + "gen_ai.tool.call.arguments": CANARY, + "gen_ai.tool.call.result": CANARY, + "gen_ai.prompt.0.content": CANARY, + "gen_ai.completion.0.content": CANARY, + "traceloop.entity.input": CANARY, + "traceloop.entity.output": CANARY, + "ai.prompt": CANARY, + "ai.prompt.messages": CANARY, + "ai.response.text": CANARY, + "ai.response.object": CANARY, + "ai.response.toolCalls": CANARY, + "ai.response.reasoning": CANARY, + "ai.toolCall.args": CANARY, + "ai.toolCall.result": CANARY, + "ai.value": CANARY, + "ai.values": [CANARY], + "pydantic_ai.all_messages": CANARY, + "final_result": CANARY, + "tool_arguments": CANARY, + "tool_response": CANARY, + "anthropic.tool_result.content": CANARY, + "events": CANARY, + "error.message": CANARY, + } + with tracer.start_as_current_span( + "gen_ai.chat", + attributes={ + **_KEPT_OTEL_ATTRIBUTES, + **content_attributes, + }, + ) as span: + span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": CANARY}) + span.record_exception(ValueError(CANARY)) + span.set_status(Status(StatusCode.ERROR, CANARY)) + + +@pytest.mark.parametrize("integration", ["processor", "exporter"]) +@pytest.mark.parametrize("privacy_source", ["option", "client", "module"]) +def test_otel_privacy_mode_sends_no_content( + integration: str, privacy_source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + pytest.importorskip("opentelemetry.exporter.otlp.proto.http") + import posthog + from opentelemetry.sdk.trace import ReadableSpan, TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + from posthog.ai.otel import PostHogSpanProcessor, PostHogTraceExporter + + monkeypatch.setattr(posthog, "privacy_mode", privacy_source == "module") + options: Dict[str, Any] = { + "privacy_mode": privacy_source == "option", + "client": ( + Client(FAKE_TEST_API_KEY, privacy_mode=True) + if privacy_source == "client" + else None + ), + } + exported: List[ReadableSpan] = [] + provider = TracerProvider() + with ( + patch("posthog.ai.otel.processor.OTLPSpanExporter"), + patch("posthog.ai.otel.processor.BatchSpanProcessor") as batch, + patch("posthog.ai.otel.exporter.OTLPSpanExporter") as otlp, + ): + batch.return_value.on_end.side_effect = exported.append + otlp.return_value.export.side_effect = exported.extend + if integration == "processor": + provider.add_span_processor( + PostHogSpanProcessor(api_key="phc_test", **options) + ) + else: + provider.add_span_processor( + SimpleSpanProcessor(PostHogTraceExporter(api_key="phc_test", **options)) + ) + _emit_canary_span(provider) + + [span] = exported + assert CANARY not in span.to_json() + assert dict(span.attributes or {}) == _KEPT_OTEL_ATTRIBUTES + [exception_event] = span.events + assert exception_event.name == "exception" + exception_attributes = dict(exception_event.attributes or {}) + assert exception_attributes["exception.type"] == "ValueError" + assert set(exception_attributes) <= {"exception.type", "exception.escaped"} + assert span.status.is_ok is False + assert span.status.description is None diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index dc2bbe85f..c908c8ce6 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -208,10 +208,12 @@ alias posthog.ai.openai_agents.processor.finalize_ai_content -> posthog.ai.utils alias posthog.ai.openai_agents.processor.setup -> posthog.setup alias posthog.ai.otel.PostHogSpanProcessor -> posthog.ai.otel.processor.PostHogSpanProcessor alias posthog.ai.otel.PostHogTraceExporter -> posthog.ai.otel.exporter.PostHogTraceExporter +alias posthog.ai.otel.exporter.Client -> posthog.client.Client alias posthog.ai.otel.exporter.DEFAULT_HOST -> posthog.ai.otel.spans.DEFAULT_HOST alias posthog.ai.otel.exporter.is_ai_span -> posthog.ai.otel.spans.is_ai_span alias posthog.ai.otel.exporter.warn_if_posthog_ai_gateway_otel_attributes -> posthog.ai.gateway.warn_if_posthog_ai_gateway_otel_attributes alias posthog.ai.otel.is_ai_span -> posthog.ai.otel.spans.is_ai_span +alias posthog.ai.otel.processor.Client -> posthog.client.Client alias posthog.ai.otel.processor.DEFAULT_HOST -> posthog.ai.otel.spans.DEFAULT_HOST alias posthog.ai.otel.processor.is_ai_span -> posthog.ai.otel.spans.is_ai_span alias posthog.ai.otel.processor.warn_if_posthog_ai_gateway_otel_attributes -> posthog.ai.gateway.warn_if_posthog_ai_gateway_otel_attributes @@ -1253,8 +1255,8 @@ class posthog.ai.openai.openai_async.WrappedResponses class posthog.ai.openai.openai_providers.AsyncAzureOpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) class posthog.ai.openai.openai_providers.AzureOpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) class posthog.ai.openai_agents.processor.PostHogTracingProcessor(client: Optional[Client] = None, distinct_id: Optional[Union[str, Callable[[Trace], Optional[str]]]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, properties: Optional[Dict[str, Any]] = None) -class posthog.ai.otel.exporter.PostHogTraceExporter(api_key: str, host: str = DEFAULT_HOST) -class posthog.ai.otel.processor.PostHogSpanProcessor(api_key: str, host: str = DEFAULT_HOST) +class posthog.ai.otel.exporter.PostHogTraceExporter(api_key: str, host: str = DEFAULT_HOST, *, privacy_mode: bool = False, client: Optional[Client] = None) +class posthog.ai.otel.processor.PostHogSpanProcessor(api_key: str, host: str = DEFAULT_HOST, *, privacy_mode: bool = False, client: Optional[Client] = None) class posthog.ai.prompts.CachedPrompt(prompt: str, fetched_at: float, name: str, version: int, label: Optional[str] = None, config: Optional[Dict[str, Any]] = None) class posthog.ai.prompts.PromptFetchError(message: str, retry_after_seconds: Optional[float] = None) class posthog.ai.prompts.PromptResult(source: PromptSource, prompt: str, name: Optional[str] = None, version: Optional[int] = None, label: Optional[str] = None, config: Optional[Dict[str, Any]] = None) From 157ad38f5e4b37668d5be554961a923c48079fb9 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 10:41:40 +0200 Subject: [PATCH 02/10] fix: address privacy mode review findings --- posthog/ai/otel/_privacy.py | 65 ++++++++++++++++++---------- posthog/test/ai/test_privacy_mode.py | 13 +++++- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index af6c5f264..3e8127f4b 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -2,8 +2,11 @@ from typing import TYPE_CHECKING, Optional +from opentelemetry.attributes import BoundedAttributes from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.sdk.util import BoundedList from opentelemetry.trace.status import Status +from opentelemetry.util.types import Attributes if TYPE_CHECKING: from ...client import Client @@ -38,6 +41,8 @@ "ai.values", "input.value", "output.value", + "llm.function_call", + "llm.invocation_parameters", "exception.message", "exception.stacktrace", "error.message", @@ -51,14 +56,22 @@ "llm.prompts", "llm.input_messages", "llm.output_messages", + "llm.choices", + "llm.prompt_template.template", + "llm.prompt_template.variables", ) _EXCEPTION_EVENT_NAME = "exception" +def _is_embedding_text(key: str) -> bool: + """OpenInference stores embedding inputs as `embedding.embeddings..embedding.text`.""" + return key.startswith("embedding.embeddings.") and key.endswith(".embedding.text") + + def _is_content_attribute(key: str) -> bool: """Whether a span or event attribute can carry prompts, completions, tool data or error text.""" - if key in _CONTENT_ATTRIBUTES: + if key in _CONTENT_ATTRIBUTES or _is_embedding_text(key): return True return any( key == root or key.startswith(f"{root}.") for root in _CONTENT_ATTRIBUTE_ROOTS @@ -79,39 +92,45 @@ def _otel_privacy_mode_enabled(privacy_mode: bool, client: Optional["Client"]) - ) +def _without_content(attributes: Attributes) -> BoundedAttributes: + return BoundedAttributes( + attributes={ + key: value + for key, value in (attributes or {}).items() + if not _is_content_attribute(key) + } + ) + + def _redact_span(span: ReadableSpan) -> ReadableSpan: """ Return a copy of ``span`` without content attributes, non-exception events, exception messages and stack traces, or the status description. """ - attributes = { - key: value - for key, value in (span.attributes or {}).items() - if not _is_content_attribute(key) - } - # Other span events (older GenAI semconv messages, framework logs) carry - # content and PostHog ingestion only reads exception events. - events = [ - Event( - name=event.name, - attributes={ - key: value - for key, value in (event.attributes or {}).items() - if not _is_content_attribute(key) - }, - timestamp=event.timestamp, - ) - for event in span.events - if event.name == _EXCEPTION_EVENT_NAME - ] + # Older OTLP encoders read `.dropped` from these collections, so plain dicts and + # lists break export. + events = BoundedList.from_seq( + None, + [ + Event( + name=event.name, + attributes=_without_content(event.attributes), + timestamp=event.timestamp, + ) + # Other span events (older GenAI semconv messages, framework logs) carry + # content and PostHog ingestion only reads exception events. + for event in span.events + if event.name == _EXCEPTION_EVENT_NAME + ], + ) return ReadableSpan( name=span.name, context=span.get_span_context(), parent=span.parent, resource=span.resource, - attributes=attributes, + attributes=_without_content(span.attributes), events=events, - links=span.links, + links=BoundedList.from_seq(None, span.links), kind=span.kind, status=Status(span.status.status_code), start_time=span.start_time, diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index 2e2df5cf4..15366a850 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -519,6 +519,16 @@ def _emit_canary_span(provider: Any) -> None: "anthropic.tool_result.content": CANARY, "events": CANARY, "error.message": CANARY, + "input.value": CANARY, + "output.value": CANARY, + "llm.input_messages.0.message.content": CANARY, + "llm.output_messages.0.message.content": CANARY, + "llm.choices.0.completion.text": CANARY, + "llm.function_call": CANARY, + "llm.invocation_parameters": json.dumps({"instructions": CANARY}), + "llm.prompt_template.template": CANARY, + "llm.prompt_template.variables": CANARY, + "embedding.embeddings.0.embedding.text": CANARY, } with tracer.start_as_current_span( "gen_ai.chat", @@ -539,6 +549,7 @@ def test_otel_privacy_mode_sends_no_content( ) -> None: pytest.importorskip("opentelemetry.exporter.otlp.proto.http") import posthog + from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -573,7 +584,7 @@ def test_otel_privacy_mode_sends_no_content( _emit_canary_span(provider) [span] = exported - assert CANARY not in span.to_json() + assert CANARY.encode() not in encode_spans([span]).SerializeToString() assert dict(span.attributes or {}) == _KEPT_OTEL_ATTRIBUTES [exception_event] = span.events assert exception_event.name == "exception" From 1fec9fd01aa976dd183fdf02fedee48f3b25719b Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 11:30:19 +0200 Subject: [PATCH 03/10] fix: strip remaining otel content in privacy mode --- posthog/ai/otel/_privacy.py | 169 ++++++++++++++++++++------- posthog/test/ai/test_privacy_mode.py | 80 ++++++++++++- 2 files changed, 203 insertions(+), 46 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 3e8127f4b..5e7aaa43e 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -1,6 +1,8 @@ """Privacy mode for OpenTelemetry spans exported to PostHog.""" -from typing import TYPE_CHECKING, Optional +import json +import re +from typing import TYPE_CHECKING, Any, Dict, Optional from opentelemetry.attributes import BoundedAttributes from opentelemetry.sdk.trace import Event, ReadableSpan @@ -11,70 +13,131 @@ if TYPE_CHECKING: from ...client import Client -# Attributes PostHog ingestion maps into $ai_input, $ai_output_choices, $ai_input_state, -# $ai_output_state or $ai_error, plus the raw payloads it strips. Tool definitions -# ($ai_tools) are not content and stay, as in the SDK wrappers. +# Attributes that carry prompts, outputs, tool calls, retrieved documents or error text. +# Ingestion keeps unmapped attributes as event properties, so this also covers attributes +# it does not map. Tool definitions ($ai_tools) are not content and stay, as in the SDK +# wrappers. _CONTENT_ATTRIBUTES = frozenset( { - "gen_ai.input.messages", - "gen_ai.output.messages", - "gen_ai.system_instructions", - "gen_ai.tool.call.arguments", - "gen_ai.tool.call.result", - "anthropic.tool_result.content", - "events", - "pydantic_ai.all_messages", - "final_result", - "tool_arguments", - "tool_response", - "traceloop.entity.input", - "traceloop.entity.output", + # Vercel AI SDK "ai.prompt", "ai.prompt.messages", + "ai.request.text", + "ai.documents", "ai.response.text", "ai.response.object", "ai.response.toolCalls", "ai.response.reasoning", + "ai.response.reasoningDetails", + "ai.response.files", "ai.toolCall.args", "ai.toolCall.result", + "ai.decision.state", + "ai.decision.questions", + "ai.decision.answers", "ai.value", "ai.values", + "ai.embedding", + "ai.embeddings", + # OpenTelemetry GenAI semantic conventions + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", + "gen_ai.prompt", + "gen_ai.completion", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + "gen_ai.retrieval.documents", + "gen_ai.retrieval.query.text", + "gen_ai.memory.query.text", + "gen_ai.memory.records", + "gen_ai.evaluation.explanation", + # Traceloop / OpenLLMetry + "traceloop.entity.input", + "traceloop.entity.output", + "traceloop.prompt.template", + "gen_ai.task.input", + "gen_ai.task.output", + "gen_ai.guardrail.input", + "gen_ai.guardrail.output", + "gen_ai.guardrail.error.message", + "gen_ai.response.revised_prompt", + "mcp.response.value", + "db.marqo.search.query", + # Anthropic + "anthropic.tool_result.content", + # Pydantic AI / Logfire + "pydantic_ai.all_messages", + "model_request_parameters", + "final_result", + "tool_arguments", + "tool_response", + "inputs", + "prompt", + "events", + # OpenInference "input.value", "output.value", "llm.function_call", + "llm.prompt_template.template", + "llm.prompt_template.variables", + # Holds request parameters such as Responses `instructions`. "llm.invocation_parameters", + "reranker.query", + # Langfuse + "langfuse.trace.input", + "langfuse.trace.output", + "langfuse.observation.input", + "langfuse.observation.output", + "langfuse.observation.status_message", + "langfuse.experiment.item.expected_output", + # Errors "exception.message", "exception.stacktrace", "error.message", } ) -# Matched as the attribute itself or as a dotted prefix, e.g. `gen_ai.prompt.0.content`. -_CONTENT_ATTRIBUTE_ROOTS = ( - "gen_ai.prompt", - "gen_ai.completion", - "llm.prompts", - "llm.input_messages", - "llm.output_messages", - "llm.choices", - "llm.prompt_template.template", - "llm.prompt_template.variables", +_CONTENT_ATTRIBUTE_PREFIXES = ( + "gen_ai.prompt.", + "gen_ai.completion.", + "llm.prompts.", + "llm.completions.", + "llm.input_messages.", + "llm.output_messages.", + "llm.choices.", + "traceloop.prompt.template_variables.", ) -_EXCEPTION_EVENT_NAME = "exception" +_CONTENT_ATTRIBUTE_PATTERN = re.compile( + "|".join( + ( + r"embedding\.embeddings\.\d+\.embedding\.text", + r"retrieval\.documents\.\d+\.document\.(content|metadata)", + r"reranker\.(input|output)_documents\.\d+\.document\.content", + r"(input|output)\.images\.\d+\.image\.url", + r"db\.chroma\.(query|get|delete)\.where_document", + r"mastra\.[a-z_]+\.(input|output)", + ) + ) +) + +# GenAI semantic convention prompt metadata, not content, under the `gen_ai.prompt.` prefix. +_NON_CONTENT_ATTRIBUTES = frozenset({"gen_ai.prompt.name", "gen_ai.prompt.version"}) +_PROVIDER_METADATA_ATTRIBUTE = "ai.response.providerMetadata" -def _is_embedding_text(key: str) -> bool: - """OpenInference stores embedding inputs as `embedding.embeddings..embedding.text`.""" - return key.startswith("embedding.embeddings.") and key.endswith(".embedding.text") +_EXCEPTION_EVENT_NAME = "exception" def _is_content_attribute(key: str) -> bool: """Whether a span or event attribute can carry prompts, completions, tool data or error text.""" - if key in _CONTENT_ATTRIBUTES or _is_embedding_text(key): - return True - return any( - key == root or key.startswith(f"{root}.") for root in _CONTENT_ATTRIBUTE_ROOTS + if key in _NON_CONTENT_ATTRIBUTES: + return False + return ( + key in _CONTENT_ATTRIBUTES + or key.startswith(_CONTENT_ATTRIBUTE_PREFIXES) + or _CONTENT_ATTRIBUTE_PATTERN.fullmatch(key) is not None ) @@ -92,14 +155,36 @@ def _otel_privacy_mode_enabled(privacy_mode: bool, client: Optional["Client"]) - ) +def _gateway_cost_only(value: Any) -> Optional[str]: + """ + Provider metadata can carry content, such as Google grounding search queries. + Ingestion only reads the Vercel AI Gateway cost from it. + """ + if not isinstance(value, str): + return None + try: + metadata = json.loads(value) + except ValueError: + return None + gateway = metadata.get("gateway") if isinstance(metadata, dict) else None + cost = gateway.get("cost") if isinstance(gateway, dict) else None + if isinstance(cost, bool) or not isinstance(cost, (int, float, str)): + return None + return json.dumps({"gateway": {"cost": cost}}, separators=(",", ":")) + + def _without_content(attributes: Attributes) -> BoundedAttributes: - return BoundedAttributes( - attributes={ - key: value - for key, value in (attributes or {}).items() - if not _is_content_attribute(key) - } - ) + kept: Dict[str, Any] = {} + for key, value in (attributes or {}).items(): + if _is_content_attribute(key): + continue + if key == _PROVIDER_METADATA_ATTRIBUTE: + metadata = _gateway_cost_only(value) + if metadata is not None: + kept[key] = metadata + continue + kept[key] = value + return BoundedAttributes(attributes=kept) def _redact_span(span: ReadableSpan) -> ReadableSpan: diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index 15366a850..597520298 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -485,10 +485,12 @@ def test_openai_agents_privacy_mode_sends_no_content(private_client: Client) -> "llm.request.functions.0.name": "lookup", "ai.prompt.tools": ('{"name": "lookup"}',), "ai.prompt.toolChoice": '{"type": "auto"}', + "gen_ai.prompt.name": "support-reply", + "gen_ai.prompt.version": "3", } -def _emit_canary_span(provider: Any) -> None: +def _emit_canary_span(provider: Any, provider_metadata: str) -> None: from opentelemetry.trace import Status, StatusCode tracer = provider.get_tracer("test") @@ -529,12 +531,58 @@ def _emit_canary_span(provider: Any) -> None: "llm.prompt_template.template": CANARY, "llm.prompt_template.variables": CANARY, "embedding.embeddings.0.embedding.text": CANARY, + "ai.request.text": CANARY, + "ai.documents": CANARY, + "ai.response.reasoningDetails": CANARY, + "ai.response.files": CANARY, + "ai.decision.state": CANARY, + "ai.decision.questions": CANARY, + "ai.decision.answers": CANARY, + "ai.embedding": CANARY, + "ai.embeddings": CANARY, + "gen_ai.prompt.variable.customer": CANARY, + "gen_ai.retrieval.documents": CANARY, + "gen_ai.retrieval.query.text": CANARY, + "gen_ai.memory.query.text": CANARY, + "gen_ai.memory.records": CANARY, + "gen_ai.evaluation.explanation": CANARY, + "gen_ai.task.input": CANARY, + "gen_ai.task.output": CANARY, + "gen_ai.guardrail.input": CANARY, + "gen_ai.guardrail.output": CANARY, + "gen_ai.guardrail.error.message": CANARY, + "gen_ai.response.revised_prompt": CANARY, + "mcp.response.value": CANARY, + "traceloop.prompt.template": CANARY, + "traceloop.prompt.template_variables.customer": CANARY, + "db.marqo.search.query": CANARY, + "db.chroma.query.where_document": CANARY, + "llm.completions.0.content": CANARY, + "retrieval.documents.0.document.content": CANARY, + "retrieval.documents.0.document.metadata": CANARY, + "reranker.query": CANARY, + "reranker.input_documents.0.document.content": CANARY, + "reranker.output_documents.0.document.content": CANARY, + "input.images.0.image.url": CANARY, + "output.images.0.image.url": CANARY, + "model_request_parameters": CANARY, + "inputs": CANARY, + "prompt": CANARY, + "langfuse.trace.input": CANARY, + "langfuse.trace.output": CANARY, + "langfuse.observation.input": CANARY, + "langfuse.observation.output": CANARY, + "langfuse.observation.status_message": CANARY, + "langfuse.experiment.item.expected_output": CANARY, + "mastra.agent_run.input": CANARY, + "mastra.agent_run.output": CANARY, } with tracer.start_as_current_span( "gen_ai.chat", attributes={ **_KEPT_OTEL_ATTRIBUTES, **content_attributes, + "ai.response.providerMetadata": provider_metadata, }, ) as span: span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": CANARY}) @@ -542,10 +590,31 @@ def _emit_canary_span(provider: Any) -> None: span.set_status(Status(StatusCode.ERROR, CANARY)) +@pytest.mark.parametrize( + "provider_metadata, kept_provider_metadata", + [ + ( + json.dumps( + { + "gateway": {"cost": 0.0012, "routing": CANARY}, + "google": {"groundingMetadata": CANARY}, + } + ), + {"ai.response.providerMetadata": '{"gateway":{"cost":0.0012}}'}, + ), + (json.dumps({"google": {"groundingMetadata": CANARY}}), {}), + (f"not json {CANARY}", {}), + ], + ids=["gateway_cost", "no_gateway_cost", "unparseable"], +) @pytest.mark.parametrize("integration", ["processor", "exporter"]) @pytest.mark.parametrize("privacy_source", ["option", "client", "module"]) def test_otel_privacy_mode_sends_no_content( - integration: str, privacy_source: str, monkeypatch: pytest.MonkeyPatch + integration: str, + privacy_source: str, + provider_metadata: str, + kept_provider_metadata: Dict[str, str], + monkeypatch: pytest.MonkeyPatch, ) -> None: pytest.importorskip("opentelemetry.exporter.otlp.proto.http") import posthog @@ -581,11 +650,14 @@ def test_otel_privacy_mode_sends_no_content( provider.add_span_processor( SimpleSpanProcessor(PostHogTraceExporter(api_key="phc_test", **options)) ) - _emit_canary_span(provider) + _emit_canary_span(provider, provider_metadata) [span] = exported assert CANARY.encode() not in encode_spans([span]).SerializeToString() - assert dict(span.attributes or {}) == _KEPT_OTEL_ATTRIBUTES + assert dict(span.attributes or {}) == { + **_KEPT_OTEL_ATTRIBUTES, + **kept_provider_metadata, + } [exception_event] = span.events assert exception_event.name == "exception" exception_attributes = dict(exception_event.attributes or {}) From f3df40ea524b9dd9af631f80741d4f47d62ec9c1 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 11:51:11 +0200 Subject: [PATCH 04/10] fix: redact mcp credentials in otel tool definitions --- posthog/ai/otel/_privacy.py | 48 ++++++++++++++++++++++++++-- posthog/test/ai/test_privacy_mode.py | 29 +++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 5e7aaa43e..472bbb84d 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -2,13 +2,15 @@ import json import re -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from opentelemetry.attributes import BoundedAttributes from opentelemetry.sdk.trace import Event, ReadableSpan from opentelemetry.sdk.util import BoundedList from opentelemetry.trace.status import Status -from opentelemetry.util.types import Attributes +from opentelemetry.util.types import Attributes, AttributeValue + +from ..utils import _redact_mcp_tool, _redact_mcp_tool_credentials if TYPE_CHECKING: from ...client import Client @@ -127,6 +129,11 @@ _PROVIDER_METADATA_ATTRIBUTE = "ai.response.providerMetadata" +# Tool definitions stay under privacy, but instrumentations serialize remote MCP tools +# with their credentials into them. +_TOOL_DEFINITION_ATTRIBUTES = frozenset({"gen_ai.tool.definitions", "ai.prompt.tools"}) +_TOOL_DEFINITION_PATTERN = re.compile(r"llm\.tools\.\d+\.tool\.json_schema") + _EXCEPTION_EVENT_NAME = "exception" @@ -173,11 +180,48 @@ def _gateway_cost_only(value: Any) -> Optional[str]: return json.dumps({"gateway": {"cost": cost}}, separators=(",", ":")) +def _redact_tool_definition_json(value: str) -> Optional[str]: + """A value that cannot be parsed could still hold credentials, so it is dropped.""" + try: + parsed = json.loads(value) + except ValueError: + return None + if isinstance(parsed, list): + redacted = _redact_mcp_tool_credentials(parsed) + elif isinstance(parsed, dict): + redacted = _redact_mcp_tool(parsed) + else: + return None + return value if redacted is parsed else json.dumps(redacted) + + +def _redact_tool_definitions(value: Any) -> Optional[AttributeValue]: + if isinstance(value, str): + return _redact_tool_definition_json(value) + if isinstance(value, (list, tuple)): + redacted: List[str] = [] + for item in value: + definition = ( + _redact_tool_definition_json(item) if isinstance(item, str) else None + ) + if definition is not None: + redacted.append(definition) + return redacted + return None + + def _without_content(attributes: Attributes) -> BoundedAttributes: kept: Dict[str, Any] = {} for key, value in (attributes or {}).items(): if _is_content_attribute(key): continue + if key in _TOOL_DEFINITION_ATTRIBUTES or _TOOL_DEFINITION_PATTERN.fullmatch( + key + ): + definitions = _redact_tool_definitions(value) + if definitions is not None: + kept[key] = definitions + continue if key == _PROVIDER_METADATA_ATTRIBUTE: metadata = _gateway_cost_only(value) if metadata is not None: diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index 597520298..b70584d7b 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -490,6 +490,33 @@ def test_openai_agents_privacy_mode_sends_no_content(private_client: Client) -> } +_MCP_OTEL_TOOL = { + "type": "mcp", + "server_label": "docs", + "headers": {"Authorization": CANARY}, + "authorization": CANARY, +} + +_MCP_OTEL_TOOL_ATTRIBUTES: Dict[str, Any] = { + "ai.prompt.tools": ('{"name": "lookup"}', json.dumps(_MCP_OTEL_TOOL)), + "llm.tools.0.tool.json_schema": json.dumps(_MCP_OTEL_TOOL), + "llm.tools.1.tool.json_schema": f"not json {CANARY}", +} + +_REDACTED_MCP_OTEL_TOOL = json.dumps( + { + **_MCP_OTEL_TOOL, + "headers": _REDACTED_MCP_CREDENTIAL, + "authorization": _REDACTED_MCP_CREDENTIAL, + } +) + +_REDACTED_MCP_OTEL_TOOL_ATTRIBUTES: Dict[str, Any] = { + "ai.prompt.tools": ('{"name": "lookup"}', _REDACTED_MCP_OTEL_TOOL), + "llm.tools.0.tool.json_schema": _REDACTED_MCP_OTEL_TOOL, +} + + def _emit_canary_span(provider: Any, provider_metadata: str) -> None: from opentelemetry.trace import Status, StatusCode @@ -581,6 +608,7 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: "gen_ai.chat", attributes={ **_KEPT_OTEL_ATTRIBUTES, + **_MCP_OTEL_TOOL_ATTRIBUTES, **content_attributes, "ai.response.providerMetadata": provider_metadata, }, @@ -656,6 +684,7 @@ def test_otel_privacy_mode_sends_no_content( assert CANARY.encode() not in encode_spans([span]).SerializeToString() assert dict(span.attributes or {}) == { **_KEPT_OTEL_ATTRIBUTES, + **_REDACTED_MCP_OTEL_TOOL_ATTRIBUTES, **kept_provider_metadata, } [exception_event] = span.events From b18ce18e41a3556f740ac790877637d871a3a008 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 12:02:54 +0200 Subject: [PATCH 05/10] fix: address otel privacy review findings --- posthog/ai/otel/_privacy.py | 2 ++ posthog/ai/utils.py | 38 ++++++++++++++++++++-------- posthog/test/ai/test_privacy_mode.py | 30 ++++++++++++++++++++-- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 472bbb84d..d67011fa2 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -26,6 +26,7 @@ "ai.prompt.messages", "ai.request.text", "ai.documents", + "ai.ranking", "ai.response.text", "ai.response.object", "ai.response.toolCalls", @@ -144,6 +145,7 @@ def _is_content_attribute(key: str) -> bool: return ( key in _CONTENT_ATTRIBUTES or key.startswith(_CONTENT_ATTRIBUTE_PREFIXES) + or f"{key}." in _CONTENT_ATTRIBUTE_PREFIXES or _CONTENT_ATTRIBUTE_PATTERN.fullmatch(key) is not None ) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index b4a85bd80..dead6eea6 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -973,6 +973,29 @@ def _format_ai_error( _MCP_TOOL_CREDENTIAL_FIELDS = ("headers", "authorization") _REDACTED_MCP_CREDENTIAL = "[REDACTED]" +# Vercel AI SDK provider tools that configure a remote MCP server, keyed by their `id`. +# They reach this SDK through OpenTelemetry tool definition attributes. +_MCP_PROVIDER_TOOL_IDS = frozenset({"openai.mcp", "xai.mcp", "google.mcp_server"}) + + +def _without_mcp_credentials(config: Dict[str, Any]) -> Dict[str, Any]: + return { + key: ( + _REDACTED_MCP_CREDENTIAL + if key in _MCP_TOOL_CREDENTIAL_FIELDS and value is not None + else value + ) + for key, value in config.items() + } + + +def _is_mcp_provider_tool(tool: Dict[str, Any]) -> bool: + return ( + tool.get("type") in ("provider", "provider-defined") + and tool.get("id") in _MCP_PROVIDER_TOOL_IDS + and isinstance(tool.get("args"), dict) + ) + def _redact_mcp_tool(tool: Any) -> Any: if not isinstance(tool, dict): @@ -989,16 +1012,11 @@ def _redact_mcp_tool(tool: Any) -> Any: return redacted except Exception: return {"type": "mcp"} - if tool.get("type") != "mcp": - return tool - return { - key: ( - _REDACTED_MCP_CREDENTIAL - if key in _MCP_TOOL_CREDENTIAL_FIELDS and value is not None - else value - ) - for key, value in tool.items() - } + if tool.get("type") == "mcp": + return _without_mcp_credentials(tool) + if _is_mcp_provider_tool(tool): + return {**tool, "args": _without_mcp_credentials(tool["args"])} + return tool def _redact_mcp_tool_credentials(tools: Any) -> Any: diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index b70584d7b..1cf08748c 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -497,8 +497,19 @@ def test_openai_agents_privacy_mode_sends_no_content(private_client: Client) -> "authorization": CANARY, } +_VERCEL_MCP_OTEL_TOOL = { + "type": "provider", + "id": "openai.mcp", + "name": "docs", + "args": {"serverUrl": "https://mcp.example.com", "authorization": CANARY}, +} + _MCP_OTEL_TOOL_ATTRIBUTES: Dict[str, Any] = { - "ai.prompt.tools": ('{"name": "lookup"}', json.dumps(_MCP_OTEL_TOOL)), + "ai.prompt.tools": ( + '{"name": "lookup"}', + json.dumps(_MCP_OTEL_TOOL), + json.dumps(_VERCEL_MCP_OTEL_TOOL), + ), "llm.tools.0.tool.json_schema": json.dumps(_MCP_OTEL_TOOL), "llm.tools.1.tool.json_schema": f"not json {CANARY}", } @@ -512,7 +523,19 @@ def test_openai_agents_privacy_mode_sends_no_content(private_client: Client) -> ) _REDACTED_MCP_OTEL_TOOL_ATTRIBUTES: Dict[str, Any] = { - "ai.prompt.tools": ('{"name": "lookup"}', _REDACTED_MCP_OTEL_TOOL), + "ai.prompt.tools": ( + '{"name": "lookup"}', + _REDACTED_MCP_OTEL_TOOL, + json.dumps( + { + **_VERCEL_MCP_OTEL_TOOL, + "args": { + "serverUrl": "https://mcp.example.com", + "authorization": _REDACTED_MCP_CREDENTIAL, + }, + } + ), + ), "llm.tools.0.tool.json_schema": _REDACTED_MCP_OTEL_TOOL, } @@ -558,6 +581,9 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: "llm.prompt_template.template": CANARY, "llm.prompt_template.variables": CANARY, "embedding.embeddings.0.embedding.text": CANARY, + "llm.prompts": CANARY, + "llm.input_messages": CANARY, + "ai.ranking": CANARY, "ai.request.text": CANARY, "ai.documents": CANARY, "ai.response.reasoningDetails": CANARY, From 9abbbf3b614648f54bb517a7ff36fd532edafe3c Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 12:07:04 +0200 Subject: [PATCH 06/10] fix: always redact mcp credentials in otel tool definitions --- posthog/ai/otel/_privacy.py | 99 ++++++++++++++++++++++------ posthog/ai/otel/exporter.py | 4 +- posthog/ai/otel/processor.py | 12 ++-- posthog/test/ai/test_privacy_mode.py | 89 ++++++++++++++++++------- 4 files changed, 152 insertions(+), 52 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index d67011fa2..7e4831128 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -182,34 +182,52 @@ def _gateway_cost_only(value: Any) -> Optional[str]: return json.dumps({"gateway": {"cost": cost}}, separators=(",", ":")) -def _redact_tool_definition_json(value: str) -> Optional[str]: - """A value that cannot be parsed could still hold credentials, so it is dropped.""" +def _redact_tool_definition_json(value: str, drop_unparseable: bool) -> Optional[str]: try: parsed = json.loads(value) except ValueError: - return None + parsed = None if isinstance(parsed, list): redacted = _redact_mcp_tool_credentials(parsed) elif isinstance(parsed, dict): redacted = _redact_mcp_tool(parsed) else: - return None + return None if drop_unparseable else value return value if redacted is parsed else json.dumps(redacted) -def _redact_tool_definitions(value: Any) -> Optional[AttributeValue]: +def _redact_tool_definitions( + value: Any, drop_unparseable: bool +) -> Optional[AttributeValue]: + """ + Mask remote MCP credentials in a tool definition attribute. Under privacy mode a + value that cannot be parsed is dropped, because it could still hold credentials. + """ if isinstance(value, str): - return _redact_tool_definition_json(value) - if isinstance(value, (list, tuple)): - redacted: List[str] = [] - for item in value: - definition = ( - _redact_tool_definition_json(item) if isinstance(item, str) else None - ) - if definition is not None: - redacted.append(definition) - return redacted - return None + return _redact_tool_definition_json(value, drop_unparseable) + if not isinstance(value, (list, tuple)): + return None if drop_unparseable else value + redacted: List[Any] = [] + for item in value: + definition = ( + _redact_tool_definition_json(item, drop_unparseable) + if isinstance(item, str) + else None + if drop_unparseable + else item + ) + if definition is not None: + redacted.append(definition) + if len(redacted) == len(value) and all(a is b for a, b in zip(redacted, value)): + return value + return redacted + + +def _is_tool_definition_attribute(key: str) -> bool: + return ( + key in _TOOL_DEFINITION_ATTRIBUTES + or _TOOL_DEFINITION_PATTERN.fullmatch(key) is not None + ) def _without_content(attributes: Attributes) -> BoundedAttributes: @@ -217,10 +235,8 @@ def _without_content(attributes: Attributes) -> BoundedAttributes: for key, value in (attributes or {}).items(): if _is_content_attribute(key): continue - if key in _TOOL_DEFINITION_ATTRIBUTES or _TOOL_DEFINITION_PATTERN.fullmatch( - key - ): - definitions = _redact_tool_definitions(value) + if _is_tool_definition_attribute(key): + definitions = _redact_tool_definitions(value, drop_unparseable=True) if definitions is not None: kept[key] = definitions continue @@ -233,7 +249,7 @@ def _without_content(attributes: Attributes) -> BoundedAttributes: return BoundedAttributes(attributes=kept) -def _redact_span(span: ReadableSpan) -> ReadableSpan: +def _without_content_span(span: ReadableSpan) -> ReadableSpan: """ Return a copy of ``span`` without content attributes, non-exception events, exception messages and stack traces, or the status description. @@ -268,3 +284,44 @@ def _redact_span(span: ReadableSpan) -> ReadableSpan: end_time=span.end_time, instrumentation_scope=span.instrumentation_scope, ) + + +def _without_mcp_credentials_span(span: ReadableSpan) -> ReadableSpan: + """Return ``span`` with remote MCP credentials masked in its tool definitions.""" + attributes = dict(span.attributes or {}) + redacted: Dict[str, Any] = {} + for key, value in attributes.items(): + if not _is_tool_definition_attribute(key): + redacted[key] = value + continue + definitions = _redact_tool_definitions(value, drop_unparseable=False) + if definitions is not None: + redacted[key] = definitions + if redacted.keys() == attributes.keys() and all( + redacted[key] is attributes[key] for key in attributes + ): + return span + return ReadableSpan( + name=span.name, + context=span.get_span_context(), + parent=span.parent, + resource=span.resource, + attributes=BoundedAttributes(attributes=redacted), + events=BoundedList.from_seq(None, span.events), + links=BoundedList.from_seq(None, span.links), + kind=span.kind, + status=span.status, + start_time=span.start_time, + end_time=span.end_time, + instrumentation_scope=span.instrumentation_scope, + ) + + +def _redact_span(span: ReadableSpan, privacy_mode: bool) -> ReadableSpan: + """ + Mask remote MCP credentials in tool definitions on every export, as for + ``$ai_tools``. Privacy mode also removes all content. + """ + if privacy_mode: + return _without_content_span(span) + return _without_mcp_credentials_span(span) diff --git a/posthog/ai/otel/exporter.py b/posthog/ai/otel/exporter.py index b6cfbe83b..13ef3e3f2 100644 --- a/posthog/ai/otel/exporter.py +++ b/posthog/ai/otel/exporter.py @@ -83,8 +83,8 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: return SpanExportResult.SUCCESS for span in ai_spans: warn_if_posthog_ai_gateway_otel_attributes(span.attributes) - if _otel_privacy_mode_enabled(self._privacy_mode, self._client): - ai_spans = [_redact_span(span) for span in ai_spans] + privacy_mode = _otel_privacy_mode_enabled(self._privacy_mode, self._client) + ai_spans = [_redact_span(span, privacy_mode) for span in ai_spans] return self._exporter.export(ai_spans) def shutdown(self) -> None: diff --git a/posthog/ai/otel/processor.py b/posthog/ai/otel/processor.py index 3024b8efe..ce9aac54e 100644 --- a/posthog/ai/otel/processor.py +++ b/posthog/ai/otel/processor.py @@ -176,12 +176,12 @@ def on_end(self, span: ReadableSpan) -> None: span = with_request_context(span, captured) except Exception: log.debug("Failed to add PostHog context to span", exc_info=True) - if _otel_privacy_mode_enabled(self._privacy_mode, self._client): - try: - span = _redact_span(span) - except Exception: - log.debug("Failed to redact span, dropping it", exc_info=True) - return + privacy_mode = _otel_privacy_mode_enabled(self._privacy_mode, self._client) + try: + span = _redact_span(span, privacy_mode) + except Exception: + log.debug("Failed to redact span, dropping it", exc_info=True) + return self._processor.on_end(span) def shutdown(self) -> None: diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index 1cf08748c..e86542e0c 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -644,6 +644,36 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: span.set_status(Status(StatusCode.ERROR, CANARY)) +def _export_one_span( + integration: str, options: Dict[str, Any], emit: Callable[[Any], None] +) -> Any: + from opentelemetry.sdk.trace import ReadableSpan, TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + from posthog.ai.otel import PostHogSpanProcessor, PostHogTraceExporter + + exported: List[ReadableSpan] = [] + provider = TracerProvider() + with ( + patch("posthog.ai.otel.processor.OTLPSpanExporter"), + patch("posthog.ai.otel.processor.BatchSpanProcessor") as batch, + patch("posthog.ai.otel.exporter.OTLPSpanExporter") as otlp, + ): + batch.return_value.on_end.side_effect = exported.append + otlp.return_value.export.side_effect = exported.extend + if integration == "processor": + provider.add_span_processor( + PostHogSpanProcessor(api_key="phc_test", **options) + ) + else: + provider.add_span_processor( + SimpleSpanProcessor(PostHogTraceExporter(api_key="phc_test", **options)) + ) + emit(provider) + [span] = exported + return span + + @pytest.mark.parametrize( "provider_metadata, kept_provider_metadata", [ @@ -673,10 +703,6 @@ def test_otel_privacy_mode_sends_no_content( pytest.importorskip("opentelemetry.exporter.otlp.proto.http") import posthog from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans - from opentelemetry.sdk.trace import ReadableSpan, TracerProvider - from opentelemetry.sdk.trace.export import SimpleSpanProcessor - - from posthog.ai.otel import PostHogSpanProcessor, PostHogTraceExporter monkeypatch.setattr(posthog, "privacy_mode", privacy_source == "module") options: Dict[str, Any] = { @@ -687,26 +713,12 @@ def test_otel_privacy_mode_sends_no_content( else None ), } - exported: List[ReadableSpan] = [] - provider = TracerProvider() - with ( - patch("posthog.ai.otel.processor.OTLPSpanExporter"), - patch("posthog.ai.otel.processor.BatchSpanProcessor") as batch, - patch("posthog.ai.otel.exporter.OTLPSpanExporter") as otlp, - ): - batch.return_value.on_end.side_effect = exported.append - otlp.return_value.export.side_effect = exported.extend - if integration == "processor": - provider.add_span_processor( - PostHogSpanProcessor(api_key="phc_test", **options) - ) - else: - provider.add_span_processor( - SimpleSpanProcessor(PostHogTraceExporter(api_key="phc_test", **options)) - ) - _emit_canary_span(provider, provider_metadata) + span = _export_one_span( + integration, + options, + lambda provider: _emit_canary_span(provider, provider_metadata), + ) - [span] = exported assert CANARY.encode() not in encode_spans([span]).SerializeToString() assert dict(span.attributes or {}) == { **_KEPT_OTEL_ATTRIBUTES, @@ -720,3 +732,34 @@ def test_otel_privacy_mode_sends_no_content( assert set(exception_attributes) <= {"exception.type", "exception.escaped"} assert span.status.is_ok is False assert span.status.description is None + + +@pytest.mark.parametrize("integration", ["processor", "exporter"]) +def test_otel_masks_mcp_credentials_without_privacy_mode( + integration: str, monkeypatch: pytest.MonkeyPatch +) -> None: + pytest.importorskip("opentelemetry.exporter.otlp.proto.http") + import posthog + from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + + monkeypatch.setattr(posthog, "privacy_mode", False) + attributes = { + **_KEPT_OTEL_ATTRIBUTES, + **_MCP_OTEL_TOOL_ATTRIBUTES, + "gen_ai.input.messages": "user prompt", + } + + def emit(provider: Any) -> None: + with provider.get_tracer("test").start_as_current_span( + "gen_ai.chat", attributes=attributes + ): + pass + + span = _export_one_span(integration, {"privacy_mode": False}, emit) + + encode_spans([span]) + assert dict(span.attributes or {}) == { + **attributes, + **_REDACTED_MCP_OTEL_TOOL_ATTRIBUTES, + "llm.tools.1.tool.json_schema": f"not json {CANARY}", + } From 06218319698ae2351e8855554f3d467e135e59e5 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 12:21:43 +0200 Subject: [PATCH 07/10] fix: strip otel embedding vectors in privacy mode --- posthog/ai/otel/_privacy.py | 2 +- posthog/test/ai/test_privacy_mode.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 7e4831128..a0dbcf04e 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -115,7 +115,7 @@ _CONTENT_ATTRIBUTE_PATTERN = re.compile( "|".join( ( - r"embedding\.embeddings\.\d+\.embedding\.text", + r"embedding\.embeddings\.\d+\.embedding\.(text|vector)", r"retrieval\.documents\.\d+\.document\.(content|metadata)", r"reranker\.(input|output)_documents\.\d+\.document\.content", r"(input|output)\.images\.\d+\.image\.url", diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index e86542e0c..315464c2d 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -581,6 +581,7 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: "llm.prompt_template.template": CANARY, "llm.prompt_template.variables": CANARY, "embedding.embeddings.0.embedding.text": CANARY, + "embedding.embeddings.0.embedding.vector": (0.12, 0.34), "llm.prompts": CANARY, "llm.input_messages": CANARY, "ai.ranking": CANARY, From ff9058f0948519410dec9edf1fd55a5fb188a506 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 12:41:39 +0200 Subject: [PATCH 08/10] fix: drop only the otel span that fails redaction --- posthog/ai/otel/_privacy.py | 18 ++++++++++- posthog/ai/otel/exporter.py | 4 +-- posthog/ai/otel/processor.py | 10 ++----- posthog/test/ai/test_privacy_mode.py | 45 ++++++++++++++++++++++++---- 4 files changed, 61 insertions(+), 16 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index a0dbcf04e..20210a608 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -1,8 +1,9 @@ """Privacy mode for OpenTelemetry spans exported to PostHog.""" import json +import logging import re -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence from opentelemetry.attributes import BoundedAttributes from opentelemetry.sdk.trace import Event, ReadableSpan @@ -15,6 +16,8 @@ if TYPE_CHECKING: from ...client import Client +log = logging.getLogger("posthog") + # Attributes that carry prompts, outputs, tool calls, retrieved documents or error text. # Ingestion keeps unmapped attributes as event properties, so this also covers attributes # it does not map. Tool definitions ($ai_tools) are not content and stay, as in the SDK @@ -325,3 +328,16 @@ def _redact_span(span: ReadableSpan, privacy_mode: bool) -> ReadableSpan: if privacy_mode: return _without_content_span(span) return _without_mcp_credentials_span(span) + + +def _redacted_spans( + spans: Sequence[ReadableSpan], privacy_mode: bool +) -> List[ReadableSpan]: + """Redact each span, dropping only a span whose redaction fails rather than leaking it.""" + redacted = [] + for span in spans: + try: + redacted.append(_redact_span(span, privacy_mode)) + except Exception: + log.debug("Failed to redact span, dropping it", exc_info=True) + return redacted diff --git a/posthog/ai/otel/exporter.py b/posthog/ai/otel/exporter.py index 13ef3e3f2..f5d21183c 100644 --- a/posthog/ai/otel/exporter.py +++ b/posthog/ai/otel/exporter.py @@ -12,7 +12,7 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from ..gateway import warn_if_posthog_ai_gateway_otel_attributes -from ._privacy import _otel_privacy_mode_enabled, _redact_span +from ._privacy import _otel_privacy_mode_enabled, _redacted_spans from .spans import DEFAULT_HOST, is_ai_span if TYPE_CHECKING: @@ -84,7 +84,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: for span in ai_spans: warn_if_posthog_ai_gateway_otel_attributes(span.attributes) privacy_mode = _otel_privacy_mode_enabled(self._privacy_mode, self._client) - ai_spans = [_redact_span(span, privacy_mode) for span in ai_spans] + ai_spans = _redacted_spans(ai_spans, privacy_mode) return self._exporter.export(ai_spans) def shutdown(self) -> None: diff --git a/posthog/ai/otel/processor.py b/posthog/ai/otel/processor.py index ce9aac54e..9c3e73976 100644 --- a/posthog/ai/otel/processor.py +++ b/posthog/ai/otel/processor.py @@ -22,7 +22,7 @@ capture_request_context, with_request_context, ) -from ._privacy import _otel_privacy_mode_enabled, _redact_span +from ._privacy import _otel_privacy_mode_enabled, _redacted_spans from .spans import DEFAULT_HOST, is_ai_span if TYPE_CHECKING: @@ -177,12 +177,8 @@ def on_end(self, span: ReadableSpan) -> None: except Exception: log.debug("Failed to add PostHog context to span", exc_info=True) privacy_mode = _otel_privacy_mode_enabled(self._privacy_mode, self._client) - try: - span = _redact_span(span, privacy_mode) - except Exception: - log.debug("Failed to redact span, dropping it", exc_info=True) - return - self._processor.on_end(span) + for redacted in _redacted_spans([span], privacy_mode): + self._processor.on_end(redacted) def shutdown(self) -> None: """Shut down the underlying batch span processor.""" diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index 315464c2d..e46966ba3 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -645,9 +645,9 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: span.set_status(Status(StatusCode.ERROR, CANARY)) -def _export_one_span( +def _export_spans( integration: str, options: Dict[str, Any], emit: Callable[[Any], None] -) -> Any: +) -> List[Any]: from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor @@ -671,8 +671,7 @@ def _export_one_span( SimpleSpanProcessor(PostHogTraceExporter(api_key="phc_test", **options)) ) emit(provider) - [span] = exported - return span + return exported @pytest.mark.parametrize( @@ -714,7 +713,7 @@ def test_otel_privacy_mode_sends_no_content( else None ), } - span = _export_one_span( + [span] = _export_spans( integration, options, lambda provider: _emit_canary_span(provider, provider_metadata), @@ -756,7 +755,7 @@ def emit(provider: Any) -> None: ): pass - span = _export_one_span(integration, {"privacy_mode": False}, emit) + [span] = _export_spans(integration, {"privacy_mode": False}, emit) encode_spans([span]) assert dict(span.attributes or {}) == { @@ -764,3 +763,37 @@ def emit(provider: Any) -> None: **_REDACTED_MCP_OTEL_TOOL_ATTRIBUTES, "llm.tools.1.tool.json_schema": f"not json {CANARY}", } + + +@pytest.mark.parametrize("privacy_mode", [True, False]) +def test_otel_exporter_drops_only_the_span_that_fails_redaction( + privacy_mode: bool, +) -> None: + pytest.importorskip("opentelemetry.exporter.otlp.proto.http") + from opentelemetry.sdk.trace import TracerProvider + + from posthog.ai.otel import PostHogTraceExporter + + tracer = TracerProvider().get_tracer("test") + broken = tracer.start_span( + "gen_ai.chat", attributes={"ai.prompt.tools": json.dumps(_MCP_OTEL_TOOL)} + ) + healthy = tracer.start_span( + "gen_ai.chat", attributes={"gen_ai.request.model": "gpt-4o"} + ) + broken.end() + healthy.end() + + with ( + patch("posthog.ai.otel.exporter.OTLPSpanExporter") as otlp, + # Deeply nested JSON raises RecursionError while parsing on some Python versions. + patch("posthog.ai.otel._privacy._redact_mcp_tool", side_effect=RecursionError), + ): + PostHogTraceExporter(api_key="phc_test", privacy_mode=privacy_mode).export( + [broken, healthy] + ) + + [spans] = otlp.return_value.export.call_args.args + assert [dict(span.attributes or {}) for span in spans] == [ + {"gen_ai.request.model": "gpt-4o"} + ] From ad27041cb4acc9cfe7acfcd64aa58587b6d53340 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 14:11:41 +0200 Subject: [PATCH 09/10] fix: keep vercel rerank scores in otel privacy mode --- posthog/ai/otel/_privacy.py | 1 - posthog/test/ai/test_privacy_mode.py | 1 - 2 files changed, 2 deletions(-) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 20210a608..537c602cf 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -29,7 +29,6 @@ "ai.prompt.messages", "ai.request.text", "ai.documents", - "ai.ranking", "ai.response.text", "ai.response.object", "ai.response.toolCalls", diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index e46966ba3..e00f3c5f8 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -584,7 +584,6 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: "embedding.embeddings.0.embedding.vector": (0.12, 0.34), "llm.prompts": CANARY, "llm.input_messages": CANARY, - "ai.ranking": CANARY, "ai.request.text": CANARY, "ai.documents": CANARY, "ai.response.reasoningDetails": CANARY, From 139f84601fb522d1505f2c90454afb215f4d4b42 Mon Sep 17 00:00:00 2001 From: Carlos Marchal Date: Fri, 23 Oct 2026 14:29:49 +0200 Subject: [PATCH 10/10] fix: strip logfire payload attributes in otel privacy mode --- posthog/ai/otel/_privacy.py | 2 ++ posthog/test/ai/test_privacy_mode.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/posthog/ai/otel/_privacy.py b/posthog/ai/otel/_privacy.py index 537c602cf..5a2114ee5 100644 --- a/posthog/ai/otel/_privacy.py +++ b/posthog/ai/otel/_privacy.py @@ -73,6 +73,8 @@ "anthropic.tool_result.content", # Pydantic AI / Logfire "pydantic_ai.all_messages", + "request_data", + "response_data", "model_request_parameters", "final_result", "tool_arguments", diff --git a/posthog/test/ai/test_privacy_mode.py b/posthog/test/ai/test_privacy_mode.py index e00f3c5f8..065c79b0d 100644 --- a/posthog/test/ai/test_privacy_mode.py +++ b/posthog/test/ai/test_privacy_mode.py @@ -568,6 +568,8 @@ def _emit_canary_span(provider: Any, provider_metadata: str) -> None: "final_result": CANARY, "tool_arguments": CANARY, "tool_response": CANARY, + "request_data": CANARY, + "response_data": CANARY, "anthropic.tool_result.content": CANARY, "events": CANARY, "error.message": CANARY,