From c0e667a4d0a95da733718d36f1d1d5cb4cd96092 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Wed, 21 Oct 2026 12:50:30 -0400 Subject: [PATCH 1/4] feat(ai): record the Anthropic web search count in the LangChain callback handler Co-Authored-By: Claude Sonnet 5.5 --- .../changesets/langchain-web-search-count.md | 5 ++ posthog/ai/langchain/callbacks.py | 16 +++++ posthog/test/ai/langchain/test_callbacks.py | 62 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 .sampo/changesets/langchain-web-search-count.md diff --git a/.sampo/changesets/langchain-web-search-count.md b/.sampo/changesets/langchain-web-search-count.md new file mode 100644 index 000000000..465d4d912 --- /dev/null +++ b/.sampo/changesets/langchain-web-search-count.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +The LangChain callback handler now records `$ai_web_search_count` for Anthropic responses that report `server_tool_use.web_search_requests`, as the OpenAI, Anthropic and Gemini wrappers and `posthog-js` already do. diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 3b9e188b0..88cd6c0d0 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -706,6 +706,8 @@ def _capture_generation( ) if usage.reasoning_tokens is not None: event_properties["$ai_reasoning_tokens"] = usage.reasoning_tokens + if usage.web_search_count is not None: + event_properties["$ai_web_search_count"] = usage.web_search_count # Generation results generation_result = output.generations[-1] @@ -838,6 +840,7 @@ class ModelUsage: reasoning_tokens: Optional[int] cache_write_5m_tokens: Optional[int] = None cache_write_1h_tokens: Optional[int] = None + web_search_count: Optional[int] = None def _parse_usage_model( @@ -906,6 +909,18 @@ def _parse_usage_model( ): parsed_usage["reasoning"] = usage["output_token_details"].get("reasoning") + # Anthropic reports the exact number of web searches of a response + server_tool_use = usage.get("server_tool_use") + web_search_requests = ( + server_tool_use.get("web_search_requests") + if isinstance(server_tool_use, dict) + else getattr(server_tool_use, "web_search_requests", None) + ) + if isinstance(web_search_requests, int) and not isinstance( + web_search_requests, bool + ): + parsed_usage["web_search"] = web_search_requests + field_mapping = { "input": "input_tokens", "output": "output_tokens", @@ -920,6 +935,7 @@ def _parse_usage_model( }, cache_write_5m_tokens=parsed_usage.get("cache_write_5m"), cache_write_1h_tokens=parsed_usage.get("cache_write_1h"), + web_search_count=parsed_usage.get("web_search"), ) # For Anthropic providers, LangChain reports input_tokens as the sum of all input tokens. # Our cost calculation expects them to be separate for Anthropic, so we subtract cache tokens. diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 2556294ae..e08ff3239 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2953,3 +2953,65 @@ def test_stop_reason_resolution( props = mock_client.capture.call_args.kwargs["properties"] assert props.get("$ai_stop_reason") == expected + + +class TestWebSearchCount: + @staticmethod + def _captured_properties(llm_output): + from unittest.mock import MagicMock + from uuid import uuid4 + + from langchain_core.messages import AIMessage + from langchain_core.outputs import ChatGeneration, LLMResult + + from posthog.ai.langchain import CallbackHandler + + client = MagicMock() + client.privacy_mode = False + callback = CallbackHandler(client) + run_id = uuid4() + callback._set_llm_metadata( + serialized={}, + run_id=run_id, + messages=[{"role": "user", "content": "test"}], + metadata={"ls_provider": "anthropic", "ls_model_name": "claude-sonnet"}, + ) + response = LLMResult( + generations=[[ChatGeneration(message=AIMessage(content="r"))]], + llm_output=llm_output, + ) + callback._pop_run_and_capture_generation(run_id, None, response) + return client.capture.call_args_list[0][1]["properties"] + + def test_records_the_web_searches_of_an_anthropic_response(self): + properties = self._captured_properties( + { + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "server_tool_use": {"web_search_requests": 2}, + } + } + ) + + assert properties["$ai_web_search_count"] == 2 + + def test_records_zero_when_anthropic_reports_no_searches(self): + properties = self._captured_properties( + { + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "server_tool_use": {"web_search_requests": 0}, + } + } + ) + + assert properties["$ai_web_search_count"] == 0 + + def test_omits_the_count_when_the_provider_reports_none(self): + properties = self._captured_properties( + {"usage": {"input_tokens": 10, "output_tokens": 5}} + ) + + assert "$ai_web_search_count" not in properties From 269756af43b68d74186b7577a33573ca2c48caef Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Wed, 21 Oct 2026 15:10:04 -0400 Subject: [PATCH 2/4] chore: update public API snapshot for ModelUsage.web_search_count Co-Authored-By: Claude Opus 5.5 --- references/public_api_snapshot.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index c745fa8b6..555d19868 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -593,6 +593,7 @@ attribute posthog.ai.langchain.callbacks.ModelUsage.cache_write_tokens: Optional attribute posthog.ai.langchain.callbacks.ModelUsage.input_tokens: Optional[int] attribute posthog.ai.langchain.callbacks.ModelUsage.output_tokens: Optional[int] attribute posthog.ai.langchain.callbacks.ModelUsage.reasoning_tokens: Optional[int] +attribute posthog.ai.langchain.callbacks.ModelUsage.web_search_count: Optional[int] = None attribute posthog.ai.langchain.callbacks.RunMetadata = Union[SpanMetadata, GenerationMetadata] attribute posthog.ai.langchain.callbacks.RunMetadataStorage = Dict[UUID, RunMetadata] attribute posthog.ai.langchain.callbacks.SpanMetadata.end_time: Optional[float] From 96329def35f4c5c61502acfc3d318bd0cce4314f Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Fri, 23 Oct 2026 11:00:22 +0200 Subject: [PATCH 3/4] chore: regenerate LangChain ModelUsage API snapshot --- references/public_api_snapshot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 555d19868..0a75c17d0 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -1231,7 +1231,7 @@ class posthog.ai.gemini.gemini_converter.GeminiMessage class posthog.ai.gemini.gemini_converter.GeminiPart class posthog.ai.langchain.callbacks.CallbackHandler(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[Dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None) class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None) -class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None) +class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None, web_search_count: Optional[int] = None) class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any]) class posthog.ai.langchain.middleware.PostHogMiddleware(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[dict[str, Any]] = None) class posthog.ai.openai.openai.OpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) From aa2a6c4d17be9222f3d4106740e1957910f231b0 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Fri, 23 Oct 2026 20:07:10 -0400 Subject: [PATCH 4/4] fix(ai): read the LangChain web search count from llm_output usage and omit zero Co-Authored-By: Claude Sonnet 5.5 --- posthog/ai/langchain/callbacks.py | 39 ++++++-- posthog/test/ai/langchain/test_callbacks.py | 103 +++++++++----------- 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 6bab4639e..8f51fd6b3 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -706,7 +706,7 @@ def _capture_generation( ) if usage.reasoning_tokens is not None: event_properties["$ai_reasoning_tokens"] = usage.reasoning_tokens - if usage.web_search_count is not None: + if usage.web_search_count is not None and usage.web_search_count > 0: event_properties["$ai_web_search_count"] = usage.web_search_count # Generation results @@ -843,6 +843,22 @@ class ModelUsage: web_search_count: Optional[int] = None +def _extract_web_search_requests(usage: Union[BaseModel, dict]) -> Optional[int]: + if isinstance(usage, BaseModel): + usage = usage.__dict__ + server_tool_use = usage.get("server_tool_use") + web_search_requests = ( + server_tool_use.get("web_search_requests") + if isinstance(server_tool_use, dict) + else getattr(server_tool_use, "web_search_requests", None) + ) + if isinstance(web_search_requests, int) and not isinstance( + web_search_requests, bool + ): + return web_search_requests + return None + + def _parse_usage_model( usage: Union[BaseModel, dict], provider: Optional[str] = None, @@ -910,15 +926,8 @@ def _parse_usage_model( parsed_usage["reasoning"] = usage["output_token_details"].get("reasoning") # Anthropic reports the exact number of web searches of a response - server_tool_use = usage.get("server_tool_use") - web_search_requests = ( - server_tool_use.get("web_search_requests") - if isinstance(server_tool_use, dict) - else getattr(server_tool_use, "web_search_requests", None) - ) - if isinstance(web_search_requests, int) and not isinstance( - web_search_requests, bool - ): + web_search_requests = _extract_web_search_requests(usage) + if web_search_requests is not None: parsed_usage["web_search"] = web_search_requests field_mapping = { @@ -973,12 +982,19 @@ def _parse_usage( reasoning_tokens=None, ) + # The generations loop below can replace the llm_output usage, which is the + # only place langchain-anthropic reports server_tool_use, so read it here. + web_search_count: Optional[int] = None + if response.llm_output is not None: for key in llm_usage_keys: if response.llm_output.get(key): llm_usage = _parse_usage_model( response.llm_output[key], provider, model ) + web_search_count = _extract_web_search_requests( + response.llm_output[key] + ) break if hasattr(response, "generations"): @@ -1024,6 +1040,9 @@ def _parse_usage( llm_usage = _parse_usage_model(chunk_usage, provider, model) break + if llm_usage.web_search_count is None: + llm_usage.web_search_count = web_search_count + return llm_usage diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 00fc433ec..1980ed425 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2955,66 +2955,55 @@ def test_stop_reason_resolution( assert props.get("$ai_stop_reason") == expected -class TestWebSearchCount: - @staticmethod - def _captured_properties(llm_output): - from unittest.mock import MagicMock - from uuid import uuid4 - - from langchain_core.messages import AIMessage - from langchain_core.outputs import ChatGeneration, LLMResult - - from posthog.ai.langchain import CallbackHandler - - client = MagicMock() - client.privacy_mode = False - callback = CallbackHandler(client) - run_id = uuid4() - callback._set_llm_metadata( - serialized={}, - run_id=run_id, - messages=[{"role": "user", "content": "test"}], - metadata={"ls_provider": "anthropic", "ls_model_name": "claude-sonnet"}, - ) - response = LLMResult( - generations=[[ChatGeneration(message=AIMessage(content="r"))]], - llm_output=llm_output, - ) - callback._pop_run_and_capture_generation(run_id, None, response) - return client.capture.call_args_list[0][1]["properties"] - - def test_records_the_web_searches_of_an_anthropic_response(self): - properties = self._captured_properties( - { - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "server_tool_use": {"web_search_requests": 2}, - } - } - ) - - assert properties["$ai_web_search_count"] == 2 - - def test_records_zero_when_anthropic_reports_no_searches(self): - properties = self._captured_properties( - { - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "server_tool_use": {"web_search_requests": 0}, - } - } - ) +@pytest.mark.parametrize( + "server_tool_use,expected", + [ + # Anthropic reports the exact number of searches + ({"web_search_requests": 2}, 2), + # no search is not worth an event property + ({"web_search_requests": 0}, None), + # the provider reports nothing + (None, None), + ], +) +def test_web_search_count(mock_client, server_tool_use, expected): + from langchain_core.outputs import ChatGeneration, LLMResult - assert properties["$ai_web_search_count"] == 0 + cb = CallbackHandler(mock_client) + run_id = uuid.uuid4() + cb._set_llm_metadata( + serialized={}, + run_id=run_id, + messages=[{"role": "user", "content": "test"}], + metadata={"ls_provider": "anthropic", "ls_model_name": "claude-sonnet"}, + ) + usage = {"input_tokens": 10, "output_tokens": 5} + if server_tool_use is not None: + usage["server_tool_use"] = server_tool_use + # langchain-anthropic always sets usage_metadata on the message, which the + # generations loop prefers over llm_output["usage"] + response = LLMResult( + generations=[ + [ + ChatGeneration( + message=AIMessage( + content="r", + usage_metadata={ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + }, + ) + ) + ] + ], + llm_output={"usage": usage}, + ) - def test_omits_the_count_when_the_provider_reports_none(self): - properties = self._captured_properties( - {"usage": {"input_tokens": 10, "output_tokens": 5}} - ) + cb._pop_run_and_capture_generation(run_id, None, response) - assert "$ai_web_search_count" not in properties + props = mock_client.capture.call_args.kwargs["properties"] + assert props.get("$ai_web_search_count") == expected class TestGetHttpStatus: