Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@
"@lavamoat/preinstall-always-fail": false,
"@langfuse/otel>@opentelemetry/exporter-trace-otlp-http>@opentelemetry/otlp-transformer>protobufjs": false,
"@opentelemetry/sdk-node>@opentelemetry/exporter-logs-otlp-grpc>@grpc/grpc-js>@grpc/proto-loader>protobufjs": false,
"eslint-plugin-import-x>unrs-resolver": false
"eslint-plugin-import-x>unrs-resolver": false,
"@langfuse/otel>@opentelemetry/otlp-exporter-base>@opentelemetry/otlp-transformer>protobufjs": false
}
}
}
14 changes: 12 additions & 2 deletions src/telemetry/message-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,11 @@ describe('createMessageHandler', () => {
});

expect(tracingMocks.traceSpan).toHaveBeenCalledWith(
expect.objectContaining({ sessionId: 'session-1', userId: 'user-1' }),
expect.objectContaining({
sessionId: 'session-1',
userId: 'user-1',
traceName: 'agent-runner',
}),
expect.any(Function),
);
expect(tracingMocks.createSessionSpan).toHaveBeenCalledWith(
Expand Down Expand Up @@ -661,7 +665,7 @@ describe('createMessageHandler', () => {
expect(tracingMocks.createSessionSpan).not.toHaveBeenCalled();
});

it('propagates custom traceName to createSessionSpan', () => {
it('propagates custom traceName to the root observation and trace context', () => {
const handler = createHandler({ traceName: 'my-eval-trace' });
handler.handleMessage({ type: 'init', sessionId: 'session-1' });

Expand All @@ -671,6 +675,10 @@ describe('createMessageHandler', () => {
expect.any(Object),
false,
);
expect(tracingMocks.traceSpan).toHaveBeenCalledWith(
expect.objectContaining({ traceName: 'my-eval-trace' }),
expect.any(Function),
);
});

it('propagates metadata, tags, and version to traceSpan', () => {
Expand Down Expand Up @@ -738,6 +746,7 @@ describe('createMessageHandler', () => {

it('propagates trace attributes on generation spans', () => {
const handler = createHandler({
traceName: 'my-eval-trace',
traceTags: ['eval'],
traceVersion: '2.0.0',
});
Expand All @@ -747,6 +756,7 @@ describe('createMessageHandler', () => {
const generationTraceCall = tracingMocks.traceSpan.mock.calls[1];
expect(generationTraceCall?.[0]).toStrictEqual(
expect.objectContaining({
traceName: 'my-eval-trace',
tags: ['eval'],
version: '2.0.0',
}),
Expand Down
7 changes: 5 additions & 2 deletions src/telemetry/message-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export type MessageHandlerConfig = {
*/
initialSessionId: string | undefined;
/**
* Name for the root Langfuse trace observation.
* Name for the Langfuse trace and its root observation.
*/
traceName?: string | undefined;
/**
Expand Down Expand Up @@ -166,6 +166,7 @@ export function createMessageHandler(
config: MessageHandlerConfig,
): MessageHandler {
const scrubCredential = createCredentialScrubber();
const traceName = config.traceName ?? 'agent-runner';
/**
* Applies credential redaction followed by the caller's redactor.
*
Expand Down Expand Up @@ -244,13 +245,14 @@ export function createMessageHandler(
{
sessionId: state.langfuseSessionId,
userId: config.userId,
traceName,
metadata,
tags: config.traceTags,
version: config.traceVersion,
},
() => {
const session = createSessionSpan(
config.traceName ?? 'agent-runner',
traceName,
config.redact ? config.prompt : scrubText(config.prompt),
{
model: message.model ?? config.model,
Expand Down Expand Up @@ -284,6 +286,7 @@ export function createMessageHandler(
{
sessionId: state.langfuseSessionId,
userId: config.userId,
traceName,
metadata: config.traceMetadata,
tags: config.traceTags,
version: config.traceVersion,
Expand Down
104 changes: 104 additions & 0 deletions src/telemetry/trace-name.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import { LangfuseSpanProcessor } from '@langfuse/otel';
import { NodeSDK } from '@opentelemetry/sdk-node';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';

import { createMessageHandler } from './message-handler.js';
import { setLangfuseProcessor } from './tracing.js';

type CapturedSpan = {
name: string;
traceId: string;
attributes: Record<string, unknown>;
};

describe('Langfuse trace-name attributes', () => {
const spans: CapturedSpan[] = [];
const processor = new LangfuseSpanProcessor({
exporter: {
export: (batch, callback): void => {
for (const span of batch) {
spans.push({
name: span.name,
traceId: span.spanContext().traceId,
attributes: { ...span.attributes },
});
}
callback({ code: 0 });
},
shutdown: async (): Promise<void> => undefined,
},
});
const sdk = new NodeSDK({
autoDetectResources: false,
spanProcessors: [processor],
metricReaders: [],
logRecordProcessors: [],
});

beforeAll(() => {
sdk.start();
setLangfuseProcessor(processor);
});

beforeEach(() => {
spans.length = 0;
});

afterAll(async () => {
setLangfuseProcessor(undefined);
await sdk.shutdown();
});

it('emits a trace name on every observation without leaking it between runs in one session', async () => {
for (const traceName of ['pr-visual-validation.plan', undefined]) {
const handler = createMessageHandler({
prompt: 'Inspect a fixture.',
model: 'fixture-model',
maxTurns: 1,
redact: false,
userId: 'fixture-user',
initialSessionId: 'shared-workflow-session',
traceName,
});
handler.handleMessage({ type: 'init', sessionId: 'sdk-session' });
handler.handleMessage({
type: 'generation',
model: 'fixture-model',
text: '',
toolCalls: [
{ id: 'read-1', name: 'Read', input: { file_path: '/fixture.txt' } },
],
usage: { inputTokens: 1, outputTokens: 1 },
stopReason: 'tool_use',
});
handler.handleMessage({
type: 'tool_result',
toolUseId: 'read-1',
content: 'Fixture contents.',
isError: false,
});
handler.handleMessage({ type: 'result', success: true, result: 'Done.' });
handler.finalizeSessionSpan();
await processor.forceFlush();

const runSpans = spans.filter(
(span) => span.traceId === handler.getState().traceId,
);
const expectedName = traceName ?? 'agent-runner';
expect(runSpans.map((span) => span.name)).toStrictEqual([
'fixture-model',
'Read: fixture.txt',
expectedName,
]);
for (const span of runSpans) {
expect(span.attributes).toMatchObject({
'langfuse.trace.name': expectedName,
'session.id': 'shared-workflow-session',
});
}
}

expect(spans).toHaveLength(6);
expect(new Set(spans.map((span) => span.traceId)).size).toBe(2);
});
});
4 changes: 3 additions & 1 deletion src/telemetry/tracing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,15 @@ describe('tracing helpers', () => {
expect(callback).toHaveBeenCalledOnce();
});

it('traceSpan propagates metadata, tags, and version when provided', () => {
it('traceSpan propagates trace name, metadata, tags, and version when provided', () => {
setLangfuseProcessor({ forceFlush: vi.fn(async () => undefined) });
const callback = vi.fn();

traceSpan(
{
sessionId: 'session-1',
userId: 'user-1',
traceName: 'my-eval-trace',
metadata: { repo: 'metamask' },
tags: ['eval', 'ci'],
version: '1.0.0',
Expand All @@ -120,6 +121,7 @@ describe('tracing helpers', () => {
{
sessionId: 'session-1',
userId: 'user-1',
traceName: 'my-eval-trace',
metadata: { repo: 'metamask' },
tags: ['eval', 'ci'],
version: '1.0.0',
Expand Down
5 changes: 5 additions & 0 deletions src/telemetry/tracing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ export type TraceAttributes = {
sessionId: string | undefined;
/** User identifier for trace attribution. */
userId: string;
/** Trace name shared by all observations in the run. */
traceName?: string | undefined;
/** Arbitrary metadata attached to the trace. */
metadata?: Record<string, string> | undefined;
/** Tags for trace filtering and categorisation. */
Expand Down Expand Up @@ -103,6 +105,9 @@ export function traceSpan(attrs: TraceAttributes, fn: () => void): void {
if (attrs.version !== undefined) {
propagated.version = attrs.version;
}
if (attrs.traceName !== undefined) {
propagated.traceName = attrs.traceName;
}
propagateAttributes(propagated, fn);
} catch {
// Fire-and-forget tracing should not affect agent execution.
Expand Down
2 changes: 1 addition & 1 deletion src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@ export type AgentRunnerConfig<
* Per-run telemetry attributes propagated to Langfuse traces.
*/
export type AgentRunTelemetryAttributes = {
/** Name for the root Langfuse trace. */
/** Name for the Langfuse trace and its root observation. Defaults to `agent-runner`. */
traceName?: string;
/** User identifier propagated to traces. */
userId?: string;
Expand Down
Loading