diff --git a/docs/scarf-server-tracking.md b/docs/scarf-server-tracking.md new file mode 100644 index 000000000..c02e78136 --- /dev/null +++ b/docs/scarf-server-tracking.md @@ -0,0 +1,55 @@ +# Scarf server request tracking + +The Cloudflare Worker sends `server_request` events to our existing Scarf +`site-events` collection after serving a successful GET of HTML, Markdown, +or an `llms.txt` / `llms-full.txt` document. This includes crawlers that never +execute JavaScript. The original visitor IP is forwarded in `X-Scarf-IP`, +and the original user agent is preserved for Scarf's bot classification. + +Delivery runs through the existing Worker `waitUntil` helper, with a five-second +timeout and no retries. Failures do not delay or fail the page response. Logs +never include the visitor IP or the outgoing request. + +## Counting + +Filter the Scarf event collection by `event=server_request` and `source=worker` +for server traffic. `page`, `format`, and `status` describe the response. +Existing browser `page_view` events and per-product pixels are unchanged. + +`traffic_type` uses Cloudflare's request metadata when available: +`verified_bot`, `suspected_bot` (score 1–29), `likely_human` (score 30–99), +or `unknown` (missing or uncomputed signals). Availability depends on the +Cloudflare plan. These are classifications, not proof of a human or a company. + +These are separate measurements, not disjoint sets of visitors. A browser's +initial page request produces a server event and can also produce a browser +event and pixel. Do not sum these counts as users or visits. This implementation +does not depend on Scarf deduplicating across event types or pixels, and does +not claim to deduplicate people by IP. SPA navigations may produce browser +events without a server document request. + +Scarf recognizes known bot user agents. Confirm that the organization's bot +exclusion setting is off if bot events should be retained, and verify with +Scarf whether the dashboard supports a bot-only view. No absence of JavaScript +or browser headers is treated as proof of a bot. + +## Coverage and rollout + +Only production builds serving `tanstack.com` with a Cloudflare client IP are +eligible. DNT and GPC opt-outs are skipped. Query strings, cookies, request +bodies, and authorization headers are not forwarded. Account, auth, admin, +chat, builder project, API, and internal analytics paths are excluded. +Assets, redirects, and failed responses are excluded too. + +This measures content responses reaching the Worker. Cloudflare requests +blocked or served before this entry point will not appear. It is not a +replacement for Cloudflare's total request count or a historical log import. + +After deployment, verify `server_request` events in Scarf using normal site +traffic, including an HTML page and Markdown docs. Check the original user +agent is retained, bot filtering is configured as intended, and browser +page-view/pixel counts have not gained a second emission. Live dashboard +verification is required to confirm ingestion and filtering. + +References: [Scarf Gateway](https://docs.scarf.sh/gateway/), +[bot filtering](https://docs.scarf.sh/user_best_practices/#excluding-all-known-bot-traffic). diff --git a/src/server.ts b/src/server.ts index c52e22a04..6e17c7688 100644 --- a/src/server.ts +++ b/src/server.ts @@ -11,7 +11,12 @@ import { runWithDatabaseContext, runWithDatabaseRequest } from '~/db/client' import { runWithHostRuntimeContext, runWithHostRuntimeEnv, + scheduleHostRuntimeTask, } from '~/server/runtime/host.server' +import { + createScarfServerRequest, + sendScarfServerRequest, +} from '~/utils/analytics/scarf-server' import { installProductionFetchProbe, installProductionProcessProbe, @@ -297,9 +302,21 @@ export default { } return runWithHostRuntimeEnv(env, () => runWithHostRuntimeContext(context, () => - withDocsRequestTimings(request, async () => - finalizeDocsHtmlHeaders(request, await server.fetch(request)), - ), + withDocsRequestTimings(request, async () => { + const response = finalizeDocsHtmlHeaders( + request, + await server.fetch(request), + ) + if (import.meta.env.PROD && import.meta.env.MODE !== 'staging') { + const scarfRequest = createScarfServerRequest(request, response) + if (scarfRequest) { + scheduleHostRuntimeTask(() => + sendScarfServerRequest(scarfRequest), + ) + } + } + return response + }), ), ) }, diff --git a/src/utils/analytics/scarf-server.ts b/src/utils/analytics/scarf-server.ts new file mode 100644 index 000000000..b0b3f42cc --- /dev/null +++ b/src/utils/analytics/scarf-server.ts @@ -0,0 +1,88 @@ +// Server requests and browser page views are separate measurements. Never emit +// page_view here or add these counts to pixel views to calculate unique visitors. +export function createScarfServerRequest(request: Request, response: Response) { + const url = new URL(request.url) + const clientIp = request.headers.get('cf-connecting-ip') + if ( + url.hostname !== 'tanstack.com' || + request.method !== 'GET' || + !response.ok || + !clientIp || + request.headers.get('dnt') === '1' || + request.headers.get('sec-gpc') === '1' + ) { + return null + } + + // Keep account, application, and internal requests out of content analytics. + if ( + /^\/(?:api|_a|_server|admin|account|auth|oauth|login|dashboard|chat|chat-access)(?:\/|$)/.test( + url.pathname, + ) || + url.pathname.startsWith('/builder/') + ) { + return null + } + + const contentType = response.headers.get('content-type')?.split(';')[0].trim() + const isMarkdown = contentType === 'text/markdown' + const isAgentText = + contentType === 'text/plain' && + (url.pathname.endsWith('.md') || + /\/llms(?:-full)?\.txt$/.test(url.pathname)) + if (contentType !== 'text/html' && !isMarkdown && !isAgentText) return null + + const eventUrl = new URL('https://tanstack.gateway.scarf.sh/site-events') + eventUrl.searchParams.set('event', 'server_request') + eventUrl.searchParams.set('source', 'worker') + eventUrl.searchParams.set('page', url.pathname) + eventUrl.searchParams.set( + 'format', + contentType === 'text/html' ? 'html' : 'markdown', + ) + eventUrl.searchParams.set('status', String(response.status)) + let trafficType = 'unknown' + if ( + 'cf' in request && + typeof request.cf === 'object' && + request.cf !== null + ) { + const cf = request.cf + if ( + 'botManagement' in cf && + typeof cf.botManagement === 'object' && + cf.botManagement !== null + ) { + const bot = cf.botManagement + const score = + 'score' in bot && typeof bot.score === 'number' ? bot.score : 0 + if ('verifiedBot' in bot && bot.verifiedBot === true) { + trafficType = 'verified_bot' + } else if (score >= 1 && score <= 99) { + trafficType = score < 30 ? 'suspected_bot' : 'likely_human' + } + } + } + eventUrl.searchParams.set('traffic_type', trafficType) + + const headers = new Headers({ 'X-Scarf-IP': clientIp }) + for (const name of ['user-agent', 'dnt', 'sec-gpc']) { + const value = request.headers.get(name) + if (value) headers.set(name, value) + } + + return new Request(eventUrl, { method: 'POST', headers }) +} + +export async function sendScarfServerRequest(request: Request) { + try { + const response = await fetch(request, { signal: AbortSignal.timeout(5000) }) + await response.body?.cancel() + if (!response.ok) { + console.warn('Scarf server tracking failed', { status: response.status }) + } + } catch { + // Do not log the request, which carries the visitor's IP address. + console.warn('Scarf server tracking failed') + } +} diff --git a/tests/scarf-server.test.ts b/tests/scarf-server.test.ts new file mode 100644 index 000000000..59d54dc5c --- /dev/null +++ b/tests/scarf-server.test.ts @@ -0,0 +1,156 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + createScarfServerRequest, + sendScarfServerRequest, +} from '../src/utils/analytics/scarf-server.ts' + +function incoming( + path = '/query/latest/docs', + headers: Record = {}, +) { + return new Request(`https://tanstack.com${path}`, { + headers: { + 'cf-connecting-ip': '192.0.2.1', + 'user-agent': 'ExampleCrawler/1.0', + ...headers, + }, + }) +} + +function content(type = 'text/html', status = 200) { + return new Response('content', { status, headers: { 'content-type': type } }) +} + +test('forwards the visitor IP and user agent, without queries or cookies', () => { + const event = createScarfServerRequest( + incoming('/query/latest/docs?token=secret', { cookie: 'session=secret' }), + content(), + ) + assert.ok(event) + const url = new URL(event.url) + assert.equal(url.searchParams.get('event'), 'server_request') + assert.equal(url.searchParams.get('source'), 'worker') + assert.equal(url.searchParams.get('page'), '/query/latest/docs') + assert.equal(event.headers.get('X-Scarf-IP'), '192.0.2.1') + assert.equal(event.headers.get('user-agent'), 'ExampleCrawler/1.0') + assert.equal(event.headers.get('cookie'), null) + assert.ok(!event.url.includes('secret')) +}) + +test('browser requests stay in the server request dataset, never page_view or pixels', () => { + const event = createScarfServerRequest( + incoming('/', { + 'sec-fetch-dest': 'document', + 'user-agent': 'Mozilla/5.0', + }), + content(), + ) + assert.ok(event) + const url = new URL(event.url) + assert.equal(url.pathname, '/site-events') + assert.equal(url.searchParams.get('event'), 'server_request') + assert.equal(url.searchParams.has('x-pxid'), false) +}) + +test('counts markdown negotiation and agent text, but not assets or internal requests', () => { + for (const [path, type] of [ + ['/query/latest/docs', 'text/markdown; charset=utf-8'], + ['/query/latest/docs/overview.md', 'text/plain'], + ['/llms.txt', 'text/plain'], + ['/llms-full.txt', 'text/plain'], + ]) { + assert.ok(createScarfServerRequest(incoming(path), content(type)), path) + } + for (const path of [ + '/api/example', + '/_a/scarf/clipboard', + '/account', + '/admin/users', + '/chat/c/private', + '/builder/private-project', + ]) { + assert.equal( + createScarfServerRequest(incoming(path), content()), + null, + path, + ) + } + for (const type of [ + 'application/json', + 'image/png', + 'text/javascript', + 'text/plain', + ]) { + assert.equal( + createScarfServerRequest(incoming('/asset'), content(type)), + null, + ) + } +}) + +test('skips opt-outs, missing IPs, non-production hosts, failed responses and non-GET requests', () => { + for (const header of ['dnt', 'sec-gpc']) { + assert.equal( + createScarfServerRequest(incoming('/', { [header]: '1' }), content()), + null, + ) + } + assert.equal( + createScarfServerRequest(new Request('https://tanstack.com/'), content()), + null, + ) + assert.equal( + createScarfServerRequest( + new Request('https://preview.workers.dev/', { + headers: { 'cf-connecting-ip': '192.0.2.1' }, + }), + content(), + ), + null, + ) + for (const status of [301, 404, 500]) { + assert.equal( + createScarfServerRequest(incoming(), content('text/html', status)), + null, + ) + } + for (const method of ['HEAD', 'POST', 'OPTIONS']) { + assert.equal( + createScarfServerRequest(new Request(incoming(), { method }), content()), + null, + ) + } +}) + +test('delivery failures do not reject or leak the visitor IP', async (context) => { + const warnings: unknown[][] = [] + context.mock.method(console, 'warn', (...args: unknown[]) => + warnings.push(args), + ) + context.mock.method(globalThis, 'fetch', async () => { + throw new Error('192.0.2.1') + }) + await assert.doesNotReject(sendScarfServerRequest(incoming())) + assert.equal(warnings.length, 1) + assert.ok(!JSON.stringify(warnings).includes('192.0.2.1')) +}) + +test('uses Cloudflare bot signals without treating missing signals as human or bot', () => { + for (const [botManagement, expected] of [ + [{ verifiedBot: true, score: 99 }, 'verified_bot'], + [{ verifiedBot: false, score: 1 }, 'suspected_bot'], + [{ verifiedBot: false, score: 29 }, 'suspected_bot'], + [{ verifiedBot: false, score: 30 }, 'likely_human'], + [{ verifiedBot: false, score: 0 }, 'unknown'], + [{}, 'unknown'], + ]) { + const request = Object.assign(incoming(), { cf: { botManagement } }) + const event = createScarfServerRequest(request, content()) + assert.ok(event) + assert.equal(new URL(event.url).searchParams.get('traffic_type'), expected) + } + const event = createScarfServerRequest(incoming(), content()) + assert.ok(event) + assert.equal(new URL(event.url).searchParams.get('traffic_type'), 'unknown') +})