Track server content requests separately in Scarf

docs/scarf-server-tracking.md

@@ -0,0 +1,55 @@
+# Scarf server request tracking
+
+The Cloudflare Worker sends `server_request` events to our existing Scarf
+`site-events` collection after serving a successful GET of HTML, Markdown,
+or an `llms.txt` / `llms-full.txt` document. This includes crawlers that never
+execute JavaScript. The original visitor IP is forwarded in `X-Scarf-IP`,
+and the original user agent is preserved for Scarf's bot classification.
+
+Delivery runs through the existing Worker `waitUntil` helper, with a five-second
+timeout and no retries. Failures do not delay or fail the page response. Logs
+never include the visitor IP or the outgoing request.
+
+## Counting
+
+Filter the Scarf event collection by `event=server_request` and `source=worker`
+for server traffic. `page`, `format`, and `status` describe the response.
+Existing browser `page_view` events and per-product pixels are unchanged.
+
+`traffic_type` uses Cloudflare's request metadata when available:
+`verified_bot`, `suspected_bot` (score 1–29), `likely_human` (score 30–99),
+or `unknown` (missing or uncomputed signals). Availability depends on the
+Cloudflare plan. These are classifications, not proof of a human or a company.
+
+These are separate measurements, not disjoint sets of visitors. A browser's
+initial page request produces a server event and can also produce a browser
+event and pixel. Do not sum these counts as users or visits. This implementation
+does not depend on Scarf deduplicating across event types or pixels, and does
+not claim to deduplicate people by IP. SPA navigations may produce browser
+events without a server document request.
+
+Scarf recognizes known bot user agents. Confirm that the organization's bot
+exclusion setting is off if bot events should be retained, and verify with
+Scarf whether the dashboard supports a bot-only view. No absence of JavaScript
+or browser headers is treated as proof of a bot.
+
+## Coverage and rollout
+
+Only production builds serving `tanstack.com` with a Cloudflare client IP are
+eligible. DNT and GPC opt-outs are skipped. Query strings, cookies, request
+bodies, and authorization headers are not forwarded. Account, auth, admin,
+chat, builder project, API, and internal analytics paths are excluded.
+Assets, redirects, and failed responses are excluded too.
+
+This measures content responses reaching the Worker. Cloudflare requests
+blocked or served before this entry point will not appear. It is not a
+replacement for Cloudflare's total request count or a historical log import.
+
+After deployment, verify `server_request` events in Scarf using normal site
+traffic, including an HTML page and Markdown docs. Check the original user
+agent is retained, bot filtering is configured as intended, and browser
+page-view/pixel counts have not gained a second emission. Live dashboard
+verification is required to confirm ingestion and filtering.
+
+References: [Scarf Gateway](https://docs.scarf.sh/gateway/),
+[bot filtering](https://docs.scarf.sh/user_best_practices/#excluding-all-known-bot-traffic).

src/server.ts

@@ -11,7 +11,12 @@ import { runWithDatabaseContext, runWithDatabaseRequest } from '~/db/client'
 import {
   runWithHostRuntimeContext,
   runWithHostRuntimeEnv,
+  scheduleHostRuntimeTask,
 } from '~/server/runtime/host.server'
+import {
+  createScarfServerRequest,
+  sendScarfServerRequest,
+} from '~/utils/analytics/scarf-server'
 import {
   installProductionFetchProbe,
   installProductionProcessProbe,
@@ -297,9 +302,21 @@ export default {
     }
     return runWithHostRuntimeEnv(env, () =>
       runWithHostRuntimeContext(context, () =>
-        withDocsRequestTimings(request, async () =>
-          finalizeDocsHtmlHeaders(request, await server.fetch(request)),
-        ),
+        withDocsRequestTimings(request, async () => {
+          const response = finalizeDocsHtmlHeaders(
+            request,
+            await server.fetch(request),
+          )
+          if (import.meta.env.PROD && import.meta.env.MODE !== 'staging') {
+            const scarfRequest = createScarfServerRequest(request, response)
+            if (scarfRequest) {
+              scheduleHostRuntimeTask(() =>
+                sendScarfServerRequest(scarfRequest),
+              )
+            }
+          }
+          return response
+        }),
       ),
     )
   },

src/utils/analytics/scarf-server.ts

@@ -0,0 +1,88 @@
+// Server requests and browser page views are separate measurements. Never emit
+// page_view here or add these counts to pixel views to calculate unique visitors.
+export function createScarfServerRequest(request: Request, response: Response) {
+  const url = new URL(request.url)
+  const clientIp = request.headers.get('cf-connecting-ip')
+  if (
+    url.hostname !== 'tanstack.com' ||
+    request.method !== 'GET' ||
+    !response.ok ||
+    !clientIp ||
+    request.headers.get('dnt') === '1' ||
+    request.headers.get('sec-gpc') === '1'
+  ) {
+    return null
+  }
+
+  // Keep account, application, and internal requests out of content analytics.
+  if (
+    /^\/(?:api|_a|_server|admin|account|auth|oauth|login|dashboard|chat|chat-access)(?:\/|$)/.test(
+      url.pathname,
+    ) ||
+    url.pathname.startsWith('/builder/')
+  ) {
+    return null
+  }
+
+  const contentType = response.headers.get('content-type')?.split(';')[0].trim()
+  const isMarkdown = contentType === 'text/markdown'
+  const isAgentText =
+    contentType === 'text/plain' &&
+    (url.pathname.endsWith('.md') ||
+      /\/llms(?:-full)?\.txt$/.test(url.pathname))
+  if (contentType !== 'text/html' && !isMarkdown && !isAgentText) return null
+
+  const eventUrl = new URL('https://tanstack.gateway.scarf.sh/site-events')
+  eventUrl.searchParams.set('event', 'server_request')
+  eventUrl.searchParams.set('source', 'worker')
+  eventUrl.searchParams.set('page', url.pathname)
+  eventUrl.searchParams.set(
+    'format',
+    contentType === 'text/html' ? 'html' : 'markdown',
+  )
+  eventUrl.searchParams.set('status', String(response.status))
+  let trafficType = 'unknown'
+  if (
+    'cf' in request &&
+    typeof request.cf === 'object' &&
+    request.cf !== null
+  ) {
+    const cf = request.cf
+    if (
+      'botManagement' in cf &&
+      typeof cf.botManagement === 'object' &&
+      cf.botManagement !== null
+    ) {
+      const bot = cf.botManagement
+      const score =
+        'score' in bot && typeof bot.score === 'number' ? bot.score : 0
+      if ('verifiedBot' in bot && bot.verifiedBot === true) {
+        trafficType = 'verified_bot'
+      } else if (score >= 1 && score <= 99) {
+        trafficType = score < 30 ? 'suspected_bot' : 'likely_human'
+      }
+    }
+  }
+  eventUrl.searchParams.set('traffic_type', trafficType)
+
+  const headers = new Headers({ 'X-Scarf-IP': clientIp })
+  for (const name of ['user-agent', 'dnt', 'sec-gpc']) {
+    const value = request.headers.get(name)
+    if (value) headers.set(name, value)
+  }
+
+  return new Request(eventUrl, { method: 'POST', headers })
+}
+
+export async function sendScarfServerRequest(request: Request) {
+  try {
+    const response = await fetch(request, { signal: AbortSignal.timeout(5000) })
+    await response.body?.cancel()
+    if (!response.ok) {
+      console.warn('Scarf server tracking failed', { status: response.status })
+    }
+  } catch {
+    // Do not log the request, which carries the visitor's IP address.
+    console.warn('Scarf server tracking failed')
+  }
+}

tests/scarf-server.test.ts

@@ -0,0 +1,156 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import {
+  createScarfServerRequest,
+  sendScarfServerRequest,
+} from '../src/utils/analytics/scarf-server.ts'
+
+function incoming(
+  path = '/query/latest/docs',
+  headers: Record<string, string> = {},
+) {
+  return new Request(`https://tanstack.com${path}`, {
+    headers: {
+      'cf-connecting-ip': '192.0.2.1',
+      'user-agent': 'ExampleCrawler/1.0',
+      ...headers,
+    },
+  })
+}
+
+function content(type = 'text/html', status = 200) {
+  return new Response('content', { status, headers: { 'content-type': type } })
+}
+
+test('forwards the visitor IP and user agent, without queries or cookies', () => {
+  const event = createScarfServerRequest(
+    incoming('/query/latest/docs?token=secret', { cookie: 'session=secret' }),
+    content(),
+  )
+  assert.ok(event)
+  const url = new URL(event.url)
+  assert.equal(url.searchParams.get('event'), 'server_request')
+  assert.equal(url.searchParams.get('source'), 'worker')
+  assert.equal(url.searchParams.get('page'), '/query/latest/docs')
+  assert.equal(event.headers.get('X-Scarf-IP'), '192.0.2.1')
+  assert.equal(event.headers.get('user-agent'), 'ExampleCrawler/1.0')
+  assert.equal(event.headers.get('cookie'), null)
+  assert.ok(!event.url.includes('secret'))
+})
+
+test('browser requests stay in the server request dataset, never page_view or pixels', () => {
+  const event = createScarfServerRequest(
+    incoming('/', {
+      'sec-fetch-dest': 'document',
+      'user-agent': 'Mozilla/5.0',
+    }),
+    content(),
+  )
+  assert.ok(event)
+  const url = new URL(event.url)
+  assert.equal(url.pathname, '/site-events')
+  assert.equal(url.searchParams.get('event'), 'server_request')
+  assert.equal(url.searchParams.has('x-pxid'), false)
+})
+
+test('counts markdown negotiation and agent text, but not assets or internal requests', () => {
+  for (const [path, type] of [
+    ['/query/latest/docs', 'text/markdown; charset=utf-8'],
+    ['/query/latest/docs/overview.md', 'text/plain'],
+    ['/llms.txt', 'text/plain'],
+    ['/llms-full.txt', 'text/plain'],
+  ]) {
+    assert.ok(createScarfServerRequest(incoming(path), content(type)), path)
+  }
+  for (const path of [
+    '/api/example',
+    '/_a/scarf/clipboard',
+    '/account',
+    '/admin/users',
+    '/chat/c/private',
+    '/builder/private-project',
+  ]) {
+    assert.equal(
+      createScarfServerRequest(incoming(path), content()),
+      null,
+      path,
+    )
+  }
+  for (const type of [
+    'application/json',
+    'image/png',
+    'text/javascript',
+    'text/plain',
+  ]) {
+    assert.equal(
+      createScarfServerRequest(incoming('/asset'), content(type)),
+      null,
+    )
+  }
+})
+
+test('skips opt-outs, missing IPs, non-production hosts, failed responses and non-GET requests', () => {
+  for (const header of ['dnt', 'sec-gpc']) {
+    assert.equal(
+      createScarfServerRequest(incoming('/', { [header]: '1' }), content()),
+      null,
+    )
+  }
+  assert.equal(
+    createScarfServerRequest(new Request('https://tanstack.com/'), content()),
+    null,
+  )
+  assert.equal(
+    createScarfServerRequest(
+      new Request('https://preview.workers.dev/', {
+        headers: { 'cf-connecting-ip': '192.0.2.1' },
+      }),
+      content(),
+    ),
+    null,
+  )
+  for (const status of [301, 404, 500]) {
+    assert.equal(
+      createScarfServerRequest(incoming(), content('text/html', status)),
+      null,
+    )
+  }
+  for (const method of ['HEAD', 'POST', 'OPTIONS']) {
+    assert.equal(
+      createScarfServerRequest(new Request(incoming(), { method }), content()),
+      null,
+    )
+  }
+})
+
+test('delivery failures do not reject or leak the visitor IP', async (context) => {
+  const warnings: unknown[][] = []
+  context.mock.method(console, 'warn', (...args: unknown[]) =>
+    warnings.push(args),
+  )
+  context.mock.method(globalThis, 'fetch', async () => {
+    throw new Error('192.0.2.1')
+  })
+  await assert.doesNotReject(sendScarfServerRequest(incoming()))
+  assert.equal(warnings.length, 1)
+  assert.ok(!JSON.stringify(warnings).includes('192.0.2.1'))
+})
+
+test('uses Cloudflare bot signals without treating missing signals as human or bot', () => {
+  for (const [botManagement, expected] of [
+    [{ verifiedBot: true, score: 99 }, 'verified_bot'],
+    [{ verifiedBot: false, score: 1 }, 'suspected_bot'],
+    [{ verifiedBot: false, score: 29 }, 'suspected_bot'],
+    [{ verifiedBot: false, score: 30 }, 'likely_human'],
+    [{ verifiedBot: false, score: 0 }, 'unknown'],
+    [{}, 'unknown'],
+  ]) {
+    const request = Object.assign(incoming(), { cf: { botManagement } })
+    const event = createScarfServerRequest(request, content())
+    assert.ok(event)
+    assert.equal(new URL(event.url).searchParams.get('traffic_type'), expected)
+  }
+  const event = createScarfServerRequest(incoming(), content())
+  assert.ok(event)
+  assert.equal(new URL(event.url).searchParams.get('traffic_type'), 'unknown')
+})