diff --git a/docs/datamart/operations.md b/docs/datamart/operations.md index 2894ba9..813f9b6 100644 --- a/docs/datamart/operations.md +++ b/docs/datamart/operations.md @@ -15,10 +15,10 @@ or building the separate Next.js web app the PRD describes, is a later step; see ## Configuration -| Variable | Needed for | If unset | -| --- | --- | --- | -| `DATAGOV_API_KEY` | Data.gov tools | They return `configuration_required` (HTTP 503) with a catalog.data.gov handoff. DEMO_KEY is never used as a fallback. | -| `DATAMART_RPM_LOC`, `DATAMART_RPM_DATAGOV` | Optional per-minute request budgets | Defaults of 15/minute each | +| Variable | Needed for | If unset | +| ------------------------------------------ | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| `DATAGOV_API_KEY` | Data.gov tools | They return `configuration_required` (HTTP 503) with a catalog.data.gov handoff. DEMO_KEY is never used as a fallback. | +| `DATAMART_RPM_LOC`, `DATAMART_RPM_DATAGOV` | Optional per-minute request budgets | Defaults of 15/minute each | Put production values in the team vault and the service environment, not a committed `.env`. @@ -29,6 +29,11 @@ committed `.env`. stops calling loc.gov for an hour after any 429 or 503, because loc.gov restarts its one-hour block on every request made during it. Responses are cached (searches 30 min, items 6 h). + On 2026-10-08, right after the first deploy, loc.gov answered the production + host (152.53.47.37) with a browser challenge, while the same calls worked from + a residential IP the day before. Datamart reports `source_unavailable` and + pauses for an hour; it does not bypass the challenge. If it persists, ask the + Library of Congress to allow the host, or serve LoC from another egress. - **Data.gov**: a personal key allows 1,000 requests/hour. Datamart sends at most 15/minute and pauses for an hour after a 429. Searches are cached 10 minutes. diff --git a/src/datamart/http.js b/src/datamart/http.js index 0c0535c..d039c6f 100644 --- a/src/datamart/http.js +++ b/src/datamart/http.js @@ -14,11 +14,7 @@ import { DatamartError } from './errors.js'; export const USER_AGENT = 'datamart.help/0.1 (+https://datamart.help)'; /** Hosts adapters may call. Anything else is refused before a socket opens. */ -export const ALLOWED_HOSTS = new Set([ - 'api.gsa.gov', - 'www.loc.gov', - 'geocoding.geo.census.gov', -]); +export const ALLOWED_HOSTS = new Set(['api.gsa.gov', 'www.loc.gov', 'geocoding.geo.census.gov']); const cache = new Map(); // key -> { expires, value } const MAX_CACHE_ENTRIES = 500; @@ -72,7 +68,8 @@ function spend(provider) { */ function coolDown(provider, retryAfterSeconds) { const floor = limitsFor(provider).cooldownMs; - const asked = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 0; + const asked = + Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 0; const ms = Math.max(floor, asked); cooldowns.set(provider, Date.now() + ms); persistCooldowns(); @@ -117,7 +114,9 @@ export function resetHttpState() { function looksLikeChallenge(contentType, text) { if (/json/i.test(contentType)) return false; - return /captcha|cf-chl|challenge-platform|perfdrive|Just a moment|Access denied/i.test(text.slice(0, 4000)); + return /captcha|cf-chl|challenge-platform|perfdrive|Just a moment|Access denied/i.test( + text.slice(0, 4000) + ); } /** @@ -133,10 +132,19 @@ function looksLikeChallenge(contentType, text) { * @param {typeof fetch} [opts.fetch] */ export async function fetchJson(url, opts) { - const { provider, headers = {}, timeoutMs = 20000, cacheTtlMs = 0, fetch: f = globalThis.fetch } = opts; + const { + provider, + headers = {}, + timeoutMs = 20000, + cacheTtlMs = 0, + fetch: f = globalThis.fetch, + } = opts; const parsed = new URL(url); if (parsed.protocol !== 'https:' || !ALLOWED_HOSTS.has(parsed.hostname)) { - throw new DatamartError('source_unavailable', `Refusing to fetch non-allowlisted host ${parsed.hostname}`); + throw new DatamartError( + 'source_unavailable', + `Refusing to fetch non-allowlisted host ${parsed.hostname}` + ); } const key = opts.cacheKey || url; if (cacheTtlMs > 0) { @@ -156,15 +164,21 @@ export async function fetchJson(url, opts) { const timedOut = err?.name === 'TimeoutError' || err?.name === 'AbortError'; throw new DatamartError( 'source_unavailable', - timedOut ? `${provider} did not answer within ${timeoutMs / 1000}s` : `${provider} request failed`, + timedOut + ? `${provider} did not answer within ${timeoutMs / 1000}s` + : `${provider} request failed`, { provider } ); } if (res.status >= 300 && res.status < 400) { - throw new DatamartError('source_unavailable', `${provider} redirected (${res.status}); not followed`, { - provider, - }); + throw new DatamartError( + 'source_unavailable', + `${provider} redirected (${res.status}); not followed`, + { + provider, + } + ); } if ((COOLDOWN_STATUSES[provider] || [429]).includes(res.status)) { const wait = coolDown(provider, Number(res.headers.get('retry-after'))); @@ -177,28 +191,38 @@ export async function fetchJson(url, opts) { const contentType = res.headers.get('content-type') || ''; const text = await res.text(); if (looksLikeChallenge(contentType, text)) { + // A challenge means the provider is blocking us; keep calling and we prolong it. + const wait = coolDown(provider); throw new DatamartError( 'source_unavailable', - `${provider} answered with a browser challenge instead of data; Datamart does not bypass these`, - { provider } + `${provider} answered with a browser challenge instead of data; Datamart does not bypass these and paused calls to it`, + { provider, retry_after_seconds: wait } ); } if (res.status === 404) { throw new DatamartError('not_found', `${provider} has no such record`, { provider }); } if (res.status === 401 || res.status === 403) { - throw new DatamartError('configuration_required', `${provider} rejected Datamart's credentials`, { - provider, - }); + throw new DatamartError( + 'configuration_required', + `${provider} rejected Datamart's credentials`, + { + provider, + } + ); } if (!res.ok) { - throw new DatamartError('source_unavailable', `${provider} returned HTTP ${res.status}`, { provider }); + throw new DatamartError('source_unavailable', `${provider} returned HTTP ${res.status}`, { + provider, + }); } let value; try { value = JSON.parse(text); } catch { - throw new DatamartError('source_unavailable', `${provider} returned non-JSON content`, { provider }); + throw new DatamartError('source_unavailable', `${provider} returned non-JSON content`, { + provider, + }); } if (cacheTtlMs > 0) { diff --git a/test/datamart/providers.test.js b/test/datamart/providers.test.js index bd0d291..be4f0bc 100644 --- a/test/datamart/providers.test.js +++ b/test/datamart/providers.test.js @@ -41,12 +41,21 @@ describe('datamart http', () => { expect(called).to.equal(false); }); - it('treats a challenge page as an error, not data', async () => { - const fetch = async () => html('Radware Captcha Page'); + it('treats a challenge page as an error, not data, and pauses the provider', async () => { + let calls = 0; + const fetch = async () => { + calls++; + return html('Radware Captcha Page'); + }; await rejects( fetchJson('https://www.loc.gov/search/?fo=json', { provider: 'loc', fetch }), 'source_unavailable' ); + await rejects( + fetchJson('https://www.loc.gov/search/?fo=json&q=2', { provider: 'loc', fetch }), + 'source_rate_limited' + ); + expect(calls).to.equal(1); }); it('does not follow redirects', async () => {