diff --git a/CONFORMANCE.md b/CONFORMANCE.md index a9843d48..16a08b84 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -44,6 +44,13 @@ Adopts the covenant scale (LBTAS, v2 floor) and carries its own record instance. The reference economy. Its operator/federation model is the named reference for the substrate's frontend-as-operator design target. +The application message API supports optional retry receipts, scoped to the +authenticated caller and conversation. These are private operational metadata, +not sealed dialog or commons entries; they store hashes and message references, +not narrative. This does not promote any in-review binding above. See +[`backend/docs/retry-safe-messages.md`](backend/docs/retry-safe-messages.md) for +the guard, deployment requirements, crash limits, and executable checks. + ### Browser garden planner The Next.js `/garden-planner` page is a member-local planning aid, not a ledger diff --git a/backend/README.md b/backend/README.md index 65327b99..ef3c8d5f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -106,6 +106,11 @@ Endpoints (used by the Chat UI): - `token`: `{ id, token }` - `message`: `{ message }` +Message sending supports an optional `Idempotency-Key` header so a lost response +can be retried without another message, upload, broadcast, or AI invocation. +See [retry-safe messages](docs/retry-safe-messages.md) for the client contract, +required receipt table/permissions, failure handling, and integration checks. + Server emitters (global): - `emitToken(conversationId, id, token)` - `emitMessage(conversationId, message)` diff --git a/backend/__tests__/message_retries.test.js b/backend/__tests__/message_retries.test.js new file mode 100644 index 00000000..58a1ca07 --- /dev/null +++ b/backend/__tests__/message_retries.test.js @@ -0,0 +1,393 @@ +const { test, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); +const express = require('express'); +const request = require('supertest'); +const fs = require('node:fs'); +const path = require('node:path'); +const { randomUUID } = require('node:crypto'); +const jwt = require('../utils/jwt'); +const docClient = require('../lib/dynamodbClient'); +const Message = require('../models/message'); +const { setClient, resetClient } = require('../lib/openaiClient'); + +// Only the external DynamoDB boundary is replaced. Real routes, auth, +// controller, message persistence and AI streaming run in these tests. +function memoryDatabase() { + const tables = new Map(); + const clone = value => value === undefined ? undefined : JSON.parse(JSON.stringify(value)); + const table = name => { + if (!tables.has(name)) tables.set(name, new Map()); + return tables.get(name); + }; + const key = item => JSON.stringify([item.conversationId || null, item.id]); + const conditionalError = () => Object.assign(new Error('condition failed'), { code: 'ConditionalCheckFailedException' }); + return { + tables, + put: params => ({ promise: async () => { + const rows = table(params.TableName); + if (params.ConditionExpression && rows.has(key(params.Item))) throw conditionalError(); + rows.set(key(params.Item), clone(params.Item)); + return {}; + } }), + get: params => ({ promise: async () => ({ Item: clone(table(params.TableName).get(key(params.Key))) }) }), + update: params => ({ promise: async () => { + const row = table(params.TableName).get(key(params.Key)); + const values = params.ExpressionAttributeValues; + if (!row || row.state !== values[':processing'] || row.fingerprint !== values[':fingerprint']) throw conditionalError(); + row.state = values[':complete']; + row.outcome = clone(values[':outcome']); + return {}; + } }), + query: params => ({ promise: async () => ({ Items: [...table(params.TableName).values()] + .filter(row => row.conversationId === params.ExpressionAttributeValues[':cid']).map(clone) }) }), + batchWrite: params => ({ promise: async () => { + for (const [name, writes] of Object.entries(params.RequestItems)) { + for (const { DeleteRequest } of writes) table(name).delete(key(DeleteRequest.Key)); + } + return {}; + } }), + }; +} + +let db, app, originalClient, originalEnv, aiCalls, emitted; +const payload = { from: 'user', to: 'ai', content: 'When should I plant tomatoes?', type: 'text' }; + +function makeApp() { + const instance = express(); + instance.use(express.json()); + instance.use('/messages', require('../routes/communicationRoutes')); + return instance; +} + +function send({ key = 'send-1', body = payload, user = 'grower-1', conversation = 'conversation-1', apiKey } = {}) { + let pending = request(app).post(`/messages/${conversation}`); + if (apiKey) pending = pending.set('X-API-Key', apiKey); + else pending = pending.set('Authorization', `Bearer ${jwt.sign({ userId: user }, process.env.JWT_SECRET)}`); + if (key !== undefined && key !== null) pending = pending.set('Idempotency-Key', key); + return pending.send(body); +} + +function messages() { return [...(db.tables.get('Messages')?.values() || [])]; } + +beforeEach(() => { + originalEnv = { JWT_SECRET: process.env.JWT_SECRET, API_KEY: process.env.API_KEY }; + process.env.JWT_SECRET = 'test-only-message-retry-secret'; + process.env.API_KEY = 'test-only-api-key'; + db = memoryDatabase(); + originalClient = { ...docClient }; + Object.assign(docClient, db); + Message.setDocClient(docClient); + aiCalls = 0; + emitted = []; + global.emitMessage = (conversation, message) => emitted.push({ conversation, message }); + setClient({ chat: { completions: { create: async () => { + aiCalls++; + return (async function* () { yield { choices: [{ delta: { content: 'Plant after the last frost.' } }] }; })(); + } } } }); + app = makeApp(); +}); + +afterEach(() => { + Object.assign(docClient, originalClient); + Message.setDocClient(docClient); + resetClient(); + delete global.emitMessage; + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } +}); + +test('retries return the original message and AI reply without repeating side effects', async () => { + const first = await send(); + const retry = await send(); + assert.equal(first.status, 201); + assert.equal(retry.status, 201); + assert.deepEqual(retry.body, first.body); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); + assert.equal(emitted.length, 2); +}); + +test('reusing a key with different content conflicts without creating another message', async () => { + const first = await send(); + const changed = await send({ body: { ...payload, content: 'A different question' } }); + assert.equal(first.status, 201); + assert.equal(changed.status, 409); + assert.equal(changed.body.code, 'idempotency_key_conflict'); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); +}); + +test('keys are isolated by authenticated caller and conversation, never by body.from', async () => { + const first = await send(); + const otherUser = await send({ user: 'grower-2' }); + const otherConversation = await send({ conversation: 'conversation-2' }); + assert.equal(otherUser.status, 201); + assert.equal(otherConversation.status, 201); + assert.notEqual(otherUser.body.message.id, first.body.message.id); + assert.notEqual(otherConversation.body.message.id, first.body.message.id); + assert.deepEqual((await send()).body, first.body); + assert.equal(messages().length, 6); +}); + +test('requests without a key keep the existing new-send behavior', async () => { + const first = await send({ key: null }); + const next = await send({ key: null }); + assert.equal(first.status, 201); + assert.equal(next.status, 201); + assert.notEqual(first.body.message.id, next.body.message.id); + assert.equal(messages().length, 4); +}); + +test('invalid keys are rejected before persistence or AI work', async () => { + for (const key of ['', 'key with spaces', 'x'.repeat(129)]) { + const result = await send({ key }); + assert.equal(result.status, 400); + } + assert.equal(messages().length, 0); + assert.equal(aiCalls, 0); +}); + +for (const apiCaller of [false, true]) { + test(`parallel ${apiCaller ? 'API-key' : 'JWT'} retries cannot acquire a second send while the first AI response is running`, async () => { + const auth = apiCaller ? { apiKey: process.env.API_KEY } : {}; + let started, finish; + const running = new Promise(resolve => { started = resolve; }); + const release = new Promise(resolve => { finish = resolve; }); + setClient({ chat: { completions: { create: async () => { + aiCalls++; + started(); + await release; + return (async function* () { yield { choices: [{ delta: { content: 'Ready' } }] }; })(); + } } } }); + const first = send(auth).then(response => response); + await running; + const racing = await Promise.all(Array.from({ length: 8 }, () => send(auth))); + finish(); + const done = await first; + assert.ok(racing.every(response => response.status === 409 && response.body.code === 'message_send_pending')); + assert.deepEqual((await send(auth)).body, done.body); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); + assert.equal(emitted.length, 2); + }); +} + +test('completed receipts survive reloading the request handler', async () => { + const first = await send(); + for (const name of ['../services/retrySafeMessage', '../models/messageSend', '../controllers/communication_controller', '../routes/communicationRoutes']) { + delete require.cache[require.resolve(name)]; + } + app = makeApp(); + assert.deepEqual((await send()).body, first.body); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); +}); + +test('omitting the default type or reordering JSON fields still replays the same send', async () => { + const first = await send(); + const retry = await send({ body: { content: payload.content, to: 'ai', from: 'user' } }); + assert.deepEqual(retry.body, first.body); + assert.equal(aiCalls, 1); +}); + +test('changing sender label, recipient, type or attachment conflicts, without rerunning work', async () => { + await send(); + for (const changed of [{ from: 'farmer' }, { to: 'another' }, { type: 'file' }, + { file: { name: 'note.txt', type: 'text/plain', data: 'aGVsbG8=' } }]) { + const result = await send({ body: { ...payload, ...changed } }); + assert.equal(result.status, 409); + assert.equal(result.body.code, 'idempotency_key_conflict'); + } + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); +}); + +test('API-key callers replay within their credential scope and do not share JWT receipts', async () => { + const first = await send({ apiKey: process.env.API_KEY }); + assert.deepEqual((await send({ apiKey: process.env.API_KEY })).body, first.body); + const user = await send(); + assert.equal(user.status, 201); + assert.notEqual(user.body.message.id, first.body.message.id); + assert.equal(aiCalls, 2); +}); + +test('API-key receipts use the stable scrypt scope instead of a fast credential hash', async () => { + const first = await send({ apiKey: process.env.API_KEY }); + assert.equal(first.status, 201); + const [receipt] = db.tables.get('MessageSends').values(); + // Independent Python hashlib.scrypt fixture: password=test-only-api-key, + // salt=SHA256(JSON([message-send-api-key-v2, conversation-1, send-1])).hex, + // N=16384, r=8, p=5, dklen=32. Locks the durable replay namespace. + assert.equal(receipt.id, 'e54be044e9e970d309de6ebe2b3763f1cbb49c546d04c7379102ba52709867e6'); + assert.ok(!JSON.stringify(receipt).includes(process.env.API_KEY)); + delete require.cache[require.resolve('../services/retrySafeMessage')]; + delete require.cache[require.resolve('../controllers/communication_controller')]; + delete require.cache[require.resolve('../routes/communicationRoutes')]; + app = makeApp(); + assert.deepEqual((await send({ apiKey: process.env.API_KEY })).body, first.body); + assert.equal(aiCalls, 1); +}); + +test('API-key rotation, conversation and send keys have separate retry scopes', async () => { + const first = await send({ apiKey: process.env.API_KEY }); + const otherConversation = await send({ apiKey: process.env.API_KEY, conversation: 'conversation-2' }); + const otherSend = await send({ apiKey: process.env.API_KEY, key: 'send-2' }); + process.env.API_KEY = 'rotated-test-api-key'; + const rotated = await send({ apiKey: process.env.API_KEY }); + for (const response of [first, otherConversation, otherSend, rotated]) assert.equal(response.status, 201); + assert.equal(new Set([first, otherConversation, otherSend, rotated].map(response => response.body.message.id)).size, 4); + assert.deepEqual((await send({ apiKey: process.env.API_KEY })).body, rotated.body); + assert.equal(aiCalls, 4); +}); + +test('WIX API-key retries work without a JWT secret', async () => { + const priorWixKey = process.env.WIX_API_KEY; + delete process.env.API_KEY; + delete process.env.JWT_SECRET; + process.env.WIX_API_KEY = 'test-only-wix-api-key'; + try { + const first = await send({ apiKey: process.env.WIX_API_KEY }); + assert.equal(first.status, 201); + assert.deepEqual((await send({ apiKey: process.env.WIX_API_KEY })).body, first.body); + assert.equal(aiCalls, 1); + } finally { + if (priorWixKey === undefined) delete process.env.WIX_API_KEY; else process.env.WIX_API_KEY = priorWixKey; + } +}); + +test('an invalid API-key header cannot alter a verified JWT retry scope', async () => { + const first = await send(); + const retry = await request(app).post('/messages/conversation-1') + .set('Authorization', `Bearer ${jwt.sign({ userId: 'grower-1' }, process.env.JWT_SECRET)}`) + .set('X-API-Key', 'invalid-api-key') + .set('Idempotency-Key', 'send-1').send(payload); + assert.equal(retry.status, 201); + assert.deepEqual(retry.body, first.body); + assert.equal(aiCalls, 1); +}); + +test('authentication failures and JWTs without a stable user ID create no receipts', async () => { + assert.equal((await request(app).post('/messages/conversation-1').set('Idempotency-Key', 'key').send(payload)).status, 403); + const noIdentity = await request(app).post('/messages/conversation-1') + .set('Authorization', `Bearer ${jwt.sign({ role: 'member' }, process.env.JWT_SECRET)}`) + .set('Idempotency-Key', 'key').send(payload); + assert.equal(noIdentity.status, 400); + assert.equal(db.tables.size, 0); +}); + +test('deleting a conversation never lets an old retry resurrect its messages or retained narrative', async () => { + await send(); + await Message.removeByConversation('conversation-1'); + const retry = await send(); + assert.equal(retry.status, 410); + assert.equal(retry.body.code, 'message_send_removed'); + assert.equal(messages().length, 0); + const receipts = JSON.stringify([...(db.tables.get('MessageSends')?.values() || [])]); + assert.ok(!receipts.includes(payload.content)); + assert.ok(!receipts.includes('Plant after the last frost.')); + assert.ok(!receipts.includes('grower-1')); + assert.ok(!receipts.includes('send-1')); + assert.equal(aiCalls, 1); +}); + +test('a missing receipt table refuses keyed sends before any message or AI work', async () => { + const put = docClient.put; + docClient.put = params => params.TableName === 'MessageSends' + ? { promise: async () => { throw Object.assign(new Error('missing table'), { code: 'ResourceNotFoundException' }); } } + : put(params); + const result = await send(); + assert.equal(result.status, 503); + assert.equal(result.body.code, 'message_send_unavailable'); + assert.equal(messages().length, 0); + assert.equal(aiCalls, 0); + assert.equal((await send({ key: null })).status, 201); +}); + +test('an acknowledged message followed by a delivery failure is not sent again on retry', async () => { + docClient.query = () => ({ promise: async () => { throw new Error('history storage unavailable'); } }); + const first = await send(); + const retry = await send(); + assert.equal(first.status, 500); + assert.equal(first.body.code, 'message_send_failed'); + assert.equal(retry.status, 500); + assert.deepEqual(retry.body, first.body); + assert.equal(messages().length, 1); + assert.equal(emitted.length, 1); + assert.equal(aiCalls, 0); +}); + +test('failure to record completion leaves a durable pending receipt instead of rerunning AI', async () => { + docClient.update = () => ({ promise: async () => { throw new Error('receipt update timeout'); } }); + const first = await send(); + const retry = await send(); + assert.equal(first.status, 503); + assert.equal(first.body.code, 'message_send_uncertain'); + assert.equal(retry.status, 409); + assert.equal(retry.body.code, 'message_send_pending'); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 1); +}); + +test('attachment retries reuse one file and do not call an AI responder', async (t) => { + const filename = `retry-${randomUUID()}.txt`; + const uploads = path.join(__dirname, '../uploads'); + t.after(() => { + for (const file of fs.readdirSync(uploads).filter(name => name.endsWith(filename))) fs.unlinkSync(path.join(uploads, file)); + }); + const body = { from: 'user', to: 'buyer', type: 'file', + file: { name: filename, type: 'text/plain', data: Buffer.from('Pickup instructions').toString('base64') } }; + const first = await send({ body }); + const replay = await send({ body }); + assert.equal(first.status, 201); + assert.deepEqual(replay.body, first.body); + assert.equal(fs.readFileSync(path.join(__dirname, '..', first.body.message.file.path), 'utf8'), 'Pickup instructions'); + assert.equal(fs.readdirSync(uploads).filter(name => name.endsWith(filename)).length, 1); + assert.equal(messages().length, 1); + assert.equal(aiCalls, 0); +}); + +test('a receipt-write timeout that actually committed never permits a duplicate send', async () => { + const put = docClient.put; + let loseAck = true; + docClient.put = params => ({ promise: async () => { + const result = await put(params).promise(); + if (params.TableName === 'MessageSends' && loseAck) { + loseAck = false; + throw new Error('acknowledgment lost after commit'); + } + return result; + } }); + assert.equal((await send()).status, 503); + const retry = await send(); + assert.equal(retry.status, 409); + assert.equal(retry.body.code, 'message_send_pending'); + assert.equal(messages().length, 0); + assert.equal(aiCalls, 0); +}); + +test('Agrinet-specific replies are also reused without rebroadcasting', async () => { + const body = { ...payload, content: 'Explain the federation node layout' }; + const first = await send({ body }); + assert.equal(first.body.reply.from, 'agrinet'); + assert.deepEqual((await send({ body })).body, first.body); + assert.equal(messages().length, 2); + assert.equal(aiCalls, 0); + assert.equal(emitted.length, 2); +}); + +test('browser preflight permits the idempotency header on the existing allowed origin', async () => { + const priorMinimal = process.env.MINIMAL_SERVER; + process.env.MINIMAL_SERVER = 'true'; + try { + const { app: serverApp } = require('../server'); + const response = await request(serverApp).options('/messages/conversation-1') + .set('Origin', 'https://www.ntari.org') + .set('Access-Control-Request-Method', 'POST') + .set('Access-Control-Request-Headers', 'authorization,content-type,idempotency-key'); + assert.equal(response.status, 204); + assert.ok(response.headers['access-control-allow-headers'].toLowerCase().split(',').includes('idempotency-key')); + } finally { + if (priorMinimal === undefined) delete process.env.MINIMAL_SERVER; else process.env.MINIMAL_SERVER = priorMinimal; + } +}); diff --git a/backend/controllers/communication_controller.js b/backend/controllers/communication_controller.js index 7804c11a..2d1299e4 100644 --- a/backend/controllers/communication_controller.js +++ b/backend/controllers/communication_controller.js @@ -1,6 +1,7 @@ const Message = require('../models/message'); const agrinetResponder = require('../services/agrinetResponder'); const openAIResponder = require('../services/openAIResponder'); +const retrySafeMessage = require('../services/retrySafeMessage'); function broadcastMessage(conversationId, msg, options = {}) { if (!msg) return; @@ -57,7 +58,7 @@ function shouldAutoRespond({ from, type }) { return ['user', 'client', 'farmer'].includes(normalized); } -exports.sendMessage = async (req, res) => { +async function deliverMessage(req) { const { conversationId } = req.params; const { from, to, content, type, file } = req.body; @@ -110,11 +111,19 @@ exports.sendMessage = async (req, res) => { responsePayload.reply = formattedReply; } - res.status(201).json(responsePayload); + return responsePayload; +} + +exports.sendMessage = async (req, res) => { + if (req.headers?.['idempotency-key'] === undefined) { + return res.status(201).json(await deliverMessage(req)); + } + const result = await retrySafeMessage.send(req, () => deliverMessage(req)); + return res.status(result.status).json(result.body); }; exports.listMessages = async (req, res) => { const { conversationId } = req.params; const msgs = await Message.listMessages(conversationId); res.json(msgs); -}; \ No newline at end of file +}; diff --git a/backend/docs/retry-safe-messages.md b/backend/docs/retry-safe-messages.md new file mode 100644 index 00000000..5f5f09b6 --- /dev/null +++ b/backend/docs/retry-safe-messages.md @@ -0,0 +1,154 @@ +# Retry-safe message sending + +## Why this exists + +A client can lose its connection after the backend has saved a message but +before it receives the response. Sending the same request again previously +created another message, repeated its broadcasts, and could generate another +AI reply. A unique message ID generated by the server cannot identify that retry. + +`POST /messages/:conversationId` now accepts an optional `Idempotency-Key` header. +The client creates a fresh key for each deliberate send and keeps that same key +and payload for every retry of that send. A UUID is suitable. This backend change +does not add automatic retries or change any frontend; clients must opt in. + +```http +POST /messages/conversation-123 +Authorization: Bearer +Idempotency-Key: 573d9ba2-9d86-4da2-b752-12b085b838d1 +Content-Type: application/json + +{"from":"user","to":"ai","content":"When should I plant tomatoes?","type":"text"} +``` + +The existing `{message, ai?, reply?}` success shape and `201` status are preserved. +A completed retry returns the same persisted message and reply, including their +IDs and timestamps, without uploads, broadcasts, token streaming, or AI calls. +Clients that consume both HTTP and SSE should reconcile using message IDs. + +## Contract + +Keys contain 1–128 printable ASCII characters, without spaces. Scope is the +conversation plus the authenticated JWT `userId`, or the accepted API credential. +It is never the caller-supplied `from` label. All clients sharing an API credential +share its retry namespace; key uniqueness is their responsibility. JWT refreshes +keep the same namespace when `userId` is unchanged; rotating an API key changes it. +The middleware's existing authentication and conversation-access behavior is +unchanged. Retry scoping does not add participant authorization to conversations. + +The payload fingerprint covers `from`, `to`, `content`, `type` (default `text`), +and an attachment's name, MIME type, and base64 data. JSON property order does not +matter. Fields ignored by the existing send path do not distinguish requests. + +| Status | Code | Meaning and client action | +| --- | --- | --- | +| `201` | — | Original send completed, or its completed response was reconstructed. | +| `400` | `invalid_idempotency_key` | Fix the header before sending. | +| `400` | `idempotency_identity_required` | Keyed sends require a stable authenticated user ID or API key. | +| `409` | `idempotency_key_conflict` | Same key and scope, different payload. Do not overwrite the original send. | +| `409` | `message_send_pending` | Another request is running, or a worker stopped before recording the outcome. Retry the same key later; inspect the conversation if it remains pending. | +| `410` | `message_send_removed` | The original message or reply was deleted. A retry will not recreate it. | +| `500` | `message_send_failed` | Delivery failed after claiming the key. This failure is retained; repeating the key does not rerun side effects. Inspect the conversation. | +| `503` | `message_send_unavailable` | Receipt storage could not acknowledge a claim. This request did not start delivery. Retry only with the same key; the write may have committed despite a lost acknowledgment. | +| `503` | `message_send_uncertain` | Delivery ran, but recording its outcome failed or timed out. Retry only with the same key and inspect the conversation; do not automatically use a new key. | + +Authentication errors retain their existing responses. A database read failure +while reconstructing a completed response uses the existing generic `500` path; +the receipt remains completed, so retrying the same key does not rerun delivery. + +## How the guard works, and its limits + +A conditional DynamoDB put claims a receipt before any message or upload is +written. Competing workers read that receipt consistently. Only the successful +claimant invokes the existing delivery function. A conditional update records +completion or terminal failure; completed success receipts point to persisted +messages, not copies of their text. + +This is not an exactly-once transaction across DynamoDB, local files, SSE, and +an external AI provider. A crash after claiming the key can leave a permanent +pending receipt, including when no message was written. There is deliberately no +lease takeover or automatic reset: replaying an uncertain operation could create +the duplicates this feature prevents. Pending sends require operator inspection +of stored messages and provider activity. No recovery endpoint is added here. +The existing AI responder's fallback behavior is unchanged. + +Receipts have no automatic expiry. Deleting or expiring them, dropping their +table, or restoring it out of step with Messages removes protection against old +retries. Retain and back up both tables together. Do not automatically clear a +receipt to unblock a client or instruct the client to generate a new key without +first reconciling any messages, uploads, and external work already produced. + +## Provisioning + +Before enabling keyed sends in a client, create a separate table. No existing +Messages key schema is changed. `MESSAGE_SEND_TABLE_NAME` defaults to `MessageSends`. + +```sh +aws dynamodb create-table \ + --table-name MessageSends \ + --attribute-definitions AttributeName=id,AttributeType=S \ + --key-schema AttributeName=id,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST +``` + +Use the backend's configured AWS region and environment; provision through the +deployment's normal review process. Grant the backend `GetItem`, `PutItem`, and +`UpdateItem` on this table, and `GetItem` on Messages for response reconstruction. +The receipt table requires no GSI or TTL. Missing table/permission errors stop +keyed sends before delivery; requests without a header do not access this table. + +Receipts contain only an opaque scoped-key hash, a payload hash, processing state, +and outcome references/status. They contain no raw keys, credentials, account +profiles, attachment bytes, or chat narrative. Keep this application metadata +private to the platform, alongside its message store. Conversation deletion still +erases its message content; retained receipts prevent resurrection on retry. + +API-credential receipt IDs use asynchronous scrypt (`N=16384`, `r=8`, `p=5`, +32-byte output). The salt is the hex SHA-256 of the JSON array +`["message-send-api-key-v2", conversationId, idempotencyKey]`; credential bytes +never enter a fast hash. This makes offline credential guesses against leaked +receipts expensive. Each active derivation uses roughly 16 MiB; concurrent work +runs through Node's worker pool. JWT receipt IDs and payload fingerprints retain +SHA-256 because they do not include authentication secrets. + +This replaces the undeployed API-key ID derivation from the initial PR revision. +Disposable test receipts from that revision cannot replay under this derivation. +Do not mix the two revisions' API-key workers or reuse old test receipts as +production retry protection. + +## NTARI boundaries + +This extends the existing application message API. It does not edit a sealed +dialog, append narrative to the commons, change assessment distributions, or +alter escrow, balances, or governance. Existing `CONFORMANCE.md` in-review labels +remain in review. The new receipt state is operational application metadata, not +the append-only JFA exchange record, and is not federated or exported here. + +References reviewed: + +- [Slack: use Janus and the bylaws to govern work](https://ntari.slack.com/archives/C0AENHR5KG8/p1791982812690669) +- [Slack: local agriculture purpose and DCO on commits](https://ntari.slack.com/archives/C0AENHR5KG8/p1792416355639889) +- [Confluence: Fruitful-UI purpose](https://ntari.atlassian.net/wiki/spaces/Fruitful/pages/44761089) +- [Confluence: DynamoDB transition guidance](https://ntari.atlassian.net/wiki/spaces/Agrinet/pages/74416160) + +The transition guidance describes possible message keys; the existing code's +`{conversationId, id}` message key remains authoritative for this implementation. + +## Verification + +`npm test` includes HTTP tests using the real auth, controller, persistence, and +streaming code, with only DynamoDB and the external AI provider replaced. + +The separate integration check uses the real SDK and DynamoDB Local, two HTTP +worker processes, restart/replay, and process death during an AI call. It creates +and deletes uniquely named tables, refuses non-loopback endpoints, and uses a +local AI fixture without contacting an external provider: + +```sh +docker run --rm -d --name agrinet-retry-check \ + -p 127.0.0.1:18001:8000 amazon/dynamodb-local:latest \ + -jar DynamoDBLocal.jar -inMemory -sharedDb +DYNAMODB_TEST_ENDPOINT=http://127.0.0.1:18001 node scripts/check-message-retries.js +DYNAMODB_TEST_ENDPOINT=http://127.0.0.1:18001 node scripts/check-message-retries.js --api-key +docker stop agrinet-retry-check +``` diff --git a/backend/models/message.js b/backend/models/message.js index 3f0f0353..9bd12ae9 100644 --- a/backend/models/message.js +++ b/backend/models/message.js @@ -51,6 +51,13 @@ async function listMessages(conversationId) { return res.Items || []; } +async function getMessage(conversationId, id) { + const { Item } = await docClient.get({ + TableName: MESSAGE_TABLE_NAME, Key: { conversationId, id }, ConsistentRead: true, + }).promise(); + return Item || null; +} + async function removeByConversation(conversationId) { const messages = await listMessages(conversationId); if (!messages.length) return; @@ -66,4 +73,4 @@ function setDocClient(client) { docClient = client; } -module.exports = { sendMessage, listMessages, removeByConversation, setDocClient }; \ No newline at end of file +module.exports = { sendMessage, listMessages, getMessage, removeByConversation, setDocClient }; diff --git a/backend/models/messageSend.js b/backend/models/messageSend.js new file mode 100644 index 00000000..8796db95 --- /dev/null +++ b/backend/models/messageSend.js @@ -0,0 +1,38 @@ +const docClient = require('../lib/dynamodbClient'); + +const MESSAGE_SEND_TABLE_NAME = process.env.MESSAGE_SEND_TABLE_NAME || 'MessageSends'; + +async function begin(id, fingerprint) { + try { + await docClient.put({ + TableName: MESSAGE_SEND_TABLE_NAME, + Item: { id, fingerprint, state: 'processing' }, + ConditionExpression: 'attribute_not_exists(id)', + }).promise(); + return { claimed: true }; + } catch (error) { + if (error.code !== 'ConditionalCheckFailedException') throw error; + const { Item } = await docClient.get({ + TableName: MESSAGE_SEND_TABLE_NAME, + Key: { id }, + ConsistentRead: true, + }).promise(); + if (!Item) throw new Error('Message send receipt unavailable'); + return { claimed: false, receipt: Item }; + } +} + +async function complete(id, fingerprint, outcome) { + await docClient.update({ + TableName: MESSAGE_SEND_TABLE_NAME, + Key: { id }, + UpdateExpression: 'SET #state = :complete, outcome = :outcome', + ConditionExpression: '#state = :processing AND fingerprint = :fingerprint', + ExpressionAttributeNames: { '#state': 'state' }, + ExpressionAttributeValues: { + ':complete': 'complete', ':processing': 'processing', ':fingerprint': fingerprint, ':outcome': outcome, + }, + }).promise(); +} + +module.exports = { begin, complete }; diff --git a/backend/scripts/check-message-retries.js b/backend/scripts/check-message-retries.js new file mode 100644 index 00000000..bedef976 --- /dev/null +++ b/backend/scripts/check-message-retries.js @@ -0,0 +1,124 @@ +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { fork } = require('node:child_process'); +const { once } = require('node:events'); +const { randomUUID } = require('node:crypto'); +const { DynamoDBClient, CreateTableCommand, DeleteTableCommand, ScanCommand } = require('@aws-sdk/client-dynamodb'); +const jwt = require('../utils/jwt'); + +async function main() { + const endpoint = process.env.DYNAMODB_TEST_ENDPOINT; + if (!endpoint || !['localhost', '127.0.0.1', '[::1]'].includes(new URL(endpoint).hostname)) { + throw new Error('Set DYNAMODB_TEST_ENDPOINT to a loopback DynamoDB Local URL. This check refuses remote endpoints.'); + } + const client = new DynamoDBClient({ endpoint, region: 'us-east-1', + credentials: { accessKeyId: 'local', secretAccessKey: 'local' } }); + const suffix = randomUUID(); + const messages = `RetryMessages-${suffix}`; + const receipts = `RetrySends-${suffix}`; + const created = []; + const workers = []; + const secret = 'local-integration-secret'; + const apiKey = 'local-integration-api-key'; + const apiCaller = process.argv.includes('--api-key'); + const payload = { from: 'user', to: 'ai', content: 'When should I plant?', type: 'text' }; + + async function worker(hold) { + const child = fork(path.join(__dirname, '../test-support/message-retry-worker.js'), [], { + env: { ...process.env, DYNAMODB_ENDPOINT: endpoint, AWS_REGION: 'us-east-1', + AWS_ACCESS_KEY_ID: 'local', AWS_SECRET_ACCESS_KEY: 'local', JWT_SECRET: secret, API_KEY: apiKey, + MESSAGE_TABLE_NAME: messages, MESSAGE_SEND_TABLE_NAME: receipts, TEST_HOLD_AI: String(hold) }, + stdio: ['ignore', 'inherit', 'inherit', 'ipc'], + }); + const state = { child, ai: 0 }; + workers.push(state); + let signalAI; + state.startedAI = new Promise(resolve => { signalAI = resolve; }); + state.url = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Worker did not start')), 15000); + child.once('error', reject); + child.on('message', message => { + if (message.type === 'ready') { clearTimeout(timeout); resolve(message.url); } + if (message.type === 'ai') { state.ai++; signalAI(); } + }); + }); + return state; + } + + async function send(target, { key = 'send-1', conversation = 'conversation-1', body = payload } = {}) { + const response = await fetch(`${target.url}/messages/${conversation}`, { + method: 'POST', headers: { 'Content-Type': 'application/json', 'Idempotency-Key': key, + ...(apiCaller ? { 'X-API-Key': apiKey } + : { Authorization: `Bearer ${jwt.sign({ userId: 'local-grower' }, secret)}` }) }, + body: JSON.stringify(body), signal: AbortSignal.timeout(15000), + }); + return { status: response.status, body: await response.json() }; + } + + async function stop(target, signal = 'SIGTERM') { + const exited = once(target.child, 'exit'); + target.child.kill(signal); + await exited; + } + + async function waitForAI(target) { + let timeout; + try { + await Promise.race([target.startedAI, new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error('Worker did not reach the AI provider')), 15000); + })]); + } finally { clearTimeout(timeout); } + } + + try { + for (const [name, composite] of [[messages, true], [receipts, false]]) { + await client.send(new CreateTableCommand({ TableName: name, BillingMode: 'PAY_PER_REQUEST', + AttributeDefinitions: composite ? [{ AttributeName: 'conversationId', AttributeType: 'S' }, { AttributeName: 'id', AttributeType: 'S' }] + : [{ AttributeName: 'id', AttributeType: 'S' }], + KeySchema: composite ? [{ AttributeName: 'conversationId', KeyType: 'HASH' }, { AttributeName: 'id', KeyType: 'RANGE' }] + : [{ AttributeName: 'id', KeyType: 'HASH' }], + })); + created.push(name); + } + const firstWorker = await worker(true); + const secondWorker = await worker(false); + const first = send(firstWorker); + await waitForAI(firstWorker); + const racing = await Promise.all(Array.from({ length: 8 }, () => send(secondWorker))); + assert.ok(racing.every(response => response.status === 409 && response.body.code === 'message_send_pending')); + firstWorker.child.send('release'); + const original = await first; + assert.equal(original.status, 201); + assert.deepEqual(await send(secondWorker), original); + assert.equal((await send(secondWorker, { body: { ...payload, content: 'changed' } })).status, 409); + await stop(firstWorker); + const restarted = await worker(true); + assert.deepEqual(await send(restarted), original); + const rows = await client.send(new ScanCommand({ TableName: messages })); + assert.equal(rows.Count, 2); + assert.equal(workers.reduce((total, value) => total + value.ai, 0), 1); + console.log('PASS: real conditional writes serialize two workers; completed response survives a worker restart.'); + + // Kill a worker after it has saved the user message and invoked the + // provider. Another worker must not run that operation again. + const interrupted = send(restarted, { key: 'crash-1', conversation: 'crash-conversation' }).catch(() => null); + await waitForAI(restarted); + await stop(restarted, 'SIGKILL'); + await interrupted; + const retry = await send(secondWorker, { key: 'crash-1', conversation: 'crash-conversation' }); + assert.equal(retry.status, 409); + assert.equal(retry.body.code, 'message_send_pending'); + assert.equal(secondWorker.ai, 0); + const afterCrash = await client.send(new ScanCommand({ TableName: messages })); + assert.equal(afterCrash.Count, 3); + console.log('PASS: process death leaves a pending receipt and does not duplicate a message or AI invocation.'); + } finally { + for (const target of workers) { + if (target.child.exitCode === null && target.child.signalCode === null) await stop(target); + } + for (const name of created.reverse()) await client.send(new DeleteTableCommand({ TableName: name })); + client.destroy(); + } +} + +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/backend/server.js b/backend/server.js index 6623eac1..7a827381 100644 --- a/backend/server.js +++ b/backend/server.js @@ -34,7 +34,7 @@ app.use(cors({ }, credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization'] + allowedHeaders: ['Content-Type', 'Authorization', 'Idempotency-Key'] })); // ----------------------------------------- diff --git a/backend/services/retrySafeMessage.js b/backend/services/retrySafeMessage.js new file mode 100644 index 00000000..1657a837 --- /dev/null +++ b/backend/services/retrySafeMessage.js @@ -0,0 +1,88 @@ +const { createHash, scrypt } = require('crypto'); +const { promisify } = require('util'); +const deriveKey = promisify(scrypt); +const MessageSend = require('../models/messageSend'); +const Message = require('../models/message'); + +function hash(value) { + return createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])); + } + return value; +} + +function problem(status, code, error) { return { status, body: { code, error } }; } +const failed = () => problem(500, 'message_send_failed', + 'This send failed and will not be repeated automatically. Inspect the conversation before starting a new send.'); + +async function send(req, deliver) { + const key = req.headers['idempotency-key']; + if (typeof key !== 'string' || !/^[\x21-\x7e]{1,128}$/.test(key)) { + return problem(400, 'invalid_idempotency_key', 'Idempotency-Key must be 1–128 printable ASCII characters without spaces.'); + } + // Auth middleware has already verified the JWT or accepted the configured + // API key. Never use body.from (a display/role label) as the caller identity. + const userId = typeof req.user?.userId === 'string' && req.user.userId ? req.user.userId : null; + const apiKey = req.user ? null : req.headers['x-api-key']; + if (!userId && !apiKey) return problem(400, 'idempotency_identity_required', 'Retry-safe sends require a stable authenticated user ID or API key.'); + + const conversationId = req.params.conversationId; + const { from, to, content, type = 'text', file } = req.body; + const fingerprint = hash(canonical({ from, to, content, type, + file: file?.data ? { name: file.name, type: file.type, data: file.data } : undefined })); + + let id, claim; + try { + // Credentials must not become cheap offline guessing targets in receipts. + // The public scope supplies a deterministic salt so workers and restarts + // find the same receipt. Keep JWT IDs and non-secret fingerprints unchanged. + id = userId + ? hash(['message-send-v1', ['user', userId], conversationId, key]) + : (await deriveKey(apiKey, hash(['message-send-api-key-v2', conversationId, key]), 32, + { N: 16384, r: 8, p: 5, maxmem: 32 * 1024 * 1024 })).toString('hex'); + claim = await MessageSend.begin(id, fingerprint); + } catch { + return problem(503, 'message_send_unavailable', 'Retry protection is unavailable. No send was started by this request. Retry with the same key.'); + } + if (!claim.claimed) { + const { receipt } = claim; + if (receipt.fingerprint !== fingerprint) { + return problem(409, 'idempotency_key_conflict', 'This key was already used with a different message payload.'); + } + if (receipt.state !== 'complete') { + return problem(409, 'message_send_pending', 'This send is still running or was interrupted. Retry this key later; inspect the conversation before starting a new send.'); + } + if (receipt.outcome.status === 500) return failed(); + const message = await Message.getMessage(conversationId, receipt.outcome.messageId); + const reply = receipt.outcome.replyId ? await Message.getMessage(conversationId, receipt.outcome.replyId) : null; + if (!message || (receipt.outcome.replyId && !reply)) { + return problem(410, 'message_send_removed', 'The original message or reply is no longer available. It will not be recreated.'); + } + return { status: 201, body: reply ? { message, ai: reply, reply } : { message } }; + } + + let result, outcome; + try { + const body = await deliver(); + result = { status: 201, body }; + outcome = { status: 201, messageId: body.message.id, + ...(body.reply && { replyId: body.reply.id }) }; + } catch { + // A failure may come after the message, an upload, a broadcast, or an + // external AI call. Never assume those effects can be rolled back. + result = failed(); + outcome = { status: 500 }; + } + try { await MessageSend.complete(id, fingerprint, outcome); } catch { + return problem(503, 'message_send_uncertain', + 'The send outcome could not be recorded. Retry only with this key; inspect the conversation before starting a new send.'); + } + return result; +} + +module.exports = { send }; diff --git a/backend/test-support/message-retry-worker.js b/backend/test-support/message-retry-worker.js new file mode 100644 index 00000000..2b31916f --- /dev/null +++ b/backend/test-support/message-retry-worker.js @@ -0,0 +1,21 @@ +// Isolated HTTP worker for scripts/check-message-retries.js. Only the external +// AI provider is replaced; storage uses the real SDK against DynamoDB Local. +const express = require('express'); +const { setClient } = require('../lib/openaiClient'); + +let release; +const gate = new Promise(resolve => { release = resolve; }); +process.on('message', message => { if (message === 'release') release(); }); +setClient({ chat: { completions: { create: async () => { + process.send({ type: 'ai' }); + if (process.env.TEST_HOLD_AI === 'true') await gate; + return (async function* () { yield { choices: [{ delta: { content: 'Local integration reply' } }] }; })(); +} } } }); + +const app = express(); +app.use(express.json()); +app.use('/messages', require('../routes/communicationRoutes')); +const server = app.listen(0, '127.0.0.1', () => { + process.send({ type: 'ready', url: `http://127.0.0.1:${server.address().port}` }); +}); +process.on('SIGTERM', () => { server.closeAllConnections(); server.close(() => process.exit(0)); });