From b58d9a0aaa817f4281ac17ddb5a55aa7e6f447bb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 19:16:21 -0700 Subject: [PATCH 01/15] feat(files): search and filter the paged file list by kind, size, and uploader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser pages a folder's files from GET /api/workspaces/[id]/files, but its Type, Size, and Uploaded By filters could only narrow the rows already loaded, and its search only reached the 50-result picker endpoint. The paged list now takes them itself: - `q` — case-insensitive name substring, over every folder or within `folderId` - `recursive` — whether `folderId`'s subfolders are listed too - `types` — document/image/audio/video, comma-separated; classified in SQL from the same extension and MIME tables `resolveEffectiveMimeType` reads (a generic or empty declared type falls back to the extension), so a file's kind matches the one the app shows - `sizes` — small/medium/large, split at 1 MiB and 10 MiB as the filter labels read - `uploadedBy` — uploader user ids, comma-separated The set filters are parsed once in the contract to sorted, de-duplicated members, and every new param is bound into the cursor scope: a cursor replayed under another search or filter is a 400, one replayed under a reordered spelling of the same set resumes. The route's input mapping and presenter move to `lib/workspace-files/api/list-page.ts` so a server prefetch can mint exactly the page (cursor included) the route would. --- .../api/workspaces/[id]/files/route.test.ts | 37 +++++ .../app/api/workspaces/[id]/files/route.ts | 57 +------ apps/sim/lib/api/contracts/workspace-files.ts | 49 +++++- .../contexts/workspace/file-manager.ts | 14 ++ apps/sim/lib/uploads/utils/file.ts | 5 +- .../file-listing.integration.ts | 139 +++++++++++++++++- apps/sim/lib/workspace-files/api/list-page.ts | 96 ++++++++++++ .../application/list-workspace-files.ts | 10 ++ apps/sim/lib/workspace-files/list-filters.ts | 38 +++++ apps/sim/lib/workspace-files/query-scope.ts | 101 ++++++++++++- packages/testing/src/mocks/file-utils.mock.ts | 1 + 11 files changed, 488 insertions(+), 59 deletions(-) create mode 100644 apps/sim/lib/workspace-files/api/list-page.ts create mode 100644 apps/sim/lib/workspace-files/list-filters.ts diff --git a/apps/sim/app/api/workspaces/[id]/files/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/route.test.ts index cbbf5c074d4..252c13f25f5 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.test.ts @@ -127,5 +127,42 @@ describe('/api/workspaces/[id]/files', () => { expect(moved.status).toBe(400) expect(collection).toHaveBeenCalledTimes(2) }) + + it('binds its cursor to the search and set filters, whatever order the sets are spelled in', async () => { + collection.mockResolvedValue({ files: [FILE], nextKeys: ['notes.md', FILE.id] }) + const filtered = + '?sortBy=name&limit=1&q=notes&types=image,document&sizes=small&uploadedBy=u1,u2' + + const first = await (await list(filtered)).json() + expect(collection.mock.calls[0][0].input.page).toMatchObject({ + search: 'notes', + kinds: ['document', 'image'], + sizeBuckets: ['small'], + uploadedBy: ['u1', 'u2'], + }) + + const respelled = await list( + `?sortBy=name&limit=1&q=notes&types=document,image,image&sizes=small&uploadedBy=u2,u1&cursor=${first.nextCursor}` + ) + expect(respelled.status).toBe(200) + + for (const changed of [ + 'q=other&types=image,document&sizes=small&uploadedBy=u1,u2', + 'q=notes&types=image&sizes=small&uploadedBy=u1,u2', + 'q=notes&types=image,document&sizes=large&uploadedBy=u1,u2', + 'q=notes&types=image,document&sizes=small&uploadedBy=u1', + 'q=notes&types=image,document&sizes=small&uploadedBy=u1,u2&folderId=root&recursive=true', + ]) { + const refiltered = await list(`?sortBy=name&limit=1&${changed}&cursor=${first.nextCursor}`) + expect(refiltered.status).toBe(400) + } + expect(collection).toHaveBeenCalledTimes(2) + }) + + it('rejects a kind or size bucket the list does not know', async () => { + expect((await list('?types=spreadsheet')).status).toBe(400) + expect((await list('?sizes=huge')).status).toBe(400) + expect(collection).not.toHaveBeenCalled() + }) }) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/route.ts b/apps/sim/app/api/workspaces/[id]/files/route.ts index aaf5aa074dc..1c5af5c4c1b 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.ts @@ -1,10 +1,7 @@ import { createWorkspaceFileContract, - DEFAULT_WORKSPACE_FILE_PAGE_SIZE, listWorkspaceFilesContract, - WORKSPACE_FILE_PAGE_PARAMS, } from '@/lib/api/contracts/workspace-files' -import { cursorRoute, cursorScopeKey } from '@/lib/api/cursor-binding' import { defineInternalJsonRoute, internalRateLimits, @@ -16,6 +13,10 @@ import { internalFileErrorPolicies, internalFilePresenters, } from '@/lib/workspace-files/api' +import { + presentWorkspaceFileList, + toWorkspaceFileListInput, +} from '@/lib/workspace-files/api/list-page' import { admitCreateWorkspaceFile, createWorkspaceFile, @@ -23,21 +24,9 @@ import { import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files' import { fileOperations } from '@/lib/workspace-files/application/operations' import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration' -import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response' export const dynamic = 'force-dynamic' -/** The params besides the sort that change which files a page walk returns; the cursor is bound to them. */ -function fileCursorFilters( - workspaceId: string, - query: { scope: 'active' | 'archived'; folderId?: string | null } -) { - return cursorScopeKey(cursorRoute(listWorkspaceFilesContract, { id: workspaceId }), { - scope: query.scope, - folderId: query.folderId, - }) -} - /** * GET /api/workspaces/[id]/files — List workspace files: one bounded page with `nextCursor` * when any paging param is sent, else the whole scope. @@ -50,43 +39,9 @@ export const GET = defineInternalJsonRoute({ // can choose a policy per request. rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal file-list behavior' }), errorPolicy: internalFileErrorPolicies.default, - mapInput: ({ params, query }) => { - if (!WORKSPACE_FILE_PAGE_PARAMS.some((param) => query[param] !== undefined)) { - return { workspaceId: params.id, scope: query.scope } - } - const sortBy = query.sortBy ?? 'uploadedAt' - const sortOrder = query.sortOrder ?? 'asc' - return { - workspaceId: params.id, - scope: query.scope, - page: { - folderId: query.folderId, - sortBy, - sortOrder, - limit: query.limit ?? DEFAULT_WORKSPACE_FILE_PAGE_SIZE, - after: readSortedCursor( - query.cursor, - sortBy, - sortOrder, - fileCursorFilters(params.id, query) - ), - }, - } - }, + mapInput: ({ params, query }) => toWorkspaceFileListInput(params.id, query), useCase: listWorkspaceFileCollection, - present: (result, { input }) => { - const listed = internalFilePresenters.successFiles(result) - if (!input.page) return listed - return { - ...listed, - nextCursor: writeSortedCursor( - result.nextKeys, - input.page.sortBy, - input.page.sortOrder, - fileCursorFilters(input.workspaceId, { scope: input.scope, folderId: input.page.folderId }) - ), - } - }, + present: (result, { input }) => presentWorkspaceFileList(result, input), }) /** POST /api/workspaces/[id]/files — Create an authored workspace file. */ diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 9954ed14530..756967fb2b3 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -12,6 +12,10 @@ import { shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' import { V2_SEARCH_MAX_LENGTH, v2LimitSchema } from '@/lib/api/contracts/v2/shared' import { LIST_SORT_ORDERS } from '@/lib/api/list-query' +import { + WORKSPACE_FILE_KINDS, + WORKSPACE_FILE_SIZE_BUCKETS, +} from '@/lib/workspace-files/list-filters' /** * Client-reachable listing scopes. `all` is deliberately excluded: it drops the @@ -48,16 +52,57 @@ const workspaceFileFolderFilterSchema = z */ export const WORKSPACE_FILE_PAGE_PARAMS = [ 'folderId', + 'recursive', + 'q', + 'types', + 'sizes', + 'uploadedBy', 'sortBy', 'sortOrder', 'limit', 'cursor', ] as const +/** + * A set filter on the wire: comma-separated members, parsed to its sorted, de-duplicated + * members so `a,b` and `b,a` are one filter for the query and for the cursor bound to it. + */ +function commaSetSchema>(member: T) { + return z.preprocess( + (raw) => + typeof raw === 'string' + ? [ + ...new Set( + raw + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + ), + ].sort() + : raw, + z.array(member).max(100, 'A filter cannot list more than 100 values') + ) +} + export const listWorkspaceFilesQuerySchema = z.object({ scope: workspaceFileScopeSchema.default('active'), - /** The folder to list directly (not its subfolders); `root` for root files. Omit for every folder. */ + /** The folder to list; `root` for root files. Omit for every folder. */ folderId: workspaceFileFolderFilterSchema.optional(), + /** Whether `folderId`'s subfolders are listed too. Ignored without `folderId`. */ + recursive: booleanQueryFlagSchema.optional(), + /** Case-insensitive substring of the file name. */ + q: z + .string() + .trim() + .min(1, 'q cannot be empty') + .max(V2_SEARCH_MAX_LENGTH, 'q is too long') + .optional(), + /** File kinds, comma-separated: a file matching any of them is listed. */ + types: commaSetSchema(z.enum(WORKSPACE_FILE_KINDS)).optional(), + /** Size buckets, comma-separated: a file in any of them is listed. */ + sizes: commaSetSchema(z.enum(WORKSPACE_FILE_SIZE_BUCKETS)).optional(), + /** Uploader user ids, comma-separated. */ + uploadedBy: commaSetSchema(z.string().max(255, 'uploadedBy entry is too long')).optional(), sortBy: z.enum(WORKSPACE_FILE_SORT_FIELDS).optional(), sortOrder: z.enum(LIST_SORT_ORDERS).optional(), limit: z.coerce @@ -70,6 +115,8 @@ export const listWorkspaceFilesQuerySchema = z.object({ cursor: z.string().min(1, 'cursor must be a non-empty token').optional(), }) +export type ListWorkspaceFilesQuery = z.output + /** * Binary stream of an image embedded in a workspace markdown document, scoped to the * workspace in the path. The route serves the bytes only when the referenced file is a diff --git a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts index fd41d8c8313..f412655be9d 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts @@ -131,6 +131,7 @@ import { normalizeDisplaySegment, VfsPathError, } from '@/lib/vfs/path' +import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' import { lockFileDirectories } from '@/lib/workspace-files/locks' import { OWNED_FILE_CONTEXTS, @@ -156,6 +157,8 @@ import { workspaceFileFolderIdCondition, workspaceFileFolderKey, workspaceFileFolderScopeCondition, + workspaceFileKindCondition, + workspaceFileSizeCondition, } from '@/lib/workspace-files/query-scope' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid } from '@/executor/constants' @@ -1905,6 +1908,12 @@ export interface QueryWorkspaceFilesOptions { name?: string /** Case-insensitive substring match on the file name. */ search?: string + /** Files of any of these kinds; absent or empty applies no kind filter. */ + kinds?: readonly WorkspaceFileKind[] + /** Files in any of these size buckets; absent or empty applies no size filter. */ + sizeBuckets?: readonly WorkspaceFileSizeBucket[] + /** Files uploaded by any of these users; absent or empty applies no uploader filter. */ + uploadedBy?: readonly string[] sortBy: V2FileSortBy sortOrder: ListSortOrder limit: number @@ -1994,6 +2003,11 @@ export async function queryFileRecords( workspaceFileFolderScopeCondition(folderScope), name === undefined ? undefined : eq(workspaceFiles.originalName, name), searchFilter(workspaceFiles.originalName, search), + workspaceFileKindCondition(options.kinds), + workspaceFileSizeCondition(options.sizeBuckets), + options.uploadedBy && options.uploadedBy.length > 0 + ? inArray(workspaceFiles.userId, [...options.uploadedBy]) + : undefined, resumeAfter, ] diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index 096a1b8d1d6..805788d3ef9 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -413,7 +413,10 @@ const GENERIC_MIME_TYPE = 'application/octet-stream' const DUAL_CONTAINER_MIME: Record = { webm: 'video/webm' } /** Every MIME type that identifies no format, including the legacy `binary/` spelling. */ -const GENERIC_MIME_TYPES = new Set([GENERIC_MIME_TYPE, 'binary/octet-stream']) +export const GENERIC_MIME_TYPES: ReadonlySet = new Set([ + GENERIC_MIME_TYPE, + 'binary/octet-stream', +]) /** Whether a declared MIME type names an actual format, rather than "some bytes". */ function identifiesFormat(declared: string | undefined): declared is string { diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 233acc4e521..3de77d66f38 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -1,7 +1,7 @@ /** * Real PostgreSQL behavior of the bounded workspace-file reads the browser, v2, and the Mothership - * VFS share: keyset paging under every sort, folder filters by id, name search, batch lookup, and - * folder rollups, under `files.list` authorization. + * VFS share: keyset paging under every sort, folder filters by id, name search, kind/size/uploader + * filters, batch lookup, and folder rollups, under `files.list` authorization. */ import { db } from '@sim/db' import { permissions, user, workspace, workspaceFiles } from '@sim/db/schema' @@ -23,6 +23,7 @@ const bob = generateId() const carol = generateId() const workspaceId = generateId() const otherWorkspaceId = generateId() +const filterWorkspaceId = generateId() const session = (userId: string) => ({ kind: 'session' as const, userId, sessionId: generateId() }) /** Folder tree in the main workspace: Docs/{Sub/{Deep}}, Media; plus one in the other workspace. */ @@ -58,6 +59,8 @@ async function insertFile(input: { updatedAt?: string deleted?: boolean context?: string + contentType?: string + userId?: string }): Promise { const file: SeededFile = { id: input.id ?? `wf_${generateId()}`, @@ -71,8 +74,9 @@ async function insertFile(input: { await db.execute(sql` INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, folder_id, context, original_name, content_type, size_bytes, uploaded_at, updated_at, deleted_at) - VALUES (${file.id}, ${file.key}, ${alice}, ${input.workspaceId ?? workspaceId}, ${file.folderId}, - ${input.context ?? 'workspace'}, ${file.name}, 'text/plain', ${file.size}, + VALUES (${file.id}, ${file.key}, ${input.userId ?? alice}, ${input.workspaceId ?? workspaceId}, + ${file.folderId}, ${input.context ?? 'workspace'}, ${file.name}, + ${input.contentType ?? 'text/plain'}, ${file.size}, ${file.uploadedAt}::timestamp, ${file.updatedAt}::timestamp, ${input.deleted ? sql`now()` : sql`NULL`})`) return file @@ -138,6 +142,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { await insertWorkspaceFixture(db, [ { id: workspaceId, name: 'Listing fixture', ownerId: alice, billedAccountUserId: alice }, { id: otherWorkspaceId, name: 'Other fixture', ownerId: alice, billedAccountUserId: alice }, + { id: filterWorkspaceId, name: 'Filter fixture', ownerId: alice, billedAccountUserId: alice }, ]) await db.insert(permissions).values([ { @@ -161,6 +166,13 @@ describe('bounded workspace file reads in PostgreSQL', () => { entityId: otherWorkspaceId, permissionType: 'admin', }, + { + id: generateId(), + userId: alice, + entityType: 'workspace', + entityId: filterWorkspaceId, + permissionType: 'admin', + }, ]) await db.execute(sql` INSERT INTO folder (id, name, user_id, workspace_id, resource_type, parent_id) VALUES @@ -251,7 +263,10 @@ describe('bounded workspace file reads in PostgreSQL', () => { }) afterAll(async () => { - await deleteWorkspaceFixture(db, inArray(workspace.id, [workspaceId, otherWorkspaceId])) + await deleteWorkspaceFixture( + db, + inArray(workspace.id, [workspaceId, otherWorkspaceId, filterWorkspaceId]) + ) await db.delete(user).where(inArray(user.id, [alice, bob, carol])) }) @@ -391,6 +406,120 @@ describe('bounded workspace file reads in PostgreSQL', () => { }) }) + describe('kind, size, and uploader filters', () => { + const MIB = 1_048_576 + /** + * Each file labelled with the Type filter's own classification: the declared type when it + * names a format, else the extension's — so a generic or empty declaration falls back, a + * `.webm` with nothing declared is video, and a document is decided by extension alone. + */ + const fixtures = [ + { name: 'photo.png', contentType: 'image/png', size: 500, kinds: ['image'] }, + { + name: 'upload.png', + contentType: 'application/octet-stream', + size: 2 * MIB, + kinds: ['image'], + }, + { name: 'clip.webm', contentType: '', size: MIB - 1, kinds: ['video'] }, + { name: 'voice.webm', contentType: 'audio/webm', size: 10 * MIB, kinds: ['audio'] }, + { + name: 'song.MP3', + contentType: 'binary/octet-stream', + size: 10 * MIB + 1, + kinds: ['audio'], + }, + { name: 'movie.mov', contentType: 'video/quicktime', size: 40 * MIB, kinds: ['video'] }, + { name: 'paper.pdf', contentType: 'application/pdf', size: MIB, kinds: ['document'] }, + { name: 'chart.json', contentType: 'image/png', size: 10, kinds: ['document', 'image'] }, + { name: 'bundle.zip', contentType: 'application/zip', size: 10, kinds: [] }, + { name: 'README', contentType: 'text/plain', size: 10, kinds: [], uploader: 'bob' }, + ] as const + const seeded: { + id: string + name: string + size: number + kinds: readonly string[] + userId: string + }[] = [] + + beforeAll(async () => { + for (const [index, fixture] of fixtures.entries()) { + const userId = 'uploader' in fixture ? bob : alice + const file = await insertFile({ + workspaceId: filterWorkspaceId, + name: fixture.name, + contentType: fixture.contentType, + size: fixture.size, + userId, + uploadedAt: `2026-04-01T00:00:0${index}`, + }) + seeded.push({ ...file, kinds: fixture.kinds, userId }) + } + }) + + const filtered = async (input: Partial) => + (await page(input, { workspace: filterWorkspaceId })).files.map((file) => file.name).sort() + const names = (predicate: (file: (typeof seeded)[number]) => boolean) => + seeded + .filter(predicate) + .map((file) => file.name) + .sort() + + it.each([['document'], ['image'], ['audio'], ['video'], ['image', 'video']] as const)( + 'lists the %s files a kind filter names', + async (...kinds) => { + expect(await filtered({ kinds })).toEqual( + names((file) => file.kinds.some((kind) => (kinds as readonly string[]).includes(kind))) + ) + } + ) + + it('splits sizes into buckets at exactly 1 MiB and 10 MiB', async () => { + expect(await filtered({ sizeBuckets: ['small'] })).toEqual(names((file) => file.size < MIB)) + expect(await filtered({ sizeBuckets: ['medium'] })).toEqual( + names((file) => file.size >= MIB && file.size <= 10 * MIB) + ) + expect(await filtered({ sizeBuckets: ['small', 'large'] })).toEqual( + names((file) => file.size < MIB || file.size > 10 * MIB) + ) + }) + + it('filters by uploader, and intersects every filter with the others and the search', async () => { + expect(await filtered({ uploadedBy: [bob] })).toEqual(['README']) + expect( + await filtered({ kinds: ['image', 'audio'], sizeBuckets: ['large'], uploadedBy: [alice] }) + ).toEqual(['song.MP3']) + expect(await filtered({ kinds: ['image'], search: 'PHO' })).toEqual(['photo.png']) + }) + + it('pages a filtered list without skipping or repeating', async () => { + const ids: string[] = [] + let after: CursorKey[] | undefined + for (let guard = 0; guard < 20; guard++) { + const result = await page( + { + kinds: ['image', 'audio', 'video'], + sortBy: 'size', + sortOrder: 'desc', + limit: 2, + after, + }, + { workspace: filterWorkspaceId } + ) + ids.push(...result.files.map((file) => file.id)) + if (!result.nextKeys) break + after = result.nextKeys + } + expect(ids).toEqual( + seeded + .filter((file) => file.kinds.some((kind) => kind !== 'document')) + .sort((a, b) => b.size - a.size) + .map((file) => file.id) + ) + }) + }) + describe('batch lookup', () => { it('resolves own live files by id, key, and reference, and ignores the rest', async () => { const alpha = byName('alpha.txt', folders.docs) diff --git a/apps/sim/lib/workspace-files/api/list-page.ts b/apps/sim/lib/workspace-files/api/list-page.ts new file mode 100644 index 00000000000..f80969eac9c --- /dev/null +++ b/apps/sim/lib/workspace-files/api/list-page.ts @@ -0,0 +1,96 @@ +import { + DEFAULT_WORKSPACE_FILE_PAGE_SIZE, + type ListWorkspaceFilesQuery, + listWorkspaceFilesContract, + WORKSPACE_FILE_PAGE_PARAMS, +} from '@/lib/api/contracts/workspace-files' +import { cursorRoute, cursorScopeKey } from '@/lib/api/cursor-binding' +import type { CursorKey } from '@/lib/api/list-query' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { internalFilePresenters } from '@/lib/workspace-files/api/internal-presenters' +import type { ListWorkspaceFileCollectionInput } from '@/lib/workspace-files/application/list-workspace-files' +import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response' + +interface FileCursorScope { + scope: 'active' | 'archived' + folderId?: string | null + recursive?: boolean + search?: string + kinds?: readonly string[] + sizeBuckets?: readonly string[] + uploadedBy?: readonly string[] +} + +/** + * Every param that changes which files, in which order, a page walk returns. The set filters + * arrive sorted and de-duplicated from the contract, the one parse both halves share. + */ +function fileCursorFilters(workspaceId: string, query: FileCursorScope) { + return cursorScopeKey(cursorRoute(listWorkspaceFilesContract, { id: workspaceId }), { + scope: query.scope, + folderId: query.folderId, + recursive: query.folderId === undefined ? undefined : query.recursive, + q: query.search, + types: query.kinds, + sizes: query.sizeBuckets, + uploadedBy: query.uploadedBy, + }) +} + +/** + * The file collection a `GET /api/workspaces/[id]/files` query asks for: one bounded page when + * any paging param is sent (resuming from its cursor, which must have been minted under the same + * scope, folder, search, filters, and sort), else the whole scope. + */ +export function toWorkspaceFileListInput( + workspaceId: string, + query: ListWorkspaceFilesQuery +): ListWorkspaceFileCollectionInput { + if (!WORKSPACE_FILE_PAGE_PARAMS.some((param) => query[param] !== undefined)) { + return { workspaceId, scope: query.scope } + } + const sortBy = query.sortBy ?? 'uploadedAt' + const sortOrder = query.sortOrder ?? 'asc' + const filters = { + folderId: query.folderId, + recursive: query.recursive, + search: query.q, + kinds: query.types, + sizeBuckets: query.sizes, + uploadedBy: query.uploadedBy, + } + return { + workspaceId, + scope: query.scope, + page: { + ...filters, + sortBy, + sortOrder, + limit: query.limit ?? DEFAULT_WORKSPACE_FILE_PAGE_SIZE, + after: readSortedCursor( + query.cursor, + sortBy, + sortOrder, + fileCursorFilters(workspaceId, { scope: query.scope, ...filters }) + ), + }, + } +} + +/** The list response for {@link toWorkspaceFileListInput}'s input: a paged one carries `nextCursor`. */ +export function presentWorkspaceFileList( + result: { files: WorkspaceFileRecord[]; nextKeys: CursorKey[] | null }, + input: ListWorkspaceFileCollectionInput +) { + const listed = internalFilePresenters.successFiles(result) + if (!input.page) return listed + return { + ...listed, + nextCursor: writeSortedCursor( + result.nextKeys, + input.page.sortBy, + input.page.sortOrder, + fileCursorFilters(input.workspaceId, { scope: input.scope, ...input.page }) + ), + } +} diff --git a/apps/sim/lib/workspace-files/application/list-workspace-files.ts b/apps/sim/lib/workspace-files/application/list-workspace-files.ts index 9d9db624215..aa395836f36 100644 --- a/apps/sim/lib/workspace-files/application/list-workspace-files.ts +++ b/apps/sim/lib/workspace-files/application/list-workspace-files.ts @@ -25,6 +25,7 @@ import { } from '@/lib/uploads/contexts/workspace' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' +import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' import { resolveWorkspaceFolderScope } from '@/lib/workspace-files/resolve-folder-scope' export interface ListAllWorkspaceFilesInput { @@ -48,6 +49,12 @@ export interface QueryWorkspaceFilePageInput { /** Exact file name. */ name?: string search?: string + /** Files of any of these kinds. */ + kinds?: readonly WorkspaceFileKind[] + /** Files in any of these size buckets. */ + sizeBuckets?: readonly WorkspaceFileSizeBucket[] + /** Files uploaded by any of these users. */ + uploadedBy?: readonly string[] sortBy: 'name' | 'size' | 'uploadedAt' | 'updatedAt' sortOrder: 'asc' | 'desc' limit: number @@ -258,6 +265,9 @@ async function readWorkspaceFilePage( fileIds: input.fileId === undefined ? undefined : [input.fileId], name: input.name, search: input.search, + kinds: input.kinds, + sizeBuckets: input.sizeBuckets, + uploadedBy: input.uploadedBy, sortBy: input.sortBy, sortOrder: input.sortOrder, limit: input.limit, diff --git a/apps/sim/lib/workspace-files/list-filters.ts b/apps/sim/lib/workspace-files/list-filters.ts new file mode 100644 index 00000000000..f25b26674cc --- /dev/null +++ b/apps/sim/lib/workspace-files/list-filters.ts @@ -0,0 +1,38 @@ +/** + * The file kinds the paged file list filters by. A file can be several at once — a `.json` + * is a document whatever its stored type — so a filter naming more than one matches a file + * that is any of them. + * + * - `document`: the extension is a supported document format (`SUPPORTED_DOCUMENT_EXTENSIONS`). + * - `image` / `audio` / `video`: the file's effective MIME type (`resolveEffectiveMimeType`: + * the stored type, or the extension's when the stored one names no format) is that kind. + */ +export const WORKSPACE_FILE_KINDS = ['document', 'image', 'audio', 'video'] as const + +export type WorkspaceFileKind = (typeof WORKSPACE_FILE_KINDS)[number] + +export function isWorkspaceFileKind(value: string): value is WorkspaceFileKind { + return (WORKSPACE_FILE_KINDS as readonly string[]).includes(value) +} + +export const WORKSPACE_FILE_SIZE_BUCKETS = ['small', 'medium', 'large'] as const + +export type WorkspaceFileSizeBucket = (typeof WORKSPACE_FILE_SIZE_BUCKETS)[number] + +export function isWorkspaceFileSizeBucket(value: string): value is WorkspaceFileSizeBucket { + return (WORKSPACE_FILE_SIZE_BUCKETS as readonly string[]).includes(value) +} + +const MIB = 1_048_576 + +/** + * Each size bucket's byte range, both bounds inclusive (`max: null` is unbounded): under 1 MiB, + * 1 to 10 MiB, and over 10 MiB, as the filter's labels read. + */ +export const WORKSPACE_FILE_SIZE_BUCKET_RANGES: Readonly< + Record +> = { + small: { min: 0, max: MIB - 1 }, + medium: { min: MIB, max: 10 * MIB }, + large: { min: 10 * MIB + 1, max: null }, +} diff --git a/apps/sim/lib/workspace-files/query-scope.ts b/apps/sim/lib/workspace-files/query-scope.ts index 99098d4afa9..e0c4b491873 100644 --- a/apps/sim/lib/workspace-files/query-scope.ts +++ b/apps/sim/lib/workspace-files/query-scope.ts @@ -1,7 +1,19 @@ import { workspaceFiles } from '@sim/db/schema' -import { eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { and, eq, gte, inArray, isNull, lte, not, or, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { FolderIdScope } from '@/lib/folders/scope' +import { + EXTENSION_TO_MIME, + GENERIC_MIME_TYPES, + MIME_TYPE_MAPPING, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file' +import { SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation' +import { + WORKSPACE_FILE_SIZE_BUCKET_RANGES, + type WorkspaceFileKind, + type WorkspaceFileSizeBucket, +} from '@/lib/workspace-files/list-filters' import { OWNED_FILE_CONTEXTS } from '@/lib/workspace-files/owned-files' /** Durable files available to workspace resource pickers, reference mappings, and fork copies. */ @@ -67,3 +79,90 @@ export function workspaceFileFolderScopeCondition( ...(scope.includeRootItems ? [null] : []), ]) } + +/** + * The file's extension as `getFileExtension` reads it: everything after the last `.`, + * lowercased, or `''` for a name without one. + */ +const workspaceFileExtension = sql`(CASE WHEN strpos(${workspaceFiles.originalName}, '.') > 0 + THEN lower(regexp_replace(${workspaceFiles.originalName}, '^.*\\.', '')) ELSE '' END)` + +const declaredContentType = sql`btrim(${workspaceFiles.contentType})` + +/** Whether the stored type names a format, the condition under which `resolveEffectiveMimeType` keeps it. */ +const declaresFormat: SQL = sql`(${declaredContentType} <> '' AND ${not( + inArray(declaredContentType, [...GENERIC_MIME_TYPES]) +)})` + +/** Every extension whose own MIME type (what a file without a declared format resolves to) passes `matches`. */ +function extensionsResolvingTo(matches: (mimeType: string) => boolean): string[] { + return Object.keys(EXTENSION_TO_MIME).filter((extension) => + matches(resolveEffectiveMimeType(undefined, `file.${extension}`)) + ) +} + +/** A file whose effective MIME type passes `matches`, spelled for SQL: the declared type, else the extension's. */ +function effectiveMimeTypeCondition( + declaredMatches: SQL, + matches: (mimeType: string) => boolean +): SQL | undefined { + const extensions = extensionsResolvingTo(matches) + return or( + and(declaresFormat, declaredMatches), + extensions.length > 0 + ? and(not(declaresFormat), inArray(workspaceFileExtension, extensions)) + : undefined + ) +} + +function mappedMimeTypes(kind: 'audio' | 'video'): string[] { + return Object.entries(MIME_TYPE_MAPPING) + .filter(([, mapped]) => mapped === kind) + .map(([mimeType]) => mimeType) +} + +function workspaceFileKindPredicate(kind: WorkspaceFileKind): SQL | undefined { + switch (kind) { + case 'document': + return inArray(workspaceFileExtension, [...SUPPORTED_DOCUMENT_EXTENSIONS]) + case 'image': + return effectiveMimeTypeCondition(sql`${declaredContentType} LIKE 'image/%'`, (mimeType) => + mimeType.startsWith('image/') + ) + case 'audio': + case 'video': { + const mimeTypes = mappedMimeTypes(kind) + return effectiveMimeTypeCondition( + inArray(sql`lower(${declaredContentType})`, mimeTypes), + (mimeType) => MIME_TYPE_MAPPING[mimeType.toLowerCase()] === kind + ) + } + } +} + +/** + * The files of any of `kinds`, as the Files browser's Type filter classifies them (see + * `WORKSPACE_FILE_KINDS`), or none when `kinds` is absent or empty. Built from the extension + * and MIME tables `resolveEffectiveMimeType` reads, so a file's kind here is the one the app + * resolves for it everywhere else. + */ +export function workspaceFileKindCondition( + kinds: readonly WorkspaceFileKind[] | undefined +): SQL | undefined { + if (!kinds || kinds.length === 0) return undefined + return or(...kinds.map(workspaceFileKindPredicate)) +} + +/** The files in any of `buckets` (see `WORKSPACE_FILE_SIZE_BUCKET_RANGES`), or none when absent or empty. */ +export function workspaceFileSizeCondition( + buckets: readonly WorkspaceFileSizeBucket[] | undefined +): SQL | undefined { + if (!buckets || buckets.length === 0) return undefined + const size = sql`coalesce(${workspaceFiles.sizeBytes}, 0)` + return or( + ...buckets.map((bucket) => { + const { min, max } = WORKSPACE_FILE_SIZE_BUCKET_RANGES[bucket] + return and(gte(size, min), max === null ? undefined : lte(size, max)) + }) + ) +} diff --git a/packages/testing/src/mocks/file-utils.mock.ts b/packages/testing/src/mocks/file-utils.mock.ts index f1878846366..06202ff3c38 100644 --- a/packages/testing/src/mocks/file-utils.mock.ts +++ b/packages/testing/src/mocks/file-utils.mock.ts @@ -705,6 +705,7 @@ export const fileUtilsMockFns = { */ export const fileUtilsMock = { MIME_TYPE_MAPPING, + GENERIC_MIME_TYPES, MODEL_SUPPORTED_IMAGE_MIME_TYPES, GENERATED_DOCUMENT_SOURCE_TYPES, MAX_TEXT_EXTRACTION_BYTES: 25 * 1024 * 1024, From 1b9376fefe3a65db9f6175c69b581179f9233197 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 28 Oct 2026 19:17:40 -0700 Subject: [PATCH 02/15] perf(files): page the Files browser and resolve files by lookup instead of listing the workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every surface that needed a file read the workspace's whole file list (tens of MB for the largest workspaces, refetched on every broadcast). None does now; the full-list hook, its key, and the "300 files or nothing" seed are gone. Data layer (`hooks/queries/workspace-files.ts`, `hooks/queries/utils/workspace-file-*.ts`): - `useWorkspaceFileList` — an infinite query over the paged list, keyed by its canonical params (folder, search, kind/size/uploader filters, sort, page size). A re-sort, re-filter, or new search keeps the previous rows until the new first page lands; another folder never does. - `useWorkspaceFileSearch` — the name search pickers and menus use. - `useWorkspaceFileFolderSizes` — subtree rollups per parent folder. - `useWorkspaceFileLookup(s)` — files by id, storage key, or reference, coalesced into batched `POST /files/lookup` requests (one per workspace and 100 values of a kind). - `useWorkspaceFileRecord` — one file's record (chat uploads included), starting from the freshest copy any cached list, search, or lookup already holds. Replaces both the list-backed record hook and the addressed-record fallback. - Writes reach every cached read: rename and delete update/drop the file in list pages, searches, lookups, and its record optimistically and roll all of them back on failure; uploads, creates, restores, folder ops, shares, content writes, and the realtime room invalidate the workspace's reads (`invalidateWorkspaceFileReads`), and the room's callback still settles only once the mounted ones have refetched. - Stale-key recovery refetches the mounted reads that handed out the dead key. Files browser: a virtualized table that loads the next page on scroll; sort by name, size, created, or updated is applied by the server; search queries every folder through the paged list; Type, Size, and Uploaded By filter on the server. Folders interleave with the server-ordered files, a folder past the last loaded file waiting until a page reaches it so loading more only appends; a folder's pinned files are looked up so they still lead it; folder sizes come from the rollups. Sorting by Type or Owner is no longer offered — the server cannot order by either, and ordering only the loaded rows would misreport the folder's order. New files are named by the server's own numbering. The server prefetch seeds the first page of the list the URL opens on (or the open file's record), through the route's own use case and presenter; Home no longer seeds anything. Everywhere else: - chips, tab names, the table file cell, and tool labels resolve by batched lookup; opening and copying a chip resolves its candidates by lookup plus a name search, so a name two files share is still refused - the embedded file view, mothership view, standalone viewer, and diff embed read the record - the image-dimension adapter resolves each image by its key or id - the chat composer menus, the resource picker, markdown `@` mentions, and Cmd+K offer the most recently updated files and search by what is typed - the workflow file picker lists a page (scoped by the server to a single folder scope) and resolves its selections on their own; search-and-replace resolves file references by lookup and offers recent and matching files plus every chosen replacement - the rail flyout loads each folder level as it opens --- .../home/organization-home.tsx | 9 +- .../release-entry/release-entry.tsx | 4 +- .../[workspaceId]/chat/[chatId]/page.tsx | 23 +- .../components/resource/resource.tsx | 9 +- .../[workspaceId]/files/[fileId]/page.tsx | 14 +- .../use-file-doc-collaboration.ts | 6 +- .../rich-markdown-editor/mention/mention.ts | 22 +- .../mention/use-editor-mentions.ts | 8 +- .../mention/use-markdown-mentions.ts | 28 +- .../workspace/[workspaceId]/files/files.tsx | 492 ++++++------- .../[workspaceId]/files/list-entries.test.ts | 88 +++ .../[workspaceId]/files/list-entries.ts | 107 +++ .../workspace/[workspaceId]/files/page.tsx | 32 +- .../workspace/[workspaceId]/files/prefetch.ts | 108 ++- .../[workspaceId]/files/search-params.test.ts | 23 + .../[workspaceId]/files/search-params.ts | 59 +- .../[workspaceId]/files/untitled-title.ts | 4 +- .../components/special-tags/special-tags.tsx | 21 +- .../mothership-chat/copyable-markdown.test.ts | 5 +- .../mothership-chat/copyable-markdown.ts | 59 +- .../mothership-chat/mothership-chat.tsx | 17 +- .../add-resource-dropdown.tsx | 21 +- .../available-resources.ts | 33 +- .../organization-resource-inventory.tsx | 4 + .../resource-content/resource-content.tsx | 50 +- .../resource-invalidation.ts | 8 +- .../resource-tabs/resource-tabs.tsx | 19 +- .../mothership-view/mothership-view.tsx | 16 +- .../plus-menu-dropdown/plus-menu-dropdown.tsx | 4 +- .../app/workspace/[workspaceId]/home/home.tsx | 34 +- .../preview/use-file-preview-controller.ts | 12 +- .../app/workspace/[workspaceId]/home/page.tsx | 16 +- .../workspace/[workspaceId]/home/prefetch.ts | 27 - .../home/resolve-resource-ref.test.ts | 76 +- .../home/resolve-resource-ref.ts | 97 +++ .../[workspaceId]/lib/prefetch.test.ts | 25 +- .../lib/seed-workspace-files.test.ts | 45 -- .../[workspaceId]/lib/seed-workspace-files.ts | 48 -- .../table-grid/cells/sim-resource-cell.tsx | 11 +- .../tests/components/test-detail.tsx | 4 +- .../components/file-upload/file-upload.tsx | 127 +++- .../replacement-controls.tsx | 4 + ...e-workflow-resource-replacement-options.ts | 14 +- .../workflow-search-replace.tsx | 8 + .../collapsed-sidebar-menu.tsx | 38 +- .../rail-resource-flyout.tsx | 139 +++- .../components/search-modal/search-modal.tsx | 57 +- apps/sim/hooks/queries/projects.ts | 2 +- apps/sim/hooks/queries/public-shares.ts | 4 +- .../queries/utils/workspace-file-cache.ts | 161 +++++ .../queries/utils/workspace-file-keys.ts | 165 +++++ .../workspace-file-lookup-batcher.test.ts | 134 ++++ .../utils/workspace-file-lookup-batcher.ts | 134 ++++ .../hooks/queries/workflow-search-replace.ts | 134 ++-- .../hooks/queries/workspace-file-folders.ts | 11 +- .../hooks/queries/workspace-files.test.tsx | 287 +++++++- apps/sim/hooks/queries/workspace-files.ts | 672 +++++++++++++----- apps/sim/lib/api/contracts/workspace-files.ts | 5 +- .../tools/client/resource-display.ts | 21 +- .../__integration__/file-list.integration.ts | 23 +- .../contexts/workspace/file-manager.ts | 14 +- .../contexts/workspace/file-query.test.ts | 18 +- apps/sim/lib/uploads/utils/file.ts | 14 - apps/sim/lib/workspace-files/queries.test.ts | 81 --- apps/sim/lib/workspace-files/queries.ts | 44 -- packages/testing/src/mocks/file-utils.mock.ts | 4 - .../check-source-conventions.baseline.json | 1 - scripts/check-unused-exports.baseline.json | 3 - 68 files changed, 2893 insertions(+), 1114 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/list-entries.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/search-params.test.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/prefetch.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.test.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.ts create mode 100644 apps/sim/hooks/queries/utils/workspace-file-cache.ts create mode 100644 apps/sim/hooks/queries/utils/workspace-file-keys.ts create mode 100644 apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts create mode 100644 apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts delete mode 100644 apps/sim/lib/workspace-files/queries.test.ts delete mode 100644 apps/sim/lib/workspace-files/queries.ts diff --git a/apps/sim/app/o/[organizationId]/home/organization-home.tsx b/apps/sim/app/o/[organizationId]/home/organization-home.tsx index c48be524d9f..26cae495931 100644 --- a/apps/sim/app/o/[organizationId]/home/organization-home.tsx +++ b/apps/sim/app/o/[organizationId]/home/organization-home.tsx @@ -36,7 +36,7 @@ import { useResourcePanelController, } from '@/app/workspace/[workspaceId]/home/hooks/use-resource-panel' import { useRestoredChatEntry } from '@/app/workspace/[workspaceId]/home/hooks/use-restored-chat-entry' -import { resolveWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' +import { fetchWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' import { searchFiltersFromParams } from '@/app/workspace/[workspaceId]/home/search-params' import type { ChatRequestMode, @@ -47,7 +47,6 @@ import { useFeatureFlag } from '@/app/workspace/[workspaceId]/providers/feature- import { useFileAttachments } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments' import { mentionifyIntegrations } from '@/blocks/integration-matcher' import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats' -import { getWorkspaceFilesQueryOptions } from '@/hooks/queries/workspace-files' import { useHydrated } from '@/hooks/use-hydrated' import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store' import { useOrganizationChatModeStore } from '@/stores/organization-chat-mode/store' @@ -180,11 +179,7 @@ function OrganizationHomeContent({ toast.error('This resource is outside this organization.') return } - const files = - ref.type === 'file' - ? await queryClient.fetchQuery(getWorkspaceFilesQueryOptions(ref.workspaceId)) - : [] - const resource = resolveWorkspaceResourceRef(ref, files) + const resource = await fetchWorkspaceResourceRef(queryClient, ref.workspaceId, ref) if (!resource) { toast.error(`Couldn't find "${ref.title}" in its workspace`) return diff --git a/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx b/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx index 6e21bf049ba..d0bcf51eb1a 100644 --- a/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx +++ b/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx @@ -17,7 +17,7 @@ import Link from 'next/link' import type { ChangelogRelease } from '@/lib/api/contracts/changelog' import { ReleaseWorkflows } from '@/app/workspace/[workspaceId]/changelog/components/release-workflows/release-workflows' import { FileViewer } from '@/app/workspace/[workspaceId]/files/components/file-viewer' -import { useAddressedWorkspaceFileRecord } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' interface ReleaseEntryProps { workspaceId: string @@ -34,7 +34,7 @@ export function ReleaseEntry({ workspaceId, release, canEdit, onEditDetails }: R const saveFailedRef = useRef(false) const [editingBody, setEditingBody] = useState(false) const [savingBody, setSavingBody] = useState(false) - const file = useAddressedWorkspaceFileRecord(workspaceId, release.fileId) + const file = useWorkspaceFileRecord(workspaceId, release.fileId) return (
diff --git a/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/page.tsx b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/page.tsx index 9955b5c4f4f..34afdcb3ee4 100644 --- a/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/chat/[chatId]/page.tsx @@ -1,13 +1,10 @@ import { Suspense } from 'react' -import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' import { notFound } from 'next/navigation' import { getSession } from '@/lib/auth' import { isChatEnabled } from '@/lib/core/config/env-flags' -import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { Home } from '@/app/workspace/[workspaceId]/home/home' import { HomeFallback } from '@/app/workspace/[workspaceId]/home/home-fallback' -import { prefetchHomeSurface } from '@/app/workspace/[workspaceId]/home/prefetch' export const metadata: Metadata = { title: 'Chat', @@ -22,20 +19,20 @@ interface ChatPageProps { export default async function ChatPage({ params }: ChatPageProps) { // The layout 404s too, but pages and layouts resolve concurrently — without this - // the prefetch below still fires on its way out. + // the page still renders on its way out. if (!isChatEnabled) { notFound() } - const [{ workspaceId, chatId }, session] = await Promise.all([params, getSession()]) - const userId = session?.user?.id - const queryClient = getQueryClient() - await prefetchHomeSurface(queryClient, workspaceId, userId) + const [{ chatId }, session] = await Promise.all([params, getSession()]) return ( - - }> - - - + }> + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx index 3607187a16a..4de6fcc2960 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx @@ -216,9 +216,12 @@ interface ResourceTableProps { apiRef?: RefObject /** * Window the row list with `@tanstack/react-virtual`, keeping only the visible - * slice in the DOM. Opt-in because it removes off-screen rows — consumers that - * depend on every row being mounted (e.g. drag-and-drop drop targets) must stay - * on the full-DOM path. Enable it only for unbounded, accumulating lists (logs). + * slice in the DOM. Opt-in because it removes off-screen rows — a consumer that + * reaches rows through the DOM rather than its own row ids must stay on the + * full-DOM path. Row drag-and-drop works windowed: a drop target is only ever a + * row under the pointer, and the drag's teardown never depends on its source row + * staying mounted (`useDragTeardown`). Enable it only for unbounded, accumulating + * lists (logs, files). */ virtualized?: boolean selectable?: SelectableConfig diff --git a/apps/sim/app/workspace/[workspaceId]/files/[fileId]/page.tsx b/apps/sim/app/workspace/[workspaceId]/files/[fileId]/page.tsx index b4808ec50dd..64f225b3a74 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/[fileId]/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/[fileId]/page.tsx @@ -13,13 +13,13 @@ export const metadata: Metadata = { } /** - * File detail entry. `Files` resolves the open file out of the workspace file LIST, so this route - * needs the same prefetch its sibling list page does — without it the server can only ever render - * the "resolving the record" spinner, and the real header (breadcrumbs, actions) has to pop in a - * frame later on the client. + * File detail entry. `Files` resolves the open file from its record, so this route prefetches it + * with the browser's chrome — without it the server can only ever render the "resolving the + * record" spinner, and the real header (breadcrumbs, actions) has to pop in a frame later on the + * client. * * It also removes a whole class of hydration mismatch: which branch `Files` renders is decided by - * whether that list is in the cache, so a server render without it and a client render with it + * whether that record is in the cache, so a server render without it and a client render with it * disagree on the header's markup (a static `…` crumb vs. the file's dropdown crumb). Prefetching * here makes both sides read the same cache and pick the same branch by construction. * @@ -31,11 +31,11 @@ export default async function FilesFilePage({ }: { params: Promise<{ workspaceId: string; fileId: string }> }) { - const [{ workspaceId }, session] = await Promise.all([params, getSession()]) + const [{ workspaceId, fileId }, session] = await Promise.all([params, getSession()]) const queryClient = getQueryClient() if (session?.user?.id) { - await prefetchFilesBrowser(queryClient, workspaceId, session.user.id) + await prefetchFilesBrowser(queryClient, workspaceId, session.user.id, { fileId }) } return ( diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts index 954e9efc68a..3ac9c8ac5ae 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/use-file-doc-collaboration.ts @@ -9,7 +9,8 @@ import { getUserColor } from '@/lib/workspaces/colors' import { FileDocProvider } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-provider' import { useReportFileDocOthers } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/collaboration/file-doc-room-context' import { useSocket } from '@/app/workspace/providers/socket-provider' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' +import { workspaceFileMetadataFilter } from '@/hooks/queries/utils/workspace-file-cache' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' /** The live collaboration binding the editor wires into TipTap's Collaboration * (the {@link Y.Doc}) and CollaborationCaret (the awareness). */ @@ -120,8 +121,7 @@ export function useFileDocCollaboration({ onWriteAccessLostRef.current?.() const filters: QueryFilters[] = [ { queryKey: workspaceFilesKeys.contentFile(workspaceId, fileId) }, - { queryKey: workspaceFilesKeys.record(workspaceId, fileId) }, - { queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }, + workspaceFileMetadataFilter(workspaceId), ] await Promise.all(filters.map((filter) => queryClient.cancelQueries(filter))) await Promise.all(filters.map((filter) => queryClient.resetQueries(filter))) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention.ts index 78e91f881ec..d68c98efeb3 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention.ts @@ -12,13 +12,15 @@ import { MENTION_PLUGIN_KEY } from '@/app/workspace/[workspaceId]/files/componen /** * Per-editor storage for the `@` mention extension. The host component populates {@link store} with * the current workspace mention data and may set {@link onOpen} to lazily start fetching that data the - * first time the menu is triggered. {@link enabled} gates the menu off entirely (e.g. a field with no + * first time the menu is triggered, and {@link onQuery} to hear what is typed after the `@` (files are + * searched on the server rather than listed whole). {@link enabled} gates the menu off entirely (e.g. a field with no * workspace scope) so `@` stays literal text. {@link navigable} lets a chip Cmd/Ctrl-click to its * resource — on for the file viewer, off inside a modal field so it can't route away from an edit. */ export interface MentionStorage { store: MentionStore onOpen: (() => void) | null + onQuery: ((query: string) => void) | null enabled: boolean navigable: boolean } @@ -34,14 +36,21 @@ declare module '@tiptap/core' { * `@` inside an email/handle (`name@host`) stays literal — opens {@link MentionList}; selecting an * entity inserts it as a portable `sim:/` markdown link (same wire format as the chat * composer's `chip-clipboard-codec`), so it round-trips natively through the editor's link + markdown - * machinery. The plugin's `items` is an empty gate; the real list is sourced reactively from the store - * inside {@link MentionList}, populated by the host via the extension's `mentionMenu` storage. + * machinery. The plugin's `items` is an empty gate that only reports the query to the host; the real + * list is sourced reactively from the store inside {@link MentionList}, populated by the host via the + * extension's `mentionMenu` storage. */ export const Mention = Extension.create, MentionStorage>({ name: 'mentionMenu', addStorage() { - return { store: createMentionStore(), onOpen: null, enabled: true, navigable: false } + return { + store: createMentionStore(), + onOpen: null, + onQuery: null, + enabled: true, + navigable: false, + } }, addProseMirrorPlugins() { @@ -61,7 +70,10 @@ export const Mention = Extension.create, MentionStorage>({ if ($from.parentOffset === 0) return true return /\s/.test($from.parent.textBetween($from.parentOffset - 1, $from.parentOffset)) }, - items: () => [], + items: ({ editor, query }) => { + editor.storage.mentionMenu.onQuery?.(query) + return [] + }, command: ({ editor, range, props }) => { editor .chain() diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts index 72ebc583a18..e2b37ff309c 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts @@ -11,7 +11,8 @@ interface UseEditorMentionsOptions { /** * Wires an editor's `@` mention menu to its workspace data: gates the menu on a workspace scope, - * lazily fetches the data on the first open, and feeds it into the menu's reactive store. Shared by + * lazily fetches the data on the first open, searches files by what is typed after the `@`, and + * feeds it into the menu's reactive store. Shared by * every editor surface that mounts the mention extension (the file editor and the modal field). */ export function useEditorMentions( @@ -20,7 +21,8 @@ export function useEditorMentions( options?: UseEditorMentionsOptions ): void { const [active, setActive] = useState(false) - const items = useMarkdownMentions(workspaceId, { enabled: active }) + const [query, setQuery] = useState('') + const items = useMarkdownMentions(workspaceId, { enabled: active, fileQuery: query }) const navigable = options?.navigable ?? false const disableTagging = options?.disableTagging ?? false @@ -30,8 +32,10 @@ export function useEditorMentions( editor.storage.mentionMenu.enabled = taggingOn editor.storage.mentionMenu.navigable = navigable editor.storage.mentionMenu.onOpen = taggingOn ? () => setActive(true) : null + editor.storage.mentionMenu.onQuery = taggingOn ? setQuery : null return () => { editor.storage.mentionMenu.onOpen = null + editor.storage.mentionMenu.onQuery = null } }, [editor, workspaceId, navigable, disableTagging]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts index 1451fffe7c2..f29f8a6a881 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts @@ -1,23 +1,34 @@ import { useMemo } from 'react' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { listIntegrations } from '@/blocks/integration-matcher' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useSkills } from '@/hooks/queries/skills' import { useTablesList } from '@/hooks/queries/tables' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { useWorkflows } from '@/hooks/queries/workflows' import { useWorkspaceFileFolders } from '@/hooks/queries/workspace-file-folders' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' +import { useDebounce } from '@/hooks/use-debounce' import { mentionIcon } from './mention-icon' import type { MentionItem } from './types' +/** The files the menu offers before anything is typed: the most recently updated anywhere. */ +const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 50 }) + +const FILE_SEARCH_LIMIT = 50 + /** * Aggregates the workspace-scoped entities the `@` menu can reference, composing the canonical * per-resource React Query hooks (never the chat-coupled `useAvailableResources` aggregator). All * queries stay disabled until `enabled` flips true — the host activates it on the first `@` trigger — * so a markdown field that never opens the menu fetches nothing. + * + * Files are the one family not listed whole: the menu offers the most recently updated, and once + * `fileQuery` (the text after the `@`) settles, the files whose names match it anywhere. */ export function useMarkdownMentions( workspaceId: string | undefined, - options: { enabled: boolean } + options: { enabled: boolean; fileQuery?: string } ): MentionItem[] { const active = options.enabled && Boolean(workspaceId) // When inactive, `ws` is undefined and `wsStr` is '' so every query stays disabled until the first @@ -26,7 +37,14 @@ export function useMarkdownMentions( const ws = active ? workspaceId : undefined const wsStr = ws ?? '' - const files = useWorkspaceFiles(wsStr, 'active', { enabled: active }) + const fileQuery = useDebounce(options.fileQuery?.trim() ?? '', SEARCH_DEBOUNCE_MS) + const recentFiles = useWorkspaceFileList(wsStr, RECENT_FILES, { enabled: active }) + const matchingFiles = useWorkspaceFileSearch( + wsStr, + { q: fileQuery, limit: FILE_SEARCH_LIMIT }, + { enabled: active } + ) + const files = fileQuery ? matchingFiles.data : recentFiles.data?.pages[0]?.files const folders = useWorkspaceFileFolders(wsStr, 'active') const tables = useTablesList(ws, 'active') const knowledgeBases = useKnowledgeBasesQuery(ws, { enabled: active }) @@ -49,7 +67,7 @@ export function useMarkdownMentions( if (!active) return [] const items: MentionItem[] = [] - for (const file of files.data ?? []) + for (const file of files ?? []) items.push({ kind: 'file', id: file.id, @@ -102,7 +120,7 @@ export function useMarkdownMentions( return items }, [ active, - files.data, + files, folders.data, tables.data, knowledgeBases.data, diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 23724ba11ce..6d3f01d70b6 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -22,6 +22,7 @@ import { import { Check, Download, Link, Send } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { useQueryClient } from '@tanstack/react-query' import { useParams, useRouter } from 'next/navigation' import { useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' @@ -40,18 +41,13 @@ import { getFileExtension, getMimeTypeFromExtension, isArchiveFileName, - isAudioFileType, - isVideoFileType, resolveEffectiveMimeType, } from '@/lib/uploads/utils/file' -import { isSupportedExtension } from '@/lib/uploads/utils/validation' +import { isWorkspaceFileKind, isWorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { FindBar } from '@/app/workspace/[workspaceId]/components/find-bar/find-bar' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut' -import type { - MoveOptionNode, - SortableResource, -} from '@/app/workspace/[workspaceId]/components/folders' +import type { MoveOptionNode } from '@/app/workspace/[workspaceId]/components/folders' import { breadcrumbFolderChain, buildDescendantIndex, @@ -67,7 +63,6 @@ import { parseFolderedRowId, parseMoveOptionValue, scopeFolderedItems, - sortResources, splitFolderedRowIds, useFolderRowDragDrop, } from '@/app/workspace/[workspaceId]/components/folders' @@ -118,8 +113,10 @@ import { FileDocRoomProvider } from '@/app/workspace/[workspaceId]/files/compone import { FilesListContextMenu } from '@/app/workspace/[workspaceId]/files/components/files-list-context-menu' import { ShareModal } from '@/app/workspace/[workspaceId]/files/components/share-modal' import { useWorkspaceFilesRoom } from '@/app/workspace/[workspaceId]/files/hooks/use-workspace-files-room' +import { mergeFileListEntries } from '@/app/workspace/[workspaceId]/files/list-entries' import FilesLoading from '@/app/workspace/[workspaceId]/files/loading' import { + filesBrowserListParams, filesFilterParsers, filesFilterUrlKeys, filesListPreferenceConfig, @@ -148,11 +145,16 @@ import { type WorkspaceFileFolderApi, } from '@/hooks/queries/workspace-file-folders' import { + getWorkspaceFileRecordQueryOptions, + getWorkspaceFileSearchQueryOptions, useCreateWorkspaceFile, useDeleteWorkspaceFile, useRenameWorkspaceFile, useUploadWorkspaceFile, - useWorkspaceFiles, + useWorkspaceFileFolderSizes, + useWorkspaceFileList, + useWorkspaceFileLookups, + useWorkspaceFileRecord, } from '@/hooks/queries/workspace-files' import { useContextMenu } from '@/hooks/use-context-menu' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' @@ -168,11 +170,6 @@ type FileResourceItem = | { kind: 'file'; id: string; file: WorkspaceFileRecord } | { kind: 'folder'; id: string; folder: WorkspaceFileFolderApi } -/** One row of the merged folder+file list, before it becomes a `ResourceRow`. */ -type FileListEntry = - | { kind: 'folder'; folder: WorkspaceFileFolderApi } - | { kind: 'file'; file: WorkspaceFileRecord } - const logger = createLogger('Files') /** @@ -185,12 +182,12 @@ const FILES_HEADER = FOLDERED_RESOURCE_HEADERS.file const FOLDER_ICON = -/** Folders' value in the `type` column — also their sort key when that column is active. */ +/** Folders' value in the `type` column. */ const FOLDER_TYPE_LABEL = 'Folder' as const /** - * Debounce window for `search` URL writes and filtering; the input itself stays - * instant. Intentionally shorter than the shared `SEARCH_DEBOUNCE_MS` (300). + * Debounce window for `search` URL writes and the server search they drive; the input + * itself stays instant. Intentionally shorter than the shared `SEARCH_DEBOUNCE_MS` (300). */ const FILES_SEARCH_DEBOUNCE_MS = 200 as const @@ -232,6 +229,9 @@ const EMPTY_WORKSPACE_FILES: WorkspaceFileRecord[] = [] const EMPTY_WORKSPACE_FILE_FOLDERS: WorkspaceFileFolderApi[] = [] const EMPTY_FIND_MATCH_IDS: readonly string[] = Object.freeze([]) +/** How many similarly named files a derived title is made unique against. */ +const TITLE_SIBLING_LIMIT = 50 + const hasExternalFiles = (dataTransfer: DataTransfer): boolean => dataTransfer.types.includes('Files') @@ -287,6 +287,7 @@ function FilesContent() { const params = useParams() const router = useRouter() + const queryClient = useQueryClient() const [{ folderId: currentFolderId, new: isNewFile, shareFileId }, setFilesParams] = useQueryStates(filesParsers, filesUrlKeys) const workspaceId = params?.workspaceId as string @@ -313,12 +314,6 @@ function FilesContent() { } }, [permissionConfig.hideFilesTab, router, workspaceId]) - const { - data: files = EMPTY_WORKSPACE_FILES, - isLoading, - isPlaceholderData, - error, - } = useWorkspaceFiles(workspaceId) const { data: folders = EMPTY_WORKSPACE_FILE_FOLDERS, isSuccess: foldersLoaded, @@ -362,24 +357,7 @@ function FilesContent() { closeMenu: closeListContextMenu, } = useContextMenu() - if (error) { - logger.error('Failed to load files:', error) - } - const justCreatedFileIdRef = useRef(null) - const filesRef = useRef(files) - filesRef.current = files - /** - * Indexed once. The drag hook resolves each dragged row's placement inside `dragover`, which - * fires continuously — a linear scan there is O(selection x resources) per event. - */ - const fileById = useMemo(() => { - const byId = new Map() - for (const file of files) byId.set(file.id, file) - return byId - }, [files]) - const fileByIdRef = useRef(fileById) - fileByIdRef.current = fileById const [uploadProgress, setUploadProgress] = useState({ completed: 0, @@ -409,8 +387,8 @@ function FilesContent() { /** * The input is controlled directly by the instant nuqs value; only the URL - * write is debounced. The in-memory filter below still reads a debounced value - * so it doesn't recompute on every keystroke. + * write is debounced. The list query below reads a debounced value so it does + * not search on every keystroke. */ const setSearchTerm = useDebouncedSearchSetter( (value, options) => setFileFilters({ search: value }, options), @@ -463,8 +441,8 @@ function FilesContent() { const applyListPreference = useCallback( (preference: ResourceListPreference) => { void setFileFilters({ - type: [...preference.filters.type], - size: [...preference.filters.size], + type: preference.filters.type.filter(isWorkspaceFileKind), + size: preference.filters.size.filter(isWorkspaceFileSizeBucket), uploadedBy: [...preference.filters.uploadedBy], }) applyUrlSort(preference.sort.column, preference.sort.direction) @@ -499,13 +477,97 @@ function FilesContent() { [setListFilter] ) + /** + * A query stops scoping the list to the open folder — see {@link scopeFolderedItems}. A + * matching folder anywhere in the workspace is a result in its own right, since opening it + * is often what the user was looking for. + */ + const isSearching = isSearchingResources(debouncedSearchTerm) + const isFilteringFiles = + typeFilter.length > 0 || sizeFilter.length > 0 || uploadedByFilter.length > 0 + /** A remembered sort or filter is applied before the list is read, so it is read once. */ + const isListOpen = fileIdFromRoute === null && isListPreferenceReady + + /** + * The open folder's files (or, while searching, matches from every folder), a server-sorted, + * server-filtered page at a time; scrolling to the end of the table loads the next. + */ + const fileList = useWorkspaceFileList( + workspaceId, + filesBrowserListParams({ + folderId: currentFolderId, + search: debouncedSearchTerm, + sort: sortColumn, + dir: sortDirection, + type: typeFilter, + size: sizeFilter, + uploadedBy: uploadedByFilter, + }), + { enabled: isListOpen } + ) + const { isLoading, isPlaceholderData, error } = fileList + if (error) { + logger.error('Failed to load files:', error) + } + const pagedFiles = useMemo( + () => fileList.data?.pages.flatMap((page) => page.files) ?? EMPTY_WORKSPACE_FILES, + [fileList.data] + ) + + /** + * A folder's pinned files lead it whether or not a page has reached them, so they are looked + * up by id. A search or filter narrows the list to its own results, which pin among themselves. + */ + const pinsLeadFolder = !isSearching && !isFilteringFiles + const pinnedFileIdList = useMemo(() => [...pinnedFileIds], [pinnedFileIds]) + const pinnedFileLookups = useWorkspaceFileLookups(workspaceId, 'id', pinnedFileIdList, { + enabled: isListOpen && pinsLeadFolder, + }) + const pinnedFolderFiles = useMemo( + () => + pinsLeadFolder + ? pinnedFileIdList.flatMap((fileId) => { + const file = pinnedFileLookups.byValue.get(fileId) + return file && (file.folderId ?? null) === currentFolderId ? [file] : [] + }) + : EMPTY_WORKSPACE_FILES, + [pinsLeadFolder, pinnedFileIdList, pinnedFileLookups.byValue, currentFolderId] + ) + + /** + * Every file the list has loaded, indexed once. The drag hook resolves each dragged row's + * placement inside `dragover`, which fires continuously — a linear scan there is + * O(selection x resources) per event. Rows only ever act on files the list has shown. + */ + const fileById = useMemo(() => { + const byId = new Map() + for (const file of pinnedFolderFiles) byId.set(file.id, file) + for (const file of pagedFiles) byId.set(file.id, file) + return byId + }, [pinnedFolderFiles, pagedFiles]) + const fileByIdRef = useRef(fileById) + fileByIdRef.current = fileById + + /** The open file, from the list it was clicked in when that is cached, else its own record. */ + const selectedFileRecord = useWorkspaceFileRecord(workspaceId, fileIdFromRoute ?? '', { + enabled: fileIdFromRoute !== null, + }) + const selectedFile = fileIdFromRoute ? selectedFileRecord.data : null + + const loadMoreFilesRef = useRef(fileList) + loadMoreFilesRef.current = fileList + const loadMoreFiles = useCallback(() => { + const { isFetching, hasNextPage, fetchNextPage } = loadMoreFilesRef.current + if (!isFetching && hasNextPage) void fetchNextPage() + }, []) + const [creatingFile, setCreatingFile] = useState(false) const [isDirty, setIsDirty] = useState(false) const [saveStatus, setSaveStatus] = useState('idle') const [previewMode, setPreviewMode] = useState(() => { if (isNewFile) return 'editor' if (fileIdFromRoute) { - const file = files.find((f) => f.id === fileIdFromRoute) + const file = selectedFile if (file && isPreviewable(file)) return 'preview' return 'editor' } @@ -541,10 +603,6 @@ function FilesContent() { updateFolder.mutateAsync({ workspaceId, folderId, updates: { name } }), }) - const selectedFile = useMemo( - () => (fileIdFromRoute ? files.find((f) => f.id === fileIdFromRoute) : null), - [fileIdFromRoute, files] - ) const selectedFileRef = useRef(selectedFile) selectedFileRef.current = selectedFile @@ -556,95 +614,59 @@ function FilesContent() { * folder siblings, and renames. A no-op once the file has a real name. */ const handleDeriveTitleFromHeading = useCallback( - (fileId: string, headingText: string) => { - const currentFile = filesRef.current.find((f) => f.id === fileId) - if (!currentFile || !isUntitledName(currentFile.name)) return - const derived = deriveMarkdownFileName(headingText) - if (!derived) return - const siblingNames = new Set( - filesRef.current - .filter( - (f) => - (f.folderId ?? null) === (currentFile.folderId ?? null) && f.id !== currentFile.id - ) - .map((f) => f.name) - ) - const name = uniqueMarkdownName(derived, siblingNames) - if (name === currentFile.name) return - renameFile - .mutateAsync({ workspaceId, fileId: currentFile.id, name }) - .catch((err) => logger.error('Failed to auto-name file from heading:', err)) + async (fileId: string, headingText: string) => { + try { + const currentFile = await queryClient.fetchQuery( + getWorkspaceFileRecordQueryOptions(queryClient, workspaceId, fileId) + ) + if (!isUntitledName(currentFile.name)) return + const derived = deriveMarkdownFileName(headingText) + if (!derived) return + /** The folder's files whose names could collide: every `…` the search can name. */ + const similar = await queryClient.fetchQuery({ + ...getWorkspaceFileSearchQueryOptions(workspaceId, { + q: derived.replace(/\.md$/i, ''), + folderId: currentFile.folderId ?? null, + recursive: false, + limit: TITLE_SIBLING_LIMIT, + }), + staleTime: 0, + }) + const siblingNames = new Set( + similar.filter((file) => file.id !== currentFile.id).map((file) => file.name) + ) + const name = uniqueMarkdownName(derived, siblingNames) + if (name === currentFile.name) return + await renameFile.mutateAsync({ workspaceId, fileId: currentFile.id, name }) + } catch (err) { + logger.error('Failed to auto-name file from heading:', err) + } }, - [workspaceId] + [queryClient, workspaceId] ) - const shareFile = shareFileId ? (files.find((f) => f.id === shareFileId) ?? null) : null - const shareModal = shareFile ? ( - <ShareModal - open - onOpenChange={(open) => - !open && setFilesParams({ shareFileId: null }, { history: 'replace' }) - } - workspaceId={workspaceId} - fileId={shareFile.id} - fileName={shareFile.name} - initialShare={shareFile.share ?? null} - /> - ) : null + /** Opens for a deep link too, once the id resolves to a live file. */ + const { data: shareFile } = useWorkspaceFileRecord(workspaceId, shareFileId ?? '', { + enabled: Boolean(shareFileId), + }) + const shareModal = + shareFileId && shareFile ? ( + <ShareModal + open + onOpenChange={(open) => + !open && setFilesParams({ shareFileId: null }, { history: 'replace' }) + } + workspaceId={workspaceId} + fileId={shareFile.id} + fileName={shareFile.name} + initialShare={shareFile.share ?? null} + /> + ) : null const folderById = useMemo(() => new Map(folders.map((folder) => [folder.id, folder])), [folders]) const folderByIdRef = useRef(folderById) folderByIdRef.current = folderById - const folderSizeMap = useMemo(() => { - const directSize = new Map<string, number>() - for (const file of files) { - if (file.folderId) { - directSize.set(file.folderId, (directSize.get(file.folderId) ?? 0) + file.size) - } - } - /** - * Children indexed once rather than re-scanning `folders` per node — the roll-up visits - * every folder, so the filter made this quadratic. - */ - const childrenByParent = new Map<string, WorkspaceFileFolderApi[]>() - for (const folder of folders) { - if (!folder.parentId) continue - const siblings = childrenByParent.get(folder.parentId) - if (siblings) siblings.push(folder) - else childrenByParent.set(folder.parentId, [folder]) - } - - const totalSize = new Map<string, number>() - /** - * `visiting` terminates a parent/child cycle. The optimistic folder-move write can produce - * one in cache, and without the guard this recurses until the stack blows and takes the - * whole page down — the same guard the shared folder helpers carry. - */ - const visiting = new Set<string>() - const getTotal = (folderId: string): number => { - const cached = totalSize.get(folderId) - if (cached !== undefined) return cached - if (visiting.has(folderId)) return 0 - visiting.add(folderId) - const size = - (directSize.get(folderId) ?? 0) + - (childrenByParent.get(folderId) ?? []).reduce((sum, child) => sum + getTotal(child.id), 0) - visiting.delete(folderId) - totalSize.set(folderId, size) - return size - } - for (const folder of folders) getTotal(folder.id) - return totalSize - }, [files, folders]) - - /** - * A query stops scoping the list to the open folder — see {@link scopeFolderedItems}. A - * matching folder anywhere in the workspace is a result in its own right, since opening it - * is often what the user was looking for. - */ - const isSearching = isSearchingResources(debouncedSearchTerm) - const visibleFolders = useMemo( () => scopeFolderedItems(folders, { @@ -656,115 +678,75 @@ function FilesContent() { [folders, currentFolderId, debouncedSearchTerm] ) - const filteredFiles = useMemo(() => { - let result = scopeFolderedItems(files, { - currentFolderId, - search: debouncedSearchTerm, - getParentId: (f) => f.folderId ?? null, - getSearchText: (f) => [f.name], - }) - - if (typeFilter.length > 0) { - result = result.filter((f) => { - const ext = getFileExtension(f.name) - // Matching the raw stored type would hide every file the browser uploaded as - // `application/octet-stream` from the audio/video/image filters. - const type = resolveEffectiveMimeType(f.type, f.name) - if (typeFilter.includes('document') && isSupportedExtension(ext)) return true - if (typeFilter.includes('audio') && isAudioFileType(type)) return true - if (typeFilter.includes('video') && isVideoFileType(type)) return true - if (typeFilter.includes('image') && type.startsWith('image/')) return true - return false - }) - } - - if (sizeFilter.length > 0) { - result = result.filter((f) => { - if (sizeFilter.includes('small') && f.size < 1_048_576) return true - if (sizeFilter.includes('medium') && f.size >= 1_048_576 && f.size <= 10_485_760) - return true - if (sizeFilter.includes('large') && f.size > 10_485_760) return true - return false - }) - } - - if (uploadedByFilter.length > 0) { - result = result.filter((f) => uploadedByFilter.includes(f.uploadedBy)) - } - - return result - }, [files, currentFolderId, debouncedSearchTerm, typeFilter, sizeFilter, uploadedByFilter]) + /** Each shown folder's whole-subtree size, rolled up on the server per parent folder. */ + const folderSizeParents = useMemo( + () => + isSearching + ? [...new Set(visibleFolders.map((folder) => folder.parentId ?? null))] + : [currentFolderId], + [isSearching, visibleFolders, currentFolderId] + ) + const folderSizes = useWorkspaceFileFolderSizes(workspaceId, folderSizeParents, { + enabled: isListOpen, + }) /** - * Folders and files sort as ONE list — a folder never outranks a file it ties with, so a - * pinned file reaches the top of the list rather than the top of the file section. - * - * Decorate-sort: each row's key + pinned flag is computed ONCE (O(N)) so the comparator - * never re-runs Date parsing, `formatFileType`, or member lookups per comparison. Every - * Files column has a folder equivalent (folders carry a size roll-up and sort as type - * "Folder"), so no entry needs a null key here. + * Folders interleave with the server-ordered files as ONE list — a folder never outranks a + * file it ties with, so a pinned file reaches the top of the list rather than the top of the + * file section. See {@link mergeFileListEntries} for how a folder is placed among pages that + * have not all loaded. */ - const sortedEntries = useMemo(() => { - const entries: SortableResource<FileListEntry>[] = [] - - for (const folder of visibleFolders) { - entries.push({ - item: { kind: 'folder', folder }, - pinned: pinnedFolderIds.has(folder.id), - name: folder.name, - key: + const sortedEntries = useMemo( + () => + mergeFileListEntries<WorkspaceFileFolderApi, WorkspaceFileRecord>({ + folders: visibleFolders, + files: pagedFiles, + pinnedFiles: pinnedFolderFiles, + hasMore: fileList.hasNextPage, + isFolderPinned: (folder) => pinnedFolderIds.has(folder.id), + isFilePinned: (file) => pinnedFileIds.has(file.id), + folderId: (folder) => folder.id, + fileId: (file) => file.id, + folderName: (folder) => folder.name, + fileName: (file) => file.name, + folderKey: (folder) => sortColumn === 'size' - ? (folderSizeMap.get(folder.id) ?? 0) - : sortColumn === 'type' - ? FOLDER_TYPE_LABEL - : sortColumn === 'created' - ? new Date(folder.createdAt).getTime() - : sortColumn === 'updated' - ? new Date(folder.updatedAt).getTime() - : sortColumn === 'owner' - ? (membersById.get(folder.userId)?.name ?? null) - : folder.name, - }) - } - - for (const file of filteredFiles) { - entries.push({ - item: { kind: 'file', file }, - pinned: pinnedFileIds.has(file.id), - name: file.name, - key: + ? (folderSizes.get(folder.id)?.sizeBytes ?? 0) + : sortColumn === 'created' + ? new Date(folder.createdAt).getTime() + : sortColumn === 'updated' + ? new Date(folder.updatedAt).getTime() + : folder.name, + fileKey: (file) => sortColumn === 'size' ? file.size - : sortColumn === 'type' - ? formatFileType(file.type, file.name) - : sortColumn === 'created' - ? new Date(file.uploadedAt).getTime() - : sortColumn === 'updated' - ? new Date(file.updatedAt).getTime() - : sortColumn === 'owner' - ? (membersById.get(file.uploadedBy)?.name ?? null) - : file.name, - }) - } - - return sortResources(entries, sortDirection) - }, [ - visibleFolders, - filteredFiles, - sortColumn, - sortDirection, - membersById, - folderSizeMap, - pinnedFolderIds, - pinnedFileIds, - ]) + : sortColumn === 'created' + ? new Date(file.uploadedAt).getTime() + : sortColumn === 'updated' + ? new Date(file.updatedAt).getTime() + : file.name, + direction: sortDirection, + }), + [ + visibleFolders, + pagedFiles, + pinnedFolderFiles, + fileList.hasNextPage, + pinnedFolderIds, + pinnedFileIds, + folderSizes, + sortColumn, + sortDirection, + ] + ) const baseRows: ResourceRow[] = useMemo( () => - sortedEntries.map(({ item, pinned }): ResourceRow => { + sortedEntries.map((item): ResourceRow => { + const { pinned } = item if (item.kind === 'folder') { const { folder } = item - const totalSize = folderSizeMap.get(folder.id) ?? 0 + const totalSize = folderSizes.get(folder.id)?.sizeBytes ?? 0 return { id: folderRowId(folder.id), cells: { @@ -826,7 +808,7 @@ function FilesContent() { }, } }), - [sortedEntries, membersById, folderSizeMap, folderById, isSearching] + [sortedEntries, membersById, folderSizes, folderById, isSearching] ) const rows: ResourceRow[] = useMemo(() => { @@ -852,9 +834,10 @@ function FilesContent() { }) }, [baseRows, listRename.editingId, listRename.editValue, listRename.isSaving]) - // Find (Cmd/Ctrl+F): the shared find bar over the visible list, stepping - // through rows whose name matches. The list is client-side, so matching is - // synchronous — no debounce or loading states. + // Find (Cmd/Ctrl+F): the shared find bar over the loaded rows, stepping + // through rows whose name matches. It searches what is on screen, so matching + // is synchronous — no debounce or loading states; the search box searches the + // whole workspace. const [findOpen, setFindOpen] = useState(false) const [findQuery, setFindQuery] = useState('') const [findIndex, setFindIndex] = useState(0) @@ -1225,12 +1208,11 @@ function FilesContent() { folderIds: selectedFolderIds, name: selectionLabel( selectedFileIds.length + selectedFolderIds.length, - files.find((file) => file.id === selectedFileIds[0])?.name ?? - folders.find((folder) => folder.id === selectedFolderIds[0])?.name + fileById.get(selectedFileIds[0])?.name ?? folderById.get(selectedFolderIds[0])?.name ), }) setShowDeleteConfirm(true) - }, [selectedFileIds, selectedFolderIds, files, folders]) + }, [selectedFileIds, selectedFolderIds, fileById, folderById]) const [isDownloadingArchive, setIsDownloadingArchive] = useState(false) // Ref as well as state: two clicks in the same tick would both pass a state check, @@ -1256,7 +1238,10 @@ function FilesContent() { ) const handleBulkDownload = useCallback(async () => { - const selectedFiles = files.filter((file) => selectedFileIds.includes(file.id)) + const selectedFiles = selectedFileIds.flatMap((fileId) => { + const file = fileById.get(fileId) + return file ? [file] : [] + }) if (selectedFiles.length === 1 && selectedFolderIds.length === 0) { handleDownload(selectedFiles[0]) return @@ -1269,7 +1254,7 @@ function FilesContent() { file_count: selectedFileIds.length + selectedFolderIds.length, }) await downloadArchive({ fileIds: selectedFileIds, folderIds: selectedFolderIds }) - }, [selectedFileIds, selectedFolderIds, files, handleDownload, downloadArchive, workspaceId]) + }, [selectedFileIds, selectedFolderIds, fileById, handleDownload, downloadArchive, workspaceId]) const fileDetailBreadcrumbs = useMemo((): BreadcrumbItem[] => { if (!selectedFile) return [] @@ -1340,15 +1325,11 @@ function FilesContent() { setCreatingFile(true) try { - const existingNames = new Set( - filesRef.current.filter((f) => (f.folderId ?? null) === currentFolderId).map((f) => f.name) - ) - const name = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, existingNames) - const mimeType = getMimeTypeFromExtension('md') + /** The server numbers the name (`untitled (1).md`, …) when the folder already has it. */ const result = await createWorkspaceFile.mutateAsync({ workspaceId, - name, + name: DEFAULT_UNTITLED_NAME, contentType: mimeType, folderId: currentFolderId ?? undefined, }) @@ -1406,7 +1387,7 @@ function FilesContent() { const item = parsed.kind === 'folder' ? folders.find((folder) => folder.id === parsed.id) - : filesRef.current.find((file) => file.id === parsed.id) + : fileByIdRef.current.get(parsed.id) if (!item) return contextMenuItemRef.current = parsed.kind === 'folder' @@ -1553,14 +1534,14 @@ function FilesContent() { /** * Tracks the route target whose preview mode has been applied. Starts at * null (the list view) rather than the initial route id because on a hard - * load the files list may not have arrived when the mode initializer ran — + * load the file's record may not have arrived when the mode initializer ran — * a deep-linked previewable file would otherwise be locked into the code * editor. The effect therefore defers until the routed file is resolvable: - * either its record exists, or the files query has settled (so a missing - * id decides 'editor' instead of waiting forever). + * either its record exists, or its read has failed (so a missing id decides + * 'editor' instead of waiting forever). */ const appliedModeFileIdRef = useRef<string | null>(null) - const routedFileResolved = selectedFile != null || !isLoading + const routedFileResolved = selectedFile != null || selectedFileRecord.isError useEffect(() => { if (fileIdFromRoute === appliedModeFileIdRef.current) return const isJustCreated = @@ -1953,10 +1934,8 @@ function FilesContent() { options: [ { id: 'name', label: 'Name' }, { id: 'size', label: 'Size' }, - { id: 'type', label: 'Type' }, { id: 'created', label: 'Created' }, { id: 'updated', label: 'Last Updated' }, - { id: 'owner', label: 'Owner' }, ], active: activeSort, onSort: setListSort, @@ -1965,9 +1944,6 @@ function FilesContent() { [activeSort, setListSort, clearListSort] ) - const hasActiveFilters = - typeFilter.length > 0 || sizeFilter.length > 0 || uploadedByFilter.length > 0 - const filterContent = useMemo(() => { const typeDisplayLabel = typeFilter.length === 0 @@ -2053,7 +2029,7 @@ function FilesContent() { /> </ResourceFilterSection> )} - {hasActiveFilters && ( + {isFilteringFiles && ( <Button variant='ghost' onClick={clearFileFilters} @@ -2070,7 +2046,7 @@ function FilesContent() { uploadedByFilter, memberOptions, membersById, - hasActiveFilters, + isFilteringFiles, setTypeFilter, setSizeFilter, setUploadedByFilter, @@ -2143,7 +2119,7 @@ function FilesContent() { if (!isListPreferenceReady) return <FilesLoading /> - if (fileIdFromRoute && !selectedFile && isLoading) { + if (fileIdFromRoute && !selectedFile && selectedFileRecord.isPending) { return ( <Resource> <Resource.Header icon={FILES_HEADER.rootIcon} breadcrumbs={loadingBreadcrumbs} /> @@ -2248,6 +2224,10 @@ function FilesContent() { columns={isSearching ? SEARCH_COLUMNS : COLUMNS} rows={displayRows} apiRef={tableApiRef} + virtualized + onLoadMore={loadMoreFiles} + hasMore={fileList.hasNextPage} + isLoadingMore={fileList.isFetchingNextPage} emptyState={ listState === 'empty' ? ( <FilesEmptyState diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts new file mode 100644 index 00000000000..c70acabb13b --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest' +import { mergeFileListEntries } from '@/app/workspace/[workspaceId]/files/list-entries' + +interface Row { + id: string + name: string + size: number +} + +const folder = (id: string, size: number): Row => ({ id, name: id, size }) +const file = (id: string, size: number): Row => ({ id, name: id, size }) + +function merge(input: { + folders?: Row[] + files: Row[] + pinnedFiles?: Row[] + hasMore: boolean + pinned?: string[] + direction?: 'asc' | 'desc' +}) { + const pinned = new Set(input.pinned ?? []) + return mergeFileListEntries<Row, Row>({ + folders: input.folders ?? [], + files: input.files, + pinnedFiles: input.pinnedFiles ?? [], + hasMore: input.hasMore, + isFolderPinned: (row) => pinned.has(row.id), + isFilePinned: (row) => pinned.has(row.id), + folderId: (row) => row.id, + fileId: (row) => row.id, + folderName: (row) => row.name, + fileName: (row) => row.name, + folderKey: (row) => row.size, + fileKey: (row) => row.size, + direction: input.direction ?? 'asc', + }).map((entry) => (entry.kind === 'folder' ? entry.folder.id : entry.file.id)) +} + +describe('mergeFileListEntries', () => { + it('withholds a folder that sorts past the last loaded file, so loading more only appends', () => { + const folders = [folder('F5', 5), folder('F25', 25)] + const firstPage = [file('a10', 10), file('b20', 20)] + const secondPage = [file('c30', 30), file('d40', 40)] + + const afterOnePage = merge({ folders, files: firstPage, hasMore: true }) + const afterTwoPages = merge({ folders, files: [...firstPage, ...secondPage], hasMore: false }) + + expect(afterOnePage).toEqual(['F5', 'a10', 'b20']) + expect(afterTwoPages).toEqual(['F5', 'a10', 'b20', 'F25', 'c30', 'd40']) + expect(afterTwoPages.slice(0, afterOnePage.length)).toEqual(afterOnePage) + }) + + it('shows every folder once the last page is in, including one past every file', () => { + expect(merge({ folders: [folder('F99', 99)], files: [file('a1', 1)], hasMore: false })).toEqual( + ['a1', 'F99'] + ) + }) + + it('places folders by the descending order the server paged the files in', () => { + expect( + merge({ + folders: [folder('F35', 35), folder('F5', 5)], + files: [file('d40', 40), file('c30', 30), file('b20', 20)], + hasMore: true, + direction: 'desc', + }) + ).toEqual(['d40', 'F35', 'c30', 'b20']) + }) + + it('keeps the server order of the files even where the client would sort them otherwise', () => { + expect(merge({ folders: [], files: [file('z', 2), file('a', 1)], hasMore: false })).toEqual([ + 'z', + 'a', + ]) + }) + + it('leads with pinned rows — a pinned file the pages have not reached, and one they have, once', () => { + expect( + merge({ + folders: [folder('F5', 5), folder('Fpin', 50)], + files: [file('a10', 10), file('pinned20', 20)], + pinnedFiles: [file('pinned20', 20), file('far90', 90)], + hasMore: true, + pinned: ['Fpin', 'pinned20', 'far90'], + }) + ).toEqual(['pinned20', 'Fpin', 'far90', 'F5', 'a10']) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts new file mode 100644 index 00000000000..68745c5b797 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts @@ -0,0 +1,107 @@ +import { + type SortableResource, + sortResources, +} from '@/app/workspace/[workspaceId]/components/folders' + +/** One row of the Files list, before it becomes a `ResourceRow`. */ +export type FileListEntry<Folder, File> = + | { kind: 'folder'; folder: Folder; pinned: boolean } + | { kind: 'file'; file: File; pinned: boolean } + +interface MergeFileListEntriesInput<Folder, File> { + /** The folders the list shows, in any order. */ + folders: readonly Folder[] + /** The loaded files, in the order the server paged them. */ + files: readonly File[] + /** + * Pinned files the pages may not have reached yet (looked up by id), shown first. One that + * a page also holds is shown once, here. + */ + pinnedFiles: readonly File[] + /** Whether the server has more files past the last loaded one. */ + hasMore: boolean + isFolderPinned: (folder: Folder) => boolean + isFilePinned: (file: File) => boolean + folderId: (folder: Folder) => string + fileId: (file: File) => string + folderName: (folder: Folder) => string + fileName: (file: File) => string + /** Each row's value for the active sort column, comparable across folders and files. */ + folderKey: (folder: Folder) => string | number + fileKey: (file: File) => string | number + direction: 'asc' | 'desc' +} + +function compareKeys(a: string | number, b: string | number): number { + return typeof a === 'number' && typeof b === 'number' ? a - b : String(a).localeCompare(String(b)) +} + +/** + * Orders the Files list: pinned rows first, then folders interleaved with the server-paged + * files as ONE list by the active sort. + * + * The files keep the order the server paged them in — re-sorting them here could disagree with + * the server's collation and move a later page's rows above ones already on screen. Each folder + * is placed in front of the first file it sorts at or before. A folder that sorts after the last + * loaded file is withheld while more pages remain, since the file it belongs in front of may be + * on one of them; it appears once a page reaches past it, so loading more only ever appends. + * + * Pinned rows are ordered among themselves by the same column, as `sortResources` orders a + * fully loaded list. + */ +export function mergeFileListEntries<Folder, File>( + input: MergeFileListEntriesInput<Folder, File> +): FileListEntry<Folder, File>[] { + const { direction } = input + const pinnedFileIds = new Set<string>() + const pinned: SortableResource<FileListEntry<Folder, File>>[] = [] + const unpinnedFolders: SortableResource<FileListEntry<Folder, File>>[] = [] + + for (const folder of input.folders) { + const entry = { + item: { kind: 'folder' as const, folder, pinned: input.isFolderPinned(folder) }, + pinned: input.isFolderPinned(folder), + name: input.folderName(folder), + key: input.folderKey(folder), + } + if (entry.pinned) pinned.push(entry) + else unpinnedFolders.push(entry) + } + const addPinnedFile = (file: File) => { + const id = input.fileId(file) + if (pinnedFileIds.has(id)) return + pinnedFileIds.add(id) + pinned.push({ + item: { kind: 'file', file, pinned: true }, + pinned: true, + name: input.fileName(file), + key: input.fileKey(file), + }) + } + for (const file of input.pinnedFiles) addPinnedFile(file) + for (const file of input.files) if (input.isFilePinned(file)) addPinnedFile(file) + + const entries = sortResources(pinned, direction).map((entry) => entry.item) + const folders = sortResources(unpinnedFolders, direction) + const precedes = (folderKey: string | number, fileKey: string | number) => { + const cmp = compareKeys(folderKey, fileKey) + return direction === 'asc' ? cmp <= 0 : cmp >= 0 + } + + let nextFolder = 0 + for (const file of input.files) { + if (pinnedFileIds.has(input.fileId(file))) continue + const key = input.fileKey(file) + while (nextFolder < folders.length) { + const folder = folders[nextFolder] + if (folder.key === null || !precedes(folder.key, key)) break + entries.push(folder.item) + nextFolder++ + } + entries.push({ kind: 'file', file, pinned: false }) + } + if (!input.hasMore) { + for (; nextFolder < folders.length; nextFolder++) entries.push(folders[nextFolder].item) + } + return entries +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/page.tsx b/apps/sim/app/workspace/[workspaceId]/files/page.tsx index 11dc9fd450e..0b4e65098c8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/page.tsx @@ -1,11 +1,25 @@ import { Suspense } from 'react' import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { createSearchParamsCache } from 'nuqs/server' import { getSession } from '@/lib/auth' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { Files } from '@/app/workspace/[workspaceId]/files/files' import FilesLoading from '@/app/workspace/[workspaceId]/files/loading' import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' +import { + filesBrowserListParams, + filesFilterParsers, + filesFilterUrlKeys, + filesParsers, + filesSortParams, +} from '@/app/workspace/[workspaceId]/files/search-params' + +/** The browser's URL state, read with the same parsers (and URL keys) the client binds. */ +const filesViewCache = createSearchParamsCache( + { ...filesParsers, ...filesFilterParsers, ...filesSortParams.parsers }, + { urlKeys: filesFilterUrlKeys.urlKeys } +) export const metadata: Metadata = { title: 'Files', @@ -19,11 +33,23 @@ export const metadata: Metadata = { * table headers) so a suspend never shows a blank frame; the route-level * `loading.tsx` covers the navigation/chunk-load transition the same way. */ -export default async function FilesPage({ params }: { params: Promise<{ workspaceId: string }> }) { - const [{ workspaceId }, session] = await Promise.all([params, getSession()]) +export default async function FilesPage({ + params, + searchParams, +}: { + params: Promise<{ workspaceId: string }> + searchParams: Promise<Record<string, string | string[] | undefined>> +}) { + const [{ workspaceId }, session, view] = await Promise.all([ + params, + getSession(), + searchParams.then((raw) => filesViewCache.parse(raw)), + ]) const queryClient = getQueryClient() - await prefetchFilesBrowser(queryClient, workspaceId, session?.user?.id) + await prefetchFilesBrowser(queryClient, workspaceId, session?.user?.id, { + list: filesBrowserListParams(view), + }) return ( <HydrationBoundary state={dehydrate(queryClient)}> diff --git a/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts index 0b5668545cb..7f3361d933e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts @@ -1,38 +1,111 @@ import type { QueryClient } from '@tanstack/react-query' import { listWorkspaceFileFoldersContract } from '@/lib/api/contracts/workspace-file-folders' +import { + listWorkspaceFilesContract, + listWorkspaceFilesQuerySchema, + readWorkspaceFileContract, +} from '@/lib/api/contracts/workspace-files' +import { internalSessionAuth } from '@/lib/api/server/routes' import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/file-folder-manager' -import { listAllWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files' +import { internalFilePresenters } from '@/lib/workspace-files/api/internal-presenters' +import { + presentWorkspaceFileList, + toWorkspaceFileListInput, +} from '@/lib/workspace-files/api/list-page' +import { + listAllWorkspaceFiles, + listWorkspaceFileCollection, +} from '@/lib/workspace-files/application/list-workspace-files' +import { readWorkspaceFileContentRecord } from '@/lib/workspace-files/application/read-workspace-file-record' import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context' import { authorizeResourcePrefetch } from '@/app/workspace/[workspaceId]/lib/authorize-resource-prefetch' import { prefetchResourceListChrome } from '@/app/workspace/[workspaceId]/lib/prefetch-resource-list-chrome' -import { seedWorkspaceFiles } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' +import { + WORKSPACE_FILE_LIST_STALE_TIME, + WORKSPACE_FILE_RECORD_STALE_TIME, + type WorkspaceFileListParams, + workspaceFileListQuery, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' import { WORKSPACE_FILE_FOLDERS_STALE_TIME, workspaceFileFolderKeys, } from '@/hooks/queries/workspace-file-folders' +/** What the Files route renders: a list of files, or one open file. */ +export type FilesBrowserView = { list: WorkspaceFileListParams } | { fileId: string } + +/** + * The first page of the list the browser opens on, read through the same use case, input + * mapping, and presenter the list route runs — so the seeded page, cursor included, is the page + * a client fetch of that key would cache, and scrolling on resumes from it. + */ +function prefetchFileListPage( + queryClient: QueryClient, + workspaceId: string, + params: WorkspaceFileListParams +) { + return queryClient.prefetchInfiniteQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, params), + queryFn: async () => { + const principal = await internalSessionAuth.authenticate() + const input = toWorkspaceFileListInput( + workspaceId, + listWorkspaceFilesQuerySchema.parse(workspaceFileListQuery(params, null)) + ) + const result = await listWorkspaceFileCollection.execute({ principal, input }) + const response = listWorkspaceFilesContract.response.schema.parse( + presentWorkspaceFileList(result, input) + ) + return { files: response.files, nextCursor: response.nextCursor ?? null } + }, + initialPageParam: null as string | null, + staleTime: WORKSPACE_FILE_LIST_STALE_TIME, + }) +} + +/** The open file's record, through the record route's use case and presenter. */ +function prefetchFileRecord(queryClient: QueryClient, workspaceId: string, fileId: string) { + return queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.record(workspaceId, fileId), + queryFn: async () => { + const principal = await internalSessionAuth.authenticate() + const result = await readWorkspaceFileContentRecord.execute({ + principal, + input: { fileId, assertedWorkspaceId: workspaceId }, + }) + return readWorkspaceFileContract.response.schema.parse( + internalFilePresenters.successFile(result) + ).file + }, + staleTime: WORKSPACE_FILE_RECORD_STALE_TIME, + }) +} + /** * Prefetches what the Files browser needs on top of the workspace layout's own prefetch, so the * first frame is complete and correctly ordered: file folders, and (via * {@link prefetchResourceListChrome}) the pinned ids that drive row order plus the members behind - * the Owner column — under the same query keys their client hooks (`useWorkspaceFileFolders`) use - * (scope `active`), so the browser paints populated on first render. - * - * The file list is seeded here rather than in the layout so only the routes that render it pay for - * it. See {@link seedWorkspaceFiles} for why a large workspace seeds nothing at all. + * the Owner column — under the same query keys their client hooks use (scope `active`) — and + * either the first page of the list the URL opens on or the open file's record. The record is + * what decides whether `Files` renders the list or the file, so with it seeded the server and the + * client render read the same cache and pick the same branch. * - * Folders and the chrome reads all go through the data layer, shaped to their route contracts so a - * hydrated entry matches a client fetch. + * Folders and the chrome reads go through the data layer, shaped to their route contracts so a + * hydrated entry matches a client fetch; the files go through their routes' own use cases. A + * failed file read seeds nothing — only a settled query is dehydrated — and the client fetch + * reaches the route instead. * - * That read carries no authorization of its own, so the viewer is proved first. This reuses the - * layout's `cache`d host-context lookup rather than re-deriving the permission, so it costs no - * additional queries; a viewer without access caches nothing and the client fetch reaches the - * route for the real 403. + * The data-layer reads carry no authorization of their own, so the viewer is proved first. This + * reuses the layout's `cache`d host-context lookup rather than re-deriving the permission, so it + * costs no additional queries; a viewer without access caches nothing and the client fetch + * reaches the route for the real 403. */ export async function prefetchFilesBrowser( queryClient: QueryClient, workspaceId: string, - userId: string | undefined + userId: string | undefined, + view: FilesBrowserView ): Promise<void> { if (!userId) return const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId) @@ -46,8 +119,7 @@ export async function prefetchFilesBrowser( * Parsed through the route's own response schema rather than seeded raw. The * manager's record type and `workspaceFileFolderSchema` are two independent * declarations that happen to agree today; without this parse, adding a column - * to one silently seeds a shape a client fetch would have stripped — the exact - * divergence that put ISO strings under `workspaceFilesKeys.list`. + * to one silently seeds a shape a client fetch would have stripped. */ queryFn: async () => { const folders = await listWorkspaceFileFolders(workspaceId, { scope: 'active' }) @@ -56,6 +128,8 @@ export async function prefetchFilesBrowser( staleTime: WORKSPACE_FILE_FOLDERS_STALE_TIME, }), prefetchResourceListChrome(queryClient, workspaceId, 'file', userId), - seedWorkspaceFiles(queryClient, workspaceId), + 'fileId' in view + ? prefetchFileRecord(queryClient, workspaceId, view.fileId) + : prefetchFileListPage(queryClient, workspaceId, view.list), ]) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/search-params.test.ts b/apps/sim/app/workspace/[workspaceId]/files/search-params.test.ts new file mode 100644 index 00000000000..03f02803171 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/search-params.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' +import { filesBrowserListParams } from '@/app/workspace/[workspaceId]/files/search-params' + +const VIEW = { + folderId: 'folder-1', + search: '', + sort: 'updated', + dir: 'desc', + type: [], + size: [], + uploadedBy: [], +} as const + +describe('filesBrowserListParams', () => { + it('lists the open folder until a search is typed, then searches every folder', () => { + expect(filesBrowserListParams(VIEW)).toMatchObject({ folderId: 'folder-1' }) + expect(filesBrowserListParams(VIEW)).not.toHaveProperty('q') + + const searching = filesBrowserListParams({ ...VIEW, search: ' report ' }) + expect(searching).toMatchObject({ q: 'report' }) + expect(searching).not.toHaveProperty('folderId') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts index 0e601c15859..f9cf1f1e163 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts @@ -1,9 +1,32 @@ -import { createParser, parseAsArrayOf, parseAsString } from 'nuqs/server' +import { createParser, parseAsArrayOf, parseAsString, parseAsStringLiteral } from 'nuqs/server' import { createSortParams } from '@/lib/url-state' +import { + WORKSPACE_FILE_KINDS, + WORKSPACE_FILE_SIZE_BUCKETS, +} from '@/lib/workspace-files/list-filters' +import { + type WorkspaceFileListParams, + type WorkspaceFileListSortBy, + workspaceFileListParams, +} from '@/hooks/queries/utils/workspace-file-keys' import type { ResourceListPreferenceConfig } from '@/stores/resource-list-preferences' -/** Sortable list columns, matching the `Resource.Options` sort menu. */ -export const FILE_SORT_COLUMNS = ['name', 'size', 'type', 'created', 'owner', 'updated'] as const +/** + * Sortable list columns, matching the `Resource.Options` sort menu: the ones the server pages + * the list by. Type and owner are not among them — a sort the server cannot apply could only + * order the rows already loaded, which reads as the whole folder's order and is not. + */ +export const FILE_SORT_COLUMNS = ['name', 'size', 'created', 'updated'] as const + +type FileSortColumn = (typeof FILE_SORT_COLUMNS)[number] + +/** The server field each sort column pages by. */ +const FILE_SORT_FIELDS: Record<FileSortColumn, WorkspaceFileListSortBy> = { + name: 'name', + size: 'size', + created: 'uploadedAt', + updated: 'updatedAt', +} /** * Parser for the `new` flag. Preserves the prior `?new=1` wire format on @@ -67,8 +90,8 @@ export const filesUrlKeys = { */ export const filesFilterParsers = { search: parseAsString.withDefault(''), - type: parseAsArrayOf(parseAsString).withDefault([]), - size: parseAsArrayOf(parseAsString).withDefault([]), + type: parseAsArrayOf(parseAsStringLiteral(WORKSPACE_FILE_KINDS)).withDefault([]), + size: parseAsArrayOf(parseAsStringLiteral(WORKSPACE_FILE_SIZE_BUCKETS)).withDefault([]), uploadedBy: parseAsArrayOf(parseAsString).withDefault([]), } as const @@ -103,3 +126,29 @@ export const filesFilterUrlKeys = { clearOnDefault: true, urlKeys: filesFilterUrlKeyMap, } as const + +/** + * The paged list the Files browser shows for its URL state: the open folder's direct children, + * or — while a search is typed — matches from every folder, sorted and filtered on the server. + * Shared by the browser and its server prefetch, so the seeded first page is the page the + * browser reads. + */ +export function filesBrowserListParams(state: { + folderId: string | null + search: string + sort: FileSortColumn + dir: 'asc' | 'desc' + type: readonly (typeof WORKSPACE_FILE_KINDS)[number][] + size: readonly (typeof WORKSPACE_FILE_SIZE_BUCKETS)[number][] + uploadedBy: readonly string[] +}): WorkspaceFileListParams { + const searching = state.search.trim().length > 0 + return workspaceFileListParams({ + ...(searching ? { q: state.search } : { folderId: state.folderId }), + types: state.type, + sizes: state.size, + uploadedBy: state.uploadedBy, + sortBy: FILE_SORT_FIELDS[state.sort], + sortOrder: state.dir, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts index 4800fb13e9a..b9e4daadd45 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts @@ -2,7 +2,7 @@ import { truncate } from '@sim/utils/string' /** * The name a freshly-created markdown file is given in `handleCreateFile`: `untitled.md`, or - * `untitled (n).md` when that is taken. A file keeps this "unnamed" status until it is renamed — + * `untitled (n).md` when that is taken (the server numbers it). A file keeps this "unnamed" status until it is renamed — * while unnamed, typing a leading heading names the file (one direction only; the reverse * name→heading seed was removed as unsafe on the shared editor). See {@link isUntitledName}. */ @@ -42,7 +42,7 @@ export function deriveMarkdownFileName(headingText: string): string | null { /** * Makes `name` unique among `existingNames` by appending ` (n)` before the `.md` extension — the same - * scheme `handleCreateFile` uses for the default untitled name. + * scheme the server numbers a newly created file's name with. */ export function uniqueMarkdownName(name: string, existingNames: ReadonlySet<string>): string { if (!existingNames.has(name)) return name diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 366d0c2c192..18d4157d848 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -114,7 +114,7 @@ import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { useWorkflows } from '@/hooks/queries/workflows' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileLookup } from '@/hooks/queries/workspace-files' import { useWorkspaceUsageGate } from '@/hooks/queries/workspace-usage' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' @@ -1992,12 +1992,20 @@ function WorkspaceResourceDisplayContent({ }: WorkspaceResourceDisplayProps & { workspaceId: string; addressed?: boolean }) { const { data: workflows = [] } = useWorkflows(workspaceId) const { data: tables = [] } = useTablesList(workspaceId) - const { data: files = [] } = useWorkspaceFiles(workspaceId) const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId) + const isFile = data.type === 'file' + const { data: fileById } = useWorkspaceFileLookup(workspaceId, 'id', isFile ? data.id : undefined) + /** Only when the id names no file (or there is none): a path is the chip's fallback identity. */ + const { data: pathMatch } = useWorkspaceFileLookup( + workspaceId, + 'reference', + isFile ? data.path : undefined, + { enabled: !data.id || fileById === null } + ) + /** The reference lookup also answers by bare name; only an exact path match identifies the file. */ + const fileFromPath = findWorkspaceFileByPath(pathMatch ? [pathMatch] : undefined, data.path) const resource = useMemo<WorkspaceResourceRef>(() => { - const fileFromPath = - data.type === 'file' ? findWorkspaceFileByPath(files, data.path) : undefined const title = data.type === 'workflow' ? (workflows.find((workflow) => workflow.id === data.id)?.name ?? @@ -2008,7 +2016,7 @@ function WorkspaceResourceDisplayContent({ : data.type === 'dashboard' ? (data.title ?? fallbackWorkspaceResourceTitle(data.type)) : data.type === 'file' - ? (files.find((file) => file.id === data.id)?.name ?? + ? (fileById?.name ?? fileFromPath?.name ?? data.title ?? fallbackWorkspaceResourceTitle(data.type)) @@ -2028,7 +2036,8 @@ function WorkspaceResourceDisplayContent({ data.path, data.title, data.type, - files, + fileById, + fileFromPath, knowledgeBases, tables, workflows, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.test.ts index 7afe2494903..f65a869a188 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { toCopyableMarkdown } from '@/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown' import { parseChipLinks } from '@/app/workspace/[workspaceId]/home/components/user-input/components/chip-clipboard-codec' +import { resolveWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' const WORKSPACE_FILES: WorkspaceFileRecord[] = [ { @@ -43,7 +44,9 @@ describe('toCopyableMarkdown', () => { })}</workspace_resource>.`, ].join('') - const markdown = toCopyableMarkdown(message, WORKSPACE_FILES) + const markdown = toCopyableMarkdown(message, (ref) => + resolveWorkspaceResourceRef(ref, WORKSPACE_FILES) + ) expect(markdown).toBe( 'Read [The Bell at Low Tide.md](sim:file/file_bell) and [Checked_\\[rare\\]\\\\portal](sim:table/tbl_f26af6dae98d4222b014b250494d00fb).' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.ts index 1f4742101b1..24c8adf997b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/copyable-markdown.ts @@ -1,5 +1,5 @@ import type { ClipboardContent } from '@sim/emcn' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import type { MothershipResource, WorkspaceResourceRef } from '@/lib/mothership/resources/types' import { sanitizeChatDisplayContent } from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize' import { type ContentSegment, @@ -14,6 +14,19 @@ interface CopyableMarkdownResult { hasUnresolvedFile: boolean } +/** + * Resolves a chip to the resource it addresses: the resource, `null` when nothing answers to it, + * or `undefined` when that is not known yet. Either way an unresolved chip is copied as plain text. + */ +type ResolveResourceRef = (ref: WorkspaceResourceRef) => MothershipResource | null | undefined + +/** Non-file chips carry their own id; resolving a file chip needs the caller's reads. */ +const resolveWithoutFiles: ResolveResourceRef = (ref) => resolveWorkspaceResourceRef(ref, []) + +function toResourceRef(data: WorkspaceResourceTagData): WorkspaceResourceRef { + return { ...data, title: data.title ?? '' } +} + function workspaceResourceLabel(data: WorkspaceResourceTagData): string { if (data.title) return data.title return data.type === 'file' ? (data.path ?? data.id ?? '') : (data.id ?? '') @@ -38,10 +51,9 @@ function appendInlineReferenceMarkdown( function portableWorkspaceResourceMarkdown( data: WorkspaceResourceTagData, - workspaceFiles: readonly WorkspaceFileRecord[] + resource: MothershipResource | null | undefined ): CopyableMarkdownResult { const label = workspaceResourceLabel(data) - const resource = resolveWorkspaceResourceRef({ ...data, title: data.title ?? '' }, workspaceFiles) return { markdown: resource ? serializePortableChipLink(data.type, resource.id, resource.title || label, data.workspaceId) @@ -50,19 +62,21 @@ function portableWorkspaceResourceMarkdown( } } +function parseCopyableSegments(raw: string): ContentSegment[] { + return parseSpecialTags(sanitizeChatDisplayContent(raw), false).segments +} + function serializeCopyableMarkdown( - raw: string, - workspaceFiles: readonly WorkspaceFileRecord[] = [] + segments: ContentSegment[], + resolve: (data: WorkspaceResourceTagData) => MothershipResource | null | undefined ): CopyableMarkdownResult { - const displayContent = sanitizeChatDisplayContent(raw) - const { segments } = parseSpecialTags(displayContent, false) let hasUnresolvedFile = false const markdown = segments .reduce((markdown, segment, index) => { if (segment.type === 'text') return markdown + segment.content if (segment.type === 'workspace_resource') { - const portable = portableWorkspaceResourceMarkdown(segment.data, workspaceFiles) + const portable = portableWorkspaceResourceMarkdown(segment.data, resolve(segment.data)) hasUnresolvedFile ||= portable.hasUnresolvedFile return appendInlineReferenceMarkdown(markdown, portable.markdown, segments[index + 1]) } @@ -75,24 +89,41 @@ function serializeCopyableMarkdown( export function toCopyableMarkdown( raw: string, - workspaceFiles: readonly WorkspaceFileRecord[] = [] + resolve: ResolveResourceRef = resolveWithoutFiles ): string { - return serializeCopyableMarkdown(raw, workspaceFiles).markdown + return serializeCopyableMarkdown(parseCopyableSegments(raw), (data) => + resolve(toResourceRef(data)) + ).markdown } +/** + * The message as portable Markdown, with chips resolved from what is cached. When a file chip + * is not, the copy resolves its file chips fresh (`resolveFresh`) on demand, and keeps the + * plain-text copy if that fails. + */ export function prepareCopyableMarkdown( raw: string, - workspaceFiles: readonly WorkspaceFileRecord[], - refreshWorkspaceFiles: () => Promise<readonly WorkspaceFileRecord[]> + resolveCached: ResolveResourceRef, + resolveFresh: (ref: WorkspaceResourceRef) => Promise<MothershipResource | null> ): ClipboardContent { - const initial = serializeCopyableMarkdown(raw, workspaceFiles) + const segments = parseCopyableSegments(raw) + const initial = serializeCopyableMarkdown(segments, (data) => resolveCached(toResourceRef(data))) if (!initial.hasUnresolvedFile) return initial.markdown return { fallback: initial.markdown, prepare: async () => { try { - return toCopyableMarkdown(raw, await refreshWorkspaceFiles()) + const fresh = new Map<WorkspaceResourceTagData, MothershipResource | null>() + await Promise.all( + segments.map(async (segment) => { + if (segment.type !== 'workspace_resource' || segment.data.type !== 'file') return + fresh.set(segment.data, await resolveFresh(toResourceRef(segment.data))) + }) + ) + return serializeCopyableMarkdown(segments, (data) => + data.type === 'file' ? fresh.get(data) : resolveWithoutFiles(toResourceRef(data)) + ).markdown } catch { return initial.markdown } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index 6498303e747..35b20311bfb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -25,7 +25,6 @@ import { defaultRangeExtractor, type Range, useVirtualizer } from '@tanstack/rea import { ComputerUseActivity } from '@/components/computer-use/activity' import { SMOOTH_CHASE_RATE } from '@/lib/core/utils/smooth-bottom-chase' import type { TurnOrigin } from '@/lib/mothership/generated/protocol' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { inter } from '@/app/_styles/fonts/inter/inter' import { FindBar } from '@/app/workspace/[workspaceId]/components/find-bar' import { MessageActions } from '@/app/workspace/[workspaceId]/components/message-actions' @@ -58,6 +57,10 @@ import { type UserInputHandle, } from '@/app/workspace/[workspaceId]/home/components/user-input' import { UserMessageContent } from '@/app/workspace/[workspaceId]/home/components/user-message-content' +import { + fetchWorkspaceResourceRef, + readCachedWorkspaceResourceRef, +} from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' import type { ChatMessage, ChatMessageAttachment, @@ -68,7 +71,6 @@ import type { WorkspaceResourceRef, } from '@/app/workspace/[workspaceId]/home/types' import { useOptionalWorkspacePermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' -import { getWorkspaceFilesQueryOptions, workspaceFilesKeys } from '@/hooks/queries/workspace-files' import { useAutoScroll } from '@/hooks/use-auto-scroll' import type { ChatContext } from '@/stores/panel' import { MothershipChatSkeleton } from './components/mothership-chat-skeleton' @@ -176,7 +178,6 @@ const LAYOUT_STYLES = { } as const const EMPTY_BLOCKS: ContentBlock[] = [] -const EMPTY_WORKSPACE_FILES: readonly WorkspaceFileRecord[] = [] interface UserMessageRowProps { content: string @@ -426,14 +427,8 @@ export function MothershipChat({ workspaceId ? prepareCopyableMarkdown( content, - queryClient.getQueryData<readonly WorkspaceFileRecord[]>( - workspaceFilesKeys.list(workspaceId) - ) ?? EMPTY_WORKSPACE_FILES, - () => - queryClient.fetchQuery({ - ...getWorkspaceFilesQueryOptions(workspaceId), - staleTime: 0, - }) + (ref) => readCachedWorkspaceResourceRef(queryClient, workspaceId, ref), + (ref) => fetchWorkspaceResourceRef(queryClient, workspaceId, ref, { fresh: true }) ) : toCopyableMarkdown(content), [queryClient, workspaceId] diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx index 1773f242283..e9f74322f38 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx @@ -306,6 +306,9 @@ export function ResourceMenuSections({ interface ResourceMenuSearchProps { groups: AvailableItemsByType[] isHydrating?: boolean + /** The query, owned by the caller so its inventory can search files with it. */ + search: string + onSearchChange: (search: string) => void children: ReactNode onSelect: (resource: MothershipResource) => void } @@ -313,10 +316,11 @@ interface ResourceMenuSearchProps { function ResourceMenuSearch({ groups: available, isHydrating, + search, + onSearchChange: setSearch, children, onSelect: select, }: ResourceMenuSearchProps) { - const [search, setSearch] = useState('') const [activeIndex, setActiveIndex] = useState(0) const filtered = useMemo(() => { const q = search.toLowerCase().trim() @@ -417,9 +421,11 @@ function WorkspaceResourceMenuContent({ onSelect, subContentClassName, }: WorkspaceResourceMenuContentProps) { + const [search, setSearch] = useState('') const { groups, structureFolders, isHydrating } = useAvailableResources(workspaceId, { enabled, excludeTypes, + fileSearch: search, }) const sections = useResourceTreeSections({ groups, structureFolders, selectFolders }) const select = (resource: MothershipResource) => @@ -441,7 +447,13 @@ function WorkspaceResourceMenuContent({ </> ) return searchable ? ( - <ResourceMenuSearch groups={groups} isHydrating={isHydrating} onSelect={select}> + <ResourceMenuSearch + groups={groups} + isHydrating={isHydrating} + search={search} + onSearchChange={setSearch} + onSelect={select} + > {menu} </ResourceMenuSearch> ) : ( @@ -528,6 +540,7 @@ export function AddResourceDropdown({ [excludeTypes] ) const [inventories, setInventories] = useState<Record<string, AvailableResources>>({}) + const [organizationSearch, setOrganizationSearch] = useState('') const receiveInventory = useCallback((workspaceId: string, inventory: AvailableResources) => { setInventories((current) => current[workspaceId] === inventory ? current : { ...current, [workspaceId]: inventory } @@ -545,6 +558,7 @@ export function AddResourceDropdown({ const hasNativeResourceSurface = isBrowserAgentAvailable() || isTerminalAvailable() const closeMenu = useCallback(() => { setOpen(false) + setOrganizationSearch('') return onClose?.() ?? Promise.resolve() }, [onClose]) @@ -585,6 +599,7 @@ export function AddResourceDropdown({ <OrganizationResourceInventory key={workspace.id} workspaceId={workspace.id} + fileSearch={organizationSearch} onChange={receiveInventory} /> ))} @@ -623,6 +638,8 @@ export function AddResourceDropdown({ ...nativeGroups, ]} isHydrating={organizationInventory.isHydrating} + search={organizationSearch} + onSearchChange={setOrganizationSearch} onSelect={select} > {workspaces.map((workspace) => ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts index 404f50cf8c5..c97e3944dc9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts @@ -2,6 +2,7 @@ import { useMemo, useSyncExternalStore } from 'react' import { isBrowserAgentAvailable } from '@/lib/browser-agent/transport' import { subscribeDesktopPreferences } from '@/lib/desktop' import { isTerminalAvailable } from '@/lib/terminal/transport' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { AvailableItem } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' import { byResourceMenuOrder, @@ -17,10 +18,12 @@ import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useLogsList } from '@/hooks/queries/logs' import { useMothershipChats } from '@/hooks/queries/mothership-chats' import { useTablesList } from '@/hooks/queries/tables' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { useWorkflowTests } from '@/hooks/queries/workflow-tests' import { useWorkflows } from '@/hooks/queries/workflows' import { useWorkspaceFileFolders } from '@/hooks/queries/workspace-file-folders' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' +import { useDebounce } from '@/hooks/use-debounce' /** * Placeholder id for the Browser launcher row. It never names a resource: the @@ -72,7 +75,7 @@ interface UseAvailableResourcesOptions { * * Note this only defers the lists nothing else on the surface already needs — * the mothership tab bar independently resolves tab names from the workflow, - * table, file, knowledge-base, and folder lists, so those stay warm there. + * table, knowledge-base, and folder lists, so those stay warm there. */ enabled?: boolean /** @@ -80,6 +83,13 @@ interface UseAvailableResourcesOptions { * (a module constant) — it keys the group memo. */ excludeTypes?: readonly MothershipResourceType[] + /** + * The text the menu is filtering by, as typed. Files are not listed whole — a workspace can + * hold far more than a menu can show — so without a query the file family holds the most + * recently updated files, and with one, the files whose names match it anywhere in the + * workspace (searched once the typing settles). Other families are filtered by the caller. + */ + fileSearch?: string } /** Stable identity for the disabled result, so downstream memos never bust. */ @@ -87,6 +97,11 @@ const NO_RESOURCE_GROUPS: AvailableItemsByType[] = [] const LOG_DROPDOWN_LIMIT = 50 +/** The files a menu offers before anything is typed: the most recently updated anywhere. */ +const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 100 }) + +const FILE_SEARCH_LIMIT = 50 + const LOG_DROPDOWN_FILTERS = { timeRange: 'All time' as const, level: 'all', @@ -132,9 +147,21 @@ export function useAvailableResources( const { data: tests, isPending: testsPending } = useWorkflowTests(workspaceId, { enabled: enabled && testsEnabled && !excludeTypes?.includes('test'), }) - const { data: files, isPending: filesPending } = useWorkspaceFiles(workspaceId, 'active', { + const fileSearch = options?.fileSearch?.trim() ?? '' + const settledFileSearch = useDebounce(fileSearch, SEARCH_DEBOUNCE_MS) + const recentFiles = useWorkspaceFileList(workspaceId, RECENT_FILES, { enabled: enabled && Boolean(workspaceId), }) + const matchingFiles = useWorkspaceFileSearch( + workspaceId, + { q: settledFileSearch, limit: FILE_SEARCH_LIMIT }, + { enabled: enabled && Boolean(workspaceId) } + ) + const files = settledFileSearch ? matchingFiles.data : recentFiles.data?.pages[0]?.files + /** A typed query is still resolving until its search settles: an empty menu is not "no match" yet. */ + const filesPending = + recentFiles.isPending || + (fileSearch.length > 0 && (fileSearch !== settledFileSearch || matchingFiles.isFetching)) const { data: knowledgeBases, isPending: knowledgeBasesPending } = useKnowledgeBasesQuery( workspaceId, { enabled: enabled && Boolean(workspaceId) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/organization-resource-inventory.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/organization-resource-inventory.tsx index 9ed7bc98ba1..1890891edfb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/organization-resource-inventory.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/organization-resource-inventory.tsx @@ -9,15 +9,19 @@ import type { MothershipResourceType } from '@/app/workspace/[workspaceId]/home/ export function OrganizationResourceInventory({ workspaceId, excludeTypes, + fileSearch, onChange, }: { workspaceId: string excludeTypes?: readonly MothershipResourceType[] + /** The picker's query, which the workspace's files are searched by. */ + fileSearch?: string onChange: (workspaceId: string, inventory: AvailableResources) => void }) { const inventory = useAvailableResources(workspaceId, { includeFolderMentions: true, excludeTypes, + fileSearch, }) useEffect(() => { onChange(workspaceId, inventory) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index f6bb90f4623..a8656ac9e76 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -33,7 +33,6 @@ import { getWorkspaceUsageLimitAction } from '@/lib/billing/workspace-permission import { prefersInPlaceNavigation } from '@/lib/desktop' import type { FilePreviewSession } from '@/lib/mothership/request/session' import { stopRunToolForExecution } from '@/lib/mothership/tools/client/run-tool-execution' -import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import { type FileDownloadSource, triggerFileDownload } from '@/lib/uploads/client/download' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file' @@ -71,11 +70,12 @@ import { useLogByExecutionId, useLogDetail } from '@/hooks/queries/logs' import { useProjectFile } from '@/hooks/queries/projects' import { exportTable } from '@/hooks/queries/tables' import { fetchWorkflowEnvelope } from '@/hooks/queries/utils/fetch-workflow-envelope' +import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { workflowKeys } from '@/hooks/queries/utils/workflow-keys' import { mapWorkflow } from '@/hooks/queries/utils/workflow-list-query' import { useWorkflowTest } from '@/hooks/queries/workflow-tests' import { useWorkflows, WORKFLOW_STATE_STALE_TIME } from '@/hooks/queries/workflows' -import { useAddressedWorkspaceFileRecord, useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileLookup, useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' import { createProjectFileContentSource, createWorkspaceFileContentSource, @@ -728,6 +728,26 @@ function EmbeddedTableActions({ workspaceId, tableId }: EmbeddedTableActionsProp const fileLogger = createLogger('EmbeddedFileActions') +/** + * The file an embedded resource tab shows: its record by id — including an addressed chat upload — + * or, when that id no longer names a live file, the file now at the tab's canonical path. + */ +function useEmbeddedWorkspaceFile(workspaceId: string, fileId: string, filePath?: string) { + const record = useWorkspaceFileRecord(workspaceId, fileId) + const pathLookup = useWorkspaceFileLookup(workspaceId, 'reference', filePath, { + enabled: record.isError, + }) + /** The reference lookup also answers by bare name; only an exact path match is this file. */ + const fileAtPath = findWorkspaceFileByPath( + pathLookup.data ? [pathLookup.data] : undefined, + filePath + ) + return { + file: record.data ?? fileAtPath, + isResolving: record.isFetching || pathLookup.isFetching, + } +} + interface EmbeddedFileActionsProps { workspaceId: string fileId: string @@ -744,17 +764,7 @@ function EmbeddedFileActions({ const router = useRouter() const { canEdit } = useUserPermissionsContext() const [isShareOpen, setIsShareOpen] = useState(false) - const { data: files = [], isLoading: listLoading } = useWorkspaceFiles(workspaceId) - const listedFile = files.find( - (file) => - file.id === fileId || - (filePath && - canonicalWorkspaceFilePath({ folderPath: file.folderPath, name: file.name }) === filePath) - ) - const detail = useAddressedWorkspaceFileRecord(workspaceId, fileId, { - enabled: !listedFile && !listLoading, - }) - const file = listedFile ?? detail.data + const { file } = useEmbeddedWorkspaceFile(workspaceId, fileId, filePath) const isUpload = file?.vfsNamespace === 'uploads' const handleDownload = async () => { @@ -938,17 +948,7 @@ function EmbeddedFile({ previewContextKey, }: EmbeddedFileProps) { const { canEdit } = useUserPermissionsContext() - const { data: files = [], isLoading: listLoading } = useWorkspaceFiles(workspaceId) - const listedFile = files.find( - (file) => - file.id === fileId || - (filePath && - canonicalWorkspaceFilePath({ folderPath: file.folderPath, name: file.name }) === filePath) - ) - const detail = useAddressedWorkspaceFileRecord(workspaceId, fileId, { - enabled: !listedFile && !listLoading, - }) - const file = listedFile ?? detail.data + const { file, isResolving } = useEmbeddedWorkspaceFile(workspaceId, fileId, filePath) const isUpload = file?.vfsNamespace === 'uploads' const contentSource = useMemo( () => @@ -958,7 +958,7 @@ function EmbeddedFile({ [isUpload, workspaceId, file?.storageContext] ) - if (!file && (listLoading || detail.isFetching)) return LOADING_SKELETON + if (!file && isResolving) return LOADING_SKELETON if (!file) return FILE_NOT_FOUND diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts index b8db2ae2c5c..f6542d6cd01 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-registry/resource-invalidation.ts @@ -9,9 +9,10 @@ import { invalidateWorkflowLists } from '@/hooks/queries/utils/invalidate-workfl import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workflowKeys } from '@/hooks/queries/utils/workflow-keys' +import { invalidateWorkspaceFileReads } from '@/hooks/queries/utils/workspace-file-cache' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' import { workflowTestKeys } from '@/hooks/queries/workflow-tests' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' function invalidate(client: QueryClient, key: QueryKey): void { void client.invalidateQueries({ queryKey: key }) @@ -34,8 +35,7 @@ const RESOURCE_INVALIDATORS: Record< invalidate(qc, workflowTestKeys.details()) }, file: (qc, wId, id) => { - invalidate(qc, workspaceFilesKeys.lists()) - invalidate(qc, id ? workspaceFilesKeys.record(wId, id) : workspaceFilesKeys.records()) + void invalidateWorkspaceFileReads(qc, wId) invalidate(qc, id ? workspaceFilesKeys.contentFile(wId, id) : workspaceFilesKeys.contents()) invalidate(qc, workspaceFilesKeys.storageInfo()) }, @@ -54,7 +54,7 @@ const RESOURCE_INVALIDATORS: Record< }, filefolder: (qc, wId) => { invalidate(qc, workspaceFileFolderKeys.workspaceLists(wId)) - invalidate(qc, workspaceFilesKeys.workspaceLists(wId)) + void invalidateWorkspaceFileReads(qc, wId) invalidate(qc, workspaceFilesKeys.storageInfo()) }, task: (qc, wId, id) => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-tabs/resource-tabs.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-tabs/resource-tabs.tsx index f892938a589..4d00a94e552 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-tabs/resource-tabs.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-tabs/resource-tabs.tsx @@ -54,7 +54,7 @@ import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' import { getWorkflowListQueryOptions } from '@/hooks/queries/utils/workflow-list-query' import { useWorkflows } from '@/hooks/queries/workflows' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileLookups } from '@/hooks/queries/workspace-files' import { useSettledTerminalCommands } from '@/hooks/use-settled-terminal-commands' import { useBrowserSessionStore } from '@/stores/browser-session/store' import { useCopilotTerminalStore } from '@/stores/copilot-terminal/store' @@ -155,7 +155,7 @@ const NO_RESOURCE_NAMES = new Map<string, string>() * Builds a `type:id` -> current name lookup from live query data so resource * tabs always reflect the latest name even after a rename. Skipped entirely * when there are no tabs to label — a chat with no open resources must not - * fetch five workspace-wide lists. + * fetch four workspace-wide lists. Files resolve by id, only the open tabs'. */ function useResourceNameLookup( workspaceId: string | undefined, @@ -183,7 +183,16 @@ function useResourceNameLookup( const { data: tables } = useTablesList(workspaceId ?? '', 'active', { enabled: enabled && Boolean(workspaceId), }) - const { data: files } = useWorkspaceFiles(workspaceId ?? '', 'active', { + const fileIds = useMemo( + () => + resources.flatMap((resource) => + resource.type === 'file' && (!resource.workspaceId || resource.workspaceId === workspaceId) + ? [resource.id] + : [] + ), + [resources, workspaceId] + ) + const { byValue: filesById } = useWorkspaceFileLookups(workspaceId ?? '', 'id', fileIds, { enabled: enabled && Boolean(workspaceId), }) const { data: knowledgeBases } = useKnowledgeBasesQuery(workspaceId ?? '', { @@ -203,11 +212,11 @@ function useResourceNameLookup( map.set(`workflow:${workflow.id}`, workflow.name) } for (const t of tables ?? []) map.set(`table:${t.id}`, t.name) - for (const file of files ?? []) map.set(`file:${file.id}`, file.name) + for (const file of filesById.values()) map.set(`file:${file.id}`, file.name) for (const kb of knowledgeBases ?? []) map.set(`knowledgebase:${kb.id}`, kb.name) for (const folder of folders ?? []) map.set(`folder:${folder.id}`, folder.name) return map - }, [enabled, workflows, tables, files, knowledgeBases, folders, ownedWorkflows, owners]) + }, [enabled, workflows, tables, filesById, knowledgeBases, folders, ownedWorkflows, owners]) } interface ResourceTabsProps { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx index e36605cac7c..75e94da3865 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx @@ -35,7 +35,7 @@ import type { } from '@/app/workspace/[workspaceId]/home/types' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { useWorkspacePermissionsQuery } from '@/hooks/queries/workspace' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' import { useUserPermissions } from '@/hooks/use-user-permissions' import { EmbeddedProjectFile, ResourceActions, ResourceContent, ResourceTabs } from './components' @@ -196,14 +196,12 @@ export const MothershipView = memo( // A large CSV renders read-only (streamed) with no editor, so it must not offer the // edit/split/preview toggle. Its size lives on the file record, not the resource tab. - const { data: files, isLoading: filesLoading } = useWorkspaceFiles( + const { data: activeFileRecord, isLoading: activeFileLoading } = useWorkspaceFileRecord( activeWorkspaceId ?? '', - 'active', - { - enabled: Boolean(activeWorkspaceId) && active?.type === 'file', - } + active?.type === 'file' ? active.id : '', + { enabled: active?.type === 'file' } ) - const activeFile = active?.type === 'file' ? files?.find((f) => f.id === active.id) : undefined + const activeFile = active?.type === 'file' ? activeFileRecord : undefined const isActiveCsv = active?.type === 'file' && getFileExtension(active.title) === 'csv' const isActiveFilePreviewable = @@ -219,8 +217,8 @@ export const MothershipView = memo( !isMarkdownFile(activeFile ?? { type: '', name: active.title }) && // Only a CSV's previewability depends on its size (large = read-only, no editor). Wait for // the record before deciding so the toggle doesn't flash on for a large CSV — but don't gate - // other rich types (html, svg, …) on the file list loading. - !(isActiveCsv && filesLoading) && + // other rich types (html, svg, …) on the record loading. + !(isActiveCsv && activeFileLoading) && !(activeFile && isCsvStreamOnly(activeFile)) && // A Sim page is locked to its rendered view (the pdf model — the raw // source is not a mode this surface offers), so no toggle either. diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx index 56e54e485b4..dea848b2668 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/plus-menu-dropdown.tsx @@ -153,10 +153,12 @@ export const PlusMenuDropdown = React.memo( }) const inventoryEnabled = open || !!warm + const query = isMention ? (mentionQuery ?? '') : search const workspaceInventory = useAvailableResources(organizationId ? '' : workspaceId, { enabled: inventoryEnabled, includeFolderMentions: true, excludeTypes: COMPOSER_EXCLUDED_TYPES, + fileSearch: query, }) const { data: allWorkspaces, isPending: workspacesPending } = useOrderedWorkspacesQuery( Boolean(organizationId) && inventoryEnabled @@ -232,7 +234,6 @@ export const PlusMenuDropdown = React.memo( selectFolders: true, }) - const query = isMention ? (mentionQuery ?? '') : search const filteredItems = useMemo((): MentionCandidate[] | null => { const q = query.toLowerCase().trim() if (!isMention && !q) return null @@ -415,6 +416,7 @@ export const PlusMenuDropdown = React.memo( key={workspace.id} workspaceId={workspace.id} excludeTypes={COMPOSER_EXCLUDED_TYPES} + fileSearch={query} onChange={receiveInventory} /> ))} diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index 7ecc6146c1d..59c85bccd04 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -31,10 +31,12 @@ import { useResourcePanelController, } from '@/app/workspace/[workspaceId]/home/hooks/use-resource-panel' import { useRestoredChatEntry } from '@/app/workspace/[workspaceId]/home/hooks/use-restored-chat-entry' -import { resolveWorkspaceResourceRef } from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' +import { + fetchWorkspaceResourceRef, + readCachedWorkspaceResourceRef, +} from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' import { PermissionAccessBoundary } from '@/ee/access-requests/components/permission-access-boundary' import { useMarkMothershipChatRead } from '@/hooks/queries/mothership-chats' -import { getWorkspaceFilesQueryOptions, useWorkspaceFiles } from '@/hooks/queries/workspace-files' import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' import type { ChatContext } from '@/stores/panel' import { @@ -76,7 +78,6 @@ function HomeContent({ chatId, userName, userId }: HomeProps) { const queryClient = useQueryClient() const controller = useResourcePanelController() const firstName = userName?.split(' ')[0] ?? '' - const { data: workspaceFiles = [] } = useWorkspaceFiles(workspaceId) const posthog = usePostHog() const posthogRef = useRef(posthog) posthogRef.current = posthog @@ -360,35 +361,34 @@ function HomeContent({ chatId, userName, userId }: HomeProps) { /** * Opens the resource a message chip points at, resolving it first. A chip may - * carry only a filename — the agent names a file before the client's file - * list knows it exists — so one forced refetch closes that window. What still + * carry only a filename — the agent names a file before the client has read + * it — so a cached miss is re-read once before giving up. What still * resolves to nothing opens nothing, rather than a tab that cannot be * viewed or removed. */ async function handleWorkspaceResourceSelect(ref: WorkspaceResourceRef) { - const immediate = resolveWorkspaceResourceRef(ref, workspaceFiles) + const immediate = readCachedWorkspaceResourceRef(queryClient, workspaceId, ref) if (immediate) { openWorkspaceResource(immediate) return } if (ref.type !== 'file') return - // `staleTime: 0` forces the fetch this branch exists for — the cached list - // is what already failed to resolve. `fetchQuery` rejects on error and this - // handler is invoked as a void callback, so failure becomes null rather - // than an unhandled rejection — and stays distinct from an empty list, so - // "we could not look" is never reported as "it is not there". - const files = await queryClient - .fetchQuery({ ...getWorkspaceFilesQueryOptions(workspaceId), staleTime: 0 }) - .catch(() => null) - const resolved = files && resolveWorkspaceResourceRef(ref, files) + // `fresh` forces the reads this branch exists for — what is cached already + // failed to resolve. A failed read rejects, and this handler is invoked as a + // void callback, so failure becomes `undefined` rather than an unhandled + // rejection — distinct from `null`, so "we could not look" is never + // reported as "it is not there". + const resolved = await fetchWorkspaceResourceRef(queryClient, workspaceId, ref, { + fresh: true, + }).catch(() => undefined) if (resolved) { openWorkspaceResource(resolved) return } // The chip looks clickable, so refusing silently reads as a broken button. toast.error( - files + resolved === null ? `Couldn't find "${ref.title}" in this workspace` : `Couldn't open "${ref.title}" — check your connection and try again` ) @@ -396,7 +396,7 @@ function HomeContent({ chatId, userName, userId }: HomeProps) { type: ref.type, title: ref.title, hasPath: Boolean(ref.path), - reachedWorkspace: files !== null, + reachedWorkspace: resolved === null, }) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts index 0269dbdfa84..f9465ffd854 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-controller.ts @@ -26,7 +26,8 @@ import { } from '@/app/workspace/[workspaceId]/home/hooks/preview/use-file-preview-sessions' import { resolveFileResourceSelectionId } from '@/app/workspace/[workspaceId]/home/resource-view-policy' import type { MothershipResource } from '@/app/workspace/[workspaceId]/home/types' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' +import { findCachedWorkspaceFile } from '@/hooks/queries/utils/workspace-file-cache' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' interface FilePreviewControllerDeps { workspaceId?: string @@ -119,10 +120,11 @@ export function useFilePreviewController({ previewText ) - const activeFiles = queryClient.getQueryData<Array<{ id: string; key: string }>>( - workspaceFilesKeys.list(workspaceId, 'active') - ) - const fileKey = activeFiles?.find((file) => file.id === fileId)?.key + const fileKey = findCachedWorkspaceFile( + queryClient, + workspaceId, + (file) => file.id === fileId + )?.file.key if (fileKey) { queryClient.setQueryData( [...workspaceFilesKeys.content(workspaceId, fileId, 'text'), fileKey], diff --git a/apps/sim/app/workspace/[workspaceId]/home/page.tsx b/apps/sim/app/workspace/[workspaceId]/home/page.tsx index e768e6de37f..6a8faaf61ff 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/page.tsx @@ -1,11 +1,8 @@ import { Suspense } from 'react' -import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' import { redirect } from 'next/navigation' import { getSession } from '@/lib/auth' import { isChatEnabled } from '@/lib/core/config/env-flags' -import { getQueryClient } from '@/app/_shell/providers/get-query-client' -import { prefetchHomeSurface } from '@/app/workspace/[workspaceId]/home/prefetch' import { Home } from './home' import { HomeFallback } from './home-fallback' @@ -17,21 +14,16 @@ export default async function HomePage({ params }: { params: Promise<{ workspace const { workspaceId } = await params // The layout redirects too, but pages and layouts resolve concurrently — without - // this the prefetch below still fires on its way out. + // this the page still renders on its way out. if (!isChatEnabled) { redirect(`/workspace/${workspaceId}`) } const session = await getSession() - const userId = session?.user?.id - const queryClient = getQueryClient() - await prefetchHomeSurface(queryClient, workspaceId, userId) return ( - <HydrationBoundary state={dehydrate(queryClient)}> - <Suspense fallback={<HomeFallback />}> - <Home userName={session?.user?.name} userId={userId} /> - </Suspense> - </HydrationBoundary> + <Suspense fallback={<HomeFallback />}> + <Home userName={session?.user?.name} userId={session?.user?.id} /> + </Suspense> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts deleted file mode 100644 index b47b000b0f1..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { QueryClient } from '@tanstack/react-query' -import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context' -import { seedWorkspaceFiles } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' - -/** - * Prefetches what the Home surface needs on top of the workspace layout's own prefetch. - * - * Home reads the workspace file list on mount (resource tabs, mentions, the resource picker), so - * the list is seeded by the routes that render Home rather than by the layout: seeding it in the - * layout would pay for it on every workspace route, including the ones that never read it. - * - * The seed carries no authorization of its own, so the viewer is proved first. This reuses the - * layout's `cache`d host-context lookup rather than re-deriving the permission, so it costs no - * additional queries; a viewer without access caches nothing and the client fetch reaches the - * route for the real 403. - */ -export async function prefetchHomeSurface( - queryClient: QueryClient, - workspaceId: string, - userId: string | undefined -): Promise<void> { - if (!userId) return - const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId) - if (!hostContext) return - - await seedWorkspaceFiles(queryClient, workspaceId) -} diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts index b4ca339b20c..fc0b5fb7071 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts @@ -1,7 +1,21 @@ -import { describe, expect, it } from 'vitest' +import { + apiClientRequestMock, + apiClientRequestMockFns, +} from '@sim/testing/mocks/api-client-request.mock' +import { QueryClient } from '@tanstack/react-query' +import { describe, expect, it, vi } from 'vitest' +import { + lookupWorkspaceFilesContract, + searchWorkspaceFilesContract, +} from '@/lib/api/contracts/workspace-files' import type { WorkspaceResourceRef } from '@/lib/mothership/resources/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { resolveWorkspaceResourceRef } from './resolve-resource-ref' +import { + fetchWorkspaceResourceRef, + resolveWorkspaceResourceRef, +} from '@/app/workspace/[workspaceId]/home/resolve-resource-ref' + +vi.mock('@/lib/api/client/request', () => apiClientRequestMock) function file(overrides: Partial<WorkspaceFileRecord> & { id: string; name: string }) { return { @@ -53,3 +67,61 @@ describe('resolveWorkspaceResourceRef', () => { } }) }) + +describe('fetchWorkspaceResourceRef', () => { + /** + * A workspace answering lookups and name searches over `files` the way the server does: a + * reference resolves by id, then canonical path, then the first file with that name. + */ + function serveWorkspace(files: WorkspaceFileRecord[]) { + apiClientRequestMockFns.mockRequestJson.mockImplementation(async (contract, input) => { + if (contract === searchWorkspaceFilesContract) { + const { q } = (input as { query: { q: string } }).query + return { success: true, files: files.filter((entry) => entry.name.includes(q)) } + } + if (contract !== lookupWorkspaceFilesContract) throw new Error('unexpected request') + const body = (input as { body: { ids?: string[]; references?: string[] } }).body + const references = (body.references ?? []).map((reference) => ({ + reference, + fileId: + files.find((entry) => entry.id === reference || entry.name === reference)?.id ?? null, + })) + return { + success: true, + files: files.filter( + (entry) => + body.ids?.includes(entry.id) || + references.some((resolved) => resolved.fileId === entry.id) + ), + references, + } + }) + } + + it('still refuses a name two files share, though the server lookup picks one of them', async () => { + serveWorkspace([ + file({ id: 'wf_a', name: 'notes.md', folderPath: 'a' }), + file({ id: 'wf_b', name: 'notes.md', folderPath: 'b' }), + ]) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + await expect(fetchWorkspaceResourceRef(client, 'ws-1', ref())).resolves.toBeNull() + await expect( + fetchWorkspaceResourceRef(client, 'ws-1', ref({ id: 'wf_b' })) + ).resolves.toMatchObject({ type: 'file', id: 'wf_b', path: 'files/b/notes.md' }) + }) + + it('resolves a unique name with one batched lookup and its name search', async () => { + serveWorkspace([file({ id: 'wf_a', name: 'notes.md' }), file({ id: 'wf_c', name: 'plan.md' })]) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + await expect(fetchWorkspaceResourceRef(client, 'ws-1', ref())).resolves.toMatchObject({ + id: 'wf_a', + }) + expect( + apiClientRequestMockFns.mockRequestJson.mock.calls.filter( + ([contract]) => contract === lookupWorkspaceFilesContract + ) + ).toHaveLength(1) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts index d149c745cbf..df4f0fbb2d0 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts @@ -1,7 +1,13 @@ +import type { QueryClient } from '@tanstack/react-query' +import { V2_SEARCH_MAX_LENGTH } from '@/lib/api/contracts/v2/shared' import type { MothershipResource, WorkspaceResourceRef } from '@/lib/mothership/resources/types' import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' +import { + getWorkspaceFileLookupQueryOptions, + getWorkspaceFileSearchQueryOptions, +} from '@/hooks/queries/workspace-files' /** * The one file whose name is exactly {@link name}, or undefined. Two files can @@ -73,3 +79,94 @@ export function resolveWorkspaceResourceRef( path: canonicalWorkspaceFilePath({ folderPath: match.folderPath, name: match.name }), } } + +/** The most files a chip's name candidate is checked against for ambiguity. */ +const NAME_CANDIDATE_LIMIT = 50 + +/** The values a file chip may be identified by, in the order they are tried. */ +function fileRefCandidates(ref: WorkspaceResourceRef): string[] { + return [ + ...new Set( + [ref.id?.trim(), ref.path?.trim(), ref.title.trim()].filter((value): value is string => + Boolean(value) + ) + ), + ] +} + +/** + * The reads that together hold every file a chip's candidates can match under + * {@link resolveWorkspaceResourceRef}'s rules: each candidate as an exact id, as a reference (an + * exact VFS path resolves first), and — for a candidate that could be a bare name — every file + * whose name contains it, so a name shared by two files is still seen as ambiguous. + */ +function fileRefReads(queryClient: QueryClient, workspaceId: string, ref: WorkspaceResourceRef) { + return fileRefCandidates(ref).flatMap((candidate) => [ + getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, 'id', candidate), + getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, 'reference', candidate), + ...(candidate.includes('/') || candidate.length > V2_SEARCH_MAX_LENGTH + ? [] + : [ + getWorkspaceFileSearchQueryOptions(workspaceId, { + q: candidate, + limit: NAME_CANDIDATE_LIMIT, + }), + ]), + ]) +} + +/** The distinct files several reads returned: one file found by two reads is still one file. */ +function distinctFiles( + reads: readonly (WorkspaceFileRecord | WorkspaceFileRecord[] | null)[] +): WorkspaceFileRecord[] { + const files = new Map<string, WorkspaceFileRecord>() + for (const data of reads) { + for (const file of Array.isArray(data) ? data : data ? [data] : []) files.set(file.id, file) + } + return [...files.values()] +} + +/** + * Resolves a chip against the workspace's live files, fetching only the handful of files its + * candidates can name rather than the workspace's list. `fresh` re-reads them all: a chip may + * name a file the agent created a moment ago, which a cached miss would hide. Rejects when a + * read fails, so "we could not look" stays distinct from "it is not there". + */ +export async function fetchWorkspaceResourceRef( + queryClient: QueryClient, + workspaceId: string, + ref: WorkspaceResourceRef, + options?: { fresh?: boolean } +): Promise<MothershipResource | null> { + if (ref.type !== 'file') return resolveWorkspaceResourceRef(ref, []) + const reads = await Promise.all( + fileRefReads(queryClient, workspaceId, ref).map((read) => + queryClient.fetchQuery<WorkspaceFileRecord | WorkspaceFileRecord[] | null>({ + ...read, + ...(options?.fresh ? { staleTime: 0 } : {}), + }) + ) + ) + return resolveWorkspaceResourceRef(ref, distinctFiles(reads)) +} + +/** + * {@link fetchWorkspaceResourceRef} from the cache alone: the resolved resource, `null` when the + * cached reads name no file, or `undefined` when a read it needs has not been cached yet. + */ +export function readCachedWorkspaceResourceRef( + queryClient: QueryClient, + workspaceId: string, + ref: WorkspaceResourceRef +): MothershipResource | null | undefined { + if (ref.type !== 'file') return resolveWorkspaceResourceRef(ref, []) + const reads: (WorkspaceFileRecord | WorkspaceFileRecord[] | null)[] = [] + for (const read of fileRefReads(queryClient, workspaceId, ref)) { + const data = queryClient.getQueryData<WorkspaceFileRecord | WorkspaceFileRecord[] | null>( + read.queryKey + ) + if (data === undefined) return undefined + reads.push(data) + } + return resolveWorkspaceResourceRef(ref, distinctFiles(reads)) +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts index 703c719c305..29ee7b9e1e1 100644 --- a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -39,23 +39,18 @@ const { mockListPinnedItemsForUser, mockListWorkspacesForViewer, mockListMothershipChats, - mockListWorkspaceFilesWithShares, } = vi.hoisted(() => ({ mockGetWorkspaceHostContextForViewer: vi.fn(), mockKnowledgePresenterList: vi.fn(), mockListPinnedItemsForUser: vi.fn(), mockListWorkspacesForViewer: vi.fn(), mockListMothershipChats: vi.fn(), - mockListWorkspaceFilesWithShares: vi.fn(), })) vi.mock('@/lib/workspaces/host-context', () => ({ getWorkspaceHostContextForViewer: mockGetWorkspaceHostContextForViewer, })) vi.mock('@/lib/folders/queries', () => folderQueriesMock) -vi.mock('@/lib/workspace-files/queries', () => ({ - listWorkspaceFilesWithShares: mockListWorkspaceFilesWithShares, -})) vi.mock('@/lib/uploads/contexts/workspace/file-folder-manager', () => workspaceFileFoldersMock) vi.mock('@/lib/pinned-items/queries', () => ({ listPinnedItemsForUser: mockListPinnedItemsForUser, @@ -96,6 +91,7 @@ import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { folderKeys } from '@/hooks/queries/utils/folder-keys' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { tableKeys } from '@/hooks/queries/utils/table-keys' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' const mockAuthenticate = apiServerRoutesMockFns.mockInternalSessionAuthenticate const mockListFoldersForWorkspace = folderQueriesMockFns.mockListFoldersForWorkspace @@ -104,6 +100,7 @@ const mockListInternalKnowledgeBases = const mockListWorkflowsForUser = workflowsQueriesMockFns.mockListWorkflowsForUser const mockGetUserProfile = usersQueriesMockFns.mockGetUserProfile const mockListWorkspaceFileFolders = workspaceFileFoldersMockFns.mockListWorkspaceFileFolders +const mockListWorkspaceFileCollection = workspaceFilesListMockFns.mockListWorkspaceFileCollection /** One authorization decision shared by the table, knowledge, and file module operations. */ const mockAuthorizeResource = vi.fn() @@ -122,6 +119,11 @@ const mockGetWorkspaceMemberProfiles = permissionsMockFns.mockGetWorkspaceMember const WORKSPACE_ID = 'ws-123' const USER_ID = 'user-1' +const ROOT_FILE_LIST = workspaceFileListParams({ + folderId: null, + sortBy: 'updatedAt', + sortOrder: 'desc', +}) function makeClient() { return new QueryClient({ defaultOptions: { queries: { retry: false } } }) @@ -132,7 +134,7 @@ describe('workspace list prefetches', () => { mockAuthorizeResource.mockResolvedValue(undefined) mockGetWorkspaceHostContextForViewer.mockResolvedValue({ viewer: { permission: 'admin' } }) mockListFoldersForWorkspace.mockResolvedValue([]) - mockListWorkspaceFilesWithShares.mockResolvedValue([]) + mockListWorkspaceFileCollection.mockResolvedValue({ files: [], nextKeys: null }) mockListWorkspaceFileFolders.mockResolvedValue([]) mockListPinnedItemsForUser.mockResolvedValue([]) mockListWorkflowsForUser.mockResolvedValue([]) @@ -179,7 +181,12 @@ describe('workspace list prefetches', () => { }) }) - it.each([prefetchTables, prefetchKnowledgeBases, prefetchFilesBrowser])( + it.each([ + prefetchTables, + prefetchKnowledgeBases, + (client: QueryClient, workspaceId: string, userId: string) => + prefetchFilesBrowser(client, workspaceId, userId, { list: ROOT_FILE_LIST }), + ])( 'seeds no protected data or chrome when the module operation refuses access', async (prefetch) => { mockAuthorizeResource.mockRejectedValue( @@ -190,7 +197,7 @@ describe('workspace list prefetches', () => { expect(client.getQueryCache().getAll()).toHaveLength(0) expect(mockListTables).not.toHaveBeenCalled() expect(mockListInternalKnowledgeBases).not.toHaveBeenCalled() - expect(mockListWorkspaceFilesWithShares).not.toHaveBeenCalled() + expect(mockListWorkspaceFileCollection).not.toHaveBeenCalled() expect(mockListFoldersForWorkspace).not.toHaveBeenCalled() expect(mockListWorkspaceFileFolders).not.toHaveBeenCalled() expect(mockListPinnedItemsForUser).not.toHaveBeenCalled() @@ -263,7 +270,7 @@ describe('workspace list prefetches', () => { mockGetWorkspaceHostContextForViewer.mockResolvedValue(null) const client = makeClient() - await prefetchFilesBrowser(client, WORKSPACE_ID, USER_ID) + await prefetchFilesBrowser(client, WORKSPACE_ID, USER_ID, { list: ROOT_FILE_LIST }) expect(client.getQueryCache().getAll()).toHaveLength(0) expect(mockListWorkspaceFileFolders).not.toHaveBeenCalled() diff --git a/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.test.ts deleted file mode 100644 index 6e95a4de659..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { emcnMock } from '@sim/testing/mocks/emcn.mock' -import { describe, expect, it, vi } from 'vitest' - -const { mockListWorkspaceFilesWithShares } = vi.hoisted(() => ({ - mockListWorkspaceFilesWithShares: vi.fn(), -})) - -vi.mock('@/lib/workspace-files/queries', () => ({ - listWorkspaceFilesWithShares: mockListWorkspaceFilesWithShares, -})) - -/** The key factory lives in a `'use client'` module that pulls emcn's CSS at import. */ -vi.mock('@sim/emcn', () => emcnMock) - -import { seedWorkspaceFiles } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' - -const WORKSPACE_ID = 'ws-123' - -function makeClient() { - const store = new Map<string, unknown>() - return { - setQueryData: (key: readonly unknown[], value: unknown) => - store.set(JSON.stringify(key), value), - getQueryData: (key: readonly unknown[]) => store.get(JSON.stringify(key)), - } as never as import('@tanstack/react-query').QueryClient & { - getQueryData: (key: readonly unknown[]) => unknown - } -} - -describe('seedWorkspaceFiles', () => { - /** - * A workspace over the budget seeds NOTHING rather than the prefix that was read: the - * Files browser renders this list as the workspace's files, so a truncated seed would - * silently hide some. The client fetch reaches the route for the complete list instead. - */ - it('seeds nothing when the workspace exceeds the budget', async () => { - mockListWorkspaceFilesWithShares.mockResolvedValue(null) - const client = makeClient() - - await seedWorkspaceFiles(client, WORKSPACE_ID) - - expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toBeUndefined() - }) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.ts b/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.ts deleted file mode 100644 index d2a476f3a78..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/lib/seed-workspace-files.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import type { QueryClient } from '@tanstack/react-query' -import { listWorkspaceFilesWithShares } from '@/lib/workspace-files/queries' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' - -const logger = createLogger('SeedWorkspaceFiles') - -/** - * How many files a page is willing to inline into its document. At ~500 bytes of JSON per - * file this budgets the entry at ~150 KB. - * - * A workspace above the budget seeds NOTHING rather than a prefix: the Files browser - * renders this list as the workspace's files, so a truncated seed would silently hide some. - */ -export const WORKSPACE_FILE_SEED_MAX = 300 - -/** - * Seeds the workspace's file list for the pages that render it. - * - * Seeded rather than prefetched so it can decline to create an entry at all above - * {@link WORKSPACE_FILE_SEED_MAX} — `prefetchQuery` always creates one, and a partial - * entry would be read as the whole list. Parsed through the route's response contract, so - * a seeded entry matches what a client fetch caches. - */ -export async function seedWorkspaceFiles( - queryClient: QueryClient, - workspaceId: string -): Promise<void> { - try { - const files = await listWorkspaceFilesWithShares(workspaceId, 'active', { - maxRows: WORKSPACE_FILE_SEED_MAX, - /** - * A failed read must reach the catch below, not degrade to an empty list: seeding - * `[]` would cache "this workspace has no files" as authoritative for the entry's - * lifetime, which is worse than seeding nothing and letting the client fetch. - */ - throwOnError: true, - }) - if (!files) return - queryClient.setQueryData(workspaceFilesKeys.list(workspaceId, 'active'), files) - } catch (error) { - /** Optimization only: the client fetch reaches the route instead. */ - logger.warn('Workspace file list seed failed; client will fetch', { - error: getErrorMessage(error), - }) - } -} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/sim-resource-cell.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/sim-resource-cell.tsx index fdbd2776e97..b1da11b5550 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/sim-resource-cell.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/sim-resource-cell.tsx @@ -7,7 +7,7 @@ import type { ChatMessageContext } from '@/app/workspace/[workspaceId]/home/type import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' import { useWorkflows } from '@/hooks/queries/workflows' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileLookup } from '@/hooks/queries/workspace-files' /** Sim resource kinds a table cell URL can point to within the current workspace. */ export type SimResourceType = 'workflow' | 'table' | 'knowledge' | 'file' @@ -34,7 +34,8 @@ interface SimResourceCellProps { * workspace as a tagged-resource chip — the same icon used for @-style resource * mentions, plus the resource's name as a link. * Only the list matching `resourceType` is fetched; the other queries stay - * disabled so a sim-resource cell subscribes to a single shared list. + * disabled so a sim-resource cell subscribes to a single shared list. A file is + * looked up by id instead — a column of file cells resolves in one batched request. */ export function SimResourceCell({ workspaceId, @@ -50,7 +51,7 @@ export function SimResourceCell({ const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId, { enabled: resourceType === 'knowledge', }) - const { data: files = [] } = useWorkspaceFiles(workspaceId, 'active', { + const { data: file } = useWorkspaceFileLookup(workspaceId, 'id', resourceId, { enabled: resourceType === 'file', }) @@ -66,9 +67,9 @@ export function SimResourceCell({ case 'knowledge': return knowledgeBases.find((kb) => kb.id === resourceId)?.name case 'file': - return files.find((f) => f.id === resourceId)?.name + return file?.name } - }, [resourceType, resourceId, workflow, tables, knowledgeBases, files]) + }, [resourceType, resourceId, workflow, tables, knowledgeBases, file]) const label = name ?? FALLBACK_LABEL[resourceType] diff --git a/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx b/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx index 3ea390f13c5..e51a6aa6259 100644 --- a/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx @@ -18,7 +18,7 @@ import { useRunWorkflowTests, useWorkflowTest, } from '@/hooks/queries/workflow-tests' -import { useAddressedWorkspaceFileRecord } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' import { testRunSelectionKey, useTestRunSelectionStore } from '@/stores/workflow-tests/store' interface TestTarget { @@ -115,7 +115,7 @@ export function TestView({ workspaceId, name, previewMode }: TestViewProps) { const testKey = testRunSelectionKey(workspaceId, name) useEffect(() => () => setUnsaved(testKey, false), [setUnsaved, testKey]) const detail = query.data - const file = useAddressedWorkspaceFileRecord(workspaceId, detail?.test.fileId ?? '', { + const file = useWorkspaceFileRecord(workspaceId, detail?.test.fileId ?? '', { enabled: Boolean(detail), }) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx index 0d451c8ca53..d0f82899867 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx @@ -15,6 +15,7 @@ import { fileDeleteContract } from '@/lib/api/contracts/storage-transfer' import { ROOT_FOLDER_PATH } from '@/lib/folders/paths' import { readFolderPaths } from '@/lib/folders/selection' import { formatFileSize, getExtensionFromMimeType } from '@/lib/uploads/utils/file' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { containsReference } from '@/lib/workflows/sanitization/references' import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' import { isFileInFolderScope } from '@/lib/workspace-files/folder-path-selection' @@ -26,12 +27,18 @@ import { useActiveCanonicalSubBlockValue } from '@/app/workspace/[workspaceId]/w import { useResourceFolders } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-resource-folders' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' import { useActiveSearchTarget } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/providers/active-search-target-provider' +import { invalidateWorkspaceFileReads } from '@/hooks/queries/utils/workspace-file-cache' +import { + workspaceFileListParams, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' import { useCloudStorageConfigured, useUploadWorkspaceFile, - useWorkspaceFiles, - workspaceFilesKeys, + useWorkspaceFileList, + useWorkspaceFileSelectionCandidates, } from '@/hooks/queries/workspace-files' +import { useDebounce } from '@/hooks/use-debounce' import { getProviderAttachmentMaxBytes } from '@/providers/attachments' import { getProviderFromModel } from '@/providers/utils' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' @@ -39,6 +46,9 @@ import { useWorkflowStore } from '@/stores/workflows/workflow/store' const logger = createLogger('FileUpload') +/** How many files the picker lists per page. */ +const PICKER_PAGE_SIZE = 100 + interface FileUploadProps { blockId: string subBlockId: string @@ -131,6 +141,12 @@ interface SingleFileSelectorProps { onInputChange: (value: string) => void onClear: (e: React.MouseEvent) => void onOpenChange: (open: boolean) => void + /** Hears the typed query (and `''` when the menu closes), which the options are searched by. */ + onSearchChange: (query: string) => void + /** Another page of options can be loaded. */ + hasMore: boolean + isLoadingMore: boolean + onLoadMore: () => void disabled: boolean isLoading: boolean truncateMiddle: (text: string, start?: number, end?: number) => string @@ -150,6 +166,10 @@ function SingleFileSelector({ onInputChange, onClear, onOpenChange, + onSearchChange, + hasMore, + isLoadingMore, + onLoadMore, disabled, isLoading, truncateMiddle, @@ -180,14 +200,19 @@ function SingleFileSelector({ // User is typing to search setIsEditing(true) setSearchQuery(newValue) + onSearchChange(newValue) }} onOpenChange={(open) => { if (!open) { setIsEditing(false) setSearchQuery('') + onSearchChange('') } onOpenChange(open) }} + hasMore={hasMore} + isLoadingMore={isLoadingMore} + onLoadMore={onLoadMore} placeholder={isLoading ? 'Loading files...' : 'Select or upload file'} disabled={disabled || isDeleting} editable={true} @@ -274,13 +299,6 @@ export function FileUpload({ const activeWorkflowId = useWorkflowRegistry((state) => state.activeWorkflowId) const { workspaceId } = useWorkflowScope() - const { - data: workspaceFiles = [], - isLoading: loadingWorkspaceFiles, - isPlaceholderData: workspaceFilesArePlaceholderData, - refetch: refetchWorkspaceFiles, - } = useWorkspaceFiles(isPreview ? '' : workspaceId) - const { data: cloudConfigured, isLoading: loadingCloudStatus } = useCloudStorageConfigured( requiresCloudStorage && !isPreview ) @@ -404,18 +422,57 @@ export function FileUpload({ loadingFolderScope, ]) const folderScopeUploadBlocked = uploadTargetFolderId === undefined + /** + * The picker offers files a page at a time rather than the workspace's whole list: with a + * typed query, the files whose names match it; without one, the most recently updated. A + * single concrete folder scope is applied by the server (descending when the scope does); + * any other scope is applied to the page here, which keeps it correct but sparser. + */ + const pickerQuery = useDebounce(inputValue.trim(), SEARCH_DEBOUNCE_MS) + const scopeFolderId = + hasConcreteFolderScope && folderScopePaths.length === 1 + ? folderScopePaths[0] === ROOT_FOLDER_PATH + ? null + : folderByPath.get(folderScopePaths[0])?.id + : undefined + const pickerList = useWorkspaceFileList( + isPreview ? '' : workspaceId, + workspaceFileListParams({ + folderId: scopeFolderId, + recursive: folderScopeIncludesSubfolders, + q: pickerQuery, + sortBy: pickerQuery ? 'name' : 'updatedAt', + sortOrder: pickerQuery ? 'asc' : 'desc', + limit: PICKER_PAGE_SIZE, + }) + ) + const loadingWorkspaceFiles = pickerList.isLoading + const loadMorePickerFiles = () => { + if (!pickerList.isFetching && pickerList.hasNextPage) void pickerList.fetchNextPage() + } + const pickerFiles = useMemo( + () => pickerList.data?.pages.flatMap((page) => page.files) ?? [], + [pickerList.data] + ) const scopedWorkspaceFiles = useMemo( () => hasConcreteFolderScope - ? workspaceFiles.filter((workspaceFile) => + ? pickerFiles.filter((workspaceFile) => folderScopePaths.some((folderScopePath) => isFileInFolderScope(workspaceFile.folderPath, folderScopePath, { includeSubfolders: folderScopeIncludesSubfolders, }) ) ) - : workspaceFiles, - [folderScopeIncludesSubfolders, folderScopePaths, hasConcreteFolderScope, workspaceFiles] + : pickerFiles, + [folderScopeIncludesSubfolders, folderScopePaths, hasConcreteFolderScope, pickerFiles] + ) + + /** The selected files, resolved on their own: they need not be on the page the picker shows. */ + const selections = useWorkspaceFileSelectionCandidates(isPreview ? '' : workspaceId, filesArray) + const workspaceFiles = useMemo( + () => [...selections.candidates, ...pickerFiles], + [selections.candidates, pickerFiles] ) const selectedWorkspaceFileIds = useMemo( @@ -430,20 +487,20 @@ export function FileUpload({ ) useEffect(() => { - if ( - !folderScope || - !hasConcreteFolderScope || - loadingWorkspaceFiles || - workspaceFilesArePlaceholderData || - isPreview - ) { + if (!folderScope || !hasConcreteFolderScope || selections.isPending || isPreview) { return } - const scopedIds = new Set(scopedWorkspaceFiles.map((file) => file.id)) const nextFiles = filesArray.filter((file) => { - const workspaceFile = findSelectedWorkspaceFile(file, workspaceFiles) - return !workspaceFile || scopedIds.has(workspaceFile.id) + const workspaceFile = findSelectedWorkspaceFile(file, selections.candidates) + return ( + !workspaceFile || + folderScopePaths.some((folderScopePath) => + isFileInFolderScope(workspaceFile.folderPath, folderScopePath, { + includeSubfolders: folderScopeIncludesSubfolders, + }) + ) + ) }) if (nextFiles.length === filesArray.length) return @@ -452,13 +509,13 @@ export function FileUpload({ commitValue, filesArray, folderScope, + folderScopeIncludesSubfolders, + folderScopePaths, hasConcreteFolderScope, isPreview, - loadingWorkspaceFiles, multiple, - scopedWorkspaceFiles, - workspaceFiles, - workspaceFilesArePlaceholderData, + selections.candidates, + selections.isPending, ]) const availableWorkspaceFiles = useMemo( @@ -589,7 +646,7 @@ export function FileUpload({ setUploadError(null) if (workspaceId) { - void refetchWorkspaceFiles() + void invalidateWorkspaceFileReads(queryClient, workspaceId) void queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) } @@ -943,8 +1000,11 @@ export function FileUpload({ value={inputValue} onChange={handleComboboxChange} onOpenChange={(open) => { - if (open) void refetchWorkspaceFiles() + if (open) void pickerList.refetch() }} + hasMore={pickerList.hasNextPage} + isLoadingMore={pickerList.isFetchingNextPage} + onLoadMore={loadMorePickerFiles} placeholder={loadingWorkspaceFiles ? 'Loading files...' : '+ Add More'} disabled={disabled || loadingWorkspaceFiles} editable={true} @@ -962,8 +1022,12 @@ export function FileUpload({ onInputChange={handleComboboxChange} onClear={(e) => handleRemoveFile(filesArray[0], e)} onOpenChange={(open) => { - if (open) void refetchWorkspaceFiles() + if (open) void pickerList.refetch() }} + onSearchChange={setInputValue} + hasMore={pickerList.hasNextPage} + isLoadingMore={pickerList.isFetchingNextPage} + onLoadMore={loadMorePickerFiles} disabled={disabled} isLoading={loadingWorkspaceFiles} truncateMiddle={truncateMiddle} @@ -985,8 +1049,11 @@ export function FileUpload({ value={inputValue} onChange={handleComboboxChange} onOpenChange={(open) => { - if (open) void refetchWorkspaceFiles() + if (open) void pickerList.refetch() }} + hasMore={pickerList.hasNextPage} + isLoadingMore={pickerList.isFetchingNextPage} + onLoadMore={loadMorePickerFiles} placeholder={loadingWorkspaceFiles ? 'Loading files...' : 'Select or upload file'} disabled={disabled || loadingWorkspaceFiles} editable={true} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/components/replacement-controls/replacement-controls.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/components/replacement-controls/replacement-controls.tsx index aefe58bbe91..39eeceec6d2 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/components/replacement-controls/replacement-controls.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/components/replacement-controls/replacement-controls.tsx @@ -13,6 +13,8 @@ interface ReplacementControlsProps { canReplaceActive: boolean canReplaceAll: boolean onReplacementChange: (replacement: string) => void + /** Hears what is typed in the resource picker, which file options are searched by. */ + onReplacementSearchChange: (query: string) => void onReplaceActive: () => void onReplaceAll: () => void } @@ -27,6 +29,7 @@ export function ReplacementControls({ canReplaceActive, canReplaceAll, onReplacementChange, + onReplacementSearchChange, onReplaceActive, onReplaceAll, }: ReplacementControlsProps) { @@ -42,6 +45,7 @@ export function ReplacementControls({ onChange={onReplacementChange} placeholder='Choose replacement...' searchable + onSearchChange={onReplacementSearchChange} searchPlaceholder='Search resources...' emptyMessage='No valid replacements available' disabled={disabled || compatibleResourceOptions.length === 0} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts index 1602ef50373..0db3ed1e6ef 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts @@ -1,4 +1,5 @@ import { useMemo } from 'react' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { WorkflowSearchMatch, WorkflowSearchReplacementOption, @@ -14,23 +15,34 @@ import { useWorkflowSearchSelectorReplacementOptions, useWorkflowSearchTableReplacementOptions, } from '@/hooks/queries/workflow-search-replace' +import { useDebounce } from '@/hooks/use-debounce' interface UseWorkflowResourceReplacementOptionsParams { matches: WorkflowSearchMatch[] workspaceId?: string workflowId?: string + /** What is typed in the replacement picker; files are searched by it rather than listed whole. */ + query: string + /** Every replacement already chosen, kept on offer so each stays valid. */ + chosen: readonly string[] } export function useWorkflowResourceReplacementOptions({ matches, workspaceId, workflowId, + query, + chosen, }: UseWorkflowResourceReplacementOptionsParams): WorkflowSearchReplacementOption[] { + const fileQuery = useDebounce(query.trim(), SEARCH_DEBOUNCE_MS) const oauthOptions = useWorkflowSearchOAuthReplacementOptions(matches, workspaceId, workflowId) const knowledgeOptions = useWorkflowSearchKnowledgeReplacementOptions(matches, workspaceId) const selectorOptions = useWorkflowSearchSelectorReplacementOptions(matches) const tableOptions = useWorkflowSearchTableReplacementOptions(matches, workspaceId) - const fileOptions = useWorkflowSearchFileReplacementOptions(matches, workspaceId) + const fileOptions = useWorkflowSearchFileReplacementOptions(matches, workspaceId, { + query: fileQuery, + chosen, + }) const mcpServerOptions = useWorkflowSearchMcpServerReplacementOptions(matches, workspaceId) const mcpToolOptions = useWorkflowSearchMcpToolReplacementOptions(matches, workspaceId) const { data: personalEnvironment } = usePersonalEnvironment() diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/workflow-search-replace.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/workflow-search-replace.tsx index b975de2ac59..b11f71fd40e 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/workflow-search-replace.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/workflow-search-replace.tsx @@ -269,10 +269,17 @@ function WorkflowSearchReplacePanel({ focusRef }: WorkflowSearchReplacePanelProp workspaceId, workflowId, }) + const [replacementSearch, setReplacementSearch] = useState('') + const chosenResourceReplacements = useMemo( + () => Object.values(resourceReplacementByContext), + [resourceReplacementByContext] + ) const resourceOptions = useWorkflowResourceReplacementOptions({ matches, workspaceId, workflowId, + query: replacementSearch, + chosen: chosenResourceReplacements, }) const hydratedMatches = useMemo( @@ -674,6 +681,7 @@ function WorkflowSearchReplacePanel({ focusRef }: WorkflowSearchReplacePanelProp eligibleMatchIds.length > 0 && hasReplacement && !allReplacementIssue )} onReplacementChange={handleReplacementChange} + onReplacementSearchChange={setReplacementSearch} onReplaceActive={handleReplaceActive} onReplaceAll={handleReplaceAll} /> diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu/collapsed-sidebar-menu.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu/collapsed-sidebar-menu.tsx index f1ee1cebea3..ee655e297e5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu/collapsed-sidebar-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu/collapsed-sidebar-menu.tsx @@ -45,6 +45,19 @@ interface CollapsedResourceFlyoutProps { */ isLoading?: boolean emptyLabel: string + /** + * Fills each folder's submenu itself, on open, instead of from `entries`' nested children — + * for a resource whose folders are listed a page at a time. `isEmpty` decides which folders + * are inert, since their children are not in `entries` to count. + */ + lazyFolders?: LazyFolderContent + /** A row after the entries, e.g. a link to the rest of a list the flyout shows only part of. */ + footer?: React.ReactNode +} + +interface LazyFolderContent { + render: (folderId: string) => React.ReactNode + isEmpty: (folderId: string) => boolean } /** @@ -59,6 +72,8 @@ export function CollapsedResourceFlyout({ currentItemId, isLoading = false, emptyLabel, + lazyFolders, + footer, }: CollapsedResourceFlyoutProps) { if (isLoading) { return ( @@ -71,7 +86,16 @@ export function CollapsedResourceFlyout({ if (entries.length === 0) { return <DropdownMenuItem disabled>{emptyLabel}</DropdownMenuItem> } - return <CollapsedFlyoutRows entries={entries} currentItemId={currentItemId} /> + return ( + <> + <CollapsedFlyoutRows + entries={entries} + currentItemId={currentItemId} + lazyFolders={lazyFolders} + /> + {footer} + </> + ) } /** @@ -88,7 +112,8 @@ function PinnedGlyph() { function CollapsedFlyoutRows({ entries, currentItemId, -}: Pick<CollapsedResourceFlyoutProps, 'entries' | 'currentItemId'>) { + lazyFolders, +}: Pick<CollapsedResourceFlyoutProps, 'entries' | 'currentItemId' | 'lazyFolders'>) { return ( <> {entries.map((entry) => { @@ -103,7 +128,8 @@ function CollapsedFlyoutRows({ ) } - if (entry.children.length === 0) { + const isEmpty = lazyFolders ? lazyFolders.isEmpty(entry.id) : entry.children.length === 0 + if (isEmpty) { return ( <DropdownMenuItem key={entry.id} disabled> <OverflowText label={entry.name} /> @@ -119,7 +145,11 @@ function CollapsedFlyoutRows({ {entry.pinned && <PinnedGlyph />} </DropdownMenuSubTrigger> <DropdownMenuSubContent className={SIDEBAR_NO_MOTION_CLASS}> - <CollapsedFlyoutRows entries={entry.children} currentItemId={currentItemId} /> + {lazyFolders ? ( + lazyFolders.render(entry.id) + ) : ( + <CollapsedFlyoutRows entries={entry.children} currentItemId={currentItemId} /> + )} </DropdownMenuSubContent> </DropdownMenuSub> ) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx index 19daa684447..79b6ad8eb19 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx @@ -11,21 +11,35 @@ */ import { useMemo } from 'react' +import { DropdownMenuItem } from '@sim/emcn' +import Link from 'next/link' import { useParams } from 'next/navigation' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildFlyoutEntries, FOLDERED_RESOURCE_HEADERS, + folderedResourceListHref, } from '@/app/workspace/[workspaceId]/components/folders' import { CollapsedResourceFlyout } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu' import { useFolders } from '@/hooks/queries/folders' import { usePinnedIds } from '@/hooks/queries/pinned-items' import { useTablesList } from '@/hooks/queries/tables' -import { useWorkspaceFileFolders } from '@/hooks/queries/workspace-file-folders' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' +import { + useWorkspaceFileFolders, + type WorkspaceFileFolderApi, +} from '@/hooks/queries/workspace-file-folders' +import { + useWorkspaceFileFolderSizes, + useWorkspaceFileList, + useWorkspaceFileLookups, +} from '@/hooks/queries/workspace-files' const TABLE_META = FOLDERED_RESOURCE_HEADERS.table const FILE_META = FOLDERED_RESOURCE_HEADERS.file +const EMPTY_FILE_FOLDERS: WorkspaceFileFolderApi[] = [] + /** * A list is usable only once it has resolved for THIS workspace. Every list here keeps the * previous workspace's rows as placeholder data across a switch, and a tree built from one @@ -72,33 +86,116 @@ export function TablesRailFlyout({ workspaceId }: { workspaceId: string }) { ) } -export function FilesRailFlyout({ workspaceId }: { workspaceId: string }) { - const params = useParams() - const filesQuery = useWorkspaceFiles(workspaceId) - const foldersQuery = useWorkspaceFileFolders(workspaceId) +/** How many of a folder's files the flyout lists before linking to the rest in Files. */ +const FILE_FLYOUT_PAGE_SIZE = 50 + +interface FilesFlyoutLevelProps { + workspaceId: string + /** The folder this level lists, `null` for the root. */ + folderId: string | null + folders: readonly WorkspaceFileFolderApi[] + foldersResolved: boolean + currentItemId?: string +} + +/** + * One level of the Files flyout: the folder's subfolders and its most recently updated files, + * fetched when the level opens rather than the workspace's whole file list. A folder's submenu is + * the same component one level down, mounted only when it opens. A level with more files than it + * lists ends in a link to the folder in Files. + */ +function FilesFlyoutLevel({ + workspaceId, + folderId, + folders, + foldersResolved, + currentItemId, +}: FilesFlyoutLevelProps) { + const fileList = useWorkspaceFileList( + workspaceId, + workspaceFileListParams({ + folderId, + sortBy: 'updatedAt', + sortOrder: 'desc', + limit: FILE_FLYOUT_PAGE_SIZE, + }) + ) + const folderStats = useWorkspaceFileFolderSizes(workspaceId, [folderId]) const pinnedFileIds = usePinnedIds(workspaceId, 'file') const pinnedFolderIds = usePinnedIds(workspaceId, 'folder') - const { data: files } = filesQuery - const { data: folders } = foldersQuery + const pinnedFileIdList = useMemo(() => [...pinnedFileIds], [pinnedFileIds]) + const pinnedFiles = useWorkspaceFileLookups(workspaceId, 'id', pinnedFileIdList) - const entries = useMemo( - () => - buildFlyoutEntries({ - folders: folders ?? [], - items: files ?? [], - pinnedFolderIds, - pinnedItemIds: pinnedFileIds, - hrefForItem: (file) => `/workspace/${workspaceId}/${FILE_META.listSegment}/${file.id}`, - }), - [folders, files, pinnedFolderIds, pinnedFileIds, workspaceId] - ) + const entries = useMemo(() => { + const files = new Map<string, WorkspaceFileRecord>() + for (const fileId of pinnedFileIdList) { + const file = pinnedFiles.byValue.get(fileId) + if (file && (file.folderId ?? null) === folderId) files.set(file.id, file) + } + for (const file of fileList.data?.pages[0]?.files ?? []) files.set(file.id, file) + return buildFlyoutEntries({ + folders: folders + .filter((folder) => (folder.parentId ?? null) === folderId) + .map((folder) => ({ ...folder, parentId: null })), + items: [...files.values()].map((file) => ({ ...file, folderId: null })), + pinnedFolderIds, + pinnedItemIds: pinnedFileIds, + hrefForItem: (file) => `/workspace/${workspaceId}/${FILE_META.listSegment}/${file.id}`, + }) + }, [ + fileList.data, + folders, + folderId, + pinnedFiles.byValue, + pinnedFileIdList, + pinnedFolderIds, + pinnedFileIds, + workspaceId, + ]) + + const hasSubfolders = (id: string) => folders.some((folder) => folder.parentId === id) return ( <CollapsedResourceFlyout entries={entries} + currentItemId={currentItemId} + isLoading={isResolving(fileList) || !foldersResolved} + emptyLabel={folderId ? 'No files in this folder' : 'No files yet'} + lazyFolders={{ + isEmpty: (id) => !hasSubfolders(id) && folderStats.get(id)?.fileCount === 0, + render: (id) => ( + <FilesFlyoutLevel + workspaceId={workspaceId} + folderId={id} + folders={folders} + foldersResolved={foldersResolved} + currentItemId={currentItemId} + /> + ), + }} + footer={ + fileList.hasNextPage ? ( + <DropdownMenuItem asChild> + <Link href={folderedResourceListHref('file', workspaceId, folderId)}> + Show all in Files + </Link> + </DropdownMenuItem> + ) : null + } + /> + ) +} + +export function FilesRailFlyout({ workspaceId }: { workspaceId: string }) { + const params = useParams() + const foldersQuery = useWorkspaceFileFolders(workspaceId) + return ( + <FilesFlyoutLevel + workspaceId={workspaceId} + folderId={null} + folders={foldersQuery.data ?? EMPTY_FILE_FOLDERS} + foldersResolved={!isResolving(foldersQuery)} currentItemId={typeof params.fileId === 'string' ? params.fileId : undefined} - isLoading={isResolving(filesQuery) || isResolving(foldersQuery)} - emptyLabel='No files yet' /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx index cb13ddcef6d..14762fb4de8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx @@ -47,6 +47,7 @@ import { getFolderPathNames } from '@/lib/folders/tree' import { sendMothershipMessage } from '@/lib/mothership/events' import { captureEvent } from '@/lib/posthog/client' import { toSearchToken } from '@/lib/search/tokens' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { hasTriggerCapability } from '@/lib/workflows/triggers/utils' import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' import { useInvokeGlobalCommand } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' @@ -92,7 +93,9 @@ import { useWorkspaceAccessRequestFeatures } from '@/ee/access-requests/componen import { useFolderMap } from '@/hooks/queries/folders' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' -import { useWorkspaceFiles } from '@/hooks/queries/workspace-files' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' +import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' +import { useDebounce } from '@/hooks/use-debounce' import { usePermissionConfig } from '@/hooks/use-permission-config' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' import type { WorkflowFolder } from '@/stores/folders/types' @@ -123,6 +126,11 @@ const MAX_SEARCH_RESULTS = 50 /** Stable empty default so a pending folder map does not remount the memos below. */ const EMPTY_FOLDER_MAP: Record<string, WorkflowFolder> = {} +/** The files the palette offers before anything is typed: the most recently updated anywhere. */ +const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 50 }) + +const FILE_SEARCH_LIMIT = MAX_SEARCH_RESULTS + export type { SearchModalProps } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/utils' type SearchModalContentProps = Omit<SearchModalProps, 'open'> @@ -189,9 +197,6 @@ function SearchModalContent({ const { data: fetchedTables = [] } = useTablesList(workspaceId, 'active', { enabled: !permissionConfig.hideTablesTab, }) - const { data: fetchedFiles = [] } = useWorkspaceFiles(workspaceId, 'active', { - enabled: !permissionConfig.hideFilesTab, - }) const { data: fetchedKnowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId, { enabled: !permissionConfig.hideKnowledgeBaseTab, }) @@ -223,21 +228,6 @@ function SearchModalContent({ [fetchedTables, tableFolderMap, workspaceId, permissionConfig.hideTablesTab] ) - const files = useMemo( - () => - permissionConfig.hideFilesTab - ? [] - : fetchedFiles.map((f) => ({ - id: f.id, - name: f.name, - href: `/workspace/${workspaceId}/files/${f.id}`, - folderPath: f.folderPath - ? parseWorkspaceFileFolderDisplayPath(f.folderPath) - : undefined, - })), - [fetchedFiles, workspaceId, permissionConfig.hideFilesTab] - ) - const knowledgeBases = useMemo( () => permissionConfig.hideKnowledgeBaseTab @@ -693,6 +683,35 @@ function SearchModalContent({ const searchRef = useRef(search) searchRef.current = search + /** + * Files are not listed whole — a workspace can hold far more than the palette ranks. Browsing + * offers the most recently updated; a query, once typing settles, searches every file by name. + */ + const fileQuery = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) + const recentFiles = useWorkspaceFileList(workspaceId, RECENT_FILES, { + enabled: !permissionConfig.hideFilesTab, + }) + const matchingFiles = useWorkspaceFileSearch( + workspaceId, + { q: fileQuery, limit: FILE_SEARCH_LIMIT }, + { enabled: !permissionConfig.hideFilesTab } + ) + const fetchedFiles = fileQuery ? matchingFiles.data : recentFiles.data?.pages[0]?.files + const files = useMemo( + () => + permissionConfig.hideFilesTab + ? [] + : (fetchedFiles ?? []).map((f) => ({ + id: f.id, + name: f.name, + href: `/workspace/${workspaceId}/files/${f.id}`, + folderPath: f.folderPath + ? parseWorkspaceFileFolderDisplayPath(f.folderPath) + : undefined, + })), + [fetchedFiles, workspaceId, permissionConfig.hideFilesTab] + ) + /** * Focus once the dialog is actually visible: under atomic browser-panel * occlusion `autoFocus` is suppressed, and `.focus()` is a no-op while the diff --git a/apps/sim/hooks/queries/projects.ts b/apps/sim/hooks/queries/projects.ts index b3c2c32a374..ff65a0c7413 100644 --- a/apps/sim/hooks/queries/projects.ts +++ b/apps/sim/hooks/queries/projects.ts @@ -4,7 +4,7 @@ import { requestJson } from '@/lib/api/client/request' import { getProjectFileContract } from '@/lib/api/contracts/project-files' import { listProjectsContract } from '@/lib/api/contracts/projects' import { useFeatureFlag } from '@/app/workspace/[workspaceId]/providers/feature-flags-provider' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' const PROJECT_LIST_STALE_TIME = 30_000 const PROJECT_FILE_STALE_TIME = 30_000 diff --git a/apps/sim/hooks/queries/public-shares.ts b/apps/sim/hooks/queries/public-shares.ts index 5f33d20d163..8fd43ea7168 100644 --- a/apps/sim/hooks/queries/public-shares.ts +++ b/apps/sim/hooks/queries/public-shares.ts @@ -12,7 +12,7 @@ import { type VerifyPublicFileOtpResponse, verifyPublicFileOtpContract, } from '@/lib/api/contracts/public-shares' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' +import { invalidateWorkspaceFileReads } from '@/hooks/queries/utils/workspace-file-cache' export const FILE_SHARE_STALE_TIME = 30 * 1000 @@ -68,7 +68,7 @@ export function useUpsertFileShare() { toast.error(error.message) }, onSettled: (_data, _error, { workspaceId }) => { - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }) + void invalidateWorkspaceFileReads(queryClient, workspaceId) }, }) } diff --git a/apps/sim/hooks/queries/utils/workspace-file-cache.ts b/apps/sim/hooks/queries/utils/workspace-file-cache.ts new file mode 100644 index 00000000000..02fec4346da --- /dev/null +++ b/apps/sim/hooks/queries/utils/workspace-file-cache.ts @@ -0,0 +1,161 @@ +import type { InfiniteData, QueryClient, QueryKey } from '@tanstack/react-query' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { + type WorkspaceFileListPage, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' + +/** + * The cache entries that hold workspace file metadata, one per read shape. A file can sit in + * several at once — a list page, a search result, a lookup, its own record — so a write has to + * reach all of them, and a reader looking for a record it already has can find it in any. + */ +const METADATA_READS = ['list', 'search', 'lookup', 'record'] as const + +type ListData = InfiniteData<WorkspaceFileListPage, string | null> +type SearchData = WorkspaceFileRecord[] +type LookupData = WorkspaceFileRecord | null +type RecordData = WorkspaceFileRecord + +/** Whether `queryKey` is one of {@link METADATA_READS} for `workspaceId`. */ +export function isWorkspaceFileMetadataKey(queryKey: QueryKey, workspaceId: string): boolean { + return ( + queryKey[0] === workspaceFilesKeys.all[0] && + METADATA_READS.some((read) => read === queryKey[1]) && + queryKey[2] === workspaceId + ) +} + +/** Every metadata read of the workspace, for `refetchQueries`/`isFetching`/`cancelQueries`. */ +export function workspaceFileMetadataFilter(workspaceId: string) { + return { + predicate: (query: { queryKey: QueryKey }) => + isWorkspaceFileMetadataKey(query.queryKey, workspaceId), + } +} + +/** + * Invalidates every metadata read of the workspace — list walks, searches, folder rollups, + * lookups, and records — and settles once the mounted ones have refetched. + */ +export function invalidateWorkspaceFileReads(queryClient: QueryClient, workspaceId: string) { + return Promise.all([ + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }), + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceSearches(workspaceId) }), + queryClient.invalidateQueries({ + queryKey: workspaceFilesKeys.workspaceFolderStats(workspaceId), + }), + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLookups(workspaceId) }), + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceRecords(workspaceId) }), + ]) +} + +/** The files a cached metadata read holds, by the read's shape (its key's second segment). */ +function filesOf(queryKey: QueryKey, data: unknown): readonly WorkspaceFileRecord[] { + if (!data) return [] + switch (queryKey[1]) { + case 'list': + return (data as ListData).pages.flatMap((page) => page.files) + case 'search': + return data as SearchData + case 'lookup': + case 'record': + return [data as RecordData] + default: + return [] + } +} + +/** + * The freshest copy of a file any cached read of its workspace holds, and when that read last + * succeeded, without fetching. `undefined` when no read has seen it. A record a reader opens + * this way starts from what the list it was clicked in already showed, aged as that list is. + */ +export function findCachedWorkspaceFile( + queryClient: QueryClient, + workspaceId: string, + match: (file: WorkspaceFileRecord) => boolean +): { file: WorkspaceFileRecord; updatedAt: number } | undefined { + let found: { file: WorkspaceFileRecord; updatedAt: number } | undefined + for (const query of queryClient + .getQueryCache() + .findAll(workspaceFileMetadataFilter(workspaceId))) { + const { data, dataUpdatedAt } = query.state + if (found && dataUpdatedAt <= found.updatedAt) continue + // Each read's data shape is fixed by its key segment, which `filesOf` switches on. + const file = filesOf(query.queryKey, data).find(match) + if (file) found = { file, updatedAt: dataUpdatedAt } + } + return found +} + +/** The cached metadata entries of one workspace, as `getQueriesData` returns them, for a rollback. */ +export type WorkspaceFileReadSnapshot = [QueryKey, unknown][] + +/** Every cached metadata read of the workspace, for an optimistic write to restore on failure. */ +export function snapshotWorkspaceFileReads( + queryClient: QueryClient, + workspaceId: string +): WorkspaceFileReadSnapshot { + return queryClient.getQueriesData(workspaceFileMetadataFilter(workspaceId)) +} + +export function restoreWorkspaceFileReads( + queryClient: QueryClient, + snapshot: WorkspaceFileReadSnapshot +): void { + for (const [queryKey, data] of snapshot) queryClient.setQueryData(queryKey, data) +} + +/** + * Rewrites one file in every cached read of its workspace: `update` returns the new record, or + * `null` to drop the file (a delete). Lists, searches, and lookups lose a dropped file; its own + * record is left for the caller, since a reader still on it is mid-navigation away. + */ +export function updateCachedWorkspaceFile( + queryClient: QueryClient, + workspaceId: string, + fileId: string, + update: (file: WorkspaceFileRecord) => WorkspaceFileRecord | null +): void { + const apply = (files: WorkspaceFileRecord[]) => { + let changed = false + const next: WorkspaceFileRecord[] = [] + for (const file of files) { + if (file.id !== fileId) { + next.push(file) + continue + } + changed = true + const updated = update(file) + if (updated) next.push(updated) + } + return changed ? next : files + } + + queryClient.setQueriesData<ListData>( + { queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }, + (data) => { + if (!data) return data + let changed = false + const pages = data.pages.map((page) => { + const files = apply(page.files) + if (files === page.files) return page + changed = true + return { ...page, files } + }) + return changed ? { ...data, pages } : data + } + ) + queryClient.setQueriesData<SearchData>( + { queryKey: workspaceFilesKeys.workspaceSearches(workspaceId) }, + (data) => (data ? apply(data) : data) + ) + queryClient.setQueriesData<LookupData>( + { queryKey: workspaceFilesKeys.workspaceLookups(workspaceId) }, + (data) => (data?.id === fileId ? update(data) : data) + ) + queryClient.setQueryData<RecordData>(workspaceFilesKeys.record(workspaceId, fileId), (data) => + data ? (update(data) ?? data) : data + ) +} diff --git a/apps/sim/hooks/queries/utils/workspace-file-keys.ts b/apps/sim/hooks/queries/utils/workspace-file-keys.ts new file mode 100644 index 00000000000..9cd9bcd53c3 --- /dev/null +++ b/apps/sim/hooks/queries/utils/workspace-file-keys.ts @@ -0,0 +1,165 @@ +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' + +export type WorkspaceFileListScope = 'active' | 'archived' +export type WorkspaceFileListSortBy = 'name' | 'size' | 'uploadedAt' | 'updatedAt' +export type WorkspaceFileListSortOrder = 'asc' | 'desc' + +/** The default page size of a workspace file list: one round trip for nearly every folder. */ +const WORKSPACE_FILE_LIST_PAGE_SIZE = 200 + +export const WORKSPACE_FILE_LIST_STALE_TIME = 30 * 1000 +export const WORKSPACE_FILE_SEARCH_STALE_TIME = 30 * 1000 +export const WORKSPACE_FILE_FOLDER_STATS_STALE_TIME = 30 * 1000 +export const WORKSPACE_FILE_RECORD_STALE_TIME = 30 * 1000 +export const WORKSPACE_FILE_LOOKUP_STALE_TIME = 30 * 1000 + +/** + * One paged file list: which files (folder, search, filters), in which order, and how many per + * page. Built by {@link workspaceFileListParams} so two spellings of one list share a cache entry. + */ +export interface WorkspaceFileListParams { + scope: WorkspaceFileListScope + /** The folder listed, `null` for the workspace root; absent lists every folder. */ + folderId?: string | null + /** Whether `folderId`'s subfolders are listed too. */ + recursive?: boolean + /** Case-insensitive substring of the file name. */ + q?: string + types?: readonly WorkspaceFileKind[] + sizes?: readonly WorkspaceFileSizeBucket[] + uploadedBy?: readonly string[] + sortBy: WorkspaceFileListSortBy + sortOrder: WorkspaceFileListSortOrder + limit: number +} + +export interface WorkspaceFileListPage { + files: WorkspaceFileRecord[] + /** The next page's cursor, `null` on the last page. */ + nextCursor: string | null +} + +export interface WorkspaceFileSearchParams { + q: string + /** The folder searched, `null` for the root; absent searches the workspace. */ + folderId?: string | null + /** Whether `folderId`'s subfolders are searched too. */ + recursive?: boolean + limit: number +} + +/** Which identity a lookup resolves a file by: its id, its storage key, or an id/path/name reference. */ +export type WorkspaceFileLookupKind = 'id' | 'key' | 'reference' + +function canonicalSet<T extends string>(values: readonly T[] | undefined): T[] | undefined { + if (!values || values.length === 0) return undefined + return [...new Set(values)].sort() +} + +/** + * The canonical form of a list's params: the search trimmed and dropped when blank, set filters + * sorted, de-duplicated, and dropped when empty, and `recursive` kept only with a folder. The + * query key hashes this object, so equivalent lists land on one cache entry and the server + * fetch reads exactly what the key names. + */ +export function workspaceFileListParams(input: { + scope?: WorkspaceFileListScope + folderId?: string | null + recursive?: boolean + q?: string + types?: readonly WorkspaceFileKind[] + sizes?: readonly WorkspaceFileSizeBucket[] + uploadedBy?: readonly string[] + sortBy: WorkspaceFileListSortBy + sortOrder: WorkspaceFileListSortOrder + limit?: number +}): WorkspaceFileListParams { + const q = input.q?.trim() + return { + scope: input.scope ?? 'active', + ...(input.folderId === undefined ? {} : { folderId: input.folderId }), + ...(input.folderId !== undefined && input.recursive ? { recursive: true } : {}), + ...(q ? { q } : {}), + ...(canonicalSet(input.types) ? { types: canonicalSet(input.types) } : {}), + ...(canonicalSet(input.sizes) ? { sizes: canonicalSet(input.sizes) } : {}), + ...(canonicalSet(input.uploadedBy) ? { uploadedBy: canonicalSet(input.uploadedBy) } : {}), + sortBy: input.sortBy, + sortOrder: input.sortOrder, + limit: input.limit ?? WORKSPACE_FILE_LIST_PAGE_SIZE, + } +} + +/** + * The `GET /api/workspaces/[id]/files` query for one page of a list: a `null` folder is spelled + * `root`, set filters comma-joined. The client fetch and the server prefetch both send this, so + * a prefetched page is the page its key names. + */ +export function workspaceFileListQuery(params: WorkspaceFileListParams, cursor: string | null) { + return { + scope: params.scope, + ...(params.folderId === undefined + ? {} + : { folderId: params.folderId ?? 'root', recursive: params.recursive }), + q: params.q, + types: params.types?.join(','), + sizes: params.sizes?.join(','), + uploadedBy: params.uploadedBy?.join(','), + sortBy: params.sortBy, + sortOrder: params.sortOrder, + limit: params.limit, + ...(cursor ? { cursor } : {}), + } +} + +/** + * Query keys for workspace files. Every metadata read — a list page walk, a search, a folder + * rollup, a lookup, a record — is keyed under its workspace, so one file write can invalidate + * exactly that workspace's reads (`invalidateWorkspaceFileReads`). + * + * Standalone, outside the hooks module, because server prefetches seed these keys. + */ +export const workspaceFilesKeys = { + all: ['workspaceFiles'] as const, + lists: () => [...workspaceFilesKeys.all, 'list'] as const, + /** Every list in one workspace — also the scope a previous list may be carried within. */ + workspaceLists: (workspaceId: string) => [...workspaceFilesKeys.lists(), workspaceId] as const, + list: (workspaceId: string, params: WorkspaceFileListParams) => + [...workspaceFilesKeys.workspaceLists(workspaceId), params] as const, + searches: () => [...workspaceFilesKeys.all, 'search'] as const, + workspaceSearches: (workspaceId: string) => + [...workspaceFilesKeys.searches(), workspaceId] as const, + search: (workspaceId: string, params: WorkspaceFileSearchParams) => + [...workspaceFilesKeys.workspaceSearches(workspaceId), params] as const, + folderStats: () => [...workspaceFilesKeys.all, 'folderStats'] as const, + workspaceFolderStats: (workspaceId: string) => + [...workspaceFilesKeys.folderStats(), workspaceId] as const, + folderStat: (workspaceId: string, parentId: string | null) => + [...workspaceFilesKeys.workspaceFolderStats(workspaceId), parentId ?? 'root'] as const, + lookups: () => [...workspaceFilesKeys.all, 'lookup'] as const, + workspaceLookups: (workspaceId: string) => + [...workspaceFilesKeys.lookups(), workspaceId] as const, + lookup: (workspaceId: string, kind: WorkspaceFileLookupKind, value: string) => + [...workspaceFilesKeys.workspaceLookups(workspaceId), kind, value] as const, + records: () => [...workspaceFilesKeys.all, 'record'] as const, + workspaceRecords: (workspaceId: string) => + [...workspaceFilesKeys.records(), workspaceId] as const, + record: (workspaceId: string, fileId: string) => + [...workspaceFilesKeys.workspaceRecords(workspaceId), fileId] as const, + contents: () => [...workspaceFilesKeys.all, 'content'] as const, + contentFile: (workspaceId: string, fileId: string) => + [...workspaceFilesKeys.contents(), workspaceId, fileId] as const, + content: ( + workspaceId: string, + fileId: string, + mode: 'text' | 'raw' | 'binary' = 'text', + storageKey?: string + ) => + [ + ...workspaceFilesKeys.contentFile(workspaceId, fileId), + mode, + ...(storageKey ? [storageKey] : []), + ] as const, + storageInfo: () => [...workspaceFilesKeys.all, 'storageInfo'] as const, + cloudConfigured: () => [...workspaceFilesKeys.all, 'cloudConfigured'] as const, +} diff --git a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts new file mode 100644 index 00000000000..1c232da5b92 --- /dev/null +++ b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts @@ -0,0 +1,134 @@ +import { flushMicrotasks } from '@sim/testing/helpers/async' +import { createDeferred } from '@sim/testing/helpers/deferred' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { createWorkspaceFileLookupBatcher } from '@/hooks/queries/utils/workspace-file-lookup-batcher' + +interface LookupBody { + ids?: string[] + keys?: string[] + references?: string[] +} + +function record(id: string, key = `workspace/ws-1/${id}`): WorkspaceFileRecord { + return { + id, + workspaceId: 'ws-1', + name: `${id}.md`, + key, + path: `/api/files/serve/${key}`, + size: 1, + type: 'text/markdown', + uploadedBy: 'user-1', + folderId: null, + uploadedAt: new Date(0), + updatedAt: new Date(0), + } +} + +/** A lookup endpoint over `files`, answering references by id only, as the real one does first. */ +function serve(files: WorkspaceFileRecord[]) { + return vi.fn(async (_workspaceId: string, body: LookupBody) => ({ + files: files.filter( + (file) => + body.ids?.includes(file.id) || + body.keys?.includes(file.key) || + body.references?.includes(file.id) + ), + references: (body.references ?? []).map((reference) => ({ + reference, + fileId: files.find((file) => file.id === reference)?.id ?? null, + })), + })) +} + +describe('createWorkspaceFileLookupBatcher', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('sends the lookups of one task as one request per workspace, each value once', async () => { + const a = record('wf_a') + const b = record('wf_b') + const fetchLookup = serve([a, b]) + const batcher = createWorkspaceFileLookupBatcher(fetchLookup) + + const results = Promise.all([ + batcher.load('ws-1', 'id', a.id), + batcher.load('ws-1', 'id', a.id), + batcher.load('ws-1', 'key', b.key), + batcher.load('ws-1', 'reference', b.id), + batcher.load('ws-1', 'id', 'wf_missing'), + batcher.load('ws-2', 'id', a.id), + ]) + await vi.runAllTimersAsync() + + expect(await results).toEqual([a, a, b, b, null, a]) + expect(fetchLookup).toHaveBeenCalledTimes(2) + expect(fetchLookup).toHaveBeenCalledWith('ws-1', { + ids: [a.id, 'wf_missing'], + keys: [b.key], + references: [b.id], + }) + expect(fetchLookup).toHaveBeenCalledWith('ws-2', { ids: [a.id] }) + }) + + it('splits a kind past the per-request cap across requests, resolving every value', async () => { + const files = Array.from({ length: 150 }, (_, index) => record(`wf_${index}`)) + const fetchLookup = serve(files) + const batcher = createWorkspaceFileLookupBatcher(fetchLookup) + + const results = Promise.all(files.map((file) => batcher.load('ws-1', 'id', file.id))) + await vi.runAllTimersAsync() + + expect(await results).toEqual(files) + expect(fetchLookup.mock.calls.map(([, body]) => body.ids?.length)).toEqual([100, 50]) + }) + + it('drops a lookup aborted before its batch is sent, and rejects one aborted after alone', async () => { + const a = record('wf_a') + const b = record('wf_b') + const response = createDeferred<{ + files: WorkspaceFileRecord[] + references: { reference: string; fileId: string | null }[] + }>() + const fetchLookup = vi.fn(() => response.promise) + const batcher = createWorkspaceFileLookupBatcher(fetchLookup) + const early = new AbortController() + const late = new AbortController() + + const dropped = batcher.load('ws-1', 'id', a.id, early.signal) + const abandoned = batcher.load('ws-1', 'id', b.id, late.signal) + const kept = batcher.load('ws-1', 'id', b.id) + early.abort() + await expect(dropped).rejects.toBe(early.signal.reason) + await vi.runAllTimersAsync() + expect(fetchLookup).toHaveBeenCalledExactlyOnceWith('ws-1', { ids: [b.id] }) + + late.abort() + await expect(abandoned).rejects.toBe(late.signal.reason) + response.resolve({ files: [b], references: [] }) + expect(await kept).toBe(b) + }) + + it('rejects the waiters of a failed request only', async () => { + const a = record('wf_a') + const fetchLookup = vi.fn(async (workspaceId: string) => { + if (workspaceId === 'ws-down') throw new Error('lookup failed') + return { files: [a], references: [] } + }) + const batcher = createWorkspaceFileLookupBatcher(fetchLookup) + + const failed = batcher.load('ws-down', 'id', a.id) + const served = batcher.load('ws-1', 'id', a.id) + failed.catch(() => {}) + await vi.runAllTimersAsync() + await flushMicrotasks() + + await expect(failed).rejects.toThrow('lookup failed') + expect(await served).toBe(a) + }) +}) diff --git a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts new file mode 100644 index 00000000000..f3a717e292c --- /dev/null +++ b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts @@ -0,0 +1,134 @@ +import { MAX_WORKSPACE_FILE_LOOKUP_ENTRIES } from '@/lib/api/contracts/workspace-files' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import type { WorkspaceFileLookupKind } from '@/hooks/queries/utils/workspace-file-keys' + +/** One `POST /files/lookup` body: at most {@link MAX_WORKSPACE_FILE_LOOKUP_ENTRIES} of each kind. */ +interface WorkspaceFileLookupBody { + ids?: string[] + keys?: string[] + references?: string[] +} + +interface WorkspaceFileLookupResponse { + files: WorkspaceFileRecord[] + references: { reference: string; fileId: string | null }[] +} + +type FetchLookup = ( + workspaceId: string, + body: WorkspaceFileLookupBody +) => Promise<WorkspaceFileLookupResponse> + +interface PendingLookup { + kind: WorkspaceFileLookupKind + value: string + resolve: (file: WorkspaceFileRecord | null) => void + reject: (error: unknown) => void +} + +const BODY_FIELD = { id: 'ids', key: 'keys', reference: 'references' } as const + +/** Splits a workspace's pending lookups into request bodies, deduplicated and within the per-kind cap. */ +function chunkLookups(entries: PendingLookup[]): WorkspaceFileLookupBody[] { + const chunks: Record<WorkspaceFileLookupKind, Set<string>>[] = [] + for (const { kind, value } of entries) { + if (chunks.some((chunk) => chunk[kind].has(value))) continue + let chunk = chunks.find((candidate) => candidate[kind].size < MAX_WORKSPACE_FILE_LOOKUP_ENTRIES) + if (!chunk) { + chunk = { id: new Set(), key: new Set(), reference: new Set() } + chunks.push(chunk) + } + chunk[kind].add(value) + } + return chunks.map((chunk) => { + const body: WorkspaceFileLookupBody = {} + for (const kind of ['id', 'key', 'reference'] as const) { + if (chunk[kind].size > 0) body[BODY_FIELD[kind]] = [...chunk[kind]] + } + return body + }) +} + +function resolveFrom( + response: WorkspaceFileLookupResponse, + kind: WorkspaceFileLookupKind, + value: string +): WorkspaceFileRecord | null { + if (kind === 'id') return response.files.find((file) => file.id === value) ?? null + if (kind === 'key') return response.files.find((file) => file.key === value) ?? null + const fileId = response.references.find((entry) => entry.reference === value)?.fileId + return fileId ? (response.files.find((file) => file.id === fileId) ?? null) : null +} + +/** + * Coalesces individual file lookups into batched `POST /files/lookup` requests. + * + * Many surfaces resolve files one at a time — a chat full of file chips, a table column of file + * cells — and each mounts its own query. Lookups requested in the same task are collected and + * sent together after it, one request per workspace and per {@link MAX_WORKSPACE_FILE_LOOKUP_ENTRIES} + * values of a kind, so N chips cost one round trip instead of N, and none of them needs the + * workspace's whole file list. + * + * A caller that aborts before the batch is sent is dropped from it; one that aborts after is + * rejected at once while the shared request carries on for the rest. + */ +export function createWorkspaceFileLookupBatcher(fetchLookup: FetchLookup) { + let pending = new Map<string, PendingLookup[]>() + let scheduled = false + + function flush() { + scheduled = false + const batch = pending + pending = new Map() + for (const [workspaceId, entries] of batch) { + for (const body of chunkLookups(entries)) { + const inBody = (entry: PendingLookup) => + body[BODY_FIELD[entry.kind]]?.includes(entry.value) ?? false + const waiters = entries.filter(inBody) + fetchLookup(workspaceId, body).then( + (response) => { + for (const waiter of waiters) + waiter.resolve(resolveFrom(response, waiter.kind, waiter.value)) + }, + (error: unknown) => { + for (const waiter of waiters) waiter.reject(error) + } + ) + } + } + } + + return { + load( + workspaceId: string, + kind: WorkspaceFileLookupKind, + value: string, + signal?: AbortSignal + ): Promise<WorkspaceFileRecord | null> { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason) + return + } + const entry: PendingLookup = { kind, value, resolve, reject } + const entries = pending.get(workspaceId) + if (entries) entries.push(entry) + else pending.set(workspaceId, [entry]) + signal?.addEventListener( + 'abort', + () => { + const queued = pending.get(workspaceId) + const index = queued?.indexOf(entry) ?? -1 + if (queued && index !== -1) queued.splice(index, 1) + reject(signal.reason) + }, + { once: true } + ) + if (!scheduled) { + scheduled = true + setTimeout(flush, 0) + } + }) + }, + } +} diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index ecf4787ba08..f7b4df76a72 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -1,4 +1,5 @@ import { useMemo, useRef } from 'react' +import { isRecordLike } from '@sim/utils/object' import { useQueries, useQuery } from '@tanstack/react-query' import { requestJson } from '@/lib/api/client/request' import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge' @@ -15,10 +16,6 @@ import { type ListTablesResponse, listTablesContract, } from '@/lib/api/contracts/tables' -import { - type ListWorkspaceFilesResponse, - listWorkspaceFilesContract, -} from '@/lib/api/contracts/workspace-files' import { createMcpToolId } from '@/lib/mcp/shared' import type { Credential } from '@/lib/oauth' import { @@ -33,6 +30,8 @@ import { type SelectorKey, } from '@/lib/selectors/manifest' import type { SelectorOption, SelectorScope } from '@/lib/selectors/types' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { extractStorageKey } from '@/lib/uploads/utils/file' import { getWorkflowSearchMatchResourceGroupKey } from '@/lib/workflows/search-replace/resources' import type { WorkflowSearchMatch, @@ -46,6 +45,12 @@ import { } from '@/hooks/queries/oauth/credentials' import { collectDuplicateNames, disambiguateLabelByFolder } from '@/hooks/queries/utils/folder-tree' import { selectorQueryRoots } from '@/hooks/queries/utils/selector-keys' +import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' +import { + useWorkspaceFileList, + useWorkspaceFileLookups, + useWorkspaceFileSearch, +} from '@/hooks/queries/workspace-files' import type { WorkflowFolder } from '@/stores/folders/types' /** Stable identity while a folder list loads, so `select` isn't re-keyed on it. */ @@ -89,11 +94,6 @@ export const workflowSearchReplaceKeys = { [...workflowSearchReplaceKeys.tableDetails(), workspaceId ?? '', tableId ?? ''] as const, tableReplacementOptions: (workspaceId?: string) => [...workflowSearchReplaceKeys.replacementOptions(), 'table', workspaceId ?? ''] as const, - fileDetails: () => [...workflowSearchReplaceKeys.resourceDetails(), 'file'] as const, - fileListDetails: (workspaceId?: string) => - [...workflowSearchReplaceKeys.fileDetails(), 'list', workspaceId ?? ''] as const, - fileReplacementOptions: (workspaceId?: string) => - [...workflowSearchReplaceKeys.replacementOptions(), 'file', workspaceId ?? ''] as const, mcpServerDetails: () => [...workflowSearchReplaceKeys.resourceDetails(), 'mcp-server'] as const, mcpServerListDetails: (workspaceId?: string) => [...workflowSearchReplaceKeys.mcpServerDetails(), 'list', workspaceId ?? ''] as const, @@ -127,14 +127,12 @@ export const workflowSearchReplaceKeys = { export const WORKFLOW_SEARCH_OAUTH_DETAIL_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_KNOWLEDGE_DETAIL_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_TABLE_DETAIL_STALE_TIME = 60 * 1000 -export const WORKFLOW_SEARCH_FILE_LIST_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_MCP_SERVER_LIST_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_MCP_TOOL_LIST_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_SELECTOR_DETAIL_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_OAUTH_REPLACEMENT_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_KNOWLEDGE_REPLACEMENT_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_TABLE_REPLACEMENT_STALE_TIME = 60 * 1000 -export const WORKFLOW_SEARCH_FILE_REPLACEMENT_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_MCP_SERVER_REPLACEMENT_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_MCP_TOOL_REPLACEMENT_STALE_TIME = 60 * 1000 export const WORKFLOW_SEARCH_SELECTOR_REPLACEMENT_STALE_TIME = 60 * 1000 @@ -358,27 +356,32 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor ), [matches] ) - - const filesQuery = useQuery({ - queryKey: workflowSearchReplaceKeys.fileListDetails(workspaceId), - queryFn: ({ signal }: { signal: AbortSignal }) => - requestJson(listWorkspaceFilesContract, { - params: { id: workspaceId as string }, - query: { scope: 'active' }, - signal, - }), - enabled: Boolean(workspaceId && fileMatches.length > 0), - staleTime: WORKFLOW_SEARCH_FILE_LIST_STALE_TIME, + /** A stored file value is an id, a storage key, a serve path, or a name: each is looked up as each. */ + const rawValues = useMemo( + () => [...new Set(fileMatches.map((match) => match.rawValue))], + [fileMatches] + ) + const storageKeys = useMemo( + () => [...new Set(rawValues.map((value) => extractStorageKey(value)))], + [rawValues] + ) + const enabled = rawValues.length > 0 + const byId = useWorkspaceFileLookups(workspaceId ?? '', 'id', rawValues, { enabled }) + const byKey = useWorkspaceFileLookups(workspaceId ?? '', 'key', storageKeys, { enabled }) + const byReference = useWorkspaceFileLookups(workspaceId ?? '', 'reference', rawValues, { + enabled, }) + const settled = !byId.isPending && !byKey.isPending && !byReference.isPending return useMemo( () => fileMatches.map((match) => { - const file = filesQuery.data?.files.find((item) => - [item.id, item.key, item.path, item.name].includes(match.rawValue) - ) + const file = + byId.byValue.get(match.rawValue) ?? + byKey.byValue.get(extractStorageKey(match.rawValue)) ?? + byReference.byValue.get(match.rawValue) return { - data: filesQuery.data + data: settled ? { matchRawValue: match.rawValue, resourceGroupKey: match.resource?.resourceGroupKey, @@ -389,7 +392,7 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor : undefined, } }), - [fileMatches, filesQuery.data] + [fileMatches, byId.byValue, byKey.byValue, byReference.byValue, settled] ) } @@ -666,30 +669,66 @@ export function useWorkflowSearchTableReplacementOptions( }) } +/** The files a replacement can be chosen from before anything is typed: the most recently updated. */ +const RECENT_REPLACEMENT_FILES = workspaceFileListParams({ + sortBy: 'updatedAt', + sortOrder: 'desc', + limit: 100, +}) + +const REPLACEMENT_FILE_SEARCH_LIMIT = 50 + +/** The storage key a chosen file replacement value carries, if it is one. */ +function replacementFileKey(value: string): string | undefined { + try { + const parsed: unknown = JSON.parse(value) + return isRecordLike(parsed) && typeof parsed.key === 'string' ? parsed.key : undefined + } catch { + return undefined + } +} + +/** + * The files a file reference can be replaced with: those matching `options.query` (settled), + * else the most recently updated — never the workspace's whole list — plus every file already + * chosen as a replacement (`options.chosen`), so a choice stays valid after the query changes. + */ export function useWorkflowSearchFileReplacementOptions( matches: WorkflowSearchMatch[], - workspaceId?: string + workspaceId: string | undefined, + options: { query: string; chosen: readonly string[] } ) { const fileGroups = useMemo( () => uniqueResourceOptionGroups(matches, 'file', (match) => !match.resource?.selectorKey), [matches] ) - - return useQueries({ - queries: [ + const enabled = Boolean(workspaceId && fileGroups.length > 0) + const recentFiles = useWorkspaceFileList(workspaceId ?? '', RECENT_REPLACEMENT_FILES, { enabled }) + const matchingFiles = useWorkspaceFileSearch( + workspaceId ?? '', + { q: options.query, limit: REPLACEMENT_FILE_SEARCH_LIMIT }, + { enabled } + ) + const chosenKeys = useMemo( + () => + options.chosen.flatMap((value) => { + const key = replacementFileKey(value) + return key ? [key] : [] + }), + [options.chosen] + ) + const chosenFiles = useWorkspaceFileLookups(workspaceId ?? '', 'key', chosenKeys, { enabled }) + const offered = options.query.trim() ? matchingFiles.data : recentFiles.data?.pages[0]?.files + + return useMemo(() => { + const files = new Map<string, WorkspaceFileRecord>() + for (const file of chosenFiles.byValue.values()) files.set(file.id, file) + for (const file of offered ?? []) files.set(file.id, file) + return [ { - queryKey: workflowSearchReplaceKeys.fileReplacementOptions(workspaceId), - queryFn: ({ signal }: { signal: AbortSignal }) => - requestJson(listWorkspaceFilesContract, { - params: { id: workspaceId as string }, - query: { scope: 'active' }, - signal, - }), - enabled: Boolean(workspaceId && fileGroups.length > 0), - staleTime: WORKFLOW_SEARCH_FILE_REPLACEMENT_STALE_TIME, - select: (response: ListWorkspaceFilesResponse): WorkflowSearchReplacementOption[] => - fileGroups.flatMap((match) => - response.files.map((file) => ({ + data: fileGroups.flatMap((match) => + [...files.values()].map( + (file): WorkflowSearchReplacementOption => ({ kind: 'file', value: JSON.stringify({ name: file.name, @@ -700,11 +739,12 @@ export function useWorkflowSearchFileReplacementOptions( }), label: file.name, resourceGroupKey: match.resource?.resourceGroupKey, - })) - ), + }) + ) + ), }, - ], - }) + ] + }, [chosenFiles.byValue, offered, fileGroups]) } export function useWorkflowSearchMcpServerReplacementOptions( diff --git a/apps/sim/hooks/queries/workspace-file-folders.ts b/apps/sim/hooks/queries/workspace-file-folders.ts index 0ac43eb5d42..4dc928b63ff 100644 --- a/apps/sim/hooks/queries/workspace-file-folders.ts +++ b/apps/sim/hooks/queries/workspace-file-folders.ts @@ -17,7 +17,8 @@ import { parseWorkspaceFileFolderDisplayPath, } from '@/lib/workspace-files/folder-display-path' import { recentlyDeletedKeys } from '@/hooks/queries/utils/recently-deleted-keys' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' +import { invalidateWorkspaceFileReads } from '@/hooks/queries/utils/workspace-file-cache' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' type WorkspaceFileFolderScope = 'active' | 'archived' | 'all' export type { WorkspaceFileFolderApi } @@ -47,7 +48,11 @@ async function fetchWorkspaceFileFolders( return data.folders } -/** Settles once the active file-browser queries have refetched. */ +/** + * Invalidates everything a file browser shows for the workspace — its folder tree and every file + * read (list walks, searches, folder rollups, lookups, records) — and storage usage. Settles once + * the mounted ones have refetched. + */ export function invalidateWorkspaceFileBrowsers( queryClient: ReturnType<typeof useQueryClient>, workspaceId: string @@ -56,7 +61,7 @@ export function invalidateWorkspaceFileBrowsers( queryClient.invalidateQueries({ queryKey: workspaceFileFolderKeys.workspaceLists(workspaceId), }), - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }), + invalidateWorkspaceFileReads(queryClient, workspaceId), queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }), ]) } diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index c07fe6ecbba..ae6a10ef73d 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -9,6 +9,8 @@ */ import { act, type ReactNode } from 'react' +import { flushMacrotask } from '@sim/testing/helpers/async' +import { createDeferred } from '@sim/testing/helpers/deferred' import { apiClientRequestMock, apiClientRequestMockFns, @@ -18,18 +20,27 @@ import { focusManager, QueryClient, QueryClientProvider } from '@tanstack/react- import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { + listWorkspaceFilesContract, readWorkspaceFileContract, + renameWorkspaceFileContract, + searchWorkspaceFilesContract, updateWorkspaceFileContentContract, } from '@/lib/api/contracts/workspace-files' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { - useAddressedWorkspaceFileRecord, + type WorkspaceFileListParams, + workspaceFileListParams, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' +import { invalidateWorkspaceFileBrowsers } from '@/hooks/queries/workspace-file-folders' +import { useReloadWorkspaceFileContent, + useRenameWorkspaceFile, useUpdateWorkspaceFileContent, useWorkspaceFileContent, - useWorkspaceFiles, + useWorkspaceFileList, + useWorkspaceFileRecord, type WorkspaceFileContentResult, - workspaceFilesKeys, } from '@/hooks/queries/workspace-files' const mockRequestJson = apiClientRequestMockFns.mockRequestJson @@ -147,7 +158,7 @@ describe('useReloadWorkspaceFileContent', () => { }, }) root = createRoot(document.createElement('div')) - mockRequestJson.mockResolvedValue({ success: true, files: [file] }) + mockRequestJson.mockResolvedValue({ success: true, file }) function Probe() { mutation = useReloadWorkspaceFileContent() return null @@ -179,8 +190,8 @@ describe('useReloadWorkspaceFileContent', () => { mutations: { retryDelay: 0 }, }) mockRequestJson - .mockResolvedValueOnce({ success: true, files: [file] }) - .mockResolvedValueOnce({ success: true, files: [nextFile] }) + .mockResolvedValueOnce({ success: true, file }) + .mockResolvedValueOnce({ success: true, file: nextFile }) const fetchMock = vi .fn() .mockResolvedValueOnce(new Response('gone', { status: 404 })) @@ -259,7 +270,7 @@ describe('useWorkspaceFileContent stale storage key', () => { expect(fetchCount).toBe(1) expect(refetchQueries).toHaveBeenCalledWith( - { queryKey: workspaceFilesKeys.workspaceLists('ws-1') }, + expect.objectContaining({ type: 'active', predicate: expect.any(Function) }), { cancelRefetch: true } ) // The RECORD is re-resolved, never this query — re-driving the read against the same dead key @@ -287,7 +298,7 @@ describe('useWorkspaceFileContent while a superseded key is being re-resolved', mockRequestJson.mockImplementation(async () => { // First call is the initial record load; the one the 404 triggers is held open. if (++call > 1) await new Promise<void>((resolve) => (releaseRecord = resolve)) - return { success: true, files: [] } + return { success: true, file: { id: 'file-1', key: 'workspace/ws-1/123-abc-doc.md' } } }) const queryClient = new QueryClient({ @@ -297,7 +308,7 @@ describe('useWorkspaceFileContent while a superseded key is being re-resolved', let result: WorkspaceFileContentResult | undefined function Probe() { - useWorkspaceFiles('ws-1') + useWorkspaceFileRecord('ws-1', 'file-1') result = useWorkspaceFileContent('ws-1', 'file-1', 'workspace/ws-1/123-abc-doc.md', false) return null } @@ -355,7 +366,7 @@ describe('addressed file metadata fallback', () => { const container = document.createElement('div') const root = createRoot(container) function Probe() { - const record = useAddressedWorkspaceFileRecord('ws-1', 'upload-1') + const record = useWorkspaceFileRecord('ws-1', 'upload-1') return <div>{record.data?.id}</div> } try { @@ -373,10 +384,264 @@ describe('addressed file metadata fallback', () => { params: { id: 'ws-1', fileId: 'upload-1' }, signal: expect.any(AbortSignal), }) - expect(client.getQueryData(workspaceFilesKeys.list('ws-1'))).toBeUndefined() + expect(client.getQueryCache().findAll({ queryKey: workspaceFilesKeys.lists() })).toEqual([]) } finally { act(() => root.unmount()) client.clear() } }) }) + +function fileRecord(id: string, overrides: Partial<WorkspaceFileRecord> = {}): WorkspaceFileRecord { + return { + id, + workspaceId: 'ws-1', + name: `${id}.md`, + key: `workspace/ws-1/${id}.md`, + path: `/api/files/serve/workspace%2Fws-1%2F${id}.md`, + size: 1, + type: 'text/markdown', + uploadedBy: 'user-1', + folderId: null, + uploadedAt: new Date('2026-09-01T00:00:00.000Z'), + updatedAt: new Date('2026-09-01T00:00:00.000Z'), + ...overrides, + } +} + +/** Mounts `useHook` under a fresh client, exposing its latest result. */ +function mountHook<T>(useHook: () => T) { + return mountHookWith( + new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }), + useHook + ) +} + +/** Mounts `useHook` under `client`, exposing its latest result. */ +function mountHookWith<T>(client: QueryClient, useHook: () => T) { + const root: Root = createRoot(document.createElement('div')) + const state: { current?: T } = {} + function Probe() { + state.current = useHook() + return null + } + act(() => { + root.render( + <QueryClientProvider client={client}> + <Probe /> + </QueryClientProvider> + ) + }) + return { + client, + result: () => { + if (state.current === undefined) throw new Error('Hook did not render') + return state.current + }, + rerender: () => + act(() => { + root.render( + <QueryClientProvider client={client}> + <Probe /> + </QueryClientProvider> + ) + }), + unmount: () => { + act(() => root.unmount()) + client.clear() + }, + } +} + +/** Answers list pages from `pages` in order, recording each request's query. */ +function servePages(pages: { files: WorkspaceFileRecord[]; nextCursor: string | null }[]) { + const queries: Record<string, unknown>[] = [] + mockRequestJson.mockImplementation(async (contract, input) => { + if (contract !== listWorkspaceFilesContract) throw new Error('unexpected request') + const query = (input as { query: Record<string, unknown> }).query + queries.push(query) + const index = query.cursor ? Number(String(query.cursor).replace('c', '')) : 0 + return { success: true, ...pages[index] } + }) + return queries +} + +describe('useWorkspaceFileList paging', () => { + it('walks a folder a page at a time, resuming each page from the last cursor', async () => { + const queries = servePages([ + { files: [fileRecord('a')], nextCursor: 'c1' }, + { files: [fileRecord('b')], nextCursor: null }, + ]) + const params = workspaceFileListParams({ + folderId: null, + types: ['image', 'document', 'image'], + sortBy: 'name', + sortOrder: 'asc', + limit: 1, + }) + const hook = mountHook(() => useWorkspaceFileList('ws-1', params)) + try { + await vi.waitFor(() => expect(hook.result().data?.pages).toHaveLength(1)) + expect(hook.result().hasNextPage).toBe(true) + await act(async () => { + await hook.result().fetchNextPage() + }) + await vi.waitFor(() => expect(hook.result().hasNextPage).toBe(false)) + expect( + hook.result().data?.pages.flatMap((page) => page.files.map((file) => file.id)) + ).toEqual(['a', 'b']) + expect(queries).toEqual([ + expect.objectContaining({ folderId: 'root', types: 'document,image', limit: 1 }), + expect.objectContaining({ folderId: 'root', cursor: 'c1' }), + ]) + expect(queries[0]).not.toHaveProperty('cursor') + } finally { + hook.unmount() + } + }) + + it("restarts from the first page on a re-sort, showing the old rows until it lands, but never another folder's", async () => { + const release = { current: () => {} } + let call = 0 + mockRequestJson.mockImplementation(async (_contract, input) => { + const query = (input as { query: Record<string, unknown> }).query + call++ + if (call > 1) await new Promise<void>((resolve) => (release.current = resolve)) + return { + success: true, + files: [fileRecord(`${String(query.folderId)}-${String(query.sortBy)}`)], + nextCursor: null, + } + }) + let params: WorkspaceFileListParams = workspaceFileListParams({ + folderId: null, + sortBy: 'updatedAt', + sortOrder: 'desc', + }) + const hook = mountHook(() => useWorkspaceFileList('ws-1', params)) + try { + await vi.waitFor(() => expect(hook.result().data).toBeDefined()) + + params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + hook.rerender() + expect(hook.result().isPlaceholderData).toBe(true) + expect(hook.result().data?.pages[0].files[0].id).toBe('root-updatedAt') + await act(async () => release.current()) + await vi.waitFor(() => expect(hook.result().isPlaceholderData).toBe(false)) + expect(hook.result().data?.pages[0].files[0].id).toBe('root-name') + expect(mockRequestJson.mock.calls[1][1]).not.toHaveProperty('query.cursor') + + params = workspaceFileListParams({ folderId: 'folder-2', sortBy: 'name', sortOrder: 'asc' }) + hook.rerender() + expect(hook.result().data).toBeUndefined() + await act(async () => release.current()) + } finally { + hook.unmount() + } + }) +}) + +describe('workspace file records', () => { + it('opens a file from the list page it was clicked in, without a request', async () => { + servePages([{ files: [fileRecord('a')], nextCursor: null }]) + const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + const list = mountHook(() => useWorkspaceFileList('ws-1', params)) + try { + await vi.waitFor(() => expect(list.result().data).toBeDefined()) + mockRequestJson.mockClear() + const record = mountHookWith(list.client, () => useWorkspaceFileRecord('ws-1', 'a')) + expect(record.result().data?.id).toBe('a') + await flushMacrotask() + expect(mockRequestJson).not.toHaveBeenCalled() + record.unmount() + } finally { + list.unmount() + } + }) +}) + +describe('useRenameWorkspaceFile', () => { + it('renames the file in every cached read at once, and restores them all when the rename fails', async () => { + const hook = mountHook(() => useRenameWorkspaceFile()) + const { client } = hook + const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + const searchKey = workspaceFilesKeys.search('ws-1', { q: 'a', limit: 20 }) + client.setQueryData(workspaceFilesKeys.list('ws-1', params), { + pages: [{ files: [fileRecord('a'), fileRecord('b')], nextCursor: null }], + pageParams: [null], + }) + client.setQueryData(searchKey, [fileRecord('a')]) + client.setQueryData(workspaceFilesKeys.record('ws-1', 'a'), fileRecord('a')) + const names = () => [ + ( + client.getQueryData(workspaceFilesKeys.list('ws-1', params)) as { + pages: { files: WorkspaceFileRecord[] }[] + } + ).pages[0].files.map((file) => file.name), + (client.getQueryData(searchKey) as WorkspaceFileRecord[])[0].name, + (client.getQueryData(workspaceFilesKeys.record('ws-1', 'a')) as WorkspaceFileRecord).name, + ] + const rename = createDeferred<never>() + mockRequestJson.mockImplementation((contract) => + contract === renameWorkspaceFileContract ? rename.promise : new Promise(() => {}) + ) + try { + let settled: Promise<unknown> = Promise.resolve() + act(() => { + settled = hook + .result() + .mutateAsync({ workspaceId: 'ws-1', fileId: 'a', name: 'renamed.md' }) + .catch(() => {}) + }) + await vi.waitFor(() => + expect(names()).toEqual([['renamed.md', 'b.md'], 'renamed.md', 'renamed.md']) + ) + await act(async () => { + rename.reject(new Error('Name taken')) + await settled + }) + expect(names()).toEqual([['a.md', 'b.md'], 'a.md', 'a.md']) + } finally { + hook.unmount() + } + }) +}) + +describe('invalidateWorkspaceFileBrowsers', () => { + it('settles only once the mounted list and search reads have refetched', async () => { + let listCalls = 0 + let searchCalls = 0 + const refetch = createDeferred<void>() + mockRequestJson.mockImplementation(async (contract) => { + if (contract === listWorkspaceFilesContract) { + if (++listCalls > 1) await refetch.promise + return { success: true, files: [fileRecord('a')], nextCursor: null } + } + if (contract === searchWorkspaceFilesContract) { + searchCalls++ + return { success: true, files: [fileRecord('a')] } + } + return { success: true, folders: [] } + }) + const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + const hook = mountHook(() => useWorkspaceFileList('ws-1', params)) + try { + await vi.waitFor(() => expect(hook.result().data).toBeDefined()) + let done = false + const invalidation = invalidateWorkspaceFileBrowsers(hook.client, 'ws-1').then(() => { + done = true + }) + await vi.waitFor(() => expect(listCalls).toBe(2)) + await flushMacrotask() + expect(done).toBe(false) + refetch.resolve() + await invalidation + expect(done).toBe(true) + expect(searchCalls).toBe(0) + } finally { + hook.unmount() + } + }) +}) diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 2a8b2bff54c..fddeade2fa5 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -1,15 +1,19 @@ -import { useCallback, useMemo } from 'react' +import { useCallback, useMemo, useRef, useSyncExternalStore } from 'react' import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { backoffWithJitter } from '@sim/utils/retry' import { keepPreviousData, + type QueryClient, + useInfiniteQuery, useIsFetching, useMutation, + useQueries, useQuery, useQueryClient, } from '@tanstack/react-query' +import { isApiClientError } from '@/lib/api/client/errors' import { requestJson } from '@/lib/api/client/request' import { fileStorageStatusContract } from '@/lib/api/contracts/storage-transfer' import { @@ -17,171 +21,484 @@ import { createWorkspaceFileContract, deleteWorkspaceFileContract, listWorkspaceFilesContract, + lookupWorkspaceFilesContract, readWorkspaceFileContract, renameWorkspaceFileContract, restoreWorkspaceFileContract, + searchWorkspaceFilesContract, type UpdateWorkspaceFileContentBody, updateWorkspaceFileContentContract, updateWorkspaceFileDimensionsContract, + workspaceFileFolderStatsContract, } from '@/lib/api/contracts/workspace-files' import { uploadWorkspaceFileSession } from '@/lib/uploads/client/session-upload' import { FileTextSizeError, readFileText } from '@/lib/uploads/client/text-content' import type { UploadProgressEvent } from '@/lib/uploads/client/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' +import { extractStorageKey } from '@/lib/uploads/utils/file' +import type { WorkspaceFileSelection } from '@/lib/workspace-files/selection' import type { UserFile } from '@/executor/types' -import { findWorkspaceFileBySrc } from '@/hooks/queries/utils/find-workspace-file-by-src' import { recentlyDeletedKeys } from '@/hooks/queries/utils/recently-deleted-keys' +import { + findCachedWorkspaceFile, + invalidateWorkspaceFileReads, + restoreWorkspaceFileReads, + snapshotWorkspaceFileReads, + updateCachedWorkspaceFile, + workspaceFileMetadataFilter, +} from '@/hooks/queries/utils/workspace-file-cache' +import { + WORKSPACE_FILE_FOLDER_STATS_STALE_TIME, + WORKSPACE_FILE_LIST_STALE_TIME, + WORKSPACE_FILE_LOOKUP_STALE_TIME, + WORKSPACE_FILE_RECORD_STALE_TIME, + WORKSPACE_FILE_SEARCH_STALE_TIME, + type WorkspaceFileListPage, + type WorkspaceFileListParams, + type WorkspaceFileLookupKind, + type WorkspaceFileSearchParams, + workspaceFileListQuery, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' +import { createWorkspaceFileLookupBatcher } from '@/hooks/queries/utils/workspace-file-lookup-batcher' import { type ImageDimensionsSource, useFileContentSource } from '@/hooks/use-file-content-source' const logger = createLogger('WorkspaceFilesQuery') -type WorkspaceFileQueryScope = 'active' | 'archived' - -/** - * Query key factories for workspace files - */ -export const workspaceFilesKeys = { - all: ['workspaceFiles'] as const, - lists: () => [...workspaceFilesKeys.all, 'list'] as const, - workspaceLists: (workspaceId: string) => [...workspaceFilesKeys.lists(), workspaceId] as const, - list: (workspaceId: string, scope: WorkspaceFileQueryScope = 'active') => - [...workspaceFilesKeys.workspaceLists(workspaceId), scope] as const, - records: () => [...workspaceFilesKeys.all, 'record'] as const, - record: (workspaceId: string, fileId: string) => - [...workspaceFilesKeys.records(), workspaceId, fileId] as const, - contents: () => [...workspaceFilesKeys.all, 'content'] as const, - contentFile: (workspaceId: string, fileId: string) => - [...workspaceFilesKeys.contents(), workspaceId, fileId] as const, - content: ( - workspaceId: string, - fileId: string, - mode: 'text' | 'raw' | 'binary' = 'text', - storageKey?: string - ) => - [ - ...workspaceFilesKeys.contentFile(workspaceId, fileId), - mode, - ...(storageKey ? [storageKey] : []), - ] as const, - storageInfo: () => [...workspaceFilesKeys.all, 'storageInfo'] as const, - cloudConfigured: () => [...workspaceFilesKeys.all, 'cloudConfigured'] as const, -} - -const WORKSPACE_FILES_LIST_STALE_TIME = 30 * 1000 export const WORKSPACE_FILE_CONTENT_STALE_TIME = 30 * 1000 export const WORKSPACE_FILE_BINARY_STALE_TIME = 30 * 1000 /** Cloud storage (S3/Blob) is env-driven and does not change at runtime. */ export const CLOUD_STORAGE_CONFIGURED_STALE_TIME = Number.POSITIVE_INFINITY -/** - * Hook to fetch a single workspace file record by ID. - * Shares the `list(workspaceId, 'active')` query key with {@link useWorkspaceFiles} so no extra - * network request is made when the list is already cached (warm path). - * On a cold path (e.g. direct navigation to a file URL), this fetches the full active file list - * for the workspace and selects the matching record via `select`. - */ -export function useWorkspaceFileRecord(workspaceId: string, fileId: string) { - return useQuery({ - queryKey: workspaceFilesKeys.list(workspaceId, 'active'), - queryFn: ({ signal }) => fetchWorkspaceFiles(workspaceId, 'active', signal), - enabled: !!workspaceId && !!fileId, - staleTime: WORKSPACE_FILES_LIST_STALE_TIME, - select: (files) => files.find((f) => f.id === fileId) ?? null, +async function fetchWorkspaceFileListPage( + workspaceId: string, + params: WorkspaceFileListParams, + cursor: string | null, + signal?: AbortSignal +): Promise<WorkspaceFileListPage> { + const data = await requestJson(listWorkspaceFilesContract, { + params: { id: workspaceId }, + query: workspaceFileListQuery(params, cursor), + signal, }) + return { files: data.files, nextCursor: data.nextCursor ?? null } +} + +/** The options behind one paged list. */ +function getWorkspaceFileListQueryOptions(workspaceId: string, params: WorkspaceFileListParams) { + return { + queryKey: workspaceFilesKeys.list(workspaceId, params), + queryFn: ({ pageParam, signal }: { pageParam: string | null; signal?: AbortSignal }) => + fetchWorkspaceFileListPage(workspaceId, params, pageParam, signal), + initialPageParam: null as string | null, + getNextPageParam: (lastPage: WorkspaceFileListPage) => lastPage.nextCursor, + staleTime: WORKSPACE_FILE_LIST_STALE_TIME, + } } -/** Canonical read fallback for addressed resources absent from the workspace file inventory. */ -export function useAddressedWorkspaceFileRecord( +/** Whether two lists show the same place: one folder (or the same whole-workspace span). */ +function isSameListPlace(previous: unknown, next: WorkspaceFileListParams): boolean { + if (typeof previous !== 'object' || previous === null) return false + const prior: Partial<WorkspaceFileListParams> = previous + return ( + prior.scope === next.scope && + prior.folderId === next.folderId && + prior.recursive === next.recursive + ) +} + +/** + * One server-sorted, server-filtered file list, a page at a time (`fetchNextPage` loads the + * next). Build `params` with `workspaceFileListParams`. + * + * A re-sort, a re-filter, or a new search keeps showing the previous rows until the new first + * page lands, as long as the list still shows the same place. Moving to another folder never + * carries the last folder's rows over: they would read as the new folder's contents. + */ +export function useWorkspaceFileList( workspaceId: string, - fileId: string, + params: WorkspaceFileListParams, options?: { enabled?: boolean } ) { - return useQuery({ - queryKey: workspaceFilesKeys.record(workspaceId, fileId), - queryFn: async ({ signal }) => { - const result = await requestJson(readWorkspaceFileContract, { - params: { id: workspaceId, fileId }, - signal, - }) - return result.file + return useInfiniteQuery({ + ...getWorkspaceFileListQueryOptions(workspaceId, params), + enabled: Boolean(workspaceId) && (options?.enabled ?? true), + placeholderData: (previous, previousQuery) => { + const key = previousQuery?.queryKey + return key?.[2] === workspaceId && isSameListPlace(key[3], params) ? previous : undefined }, - enabled: !!workspaceId && !!fileId && (options?.enabled ?? true), - staleTime: WORKSPACE_FILES_LIST_STALE_TIME, - retry: false, }) } +async function fetchWorkspaceFileSearch( + workspaceId: string, + params: WorkspaceFileSearchParams, + signal?: AbortSignal +): Promise<WorkspaceFileRecord[]> { + const data = await requestJson(searchWorkspaceFilesContract, { + params: { id: workspaceId }, + query: { + q: params.q, + ...(params.folderId === undefined + ? {} + : { folderId: params.folderId ?? 'root', recursive: params.recursive }), + limit: params.limit, + }, + signal, + }) + return data.files +} + +/** Shared options for one name search; `params.q` must already be trimmed and non-empty. */ +export function getWorkspaceFileSearchQueryOptions( + workspaceId: string, + params: WorkspaceFileSearchParams +) { + return { + queryKey: workspaceFilesKeys.search(workspaceId, params), + queryFn: ({ signal }: { signal?: AbortSignal }) => + fetchWorkspaceFileSearch(workspaceId, params, signal), + staleTime: WORKSPACE_FILE_SEARCH_STALE_TIME, + } +} + /** - * Fetch workspace files from API + * Up to `params.limit` live files whose name contains `params.q`, by name, for pickers and + * mention menus. Disabled while the trimmed query is empty; pass an already-debounced query. */ -async function fetchWorkspaceFiles( +export function useWorkspaceFileSearch( + workspaceId: string, + params: WorkspaceFileSearchParams, + options?: { enabled?: boolean } +) { + const q = params.q.trim() + return useQuery({ + ...getWorkspaceFileSearchQueryOptions(workspaceId, { ...params, q }), + enabled: Boolean(workspaceId) && q.length > 0 && (options?.enabled ?? true), + placeholderData: keepPreviousData, + }) +} + +export interface WorkspaceFileFolderStat { + fileCount: number + sizeBytes: number +} + +async function fetchWorkspaceFileFolderStats( workspaceId: string, - scope: WorkspaceFileQueryScope = 'active', + parentId: string | null, signal?: AbortSignal -): Promise<WorkspaceFileRecord[]> { - const data = await requestJson(listWorkspaceFilesContract, { +): Promise<Map<string, WorkspaceFileFolderStat>> { + const data = await requestJson(workspaceFileFolderStatsContract, { params: { id: workspaceId }, - query: { scope }, + query: { parentId: parentId ?? 'root' }, signal, }) - return data.success ? data.files : [] + return new Map( + data.folders.map(({ folderId, fileCount, sizeBytes }) => [folderId, { fileCount, sizeBytes }]) + ) +} + +function combineFolderSizes( + results: { data: Map<string, WorkspaceFileFolderStat> | undefined }[] +): ReadonlyMap<string, WorkspaceFileFolderStat> { + if (results.length === 1) return results[0].data ?? NO_FOLDER_STATS + const merged = new Map<string, WorkspaceFileFolderStat>() + for (const result of results) { + for (const [folderId, stat] of result.data ?? []) merged.set(folderId, stat) + } + return merged +} + +const NO_FOLDER_STATS: ReadonlyMap<string, WorkspaceFileFolderStat> = new Map() + +/** + * The live file count and bytes under each child folder of every parent in `parentIds` (`null` + * for the root) — each folder's whole subtree rolled up, what a folder row shows as its size — + * merged into one map by folder id. Several parents because search results span folders. + */ +export function useWorkspaceFileFolderSizes( + workspaceId: string, + parentIds: readonly (string | null)[], + options?: { enabled?: boolean } +): ReadonlyMap<string, WorkspaceFileFolderStat> { + const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + return useQueries({ + queries: parentIds.map((parentId) => ({ + queryKey: workspaceFilesKeys.folderStat(workspaceId, parentId), + queryFn: ({ signal }: { signal?: AbortSignal }) => + fetchWorkspaceFileFolderStats(workspaceId, parentId, signal), + enabled, + staleTime: WORKSPACE_FILE_FOLDER_STATS_STALE_TIME, + })), + combine: combineFolderSizes, + }) } +const workspaceFileLookups = createWorkspaceFileLookupBatcher((workspaceId, body) => + requestJson(lookupWorkspaceFilesContract, { params: { id: workspaceId }, body }) +) + /** - * Shared options for the workspace-file list, so an imperative caller can - * `fetchQuery` the same cache entry {@link useWorkspaceFiles} populates instead - * of refetching by key and reading the result back out of the cache. + * Shared options for resolving one file by id, storage key, or reference (an id, a VFS path, + * or a bare name, resolved as the agent resolves it). Lookups requested together are sent as + * one batched request. Resolves `null` for a value no live file answers to. */ -export function getWorkspaceFilesQueryOptions( +export function getWorkspaceFileLookupQueryOptions( + queryClient: QueryClient, workspaceId: string, - scope: WorkspaceFileQueryScope = 'active' + kind: WorkspaceFileLookupKind, + value: string ) { + const cached = + kind === 'id' + ? () => findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.id === value) + : kind === 'key' + ? () => findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.key === value) + : undefined return { - queryKey: workspaceFilesKeys.list(workspaceId, scope), + queryKey: workspaceFilesKeys.lookup(workspaceId, kind, value), queryFn: ({ signal }: { signal?: AbortSignal }) => - fetchWorkspaceFiles(workspaceId, scope, signal), - staleTime: WORKSPACE_FILES_LIST_STALE_TIME, // 30 seconds - files can change frequently + workspaceFileLookups.load(workspaceId, kind, value, signal), + staleTime: WORKSPACE_FILE_LOOKUP_STALE_TIME, + initialData: cached ? () => cached()?.file : undefined, + initialDataUpdatedAt: cached ? () => cached()?.updatedAt : undefined, + } +} + +/** One file by id, storage key, or reference; `data` is `null` when nothing live answers to it. */ +export function useWorkspaceFileLookup( + workspaceId: string, + kind: WorkspaceFileLookupKind, + value: string | undefined, + options?: { enabled?: boolean } +) { + const queryClient = useQueryClient() + return useQuery({ + ...getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value ?? ''), + enabled: Boolean(workspaceId) && Boolean(value) && (options?.enabled ?? true), + }) +} + +export interface WorkspaceFileLookups { + /** Each resolved value's file; a value still loading or resolving to nothing is absent. */ + byValue: ReadonlyMap<string, WorkspaceFileRecord> + /** True while any value is still on its first load. */ + isPending: boolean +} + +function combineLookups( + values: readonly string[], + results: { data: WorkspaceFileRecord | null | undefined; isPending: boolean }[] +): WorkspaceFileLookups { + const byValue = new Map<string, WorkspaceFileRecord>() + let isPending = false + results.forEach((result, index) => { + if (result.data) byValue.set(values[index], result.data) + if (result.isPending) isPending = true + }) + return { byValue, isPending } +} + +/** + * Many files by id, storage key, or reference, sent as batched lookups and cached per value. + * `values` must keep its identity while unchanged (memoize it), or the result is rebuilt each render. + */ +export function useWorkspaceFileLookups( + workspaceId: string, + kind: WorkspaceFileLookupKind, + values: readonly string[], + options?: { enabled?: boolean } +): WorkspaceFileLookups { + const queryClient = useQueryClient() + const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + /** Stable while `values` is, so the combined result keeps its identity between renders. */ + const combine = useCallback( + (results: { data: WorkspaceFileRecord | null | undefined; isPending: boolean }[]) => + combineLookups(values, results), + [values] + ) + return useQueries({ + queries: values.map((value) => ({ + ...getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value), + enabled, + })), + combine, + }) +} + +const SELECTION_NAME_CANDIDATE_LIMIT = 50 + +export interface WorkspaceFileSelectionCandidates { + /** Every live file a selection could resolve to under `findSelectedWorkspaceFile`'s rules. */ + candidates: WorkspaceFileRecord[] + /** True while any selection's candidates are still on their first load. */ + isPending: boolean +} + +/** + * The live files persisted picker selections can resolve to, without the workspace's list: each + * selection's file by id, or — for older values — by storage key (stated, or read off the serve + * path), and for the oldest, name-only shape, every file whose name contains that name, so + * `findSelectedWorkspaceFile` can still refuse a name two files share. + */ +export function useWorkspaceFileSelectionCandidates( + workspaceId: string, + selections: readonly WorkspaceFileSelection[], + options?: { enabled?: boolean } +): WorkspaceFileSelectionCandidates { + const { ids, keys, names } = useMemo(() => { + const ids = new Set<string>() + const keys = new Set<string>() + const names = new Set<string>() + for (const selection of selections) { + if (selection.id) ids.add(selection.id) + else if (selection.key) keys.add(selection.key) + else if (selection.path) keys.add(extractStorageKey(selection.path)) + else if (selection.name.trim()) names.add(selection.name.trim()) + } + return { ids: [...ids], keys: [...keys], names: [...names] } + }, [selections]) + const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + const byId = useWorkspaceFileLookups(workspaceId, 'id', ids, { enabled }) + const byKey = useWorkspaceFileLookups(workspaceId, 'key', keys, { enabled }) + const byName = useQueries({ + queries: names.map((name) => ({ + ...getWorkspaceFileSearchQueryOptions(workspaceId, { + q: name, + limit: SELECTION_NAME_CANDIDATE_LIMIT, + }), + enabled, + })), + combine: combineSearches, + }) + const candidates = useMemo( + () => [...byId.byValue.values(), ...byKey.byValue.values(), ...byName.files], + [byId.byValue, byKey.byValue, byName.files] + ) + return { + candidates, + isPending: byId.isPending || byKey.isPending || byName.isPending, + } +} + +function combineSearches( + results: { data: WorkspaceFileRecord[] | undefined; isPending: boolean }[] +): { files: WorkspaceFileRecord[]; isPending: boolean } { + return { + files: results.flatMap((result) => result.data ?? []), + isPending: results.some((result) => result.isPending), } } +async function fetchWorkspaceFileRecord( + workspaceId: string, + fileId: string, + signal?: AbortSignal +): Promise<WorkspaceFileRecord> { + const result = await requestJson(readWorkspaceFileContract, { + params: { id: workspaceId, fileId }, + signal, + }) + return result.file +} + /** - * Hook to fetch workspace files + * Shared options for one file's canonical record — including addressed chat uploads, which no + * listing returns. Starts from the freshest copy any cached list, search, or lookup of the + * workspace already holds, so opening a file from a list costs no request until that copy ages. */ -export function useWorkspaceFiles( +export function getWorkspaceFileRecordQueryOptions( + queryClient: QueryClient, + workspaceId: string, + fileId: string +) { + const cached = () => + findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.id === fileId) + return { + queryKey: workspaceFilesKeys.record(workspaceId, fileId), + queryFn: ({ signal }: { signal?: AbortSignal }) => + fetchWorkspaceFileRecord(workspaceId, fileId, signal), + staleTime: WORKSPACE_FILE_RECORD_STALE_TIME, + initialData: () => cached()?.file, + initialDataUpdatedAt: () => cached()?.updatedAt, + retry: false, + } +} + +/** One live file's record by id; a missing or inaccessible file settles as an error. */ +export function useWorkspaceFileRecord( workspaceId: string, - scope: WorkspaceFileQueryScope = 'active', + fileId: string, options?: { enabled?: boolean } ) { + const queryClient = useQueryClient() return useQuery({ - ...getWorkspaceFilesQueryOptions(workspaceId, scope), - enabled: !!workspaceId && (options?.enabled ?? true), - placeholderData: keepPreviousData, // Show cached data immediately + ...getWorkspaceFileRecordQueryOptions(queryClient, workspaceId, fileId), + enabled: Boolean(workspaceId) && Boolean(fileId) && (options?.enabled ?? true), }) } /** - * Back the file content source's image-dimension capability with workspace file metadata. Subscribes to - * the active file list ({@link useWorkspaceFiles}) and reads each image's stored intrinsic dimensions from - * it, so a stored image reserves its box before it downloads. A reactive read (not a one-shot - * `getQueryData`), so it also works on a cold direct file-view load where the list isn't cached until after - * the image first renders: the subscription re-runs the node view's dimension read once the list resolves. - * Persists the browser's measured dimensions when they're absent or disagree with what's stored — an - * overwrite, so a stale value (left over after a content swap, or a non-EXIF-corrected one) self-corrects - * rather than sticking. The write is fire-and-forget and de-duped (an exact-match cache check plus - * mismatch-only reporting from the caller), so it never storms, never blocks render, and never touches the - * collaborative document. `options.enabled` turns the subscription off for callers that supply their own - * content source (the public share page, whose `workspaceId` is a share token that would 404). + * Back the file content source's image-dimension capability with workspace file metadata. Each + * embedded image is resolved by the storage key or file id its `src` carries — from any cached + * read of the file, else one batched lookup — so a stored image reserves its box before it + * downloads without the workspace's whole file list. The adapter takes a new identity whenever + * a lookup of this workspace lands, which re-runs the node views' dimension reads: on a cold + * direct file-view load the first read misses and the second, once the lookup resolves, hits. + * + * Persists the browser's measured dimensions when they're absent or disagree with what's stored — + * an overwrite, so a stale value (left over after a content swap, or a non-EXIF-corrected one) + * self-corrects rather than sticking. The write is fire-and-forget and de-duped (an exact-match + * cache check plus mismatch-only reporting from the caller), so it never storms, never blocks + * render, and never touches the collaborative document. `options.enabled` turns it off for + * callers that supply their own content source (the public share page, whose `workspaceId` is + * a share token that would 404). */ export function useWorkspaceImageDimensionsAdapter( workspaceId: string, options?: { enabled?: boolean } ): ImageDimensionsSource { const queryClient = useQueryClient() - const { data: files } = useWorkspaceFiles(workspaceId, 'active', options) + const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + const versionRef = useRef(0) + const subscribe = useCallback( + (onChange: () => void) => + queryClient.getQueryCache().subscribe((event) => { + if ( + event.type === 'updated' && + event.action.type === 'success' && + event.query.queryKey[1] === 'lookup' && + event.query.queryKey[2] === workspaceId + ) { + versionRef.current += 1 + onChange() + } + }), + [queryClient, workspaceId] + ) + const version = useSyncExternalStore( + subscribe, + () => versionRef.current, + () => 0 + ) + return useMemo<ImageDimensionsSource>(() => { - const listKey = workspaceFilesKeys.list(workspaceId, 'active') - const findRecord = (src: string | undefined): WorkspaceFileRecord | undefined => - findWorkspaceFileBySrc(files, src) + /** The cached record for `src`, warming its lookup (outside render) on a miss. */ + const findRecord = (src: string | undefined): WorkspaceFileRecord | undefined => { + const ref = enabled && src ? extractEmbeddedFileRef(src) : null + if (!ref) return undefined + const [kind, value] = + 'key' in ref ? (['key', ref.key] as const) : (['id', ref.fileId] as const) + const lookup = getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value) + const resolved = queryClient.getQueryData<WorkspaceFileRecord | null>(lookup.queryKey) + if (resolved !== undefined) return resolved ?? undefined + const cached = lookup.initialData?.() + if (cached) return cached + queueMicrotask(() => void queryClient.prefetchQuery(lookup)) + return undefined + } return { getImageDimensions: (src) => { const record = findRecord(src) @@ -197,10 +514,11 @@ export function useWorkspaceImageDimensionsAdapter( // on a real mismatch, so we overwrite to self-correct. if (!record || (record.width === dimensions.width && record.height === dimensions.height)) return - // Populate the cache so this and sibling views reserve space immediately. - queryClient.setQueryData<WorkspaceFileRecord[]>(listKey, (previous) => - previous?.map((entry) => (entry.id === record.id ? { ...entry, ...dimensions } : entry)) - ) + // Populate every cached read so this and sibling views reserve space immediately. + updateCachedWorkspaceFile(queryClient, workspaceId, record.id, (entry) => ({ + ...entry, + ...dimensions, + })) void requestJson(updateWorkspaceFileDimensionsContract, { params: { id: workspaceId, fileId: record.id }, // Send the key we measured against; the server rejects the write if the row's content (key) has @@ -212,15 +530,16 @@ export function useWorkspaceImageDimensionsAdapter( // our optimistic patch is now for superseded bytes, so refetch to reconcile the cache with the // new content (its real size is persisted when the replaced image next loads). Do NOT re-send // this measurement: it's of the old bytes and would write the wrong size under the new key. - if (!response.success) void queryClient.invalidateQueries({ queryKey: listKey }) + if (!response.success) void invalidateWorkspaceFileReads(queryClient, workspaceId) }) // A transport error / 403 for a read-only member leaves the optimistic value in place: the - // measurement is the real displayed size, correct whether or not it persisted; a later list + // measurement is the real displayed size, correct whether or not it persisted; a later // refetch reconciles it. .catch(() => {}) }, } - }, [files, queryClient, workspaceId]) + // `version` is the dependency that matters: a new identity is what re-runs the node views' reads. + }, [enabled, queryClient, workspaceId, version]) } /** @@ -240,19 +559,25 @@ class StaleStorageKeyError extends Error { } } +/** The mounted metadata reads of a workspace — whichever one handed the reader its storage key. */ +function activeWorkspaceFileReads(workspaceId: string) { + return { ...workspaceFileMetadataFilter(workspaceId), type: 'active' as const } +} + /** * Re-resolve a workspace's file records after a read found its storage key superseded. * - * The key a content read addresses comes from the cached file list, and nothing invalidates that - * list when a write rotates the key — the collaborative relay projects an open document back to - * markdown every few seconds, entirely server-side, so an open tab's key can be replaced many times - * over without a single client-visible event. Recovering on the 404 makes the rotation cost exactly - * one request instead of stranding the reader on a dead key until a full reload. + * The key a content read addresses comes from a cached record — the file's own, or the list, + * search, or lookup the reader opened it from — and nothing invalidates those when a write rotates + * the key: the collaborative relay projects an open document back to markdown every few seconds, + * entirely server-side, so an open tab's key can be replaced many times over without a single + * client-visible event. Recovering on the 404 makes the rotation cost one refetch of the mounted + * reads instead of stranding the reader on a dead key until a full reload. * - * Re-reads the RECORD, never the failed query: the refetched list either hands back a new key — which - * re-keys the read onto a fresh cache entry that fetches once — or the same one, in which case nothing - * refetches and the failure stands. That asymmetry is what makes this loop-proof, and it is why a - * genuinely deleted file settles instead of retrying: the list simply stops containing it. + * Re-reads the RECORD, never the failed query: the refetched record either hands back a new key — + * which re-keys the read onto a fresh cache entry that fetches once — or the same one, in which case + * nothing refetches and the failure stands. That asymmetry is what makes this loop-proof, and it is why + * a genuinely deleted file settles instead of retrying: its record stops resolving. * * Both details below exist because this runs from inside the failing read's own `queryFn`, and each was * measured: without them the recovery is requested and no fetch happens at all, so the reader is left @@ -271,10 +596,7 @@ function useStaleKeyRecovery(workspaceId: string): (error: unknown) => void { // `cancelRefetch` because a re-resolution has to OBSERVE the rotation: a record read already in // flight was started before it, so it can only hand back the key we already know is dead. void Promise.resolve().then(() => - queryClient.refetchQueries( - { queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }, - { cancelRefetch: true } - ) + queryClient.refetchQueries(activeWorkspaceFileReads(workspaceId), { cancelRefetch: true }) ) }, [queryClient, workspaceId] @@ -374,9 +696,7 @@ function useStaleKeyRecoveryState( isLoading: boolean, error: unknown ): { isLoading: boolean; error: Error | null } { - const resolvingRecord = useIsFetching({ - queryKey: workspaceFilesKeys.workspaceLists(workspaceId), - }) + const resolvingRecord = useIsFetching(workspaceFileMetadataFilter(workspaceId)) const recovering = error instanceof StaleStorageKeyError && resolvingRecord > 0 return { isLoading: isLoading || recovering, @@ -510,6 +830,14 @@ export function useCloudStorageConfigured(enabled = true) { }) } +/** Every metadata read of the workspace plus storage usage, after a write that changes files. */ +function invalidateAfterFileWrite(queryClient: QueryClient, workspaceId: string) { + return Promise.all([ + invalidateWorkspaceFileReads(queryClient, workspaceId), + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }), + ]) +} + /** * Upload workspace file mutation */ @@ -564,10 +892,7 @@ export function useUploadWorkspaceFile() { uploadWorkspaceFile(workspaceId, file, folderId, onProgress, signal), onSettled: (_data, _error, variables) => { if (variables.skipInvalidation) return - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + void invalidateAfterFileWrite(queryClient, variables.workspaceId) }, onSuccess: (_data, variables) => { if (!variables.skipToast) { @@ -600,10 +925,7 @@ export function useCreateWorkspaceFile() { body, }), onSettled: (_data, _error, variables) => { - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + void invalidateAfterFileWrite(queryClient, variables.workspaceId) }, onError: (error) => { logger.error('Failed to create file:', error) @@ -635,13 +957,7 @@ export function useUpdateWorkspaceFileContent() { queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.contentFile(variables.workspaceId, variables.fileId), }) - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.record(variables.workspaceId, variables.fileId), - }) - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + void invalidateAfterFileWrite(queryClient, variables.workspaceId) }, onError: (error) => { logger.error('Failed to update file content:', error) @@ -663,13 +979,16 @@ export function useReloadWorkspaceFileContent() { fileId: string raw: boolean }) => { - await queryClient.cancelQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }) - const files = await queryClient.fetchQuery({ - ...getWorkspaceFilesQueryOptions(workspaceId), - staleTime: 0, - }) - const file = files.find((record) => record.id === fileId) - if (!file) throw new Error('File no longer exists') + const recordOptions = getWorkspaceFileRecordQueryOptions(queryClient, workspaceId, fileId) + await queryClient.cancelQueries({ queryKey: recordOptions.queryKey }) + const file = await queryClient + .fetchQuery({ ...recordOptions, staleTime: 0 }) + .catch((error: unknown) => { + if (isApiClientError(error) && error.status === 404) { + throw new Error('File no longer exists') + } + throw error + }) if (!file.contentUpdatedAt) throw new Error('The latest file version is unavailable') const content = await queryClient.fetchQuery({ queryKey: workspaceFilesKeys.content(workspaceId, fileId, raw ? 'raw' : 'text', file.key), @@ -703,31 +1022,17 @@ export function useRenameWorkspaceFile() { body: { name }, }), onMutate: async ({ workspaceId, fileId, name }) => { - await queryClient.cancelQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }) - const previous = queryClient.getQueryData<WorkspaceFileRecord[]>( - workspaceFilesKeys.list(workspaceId, 'active') - ) - if (previous) { - queryClient.setQueryData<WorkspaceFileRecord[]>( - workspaceFilesKeys.list(workspaceId, 'active'), - previous.map((f) => (f.id === fileId ? { ...f, name } : f)) - ) - } + await queryClient.cancelQueries(workspaceFileMetadataFilter(workspaceId)) + const previous = snapshotWorkspaceFileReads(queryClient, workspaceId) + updateCachedWorkspaceFile(queryClient, workspaceId, fileId, (file) => ({ ...file, name })) return { previous } }, - onError: (error, variables, context) => { - if (context?.previous) { - queryClient.setQueryData( - workspaceFilesKeys.list(variables.workspaceId, 'active'), - context.previous - ) - } + onError: (error, _variables, context) => { + if (context) restoreWorkspaceFileReads(queryClient, context.previous) toast.error(error.message, { duration: 5000 }) }, onSettled: (_data, _error, variables) => { - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) + void invalidateWorkspaceFileReads(queryClient, variables.workspaceId) }, }) } @@ -749,42 +1054,24 @@ export function useDeleteWorkspaceFile() { params: { id: workspaceId, fileId }, }), onMutate: async ({ workspaceId, fileId }) => { - await queryClient.cancelQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }) - - const previousFiles = queryClient.getQueryData<WorkspaceFileRecord[]>( - workspaceFilesKeys.list(workspaceId, 'active') - ) - - if (previousFiles) { - queryClient.setQueryData<WorkspaceFileRecord[]>( - workspaceFilesKeys.list(workspaceId, 'active'), - previousFiles.filter((f) => f.id !== fileId) - ) - } - - return { previousFiles } + await queryClient.cancelQueries(workspaceFileMetadataFilter(workspaceId)) + const previous = snapshotWorkspaceFileReads(queryClient, workspaceId) + updateCachedWorkspaceFile(queryClient, workspaceId, fileId, () => null) + return { previous } }, - onError: (_err, variables, context) => { - if (context?.previousFiles) { - queryClient.setQueryData( - workspaceFilesKeys.list(variables.workspaceId, 'active'), - context.previousFiles - ) - } + onError: (error, _variables, context) => { + if (context) restoreWorkspaceFileReads(queryClient, context.previous) logger.error('Failed to delete file') - toast.error(toError(_err).message) + toast.error(toError(error).message) }, onSuccess: () => { toast.success('File moved to trash') }, onSettled: (_data, _error, variables) => { - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) + void invalidateAfterFileWrite(queryClient, variables.workspaceId) queryClient.removeQueries({ queryKey: workspaceFilesKeys.contentFile(variables.workspaceId, variables.fileId), }) - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) }, }) } @@ -805,10 +1092,7 @@ export function useRestoreWorkspaceFile() { }, onSettled: (_data, _error, variables) => { queryClient.invalidateQueries({ queryKey: recentlyDeletedKeys.lists() }) - queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), - }) - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + void invalidateAfterFileWrite(queryClient, variables.workspaceId) }, }) } diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 756967fb2b3..d2ef95aa856 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -266,8 +266,6 @@ const listWorkspaceFilesResponseSchema = workspaceFilesResponseSchema.extend({ nextCursor: z.string().nullable().optional(), }) -export type ListWorkspaceFilesResponse = z.output<typeof listWorkspaceFilesResponseSchema> - export const extractWorkspaceFileResponseSchema = workspaceFileSuccessSchema.extend({ folderName: z.string(), extractedCount: z.number().int().nonnegative(), @@ -310,7 +308,8 @@ export const searchWorkspaceFilesContract = defineRouteContract({ response: { mode: 'json', schema: workspaceFilesResponseSchema }, }) -const MAX_WORKSPACE_FILE_LOOKUP_ENTRIES = 100 +/** The most entries each of a lookup's `ids`, `keys`, and `references` may carry. */ +export const MAX_WORKSPACE_FILE_LOOKUP_ENTRIES = 100 const lookupEntriesSchema = (field: string) => z diff --git a/apps/sim/lib/mothership/tools/client/resource-display.ts b/apps/sim/lib/mothership/tools/client/resource-display.ts index 71a85258c41..b50ca5d864b 100644 --- a/apps/sim/lib/mothership/tools/client/resource-display.ts +++ b/apps/sim/lib/mothership/tools/client/resource-display.ts @@ -17,9 +17,13 @@ import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { mcpKeys } from '@/hooks/queries/utils/mcp-keys' import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workflowKeys } from '@/hooks/queries/utils/workflow-keys' +import { + findCachedWorkspaceFile, + isWorkspaceFileMetadataKey, +} from '@/hooks/queries/utils/workspace-file-cache' +import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' import { workspaceKeys } from '@/hooks/queries/workspace' import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' -import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' export interface ToolResourceContext { workspaceId?: string @@ -41,11 +45,6 @@ function inventoryKeys(type: NamedResource, workspaceId: string): readonly Query knowledgeKeys.countedList(workspaceId), knowledgeKeys.list(workspaceId, 'archived'), ] - case 'file': - return [ - workspaceFilesKeys.list(workspaceId), - workspaceFilesKeys.list(workspaceId, 'archived'), - ] case 'filefolder': return [workspaceFileFolderKeys.list(workspaceId)] case 'folder': @@ -100,6 +99,12 @@ export function resolveResourceDisplayName( const fallback = title && title !== resourceId ? title : undefined const workspaceId = context.workspaceId ?? resource?.workspaceId if (!workspaceId && type !== 'workspace') return fallback + if (type === 'file' && workspaceId) { + return ( + findCachedWorkspaceFile(getQueryClient(), workspaceId, (file) => file.id === resourceId)?.file + .name ?? fallback + ) + } for (const key of inventoryKeys(type, workspaceId ?? '')) { for (const row of inventoryRows(getQueryClient().getQueryData(key))) { if (!isRecordLike(row) || row.id !== resourceId) continue @@ -113,12 +118,14 @@ export function resolveResourceDisplayName( /** Only successful inventory writes should refresh labels, never chat or tool-stream writes. */ export function isResourceNameQuery(key: QueryKey, workspaceId?: string): boolean { if (key[0] === workspaceKeys.all[0]) return key[1] === 'list' || key[1] === 'adminList' + if (key[0] === workspaceFilesKeys.all[0]) { + return isWorkspaceFileMetadataKey(key, workspaceId ?? String(key[2])) + } return ( [ workflowKeys.all[0], tableKeys.all[0], knowledgeKeys.all[0], - workspaceFilesKeys.all[0], workspaceFileFolderKeys.all[0], folderKeys.all[0], skillsKeys.all[0], diff --git a/apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts b/apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts index 50c748d0632..91b64d2a2a1 100644 --- a/apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts +++ b/apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts @@ -14,7 +14,6 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock) import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager' -import { WORKSPACE_FILE_SEED_MAX } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files' const ownerId = generateId() const workspaceId = generateId() @@ -121,17 +120,15 @@ describe('workspace file list in PostgreSQL', () => { * With every sorting plan ruled out, an ordered plan exists only if an index's predicate is * implied by the list's filter and its key gives the list's order. */ - it('reads the live list, whole or bounded, in order from its workspace index', async () => { - for (const limit of [undefined, WORKSPACE_FILE_SEED_MAX + 1]) { - const nodes = await explainCaptured( - db, - () => listWorkspaceFiles(workspaceId, { limit, throwOnError: true }), - { match: /from "workspace_files"/, gucs: { enable_sort: 'off' } } - ) - expect(nodes.map((node) => node['Node Type'])).not.toContain('Sort') - expect(nodes.map((node) => node['Index Name'])).toContain( - 'workspace_files_workspace_active_uploaded_idx' - ) - } + it('reads the live list in order from its workspace index', async () => { + const nodes = await explainCaptured( + db, + () => listWorkspaceFiles(workspaceId, { throwOnError: true }), + { match: /from "workspace_files"/, gucs: { enable_sort: 'off' } } + ) + expect(nodes.map((node) => node['Node Type'])).not.toContain('Sort') + expect(nodes.map((node) => node['Index Name'])).toContain( + 'workspace_files_workspace_active_uploaded_idx' + ) }) }) diff --git a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts index f412655be9d..24a3fb9646b 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts @@ -166,7 +166,7 @@ import type { UserFile } from '@/executor/types' const logger = createLogger('WorkspaceFileStorage') -export type WorkspaceFileScope = 'active' | 'archived' | 'all' +type WorkspaceFileScope = 'active' | 'archived' | 'all' /** * An {@link OrchestrationError} so every surface reaches 409 by class rather than by @@ -265,12 +265,6 @@ interface ListWorkspaceFilesOptions { hydrateFolderPaths?: boolean /** Propagate storage errors when an incomplete list would be unsafe. */ throwOnError?: boolean - /** - * Row cap for callers that only need to know whether the workspace fits a budget. - * The result is a prefix of the full list, so a caller that reads it as "the - * workspace's files" must not set this. - */ - limit?: number } /** @@ -1810,13 +1804,11 @@ export async function listWorkspaceFiles( options?: ListWorkspaceFilesOptions ): Promise<WorkspaceFileRecord[]> { try { - const { scope = 'active', limit } = options ?? {} - const query = db + const files = await db .select(workspaceFileListColumns) .from(workspaceFiles) - .where(workspaceFileScopeCondition(workspaceId, scope)) + .where(workspaceFileScopeCondition(workspaceId, options?.scope ?? 'active')) .orderBy(workspaceFiles.uploadedAt, workspaceFiles.id) - const files = await (limit === undefined ? query : query.limit(limit)) return await hydrateWorkspaceFilePaths(files, workspaceId, options) } catch (error) { diff --git a/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts b/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts index 6a24cb4b6cf..473583ca76f 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts @@ -26,10 +26,7 @@ vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) vi.mock('@/lib/uploads/contexts/workspace/file-folder-manager', () => workspaceFileFoldersMock) -import { - listWorkspaceFiles, - queryWorkspaceFiles, -} from '@/lib/uploads/contexts/workspace/file-manager' +import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager' uploadsMockFns.mockGetServePathPrefix.mockImplementation(() => '/api/files/serve/s3/') @@ -144,16 +141,3 @@ describe('queryWorkspaceFiles', () => { * stays bounded: the projection, and the optional row cap its one budgeted caller * (the workspace layout's server seed) uses to detect an oversized workspace. */ -describe('listWorkspaceFiles', () => { - beforeEach(() => { - resetDbChainMock() - }) - - it('caps the rows read when the caller only needs to fit a budget', async () => { - queueTableRows(schemaMock.workspaceFiles, [buildRow()]) - - await listWorkspaceFiles(WS, { limit: 2 }) - - expect(dbChainMockFns.limit).toHaveBeenCalledWith(2) - }) -}) diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index 805788d3ef9..be43f05152d 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -109,20 +109,6 @@ export function isImageFileType(mimeType: string): boolean { return IMAGE_MIME_TYPES.has(mimeType.toLowerCase()) } -/** - * Check if a MIME type is an audio type - */ -export function isAudioFileType(mimeType: string): boolean { - return getContentType(mimeType) === 'audio' -} - -/** - * Check if a MIME type is a video type - */ -export function isVideoFileType(mimeType: string): boolean { - return getContentType(mimeType) === 'video' -} - /** * Convert a file buffer to base64 */ diff --git a/apps/sim/lib/workspace-files/queries.test.ts b/apps/sim/lib/workspace-files/queries.test.ts deleted file mode 100644 index 1712afc0340..00000000000 --- a/apps/sim/lib/workspace-files/queries.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { publicSharesMock, publicSharesMockFns } from '@sim/testing/mocks/public-shares.mock' -import { - workspaceFileManagerMock, - workspaceFileManagerMockFns, -} from '@sim/testing/mocks/workspace-file-manager.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/public-shares/manager', () => publicSharesMock) -vi.mock('@/lib/uploads/contexts/workspace/file-manager', () => workspaceFileManagerMock) - -import { listWorkspaceFilesWithShares } from '@/lib/workspace-files/queries' - -const mockGetWorkspaceShares = publicSharesMockFns.mockGetWorkspaceShares - -const mockListWorkspaceFiles = workspaceFileManagerMockFns.mockListWorkspaceFiles - -const STORED_FILE = { - id: 'file-1', - workspaceId: 'ws-1', - name: 'notes.md', - key: 'ws-1/notes.md', - path: '/notes.md', - size: 12, - type: 'text/markdown', - uploadedBy: 'user-1', - folderId: null, - uploadedAt: new Date('2026-01-01T00:00:00.000Z'), - updatedAt: new Date('2026-01-02T00:00:00.000Z'), - contentUpdatedAt: new Date('2026-01-03T00:00:00.000Z'), - serverOnlyField: 'not part of the public contract', -} - -describe('listWorkspaceFilesWithShares', () => { - beforeEach(() => { - mockListWorkspaceFiles.mockResolvedValue([STORED_FILE]) - mockGetWorkspaceShares.mockResolvedValue(new Map()) - }) - - /** - * The route and the server prefetch both cache this under one query key, and the client - * parses the response through the same contract. A field the contract does not declare - * would sit in a hydrated entry and vanish on the first refetch. - */ - it('strips fields the response contract does not declare', async () => { - const [file] = await listWorkspaceFilesWithShares('ws-1', 'active') - - expect(file).not.toHaveProperty('serverOnlyField') - expect(file.contentUpdatedAt).toEqual(STORED_FILE.contentUpdatedAt) - expect(file.id).toBe('file-1') - expect(file.uploadedAt).toEqual(new Date('2026-01-01T00:00:00.000Z')) - }) - - /** - * `maxRows` exists for a caller that will only use the list if the whole workspace fits - * its payload budget, so overflow must be reported as `null` — a prefix returned here - * would be presented as the workspace's complete file list. The share read still runs - * concurrently and is discarded: the under-budget workspaces are the common case, and - * serializing the two reads to save this one would tax every normal request. - */ - it('returns null when the workspace exceeds maxRows', async () => { - mockListWorkspaceFiles.mockResolvedValue([STORED_FILE, STORED_FILE, STORED_FILE]) - - const result = await listWorkspaceFilesWithShares('ws-1', 'active', { maxRows: 2 }) - - expect(result).toBeNull() - expect(mockListWorkspaceFiles).toHaveBeenCalledWith('ws-1', { scope: 'active', limit: 3 }) - }) - - /** - * The file read swallows errors and returns `[]` by default. A caller seeding a cache - * must not receive that: an empty list would be cached as "this workspace has no files". - */ - it('propagates a failed read instead of degrading to an empty list', async () => { - await listWorkspaceFilesWithShares('ws-1', 'active', { throwOnError: true }) - - expect(mockListWorkspaceFiles).toHaveBeenCalledWith( - 'ws-1', - expect.objectContaining({ throwOnError: true }) - ) - }) -}) diff --git a/apps/sim/lib/workspace-files/queries.ts b/apps/sim/lib/workspace-files/queries.ts deleted file mode 100644 index 26a286ed913..00000000000 --- a/apps/sim/lib/workspace-files/queries.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { listWorkspaceFilesContract } from '@/lib/api/contracts/workspace-files' -import { attachShares } from '@/lib/public-shares/attach' -import { getWorkspaceShares } from '@/lib/public-shares/manager' -import { - listWorkspaceFiles, - type WorkspaceFileScope, -} from '@/lib/uploads/contexts/workspace/file-manager' - -/** - * Lists a workspace's files with each file's public share joined on, parsed through the - * `GET /api/workspaces/[id]/files` response contract so the workspace layout's server seed - * caches exactly the shape that route returns. - * - * The shared response schema keeps server-prefetched and client-fetched entries identical, - * including the content version used for conditional saves. - * - * Callers authorize the viewer against `workspaceId` first. - * - * `maxRows` bounds the result for a caller that only uses the list if the whole workspace - * fits a payload budget: on overflow it returns `null` rather than the prefix, so a - * truncated read is never presented as the workspace's files. The two reads still run - * concurrently, since under-budget workspaces are the common case. - * - * `throwOnError` propagates a failed file read instead of degrading to an empty list, - * which a cache seed would store as authoritative. - */ -export async function listWorkspaceFilesWithShares( - workspaceId: string, - scope: WorkspaceFileScope, - options?: { maxRows?: number; throwOnError?: boolean } -) { - const maxRows = options?.maxRows - const [files, shares] = await Promise.all([ - listWorkspaceFiles(workspaceId, { - scope, - limit: maxRows === undefined ? undefined : maxRows + 1, - throwOnError: options?.throwOnError, - }), - getWorkspaceShares('file', workspaceId), - ]) - if (maxRows !== undefined && files.length > maxRows) return null - - return listWorkspaceFilesContract.response.schema.shape.files.parse(attachShares(files, shares)) -} diff --git a/packages/testing/src/mocks/file-utils.mock.ts b/packages/testing/src/mocks/file-utils.mock.ts index 06202ff3c38..46b634c04c1 100644 --- a/packages/testing/src/mocks/file-utils.mock.ts +++ b/packages/testing/src/mocks/file-utils.mock.ts @@ -498,8 +498,6 @@ export const fileUtilsMockFns = { mockGetContentType: vi.fn(getContentType), mockIsSupportedFileType: vi.fn((mimeType: string) => mimeType.toLowerCase() in MIME_TYPE_MAPPING), mockIsImageFileType: vi.fn((mimeType: string) => getContentType(mimeType) === 'image'), - mockIsAudioFileType: vi.fn((mimeType: string) => getContentType(mimeType) === 'audio'), - mockIsVideoFileType: vi.fn((mimeType: string) => getContentType(mimeType) === 'video'), mockBufferToBase64: vi.fn((buffer: Buffer) => buffer.toString('base64')), mockCreateFileContent: vi.fn((fileBuffer: Buffer, mimeType: string) => createFileContentFromBase64(fileBuffer.toString('base64'), mimeType) @@ -713,8 +711,6 @@ export const fileUtilsMock = { getContentType: fileUtilsMockFns.mockGetContentType, isSupportedFileType: fileUtilsMockFns.mockIsSupportedFileType, isImageFileType: fileUtilsMockFns.mockIsImageFileType, - isAudioFileType: fileUtilsMockFns.mockIsAudioFileType, - isVideoFileType: fileUtilsMockFns.mockIsVideoFileType, bufferToBase64: fileUtilsMockFns.mockBufferToBase64, createFileContent: fileUtilsMockFns.mockCreateFileContent, createFileContentFromBase64: fileUtilsMockFns.mockCreateFileContentFromBase64, diff --git a/scripts/check-source-conventions.baseline.json b/scripts/check-source-conventions.baseline.json index 55220dc068b..e2b45dc1e08 100644 --- a/scripts/check-source-conventions.baseline.json +++ b/scripts/check-source-conventions.baseline.json @@ -69,7 +69,6 @@ "apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.test.ts": 2, "apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts": 5, "apps/sim/app/workspace/[workspaceId]/home/page.tsx": 2, - "apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts": 1, "apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/dashboard.tsx": 1, "apps/sim/app/workspace/[workspaceId]/logs/logs.tsx": 2, "apps/sim/app/workspace/[workspaceId]/logs/utils.test.ts": 1, diff --git a/scripts/check-unused-exports.baseline.json b/scripts/check-unused-exports.baseline.json index 4eebc887450..8add1756970 100644 --- a/scripts/check-unused-exports.baseline.json +++ b/scripts/check-unused-exports.baseline.json @@ -1131,8 +1131,6 @@ "apps/sim/hooks/queries/workflow-mcp-servers.ts#WORKFLOW_MCP_SERVERS_LIST_STALE_TIME", "apps/sim/hooks/queries/workflow-mcp-servers.ts#WORKFLOW_MCP_SERVER_DETAIL_STALE_TIME", "apps/sim/hooks/queries/workflow-mcp-servers.ts#WORKFLOW_MCP_TOOLS_STALE_TIME", - "apps/sim/hooks/queries/workflow-search-replace.ts#WORKFLOW_SEARCH_FILE_LIST_STALE_TIME", - "apps/sim/hooks/queries/workflow-search-replace.ts#WORKFLOW_SEARCH_FILE_REPLACEMENT_STALE_TIME", "apps/sim/hooks/queries/workflow-search-replace.ts#WORKFLOW_SEARCH_KNOWLEDGE_DETAIL_STALE_TIME", "apps/sim/hooks/queries/workflow-search-replace.ts#WORKFLOW_SEARCH_KNOWLEDGE_REPLACEMENT_STALE_TIME", "apps/sim/hooks/queries/workflow-search-replace.ts#WORKFLOW_SEARCH_MCP_SERVER_LIST_STALE_TIME", @@ -3923,7 +3921,6 @@ "apps/sim/lib/api/contracts/workspace-file-table.ts#workspaceCsvPreviewParamsSchema", "apps/sim/lib/api/contracts/workspace-files.ts#ExportWorkspaceFileSnapshotBody", "apps/sim/lib/api/contracts/workspace-files.ts#UpdateWorkspaceFileDimensionsBody", - "apps/sim/lib/api/contracts/workspace-files.ts#listWorkspaceFilesQuerySchema", "apps/sim/lib/api/contracts/workspace-files.ts#renameWorkspaceFileErrorSchema", "apps/sim/lib/api/contracts/workspace-files.ts#updateWorkspaceFileDimensionsBodySchema", "apps/sim/lib/api/contracts/workspace-files.ts#workspaceFilesParamsSchema", From da95a834b799c81fcfb37f39bc7d18e0b2be61fc Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 19:53:35 -0700 Subject: [PATCH 03/15] fix(files): settle deleted files, refused lookups, and stale saved filters in the paged browser Review fixes on top of the paged file reads: - A record that 404s now resolves `null` instead of erroring. An error kept the copy the record was opened from (seeded from a list page), so a file deleted while open stayed on screen in the Files editor and the mothership tab. `/files/<id>` for a missing id now falls back to the list instead of an empty one, and test and release views say the file is not found. - A lookup value the endpoint would refuse (empty, or over 1024 characters) resolves `null` on its own instead of failing every other lookup batched with it. - Picker selection candidates are distinct by id: the same file found by two reads made a unique legacy name look shared. - The table's load-more sentinel re-arms after each load settles, and the Files list counts a background refetch as loading, so reaching the bottom mid-refetch no longer stalls the scroll. - A remembered Type or Size filter value the URL parser drops is dropped on restore too; kept, it never matched the URL and held the page on its loading skeleton. - Only live list pages seed lookups and records; a chip whose id lookup failed still falls back to its path. --- .../release-entry/release-entry.tsx | 4 +- .../components/resource/resource.tsx | 10 ++++- .../workspace/[workspaceId]/files/files.tsx | 22 +++++----- .../[workspaceId]/files/search-params.ts | 1 + .../components/special-tags/special-tags.tsx | 5 ++- .../resource-content/resource-content.tsx | 2 +- .../mothership-view/mothership-view.tsx | 2 +- .../tests/components/test-detail.tsx | 6 ++- .../components/file-upload/file-upload.tsx | 7 ++- .../queries/utils/workspace-file-cache.ts | 10 +++-- .../workspace-file-lookup-batcher.test.ts | 16 +++++++ .../utils/workspace-file-lookup-batcher.ts | 10 ++++- .../hooks/queries/workspace-files.test.tsx | 25 +++++++++++ apps/sim/hooks/queries/workspace-files.ts | 43 ++++++++++++------- .../use-resource-list-preferences.test.tsx | 19 ++++++++ .../hooks/use-resource-list-preferences.ts | 6 ++- apps/sim/lib/api/contracts/workspace-files.ts | 8 +++- .../stores/resource-list-preferences/types.ts | 6 +++ 18 files changed, 160 insertions(+), 42 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx b/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx index d0bcf51eb1a..7da50b531ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx +++ b/apps/sim/app/workspace/[workspaceId]/changelog/components/release-entry/release-entry.tsx @@ -95,9 +95,9 @@ export function ReleaseEntry({ workspaceId, release, canEdit, onEditDetails }: R ))} </div> - {file.error ? ( + {file.error || file.data === null ? ( <p role='alert' className='text-[var(--text-error)] text-small'> - {file.error.message} + {file.error?.message ?? 'File not found'} </p> ) : file.data ? ( <div diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx index 4de6fcc2960..54694858846 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx @@ -291,7 +291,15 @@ const ResourceTable = memo(function ResourceTable({ emptyState, }: ResourceTableProps) { const scrollRef = useRef<HTMLDivElement>(null) - const loadMoreRef = useLoadMoreSentinel({ onLoadMore, enabled: Boolean(hasMore) }) + /** + * Re-armed whenever a load settles: an observer only reports a change in intersection, so a + * sentinel still on screen after the rows it revealed (or a request it was refused during) + * would otherwise never ask again. + */ + const loadMoreRef = useLoadMoreSentinel({ + onLoadMore, + enabled: Boolean(hasMore) && !isLoadingMore, + }) const showEmptyState = rows.length === 0 && Boolean(emptyState) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 6d3f01d70b6..502a7fc153e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -485,8 +485,16 @@ function FilesContent() { const isSearching = isSearchingResources(debouncedSearchTerm) const isFilteringFiles = typeFilter.length > 0 || sizeFilter.length > 0 || uploadedByFilter.length > 0 + /** The open file, from the list it was clicked in when that is cached, else its own record. */ + const selectedFileRecord = useWorkspaceFileRecord(workspaceId, fileIdFromRoute ?? '', { + enabled: fileIdFromRoute !== null, + }) + const selectedFile = fileIdFromRoute ? selectedFileRecord.data : null + /** A route naming no live file falls back to the list, which is then what is shown. */ + const isRoutedFileMissing = + fileIdFromRoute !== null && (selectedFileRecord.data === null || selectedFileRecord.isError) /** A remembered sort or filter is applied before the list is read, so it is read once. */ - const isListOpen = fileIdFromRoute === null && isListPreferenceReady + const isListOpen = (fileIdFromRoute === null || isRoutedFileMissing) && isListPreferenceReady /** * The open folder's files (or, while searching, matches from every folder), a server-sorted, @@ -548,12 +556,6 @@ function FilesContent() { const fileByIdRef = useRef(fileById) fileByIdRef.current = fileById - /** The open file, from the list it was clicked in when that is cached, else its own record. */ - const selectedFileRecord = useWorkspaceFileRecord(workspaceId, fileIdFromRoute ?? '', { - enabled: fileIdFromRoute !== null, - }) - const selectedFile = fileIdFromRoute ? selectedFileRecord.data : null - const loadMoreFilesRef = useRef(fileList) loadMoreFilesRef.current = fileList const loadMoreFiles = useCallback(() => { @@ -619,7 +621,7 @@ function FilesContent() { const currentFile = await queryClient.fetchQuery( getWorkspaceFileRecordQueryOptions(queryClient, workspaceId, fileId) ) - if (!isUntitledName(currentFile.name)) return + if (!currentFile || !isUntitledName(currentFile.name)) return const derived = deriveMarkdownFileName(headingText) if (!derived) return /** The folder's files whose names could collide: every `<title>…` the search can name. */ @@ -1541,7 +1543,7 @@ function FilesContent() { * 'editor' instead of waiting forever). */ const appliedModeFileIdRef = useRef<string | null>(null) - const routedFileResolved = selectedFile != null || selectedFileRecord.isError + const routedFileResolved = selectedFile != null || isRoutedFileMissing useEffect(() => { if (fileIdFromRoute === appliedModeFileIdRef.current) return const isJustCreated = @@ -2227,7 +2229,7 @@ function FilesContent() { virtualized onLoadMore={loadMoreFiles} hasMore={fileList.hasNextPage} - isLoadingMore={fileList.isFetchingNextPage} + isLoadingMore={fileList.isFetching} emptyState={ listState === 'empty' ? ( <FilesEmptyState diff --git a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts index f9cf1f1e163..5547b785ab3 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts @@ -112,6 +112,7 @@ export const filesListPreferenceConfig = { module: 'files', sortColumns: FILE_SORT_COLUMNS, filterKeys: ['type', 'size', 'uploadedBy'], + filterValues: { type: WORKSPACE_FILE_KINDS, size: WORKSPACE_FILE_SIZE_BUCKETS }, preferenceUrlKeys: filesFilterUrlKeyMap, defaultPreference: { sort: filesSortParams.default, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 18d4157d848..430fd7e6ae2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -1994,13 +1994,14 @@ function WorkspaceResourceDisplayContent({ const { data: tables = [] } = useTablesList(workspaceId) const { data: knowledgeBases = [] } = useKnowledgeBasesQuery(workspaceId) const isFile = data.type === 'file' - const { data: fileById } = useWorkspaceFileLookup(workspaceId, 'id', isFile ? data.id : undefined) + const idLookup = useWorkspaceFileLookup(workspaceId, 'id', isFile ? data.id : undefined) + const fileById = idLookup.data /** Only when the id names no file (or there is none): a path is the chip's fallback identity. */ const { data: pathMatch } = useWorkspaceFileLookup( workspaceId, 'reference', isFile ? data.path : undefined, - { enabled: !data.id || fileById === null } + { enabled: !data.id || fileById === null || idLookup.isError } ) /** The reference lookup also answers by bare name; only an exact path match identifies the file. */ const fileFromPath = findWorkspaceFileByPath(pathMatch ? [pathMatch] : undefined, data.path) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index a8656ac9e76..92ec97a48f8 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -735,7 +735,7 @@ const fileLogger = createLogger('EmbeddedFileActions') function useEmbeddedWorkspaceFile(workspaceId: string, fileId: string, filePath?: string) { const record = useWorkspaceFileRecord(workspaceId, fileId) const pathLookup = useWorkspaceFileLookup(workspaceId, 'reference', filePath, { - enabled: record.isError, + enabled: record.data === null, }) /** The reference lookup also answers by bare name; only an exact path match is this file. */ const fileAtPath = findWorkspaceFileByPath( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx index 75e94da3865..c4f55ca3292 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/mothership-view.tsx @@ -201,7 +201,7 @@ export const MothershipView = memo( active?.type === 'file' ? active.id : '', { enabled: active?.type === 'file' } ) - const activeFile = active?.type === 'file' ? activeFileRecord : undefined + const activeFile = active?.type === 'file' ? (activeFileRecord ?? undefined) : undefined const isActiveCsv = active?.type === 'file' && getFileExtension(active.title) === 'csv' const isActiveFilePreviewable = diff --git a/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx b/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx index e51a6aa6259..de2e78f9e47 100644 --- a/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tests/components/test-detail.tsx @@ -21,6 +21,8 @@ import { import { useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' import { testRunSelectionKey, useTestRunSelectionStore } from '@/stores/workflow-tests/store' +const FILE_NOT_FOUND_MESSAGE = 'File not found' + interface TestTarget { workspaceId: string name: string @@ -120,10 +122,10 @@ export function TestView({ workspaceId, name, previewMode }: TestViewProps) { }) const error = query.error ?? file.error - if (error) { + if (error || file.data === null) { return ( <p role='alert' className='p-6 text-[var(--text-error)] text-small'> - {error.message} + {error?.message ?? FILE_NOT_FOUND_MESSAGE} </p> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx index d0f82899867..8401c7e01de 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx @@ -470,8 +470,13 @@ export function FileUpload({ /** The selected files, resolved on their own: they need not be on the page the picker shows. */ const selections = useWorkspaceFileSelectionCandidates(isPreview ? '' : workspaceId, filesArray) + /** Distinct by id: the same file on the page and among the candidates is one file. */ const workspaceFiles = useMemo( - () => [...selections.candidates, ...pickerFiles], + () => [ + ...new Map( + [...selections.candidates, ...pickerFiles].map((file) => [file.id, file]) + ).values(), + ], [selections.candidates, pickerFiles] ) diff --git a/apps/sim/hooks/queries/utils/workspace-file-cache.ts b/apps/sim/hooks/queries/utils/workspace-file-cache.ts index 02fec4346da..a35dd8aeefa 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-cache.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-cache.ts @@ -1,3 +1,4 @@ +import { isRecordLike } from '@sim/utils/object' import type { InfiniteData, QueryClient, QueryKey } from '@tanstack/react-query' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { @@ -15,7 +16,7 @@ const METADATA_READS = ['list', 'search', 'lookup', 'record'] as const type ListData = InfiniteData<WorkspaceFileListPage, string | null> type SearchData = WorkspaceFileRecord[] type LookupData = WorkspaceFileRecord | null -type RecordData = WorkspaceFileRecord +type RecordData = WorkspaceFileRecord | null /** Whether `queryKey` is one of {@link METADATA_READS} for `workspaceId`. */ export function isWorkspaceFileMetadataKey(queryKey: QueryKey, workspaceId: string): boolean { @@ -55,12 +56,15 @@ function filesOf(queryKey: QueryKey, data: unknown): readonly WorkspaceFileRecor if (!data) return [] switch (queryKey[1]) { case 'list': - return (data as ListData).pages.flatMap((page) => page.files) + /** Only a live list seeds a live read: an archived one holds deleted files. */ + return isRecordLike(queryKey[3]) && queryKey[3].scope === 'active' + ? (data as ListData).pages.flatMap((page) => page.files) + : [] case 'search': return data as SearchData case 'lookup': case 'record': - return [data as RecordData] + return [data as WorkspaceFileRecord] default: return [] } diff --git a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts index 1c232da5b92..351506718b0 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts @@ -114,6 +114,22 @@ describe('createWorkspaceFileLookupBatcher', () => { expect(await kept).toBe(b) }) + it('answers a value the endpoint would refuse with no file, keeping it out of the shared request', async () => { + const a = record('wf_a') + const fetchLookup = serve([a]) + const batcher = createWorkspaceFileLookupBatcher(fetchLookup) + + const results = Promise.all([ + batcher.load('ws-1', 'key', 'x'.repeat(1025)), + batcher.load('ws-1', 'reference', ''), + batcher.load('ws-1', 'id', a.id), + ]) + await vi.runAllTimersAsync() + + expect(await results).toEqual([null, null, a]) + expect(fetchLookup).toHaveBeenCalledExactlyOnceWith('ws-1', { ids: [a.id] }) + }) + it('rejects the waiters of a failed request only', async () => { const a = record('wf_a') const fetchLookup = vi.fn(async (workspaceId: string) => { diff --git a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts index f3a717e292c..8e1aa38f800 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.ts @@ -1,4 +1,7 @@ -import { MAX_WORKSPACE_FILE_LOOKUP_ENTRIES } from '@/lib/api/contracts/workspace-files' +import { + MAX_WORKSPACE_FILE_LOOKUP_ENTRIES, + MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH, +} from '@/lib/api/contracts/workspace-files' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import type { WorkspaceFileLookupKind } from '@/hooks/queries/utils/workspace-file-keys' @@ -110,6 +113,11 @@ export function createWorkspaceFileLookupBatcher(fetchLookup: FetchLookup) { reject(signal.reason) return } + /** No file answers to a value the endpoint refuses; batched, it would fail its neighbours. */ + if (value.length === 0 || value.length > MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH) { + resolve(null) + return + } const entry: PendingLookup = { kind, value, resolve, reject } const entries = pending.get(workspaceId) if (entries) entries.push(entry) diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index ae6a10ef73d..f080e219628 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -19,6 +19,7 @@ import { sleep } from '@sim/utils/helpers' import { focusManager, QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ApiClientError } from '@/lib/api/client/errors' import { listWorkspaceFilesContract, readWorkspaceFileContract, @@ -562,6 +563,30 @@ describe('workspace file records', () => { }) }) +describe('a file deleted while open', () => { + it('reads as gone once its record 404s, not as the list copy it was opened from', async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + client.setQueryData(workspaceFilesKeys.list('ws-1', params), { + pages: [{ files: [fileRecord('a')], nextCursor: null }], + pageParams: [null], + }) + mockRequestJson.mockRejectedValue( + new ApiClientError({ message: 'File not found', status: 404, body: {} }) + ) + const record = mountHookWith(client, () => useWorkspaceFileRecord('ws-1', 'a')) + try { + expect(record.result().data?.id).toBe('a') + await act(async () => { + await client.invalidateQueries({ queryKey: workspaceFilesKeys.record('ws-1', 'a') }) + }) + await vi.waitFor(() => expect(record.result().data).toBeNull()) + } finally { + record.unmount() + } + }) +}) + describe('useRenameWorkspaceFile', () => { it('renames the file in every cached read at once, and restores them all when the rename fails', async () => { const hook = mountHook(() => useRenameWorkspaceFile()) diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index fddeade2fa5..a50ad01e257 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -373,8 +373,16 @@ export function useWorkspaceFileSelectionCandidates( })), combine: combineSearches, }) + /** Distinct by id: a file two reads found is one candidate, or a unique name reads as shared. */ const candidates = useMemo( - () => [...byId.byValue.values(), ...byKey.byValue.values(), ...byName.files], + () => [ + ...new Map( + [...byId.byValue.values(), ...byKey.byValue.values(), ...byName.files].map((file) => [ + file.id, + file, + ]) + ).values(), + ], [byId.byValue, byKey.byValue, byName.files] ) return { @@ -392,16 +400,22 @@ function combineSearches( } } +/** The file's record, or `null` once no live file has that id (deleted, or never existed). */ async function fetchWorkspaceFileRecord( workspaceId: string, fileId: string, signal?: AbortSignal -): Promise<WorkspaceFileRecord> { - const result = await requestJson(readWorkspaceFileContract, { - params: { id: workspaceId, fileId }, - signal, - }) - return result.file +): Promise<WorkspaceFileRecord | null> { + try { + const result = await requestJson(readWorkspaceFileContract, { + params: { id: workspaceId, fileId }, + signal, + }) + return result.file + } catch (error) { + if (isApiClientError(error) && error.status === 404) return null + throw error + } } /** @@ -427,7 +441,10 @@ export function getWorkspaceFileRecordQueryOptions( } } -/** One live file's record by id; a missing or inaccessible file settles as an error. */ +/** + * One live file's record by id. `data` is `null` once the file is gone — a 404 is an answer, not + * a failure, so a file deleted while open does not keep showing the record it was opened from. + */ export function useWorkspaceFileRecord( workspaceId: string, fileId: string, @@ -981,14 +998,8 @@ export function useReloadWorkspaceFileContent() { }) => { const recordOptions = getWorkspaceFileRecordQueryOptions(queryClient, workspaceId, fileId) await queryClient.cancelQueries({ queryKey: recordOptions.queryKey }) - const file = await queryClient - .fetchQuery({ ...recordOptions, staleTime: 0 }) - .catch((error: unknown) => { - if (isApiClientError(error) && error.status === 404) { - throw new Error('File no longer exists') - } - throw error - }) + const file = await queryClient.fetchQuery({ ...recordOptions, staleTime: 0 }) + if (!file) throw new Error('File no longer exists') if (!file.contentUpdatedAt) throw new Error('The latest file version is unavailable') const content = await queryClient.fetchQuery({ queryKey: workspaceFilesKeys.content(workspaceId, fileId, raw ? 'raw' : 'text', file.key), diff --git a/apps/sim/hooks/use-resource-list-preferences.test.tsx b/apps/sim/hooks/use-resource-list-preferences.test.tsx index 1a4ce80b2cc..fdbe4eb97c5 100644 --- a/apps/sim/hooks/use-resource-list-preferences.test.tsx +++ b/apps/sim/hooks/use-resource-list-preferences.test.tsx @@ -143,6 +143,25 @@ describe('useResourceListPreferences', () => { expect(useResourceListPreferencesStore.getState().preferences).toEqual({}) }) + it('restores a saved filter without the values its URL parser drops, so the list becomes ready', async () => { + seedPreference({ + sort: { column: 'name', direction: 'asc' }, + filters: { type: ['document', 'spreadsheet'], size: ['huge'], uploadedBy: [] }, + }) + const applyPreference = vi.fn() + const result = renderPreferenceHook({ preference: defaultPreference, applyPreference }) + await flushEffects() + + const restored: ResourceListPreference = { + sort: { column: 'name', direction: 'asc' }, + filters: { type: ['document'], size: [], uploadedBy: [] }, + } + expect(applyPreference).toHaveBeenCalledWith(restored) + result.rerender({ preference: restored, applyPreference }) + await flushEffects() + expect(result.current.isReady).toBe(true) + }) + it('keeps URL commits working when localStorage writes fail', async () => { const applyPreference = vi.fn() const result = renderPreferenceHook({ preference: defaultPreference, applyPreference }) diff --git a/apps/sim/hooks/use-resource-list-preferences.ts b/apps/sim/hooks/use-resource-list-preferences.ts index b78963a7920..faca27a40ee 100644 --- a/apps/sim/hooks/use-resource-list-preferences.ts +++ b/apps/sim/hooks/use-resource-list-preferences.ts @@ -58,7 +58,11 @@ function normalizePreference( return { sort: { ...preference.sort }, filters: Object.fromEntries( - config.filterKeys.map((key) => [key, [...preference.filters[key]]]) + config.filterKeys.map((key) => { + const allowed = config.filterValues?.[key] + const values = preference.filters[key] + return [key, allowed ? values.filter((value) => allowed.includes(value)) : [...values]] + }) ), } } diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index d2ef95aa856..9c73515a3fa 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -311,10 +311,16 @@ export const searchWorkspaceFilesContract = defineRouteContract({ /** The most entries each of a lookup's `ids`, `keys`, and `references` may carry. */ export const MAX_WORKSPACE_FILE_LOOKUP_ENTRIES = 100 +/** The longest id, key, or reference a lookup accepts. */ +export const MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH = 1024 + const lookupEntriesSchema = (field: string) => z .array( - z.string().min(1, `${field} entries cannot be empty`).max(1024, `${field} entry is too long`) + z + .string() + .min(1, `${field} entries cannot be empty`) + .max(MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH, `${field} entry is too long`) ) .max( MAX_WORKSPACE_FILE_LOOKUP_ENTRIES, diff --git a/apps/sim/stores/resource-list-preferences/types.ts b/apps/sim/stores/resource-list-preferences/types.ts index a3a564d3a68..30d70e12c7c 100644 --- a/apps/sim/stores/resource-list-preferences/types.ts +++ b/apps/sim/stores/resource-list-preferences/types.ts @@ -21,6 +21,12 @@ export interface ResourceListPreferenceConfig { module: ResourceListModule sortColumns: readonly string[] filterKeys: readonly string[] + /** + * The values a filter accepts, for filters whose URL parser admits only a closed set. A + * remembered value outside it is dropped on restore, as the URL parser would drop it — kept, + * it could never match the URL state it was applied to. + */ + filterValues?: Readonly<Record<string, readonly string[]>> preferenceUrlKeys?: Readonly<Record<string, string>> defaultPreference: ResourceListPreference } From 8ba63c52b07dd9363bce5da8c0ec5c9220c5b19c Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 20:02:36 -0700 Subject: [PATCH 04/15] feat(files): sort the paged list by type and owner, and rank file search like the palette MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two server capabilities the Files browser and the Cmd+K palette had on the client before the list was paged, so neither is lost: - `sortBy=type` orders by the label the Type column shows: one `workspaceFileTypeLabel` shared by the column and spelled in SQL (`workspaceFileTypeLabelExpression`) from the same label table and MIME resolution. `sortBy=owner` orders by the uploader's workspace member name, with a former member's files last in both directions. Both break ties by file name A→Z whichever way the column runs, as the browser always did, then by id. Their cursors carry the sort value like every other sort, and the key order honours each key's own direction. Neither key is indexed, so a page sorts its scope (one folder's rows, reached through the folder index) rather than seeking. - `GET /files/search` now finds what the palette's matcher found, ranked: name equal to the query, then prefix, then containing it or all its words in any order, then its letters in order from a word start (`qrep` → Quarterly-Report) or a fuzzy match (pg_trgm `<%` word similarity, so a typo still finds the file), then files under any folder whose name matches the same way — the palette matched a file's folder path. Within a tier, closer word similarity first, then by name. Same limit. Substring, word, and fuzzy predicates are served by the trigram name index; the initials pattern is checked per file of the workspace. --- .../api/workspaces/[id]/files/search/route.ts | 22 +- apps/sim/lib/api/contracts/workspace-files.ts | 20 +- .../contexts/workspace/file-manager.ts | 290 +++++++++++++++--- .../file-listing.integration.ts | 216 ++++++++++++- .../application/list-workspace-files.ts | 58 +++- apps/sim/lib/workspace-files/query-scope.ts | 46 +++ apps/sim/lib/workspace-files/type-label.ts | 36 +++ .../src/mocks/workspace-files-list.mock.ts | 8 + 8 files changed, 633 insertions(+), 63 deletions(-) create mode 100644 apps/sim/lib/workspace-files/type-label.ts diff --git a/apps/sim/app/api/workspaces/[id]/files/search/route.ts b/apps/sim/app/api/workspaces/[id]/files/search/route.ts index 2ab31c72123..bd8ea5b7d77 100644 --- a/apps/sim/app/api/workspaces/[id]/files/search/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/search/route.ts @@ -5,7 +5,7 @@ import { internalSessionAuth, } from '@/lib/api/server/routes' import { internalFileErrorPolicies, internalFilePresenters } from '@/lib/workspace-files/api' -import { listWorkspaceFileCollection } from '@/lib/workspace-files/application/list-workspace-files' +import { searchWorkspaceFiles } from '@/lib/workspace-files/application/list-workspace-files' import { fileOperations } from '@/lib/workspace-files/application/operations' export const dynamic = 'force-dynamic' @@ -14,8 +14,9 @@ export const dynamic = 'force-dynamic' const SEARCH_TYPE_AHEAD_LIMIT = { maxTokens: 120, refillRate: 120, refillIntervalMs: 60_000 } /** - * GET /api/workspaces/[id]/files/search — live files whose name contains `q`, by name, each - * carrying its `folderPath`; within `folderId` (and, by default, its subfolders) when given. + * GET /api/workspaces/[id]/files/search — live files `q` finds, best match first: by name + * (exact, prefix, substring, then fuzzy), then by the folders they sit under; each carries its + * `folderPath`. Within `folderId` (and, by default, its subfolders) when given. */ export const GET = defineInternalJsonRoute({ contract: searchWorkspaceFilesContract, @@ -28,16 +29,11 @@ export const GET = defineInternalJsonRoute({ errorPolicy: internalFileErrorPolicies.default, mapInput: ({ params, query }) => ({ workspaceId: params.id, - scope: 'active' as const, - page: { - search: query.q, - folderId: query.folderId, - recursive: query.recursive, - sortBy: 'name' as const, - sortOrder: 'asc' as const, - limit: query.limit, - }, + query: query.q, + folderId: query.folderId, + recursive: query.recursive, + limit: query.limit, }), - useCase: listWorkspaceFileCollection, + useCase: searchWorkspaceFiles, present: internalFilePresenters.successFiles, }) diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 9c73515a3fa..64502cee078 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -33,8 +33,18 @@ export const workspaceFileParamsSchema = workspaceFilesParamsSchema.extend({ fileId: z.string({ error: 'File ID is required' }).min(1, 'File ID is required'), }) -/** The fields a paged list sorts by: the v2 list's, which the shared keyset implements. */ -const WORKSPACE_FILE_SORT_FIELDS = ['name', 'size', 'uploadedAt', 'updatedAt'] as const +/** + * The fields a paged list sorts by: the v2 list's, plus the browser's Type (the label its Type + * column shows) and Owner (the uploader's member name, a non-member uploader last). + */ +const WORKSPACE_FILE_SORT_FIELDS = [ + 'name', + 'size', + 'uploadedAt', + 'updatedAt', + 'type', + 'owner', +] as const const MAX_WORKSPACE_FILE_PAGE_SIZE = 500 @@ -297,8 +307,10 @@ const searchWorkspaceFilesQuerySchema = z.object({ }) /** - * GET — case-insensitive file-name search within a workspace, optionally within one folder: - * live files whose name contains `q`, by name, each carrying `folderPath`. + * GET — palette-style file search within a workspace, optionally within one folder: live files + * `q` finds, best match first — name equal to it, starting with it, containing it or all its + * words (case-insensitive), its initials or a fuzzy match — then files under a folder whose name + * matches it. Each carries `folderPath`. */ export const searchWorkspaceFilesContract = defineRouteContract({ method: 'GET', diff --git a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts index 24a3fb9646b..eb13c358ec3 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts @@ -8,7 +8,9 @@ import { db } from '@sim/db' import { displaySegmentKey, folder, + permissions, uploadSession, + user, type WorkspaceFileRow, workspace, workspaceFiles, @@ -24,8 +26,20 @@ import { } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { omit } from '@sim/utils/object' -import { escapeLikePattern } from '@sim/utils/string' -import { and, desc, eq, inArray, isNotNull, isNull, ne, or, type SQL, sql } from 'drizzle-orm' +import { escapeLikePattern, escapeRegExp } from '@sim/utils/string' +import { + and, + desc, + eq, + inArray, + isNotNull, + isNull, + ne, + or, + type SQL, + type SQLWrapper, + sql, +} from 'drizzle-orm' import type { ShareRecord } from '@/lib/api/contracts/public-shares' import type { V2FileSortBy } from '@/lib/api/contracts/v2/files' import { @@ -34,9 +48,8 @@ import { INVALID_CURSOR_MESSAGE, type KeysetKey, keysetAfter, - keysetColumns, + keysetOrderBy, type ListSortOrder, - listOrderBy, numberKey, preciseTimestampText, resumableTimestampKey, @@ -159,6 +172,7 @@ import { workspaceFileFolderScopeCondition, workspaceFileKindCondition, workspaceFileSizeCondition, + workspaceFileTypeLabelExpression, } from '@/lib/workspace-files/query-scope' import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' import { isUuid } from '@/executor/constants' @@ -1824,10 +1838,11 @@ const TIMESTAMP_SORT_COLUMNS = { } as const /** - * A page row: the listing columns plus, under a timestamp sort, that column's full-precision - * cursor text (see {@link resumableTimestampKey}); other sorts select an empty placeholder. + * A page row's columns: the listing columns plus, under a timestamp sort, that column's + * full-precision cursor text (see {@link resumableTimestampKey}); other sorts select an empty + * placeholder. */ -function workspaceFilePageColumns(sortBy: V2FileSortBy) { +function workspaceFilePageColumns(sortBy: WorkspaceFileSortBy) { const column = sortBy === 'uploadedAt' || sortBy === 'updatedAt' ? TIMESTAMP_SORT_COLUMNS[sortBy] : null return { @@ -1836,48 +1851,129 @@ function workspaceFilePageColumns(sortBy: V2FileSortBy) { } } -type WorkspaceFilePageRow = WorkspaceFileListRow & { timestampCursor: string } +/** Every field a workspace file page can be sorted by: the v2 list's, plus the browser's Type and Owner. */ +export type WorkspaceFileSortBy = V2FileSortBy | 'type' | 'owner' + +/** + * The name of the file's uploader as the workspace's member list shows it, or `NULL` for an + * uploader no longer a member (whose Owner cell is blank). A Project has no member list, so its + * files show the user's name. + */ +function fileOwnerNameExpression(owner: EditableFileOwner): SQL<string | null> { + if (owner.entityType !== 'workspace') { + return sql< + string | null + >`(SELECT ${user.name} FROM ${user} WHERE ${user.id} = ${workspaceFiles.userId})` + } + return sql<string | null>`(SELECT ${user.name} FROM ${permissions} + INNER JOIN ${user} ON ${user.id} = ${permissions.userId} + WHERE ${permissions.entityType} = 'workspace' AND ${permissions.entityId} = ${owner.entityId} + AND ${permissions.userId} = ${workspaceFiles.userId} + LIMIT 1)` +} + +type WorkspaceFilePageRow = WorkspaceFileListRow & { + timestampCursor: string + /** The Type label or owner name a `type`/`owner` page sorts by, selected only for that sort. */ + sortLabel?: string + /** `1` when an `owner` page's file has no member uploader, which sorts last either way. */ + sortLast?: number +} /** - * The keyset behind {@link queryFileRecords}' sortable fields. The `Record` makes it total over - * the contract enum: a new sortable field fails to compile until it has a keyset here. + * The keyset behind {@link queryFileRecords}' sortable fields, and the columns it reads that the + * listing does not already select. The `Record` makes it total over the sortable fields: a new + * one fails to compile until it has a keyset here. * - * Every key column is `NOT NULL`, and `id` closes each keyset so a page boundary inside a run of - * equal names/sizes/timestamps is still stable. The name key bounds its scan so a later page - * seeks the `(owner, folder, original_name)` index instead of filtering down from its top; the - * timestamp keys order on the raw column for the upload-order indexes (see `schema.ts`). + * Every key is `NOT NULL`, and `id` closes each keyset so a page boundary inside a run of equal + * values is still stable. The name key bounds its scan so a later page seeks the + * `(owner, folder, original_name)` index instead of filtering down from its top; the timestamp + * keys order on the raw column for the upload-order indexes (see `schema.ts`). + * + * Type and Owner order as the browser always has: by the label or member name in the requested + * direction, then by file name A→Z whichever way the column runs, a file without a member owner + * last in both directions. Neither is an indexed column, so their pages sort the scoped files + * (one folder's, or a search's matches) rather than seeking an index. */ -function workspaceFileSortKeys( - sortBy: V2FileSortBy, - after: CursorKey[] | undefined -): KeysetKey<WorkspaceFilePageRow>[] { +function workspaceFileSortPlan( + sortBy: WorkspaceFileSortBy, + after: CursorKey[] | undefined, + owner: EditableFileOwner +): { keys: KeysetKey<WorkspaceFilePageRow>[]; columns: Record<string, SQL> } { const fileId = textKey<WorkspaceFilePageRow>(workspaceFiles.id, (row) => row.id) - const sorts: Record<V2FileSortBy, () => KeysetKey<WorkspaceFilePageRow>> = { - name: () => ({ - ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), - boundsScan: true, - }), + const nameAscending: KeysetKey<WorkspaceFilePageRow> = { + ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), + direction: 'asc', + } + const idAscending: KeysetKey<WorkspaceFilePageRow> = { ...fileId, direction: 'asc' } + const sortLabel = (row: WorkspaceFilePageRow) => row.sortLabel ?? '' + const single = (key: KeysetKey<WorkspaceFilePageRow>) => ({ + keys: [key, fileId], + columns: {}, + }) + const sorts: Record< + WorkspaceFileSortBy, + () => { keys: KeysetKey<WorkspaceFilePageRow>[]; columns: Record<string, SQL> } + > = { + name: () => + single({ + ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), + boundsScan: true, + }), size: () => - numberKey<WorkspaceFilePageRow>( - sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number), - (row) => getWorkspaceFileSize(row) + single( + numberKey<WorkspaceFilePageRow>( + sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number), + (row) => getWorkspaceFileSize(row) + ) ), uploadedAt: () => - resumableTimestampKey( - workspaceFiles.uploadedAt, - (row) => row.uploadedAt, - (row) => row.timestampCursor, - after + single( + resumableTimestampKey( + workspaceFiles.uploadedAt, + (row) => row.uploadedAt, + (row) => row.timestampCursor, + after + ) ), updatedAt: () => - resumableTimestampKey( - workspaceFiles.updatedAt, - (row) => row.updatedAt, - (row) => row.timestampCursor, - after + single( + resumableTimestampKey( + workspaceFiles.updatedAt, + (row) => row.updatedAt, + (row) => row.timestampCursor, + after + ) ), + type: () => ({ + keys: [ + textKey<WorkspaceFilePageRow>(workspaceFileTypeLabelExpression, sortLabel), + nameAscending, + idAscending, + ], + columns: { sortLabel: workspaceFileTypeLabelExpression }, + }), + owner: () => { + const ownerName = fileOwnerNameExpression(owner) + const missing = sql<number>`(CASE WHEN ${ownerName} IS NULL THEN 1 ELSE 0 END)`.mapWith( + Number + ) + const name = sql<string>`coalesce(${ownerName}, '')` + return { + keys: [ + { + ...numberKey<WorkspaceFilePageRow>(missing, (row) => row.sortLast ?? 0), + direction: 'asc', + }, + textKey<WorkspaceFilePageRow>(name, sortLabel), + nameAscending, + idAscending, + ], + columns: { sortLast: missing, sortLabel: name }, + } + }, } - return [sorts[sortBy](), fileId] + return sorts[sortBy]() } export interface QueryWorkspaceFilesOptions { @@ -1906,7 +2002,7 @@ export interface QueryWorkspaceFilesOptions { sizeBuckets?: readonly WorkspaceFileSizeBucket[] /** Files uploaded by any of these users; absent or empty applies no uploader filter. */ uploadedBy?: readonly string[] - sortBy: V2FileSortBy + sortBy: WorkspaceFileSortBy sortOrder: ListSortOrder limit: number /** Keyset values from a cursor, in the sort's key order. */ @@ -1974,7 +2070,7 @@ export async function queryFileRecords<const O extends EditableFileOwner>( if (folderId !== undefined && folderScope !== undefined) { throw new OrchestrationError('validation', 'Specify either folderId or folderScope, not both') } - const keys = workspaceFileSortKeys(sortBy, after) + const { keys, columns } = workspaceFileSortPlan(sortBy, after, owner) let resumeAfter: SQL | undefined if (after) { @@ -2004,10 +2100,10 @@ export async function queryFileRecords<const O extends EditableFileOwner>( ] const rows: WorkspaceFilePageRow[] = await client - .select(workspaceFilePageColumns(sortBy)) + .select({ ...workspaceFilePageColumns(sortBy), ...columns }) .from(workspaceFiles) .where(and(...conditions)) - .orderBy(...listOrderBy(keysetColumns(keys), sortOrder)) + .orderBy(...keysetOrderBy(keys, sortOrder)) .limit(limit + 1) const hasMore = rows.length > limit @@ -2405,6 +2501,118 @@ export async function lookupActiveWorkspaceFiles( } } +/** + * How a palette query can match a name, each spelled for SQL, mirroring the Cmd+K matcher the + * file list used to run in the browser: the name containing the query; for a multi-word query, + * containing every word in any order; its letters in order starting at a word boundary (`qrep` + * finds `Quarterly-Report`); and, beyond what that matcher had, a fuzzy match (`query <%` name, + * pg_trgm word similarity) so a typo still finds the file. All case-insensitive. + */ +function paletteNameMatches(column: SQLWrapper, query: string) { + const escaped = escapeLikePattern(query) + const words = query.split(/\s+/).filter(Boolean) + const letters = [...query.replace(/\s+/g, '')] + return { + exact: sql`${column} ILIKE ${escaped}`, + prefix: sql`${column} ILIKE ${`${escaped}%`}`, + contains: sql`${column} ILIKE ${`%${escaped}%`}`, + allWords: + words.length > 1 + ? and(...words.map((word) => sql`${column} ILIKE ${`%${escapeLikePattern(word)}%`}`)) + : undefined, + initials: + letters.length > 1 + ? sql`${column} ~* ${`(^|[ _/.:()-])${letters.map(escapeRegExp).join('.*')}`}` + : undefined, + fuzzy: sql`${query} <% ${column}`, + } +} + +/** + * The ids of the workspace's live file folders whose name a palette query matches (see + * {@link paletteNameMatches}), for a search that also finds files by the folders they sit in. + */ +export async function findWorkspaceFileFoldersMatching( + workspaceId: string, + query: string +): Promise<string[]> { + const matches = paletteNameMatches(folder.name, query) + const rows = await db + .select({ id: folder.id }) + .from(folder) + .where( + and( + eq(folder.workspaceId, workspaceId), + eq(folder.resourceType, 'file'), + isNull(folder.deletedAt), + or(matches.contains, matches.allWords, matches.initials, matches.fuzzy) + ) + ) + return rows.map((row) => row.id) +} + +export interface SearchWorkspaceFileRecordsOptions { + query: string + /** The folder scope searched: one id, `null` for the root, or several ids; absent, every folder. */ + folderId?: string | null | readonly string[] + /** Folders whose files match by path: files in any of them count as matches. */ + pathFolderIds: readonly string[] + limit: number +} + +/** + * Live workspace files a palette query finds (see {@link paletteNameMatches}), best match first: + * the name equal to the query, then starting with it, then containing it or every word of it, + * then its initials or a fuzzy match, then files under a folder that matched (`pathFolderIds`). + * Within a tier, closer word similarity first, then by name. + * + * Substring, word, and fuzzy predicates are the ones the `(workspace_id, original_name + * gin_trgm_ops)` index answers. An initials pattern has too few letters to index, so for a + * multi-letter query the index narrows to the workspace and the pattern is checked per file. + */ +export async function searchWorkspaceFileRecords( + workspaceId: string, + options: SearchWorkspaceFileRecordsOptions +): Promise<WorkspaceFileRecord[]> { + const { query, limit } = options + const name = workspaceFiles.originalName + const matches = paletteNameMatches(name, query) + const containsAll = or(matches.contains, matches.allWords) ?? matches.contains + const loose = or(matches.initials, matches.fuzzy) ?? matches.fuzzy + const tier = sql`(CASE + WHEN ${matches.exact} THEN 0 + WHEN ${matches.prefix} THEN 1 + WHEN ${containsAll} THEN 2 + WHEN ${loose} THEN 3 + ELSE 4 END)` + const rows = await db + .select(workspaceFileListColumns) + .from(workspaceFiles) + .where( + and( + workspaceFileScopeCondition(workspaceId, 'active'), + options.folderId === undefined + ? undefined + : workspaceFileFolderIdCondition(options.folderId), + or( + containsAll, + loose, + options.pathFolderIds.length > 0 + ? workspaceFileFolderIdCondition(options.pathFolderIds) + : undefined + ) + ) + ) + .orderBy( + tier, + sql`word_similarity(${query}, ${name}) DESC`, + workspaceFiles.originalName, + workspaceFiles.id + ) + .limit(limit) + return hydrateWorkspaceFilePaths(rows, workspaceId) +} + export interface WorkspaceFileFolderUsage { fileCount: number sizeBytes: number diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 3de77d66f38..005bd042e10 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -5,6 +5,7 @@ */ import { db } from '@sim/db' import { permissions, user, workspace, workspaceFiles } from '@sim/db/schema' +import { explainCaptured } from '@sim/db/testing/explain' import { assertDisposableTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures' import { generateId } from '@sim/utils/id' @@ -16,7 +17,10 @@ import { getWorkspaceFileFolderStats, listWorkspaceFileCollection, lookupWorkspaceFiles, + searchWorkspaceFiles, } from '@/lib/workspace-files/application/list-workspace-files' +import { workspaceFileTypeLabelExpression } from '@/lib/workspace-files/query-scope' +import { workspaceFileTypeLabel } from '@/lib/workspace-files/type-label' const alice = generateId() const bob = generateId() @@ -24,6 +28,11 @@ const carol = generateId() const workspaceId = generateId() const otherWorkspaceId = generateId() const filterWorkspaceId = generateId() +const sortWorkspaceId = generateId() +/** Uploaders with fixed display names: two members and one who has left the workspace. */ +const ann = generateId() +const ben = generateId() +const zed = generateId() const session = (userId: string) => ({ kind: 'session' as const, userId, sessionId: generateId() }) /** Folder tree in the main workspace: Docs/{Sub/{Deep}}, Media; plus one in the other workspace. */ @@ -130,9 +139,9 @@ describe('bounded workspace file reads in PostgreSQL', () => { beforeAll(async () => { assertDisposableTestDatabaseUrl(process.env.DATABASE_URL ?? '') await db.insert(user).values( - [alice, bob, carol].map((id) => ({ + [alice, bob, carol, ann, ben, zed].map((id) => ({ id, - name: `Fixture ${id}`, + name: id === ann ? 'Ann' : id === ben ? 'Ben' : id === zed ? 'Zed' : `Fixture ${id}`, email: `${id}@fixture.test`, emailVerified: true, createdAt: new Date(), @@ -143,6 +152,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { { id: workspaceId, name: 'Listing fixture', ownerId: alice, billedAccountUserId: alice }, { id: otherWorkspaceId, name: 'Other fixture', ownerId: alice, billedAccountUserId: alice }, { id: filterWorkspaceId, name: 'Filter fixture', ownerId: alice, billedAccountUserId: alice }, + { id: sortWorkspaceId, name: 'Sort fixture', ownerId: alice, billedAccountUserId: alice }, ]) await db.insert(permissions).values([ { @@ -173,6 +183,13 @@ describe('bounded workspace file reads in PostgreSQL', () => { entityId: filterWorkspaceId, permissionType: 'admin', }, + ...[alice, ann, ben].map((userId) => ({ + id: generateId(), + userId, + entityType: 'workspace', + entityId: sortWorkspaceId, + permissionType: 'admin' as const, + })), ]) await db.execute(sql` INSERT INTO folder (id, name, user_id, workspace_id, resource_type, parent_id) VALUES @@ -265,9 +282,9 @@ describe('bounded workspace file reads in PostgreSQL', () => { afterAll(async () => { await deleteWorkspaceFixture( db, - inArray(workspace.id, [workspaceId, otherWorkspaceId, filterWorkspaceId]) + inArray(workspace.id, [workspaceId, otherWorkspaceId, filterWorkspaceId, sortWorkspaceId]) ) - await db.delete(user).where(inArray(user.id, [alice, bob, carol])) + await db.delete(user).where(inArray(user.id, [alice, bob, carol, ann, ben, zed])) }) const byName = (name: string, folderId: string | null = null) => { @@ -520,6 +537,197 @@ describe('bounded workspace file reads in PostgreSQL', () => { }) }) + describe('Type and Owner sorts', () => { + const fixtures = [ + { name: 'b.pdf', contentType: 'application/pdf', userId: ben }, + { name: 'a.pdf', contentType: 'application/pdf', userId: zed }, + { name: 'd.pdf', contentType: 'application/pdf', userId: ann }, + { name: 'c.txt', contentType: 'text/plain', userId: ann }, + { name: 'x.png', contentType: 'application/octet-stream', userId: ann }, + { name: 'y.webm', contentType: '', userId: ben }, + { name: 'zz.foo', contentType: 'application/octet-stream', userId: zed }, + { name: 'noext', contentType: 'application/x-custom', userId: ben }, + { name: 'e.md', contentType: 'text/markdown', userId: ben }, + ] + const ownerNames = new Map([ + [ann, 'Ann'], + [ben, 'Ben'], + ]) + const seeded: { id: string; name: string; label: string; owner: string | null }[] = [] + + beforeAll(async () => { + for (const [index, fixture] of fixtures.entries()) { + const file = await insertFile({ + workspaceId: sortWorkspaceId, + name: fixture.name, + contentType: fixture.contentType, + userId: fixture.userId, + uploadedAt: `2026-05-01T00:00:0${index}`, + }) + seeded.push({ + id: file.id, + name: fixture.name, + label: workspaceFileTypeLabel(fixture.contentType, fixture.name), + owner: ownerNames.get(fixture.userId) ?? null, + }) + } + }) + + async function walkSorted(sortBy: 'type' | 'owner', sortOrder: 'asc' | 'desc') { + const ids: string[] = [] + let after: CursorKey[] | undefined + for (let guard = 0; guard < 20; guard++) { + const result = await page( + { folderId: null, sortBy, sortOrder, limit: 2, after }, + { workspace: sortWorkspaceId } + ) + ids.push(...result.files.map((file) => file.id)) + if (!result.nextKeys) return ids + after = result.nextKeys + } + throw new Error('Pagination did not terminate') + } + + /** + * The browser's order, collated by the database the server sorts in: the key in the + * requested direction, then the name A→Z, a file with no member owner last either way. + */ + async function expectedOrder(key: 'label' | 'owner', sortOrder: 'asc' | 'desc') { + const rows = sql.join( + seeded.map( + (file) => + sql`(${file.id}, ${key === 'label' ? file.label : file.owner}::text, ${file.name})` + ), + sql`, ` + ) + const result = await db.execute<{ id: string }>(sql` + SELECT id FROM (VALUES ${rows}) AS expected(id, key, name) + ORDER BY (key IS NULL) ASC, key ${sql.raw(sortOrder)}, name ASC, id ASC`) + return result.map((row) => row.id) + } + + it.each(['asc', 'desc'] as const)( + 'pages the Type sort by the label each file shows, ties A→Z by name (%s)', + async (sortOrder) => { + expect(await walkSorted('type', sortOrder)).toEqual(await expectedOrder('label', sortOrder)) + } + ) + + it.each(['asc', 'desc'] as const)( + 'pages the Owner sort by member name, a former member last (%s)', + async (sortOrder) => { + expect(await walkSorted('owner', sortOrder)).toEqual( + await expectedOrder('owner', sortOrder) + ) + } + ) + + it('labels each file as its Type column does', async () => { + const rows = await db.execute<{ id: string; label: string }>(sql` + SELECT id, ${workspaceFileTypeLabelExpression} AS label FROM ${workspaceFiles} + WHERE workspace_id = ${sortWorkspaceId}`) + expect(new Map(rows.map((row) => [row.id, row.label]))).toEqual( + new Map(seeded.map((file) => [file.id, file.label])) + ) + }) + + /** + * Neither key is indexed, so a page sorts its scope — but only its scope: the folder + * predicate must still seek the folder index rather than reading every live file. + */ + it("sorts one folder's rows, reached through the folder index", async () => { + const padFolder = generateId() + await db.execute(sql` + INSERT INTO folder (id, name, user_id, workspace_id, resource_type, parent_id) + VALUES (${padFolder}, 'Pad', ${ann}, ${sortWorkspaceId}, 'file', NULL)`) + await db.execute(sql` + INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, folder_id, context, original_name, content_type, size_bytes) + SELECT 'wf_sortpad_' || n, 'sortpad/' || n || '/' || ${sortWorkspaceId}, ${ann}, + ${sortWorkspaceId}, ${padFolder}, 'workspace', 'pad-' || n || '.txt', 'text/plain', 1 + FROM generate_series(1, 3000) AS n`) + await db.execute(sql`ANALYZE ${workspaceFiles}`) + try { + for (const sortBy of ['type', 'owner'] as const) { + const nodes = await explainCaptured( + db, + () => + queryWorkspaceFiles(sortWorkspaceId, { + folderId: null, + sortBy, + sortOrder: 'asc', + limit: 50, + }), + { match: /order by/ } + ) + expect(nodes.some((node) => node['Index Cond']?.includes('COALESCE(folder_id'))).toBe( + true + ) + } + } finally { + await db.execute( + sql`DELETE FROM ${workspaceFiles} WHERE id LIKE 'wf_sortpad_%' AND workspace_id = ${sortWorkspaceId}` + ) + } + }) + }) + + describe('palette search', () => { + const search = (query: string, extra: { folderId?: string | null; recursive?: boolean } = {}) => + searchWorkspaceFiles.execute({ + principal: session(alice), + input: { + workspaceId, + query, + folderId: extra.folderId, + recursive: extra.recursive ?? true, + limit: 20, + }, + }) + const names = async ( + query: string, + extra?: { folderId?: string | null; recursive?: boolean } + ) => (await search(query, extra)).files.map((file) => file.name) + + it('ranks an exact name, then a prefix, then a substring, then a fuzzy match', async () => { + expect((await names('alpha.txt')).slice(0, 2)).toEqual(['alpha.txt', 'alpha.txt']) + const ranked = await names('report') + expect(ranked[0]).toBe('report-draft.md') + expect(ranked.slice(1, 3).sort()).toEqual(['Quarterly-Report.csv', 'nested report.md']) + }) + + it('tolerates a typo in the name', async () => { + expect(await names('repot')).toEqual( + expect.arrayContaining(['report-draft.md', 'nested report.md']) + ) + expect(await names('qarterly')).toContain('Quarterly-Report.csv') + }) + + it('matches words in any order and initials from a word start, as the palette did', async () => { + expect(await names('report nested')).toContain('nested report.md') + expect(await names('qrep')).toContain('Quarterly-Report.csv') + expect(await names('uarep')).not.toContain('Quarterly-Report.csv') + }) + + it('finds the files beneath a folder the query names', async () => { + expect((await names('Sub')).sort()).toEqual(['deep.bin', 'nested report.md']) + expect(await names('media')).toEqual(['photo.png']) + const docs = await names('docs') + expect(docs.sort()).toEqual( + ['alpha.txt', 'deep.bin', 'nested report.md', 'report-draft.md'].sort() + ) + }) + + it('stays within the folder scope, and never returns deleted or chat files', async () => { + expect(await names('report', { folderId: folders.docs, recursive: false })).toEqual([ + 'report-draft.md', + ]) + const all = await names('report') + expect(all).not.toContain('old report.txt') + expect(all).not.toContain('chat report.txt') + expect(await names('report', { folderId: generateId() })).toEqual([]) + }) + }) + describe('batch lookup', () => { it('resolves own live files by id, key, and reference, and ignores the rest', async () => { const alpha = byName('alpha.txt', folders.docs) diff --git a/apps/sim/lib/workspace-files/application/list-workspace-files.ts b/apps/sim/lib/workspace-files/application/list-workspace-files.ts index aa395836f36..32784662cc5 100644 --- a/apps/sim/lib/workspace-files/application/list-workspace-files.ts +++ b/apps/sim/lib/workspace-files/application/list-workspace-files.ts @@ -17,11 +17,14 @@ import { getWorkspaceShares, getWorkspaceSharesForResources } from '@/lib/public import { aggregateWorkspaceFileFolderUsage, findListedWorkspaceFiles, + findWorkspaceFileFoldersMatching, listWorkspaceFiles, loadActiveWorkspaceContext, lookupActiveWorkspaceFiles, queryWorkspaceFiles, + searchWorkspaceFileRecords, type WorkspaceFileRecord, + type WorkspaceFileSortBy, } from '@/lib/uploads/contexts/workspace' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' @@ -55,7 +58,7 @@ export interface QueryWorkspaceFilePageInput { sizeBuckets?: readonly WorkspaceFileSizeBucket[] /** Files uploaded by any of these users. */ uploadedBy?: readonly string[] - sortBy: 'name' | 'size' | 'uploadedAt' | 'updatedAt' + sortBy: WorkspaceFileSortBy sortOrder: 'asc' | 'desc' limit: number after?: CursorKey[] @@ -221,6 +224,59 @@ export const getWorkspaceFileFolderStats = defineAuthorizedWorkspaceFileUseCase( }, }) +export interface SearchWorkspaceFilesInput { + workspaceId: string + query: string + /** The folder searched, `null` for the root; absent, the whole workspace. */ + folderId?: string | null + /** Whether `folderId`'s subfolders are searched too. */ + recursive: boolean + limit: number +} + +/** + * Live files a palette-style query finds, best match first (see `searchWorkspaceFileRecords`): + * by name — exactly, by prefix, by substring, then fuzzily — and then by the folders they sit + * under, so a query naming a folder finds the files anywhere beneath it. Each file carries its + * public share state. A folder id naming no active folder of this workspace finds nothing. + */ +export const searchWorkspaceFiles = defineAuthorizedWorkspaceFileUseCase({ + operation: fileOperations.list, + resolveContext: ({ input }: { input: SearchWorkspaceFilesInput }) => + resolveListWorkspaceFileContext(input.workspaceId), + async execute({ input, context }) { + const folderIndex = await loadFileFolderIndex(context.workspaceId) + let folderId: string | null | string[] | undefined + if (input.folderId === null) { + folderId = input.recursive ? undefined : null + } else if (input.folderId !== undefined) { + if (!folderIndex.rowById.has(input.folderId)) return { files: [] } + folderId = input.recursive + ? resolveFolderScope(folderIndex, { kind: 'folder', folderId: input.folderId }, true) + : input.folderId + } + const childrenByParent = indexFolderChildren([...folderIndex.rowById.values()]) + const pathFolderIds = new Set<string>() + for (const matched of await findWorkspaceFileFoldersMatching( + context.workspaceId, + input.query + )) { + if (!folderIndex.rowById.has(matched)) continue + pathFolderIds.add(matched) + for (const id of collectDescendantFolderIdsFrom(childrenByParent, matched)) { + pathFolderIds.add(id) + } + } + const files = await searchWorkspaceFileRecords(context.workspaceId, { + query: input.query, + folderId, + pathFolderIds: [...pathFolderIds], + limit: input.limit, + }) + return { files: await withShares(context.workspaceId, files) } + }, +}) + /** * One filtered, sorted, bounded page of a workspace's files, the folder filter given by path, * by id, or pre-resolved. The body every paged list surface shares once authorized. diff --git a/apps/sim/lib/workspace-files/query-scope.ts b/apps/sim/lib/workspace-files/query-scope.ts index e0c4b491873..f2efc0ad5f4 100644 --- a/apps/sim/lib/workspace-files/query-scope.ts +++ b/apps/sim/lib/workspace-files/query-scope.ts @@ -15,6 +15,7 @@ import { type WorkspaceFileSizeBucket, } from '@/lib/workspace-files/list-filters' import { OWNED_FILE_CONTEXTS } from '@/lib/workspace-files/owned-files' +import { WORKSPACE_FILE_TYPE_LABELS } from '@/lib/workspace-files/type-label' /** Durable files available to workspace resource pickers, reference mappings, and fork copies. */ export function activeWorkspaceFileConditions(workspaceIds: string[]) { @@ -166,3 +167,48 @@ export function workspaceFileSizeCondition( }) ) } + +/** + * A string constant of this module as an SQL literal. Only for the label and MIME tables below, + * which are code: inlining them keeps a query that repeats the expression (select, order, and + * keyset) from binding hundreds of parameters. + */ +function literal(value: string): SQL { + return sql.raw(`'${value.replaceAll("'", "''")}'`) +} + +/** A `CASE <subject> WHEN … THEN … ELSE <otherwise> END` over a constant value-to-value table. */ +function caseOf(subject: SQL, table: readonly (readonly [string, string])[], otherwise: SQL): SQL { + if (table.length === 0) return otherwise + return sql`(CASE ${subject} ${sql.join( + table.map(([when, then]) => sql`WHEN ${literal(when)} THEN ${literal(then)}`), + sql` ` + )} ELSE ${otherwise} END)` +} + +/** The MIME type `resolveEffectiveMimeType` gives the file, spelled for SQL. */ +const workspaceFileEffectiveMimeType: SQL = sql`(CASE WHEN ${declaresFormat} THEN ${declaredContentType} ELSE ${caseOf( + workspaceFileExtension, + Object.keys(EXTENSION_TO_MIME).map( + (extension) => [extension, resolveEffectiveMimeType(undefined, `file.${extension}`)] as const + ), + literal(resolveEffectiveMimeType(undefined, 'file')) +)} END)` + +const resolvedMime = sql.raw('resolved.mime') +const resolvedExtension = sql.raw('resolved.extension') + +/** + * The file's Type column label (`workspaceFileTypeLabel`), spelled for SQL: what the list's Type + * sort orders by. Built from the same label table and MIME resolution, so a file sorts under the + * label it shows. The MIME type is resolved once per row, in a scalar subquery. + */ +export const workspaceFileTypeLabelExpression: SQL<string> = sql<string>`(SELECT ${caseOf( + resolvedMime, + Object.entries(WORKSPACE_FILE_TYPE_LABELS), + sql`(CASE WHEN ${resolvedMime} LIKE 'audio/%' THEN 'Audio' + WHEN ${resolvedMime} LIKE 'video/%' THEN 'Video' + WHEN ${resolvedMime} LIKE 'image/%' THEN 'Image' + WHEN ${resolvedExtension} <> '' THEN upper(${resolvedExtension}) + ELSE ${workspaceFiles.contentType} END)` +)} FROM (SELECT ${workspaceFileEffectiveMimeType} AS mime, ${workspaceFileExtension} AS extension) AS resolved)` diff --git a/apps/sim/lib/workspace-files/type-label.ts b/apps/sim/lib/workspace-files/type-label.ts new file mode 100644 index 00000000000..57624d16a49 --- /dev/null +++ b/apps/sim/lib/workspace-files/type-label.ts @@ -0,0 +1,36 @@ +import { getFileExtension, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' + +/** The Type column's label for each MIME type it names specifically. */ +export const WORKSPACE_FILE_TYPE_LABELS: Readonly<Record<string, string>> = { + 'application/pdf': 'PDF', + 'application/zip': 'ZIP', + 'application/msword': 'Word', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'Word', + 'application/vnd.ms-excel': 'Excel', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'Excel', + 'application/vnd.ms-powerpoint': 'PowerPoint', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'PowerPoint', + 'application/json': 'JSON', + 'application/x-yaml': 'YAML', + 'text/csv': 'CSV', + 'text/plain': 'Text', + 'text/html': 'HTML', + 'text/x-sim-page': 'Page', + 'text/markdown': 'Markdown', +} + +/** + * The file's Type column label: a specific label for its effective MIME type, else its media + * family (Audio, Video, Image), else its extension uppercased, else the stored type. The list's + * Type sort orders by this same label, spelled in SQL (`workspaceFileTypeLabelExpression`). + */ +export function workspaceFileTypeLabel(storedType: string, filename: string): string { + const mimeType = resolveEffectiveMimeType(storedType, filename) + const label = WORKSPACE_FILE_TYPE_LABELS[mimeType] + if (label) return label + if (mimeType.startsWith('audio/')) return 'Audio' + if (mimeType.startsWith('video/')) return 'Video' + if (mimeType.startsWith('image/')) return 'Image' + const extension = getFileExtension(filename) + return extension ? extension.toUpperCase() : storedType +} diff --git a/packages/testing/src/mocks/workspace-files-list.mock.ts b/packages/testing/src/mocks/workspace-files-list.mock.ts index 6a2fcdd0e46..ef12a3f4a6a 100644 --- a/packages/testing/src/mocks/workspace-files-list.mock.ts +++ b/packages/testing/src/mocks/workspace-files-list.mock.ts @@ -50,6 +50,8 @@ export const workspaceFilesListMockFns = { mockLookupWorkspaceFilesAuthorize: vi.fn(), mockGetWorkspaceFileFolderStats: vi.fn(), mockGetWorkspaceFileFolderStatsAuthorize: vi.fn(), + mockSearchWorkspaceFiles: vi.fn(), + mockSearchWorkspaceFilesAuthorize: vi.fn(), } const fns = workspaceFilesListMockFns @@ -106,4 +108,10 @@ export const workspaceFilesListMock = { authorize: fns.mockGetWorkspaceFileFolderStatsAuthorize, execute: fns.mockGetWorkspaceFileFolderStats, }, + searchWorkspaceFiles: { + operation: FILES_LIST_OPERATION, + delegationAudience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + authorize: fns.mockSearchWorkspaceFilesAuthorize, + execute: fns.mockSearchWorkspaceFiles, + }, } From 19e36dfd9ea339989d968fa2d81eaef9cbaa1407 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 20:06:51 -0700 Subject: [PATCH 05/15] feat(files): offer the Type and Owner sorts again on the paged browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server now pages by both, so the Files sort menu, `sort=type|owner` links, and remembered preferences come back as they were. Folders interleave with the paged files on the same keys — "Folder" for Type, the member name for Owner — and a folder tying with a file on the column goes by name A→Z, a row without a value (a file whose uploader has left) last in both directions, as the fully loaded list always sorted. The Type label the column shows and the server sorts by is one shared function. --- .../workspace/[workspaceId]/files/files.tsx | 73 ++++++------------- .../[workspaceId]/files/list-entries.test.ts | 30 ++++++++ .../[workspaceId]/files/list-entries.ts | 29 ++++++-- .../[workspaceId]/files/search-params.ts | 10 +-- .../components/search-modal/search-modal.tsx | 3 +- .../queries/utils/workspace-file-keys.ts | 8 +- 6 files changed, 88 insertions(+), 65 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 502a7fc153e..253b8b64214 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -38,13 +38,12 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' import { formatFileSize, - getFileExtension, getMimeTypeFromExtension, isArchiveFileName, - resolveEffectiveMimeType, } from '@/lib/uploads/utils/file' import { isWorkspaceFileKind, isWorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' +import { workspaceFileTypeLabel } from '@/lib/workspace-files/type-label' import { FindBar } from '@/app/workspace/[workspaceId]/components/find-bar/find-bar' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut' import type { MoveOptionNode } from '@/app/workspace/[workspaceId]/components/folders' @@ -182,7 +181,7 @@ const FILES_HEADER = FOLDERED_RESOURCE_HEADERS.file const FOLDER_ICON = <Folder className='size-[14px]' /> -/** Folders' value in the `type` column. */ +/** Folders' value in the `type` column — also their sort key when that column is active. */ const FOLDER_TYPE_LABEL = 'Folder' as const /** @@ -207,24 +206,6 @@ const COLUMNS: ResourceColumn[] = [ */ const SEARCH_COLUMNS: ResourceColumn[] = [...COLUMNS, FOLDER_LOCATION_COLUMN] -const MIME_TYPE_LABELS: Record<string, string> = { - 'application/pdf': 'PDF', - 'application/zip': 'ZIP', - 'application/msword': 'Word', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'Word', - 'application/vnd.ms-excel': 'Excel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'Excel', - 'application/vnd.ms-powerpoint': 'PowerPoint', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'PowerPoint', - 'application/json': 'JSON', - 'application/x-yaml': 'YAML', - 'text/csv': 'CSV', - 'text/plain': 'Text', - 'text/html': 'HTML', - 'text/x-sim-page': 'Page', - 'text/markdown': 'Markdown', -} - const EMPTY_WORKSPACE_FILES: WorkspaceFileRecord[] = [] const EMPTY_WORKSPACE_FILE_FOLDERS: WorkspaceFileFolderApi[] = [] const EMPTY_FIND_MATCH_IDS: readonly string[] = Object.freeze([]) @@ -235,23 +216,6 @@ const TITLE_SIBLING_LIMIT = 50 const hasExternalFiles = (dataTransfer: DataTransfer): boolean => dataTransfer.types.includes('Files') -function formatFileType(storedType: string | null, filename: string): string { - const mimeType = resolveEffectiveMimeType(storedType, filename) - - if (MIME_TYPE_LABELS[mimeType]) { - return MIME_TYPE_LABELS[mimeType] - } - - if (mimeType.startsWith('audio/')) return 'Audio' - if (mimeType.startsWith('video/')) return 'Video' - if (mimeType.startsWith('image/')) return 'Image' - - const ext = getFileExtension(filename) - if (ext) return ext.toUpperCase() - - return storedType ?? 'File' -} - function getDroppedFiles(dataTransfer: DataTransfer): File[] { if (dataTransfer.items.length === 0) return Array.from(dataTransfer.files) const files: File[] = [] @@ -714,19 +678,27 @@ function FilesContent() { folderKey: (folder) => sortColumn === 'size' ? (folderSizes.get(folder.id)?.sizeBytes ?? 0) - : sortColumn === 'created' - ? new Date(folder.createdAt).getTime() - : sortColumn === 'updated' - ? new Date(folder.updatedAt).getTime() - : folder.name, + : sortColumn === 'type' + ? FOLDER_TYPE_LABEL + : sortColumn === 'created' + ? new Date(folder.createdAt).getTime() + : sortColumn === 'updated' + ? new Date(folder.updatedAt).getTime() + : sortColumn === 'owner' + ? (membersById.get(folder.userId)?.name ?? null) + : folder.name, fileKey: (file) => sortColumn === 'size' ? file.size - : sortColumn === 'created' - ? new Date(file.uploadedAt).getTime() - : sortColumn === 'updated' - ? new Date(file.updatedAt).getTime() - : file.name, + : sortColumn === 'type' + ? workspaceFileTypeLabel(file.type, file.name) + : sortColumn === 'created' + ? new Date(file.uploadedAt).getTime() + : sortColumn === 'updated' + ? new Date(file.updatedAt).getTime() + : sortColumn === 'owner' + ? (membersById.get(file.uploadedBy)?.name ?? null) + : file.name, direction: sortDirection, }), [ @@ -737,6 +709,7 @@ function FilesContent() { pinnedFolderIds, pinnedFileIds, folderSizes, + membersById, sortColumn, sortDirection, ] @@ -799,7 +772,7 @@ function FilesContent() { }, type: { icon: <Icon className='size-[14px]' />, - label: formatFileType(file.type, file.name), + label: workspaceFileTypeLabel(file.type, file.name), }, created: timeCell(file.uploadedAt), owner: ownerCell(file.uploadedBy, membersById), @@ -1936,8 +1909,10 @@ function FilesContent() { options: [ { id: 'name', label: 'Name' }, { id: 'size', label: 'Size' }, + { id: 'type', label: 'Type' }, { id: 'created', label: 'Created' }, { id: 'updated', label: 'Last Updated' }, + { id: 'owner', label: 'Owner' }, ], active: activeSort, onSort: setListSort, diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts index c70acabb13b..f84bdf869d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts @@ -85,4 +85,34 @@ describe('mergeFileListEntries', () => { }) ).toEqual(['pinned20', 'Fpin', 'far90', 'F5', 'a10']) }) + + it('places a folder tying on the column by name, and rows without a value last either way', () => { + const owned = (id: string, owner: string | null): Row & { owner: string | null } => ({ + ...file(id, 0), + owner, + }) + const order = (direction: 'asc' | 'desc', files: (Row & { owner: string | null })[]) => + mergeFileListEntries<Row & { owner: string | null }, Row & { owner: string | null }>({ + folders: [owned('b-folder', 'Ann'), owned('z-folder', null)], + files, + pinnedFiles: [], + hasMore: false, + isFolderPinned: () => false, + isFilePinned: () => false, + folderId: (row) => row.id, + fileId: (row) => row.id, + folderName: (row) => row.name, + fileName: (row) => row.name, + folderKey: (row) => row.owner, + fileKey: (row) => row.owner, + direction, + }).map((entry) => (entry.kind === 'folder' ? entry.folder.id : entry.file.id)) + + expect( + order('asc', [owned('a-file', 'Ann'), owned('c-file', 'Ann'), owned('y-file', null)]) + ).toEqual(['a-file', 'b-folder', 'c-file', 'y-file', 'z-folder']) + expect( + order('desc', [owned('c-file', 'Ben'), owned('a-file', 'Ann'), owned('y-file', null)]) + ).toEqual(['c-file', 'a-file', 'b-folder', 'y-file', 'z-folder']) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts index 68745c5b797..92dfc4f083b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts @@ -27,8 +27,9 @@ interface MergeFileListEntriesInput<Folder, File> { folderName: (folder: Folder) => string fileName: (file: File) => string /** Each row's value for the active sort column, comparable across folders and files. */ - folderKey: (folder: Folder) => string | number - fileKey: (file: File) => string | number + /** `null` for a row the column has no value for (a file without a member owner): last either way. */ + folderKey: (folder: Folder) => string | number | null + fileKey: (file: File) => string | number | null direction: 'asc' | 'desc' } @@ -46,8 +47,9 @@ function compareKeys(a: string | number, b: string | number): number { * loaded file is withheld while more pages remain, since the file it belongs in front of may be * on one of them; it appears once a page reaches past it, so loading more only ever appends. * - * Pinned rows are ordered among themselves by the same column, as `sortResources` orders a - * fully loaded list. + * A folder tying with a file on the column goes by name A→Z, and a row without a value goes + * last in either direction — the rules `sortResources` orders a fully loaded list by, which also + * orders the pinned rows among themselves. */ export function mergeFileListEntries<Folder, File>( input: MergeFileListEntriesInput<Folder, File> @@ -83,18 +85,29 @@ export function mergeFileListEntries<Folder, File>( const entries = sortResources(pinned, direction).map((entry) => entry.item) const folders = sortResources(unpinnedFolders, direction) - const precedes = (folderKey: string | number, fileKey: string | number) => { - const cmp = compareKeys(folderKey, fileKey) - return direction === 'asc' ? cmp <= 0 : cmp >= 0 + /** Whether `folder` sorts at or before a file whose key is `fileKey` and name is `fileName`. */ + const precedes = ( + folder: SortableResource<FileListEntry<Folder, File>>, + fileKey: string | number | null, + fileName: string + ) => { + if (folder.key === null || fileKey === null) { + if (folder.key !== fileKey) return fileKey === null + return folder.name.localeCompare(fileName) <= 0 + } + const cmp = compareKeys(folder.key, fileKey) + if (cmp !== 0) return direction === 'asc' ? cmp < 0 : cmp > 0 + return folder.name.localeCompare(fileName) <= 0 } let nextFolder = 0 for (const file of input.files) { if (pinnedFileIds.has(input.fileId(file))) continue const key = input.fileKey(file) + const name = input.fileName(file) while (nextFolder < folders.length) { const folder = folders[nextFolder] - if (folder.key === null || !precedes(folder.key, key)) break + if (!precedes(folder, key, name)) break entries.push(folder.item) nextFolder++ } diff --git a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts index 5547b785ab3..418beb489db 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts @@ -11,12 +11,8 @@ import { } from '@/hooks/queries/utils/workspace-file-keys' import type { ResourceListPreferenceConfig } from '@/stores/resource-list-preferences' -/** - * Sortable list columns, matching the `Resource.Options` sort menu: the ones the server pages - * the list by. Type and owner are not among them — a sort the server cannot apply could only - * order the rows already loaded, which reads as the whole folder's order and is not. - */ -export const FILE_SORT_COLUMNS = ['name', 'size', 'created', 'updated'] as const +/** Sortable list columns, matching the `Resource.Options` sort menu; the server pages by each. */ +export const FILE_SORT_COLUMNS = ['name', 'size', 'type', 'created', 'owner', 'updated'] as const type FileSortColumn = (typeof FILE_SORT_COLUMNS)[number] @@ -24,7 +20,9 @@ type FileSortColumn = (typeof FILE_SORT_COLUMNS)[number] const FILE_SORT_FIELDS: Record<FileSortColumn, WorkspaceFileListSortBy> = { name: 'name', size: 'size', + type: 'type', created: 'uploadedAt', + owner: 'owner', updated: 'updatedAt', } diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx index 14762fb4de8..0eeb9ec8ac6 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx @@ -685,7 +685,8 @@ function SearchModalContent({ /** * Files are not listed whole — a workspace can hold far more than the palette ranks. Browsing - * offers the most recently updated; a query, once typing settles, searches every file by name. + * offers the most recently updated; a query, once typing settles, asks the server for the + * best matches by name and folder path, which the palette then ranks as it ranks everything. */ const fileQuery = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) const recentFiles = useWorkspaceFileList(workspaceId, RECENT_FILES, { diff --git a/apps/sim/hooks/queries/utils/workspace-file-keys.ts b/apps/sim/hooks/queries/utils/workspace-file-keys.ts index 9cd9bcd53c3..1f6aca40f9b 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-keys.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-keys.ts @@ -2,7 +2,13 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' export type WorkspaceFileListScope = 'active' | 'archived' -export type WorkspaceFileListSortBy = 'name' | 'size' | 'uploadedAt' | 'updatedAt' +export type WorkspaceFileListSortBy = + | 'name' + | 'size' + | 'uploadedAt' + | 'updatedAt' + | 'type' + | 'owner' export type WorkspaceFileListSortOrder = 'asc' | 'desc' /** The default page size of a workspace file list: one round trip for nearly every folder. */ From f76c9d3f50184ddd3c35f63f35b14710496968d6 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 20:42:58 -0700 Subject: [PATCH 06/15] refactor(files): share the file browsers' type rules and tighten the paged reads - One source for the Type/Size vocabulary and its SQL. The Files page and the Project files browser now read the same label table, kinds, size boundaries (`lib/workspace-files/browser.ts`) and the same SQL builders for the extension, effective MIME type, label, kind, and size bucket (`lib/workspace-files/file-type-sql.ts`); `type-label.ts` and `list-filters.ts` are gone. Every kind resolves through the effective MIME type. The one remaining difference is deliberate and kept: the Project browser normalizes the declared type (lowercased, parameters dropped) before resolving it, while the Files page trims it, as its column always has. An integration test checks the SQL label and every kind against the JS rules for each extension the MIME table knows, declared three ways. - One list query param parser, `queryListSchema`, for repeated or comma-separated values, canonicalized for set filters; it replaces the workspace list's and the folder download's local copies. - The Type and Owner sorts compute their value once per scoped row, in a fenced subquery that the order, keyset, and cursor all read, instead of re-running the label and member-name subqueries per comparison. - File search computes each row's tier and similarity once, runs the unindexable initials pattern only when the indexed read leaves room, loads the folder tree and the matched folders together, and hydrates paths from that tree instead of reading it again. --- .../workspace/[workspaceId]/files/files.tsx | 15 +- .../[workspaceId]/files/search-params.ts | 15 +- .../queries/utils/workspace-file-keys.ts | 10 +- apps/sim/lib/api/contracts/primitives.ts | 26 ++ .../api/contracts/workspace-file-folders.ts | 13 +- apps/sim/lib/api/contracts/workspace-files.ts | 36 +- .../contexts/workspace/file-manager.ts | 361 ++++++++++-------- .../file-listing.integration.ts | 80 +++- .../application/list-workspace-files.ts | 20 +- apps/sim/lib/workspace-files/browser-query.ts | 69 +--- apps/sim/lib/workspace-files/browser.ts | 34 ++ apps/sim/lib/workspace-files/file-type-sql.ts | 82 ++++ apps/sim/lib/workspace-files/list-filters.ts | 38 -- apps/sim/lib/workspace-files/query-scope.ts | 159 ++------ apps/sim/lib/workspace-files/type-label.ts | 36 -- 15 files changed, 508 insertions(+), 486 deletions(-) create mode 100644 apps/sim/lib/workspace-files/file-type-sql.ts delete mode 100644 apps/sim/lib/workspace-files/list-filters.ts delete mode 100644 apps/sim/lib/workspace-files/type-label.ts diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 253b8b64214..9aced945793 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -41,9 +41,12 @@ import { getMimeTypeFromExtension, isArchiveFileName, } from '@/lib/uploads/utils/file' -import { isWorkspaceFileKind, isWorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' +import { + fileBrowserTypeLabel, + isFileBrowserSize, + isFileBrowserType, +} from '@/lib/workspace-files/browser' import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' -import { workspaceFileTypeLabel } from '@/lib/workspace-files/type-label' import { FindBar } from '@/app/workspace/[workspaceId]/components/find-bar/find-bar' import { useFindShortcut } from '@/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut' import type { MoveOptionNode } from '@/app/workspace/[workspaceId]/components/folders' @@ -405,8 +408,8 @@ function FilesContent() { const applyListPreference = useCallback( (preference: ResourceListPreference) => { void setFileFilters({ - type: preference.filters.type.filter(isWorkspaceFileKind), - size: preference.filters.size.filter(isWorkspaceFileSizeBucket), + type: preference.filters.type.filter(isFileBrowserType), + size: preference.filters.size.filter(isFileBrowserSize), uploadedBy: [...preference.filters.uploadedBy], }) applyUrlSort(preference.sort.column, preference.sort.direction) @@ -691,7 +694,7 @@ function FilesContent() { sortColumn === 'size' ? file.size : sortColumn === 'type' - ? workspaceFileTypeLabel(file.type, file.name) + ? fileBrowserTypeLabel(file.type, file.name) : sortColumn === 'created' ? new Date(file.uploadedAt).getTime() : sortColumn === 'updated' @@ -772,7 +775,7 @@ function FilesContent() { }, type: { icon: <Icon className='size-[14px]' />, - label: workspaceFileTypeLabel(file.type, file.name), + label: fileBrowserTypeLabel(file.type, file.name), }, created: timeCell(file.uploadedAt), owner: ownerCell(file.uploadedBy, membersById), diff --git a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts index 418beb489db..b4457e4b1e2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/search-params.ts @@ -1,9 +1,6 @@ import { createParser, parseAsArrayOf, parseAsString, parseAsStringLiteral } from 'nuqs/server' import { createSortParams } from '@/lib/url-state' -import { - WORKSPACE_FILE_KINDS, - WORKSPACE_FILE_SIZE_BUCKETS, -} from '@/lib/workspace-files/list-filters' +import { FILE_BROWSER_SIZES, FILE_BROWSER_TYPES } from '@/lib/workspace-files/browser' import { type WorkspaceFileListParams, type WorkspaceFileListSortBy, @@ -88,8 +85,8 @@ export const filesUrlKeys = { */ export const filesFilterParsers = { search: parseAsString.withDefault(''), - type: parseAsArrayOf(parseAsStringLiteral(WORKSPACE_FILE_KINDS)).withDefault([]), - size: parseAsArrayOf(parseAsStringLiteral(WORKSPACE_FILE_SIZE_BUCKETS)).withDefault([]), + type: parseAsArrayOf(parseAsStringLiteral(FILE_BROWSER_TYPES)).withDefault([]), + size: parseAsArrayOf(parseAsStringLiteral(FILE_BROWSER_SIZES)).withDefault([]), uploadedBy: parseAsArrayOf(parseAsString).withDefault([]), } as const @@ -110,7 +107,7 @@ export const filesListPreferenceConfig = { module: 'files', sortColumns: FILE_SORT_COLUMNS, filterKeys: ['type', 'size', 'uploadedBy'], - filterValues: { type: WORKSPACE_FILE_KINDS, size: WORKSPACE_FILE_SIZE_BUCKETS }, + filterValues: { type: FILE_BROWSER_TYPES, size: FILE_BROWSER_SIZES }, preferenceUrlKeys: filesFilterUrlKeyMap, defaultPreference: { sort: filesSortParams.default, @@ -137,8 +134,8 @@ export function filesBrowserListParams(state: { search: string sort: FileSortColumn dir: 'asc' | 'desc' - type: readonly (typeof WORKSPACE_FILE_KINDS)[number][] - size: readonly (typeof WORKSPACE_FILE_SIZE_BUCKETS)[number][] + type: readonly (typeof FILE_BROWSER_TYPES)[number][] + size: readonly (typeof FILE_BROWSER_SIZES)[number][] uploadedBy: readonly string[] }): WorkspaceFileListParams { const searching = state.search.trim().length > 0 diff --git a/apps/sim/hooks/queries/utils/workspace-file-keys.ts b/apps/sim/hooks/queries/utils/workspace-file-keys.ts index 1f6aca40f9b..fef15074de3 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-keys.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-keys.ts @@ -1,5 +1,5 @@ import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' +import type { FileBrowserSize, FileBrowserType } from '@/lib/workspace-files/browser' export type WorkspaceFileListScope = 'active' | 'archived' export type WorkspaceFileListSortBy = @@ -32,8 +32,8 @@ export interface WorkspaceFileListParams { recursive?: boolean /** Case-insensitive substring of the file name. */ q?: string - types?: readonly WorkspaceFileKind[] - sizes?: readonly WorkspaceFileSizeBucket[] + types?: readonly FileBrowserType[] + sizes?: readonly FileBrowserSize[] uploadedBy?: readonly string[] sortBy: WorkspaceFileListSortBy sortOrder: WorkspaceFileListSortOrder @@ -74,8 +74,8 @@ export function workspaceFileListParams(input: { folderId?: string | null recursive?: boolean q?: string - types?: readonly WorkspaceFileKind[] - sizes?: readonly WorkspaceFileSizeBucket[] + types?: readonly FileBrowserType[] + sizes?: readonly FileBrowserSize[] uploadedBy?: readonly string[] sortBy: WorkspaceFileListSortBy sortOrder: WorkspaceFileListSortOrder diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index c583cab62a3..ee1be2dca98 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -605,6 +605,32 @@ export const retentionOverridesSchema = z export type RetentionOverrides = z.output<typeof retentionOverridesSchema> +/** + * A list-valued query param, sent repeated (`a=x&a=y`, how `requestJson` serializes an array) or + * comma-separated (`a=x,y`): entries trimmed, empty ones dropped. `canonical` also de-duplicates + * and sorts them, for a filter the query treats as a set — so `x,y` and `y,x` are one filter, for + * the query and for a cursor bound to it. + */ +export function queryListSchema<T extends z.ZodType<string>>( + member: T, + options: { max?: number; canonical?: boolean } = {} +) { + const entries = z.array(member) + return z.preprocess( + (raw) => { + if (typeof raw !== 'string' && !Array.isArray(raw)) return raw + const values = (Array.isArray(raw) ? raw : [raw]) + .flatMap((entry) => (typeof entry === 'string' ? entry.split(',') : [entry])) + .map((entry) => (typeof entry === 'string' ? entry.trim() : entry)) + .filter((entry) => entry !== '') + return options.canonical ? [...new Set(values)].sort() : values + }, + options.max === undefined + ? entries + : entries.max(options.max, `Cannot list more than ${options.max} values`) + ) +} + export const booleanQueryFlagSchema = z.preprocess( (value) => { if (typeof value === 'boolean') return value diff --git a/apps/sim/lib/api/contracts/workspace-file-folders.ts b/apps/sim/lib/api/contracts/workspace-file-folders.ts index 66620f36363..8433e070eb8 100644 --- a/apps/sim/lib/api/contracts/workspace-file-folders.ts +++ b/apps/sim/lib/api/contracts/workspace-file-folders.ts @@ -1,4 +1,5 @@ import { z } from 'zod' +import { queryListSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { MAX_WORKSPACE_FILE_BULK_REQUEST_IDS } from '@/lib/workspace-files/limits' @@ -86,17 +87,7 @@ export const bulkArchiveWorkspaceFileItemsBodySchema = z message: 'At least one file or folder must be selected', }) -const queryIdListSchema = z - .union([z.string(), z.array(z.string())]) - .optional() - .transform((value) => { - if (!value) return [] - const values = Array.isArray(value) ? value : [value] - return values - .flatMap((entry) => entry.split(',')) - .map((entry) => entry.trim()) - .filter(Boolean) - }) +const queryIdListSchema = queryListSchema(z.string()).default([]) export const downloadWorkspaceFileItemsQuerySchema = z.object({ fileIds: queryIdListSchema, diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 64502cee078..7affae0dd6d 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -5,6 +5,7 @@ import { folderIdSchema, inlineFileRefQuerySchema, isCanonicalBase64, + queryListSchema, workspaceFileNameSchema, workspaceIdSchema, } from '@/lib/api/contracts/primitives' @@ -12,10 +13,7 @@ import { shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' import { V2_SEARCH_MAX_LENGTH, v2LimitSchema } from '@/lib/api/contracts/v2/shared' import { LIST_SORT_ORDERS } from '@/lib/api/list-query' -import { - WORKSPACE_FILE_KINDS, - WORKSPACE_FILE_SIZE_BUCKETS, -} from '@/lib/workspace-files/list-filters' +import { FILE_BROWSER_SIZES, FILE_BROWSER_TYPES } from '@/lib/workspace-files/browser' /** * Client-reachable listing scopes. `all` is deliberately excluded: it drops the @@ -73,27 +71,6 @@ export const WORKSPACE_FILE_PAGE_PARAMS = [ 'cursor', ] as const -/** - * A set filter on the wire: comma-separated members, parsed to its sorted, de-duplicated - * members so `a,b` and `b,a` are one filter for the query and for the cursor bound to it. - */ -function commaSetSchema<T extends z.ZodType<string>>(member: T) { - return z.preprocess( - (raw) => - typeof raw === 'string' - ? [ - ...new Set( - raw - .split(',') - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) - ), - ].sort() - : raw, - z.array(member).max(100, 'A filter cannot list more than 100 values') - ) -} - export const listWorkspaceFilesQuerySchema = z.object({ scope: workspaceFileScopeSchema.default('active'), /** The folder to list; `root` for root files. Omit for every folder. */ @@ -108,11 +85,14 @@ export const listWorkspaceFilesQuerySchema = z.object({ .max(V2_SEARCH_MAX_LENGTH, 'q is too long') .optional(), /** File kinds, comma-separated: a file matching any of them is listed. */ - types: commaSetSchema(z.enum(WORKSPACE_FILE_KINDS)).optional(), + types: queryListSchema(z.enum(FILE_BROWSER_TYPES), { canonical: true }).optional(), /** Size buckets, comma-separated: a file in any of them is listed. */ - sizes: commaSetSchema(z.enum(WORKSPACE_FILE_SIZE_BUCKETS)).optional(), + sizes: queryListSchema(z.enum(FILE_BROWSER_SIZES), { canonical: true }).optional(), /** Uploader user ids, comma-separated. */ - uploadedBy: commaSetSchema(z.string().max(255, 'uploadedBy entry is too long')).optional(), + uploadedBy: queryListSchema(z.string().max(255, 'uploadedBy entry is too long'), { + max: 100, + canonical: true, + }).optional(), sortBy: z.enum(WORKSPACE_FILE_SORT_FIELDS).optional(), sortOrder: z.enum(LIST_SORT_ORDERS).optional(), limit: z.coerce diff --git a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts index eb13c358ec3..c0b6c9078f4 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-manager.ts @@ -35,6 +35,7 @@ import { isNotNull, isNull, ne, + not, or, type SQL, type SQLWrapper, @@ -144,7 +145,7 @@ import { normalizeDisplaySegment, VfsPathError, } from '@/lib/vfs/path' -import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' +import type { FileBrowserSize, FileBrowserType } from '@/lib/workspace-files/browser' import { lockFileDirectories } from '@/lib/workspace-files/locks' import { OWNED_FILE_CONTEXTS, @@ -1798,14 +1799,17 @@ type WorkspaceFileListRow = Pick<WorkspaceFileRow, keyof typeof workspaceFileLis async function hydrateWorkspaceFilePaths( files: WorkspaceFileListRow[], workspaceId: string, - options?: { folders?: WorkspaceFileFolderRecord[]; hydrateFolderPaths?: boolean } + options?: { + folders?: readonly Pick<WorkspaceFileFolderRecord, 'id' | 'name' | 'parentId'>[] + hydrateFolderPaths?: boolean + } ): Promise<WorkspaceFileRecord[]> { const needsFolderPaths = files.some((file) => file.folderId) && (options?.hydrateFolderPaths ?? true) const folders = needsFolderPaths ? (options?.folders ?? (await listWorkspaceFileFolders(workspaceId, { scope: 'all' }))) : [] - const folderPaths = needsFolderPaths ? buildWorkspaceFileFolderPathMap(folders) : new Map() + const folderPaths = needsFolderPaths ? buildWorkspaceFileFolderPathMap([...folders]) : new Map() return files.map((file) => mapWorkspaceFileRecord(file, workspaceId, folderPaths)) } @@ -1872,108 +1876,48 @@ function fileOwnerNameExpression(owner: EditableFileOwner): SQL<string | null> { LIMIT 1)` } -type WorkspaceFilePageRow = WorkspaceFileListRow & { - timestampCursor: string - /** The Type label or owner name a `type`/`owner` page sorts by, selected only for that sort. */ - sortLabel?: string - /** `1` when an `owner` page's file has no member uploader, which sorts last either way. */ - sortLast?: number -} +type WorkspaceFilePageRow = WorkspaceFileListRow & { timestampCursor: string } /** - * The keyset behind {@link queryFileRecords}' sortable fields, and the columns it reads that the - * listing does not already select. The `Record` makes it total over the sortable fields: a new - * one fails to compile until it has a keyset here. + * The keyset behind {@link queryFileRecords}' indexed sorts. The `Record` makes it total over + * the v2 sort enum: a new sortable field fails to compile until it has a keyset here. * - * Every key is `NOT NULL`, and `id` closes each keyset so a page boundary inside a run of equal - * values is still stable. The name key bounds its scan so a later page seeks the - * `(owner, folder, original_name)` index instead of filtering down from its top; the timestamp - * keys order on the raw column for the upload-order indexes (see `schema.ts`). - * - * Type and Owner order as the browser always has: by the label or member name in the requested - * direction, then by file name A→Z whichever way the column runs, a file without a member owner - * last in both directions. Neither is an indexed column, so their pages sort the scoped files - * (one folder's, or a search's matches) rather than seeking an index. + * Every key column is `NOT NULL`, and `id` closes each keyset so a page boundary inside a run of + * equal names/sizes/timestamps is still stable. The name key bounds its scan so a later page + * seeks the `(owner, folder, original_name)` index instead of filtering down from its top; the + * timestamp keys order on the raw column for the upload-order indexes (see `schema.ts`). */ -function workspaceFileSortPlan( - sortBy: WorkspaceFileSortBy, - after: CursorKey[] | undefined, - owner: EditableFileOwner -): { keys: KeysetKey<WorkspaceFilePageRow>[]; columns: Record<string, SQL> } { +function workspaceFileSortKeys( + sortBy: V2FileSortBy, + after: CursorKey[] | undefined +): KeysetKey<WorkspaceFilePageRow>[] { const fileId = textKey<WorkspaceFilePageRow>(workspaceFiles.id, (row) => row.id) - const nameAscending: KeysetKey<WorkspaceFilePageRow> = { - ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), - direction: 'asc', - } - const idAscending: KeysetKey<WorkspaceFilePageRow> = { ...fileId, direction: 'asc' } - const sortLabel = (row: WorkspaceFilePageRow) => row.sortLabel ?? '' - const single = (key: KeysetKey<WorkspaceFilePageRow>) => ({ - keys: [key, fileId], - columns: {}, - }) - const sorts: Record< - WorkspaceFileSortBy, - () => { keys: KeysetKey<WorkspaceFilePageRow>[]; columns: Record<string, SQL> } - > = { - name: () => - single({ - ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), - boundsScan: true, - }), + const sorts: Record<V2FileSortBy, () => KeysetKey<WorkspaceFilePageRow>> = { + name: () => ({ + ...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName), + boundsScan: true, + }), size: () => - single( - numberKey<WorkspaceFilePageRow>( - sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number), - (row) => getWorkspaceFileSize(row) - ) + numberKey<WorkspaceFilePageRow>( + sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number), + (row) => getWorkspaceFileSize(row) ), uploadedAt: () => - single( - resumableTimestampKey( - workspaceFiles.uploadedAt, - (row) => row.uploadedAt, - (row) => row.timestampCursor, - after - ) + resumableTimestampKey( + workspaceFiles.uploadedAt, + (row) => row.uploadedAt, + (row) => row.timestampCursor, + after ), updatedAt: () => - single( - resumableTimestampKey( - workspaceFiles.updatedAt, - (row) => row.updatedAt, - (row) => row.timestampCursor, - after - ) + resumableTimestampKey( + workspaceFiles.updatedAt, + (row) => row.updatedAt, + (row) => row.timestampCursor, + after ), - type: () => ({ - keys: [ - textKey<WorkspaceFilePageRow>(workspaceFileTypeLabelExpression, sortLabel), - nameAscending, - idAscending, - ], - columns: { sortLabel: workspaceFileTypeLabelExpression }, - }), - owner: () => { - const ownerName = fileOwnerNameExpression(owner) - const missing = sql<number>`(CASE WHEN ${ownerName} IS NULL THEN 1 ELSE 0 END)`.mapWith( - Number - ) - const name = sql<string>`coalesce(${ownerName}, '')` - return { - keys: [ - { - ...numberKey<WorkspaceFilePageRow>(missing, (row) => row.sortLast ?? 0), - direction: 'asc', - }, - textKey<WorkspaceFilePageRow>(name, sortLabel), - nameAscending, - idAscending, - ], - columns: { sortLast: missing, sortLabel: name }, - } - }, } - return sorts[sortBy]() + return [sorts[sortBy](), fileId] } export interface QueryWorkspaceFilesOptions { @@ -1997,9 +1941,9 @@ export interface QueryWorkspaceFilesOptions { /** Case-insensitive substring match on the file name. */ search?: string /** Files of any of these kinds; absent or empty applies no kind filter. */ - kinds?: readonly WorkspaceFileKind[] + kinds?: readonly FileBrowserType[] /** Files in any of these size buckets; absent or empty applies no size filter. */ - sizeBuckets?: readonly WorkspaceFileSizeBucket[] + sizeBuckets?: readonly FileBrowserSize[] /** Files uploaded by any of these users; absent or empty applies no uploader filter. */ uploadedBy?: readonly string[] sortBy: WorkspaceFileSortBy @@ -2056,30 +2000,11 @@ export async function queryFileRecords<const O extends EditableFileOwner>( options: QueryWorkspaceFilesOptions, client: DbOrTx = db ): Promise<{ files: OwnedFileRecord<O>[]; nextKeys: CursorKey[] | null }> { - const { - scope = 'active', - folderId, - folderScope, - name, - search, - sortBy, - sortOrder, - limit, - after, - } = options + const { scope = 'active', folderId, folderScope, name, search, sortBy } = options if (folderId !== undefined && folderScope !== undefined) { throw new OrchestrationError('validation', 'Specify either folderId or folderScope, not both') } - const { keys, columns } = workspaceFileSortPlan(sortBy, after, owner) - - let resumeAfter: SQL | undefined - if (after) { - const condition = keysetAfter(keys, after, sortOrder) - if (!condition) throw new OrchestrationError('validation', INVALID_CURSOR_MESSAGE) - resumeAfter = condition - } - - const conditions = [ + const scoped = [ fileOwnerCondition(owner), options.fileIds ? inArray(workspaceFiles.id, [...options.fileIds]) : undefined, scope === 'all' @@ -2096,26 +2021,116 @@ export async function queryFileRecords<const O extends EditableFileOwner>( options.uploadedBy && options.uploadedBy.length > 0 ? inArray(workspaceFiles.userId, [...options.uploadedBy]) : undefined, - resumeAfter, ] + const { page, nextKeys } = + sortBy === 'type' || sortBy === 'owner' + ? await readComputedSortPage(owner, sortBy, scoped, options, client) + : await readIndexedSortPage(sortBy, scoped, options, client) - const rows: WorkspaceFilePageRow[] = await client - .select({ ...workspaceFilePageColumns(sortBy), ...columns }) - .from(workspaceFiles) - .where(and(...conditions)) - .orderBy(...keysetOrderBy(keys, sortOrder)) - .limit(limit + 1) - - const hasMore = rows.length > limit - const page = rows.slice(0, limit) const folders = !page.some((file) => file.folderId) ? [] : (options.folders ?? (await listFileFolders(owner, { scope: 'all' }, client))) const paths = buildWorkspaceFileFolderPathMap([...folders]) const files = page.map((file) => mapFileRecord(file, owner, paths)) + return { files, nextKeys } +} + +/** One page of `rows` over-fetched by one, and the keys that resume after it. */ +function cutPage<Row>(keys: readonly KeysetKey<Row>[], rows: Row[], limit: number) { + const page = rows.slice(0, limit) const last = page.at(-1) + return { page, nextKeys: rows.length > limit && last ? encodeKeyset(keys, last) : null } +} + +/** The cursor's `WHERE` half, or a classified 400 when the cursor does not fit the sort. */ +function resumeCondition<Row>( + keys: readonly KeysetKey<Row>[], + after: CursorKey[] | undefined, + sortOrder: ListSortOrder +): SQL | undefined { + if (!after) return undefined + const condition = keysetAfter(keys, after, sortOrder) + if (!condition) throw new OrchestrationError('validation', INVALID_CURSOR_MESSAGE) + return condition +} + +/** A page under a sort an index serves: name, size, or a timestamp. */ +async function readIndexedSortPage( + sortBy: V2FileSortBy, + scoped: (SQL | undefined)[], + options: QueryWorkspaceFilesOptions, + client: DbOrTx +) { + const keys = workspaceFileSortKeys(sortBy, options.after) + const rows: WorkspaceFilePageRow[] = await client + .select(workspaceFilePageColumns(sortBy)) + .from(workspaceFiles) + .where(and(...scoped, resumeCondition(keys, options.after, options.sortOrder))) + .orderBy(...keysetOrderBy(keys, options.sortOrder)) + .limit(options.limit + 1) + return cutPage(keys, rows, options.limit) +} - return { files, nextKeys: hasMore && last ? encodeKeyset(keys, last) : null } +/** + * A page under the Type or Owner sort, which order by a computed value: the label the Type column + * shows, or the uploader's member name. Both order as the browser always has: by the value in the + * requested direction, then by file name A→Z whichever way the column runs, then by id; a file + * without a member owner last in both directions. + * + * Neither value is indexed, so the page sorts its scope (one folder's rows, reached through the + * folder index). The value is computed once per scoped row, in a fenced subquery (`OFFSET 0` + * keeps the planner from inlining it into the order and the cursor predicate), and the order, + * keyset, and cursor all read that one column. + */ +async function readComputedSortPage( + owner: EditableFileOwner, + sortBy: 'type' | 'owner', + scoped: (SQL | undefined)[], + options: QueryWorkspaceFilesOptions, + client: DbOrTx +) { + const value = + sortBy === 'type' ? workspaceFileTypeLabelExpression : fileOwnerNameExpression(owner) + const sorted = client + .select({ + ...workspaceFileListColumns, + sortValue: sql<string | null>`${value}`.as('sort_value'), + }) + .from(workspaceFiles) + .where(and(...scoped)) + .offset(0) + .as('sorted') + type SortedRow = WorkspaceFileListRow & { sortValue: string | null } + const name: KeysetKey<SortedRow> = { + ...textKey<SortedRow>(sorted.originalName, (row) => row.originalName), + direction: 'asc', + } + const id: KeysetKey<SortedRow> = { + ...textKey<SortedRow>(sorted.id, (row) => row.id), + direction: 'asc', + } + const keys: KeysetKey<SortedRow>[] = + sortBy === 'type' + ? [textKey<SortedRow>(sql`${sorted.sortValue}`, (row) => row.sortValue ?? ''), name, id] + : [ + { + ...numberKey<SortedRow>( + sql`(case when ${sorted.sortValue} is null then 1 else 0 end)`, + (row) => (row.sortValue === null ? 1 : 0) + ), + direction: 'asc', + }, + textKey<SortedRow>(sql`coalesce(${sorted.sortValue}, '')`, (row) => row.sortValue ?? ''), + name, + id, + ] + const rows: SortedRow[] = await client + .select() + .from(sorted) + .where(resumeCondition(keys, options.after, options.sortOrder)) + .orderBy(...keysetOrderBy(keys, options.sortOrder)) + .limit(options.limit + 1) + return cutPage(keys, rows, options.limit) } function normalizeWorkspaceFileReferenceSegments(fileReference: string): string[] { @@ -2558,17 +2573,24 @@ export interface SearchWorkspaceFileRecordsOptions { /** Folders whose files match by path: files in any of them count as matches. */ pathFolderIds: readonly string[] limit: number + /** The workspace's active folders, to hydrate folder paths without reading the tree again. */ + folders?: readonly Pick<WorkspaceFileFolderRecord, 'id' | 'name' | 'parentId'>[] } +/** How well a file matched, lowest first; `initials` sits with `loose`, after it. */ +const SEARCH_TIER = { exact: 0, prefix: 1, contains: 2, loose: 3, path: 4 } as const + /** * Live workspace files a palette query finds (see {@link paletteNameMatches}), best match first: * the name equal to the query, then starting with it, then containing it or every word of it, - * then its initials or a fuzzy match, then files under a folder that matched (`pathFolderIds`). + * then a fuzzy match or its initials, then files under a folder that matched (`pathFolderIds`). * Within a tier, closer word similarity first, then by name. * - * Substring, word, and fuzzy predicates are the ones the `(workspace_id, original_name - * gin_trgm_ops)` index answers. An initials pattern has too few letters to index, so for a - * multi-letter query the index narrows to the workspace and the pattern is checked per file. + * The first read is the one the `(workspace_id, original_name gin_trgm_ops)` index answers — + * substring, word, and fuzzy predicates, plus the matched folders — with each row's tier and + * similarity computed once. An initials pattern holds too few letters to index, so it runs as a + * second read only when the first leaves room under `limit`, and its files rank after the first + * read's fuzzy matches. */ export async function searchWorkspaceFileRecords( workspaceId: string, @@ -2578,39 +2600,62 @@ export async function searchWorkspaceFileRecords( const name = workspaceFiles.originalName const matches = paletteNameMatches(name, query) const containsAll = or(matches.contains, matches.allWords) ?? matches.contains - const loose = or(matches.initials, matches.fuzzy) ?? matches.fuzzy - const tier = sql`(CASE - WHEN ${matches.exact} THEN 0 - WHEN ${matches.prefix} THEN 1 - WHEN ${containsAll} THEN 2 - WHEN ${loose} THEN 3 - ELSE 4 END)` - const rows = await db - .select(workspaceFileListColumns) + const inPath = + options.pathFolderIds.length > 0 + ? workspaceFileFolderIdCondition(options.pathFolderIds) + : undefined + const scope = and( + workspaceFileScopeCondition(workspaceId, 'active'), + options.folderId === undefined ? undefined : workspaceFileFolderIdCondition(options.folderId) + ) + const tier = sql<number>`(case + when ${matches.exact} then ${SEARCH_TIER.exact} + when ${matches.prefix} then ${SEARCH_TIER.prefix} + when ${containsAll} then ${SEARCH_TIER.contains} + when ${matches.fuzzy} then ${SEARCH_TIER.loose} + else ${SEARCH_TIER.path} end)`.mapWith(Number) + const similarity = sql<number>`word_similarity(${query}, ${name})` + const indexed = await db + .select({ + ...workspaceFileListColumns, + tier: tier.as('search_tier'), + similarity: similarity.as('search_similarity'), + }) .from(workspaceFiles) - .where( - and( - workspaceFileScopeCondition(workspaceId, 'active'), - options.folderId === undefined - ? undefined - : workspaceFileFolderIdCondition(options.folderId), - or( - containsAll, - loose, - options.pathFolderIds.length > 0 - ? workspaceFileFolderIdCondition(options.pathFolderIds) - : undefined - ) - ) - ) + .where(and(scope, or(containsAll, matches.fuzzy, inPath))) .orderBy( - tier, - sql`word_similarity(${query}, ${name}) DESC`, + sql`search_tier`, + sql`search_similarity desc`, workspaceFiles.originalName, workspaceFiles.id ) .limit(limit) - return hydrateWorkspaceFilePaths(rows, workspaceId) + + let rows: WorkspaceFileListRow[] = indexed + if (matches.initials && indexed.length < limit) { + const byInitials = await db + .select(workspaceFileListColumns) + .from(workspaceFiles) + .where( + and( + scope, + matches.initials, + indexed.length > 0 + ? not( + inArray( + workspaceFiles.id, + indexed.map((row) => row.id) + ) + ) + : undefined + ) + ) + .orderBy(desc(similarity), workspaceFiles.originalName, workspaceFiles.id) + .limit(limit - indexed.length) + const before = indexed.filter((row) => row.tier <= SEARCH_TIER.loose) + rows = [...before, ...byInitials, ...indexed.slice(before.length)] + } + return hydrateWorkspaceFilePaths(rows, workspaceId, { folders: options.folders }) } export interface WorkspaceFileFolderUsage { diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 005bd042e10..7bd04f6fe72 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -13,14 +13,24 @@ import { inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import type { CursorKey } from '@/lib/api/list-query' import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager' +import { + EXTENSION_TO_MIME, + getFileExtension, + MIME_TYPE_MAPPING, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file' +import { isSupportedExtension } from '@/lib/uploads/utils/validation' import { getWorkspaceFileFolderStats, listWorkspaceFileCollection, lookupWorkspaceFiles, searchWorkspaceFiles, } from '@/lib/workspace-files/application/list-workspace-files' -import { workspaceFileTypeLabelExpression } from '@/lib/workspace-files/query-scope' -import { workspaceFileTypeLabel } from '@/lib/workspace-files/type-label' +import { fileBrowserTypeLabel } from '@/lib/workspace-files/browser' +import { + workspaceFileKindCondition, + workspaceFileTypeLabelExpression, +} from '@/lib/workspace-files/query-scope' const alice = generateId() const bob = generateId() @@ -29,10 +39,18 @@ const workspaceId = generateId() const otherWorkspaceId = generateId() const filterWorkspaceId = generateId() const sortWorkspaceId = generateId() +const parityWorkspaceId = generateId() /** Uploaders with fixed display names: two members and one who has left the workspace. */ const ann = generateId() const ben = generateId() const zed = generateId() +/** The four kind columns of a parity row. */ +const pick = (row: Record<string, string | boolean>) => ({ + document: row.document, + image: row.image, + audio: row.audio, + video: row.video, +}) const session = (userId: string) => ({ kind: 'session' as const, userId, sessionId: generateId() }) /** Folder tree in the main workspace: Docs/{Sub/{Deep}}, Media; plus one in the other workspace. */ @@ -153,6 +171,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { { id: otherWorkspaceId, name: 'Other fixture', ownerId: alice, billedAccountUserId: alice }, { id: filterWorkspaceId, name: 'Filter fixture', ownerId: alice, billedAccountUserId: alice }, { id: sortWorkspaceId, name: 'Sort fixture', ownerId: alice, billedAccountUserId: alice }, + { id: parityWorkspaceId, name: 'Parity fixture', ownerId: alice, billedAccountUserId: alice }, ]) await db.insert(permissions).values([ { @@ -282,7 +301,13 @@ describe('bounded workspace file reads in PostgreSQL', () => { afterAll(async () => { await deleteWorkspaceFixture( db, - inArray(workspace.id, [workspaceId, otherWorkspaceId, filterWorkspaceId, sortWorkspaceId]) + inArray(workspace.id, [ + workspaceId, + otherWorkspaceId, + filterWorkspaceId, + sortWorkspaceId, + parityWorkspaceId, + ]) ) await db.delete(user).where(inArray(user.id, [alice, bob, carol, ann, ben, zed])) }) @@ -567,7 +592,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { seeded.push({ id: file.id, name: fixture.name, - label: workspaceFileTypeLabel(fixture.contentType, fixture.name), + label: fileBrowserTypeLabel(fixture.contentType, fixture.name), owner: ownerNames.get(fixture.userId) ?? null, }) } @@ -671,6 +696,53 @@ describe('bounded workspace file reads in PostgreSQL', () => { }) }) + describe('SQL type labels and kinds', () => { + /** + * Every extension the MIME table knows, stored three ways: declaring nothing (so the + * extension decides), declaring the generic type (likewise), and declaring its own type (so + * the declared type decides). The SQL must agree with the JS rules for each. + */ + it('labels and classifies a file of every known extension as the JS rules do', async () => { + const cases = Object.keys(EXTENSION_TO_MIME).flatMap((extension) => + ['', 'application/octet-stream', EXTENSION_TO_MIME[extension]].map((contentType) => ({ + name: `file.${extension}`, + contentType, + })) + ) + const ids = new Map<string, { name: string; contentType: string }>() + for (const [index, fixture] of cases.entries()) { + const file = await insertFile({ + workspaceId: parityWorkspaceId, + name: `${index}-${fixture.name}`, + contentType: fixture.contentType, + uploadedAt: '2026-06-01T00:00:00', + }) + ids.set(file.id, { name: file.name, contentType: fixture.contentType }) + } + const rows = await db.execute<Record<string, string | boolean>>(sql` + SELECT id, ${workspaceFileTypeLabelExpression} AS label, + ${workspaceFileKindCondition(['document'])} AS document, + ${workspaceFileKindCondition(['image'])} AS image, + ${workspaceFileKindCondition(['audio'])} AS audio, + ${workspaceFileKindCondition(['video'])} AS video + FROM ${workspaceFiles} WHERE workspace_id = ${parityWorkspaceId}`) + expect(rows).toHaveLength(cases.length) + for (const row of rows) { + const fixture = ids.get(String(row.id)) + if (!fixture) throw new Error('Unexpected row') + const mime = resolveEffectiveMimeType(fixture.contentType, fixture.name) + expect({ id: row.id, label: row.label, ...pick(row) }).toEqual({ + id: row.id, + label: fileBrowserTypeLabel(fixture.contentType, fixture.name), + document: isSupportedExtension(getFileExtension(fixture.name)), + image: mime.startsWith('image/'), + audio: MIME_TYPE_MAPPING[mime.toLowerCase()] === 'audio', + video: MIME_TYPE_MAPPING[mime.toLowerCase()] === 'video', + }) + } + }) + }) + describe('palette search', () => { const search = (query: string, extra: { folderId?: string | null; recursive?: boolean } = {}) => searchWorkspaceFiles.execute({ diff --git a/apps/sim/lib/workspace-files/application/list-workspace-files.ts b/apps/sim/lib/workspace-files/application/list-workspace-files.ts index 32784662cc5..52e790e8f9e 100644 --- a/apps/sim/lib/workspace-files/application/list-workspace-files.ts +++ b/apps/sim/lib/workspace-files/application/list-workspace-files.ts @@ -28,7 +28,7 @@ import { } from '@/lib/uploads/contexts/workspace' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' -import type { WorkspaceFileKind, WorkspaceFileSizeBucket } from '@/lib/workspace-files/list-filters' +import type { FileBrowserSize, FileBrowserType } from '@/lib/workspace-files/browser' import { resolveWorkspaceFolderScope } from '@/lib/workspace-files/resolve-folder-scope' export interface ListAllWorkspaceFilesInput { @@ -53,9 +53,9 @@ export interface QueryWorkspaceFilePageInput { name?: string search?: string /** Files of any of these kinds. */ - kinds?: readonly WorkspaceFileKind[] + kinds?: readonly FileBrowserType[] /** Files in any of these size buckets. */ - sizeBuckets?: readonly WorkspaceFileSizeBucket[] + sizeBuckets?: readonly FileBrowserSize[] /** Files uploaded by any of these users. */ uploadedBy?: readonly string[] sortBy: WorkspaceFileSortBy @@ -245,7 +245,10 @@ export const searchWorkspaceFiles = defineAuthorizedWorkspaceFileUseCase({ resolveContext: ({ input }: { input: SearchWorkspaceFilesInput }) => resolveListWorkspaceFileContext(input.workspaceId), async execute({ input, context }) { - const folderIndex = await loadFileFolderIndex(context.workspaceId) + const [folderIndex, matchedFolders] = await Promise.all([ + loadFileFolderIndex(context.workspaceId), + findWorkspaceFileFoldersMatching(context.workspaceId, input.query), + ]) let folderId: string | null | string[] | undefined if (input.folderId === null) { folderId = input.recursive ? undefined : null @@ -255,12 +258,10 @@ export const searchWorkspaceFiles = defineAuthorizedWorkspaceFileUseCase({ ? resolveFolderScope(folderIndex, { kind: 'folder', folderId: input.folderId }, true) : input.folderId } - const childrenByParent = indexFolderChildren([...folderIndex.rowById.values()]) + const folders = [...folderIndex.rowById.values()] + const childrenByParent = indexFolderChildren(folders) const pathFolderIds = new Set<string>() - for (const matched of await findWorkspaceFileFoldersMatching( - context.workspaceId, - input.query - )) { + for (const matched of matchedFolders) { if (!folderIndex.rowById.has(matched)) continue pathFolderIds.add(matched) for (const id of collectDescendantFolderIdsFrom(childrenByParent, matched)) { @@ -272,6 +273,7 @@ export const searchWorkspaceFiles = defineAuthorizedWorkspaceFileUseCase({ folderId, pathFolderIds: [...pathFolderIds], limit: input.limit, + folders, }) return { files: await withShares(context.workspaceId, files) } }, diff --git a/apps/sim/lib/workspace-files/browser-query.ts b/apps/sim/lib/workspace-files/browser-query.ts index d196f86d6ba..da74ec5aef7 100644 --- a/apps/sim/lib/workspace-files/browser-query.ts +++ b/apps/sim/lib/workspace-files/browser-query.ts @@ -17,20 +17,19 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx } from '@/lib/db/types' import { MAX_FOLDER_PATH_SEGMENTS } from '@/lib/folders/paths' import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' -import { - EXTENSION_TO_MIME, - MIME_TYPE_MAPPING, - resolveEffectiveMimeType, -} from '@/lib/uploads/utils/file' -import { SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation' -import { - FILE_BROWSER_MIME_LABELS, - FILE_BROWSER_SIZE_BOUNDARIES, - type FileBrowserCreator, - type FileBrowserFilters, - type FileBrowserItem, - type FileBrowserSort, +import type { + FileBrowserCreator, + FileBrowserFilters, + FileBrowserItem, + FileBrowserSort, } from '@/lib/workspace-files/browser' +import { + effectiveMimeSql, + fileExtensionSql, + fileKindSql, + fileSizeBucketSql, + fileTypeLabelSql, +} from '@/lib/workspace-files/file-type-sql' import type { EditableFileOwner } from '@/lib/workspace-files/ownership' import { fileFolderOwnerCondition, fileOwnerCondition } from '@/lib/workspace-files/ownership-query' @@ -59,19 +58,6 @@ type BrowserRow = { creator_deleted: boolean } -const EFFECTIVE_EXTENSION_MIMES = Object.fromEntries( - Object.keys(EXTENSION_TO_MIME).map((extension) => [ - extension, - resolveEffectiveMimeType(null, `file.${extension}`), - ]) -) -const AUDIO_MIMES = Object.keys(MIME_TYPE_MAPPING).filter( - (mime) => MIME_TYPE_MAPPING[mime] === 'audio' -) -const VIDEO_MIMES = Object.keys(MIME_TYPE_MAPPING).filter( - (mime) => MIME_TYPE_MAPPING[mime] === 'video' -) - const BROWSER_SORTS: Record<FileBrowserSort, KeysetKey<BrowserRow>> = { name: textKey<BrowserRow>(sql`name collate "C"`, (row) => row.name), size: numberKey(sql`size`, (row) => Number(row.size)), @@ -122,7 +108,7 @@ export async function queryFileBrowserItems( owned_files as ( select id, original_name as name, folder_id as parent_id, size_bytes as size, content_type, user_id, uploaded_at, updated_at, - case when strpos(original_name, '.') > 0 then lower(regexp_replace(original_name, '^.*[.]', '')) else '' end as extension + ${fileExtensionSql(sql`original_name`)} as extension from ${workspaceFiles} where ${fileOwnerCondition(owner)} and ${activeFiles} ), descendants as ( @@ -137,15 +123,12 @@ export async function queryFileBrowserItems( join owned_files files on files.parent_id = tree.id group by tree.root_id ), effective_files as ( - select *, case when lower(btrim(split_part(content_type, ';', 1))) not in ('', 'application/octet-stream', 'binary/octet-stream') - then lower(btrim(split_part(content_type, ';', 1))) else coalesce(${JSON.stringify(EFFECTIVE_EXTENSION_MIMES)}::jsonb ->> extension, 'application/octet-stream') end as mime + select *, ${effectiveMimeSql(sql`lower(btrim(split_part(content_type, ';', 1)))`, sql`extension`)} as mime from owned_files ), items as ( select id, 'file'::text as kind, name, parent_id, size, - coalesce(${JSON.stringify(FILE_BROWSER_MIME_LABELS)}::jsonb ->> mime, - case when mime like 'audio/%' then 'Audio' when mime like 'video/%' then 'Video' - when mime like 'image/%' then 'Image' when extension <> '' then upper(extension) else coalesce(content_type, 'File') end) as type, + ${fileTypeLabelSql(sql`mime`, sql`extension`, sql`content_type`)} as type, user_id, trunc(extract(epoch from uploaded_at) * 1000) as created_ms, trunc(extract(epoch from updated_at) * 1000) as updated_ms, extension, mime from effective_files @@ -191,26 +174,10 @@ export async function queryFileBrowserItems( input.folderId === null ? sql`parent_id is null` : sql`parent_id = ${input.folderId}` ) if (input.search) clauses.push(sql`name ilike ${`%${escapeLikePattern(input.search)}%`}`) - const typeClauses = (input.types ?? []).map((type) => { - if (type === 'document') - return sql`extension in (${sql.join( - SUPPORTED_DOCUMENT_EXTENSIONS.map((extension) => sql`${extension}`), - sql`, ` - )})` - if (type === 'image') return sql`mime like 'image/%'` - const mimes = type === 'audio' ? AUDIO_MIMES : VIDEO_MIMES - return sql`lower(mime) in (${sql.join( - mimes.map((mime) => sql`${mime}`), - sql`, ` - )})` - }) - const sizeClauses = (input.sizes ?? []).map((size) => - size === 'small' - ? sql`size < ${FILE_BROWSER_SIZE_BOUNDARIES.small}` - : size === 'medium' - ? sql`size >= ${FILE_BROWSER_SIZE_BOUNDARIES.small} and size <= ${FILE_BROWSER_SIZE_BOUNDARIES.medium}` - : sql`size > ${FILE_BROWSER_SIZE_BOUNDARIES.medium}` + const typeClauses = (input.types ?? []).map((type) => + fileKindSql(type, sql`mime`, sql`extension`) ) + const sizeClauses = (input.sizes ?? []).map((size) => fileSizeBucketSql(size, sql`size`)) const fileFilters: SQL[] = [] if (typeClauses.length) fileFilters.push(sql`(${sql.join(typeClauses, sql` or `)})`) if (sizeClauses.length) fileFilters.push(sql`(${sql.join(sizeClauses, sql` or `)})`) diff --git a/apps/sim/lib/workspace-files/browser.ts b/apps/sim/lib/workspace-files/browser.ts index d18d7be4dd5..21646d01abf 100644 --- a/apps/sim/lib/workspace-files/browser.ts +++ b/apps/sim/lib/workspace-files/browser.ts @@ -1,9 +1,27 @@ +import { getFileExtension, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' + export const FILE_BROWSER_SORTS = ['name', 'size', 'type', 'created', 'owner', 'updated'] as const export type FileBrowserSort = (typeof FILE_BROWSER_SORTS)[number] +/** + * The file kinds the file browsers filter by. A file can be several at once — a `.json` is a + * document whatever its stored type: `document` is decided by extension + * (`SUPPORTED_DOCUMENT_EXTENSIONS`), the media kinds by the file's effective MIME type. + */ export const FILE_BROWSER_TYPES = ['document', 'image', 'audio', 'video'] as const +export type FileBrowserType = (typeof FILE_BROWSER_TYPES)[number] export const FILE_BROWSER_SIZES = ['small', 'medium', 'large'] as const +export type FileBrowserSize = (typeof FILE_BROWSER_SIZES)[number] +/** Small is under 1 MiB, medium 1 to 10 MiB inclusive, large over 10 MiB. */ export const FILE_BROWSER_SIZE_BOUNDARIES = { small: 1_048_576, medium: 10_485_760 } as const +export function isFileBrowserType(value: string): value is FileBrowserType { + return (FILE_BROWSER_TYPES as readonly string[]).includes(value) +} + +export function isFileBrowserSize(value: string): value is FileBrowserSize { + return (FILE_BROWSER_SIZES as readonly string[]).includes(value) +} + export const FILE_BROWSER_MIME_LABELS: Readonly<Record<string, string>> = { 'application/pdf': 'PDF', 'application/zip': 'ZIP', @@ -46,3 +64,19 @@ export interface FileBrowserFilters { sizes?: readonly (typeof FILE_BROWSER_SIZES)[number][] creatorIds?: readonly string[] } + +/** + * A file's Type column label: a specific label for its effective MIME type, else its media + * family (Audio, Video, Image), else its extension uppercased, else the stored type. The SQL + * twin is `fileTypeLabelSql`, which the browsers' Type sorts order by. + */ +export function fileBrowserTypeLabel(storedType: string, filename: string): string { + const mimeType = resolveEffectiveMimeType(storedType, filename) + const label = FILE_BROWSER_MIME_LABELS[mimeType] + if (label) return label + if (mimeType.startsWith('audio/')) return 'Audio' + if (mimeType.startsWith('video/')) return 'Video' + if (mimeType.startsWith('image/')) return 'Image' + const extension = getFileExtension(filename) + return extension ? extension.toUpperCase() : storedType +} diff --git a/apps/sim/lib/workspace-files/file-type-sql.ts b/apps/sim/lib/workspace-files/file-type-sql.ts new file mode 100644 index 00000000000..a69193d93ca --- /dev/null +++ b/apps/sim/lib/workspace-files/file-type-sql.ts @@ -0,0 +1,82 @@ +import { type SQL, type SQLWrapper, sql } from 'drizzle-orm' +import { + EXTENSION_TO_MIME, + GENERIC_MIME_TYPES, + MIME_TYPE_MAPPING, + resolveEffectiveMimeType, +} from '@/lib/uploads/utils/file' +import { SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation' +import { + FILE_BROWSER_MIME_LABELS, + FILE_BROWSER_SIZE_BOUNDARIES, + type FileBrowserSize, + type FileBrowserType, +} from '@/lib/workspace-files/browser' + +/** + * The SQL the file browsers classify, label, and bucket files with — one spelling of the rules + * `resolveEffectiveMimeType`, `fileBrowserTypeLabel`, and the browsers' filters apply, so the + * Files page and the Project files page cannot drift apart. + */ + +/** Each known extension's own MIME type: what a file declaring no format resolves to. */ +const EFFECTIVE_EXTENSION_MIMES = JSON.stringify( + Object.fromEntries( + Object.keys(EXTENSION_TO_MIME).map((extension) => [ + extension, + resolveEffectiveMimeType(null, `file.${extension}`), + ]) + ) +) +const LABELS = JSON.stringify(FILE_BROWSER_MIME_LABELS) +const GENERIC_MIME = resolveEffectiveMimeType(null, 'file') + +function mimesOfKind(kind: 'audio' | 'video'): string[] { + return Object.keys(MIME_TYPE_MAPPING).filter((mime) => MIME_TYPE_MAPPING[mime] === kind) +} + +function list(values: readonly string[]): SQL { + return sql.join( + values.map((value) => sql`${value}`), + sql`, ` + ) +} + +/** The extension `getFileExtension` reads: after the last `.`, lowercased; `''` without one. */ +export function fileExtensionSql(name: SQLWrapper): SQL<string> { + return sql<string>`(case when strpos(${name}, '.') > 0 then lower(regexp_replace(${name}, '^.*[.]', '')) else '' end)` +} + +/** + * The effective MIME type: `declared` when it names a format, else the extension's own. The + * caller decides how `declared` is normalized — the Files page trims the stored type, as + * `resolveEffectiveMimeType` does; the Project browser also lowercases it and drops parameters. + */ +export function effectiveMimeSql(declared: SQL, extension: SQL): SQL<string> { + return sql<string>`(case when ${declared} not in ('', ${list([...GENERIC_MIME_TYPES])}) then ${declared} + else coalesce(${EFFECTIVE_EXTENSION_MIMES}::jsonb ->> ${extension}, ${GENERIC_MIME}) end)` +} + +/** The Type column label for a file of effective MIME type `mime` (see `fileBrowserTypeLabel`). */ +export function fileTypeLabelSql(mime: SQL, extension: SQL, contentType: SQLWrapper): SQL<string> { + return sql<string>`coalesce(${LABELS}::jsonb ->> ${mime}, + case when ${mime} like 'audio/%' then 'Audio' when ${mime} like 'video/%' then 'Video' + when ${mime} like 'image/%' then 'Image' when ${extension} <> '' then upper(${extension}) + else coalesce(${contentType}, 'File') end)` +} + +/** Whether a file of effective MIME type `mime` is of `kind`. */ +export function fileKindSql(kind: FileBrowserType, mime: SQL, extension: SQL): SQL { + if (kind === 'document') return sql`${extension} in (${list(SUPPORTED_DOCUMENT_EXTENSIONS)})` + if (kind === 'image') return sql`${mime} like 'image/%'` + return sql`lower(${mime}) in (${list(mimesOfKind(kind))})` +} + +/** Whether a file of `size` bytes is in `bucket`. */ +export function fileSizeBucketSql(bucket: FileBrowserSize, size: SQL): SQL { + if (bucket === 'small') return sql`${size} < ${FILE_BROWSER_SIZE_BOUNDARIES.small}` + if (bucket === 'medium') { + return sql`${size} >= ${FILE_BROWSER_SIZE_BOUNDARIES.small} and ${size} <= ${FILE_BROWSER_SIZE_BOUNDARIES.medium}` + } + return sql`${size} > ${FILE_BROWSER_SIZE_BOUNDARIES.medium}` +} diff --git a/apps/sim/lib/workspace-files/list-filters.ts b/apps/sim/lib/workspace-files/list-filters.ts deleted file mode 100644 index f25b26674cc..00000000000 --- a/apps/sim/lib/workspace-files/list-filters.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * The file kinds the paged file list filters by. A file can be several at once — a `.json` - * is a document whatever its stored type — so a filter naming more than one matches a file - * that is any of them. - * - * - `document`: the extension is a supported document format (`SUPPORTED_DOCUMENT_EXTENSIONS`). - * - `image` / `audio` / `video`: the file's effective MIME type (`resolveEffectiveMimeType`: - * the stored type, or the extension's when the stored one names no format) is that kind. - */ -export const WORKSPACE_FILE_KINDS = ['document', 'image', 'audio', 'video'] as const - -export type WorkspaceFileKind = (typeof WORKSPACE_FILE_KINDS)[number] - -export function isWorkspaceFileKind(value: string): value is WorkspaceFileKind { - return (WORKSPACE_FILE_KINDS as readonly string[]).includes(value) -} - -export const WORKSPACE_FILE_SIZE_BUCKETS = ['small', 'medium', 'large'] as const - -export type WorkspaceFileSizeBucket = (typeof WORKSPACE_FILE_SIZE_BUCKETS)[number] - -export function isWorkspaceFileSizeBucket(value: string): value is WorkspaceFileSizeBucket { - return (WORKSPACE_FILE_SIZE_BUCKETS as readonly string[]).includes(value) -} - -const MIB = 1_048_576 - -/** - * Each size bucket's byte range, both bounds inclusive (`max: null` is unbounded): under 1 MiB, - * 1 to 10 MiB, and over 10 MiB, as the filter's labels read. - */ -export const WORKSPACE_FILE_SIZE_BUCKET_RANGES: Readonly< - Record<WorkspaceFileSizeBucket, { min: number; max: number | null }> -> = { - small: { min: 0, max: MIB - 1 }, - medium: { min: MIB, max: 10 * MIB }, - large: { min: 10 * MIB + 1, max: null }, -} diff --git a/apps/sim/lib/workspace-files/query-scope.ts b/apps/sim/lib/workspace-files/query-scope.ts index f2efc0ad5f4..9233ad7974e 100644 --- a/apps/sim/lib/workspace-files/query-scope.ts +++ b/apps/sim/lib/workspace-files/query-scope.ts @@ -1,21 +1,16 @@ import { workspaceFiles } from '@sim/db/schema' -import { and, eq, gte, inArray, isNull, lte, not, or, type SQL, sql } from 'drizzle-orm' +import { eq, inArray, isNull, or, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { FolderIdScope } from '@/lib/folders/scope' +import type { FileBrowserSize, FileBrowserType } from '@/lib/workspace-files/browser' import { - EXTENSION_TO_MIME, - GENERIC_MIME_TYPES, - MIME_TYPE_MAPPING, - resolveEffectiveMimeType, -} from '@/lib/uploads/utils/file' -import { SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation' -import { - WORKSPACE_FILE_SIZE_BUCKET_RANGES, - type WorkspaceFileKind, - type WorkspaceFileSizeBucket, -} from '@/lib/workspace-files/list-filters' + effectiveMimeSql, + fileExtensionSql, + fileKindSql, + fileSizeBucketSql, + fileTypeLabelSql, +} from '@/lib/workspace-files/file-type-sql' import { OWNED_FILE_CONTEXTS } from '@/lib/workspace-files/owned-files' -import { WORKSPACE_FILE_TYPE_LABELS } from '@/lib/workspace-files/type-label' /** Durable files available to workspace resource pickers, reference mappings, and fork copies. */ export function activeWorkspaceFileConditions(workspaceIds: string[]) { @@ -81,134 +76,36 @@ export function workspaceFileFolderScopeCondition( ]) } -/** - * The file's extension as `getFileExtension` reads it: everything after the last `.`, - * lowercased, or `''` for a name without one. - */ -const workspaceFileExtension = sql<string>`(CASE WHEN strpos(${workspaceFiles.originalName}, '.') > 0 - THEN lower(regexp_replace(${workspaceFiles.originalName}, '^.*\\.', '')) ELSE '' END)` - -const declaredContentType = sql<string>`btrim(${workspaceFiles.contentType})` - -/** Whether the stored type names a format, the condition under which `resolveEffectiveMimeType` keeps it. */ -const declaresFormat: SQL = sql`(${declaredContentType} <> '' AND ${not( - inArray(declaredContentType, [...GENERIC_MIME_TYPES]) -)})` +const workspaceFileExtension = fileExtensionSql(workspaceFiles.originalName) -/** Every extension whose own MIME type (what a file without a declared format resolves to) passes `matches`. */ -function extensionsResolvingTo(matches: (mimeType: string) => boolean): string[] { - return Object.keys(EXTENSION_TO_MIME).filter((extension) => - matches(resolveEffectiveMimeType(undefined, `file.${extension}`)) - ) -} - -/** A file whose effective MIME type passes `matches`, spelled for SQL: the declared type, else the extension's. */ -function effectiveMimeTypeCondition( - declaredMatches: SQL, - matches: (mimeType: string) => boolean -): SQL | undefined { - const extensions = extensionsResolvingTo(matches) - return or( - and(declaresFormat, declaredMatches), - extensions.length > 0 - ? and(not(declaresFormat), inArray(workspaceFileExtension, extensions)) - : undefined - ) -} - -function mappedMimeTypes(kind: 'audio' | 'video'): string[] { - return Object.entries(MIME_TYPE_MAPPING) - .filter(([, mapped]) => mapped === kind) - .map(([mimeType]) => mimeType) -} - -function workspaceFileKindPredicate(kind: WorkspaceFileKind): SQL | undefined { - switch (kind) { - case 'document': - return inArray(workspaceFileExtension, [...SUPPORTED_DOCUMENT_EXTENSIONS]) - case 'image': - return effectiveMimeTypeCondition(sql`${declaredContentType} LIKE 'image/%'`, (mimeType) => - mimeType.startsWith('image/') - ) - case 'audio': - case 'video': { - const mimeTypes = mappedMimeTypes(kind) - return effectiveMimeTypeCondition( - inArray(sql`lower(${declaredContentType})`, mimeTypes), - (mimeType) => MIME_TYPE_MAPPING[mimeType.toLowerCase()] === kind - ) - } - } -} +/** The MIME type `resolveEffectiveMimeType` gives the file: the trimmed stored type, else the extension's. */ +const workspaceFileEffectiveMime = effectiveMimeSql( + sql`btrim(${workspaceFiles.contentType})`, + workspaceFileExtension +) -/** - * The files of any of `kinds`, as the Files browser's Type filter classifies them (see - * `WORKSPACE_FILE_KINDS`), or none when `kinds` is absent or empty. Built from the extension - * and MIME tables `resolveEffectiveMimeType` reads, so a file's kind here is the one the app - * resolves for it everywhere else. - */ +/** The files of any of `kinds` (see `FILE_BROWSER_TYPES`), or none when `kinds` is absent or empty. */ export function workspaceFileKindCondition( - kinds: readonly WorkspaceFileKind[] | undefined + kinds: readonly FileBrowserType[] | undefined ): SQL | undefined { if (!kinds || kinds.length === 0) return undefined - return or(...kinds.map(workspaceFileKindPredicate)) + return or( + ...kinds.map((kind) => fileKindSql(kind, workspaceFileEffectiveMime, workspaceFileExtension)) + ) } -/** The files in any of `buckets` (see `WORKSPACE_FILE_SIZE_BUCKET_RANGES`), or none when absent or empty. */ +/** The files in any of `buckets` (see `FILE_BROWSER_SIZE_BOUNDARIES`), or none when absent or empty. */ export function workspaceFileSizeCondition( - buckets: readonly WorkspaceFileSizeBucket[] | undefined + buckets: readonly FileBrowserSize[] | undefined ): SQL | undefined { if (!buckets || buckets.length === 0) return undefined - const size = sql<number>`coalesce(${workspaceFiles.sizeBytes}, 0)` - return or( - ...buckets.map((bucket) => { - const { min, max } = WORKSPACE_FILE_SIZE_BUCKET_RANGES[bucket] - return and(gte(size, min), max === null ? undefined : lte(size, max)) - }) - ) -} - -/** - * A string constant of this module as an SQL literal. Only for the label and MIME tables below, - * which are code: inlining them keeps a query that repeats the expression (select, order, and - * keyset) from binding hundreds of parameters. - */ -function literal(value: string): SQL { - return sql.raw(`'${value.replaceAll("'", "''")}'`) -} - -/** A `CASE <subject> WHEN … THEN … ELSE <otherwise> END` over a constant value-to-value table. */ -function caseOf(subject: SQL, table: readonly (readonly [string, string])[], otherwise: SQL): SQL { - if (table.length === 0) return otherwise - return sql`(CASE ${subject} ${sql.join( - table.map(([when, then]) => sql`WHEN ${literal(when)} THEN ${literal(then)}`), - sql` ` - )} ELSE ${otherwise} END)` + const size = sql`coalesce(${workspaceFiles.sizeBytes}, 0)` + return or(...buckets.map((bucket) => fileSizeBucketSql(bucket, size))) } -/** The MIME type `resolveEffectiveMimeType` gives the file, spelled for SQL. */ -const workspaceFileEffectiveMimeType: SQL = sql`(CASE WHEN ${declaresFormat} THEN ${declaredContentType} ELSE ${caseOf( +/** The file's Type column label (`fileBrowserTypeLabel`), what the list's Type sort orders by. */ +export const workspaceFileTypeLabelExpression: SQL<string> = fileTypeLabelSql( + workspaceFileEffectiveMime, workspaceFileExtension, - Object.keys(EXTENSION_TO_MIME).map( - (extension) => [extension, resolveEffectiveMimeType(undefined, `file.${extension}`)] as const - ), - literal(resolveEffectiveMimeType(undefined, 'file')) -)} END)` - -const resolvedMime = sql.raw('resolved.mime') -const resolvedExtension = sql.raw('resolved.extension') - -/** - * The file's Type column label (`workspaceFileTypeLabel`), spelled for SQL: what the list's Type - * sort orders by. Built from the same label table and MIME resolution, so a file sorts under the - * label it shows. The MIME type is resolved once per row, in a scalar subquery. - */ -export const workspaceFileTypeLabelExpression: SQL<string> = sql<string>`(SELECT ${caseOf( - resolvedMime, - Object.entries(WORKSPACE_FILE_TYPE_LABELS), - sql`(CASE WHEN ${resolvedMime} LIKE 'audio/%' THEN 'Audio' - WHEN ${resolvedMime} LIKE 'video/%' THEN 'Video' - WHEN ${resolvedMime} LIKE 'image/%' THEN 'Image' - WHEN ${resolvedExtension} <> '' THEN upper(${resolvedExtension}) - ELSE ${workspaceFiles.contentType} END)` -)} FROM (SELECT ${workspaceFileEffectiveMimeType} AS mime, ${workspaceFileExtension} AS extension) AS resolved)` + workspaceFiles.contentType +) diff --git a/apps/sim/lib/workspace-files/type-label.ts b/apps/sim/lib/workspace-files/type-label.ts deleted file mode 100644 index 57624d16a49..00000000000 --- a/apps/sim/lib/workspace-files/type-label.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getFileExtension, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' - -/** The Type column's label for each MIME type it names specifically. */ -export const WORKSPACE_FILE_TYPE_LABELS: Readonly<Record<string, string>> = { - 'application/pdf': 'PDF', - 'application/zip': 'ZIP', - 'application/msword': 'Word', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'Word', - 'application/vnd.ms-excel': 'Excel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'Excel', - 'application/vnd.ms-powerpoint': 'PowerPoint', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'PowerPoint', - 'application/json': 'JSON', - 'application/x-yaml': 'YAML', - 'text/csv': 'CSV', - 'text/plain': 'Text', - 'text/html': 'HTML', - 'text/x-sim-page': 'Page', - 'text/markdown': 'Markdown', -} - -/** - * The file's Type column label: a specific label for its effective MIME type, else its media - * family (Audio, Video, Image), else its extension uppercased, else the stored type. The list's - * Type sort orders by this same label, spelled in SQL (`workspaceFileTypeLabelExpression`). - */ -export function workspaceFileTypeLabel(storedType: string, filename: string): string { - const mimeType = resolveEffectiveMimeType(storedType, filename) - const label = WORKSPACE_FILE_TYPE_LABELS[mimeType] - if (label) return label - if (mimeType.startsWith('audio/')) return 'Audio' - if (mimeType.startsWith('video/')) return 'Video' - if (mimeType.startsWith('image/')) return 'Image' - const extension = getFileExtension(filename) - return extension ? extension.toUpperCase() : storedType -} From 2156cce49a2f9b467aff4eab006739a739ba5e3b Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 20:44:20 -0700 Subject: [PATCH 07/15] perf(files): roll up the child folders of several parents in one folder-stats read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GET /files/folder-stats` takes `parentIds` (repeated or comma-separated, `root` for the root) in place of `parentId`, so a view whose folders sit under different parents — search results — asks once instead of once per parent. One aggregate covers every parent's child subtrees; an unknown parent contributes nothing. --- .../[id]/files/folder-stats/route.test.ts | 18 +++++++++++++++++- .../[id]/files/folder-stats/route.ts | 7 ++++++- apps/sim/lib/api/contracts/workspace-files.ts | 10 +++++++++- .../file-listing.integration.ts | 18 +++++++++++++++--- .../application/list-workspace-files.ts | 17 ++++++++--------- 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts index e78df3fd32f..53e9daebe7d 100644 --- a/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.test.ts @@ -36,6 +36,22 @@ describe('GET /api/workspaces/[id]/files/folder-stats', () => { ) expect(response.status).toBe(200) - expect(stats.mock.calls[0][0].input).toEqual({ workspaceId: WORKSPACE_ID, parentId: null }) + expect(stats.mock.calls[0][0].input).toEqual({ workspaceId: WORKSPACE_ID, parentIds: [null] }) + }) + + it('reads several parents at once, the root spelled `root`, in place of parentId', async () => { + const response = await GET( + createMockRequest({ + method: 'GET', + url: `/api/workspaces/${WORKSPACE_ID}/files/folder-stats?parentId=folder-9&parentIds=folder-2,root`, + }), + createRouteContext({ id: WORKSPACE_ID }) + ) + + expect(response.status).toBe(200) + expect(stats.mock.calls[0][0].input).toEqual({ + workspaceId: WORKSPACE_ID, + parentIds: ['folder-2', null], + }) }) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts b/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts index 0cde499839a..0a3b4764a0e 100644 --- a/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/folder-stats/route.ts @@ -20,7 +20,12 @@ export const GET = defineInternalJsonRoute({ operation: fileOperations.list, rateLimit: internalRateLimits.user({ bucketName: 'workspace-files.read' }), errorPolicy: internalFileErrorPolicies.default, - mapInput: ({ params, query }) => ({ workspaceId: params.id, parentId: query.parentId }), + mapInput: ({ params, query }) => ({ + workspaceId: params.id, + parentIds: query.parentIds + ? query.parentIds.map((parentId) => (parentId === 'root' ? null : parentId)) + : [query.parentId], + }), useCase: getWorkspaceFileFolderStats, present: ({ folders }) => ({ success: true as const, folders }), }) diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 7affae0dd6d..97903d57f3b 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -353,10 +353,18 @@ export const lookupWorkspaceFilesContract = defineRouteContract({ const workspaceFileFolderStatsQuerySchema = z.object({ /** The folder whose child folders to report: `root` (default) or a folder id. */ parentId: workspaceFileFolderFilterSchema.prefault('root'), + /** + * Several folders whose child folders to report at once (`root` or folder ids), for a view + * whose folders sit under different parents; replaces `parentId` when sent. + */ + parentIds: queryListSchema(z.union([z.literal('root'), folderIdSchema]), { + max: 100, + canonical: true, + }).optional(), }) const workspaceFileFolderStatsResponseSchema = workspaceFileSuccessSchema.extend({ - /** One entry per direct child folder of `parentId`, totals covering its whole subtree. */ + /** One entry per direct child folder of each parent, totals covering its whole subtree. */ folders: z.array( z.object({ folderId: z.string(), diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 7bd04f6fe72..689c84d1b49 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -849,10 +849,10 @@ describe('bounded workspace file reads in PostgreSQL', () => { }) describe('folder stats', () => { - const stats = (parentId: string | null) => + const stats = (...parentIds: (string | null)[]) => getWorkspaceFileFolderStats.execute({ principal: session(alice), - input: { workspaceId, parentId }, + input: { workspaceId, parentIds }, }) it('rolls each child folder subtree up, excluding deleted files', async () => { @@ -870,6 +870,18 @@ describe('bounded workspace file reads in PostgreSQL', () => { expect((await stats(folders.deep)).folders).toEqual([]) }) + it('rolls up the children of several parents in one read, skipping unknown ones', async () => { + const combined = (await stats(null, folders.docs, generateId())).folders + expect(combined).toHaveLength(3) + expect(combined).toEqual( + expect.arrayContaining([ + { folderId: folders.docs, fileCount: 4, sizeBytes: 100 + 200 + 400 + 1000 }, + { folderId: folders.media, fileCount: 1, sizeBytes: 50 }, + { folderId: folders.sub, fileCount: 2, sizeBytes: 1400 }, + ]) + ) + }) + it('reports nothing for another workspace folder or an unknown folder', async () => { expect((await stats(folders.foreign)).folders).toEqual([]) expect((await stats(generateId())).folders).toEqual([]) @@ -890,7 +902,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { await expect( getWorkspaceFileFolderStats.execute({ principal: session(carol), - input: { workspaceId, parentId: null }, + input: { workspaceId, parentIds: [null] }, }) ).rejects.toMatchObject(refused) }) diff --git a/apps/sim/lib/workspace-files/application/list-workspace-files.ts b/apps/sim/lib/workspace-files/application/list-workspace-files.ts index 52e790e8f9e..4009b383b76 100644 --- a/apps/sim/lib/workspace-files/application/list-workspace-files.ts +++ b/apps/sim/lib/workspace-files/application/list-workspace-files.ts @@ -178,14 +178,15 @@ export const lookupWorkspaceFiles = defineAuthorizedWorkspaceFileUseCase({ export interface GetWorkspaceFileFolderStatsInput { workspaceId: string - /** The folder whose child folders to report, `null` for the workspace root. */ - parentId: string | null + /** The folders whose child folders to report, `null` for the workspace root. */ + parentIds: readonly (string | null)[] } /** - * For each direct child folder of `parentId`, the live files and bytes in its whole subtree. - * One grouped aggregate over the subtrees' folder ids, rolled up against the active folder - * tree; root files are never read. A `parentId` naming no active folder reports no children. + * For each direct child folder of every parent in `parentIds`, the live files and bytes in its + * whole subtree. One grouped aggregate over the subtrees' folder ids, rolled up against the + * active folder tree; root files are never read. A parent naming no active folder reports no + * children. */ export const getWorkspaceFileFolderStats = defineAuthorizedWorkspaceFileUseCase({ operation: fileOperations.list, @@ -193,14 +194,12 @@ export const getWorkspaceFileFolderStats = defineAuthorizedWorkspaceFileUseCase( resolveListWorkspaceFileContext(input.workspaceId), async execute({ input, context }) { const folderIndex = await loadFileFolderIndex(context.workspaceId) - if (input.parentId !== null && !folderIndex.rowById.has(input.parentId)) { - return { folders: [] } - } + const parents = new Set(input.parentIds) const rows = [...folderIndex.rowById.values()] // One child index for every subtree: `resolveFolderScope` would rebuild it per child. const childrenByParent = indexFolderChildren(rows) const subtrees = rows - .filter((row) => (row.parentId ?? null) === input.parentId) + .filter((row) => parents.has(row.parentId ?? null)) .map((row) => ({ folderId: row.id, ids: [row.id, ...collectDescendantFolderIdsFrom(childrenByParent, row.id)], From 1d14e47666367d22d3c6d3dfb5f9b227d0fefc9b Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 21:07:14 -0700 Subject: [PATCH 08/15] refactor(files): consolidate the paged file reads' client hooks and narrow their cache writes Hooks - `useWorkspaceFileList` returns `data.files` (selected once per page set) and a `loadMore` that skips while a page or refetch is in flight. - `useWorkspaceFilePickerOptions` replaces the four recent-or-search copies (markdown mentions, the search palette, the add-resource menu, workflow search-replace): recent files untyped, the server's best matches once the query settles via `useSearchFilterValue`, one limit constant. - `usePinnedWorkspaceFiles` serves the Files page and the rail flyout; `useWorkspaceFileAtPath` serves special tags and the resource tab; `distinctFiles` is the one de-duplicator. - Folder sizes read every parent in one `parentIds` request. - The file-upload picker reads its query through `useSearchFilterValue`, shares one folder-scope predicate, and refetches on open only when stale. Cache writes - Folder sizes join the metadata filter, and `invalidateWorkspaceFileReads` is one invalidation; an unobserved list is cut to its first page first, so its next mount refetches one page. - Optimistic writes patch only reads whose membership and order the change cannot move, and snapshot only those. A rename rewrites the record and id/key lookups, then refetches lists, searches, and name references once it lands (the mutation settles after the mounted ones have), so a renamed file never sits at its old position or in a search it no longer matches. A delete drops the file from live reads but not from archived lists. A content save writes the returned record through and refetches only the listings and folder sizes. - A superseded storage key re-resolves only the file's record and the lookups that found it. - Cached seeds are read once per query (and once per batch for multi-value lookups). - The image-dimension adapter re-runs its reads only when a lookup it asked for lands. - `isWorkspaceFileRecordKey` takes an optional workspace, replacing the any-workspace fallback in resource labels. Files page - The list merge takes pinned id sets and `{ id, name }` rows, and places folders with `compareSortableResources`, the comparator `sortResources` now exports. Type labels are computed once per record and file rows cached per record, so loading a page appends rows without rebuilding the ones on screen. Folders still interleave with files, as they always have. Workflow search-replace looks a stored value up only as what it can be: a reference (which answers an exact id) for every value, a storage key only for a serve path or a value with a `/`. The dead `findWorkspaceFileBySrc` is removed. --- .../[workspaceId]/components/folders/index.ts | 2 +- .../components/folders/resource-sort.ts | 33 +- .../mention/use-markdown-mentions.ts | 21 +- .../workspace/[workspaceId]/files/files.tsx | 80 ++-- .../[workspaceId]/files/list-entries.test.ts | 16 +- .../[workspaceId]/files/list-entries.ts | 89 ++-- .../components/special-tags/special-tags.tsx | 12 +- .../available-resources.ts | 27 +- .../resource-content/resource-content.tsx | 16 +- .../components/file-upload/file-upload.tsx | 74 ++-- .../rail-resource-flyout.tsx | 38 +- .../components/search-modal/search-modal.tsx | 20 +- .../hooks/queries/pinned-workspace-files.ts | 27 ++ .../utils/find-workspace-file-by-src.test.ts | 26 -- .../utils/find-workspace-file-by-src.ts | 18 - .../utils/workspace-file-cache.test.ts | 159 +++++++ .../queries/utils/workspace-file-cache.ts | 250 ++++++++--- .../queries/utils/workspace-file-keys.ts | 41 +- .../queries/workflow-search-replace.test.ts | 26 +- .../hooks/queries/workflow-search-replace.ts | 77 ++-- .../hooks/queries/workspace-files.test.tsx | 240 +++++++++-- apps/sim/hooks/queries/workspace-files.ts | 387 +++++++++++++----- .../tools/client/resource-display.ts | 4 +- 23 files changed, 1145 insertions(+), 538 deletions(-) create mode 100644 apps/sim/hooks/queries/pinned-workspace-files.ts delete mode 100644 apps/sim/hooks/queries/utils/find-workspace-file-by-src.test.ts create mode 100644 apps/sim/hooks/queries/utils/workspace-file-cache.test.ts diff --git a/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts b/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts index fb012cfb42b..36b951ba62a 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts @@ -32,7 +32,7 @@ export { renderMoveOptions, } from './move-options' export type { SortableResource } from './resource-sort' -export { sortResources } from './resource-sort' +export { compareSortableResources, sortResources } from './resource-sort' export { folderNavParsers, folderNavUrlKeys } from './search-params' export { useDragTeardown } from './use-drag-teardown' export type { FolderAncestors, UseFolderAncestorsOptions } from './use-folder-ancestors' diff --git a/apps/sim/app/workspace/[workspaceId]/components/folders/resource-sort.ts b/apps/sim/app/workspace/[workspaceId]/components/folders/resource-sort.ts index c465c9b28fb..ea2490b8976 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/folders/resource-sort.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/folders/resource-sort.ts @@ -36,19 +36,26 @@ export function sortResources<T>( entries: SortableResource<T>[], direction: SortDirection ): SortableResource<T>[] { - return entries.sort((a, b) => { - if (a.pinned !== b.pinned) return a.pinned ? -1 : 1 + return entries.sort((a, b) => compareSortableResources(a, b, direction)) +} + +/** The order {@link sortResources} applies: negative when `a` sorts before `b`. */ +export function compareSortableResources( + a: Omit<SortableResource<unknown>, 'item'>, + b: Omit<SortableResource<unknown>, 'item'>, + direction: SortDirection +): number { + if (a.pinned !== b.pinned) return a.pinned ? -1 : 1 - if (a.key === null || b.key === null) { - if (a.key !== b.key) return a.key === null ? 1 : -1 - } else { - const cmp = - typeof a.key === 'number' && typeof b.key === 'number' - ? a.key - b.key - : String(a.key).localeCompare(String(b.key)) - if (cmp !== 0) return direction === 'asc' ? cmp : -cmp - } + if (a.key === null || b.key === null) { + if (a.key !== b.key) return a.key === null ? 1 : -1 + } else { + const cmp = + typeof a.key === 'number' && typeof b.key === 'number' + ? a.key - b.key + : String(a.key).localeCompare(String(b.key)) + if (cmp !== 0) return direction === 'asc' ? cmp : -cmp + } - return a.name.localeCompare(b.name) - }) + return a.name.localeCompare(b.name) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts index f29f8a6a881..5ae376ffad9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts @@ -1,22 +1,14 @@ import { useMemo } from 'react' -import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { listIntegrations } from '@/blocks/integration-matcher' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useSkills } from '@/hooks/queries/skills' import { useTablesList } from '@/hooks/queries/tables' -import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { useWorkflows } from '@/hooks/queries/workflows' import { useWorkspaceFileFolders } from '@/hooks/queries/workspace-file-folders' -import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' -import { useDebounce } from '@/hooks/use-debounce' +import { useWorkspaceFilePickerOptions } from '@/hooks/queries/workspace-files' import { mentionIcon } from './mention-icon' import type { MentionItem } from './types' -/** The files the menu offers before anything is typed: the most recently updated anywhere. */ -const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 50 }) - -const FILE_SEARCH_LIMIT = 50 - /** * Aggregates the workspace-scoped entities the `@` menu can reference, composing the canonical * per-resource React Query hooks (never the chat-coupled `useAvailableResources` aggregator). All @@ -37,14 +29,9 @@ export function useMarkdownMentions( const ws = active ? workspaceId : undefined const wsStr = ws ?? '' - const fileQuery = useDebounce(options.fileQuery?.trim() ?? '', SEARCH_DEBOUNCE_MS) - const recentFiles = useWorkspaceFileList(wsStr, RECENT_FILES, { enabled: active }) - const matchingFiles = useWorkspaceFileSearch( - wsStr, - { q: fileQuery, limit: FILE_SEARCH_LIMIT }, - { enabled: active } - ) - const files = fileQuery ? matchingFiles.data : recentFiles.data?.pages[0]?.files + const { files } = useWorkspaceFilePickerOptions(wsStr, options.fileQuery ?? '', { + enabled: active, + }) const folders = useWorkspaceFileFolders(wsStr, 'active') const tables = useTablesList(ws, 'active') const knowledgeBases = useKnowledgeBasesQuery(ws, { enabled: active }) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 9aced945793..dd2fa70d3bc 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -136,6 +136,7 @@ import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/provide import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { PermissionAccessBoundary } from '@/ee/access-requests/components/permission-access-boundary' import { usePinItem, usePinnedIds, useUnpinItem } from '@/hooks/queries/pinned-items' +import { usePinnedWorkspaceFiles } from '@/hooks/queries/pinned-workspace-files' import { useWorkspaceMembersQuery, type WorkspaceMember } from '@/hooks/queries/workspace' import { useBulkArchiveWorkspaceFileItems, @@ -155,7 +156,6 @@ import { useUploadWorkspaceFile, useWorkspaceFileFolderSizes, useWorkspaceFileList, - useWorkspaceFileLookups, useWorkspaceFileRecord, } from '@/hooks/queries/workspace-files' import { useContextMenu } from '@/hooks/use-context-menu' @@ -210,6 +210,21 @@ const COLUMNS: ResourceColumn[] = [ const SEARCH_COLUMNS: ResourceColumn[] = [...COLUMNS, FOLDER_LOCATION_COLUMN] const EMPTY_WORKSPACE_FILES: WorkspaceFileRecord[] = [] + +/** + * Each file record's Type label, computed once: cached records are immutable, so a label keyed + * by the record holds until the record itself is replaced. + */ +const TYPE_LABELS = new WeakMap<WorkspaceFileRecord, string>() + +function typeLabelOf(file: WorkspaceFileRecord): string { + let label = TYPE_LABELS.get(file) + if (label === undefined) { + label = fileBrowserTypeLabel(file.type, file.name) + TYPE_LABELS.set(file, label) + } + return label +} const EMPTY_WORKSPACE_FILE_FOLDERS: WorkspaceFileFolderApi[] = [] const EMPTY_FIND_MATCH_IDS: readonly string[] = Object.freeze([]) @@ -484,30 +499,17 @@ function FilesContent() { if (error) { logger.error('Failed to load files:', error) } - const pagedFiles = useMemo( - () => fileList.data?.pages.flatMap((page) => page.files) ?? EMPTY_WORKSPACE_FILES, - [fileList.data] - ) + const pagedFiles = fileList.data?.files ?? EMPTY_WORKSPACE_FILES /** * A folder's pinned files lead it whether or not a page has reached them, so they are looked * up by id. A search or filter narrows the list to its own results, which pin among themselves. */ const pinsLeadFolder = !isSearching && !isFilteringFiles - const pinnedFileIdList = useMemo(() => [...pinnedFileIds], [pinnedFileIds]) - const pinnedFileLookups = useWorkspaceFileLookups(workspaceId, 'id', pinnedFileIdList, { + const pinnedInFolder = usePinnedWorkspaceFiles(workspaceId, currentFolderId, { enabled: isListOpen && pinsLeadFolder, }) - const pinnedFolderFiles = useMemo( - () => - pinsLeadFolder - ? pinnedFileIdList.flatMap((fileId) => { - const file = pinnedFileLookups.byValue.get(fileId) - return file && (file.folderId ?? null) === currentFolderId ? [file] : [] - }) - : EMPTY_WORKSPACE_FILES, - [pinsLeadFolder, pinnedFileIdList, pinnedFileLookups.byValue, currentFolderId] - ) + const pinnedFolderFiles = pinsLeadFolder ? pinnedInFolder.files : EMPTY_WORKSPACE_FILES /** * Every file the list has loaded, indexed once. The drag hook resolves each dragged row's @@ -523,23 +525,12 @@ function FilesContent() { const fileByIdRef = useRef(fileById) fileByIdRef.current = fileById - const loadMoreFilesRef = useRef(fileList) - loadMoreFilesRef.current = fileList - const loadMoreFiles = useCallback(() => { - const { isFetching, hasNextPage, fetchNextPage } = loadMoreFilesRef.current - if (!isFetching && hasNextPage) void fetchNextPage() - }, []) - const [creatingFile, setCreatingFile] = useState(false) const [isDirty, setIsDirty] = useState(false) const [saveStatus, setSaveStatus] = useState<SaveStatus>('idle') const [previewMode, setPreviewMode] = useState<PreviewMode>(() => { if (isNewFile) return 'editor' - if (fileIdFromRoute) { - const file = selectedFile - if (file && isPreviewable(file)) return 'preview' - return 'editor' - } + if (fileIdFromRoute) return selectedFile && isPreviewable(selectedFile) ? 'preview' : 'editor' return 'preview' }) const [showUnsavedChangesAlert, setShowUnsavedChangesAlert] = useState(false) @@ -672,12 +663,8 @@ function FilesContent() { files: pagedFiles, pinnedFiles: pinnedFolderFiles, hasMore: fileList.hasNextPage, - isFolderPinned: (folder) => pinnedFolderIds.has(folder.id), - isFilePinned: (file) => pinnedFileIds.has(file.id), - folderId: (folder) => folder.id, - fileId: (file) => file.id, - folderName: (folder) => folder.name, - fileName: (file) => file.name, + pinnedFolderIds, + pinnedFileIds, folderKey: (folder) => sortColumn === 'size' ? (folderSizes.get(folder.id)?.sizeBytes ?? 0) @@ -694,7 +681,7 @@ function FilesContent() { sortColumn === 'size' ? file.size : sortColumn === 'type' - ? fileBrowserTypeLabel(file.type, file.name) + ? typeLabelOf(file) : sortColumn === 'created' ? new Date(file.uploadedAt).getTime() : sortColumn === 'updated' @@ -718,6 +705,15 @@ function FilesContent() { ] ) + /** + * Each file's row, built once per record and pin state: loading another page appends rows + * without rebuilding the ones already on screen. Reset when what every row reads changes. + */ + const fileRowCache = useMemo( + () => new WeakMap<WorkspaceFileRecord, { pinned: boolean; row: ResourceRow }>(), + [membersById, folderById, isSearching] + ) + const baseRows: ResourceRow[] = useMemo( () => sortedEntries.map((item): ResourceRow => { @@ -761,8 +757,10 @@ function FilesContent() { } const { file } = item + const cached = fileRowCache.get(file) + if (cached && cached.pinned === pinned) return cached.row const Icon = getDocumentIcon(file.type || '', file.name) - return { + const row: ResourceRow = { id: file.id, cells: { name: { @@ -775,7 +773,7 @@ function FilesContent() { }, type: { icon: <Icon className='size-[14px]' />, - label: fileBrowserTypeLabel(file.type, file.name), + label: typeLabelOf(file), }, created: timeCell(file.uploadedAt), owner: ownerCell(file.uploadedBy, membersById), @@ -785,8 +783,10 @@ function FilesContent() { : EMPTY_LOCATION_CELL, }, } + fileRowCache.set(file, { pinned, row }) + return row }), - [sortedEntries, membersById, folderSizes, folderById, isSearching] + [sortedEntries, membersById, folderSizes, folderById, isSearching, fileRowCache] ) const rows: ResourceRow[] = useMemo(() => { @@ -2205,7 +2205,7 @@ function FilesContent() { rows={displayRows} apiRef={tableApiRef} virtualized - onLoadMore={loadMoreFiles} + onLoadMore={fileList.loadMore} hasMore={fileList.hasNextPage} isLoadingMore={fileList.isFetching} emptyState={ diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts index f84bdf869d9..cd6cf3a3a8a 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts @@ -24,12 +24,8 @@ function merge(input: { files: input.files, pinnedFiles: input.pinnedFiles ?? [], hasMore: input.hasMore, - isFolderPinned: (row) => pinned.has(row.id), - isFilePinned: (row) => pinned.has(row.id), - folderId: (row) => row.id, - fileId: (row) => row.id, - folderName: (row) => row.name, - fileName: (row) => row.name, + pinnedFolderIds: pinned, + pinnedFileIds: pinned, folderKey: (row) => row.size, fileKey: (row) => row.size, direction: input.direction ?? 'asc', @@ -97,12 +93,8 @@ describe('mergeFileListEntries', () => { files, pinnedFiles: [], hasMore: false, - isFolderPinned: () => false, - isFilePinned: () => false, - folderId: (row) => row.id, - fileId: (row) => row.id, - folderName: (row) => row.name, - fileName: (row) => row.name, + pinnedFolderIds: new Set(), + pinnedFileIds: new Set(), folderKey: (row) => row.owner, fileKey: (row) => row.owner, direction, diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts index 92dfc4f083b..bc94cf3ea35 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.ts @@ -1,4 +1,5 @@ import { + compareSortableResources, type SortableResource, sortResources, } from '@/app/workspace/[workspaceId]/components/folders' @@ -8,7 +9,12 @@ export type FileListEntry<Folder, File> = | { kind: 'folder'; folder: Folder; pinned: boolean } | { kind: 'file'; file: File; pinned: boolean } -interface MergeFileListEntriesInput<Folder, File> { +interface Named { + id: string + name: string +} + +interface MergeFileListEntriesInput<Folder extends Named, File extends Named> { /** The folders the list shows, in any order. */ folders: readonly Folder[] /** The loaded files, in the order the server paged them. */ @@ -20,95 +26,72 @@ interface MergeFileListEntriesInput<Folder, File> { pinnedFiles: readonly File[] /** Whether the server has more files past the last loaded one. */ hasMore: boolean - isFolderPinned: (folder: Folder) => boolean - isFilePinned: (file: File) => boolean - folderId: (folder: Folder) => string - fileId: (file: File) => string - folderName: (folder: Folder) => string - fileName: (file: File) => string - /** Each row's value for the active sort column, comparable across folders and files. */ - /** `null` for a row the column has no value for (a file without a member owner): last either way. */ + pinnedFolderIds: ReadonlySet<string> + pinnedFileIds: ReadonlySet<string> + /** + * Each row's value for the active sort column, comparable across folders and files; `null` + * for a row the column has no value for (a file without a member owner), last either way. + */ folderKey: (folder: Folder) => string | number | null fileKey: (file: File) => string | number | null direction: 'asc' | 'desc' } -function compareKeys(a: string | number, b: string | number): number { - return typeof a === 'number' && typeof b === 'number' ? a - b : String(a).localeCompare(String(b)) -} - /** * Orders the Files list: pinned rows first, then folders interleaved with the server-paged * files as ONE list by the active sort. * * The files keep the order the server paged them in — re-sorting them here could disagree with * the server's collation and move a later page's rows above ones already on screen. Each folder - * is placed in front of the first file it sorts at or before. A folder that sorts after the last - * loaded file is withheld while more pages remain, since the file it belongs in front of may be - * on one of them; it appears once a page reaches past it, so loading more only ever appends. - * - * A folder tying with a file on the column goes by name A→Z, and a row without a value goes - * last in either direction — the rules `sortResources` orders a fully loaded list by, which also - * orders the pinned rows among themselves. + * is placed in front of the first file it sorts at or before, by `compareSortableResources` — + * the order `sortResources` gives a fully loaded list, which also orders the pinned rows. A + * folder that sorts after the last loaded file is withheld while more pages remain, since the + * file it belongs in front of may be on one of them; it appears once a page reaches past it, so + * loading more only ever appends. */ -export function mergeFileListEntries<Folder, File>( +export function mergeFileListEntries<Folder extends Named, File extends Named>( input: MergeFileListEntriesInput<Folder, File> ): FileListEntry<Folder, File>[] { const { direction } = input - const pinnedFileIds = new Set<string>() + const shownPinnedFiles = new Set<string>() const pinned: SortableResource<FileListEntry<Folder, File>>[] = [] const unpinnedFolders: SortableResource<FileListEntry<Folder, File>>[] = [] for (const folder of input.folders) { + const isPinned = input.pinnedFolderIds.has(folder.id) const entry = { - item: { kind: 'folder' as const, folder, pinned: input.isFolderPinned(folder) }, - pinned: input.isFolderPinned(folder), - name: input.folderName(folder), + item: { kind: 'folder' as const, folder, pinned: isPinned }, + pinned: isPinned, + name: folder.name, key: input.folderKey(folder), } - if (entry.pinned) pinned.push(entry) - else unpinnedFolders.push(entry) + ;(isPinned ? pinned : unpinnedFolders).push(entry) } const addPinnedFile = (file: File) => { - const id = input.fileId(file) - if (pinnedFileIds.has(id)) return - pinnedFileIds.add(id) + if (shownPinnedFiles.has(file.id)) return + shownPinnedFiles.add(file.id) pinned.push({ item: { kind: 'file', file, pinned: true }, pinned: true, - name: input.fileName(file), + name: file.name, key: input.fileKey(file), }) } for (const file of input.pinnedFiles) addPinnedFile(file) - for (const file of input.files) if (input.isFilePinned(file)) addPinnedFile(file) + for (const file of input.files) if (input.pinnedFileIds.has(file.id)) addPinnedFile(file) const entries = sortResources(pinned, direction).map((entry) => entry.item) const folders = sortResources(unpinnedFolders, direction) - /** Whether `folder` sorts at or before a file whose key is `fileKey` and name is `fileName`. */ - const precedes = ( - folder: SortableResource<FileListEntry<Folder, File>>, - fileKey: string | number | null, - fileName: string - ) => { - if (folder.key === null || fileKey === null) { - if (folder.key !== fileKey) return fileKey === null - return folder.name.localeCompare(fileName) <= 0 - } - const cmp = compareKeys(folder.key, fileKey) - if (cmp !== 0) return direction === 'asc' ? cmp < 0 : cmp > 0 - return folder.name.localeCompare(fileName) <= 0 - } let nextFolder = 0 for (const file of input.files) { - if (pinnedFileIds.has(input.fileId(file))) continue - const key = input.fileKey(file) - const name = input.fileName(file) - while (nextFolder < folders.length) { - const folder = folders[nextFolder] - if (!precedes(folder, key, name)) break - entries.push(folder.item) + if (shownPinnedFiles.has(file.id)) continue + const position = { pinned: false, name: file.name, key: input.fileKey(file) } + while ( + nextFolder < folders.length && + compareSortableResources(folders[nextFolder], position, direction) <= 0 + ) { + entries.push(folders[nextFolder].item) nextFolder++ } entries.push({ kind: 'file', file, pinned: false }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx index 430fd7e6ae2..b3d2a58df18 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.tsx @@ -112,9 +112,8 @@ import { } from '@/hooks/queries/environment' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' -import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { useWorkflows } from '@/hooks/queries/workflows' -import { useWorkspaceFileLookup } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileAtPath, useWorkspaceFileLookup } from '@/hooks/queries/workspace-files' import { useWorkspaceUsageGate } from '@/hooks/queries/workspace-usage' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' @@ -1997,14 +1996,13 @@ function WorkspaceResourceDisplayContent({ const idLookup = useWorkspaceFileLookup(workspaceId, 'id', isFile ? data.id : undefined) const fileById = idLookup.data /** Only when the id names no file (or there is none): a path is the chip's fallback identity. */ - const { data: pathMatch } = useWorkspaceFileLookup( + const { file: fileFromPath } = useWorkspaceFileAtPath( workspaceId, - 'reference', isFile ? data.path : undefined, - { enabled: !data.id || fileById === null || idLookup.isError } + { + enabled: !data.id || fileById === null || idLookup.isError, + } ) - /** The reference lookup also answers by bare name; only an exact path match identifies the file. */ - const fileFromPath = findWorkspaceFileByPath(pathMatch ? [pathMatch] : undefined, data.path) const resource = useMemo<WorkspaceResourceRef>(() => { const title = diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts index c97e3944dc9..3297e3a50d7 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts @@ -2,7 +2,6 @@ import { useMemo, useSyncExternalStore } from 'react' import { isBrowserAgentAvailable } from '@/lib/browser-agent/transport' import { subscribeDesktopPreferences } from '@/lib/desktop' import { isTerminalAvailable } from '@/lib/terminal/transport' -import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { AvailableItem } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' import { byResourceMenuOrder, @@ -18,12 +17,10 @@ import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useLogsList } from '@/hooks/queries/logs' import { useMothershipChats } from '@/hooks/queries/mothership-chats' import { useTablesList } from '@/hooks/queries/tables' -import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { useWorkflowTests } from '@/hooks/queries/workflow-tests' import { useWorkflows } from '@/hooks/queries/workflows' import { useWorkspaceFileFolders } from '@/hooks/queries/workspace-file-folders' -import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' -import { useDebounce } from '@/hooks/use-debounce' +import { useWorkspaceFilePickerOptions } from '@/hooks/queries/workspace-files' /** * Placeholder id for the Browser launcher row. It never names a resource: the @@ -97,10 +94,8 @@ const NO_RESOURCE_GROUPS: AvailableItemsByType[] = [] const LOG_DROPDOWN_LIMIT = 50 -/** The files a menu offers before anything is typed: the most recently updated anywhere. */ -const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 100 }) - -const FILE_SEARCH_LIMIT = 50 +/** How many of the most recently updated files a menu offers before anything is typed. */ +const RECENT_FILE_LIMIT = 100 const LOG_DROPDOWN_FILTERS = { timeRange: 'All time' as const, @@ -147,21 +142,11 @@ export function useAvailableResources( const { data: tests, isPending: testsPending } = useWorkflowTests(workspaceId, { enabled: enabled && testsEnabled && !excludeTypes?.includes('test'), }) - const fileSearch = options?.fileSearch?.trim() ?? '' - const settledFileSearch = useDebounce(fileSearch, SEARCH_DEBOUNCE_MS) - const recentFiles = useWorkspaceFileList(workspaceId, RECENT_FILES, { - enabled: enabled && Boolean(workspaceId), - }) - const matchingFiles = useWorkspaceFileSearch( + const { files, isPending: filesPending } = useWorkspaceFilePickerOptions( workspaceId, - { q: settledFileSearch, limit: FILE_SEARCH_LIMIT }, - { enabled: enabled && Boolean(workspaceId) } + options?.fileSearch ?? '', + { limit: RECENT_FILE_LIMIT, enabled } ) - const files = settledFileSearch ? matchingFiles.data : recentFiles.data?.pages[0]?.files - /** A typed query is still resolving until its search settles: an empty menu is not "no match" yet. */ - const filesPending = - recentFiles.isPending || - (fileSearch.length > 0 && (fileSearch !== settledFileSearch || matchingFiles.isFetching)) const { data: knowledgeBases, isPending: knowledgeBasesPending } = useKnowledgeBasesQuery( workspaceId, { enabled: enabled && Boolean(workspaceId) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx index 92ec97a48f8..b7c2d25ed62 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/resource-content.tsx @@ -70,12 +70,11 @@ import { useLogByExecutionId, useLogDetail } from '@/hooks/queries/logs' import { useProjectFile } from '@/hooks/queries/projects' import { exportTable } from '@/hooks/queries/tables' import { fetchWorkflowEnvelope } from '@/hooks/queries/utils/fetch-workflow-envelope' -import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { workflowKeys } from '@/hooks/queries/utils/workflow-keys' import { mapWorkflow } from '@/hooks/queries/utils/workflow-list-query' import { useWorkflowTest } from '@/hooks/queries/workflow-tests' import { useWorkflows, WORKFLOW_STATE_STALE_TIME } from '@/hooks/queries/workflows' -import { useWorkspaceFileLookup, useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' +import { useWorkspaceFileAtPath, useWorkspaceFileRecord } from '@/hooks/queries/workspace-files' import { createProjectFileContentSource, createWorkspaceFileContentSource, @@ -734,17 +733,10 @@ const fileLogger = createLogger('EmbeddedFileActions') */ function useEmbeddedWorkspaceFile(workspaceId: string, fileId: string, filePath?: string) { const record = useWorkspaceFileRecord(workspaceId, fileId) - const pathLookup = useWorkspaceFileLookup(workspaceId, 'reference', filePath, { - enabled: record.data === null, - }) - /** The reference lookup also answers by bare name; only an exact path match is this file. */ - const fileAtPath = findWorkspaceFileByPath( - pathLookup.data ? [pathLookup.data] : undefined, - filePath - ) + const atPath = useWorkspaceFileAtPath(workspaceId, filePath, { enabled: record.data === null }) return { - file: record.data ?? fileAtPath, - isResolving: record.isFetching || pathLookup.isFetching, + file: record.data ?? atPath.file, + isResolving: record.isFetching || atPath.isFetching, } } diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx index 8401c7e01de..a5c6007d61d 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx @@ -14,6 +14,7 @@ import { requestJson } from '@/lib/api/client/request' import { fileDeleteContract } from '@/lib/api/contracts/storage-transfer' import { ROOT_FOLDER_PATH } from '@/lib/folders/paths' import { readFolderPaths } from '@/lib/folders/selection' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { formatFileSize, getExtensionFromMimeType } from '@/lib/uploads/utils/file' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { containsReference } from '@/lib/workflows/sanitization/references' @@ -33,12 +34,13 @@ import { workspaceFilesKeys, } from '@/hooks/queries/utils/workspace-file-keys' import { + distinctFiles, useCloudStorageConfigured, useUploadWorkspaceFile, useWorkspaceFileList, useWorkspaceFileSelectionCandidates, } from '@/hooks/queries/workspace-files' -import { useDebounce } from '@/hooks/use-debounce' +import { useSearchFilterValue } from '@/hooks/use-search-filter-value' import { getProviderAttachmentMaxBytes } from '@/providers/attachments' import { getProviderFromModel } from '@/providers/utils' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' @@ -49,6 +51,8 @@ const logger = createLogger('FileUpload') /** How many files the picker lists per page. */ const PICKER_PAGE_SIZE = 100 +const EMPTY_PICKER_FILES: WorkspaceFileRecord[] = [] + interface FileUploadProps { blockId: string subBlockId: string @@ -428,7 +432,7 @@ export function FileUpload({ * single concrete folder scope is applied by the server (descending when the scope does); * any other scope is applied to the page here, which keeps it correct but sparser. */ - const pickerQuery = useDebounce(inputValue.trim(), SEARCH_DEBOUNCE_MS) + const pickerQuery = useSearchFilterValue(inputValue, SEARCH_DEBOUNCE_MS).trim() const scopeFolderId = hasConcreteFolderScope && folderScopePaths.length === 1 ? folderScopePaths[0] === ROOT_FOLDER_PATH @@ -447,36 +451,34 @@ export function FileUpload({ }) ) const loadingWorkspaceFiles = pickerList.isLoading - const loadMorePickerFiles = () => { - if (!pickerList.isFetching && pickerList.hasNextPage) void pickerList.fetchNextPage() - } - const pickerFiles = useMemo( - () => pickerList.data?.pages.flatMap((page) => page.files) ?? [], - [pickerList.data] + const pickerFiles = pickerList.data?.files ?? EMPTY_PICKER_FILES + /** Whether a file sits inside the block's folder scope, as the scope's settings define it. */ + const isInFolderScope = useCallback( + (folderPath: string | null | undefined) => + folderScopePaths.some((folderScopePath) => + isFileInFolderScope(folderPath, folderScopePath, { + includeSubfolders: folderScopeIncludesSubfolders, + }) + ), + [folderScopeIncludesSubfolders, folderScopePaths] ) const scopedWorkspaceFiles = useMemo( () => hasConcreteFolderScope - ? pickerFiles.filter((workspaceFile) => - folderScopePaths.some((folderScopePath) => - isFileInFolderScope(workspaceFile.folderPath, folderScopePath, { - includeSubfolders: folderScopeIncludesSubfolders, - }) - ) - ) + ? pickerFiles.filter((workspaceFile) => isInFolderScope(workspaceFile.folderPath)) : pickerFiles, - [folderScopeIncludesSubfolders, folderScopePaths, hasConcreteFolderScope, pickerFiles] + [hasConcreteFolderScope, isInFolderScope, pickerFiles] ) + /** Re-reads the picker's files when its menu opens, if they have aged past fresh. */ + const refreshPickerOnOpen = (open: boolean) => { + if (open && pickerList.isStale) void pickerList.refetch() + } /** The selected files, resolved on their own: they need not be on the page the picker shows. */ const selections = useWorkspaceFileSelectionCandidates(isPreview ? '' : workspaceId, filesArray) /** Distinct by id: the same file on the page and among the candidates is one file. */ const workspaceFiles = useMemo( - () => [ - ...new Map( - [...selections.candidates, ...pickerFiles].map((file) => [file.id, file]) - ).values(), - ], + () => distinctFiles([selections.candidates, pickerFiles]), [selections.candidates, pickerFiles] ) @@ -498,14 +500,7 @@ export function FileUpload({ const nextFiles = filesArray.filter((file) => { const workspaceFile = findSelectedWorkspaceFile(file, selections.candidates) - return ( - !workspaceFile || - folderScopePaths.some((folderScopePath) => - isFileInFolderScope(workspaceFile.folderPath, folderScopePath, { - includeSubfolders: folderScopeIncludesSubfolders, - }) - ) - ) + return !workspaceFile || isInFolderScope(workspaceFile.folderPath) }) if (nextFiles.length === filesArray.length) return @@ -514,9 +509,8 @@ export function FileUpload({ commitValue, filesArray, folderScope, - folderScopeIncludesSubfolders, - folderScopePaths, hasConcreteFolderScope, + isInFolderScope, isPreview, multiple, selections.candidates, @@ -1004,12 +998,10 @@ export function FileUpload({ options={comboboxOptions} value={inputValue} onChange={handleComboboxChange} - onOpenChange={(open) => { - if (open) void pickerList.refetch() - }} + onOpenChange={refreshPickerOnOpen} hasMore={pickerList.hasNextPage} isLoadingMore={pickerList.isFetchingNextPage} - onLoadMore={loadMorePickerFiles} + onLoadMore={pickerList.loadMore} placeholder={loadingWorkspaceFiles ? 'Loading files...' : '+ Add More'} disabled={disabled || loadingWorkspaceFiles} editable={true} @@ -1026,13 +1018,11 @@ export function FileUpload({ selectedValue={selectedFileId} onInputChange={handleComboboxChange} onClear={(e) => handleRemoveFile(filesArray[0], e)} - onOpenChange={(open) => { - if (open) void pickerList.refetch() - }} + onOpenChange={refreshPickerOnOpen} onSearchChange={setInputValue} hasMore={pickerList.hasNextPage} isLoadingMore={pickerList.isFetchingNextPage} - onLoadMore={loadMorePickerFiles} + onLoadMore={pickerList.loadMore} disabled={disabled} isLoading={loadingWorkspaceFiles} truncateMiddle={truncateMiddle} @@ -1053,12 +1043,10 @@ export function FileUpload({ options={comboboxOptions} value={inputValue} onChange={handleComboboxChange} - onOpenChange={(open) => { - if (open) void pickerList.refetch() - }} + onOpenChange={refreshPickerOnOpen} hasMore={pickerList.hasNextPage} isLoadingMore={pickerList.isFetchingNextPage} - onLoadMore={loadMorePickerFiles} + onLoadMore={pickerList.loadMore} placeholder={loadingWorkspaceFiles ? 'Loading files...' : 'Select or upload file'} disabled={disabled || loadingWorkspaceFiles} editable={true} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx index 79b6ad8eb19..2f06dc82160 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx @@ -14,7 +14,6 @@ import { useMemo } from 'react' import { DropdownMenuItem } from '@sim/emcn' import Link from 'next/link' import { useParams } from 'next/navigation' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { buildFlyoutEntries, FOLDERED_RESOURCE_HEADERS, @@ -23,6 +22,7 @@ import { import { CollapsedResourceFlyout } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu' import { useFolders } from '@/hooks/queries/folders' import { usePinnedIds } from '@/hooks/queries/pinned-items' +import { usePinnedWorkspaceFiles } from '@/hooks/queries/pinned-workspace-files' import { useTablesList } from '@/hooks/queries/tables' import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { @@ -30,9 +30,9 @@ import { type WorkspaceFileFolderApi, } from '@/hooks/queries/workspace-file-folders' import { + distinctFiles, useWorkspaceFileFolderSizes, useWorkspaceFileList, - useWorkspaceFileLookups, } from '@/hooks/queries/workspace-files' const TABLE_META = FOLDERED_RESOURCE_HEADERS.table @@ -121,39 +121,27 @@ function FilesFlyoutLevel({ }) ) const folderStats = useWorkspaceFileFolderSizes(workspaceId, [folderId]) - const pinnedFileIds = usePinnedIds(workspaceId, 'file') + const pinned = usePinnedWorkspaceFiles(workspaceId, folderId) const pinnedFolderIds = usePinnedIds(workspaceId, 'folder') - const pinnedFileIdList = useMemo(() => [...pinnedFileIds], [pinnedFileIds]) - const pinnedFiles = useWorkspaceFileLookups(workspaceId, 'id', pinnedFileIdList) const entries = useMemo(() => { - const files = new Map<string, WorkspaceFileRecord>() - for (const fileId of pinnedFileIdList) { - const file = pinnedFiles.byValue.get(fileId) - if (file && (file.folderId ?? null) === folderId) files.set(file.id, file) - } - for (const file of fileList.data?.pages[0]?.files ?? []) files.set(file.id, file) + const files = distinctFiles([pinned.files, fileList.data?.files]) return buildFlyoutEntries({ folders: folders .filter((folder) => (folder.parentId ?? null) === folderId) .map((folder) => ({ ...folder, parentId: null })), - items: [...files.values()].map((file) => ({ ...file, folderId: null })), + items: files.map((file) => ({ ...file, folderId: null })), pinnedFolderIds, - pinnedItemIds: pinnedFileIds, + pinnedItemIds: pinned.pinnedFileIds, hrefForItem: (file) => `/workspace/${workspaceId}/${FILE_META.listSegment}/${file.id}`, }) - }, [ - fileList.data, - folders, - folderId, - pinnedFiles.byValue, - pinnedFileIdList, - pinnedFolderIds, - pinnedFileIds, - workspaceId, - ]) + }, [fileList.data, folders, folderId, pinned, pinnedFolderIds, workspaceId]) - const hasSubfolders = (id: string) => folders.some((folder) => folder.parentId === id) + /** The folders that have subfolders, indexed once for the per-folder emptiness check. */ + const parentsWithSubfolders = useMemo( + () => new Set(folders.map((folder) => folder.parentId)), + [folders] + ) return ( <CollapsedResourceFlyout @@ -162,7 +150,7 @@ function FilesFlyoutLevel({ isLoading={isResolving(fileList) || !foldersResolved} emptyLabel={folderId ? 'No files in this folder' : 'No files yet'} lazyFolders={{ - isEmpty: (id) => !hasSubfolders(id) && folderStats.get(id)?.fileCount === 0, + isEmpty: (id) => !parentsWithSubfolders.has(id) && folderStats.get(id)?.fileCount === 0, render: (id) => ( <FilesFlyoutLevel workspaceId={workspaceId} diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx index 0eeb9ec8ac6..3fe38642ea8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/search-modal.tsx @@ -47,7 +47,6 @@ import { getFolderPathNames } from '@/lib/folders/tree' import { sendMothershipMessage } from '@/lib/mothership/events' import { captureEvent } from '@/lib/posthog/client' import { toSearchToken } from '@/lib/search/tokens' -import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { hasTriggerCapability } from '@/lib/workflows/triggers/utils' import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path' import { useInvokeGlobalCommand } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' @@ -93,9 +92,7 @@ import { useWorkspaceAccessRequestFeatures } from '@/ee/access-requests/componen import { useFolderMap } from '@/hooks/queries/folders' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useTablesList } from '@/hooks/queries/tables' -import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' -import { useWorkspaceFileList, useWorkspaceFileSearch } from '@/hooks/queries/workspace-files' -import { useDebounce } from '@/hooks/use-debounce' +import { useWorkspaceFilePickerOptions } from '@/hooks/queries/workspace-files' import { usePermissionConfig } from '@/hooks/use-permission-config' import { useSettingsNavigation } from '@/hooks/use-settings-navigation' import type { WorkflowFolder } from '@/stores/folders/types' @@ -126,11 +123,6 @@ const MAX_SEARCH_RESULTS = 50 /** Stable empty default so a pending folder map does not remount the memos below. */ const EMPTY_FOLDER_MAP: Record<string, WorkflowFolder> = {} -/** The files the palette offers before anything is typed: the most recently updated anywhere. */ -const RECENT_FILES = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit: 50 }) - -const FILE_SEARCH_LIMIT = MAX_SEARCH_RESULTS - export type { SearchModalProps } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/utils' type SearchModalContentProps = Omit<SearchModalProps, 'open'> @@ -688,16 +680,10 @@ function SearchModalContent({ * offers the most recently updated; a query, once typing settles, asks the server for the * best matches by name and folder path, which the palette then ranks as it ranks everything. */ - const fileQuery = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) - const recentFiles = useWorkspaceFileList(workspaceId, RECENT_FILES, { + const { files: fetchedFiles } = useWorkspaceFilePickerOptions(workspaceId, search, { + limit: MAX_SEARCH_RESULTS, enabled: !permissionConfig.hideFilesTab, }) - const matchingFiles = useWorkspaceFileSearch( - workspaceId, - { q: fileQuery, limit: FILE_SEARCH_LIMIT }, - { enabled: !permissionConfig.hideFilesTab } - ) - const fetchedFiles = fileQuery ? matchingFiles.data : recentFiles.data?.pages[0]?.files const files = useMemo( () => permissionConfig.hideFilesTab diff --git a/apps/sim/hooks/queries/pinned-workspace-files.ts b/apps/sim/hooks/queries/pinned-workspace-files.ts new file mode 100644 index 00000000000..4f8cf6f35fe --- /dev/null +++ b/apps/sim/hooks/queries/pinned-workspace-files.ts @@ -0,0 +1,27 @@ +import { useMemo } from 'react' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { usePinnedIds } from '@/hooks/queries/pinned-items' +import { useWorkspaceFileLookups } from '@/hooks/queries/workspace-files' + +/** + * The pinned files in one folder (`null` for the root), looked up by id so a folder's pins can + * lead it before any page of its list has reached them. In pin order. + */ +export function usePinnedWorkspaceFiles( + workspaceId: string, + folderId: string | null, + options?: { enabled?: boolean } +): { pinnedFileIds: ReadonlySet<string>; files: WorkspaceFileRecord[] } { + const pinnedFileIds = usePinnedIds(workspaceId, 'file') + const ids = useMemo(() => [...pinnedFileIds], [pinnedFileIds]) + const lookups = useWorkspaceFileLookups(workspaceId, 'id', ids, options) + const files = useMemo( + () => + ids.flatMap((fileId) => { + const file = lookups.byValue.get(fileId) + return file && (file.folderId ?? null) === folderId ? [file] : [] + }), + [ids, lookups.byValue, folderId] + ) + return { pinnedFileIds, files } +} diff --git a/apps/sim/hooks/queries/utils/find-workspace-file-by-src.test.ts b/apps/sim/hooks/queries/utils/find-workspace-file-by-src.test.ts deleted file mode 100644 index d3fc93c39d2..00000000000 --- a/apps/sim/hooks/queries/utils/find-workspace-file-by-src.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { findWorkspaceFileBySrc } from '@/hooks/queries/utils/find-workspace-file-by-src' - -function record(over: Partial<WorkspaceFileRecord>): WorkspaceFileRecord { - return { id: 'wf_x', key: 'workspace/ws1/x.png', ...over } as WorkspaceFileRecord -} - -const records = [ - record({ id: 'wf_a', key: 'workspace/ws1/a.png' }), - record({ id: 'wf_b', key: 'workspace/ws1/b.png' }), -] - -const serveUrl = (key: string) => `/api/files/serve/${encodeURIComponent(key)}?context=workspace` - -describe('findWorkspaceFileBySrc', () => { - it('returns undefined for a serve URL whose key is not in the list', () => { - expect(findWorkspaceFileBySrc(records, serveUrl('workspace/ws1/missing.png'))).toBeUndefined() - }) - - it('returns undefined for external, data:, and undefined srcs', () => { - expect(findWorkspaceFileBySrc(records, 'https://example.com/x.png')).toBeUndefined() - expect(findWorkspaceFileBySrc(records, 'data:image/png;base64,AAAA')).toBeUndefined() - expect(findWorkspaceFileBySrc(records, undefined)).toBeUndefined() - }) -}) diff --git a/apps/sim/hooks/queries/utils/find-workspace-file-by-src.ts b/apps/sim/hooks/queries/utils/find-workspace-file-by-src.ts index 90811ce8bad..970dd289311 100644 --- a/apps/sim/hooks/queries/utils/find-workspace-file-by-src.ts +++ b/apps/sim/hooks/queries/utils/find-workspace-file-by-src.ts @@ -1,6 +1,5 @@ import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' /** * Resolve the workspace file a canonical VFS path points at (`files/docs/a.md`). @@ -17,20 +16,3 @@ export function findWorkspaceFileByPath( canonicalWorkspaceFilePath({ folderPath: record.folderPath, name: record.name }) === path ) } - -/** - * Resolve the workspace file record an embedded image `src` points at, matching the persisted serve-URL - * shape by storage key or file id. Returns `undefined` for external / `data:` / unrecognized srcs, and - * when the file list isn't loaded — callers then fall back to on-load measurement rather than reserving - * from metadata. - */ -export function findWorkspaceFileBySrc( - records: WorkspaceFileRecord[] | undefined, - src: string | undefined -): WorkspaceFileRecord | undefined { - const ref = src ? extractEmbeddedFileRef(src) : null - if (!ref || !records) return undefined - return 'key' in ref - ? records.find((record) => record.key === ref.key) - : records.find((record) => record.id === ref.fileId) -} diff --git a/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts b/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts new file mode 100644 index 00000000000..46561c44c1a --- /dev/null +++ b/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts @@ -0,0 +1,159 @@ +/** + * @vitest-environment node + */ +import { QueryClient, QueryObserver } from '@tanstack/react-query' +import { describe, expect, it } from 'vitest' +import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' +import { + findCachedWorkspaceFile, + indexCachedWorkspaceFiles, + invalidateWorkspaceFileReads, + isWorkspaceFileRecordKey, + snapshotWorkspaceFileReads, + updateCachedWorkspaceFile, +} from '@/hooks/queries/utils/workspace-file-cache' +import { + type WorkspaceFileListPage, + workspaceFileListParams, + workspaceFilesKeys, +} from '@/hooks/queries/utils/workspace-file-keys' + +function file(id: string, overrides: Partial<WorkspaceFileRecord> = {}): WorkspaceFileRecord { + return { + id, + workspaceId: 'ws-1', + name: `${id}.md`, + key: `workspace/ws-1/${id}.md`, + path: `/api/files/serve/${id}`, + size: 1, + type: 'text/markdown', + uploadedBy: 'user-1', + folderId: null, + uploadedAt: new Date('2026-09-01T00:00:00.000Z'), + updatedAt: new Date('2026-09-01T00:00:00.000Z'), + ...overrides, + } as WorkspaceFileRecord +} + +const pages = (...files: WorkspaceFileRecord[][]) => ({ + pages: files.map( + (page, index): WorkspaceFileListPage => ({ + files: page, + nextCursor: index < files.length - 1 ? `c${index + 1}` : null, + }) + ), + pageParams: files.map((_, index) => (index === 0 ? null : `c${index}`)), +}) + +const liveKey = workspaceFilesKeys.list( + 'ws-1', + workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) +) +const archivedKey = workspaceFilesKeys.list( + 'ws-1', + workspaceFileListParams({ scope: 'archived', folderId: null, sortBy: 'name', sortOrder: 'asc' }) +) + +const listIds = (client: QueryClient, key: readonly unknown[]) => + client + .getQueryData<ReturnType<typeof pages>>(key) + ?.pages.flatMap((page) => page.files.map((entry) => entry.id)) + +describe('invalidateWorkspaceFileReads', () => { + it('cuts an unobserved list to its first page but leaves a mounted one whole', async () => { + const client = new QueryClient() + const mountedKey = workspaceFilesKeys.list( + 'ws-1', + workspaceFileListParams({ folderId: null, sortBy: 'size', sortOrder: 'asc' }) + ) + client.setQueryData(liveKey, pages([file('a')], [file('b')])) + client.setQueryData(mountedKey, pages([file('a')], [file('b')])) + const observer = new QueryObserver(client, { queryKey: mountedKey, enabled: false }) + const unsubscribe = observer.subscribe(() => {}) + const before = client.getQueryState(liveKey)?.dataUpdatedAt + + await invalidateWorkspaceFileReads(client, 'ws-1') + + expect(listIds(client, liveKey)).toEqual(['a']) + expect(client.getQueryState(liveKey)?.dataUpdatedAt).toBe(before) + expect(client.getQueryState(liveKey)?.isInvalidated).toBe(true) + expect(listIds(client, mountedKey)).toEqual(['a', 'b']) + unsubscribe() + }) + + it('invalidates folder sizes with the rest of the workspace metadata, and no other workspace', async () => { + const client = new QueryClient() + const stats = workspaceFilesKeys.folderStat('ws-1', ['root']) + const other = workspaceFilesKeys.folderStat('ws-2', ['root']) + client.setQueryData(stats, new Map()) + client.setQueryData(other, new Map()) + await invalidateWorkspaceFileReads(client, 'ws-1') + expect(client.getQueryState(stats)?.isInvalidated).toBe(true) + expect(client.getQueryState(other)?.isInvalidated).toBe(false) + }) +}) + +describe('updateCachedWorkspaceFile', () => { + it('drops a deleted file from live reads only: an archived list is where it now belongs', () => { + const client = new QueryClient() + client.setQueryData(liveKey, pages([file('a'), file('b')])) + client.setQueryData(archivedKey, pages([file('a', { deletedAt: new Date() })])) + const snapshot = snapshotWorkspaceFileReads(client, 'ws-1', ['list']) + + updateCachedWorkspaceFile(client, 'ws-1', 'a', () => null, ['list']) + + expect(listIds(client, liveKey)).toEqual(['b']) + expect(listIds(client, archivedKey)).toEqual(['a']) + expect(snapshot.map(([key]) => key)).toEqual([liveKey]) + }) + + it('patches only the reads named, so a rename leaves lists and name references to refetch', () => { + const client = new QueryClient() + const byId = workspaceFilesKeys.lookup('ws-1', 'id', 'a') + const byName = workspaceFilesKeys.lookup('ws-1', 'reference', 'a.md') + client.setQueryData(liveKey, pages([file('a')])) + client.setQueryData(byId, file('a')) + client.setQueryData(byName, file('a')) + client.setQueryData(workspaceFilesKeys.record('ws-1', 'a'), file('a')) + + updateCachedWorkspaceFile(client, 'ws-1', 'a', (entry) => ({ ...entry, name: 'z.md' }), [ + 'identityLookup', + 'record', + ]) + + const name = (key: readonly unknown[]) => client.getQueryData<WorkspaceFileRecord>(key)?.name + expect(name(byId)).toBe('z.md') + expect(name(workspaceFilesKeys.record('ws-1', 'a'))).toBe('z.md') + expect(name(byName)).toBe('a.md') + expect(client.getQueryData<ReturnType<typeof pages>>(liveKey)?.pages[0].files[0].name).toBe( + 'a.md' + ) + }) +}) + +describe('cached file seeds', () => { + it('finds the freshest copy across reads, by one scan for a batch', () => { + const client = new QueryClient() + client.setQueryData(liveKey, pages([file('a', { name: 'old.md' })]), { updatedAt: 1 }) + client.setQueryData(workspaceFilesKeys.record('ws-1', 'a'), file('a', { name: 'new.md' }), { + updatedAt: 2, + }) + client.setQueryData(archivedKey, pages([file('gone')]), { updatedAt: 3 }) + + const index = indexCachedWorkspaceFiles(client, 'ws-1', 'id') + expect(index.get('a')).toEqual({ + file: expect.objectContaining({ name: 'new.md' }), + updatedAt: 2, + }) + expect(index.has('gone')).toBe(false) + expect(findCachedWorkspaceFile(client, 'ws-1', (entry) => entry.id === 'a')).toEqual( + index.get('a') + ) + }) + + it('names file-holding reads of any workspace when no workspace is given', () => { + expect(isWorkspaceFileRecordKey(workspaceFilesKeys.record('ws-9', 'a'))).toBe(true) + expect(isWorkspaceFileRecordKey(workspaceFilesKeys.record('ws-9', 'a'), 'ws-1')).toBe(false) + expect(isWorkspaceFileRecordKey(workspaceFilesKeys.folderStat('ws-9', ['root']))).toBe(false) + }) +}) diff --git a/apps/sim/hooks/queries/utils/workspace-file-cache.ts b/apps/sim/hooks/queries/utils/workspace-file-cache.ts index a35dd8aeefa..07161adeb72 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-cache.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-cache.ts @@ -7,31 +7,67 @@ import { } from '@/hooks/queries/utils/workspace-file-keys' /** - * The cache entries that hold workspace file metadata, one per read shape. A file can sit in + * The cache entries that hold workspace file records, one per read shape. A file can sit in * several at once — a list page, a search result, a lookup, its own record — so a write has to - * reach all of them, and a reader looking for a record it already has can find it in any. + * reach each one it affects, and a reader looking for a record it already has can find it in any. */ -const METADATA_READS = ['list', 'search', 'lookup', 'record'] as const +const FILE_READS = ['list', 'search', 'lookup', 'record'] as const + +/** Every workspace file metadata read: the {@link FILE_READS} and the folder size rollups. */ +const METADATA_READS = [...FILE_READS, 'folderStats'] as const type ListData = InfiniteData<WorkspaceFileListPage, string | null> type SearchData = WorkspaceFileRecord[] type LookupData = WorkspaceFileRecord | null type RecordData = WorkspaceFileRecord | null -/** Whether `queryKey` is one of {@link METADATA_READS} for `workspaceId`. */ -export function isWorkspaceFileMetadataKey(queryKey: QueryKey, workspaceId: string): boolean { +function isReadOf( + reads: readonly string[], + queryKey: QueryKey, + workspaceId: string | undefined +): boolean { return ( queryKey[0] === workspaceFilesKeys.all[0] && - METADATA_READS.some((read) => read === queryKey[1]) && - queryKey[2] === workspaceId + reads.includes(String(queryKey[1])) && + (workspaceId === undefined || queryKey[2] === workspaceId) ) } -/** Every metadata read of the workspace, for `refetchQueries`/`isFetching`/`cancelQueries`. */ +/** Whether `queryKey` holds file records of `workspaceId` (of any workspace when omitted). */ +export function isWorkspaceFileRecordKey(queryKey: QueryKey, workspaceId?: string): boolean { + return isReadOf(FILE_READS, queryKey, workspaceId) +} + +/** Every metadata read of the workspace, for `invalidateQueries`/`refetchQueries`/`cancelQueries`. */ export function workspaceFileMetadataFilter(workspaceId: string) { return { predicate: (query: { queryKey: QueryKey }) => - isWorkspaceFileMetadataKey(query.queryKey, workspaceId), + isReadOf(METADATA_READS, query.queryKey, workspaceId), + } +} + +/** A list of live files: an archived list holds deleted files, which no live read should see. */ +function isLiveList(queryKey: QueryKey): boolean { + return queryKey[1] === 'list' && isRecordLike(queryKey[3]) && queryKey[3].scope === 'active' +} + +/** + * Cuts each unobserved list of the workspace back to its first page, so the refetch an + * invalidation leaves for its next mount reads one page instead of every page it once walked. + * A mounted list keeps its pages: the rows on screen stay where they are while it refetches. + */ +function trimUnobservedLists(queryClient: QueryClient, workspaceId: string): void { + for (const query of queryClient + .getQueryCache() + .findAll({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) })) { + // A list key's data is always its infinite query's pages. + const data = query.state.data as ListData | undefined + if (query.getObserversCount() > 0 || !data || data.pages.length <= 1) continue + queryClient.setQueryData<ListData>( + query.queryKey, + { pages: data.pages.slice(0, 1), pageParams: data.pageParams.slice(0, 1) }, + { updatedAt: query.state.dataUpdatedAt } + ) } } @@ -40,26 +76,42 @@ export function workspaceFileMetadataFilter(workspaceId: string) { * lookups, and records — and settles once the mounted ones have refetched. */ export function invalidateWorkspaceFileReads(queryClient: QueryClient, workspaceId: string) { + trimUnobservedLists(queryClient, workspaceId) + return queryClient.invalidateQueries(workspaceFileMetadataFilter(workspaceId)) +} + +/** + * Invalidates the reads whose membership or order a write to one file can change — lists, + * searches, name references, and (with `folderStats`) folder sizes — for a write that patched + * the file's own record and identity lookups instead. Settles once the mounted ones refetched. + */ +export function invalidateWorkspaceFileListings( + queryClient: QueryClient, + workspaceId: string, + options: { folderStats?: boolean } = {} +) { + trimUnobservedLists(queryClient, workspaceId) return Promise.all([ queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }), queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceSearches(workspaceId) }), queryClient.invalidateQueries({ - queryKey: workspaceFilesKeys.workspaceFolderStats(workspaceId), + queryKey: [...workspaceFilesKeys.workspaceLookups(workspaceId), 'reference'], }), - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceLookups(workspaceId) }), - queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.workspaceRecords(workspaceId) }), + options.folderStats + ? queryClient.invalidateQueries({ + queryKey: workspaceFilesKeys.workspaceFolderStats(workspaceId), + }) + : undefined, ]) } -/** The files a cached metadata read holds, by the read's shape (its key's second segment). */ +/** The files a cached read holds, by the read's shape (its key's second segment). */ function filesOf(queryKey: QueryKey, data: unknown): readonly WorkspaceFileRecord[] { if (!data) return [] + // Each read's data shape is fixed by its key segment, which this switches on. switch (queryKey[1]) { case 'list': - /** Only a live list seeds a live read: an archived one holds deleted files. */ - return isRecordLike(queryKey[3]) && queryKey[3].scope === 'active' - ? (data as ListData).pages.flatMap((page) => page.files) - : [] + return isLiveList(queryKey) ? (data as ListData).pages.flatMap((page) => page.files) : [] case 'search': return data as SearchData case 'lookup': @@ -70,38 +122,111 @@ function filesOf(queryKey: QueryKey, data: unknown): readonly WorkspaceFileRecor } } +/** A cached copy of a file and when the read holding it last succeeded. */ +export interface CachedWorkspaceFile { + file: WorkspaceFileRecord + updatedAt: number +} + /** - * The freshest copy of a file any cached read of its workspace holds, and when that read last - * succeeded, without fetching. `undefined` when no read has seen it. A record a reader opens - * this way starts from what the list it was clicked in already showed, aged as that list is. + * The freshest copy of each file any cached read of the workspace holds, by `field`, without + * fetching — one scan for a caller resolving many values. A record a reader opens this way + * starts from what the list it was clicked in already showed, aged as that list is. */ +export function indexCachedWorkspaceFiles( + queryClient: QueryClient, + workspaceId: string, + field: 'id' | 'key' +): ReadonlyMap<string, CachedWorkspaceFile> { + const index = new Map<string, CachedWorkspaceFile>() + for (const query of queryClient.getQueryCache().findAll()) { + if (!isWorkspaceFileRecordKey(query.queryKey, workspaceId)) continue + const { data, dataUpdatedAt } = query.state + for (const file of filesOf(query.queryKey, data)) { + const known = index.get(file[field]) + if (!known || dataUpdatedAt > known.updatedAt) { + index.set(file[field], { file, updatedAt: dataUpdatedAt }) + } + } + } + return index +} + +/** The freshest cached copy of one file that passes `match`; see {@link indexCachedWorkspaceFiles}. */ export function findCachedWorkspaceFile( queryClient: QueryClient, workspaceId: string, match: (file: WorkspaceFileRecord) => boolean -): { file: WorkspaceFileRecord; updatedAt: number } | undefined { - let found: { file: WorkspaceFileRecord; updatedAt: number } | undefined - for (const query of queryClient - .getQueryCache() - .findAll(workspaceFileMetadataFilter(workspaceId))) { +): CachedWorkspaceFile | undefined { + let found: CachedWorkspaceFile | undefined + for (const query of queryClient.getQueryCache().findAll()) { + if (!isWorkspaceFileRecordKey(query.queryKey, workspaceId)) continue const { data, dataUpdatedAt } = query.state if (found && dataUpdatedAt <= found.updatedAt) continue - // Each read's data shape is fixed by its key segment, which `filesOf` switches on. const file = filesOf(query.queryKey, data).find(match) if (file) found = { file, updatedAt: dataUpdatedAt } } return found } -/** The cached metadata entries of one workspace, as `getQueriesData` returns them, for a rollback. */ +/** + * A cached read a write to one file can rewrite in place: + * - `list` and `search`: the live lists and name searches holding it; + * - `identityLookup`: lookups by id or storage key, which a rename leaves pointing at the file; + * - `referenceLookup`: lookups by name or path, which a rename re-points; + * - `record`: the file's own record. + */ +export type PatchableWorkspaceFileRead = + | 'list' + | 'search' + | 'identityLookup' + | 'referenceLookup' + | 'record' + +/** Every read: what a write that changes neither a file's name nor its place can patch. */ +export const EVERY_WORKSPACE_FILE_READ: readonly PatchableWorkspaceFileRead[] = [ + 'list', + 'search', + 'identityLookup', + 'referenceLookup', + 'record', +] + +function readOf(queryKey: QueryKey): PatchableWorkspaceFileRead | undefined { + switch (queryKey[1]) { + case 'list': + return isLiveList(queryKey) ? 'list' : undefined + case 'search': + return 'search' + case 'lookup': + return queryKey[3] === 'reference' ? 'referenceLookup' : 'identityLookup' + case 'record': + return 'record' + default: + return undefined + } +} + +function patchableReadsFilter(workspaceId: string, reads: readonly PatchableWorkspaceFileRead[]) { + return { + predicate: (query: { queryKey: QueryKey }) => { + if (!isWorkspaceFileRecordKey(query.queryKey, workspaceId)) return false + const read = readOf(query.queryKey) + return read !== undefined && reads.includes(read) + }, + } +} + +/** The cached entries an optimistic write patches, as `getQueriesData` returns them, for a rollback. */ export type WorkspaceFileReadSnapshot = [QueryKey, unknown][] -/** Every cached metadata read of the workspace, for an optimistic write to restore on failure. */ +/** The cached `reads` of the workspace, for an optimistic write to restore on failure. */ export function snapshotWorkspaceFileReads( queryClient: QueryClient, - workspaceId: string + workspaceId: string, + reads: readonly PatchableWorkspaceFileRead[] ): WorkspaceFileReadSnapshot { - return queryClient.getQueriesData(workspaceFileMetadataFilter(workspaceId)) + return queryClient.getQueriesData(patchableReadsFilter(workspaceId, reads)) } export function restoreWorkspaceFileReads( @@ -112,15 +237,19 @@ export function restoreWorkspaceFileReads( } /** - * Rewrites one file in every cached read of its workspace: `update` returns the new record, or + * Rewrites one file in the cached `reads` of its workspace: `update` returns the new record, or * `null` to drop the file (a delete). Lists, searches, and lookups lose a dropped file; its own * record is left for the caller, since a reader still on it is mid-navigation away. + * + * Patch only reads whose membership and order the change cannot affect: a rename can move a + * file in a name-sorted list or out of a search, so those are refetched instead. */ export function updateCachedWorkspaceFile( queryClient: QueryClient, workspaceId: string, fileId: string, - update: (file: WorkspaceFileRecord) => WorkspaceFileRecord | null + update: (file: WorkspaceFileRecord) => WorkspaceFileRecord | null, + reads: readonly PatchableWorkspaceFileRead[] ): void { const apply = (files: WorkspaceFileRecord[]) => { let changed = false @@ -137,29 +266,38 @@ export function updateCachedWorkspaceFile( return changed ? next : files } - queryClient.setQueriesData<ListData>( - { queryKey: workspaceFilesKeys.workspaceLists(workspaceId) }, - (data) => { - if (!data) return data - let changed = false - const pages = data.pages.map((page) => { - const files = apply(page.files) - if (files === page.files) return page - changed = true - return { ...page, files } - }) - return changed ? { ...data, pages } : data + for (const { queryKey } of queryClient + .getQueryCache() + .findAll(patchableReadsFilter(workspaceId, reads))) { + switch (readOf(queryKey)) { + case 'list': + queryClient.setQueryData<ListData>(queryKey, (data) => { + if (!data) return data + let changed = false + const pages = data.pages.map((page) => { + const files = apply(page.files) + if (files === page.files) return page + changed = true + return { ...page, files } + }) + return changed ? { ...data, pages } : data + }) + break + case 'search': + queryClient.setQueryData<SearchData>(queryKey, (data) => (data ? apply(data) : data)) + break + case 'identityLookup': + case 'referenceLookup': + queryClient.setQueryData<LookupData>(queryKey, (data) => + data?.id === fileId ? update(data) : data + ) + break + case 'record': + if (queryKey[3] !== fileId) break + queryClient.setQueryData<RecordData>(queryKey, (data) => + data ? (update(data) ?? data) : data + ) + break } - ) - queryClient.setQueriesData<SearchData>( - { queryKey: workspaceFilesKeys.workspaceSearches(workspaceId) }, - (data) => (data ? apply(data) : data) - ) - queryClient.setQueriesData<LookupData>( - { queryKey: workspaceFilesKeys.workspaceLookups(workspaceId) }, - (data) => (data?.id === fileId ? update(data) : data) - ) - queryClient.setQueryData<RecordData>(workspaceFilesKeys.record(workspaceId, fileId), (data) => - data ? (update(data) ?? data) : data - ) + } } diff --git a/apps/sim/hooks/queries/utils/workspace-file-keys.ts b/apps/sim/hooks/queries/utils/workspace-file-keys.ts index fef15074de3..57d512b51e1 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-keys.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-keys.ts @@ -82,35 +82,49 @@ export function workspaceFileListParams(input: { limit?: number }): WorkspaceFileListParams { const q = input.q?.trim() + const types = canonicalSet(input.types) + const sizes = canonicalSet(input.sizes) + const uploadedBy = canonicalSet(input.uploadedBy) return { scope: input.scope ?? 'active', ...(input.folderId === undefined ? {} : { folderId: input.folderId }), ...(input.folderId !== undefined && input.recursive ? { recursive: true } : {}), ...(q ? { q } : {}), - ...(canonicalSet(input.types) ? { types: canonicalSet(input.types) } : {}), - ...(canonicalSet(input.sizes) ? { sizes: canonicalSet(input.sizes) } : {}), - ...(canonicalSet(input.uploadedBy) ? { uploadedBy: canonicalSet(input.uploadedBy) } : {}), + ...(types ? { types } : {}), + ...(sizes ? { sizes } : {}), + ...(uploadedBy ? { uploadedBy } : {}), sortBy: input.sortBy, sortOrder: input.sortOrder, limit: input.limit ?? WORKSPACE_FILE_LIST_PAGE_SIZE, } } +/** + * The folder half of a list or search query: absent for every folder, else the folder (`root` + * for `null`) and whether its subfolders count. + */ +export function workspaceFileFolderQuery(params: { + folderId?: string | null + recursive?: boolean +}): { folderId?: string; recursive?: boolean } { + return params.folderId === undefined + ? {} + : { folderId: params.folderId ?? 'root', recursive: params.recursive } +} + /** * The `GET /api/workspaces/[id]/files` query for one page of a list: a `null` folder is spelled - * `root`, set filters comma-joined. The client fetch and the server prefetch both send this, so - * a prefetched page is the page its key names. + * `root`, set filters sent as repeated params. The client fetch and the server prefetch both + * send this, so a prefetched page is the page its key names. */ export function workspaceFileListQuery(params: WorkspaceFileListParams, cursor: string | null) { return { scope: params.scope, - ...(params.folderId === undefined - ? {} - : { folderId: params.folderId ?? 'root', recursive: params.recursive }), + ...workspaceFileFolderQuery(params), q: params.q, - types: params.types?.join(','), - sizes: params.sizes?.join(','), - uploadedBy: params.uploadedBy?.join(','), + types: params.types, + sizes: params.sizes, + uploadedBy: params.uploadedBy, sortBy: params.sortBy, sortOrder: params.sortOrder, limit: params.limit, @@ -140,8 +154,9 @@ export const workspaceFilesKeys = { folderStats: () => [...workspaceFilesKeys.all, 'folderStats'] as const, workspaceFolderStats: (workspaceId: string) => [...workspaceFilesKeys.folderStats(), workspaceId] as const, - folderStat: (workspaceId: string, parentId: string | null) => - [...workspaceFilesKeys.workspaceFolderStats(workspaceId), parentId ?? 'root'] as const, + /** The rollup for a set of parents; `parentIds` sorted, de-duplicated, `root` for `null`. */ + folderStat: (workspaceId: string, parentIds: readonly string[]) => + [...workspaceFilesKeys.workspaceFolderStats(workspaceId), parentIds] as const, lookups: () => [...workspaceFilesKeys.all, 'lookup'] as const, workspaceLookups: (workspaceId: string) => [...workspaceFilesKeys.lookups(), workspaceId] as const, diff --git a/apps/sim/hooks/queries/workflow-search-replace.test.ts b/apps/sim/hooks/queries/workflow-search-replace.test.ts index d03f6edd332..ca29129fe38 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.test.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' import type { WorkflowSearchMatch } from '@/lib/workflows/search-replace/types' -import { buildWorkflowSearchMcpToolReplacementOptions } from '@/hooks/queries/workflow-search-replace' +import { + buildWorkflowSearchMcpToolReplacementOptions, + workflowSearchFileLookups, +} from '@/hooks/queries/workflow-search-replace' function createMcpToolMatch(serverId?: string): WorkflowSearchMatch { return { @@ -62,3 +65,24 @@ describe('buildWorkflowSearchMcpToolReplacementOptions', () => { ]) }) }) + +describe('workflowSearchFileLookups', () => { + it('looks a value up only as what it can be', () => { + expect( + workflowSearchFileLookups([ + 'wf_abc', + 'notes.md', + 'docs/notes.md', + 'workspace/ws-1/1731-ab-doc.md', + '/api/files/serve/workspace%2Fws-1%2F1731-ab-img.png?context=workspace', + ]) + ).toEqual({ + references: ['wf_abc', 'notes.md', 'docs/notes.md', 'workspace/ws-1/1731-ab-doc.md'], + keyValues: [ + 'docs/notes.md', + 'workspace/ws-1/1731-ab-doc.md', + 'workspace/ws-1/1731-ab-img.png', + ], + }) + }) +}) diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index f7b4df76a72..57da58d5318 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -30,7 +30,6 @@ import { type SelectorKey, } from '@/lib/selectors/manifest' import type { SelectorOption, SelectorScope } from '@/lib/selectors/types' -import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractStorageKey } from '@/lib/uploads/utils/file' import { getWorkflowSearchMatchResourceGroupKey } from '@/lib/workflows/search-replace/resources' import type { @@ -45,11 +44,10 @@ import { } from '@/hooks/queries/oauth/credentials' import { collectDuplicateNames, disambiguateLabelByFolder } from '@/hooks/queries/utils/folder-tree' import { selectorQueryRoots } from '@/hooks/queries/utils/selector-keys' -import { workspaceFileListParams } from '@/hooks/queries/utils/workspace-file-keys' import { - useWorkspaceFileList, + distinctFiles, useWorkspaceFileLookups, - useWorkspaceFileSearch, + useWorkspaceFilePickerOptions, } from '@/hooks/queries/workspace-files' import type { WorkflowFolder } from '@/stores/folders/types' @@ -347,6 +345,29 @@ export function useWorkflowSearchTableDetails( }) } +/** + * What to look a stored file value up by. It is an id, a storage key, a serve path, or a name: a + * reference lookup answers an id (its first rule), a path, and a name; a storage key — every one + * has a `/` — and a serve path also need the key lookup, which a plain name or id never would. + */ +export function workflowSearchFileLookups(rawValues: readonly string[]): { + references: string[] + keyValues: string[] +} { + const references = new Set<string>() + const keyValues = new Set<string>() + for (const rawValue of rawValues) { + const key = extractStorageKey(rawValue) + if (key !== rawValue) { + keyValues.add(key) + continue + } + references.add(rawValue) + if (rawValue.includes('/')) keyValues.add(rawValue) + } + return { references: [...references], keyValues: [...keyValues] } +} + export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], workspaceId?: string) { const fileMatches = useMemo( () => @@ -356,28 +377,21 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor ), [matches] ) - /** A stored file value is an id, a storage key, a serve path, or a name: each is looked up as each. */ - const rawValues = useMemo( - () => [...new Set(fileMatches.map((match) => match.rawValue))], + const { references, keyValues } = useMemo( + () => workflowSearchFileLookups(fileMatches.map((match) => match.rawValue)), [fileMatches] ) - const storageKeys = useMemo( - () => [...new Set(rawValues.map((value) => extractStorageKey(value)))], - [rawValues] - ) - const enabled = rawValues.length > 0 - const byId = useWorkspaceFileLookups(workspaceId ?? '', 'id', rawValues, { enabled }) - const byKey = useWorkspaceFileLookups(workspaceId ?? '', 'key', storageKeys, { enabled }) - const byReference = useWorkspaceFileLookups(workspaceId ?? '', 'reference', rawValues, { + const enabled = fileMatches.length > 0 + const byKey = useWorkspaceFileLookups(workspaceId ?? '', 'key', keyValues, { enabled }) + const byReference = useWorkspaceFileLookups(workspaceId ?? '', 'reference', references, { enabled, }) - const settled = !byId.isPending && !byKey.isPending && !byReference.isPending + const settled = !byKey.isPending && !byReference.isPending return useMemo( () => fileMatches.map((match) => { const file = - byId.byValue.get(match.rawValue) ?? byKey.byValue.get(extractStorageKey(match.rawValue)) ?? byReference.byValue.get(match.rawValue) return { @@ -392,7 +406,7 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor : undefined, } }), - [fileMatches, byId.byValue, byKey.byValue, byReference.byValue, settled] + [fileMatches, byKey.byValue, byReference.byValue, settled] ) } @@ -669,14 +683,8 @@ export function useWorkflowSearchTableReplacementOptions( }) } -/** The files a replacement can be chosen from before anything is typed: the most recently updated. */ -const RECENT_REPLACEMENT_FILES = workspaceFileListParams({ - sortBy: 'updatedAt', - sortOrder: 'desc', - limit: 100, -}) - -const REPLACEMENT_FILE_SEARCH_LIMIT = 50 +/** How many of the most recently updated files a replacement can be chosen from untyped. */ +const RECENT_REPLACEMENT_FILE_LIMIT = 100 /** The storage key a chosen file replacement value carries, if it is one. */ function replacementFileKey(value: string): string | undefined { @@ -703,12 +711,10 @@ export function useWorkflowSearchFileReplacementOptions( [matches] ) const enabled = Boolean(workspaceId && fileGroups.length > 0) - const recentFiles = useWorkspaceFileList(workspaceId ?? '', RECENT_REPLACEMENT_FILES, { enabled }) - const matchingFiles = useWorkspaceFileSearch( - workspaceId ?? '', - { q: options.query, limit: REPLACEMENT_FILE_SEARCH_LIMIT }, - { enabled } - ) + const { files: offered } = useWorkspaceFilePickerOptions(workspaceId ?? '', options.query, { + limit: RECENT_REPLACEMENT_FILE_LIMIT, + enabled, + }) const chosenKeys = useMemo( () => options.chosen.flatMap((value) => { @@ -718,16 +724,13 @@ export function useWorkflowSearchFileReplacementOptions( [options.chosen] ) const chosenFiles = useWorkspaceFileLookups(workspaceId ?? '', 'key', chosenKeys, { enabled }) - const offered = options.query.trim() ? matchingFiles.data : recentFiles.data?.pages[0]?.files return useMemo(() => { - const files = new Map<string, WorkspaceFileRecord>() - for (const file of chosenFiles.byValue.values()) files.set(file.id, file) - for (const file of offered ?? []) files.set(file.id, file) + const files = distinctFiles([...chosenFiles.byValue.values(), offered]) return [ { data: fileGroups.flatMap((match) => - [...files.values()].map( + files.map( (file): WorkflowSearchReplacementOption => ({ kind: 'file', value: JSON.stringify({ diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index f080e219628..83b4c0d5bfa 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -39,8 +39,12 @@ import { useRenameWorkspaceFile, useUpdateWorkspaceFileContent, useWorkspaceFileContent, + useWorkspaceFileFolderSizes, useWorkspaceFileList, + useWorkspaceFilePickerOptions, useWorkspaceFileRecord, + useWorkspaceFileSearch, + useWorkspaceImageDimensionsAdapter, type WorkspaceFileContentResult, } from '@/hooks/queries/workspace-files' @@ -277,6 +281,36 @@ describe('useWorkspaceFileContent stale storage key', () => { // The RECORD is re-resolved, never this query — re-driving the read against the same dead key // is what would spin. expect(refetchQueries).toHaveBeenCalledTimes(1) + /** Only the reads that can hold this file's key: its record and the lookups that found it. */ + const [filter] = refetchQueries.mock.calls[0] as unknown as [ + { + predicate: (query: { + queryHash: string + queryKey: unknown[] + state: { data: unknown } + }) => boolean + }, + ] + const query = (queryKey: readonly unknown[], data: unknown = null) => ({ + queryKey: [...queryKey], + queryHash: JSON.stringify(queryKey), + state: { data }, + }) + expect(filter.predicate(query(workspaceFilesKeys.record('ws-1', 'file-1')))).toBe(true) + expect( + filter.predicate( + query(workspaceFilesKeys.lookup('ws-1', 'reference', 'doc.md'), fileRecord('file-1')) + ) + ).toBe(true) + expect( + filter.predicate(query(workspaceFilesKeys.lookup('ws-1', 'id', 'other'), fileRecord('other'))) + ).toBe(false) + expect(filter.predicate(query(workspaceFilesKeys.record('ws-1', 'other')))).toBe(false) + expect( + filter.predicate( + query(workspaceFilesKeys.search('ws-1', { q: 'doc', limit: 20 }), [fileRecord('file-1')]) + ) + ).toBe(false) unmount() }) }) @@ -486,15 +520,11 @@ describe('useWorkspaceFileList paging', () => { try { await vi.waitFor(() => expect(hook.result().data?.pages).toHaveLength(1)) expect(hook.result().hasNextPage).toBe(true) - await act(async () => { - await hook.result().fetchNextPage() - }) + act(() => hook.result().loadMore()) await vi.waitFor(() => expect(hook.result().hasNextPage).toBe(false)) - expect( - hook.result().data?.pages.flatMap((page) => page.files.map((file) => file.id)) - ).toEqual(['a', 'b']) + expect(hook.result().data?.files.map((file) => file.id)).toEqual(['a', 'b']) expect(queries).toEqual([ - expect.objectContaining({ folderId: 'root', types: 'document,image', limit: 1 }), + expect.objectContaining({ folderId: 'root', types: ['document', 'image'], limit: 1 }), expect.objectContaining({ folderId: 'root', cursor: 'c1' }), ]) expect(queries[0]).not.toHaveProperty('cursor') @@ -588,26 +618,22 @@ describe('a file deleted while open', () => { }) describe('useRenameWorkspaceFile', () => { - it('renames the file in every cached read at once, and restores them all when the rename fails', async () => { + it('renames the file where it is addressed by identity at once, and restores it when the rename fails', async () => { const hook = mountHook(() => useRenameWorkspaceFile()) const { client } = hook const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) - const searchKey = workspaceFilesKeys.search('ws-1', { q: 'a', limit: 20 }) - client.setQueryData(workspaceFilesKeys.list('ws-1', params), { + const listKey = workspaceFilesKeys.list('ws-1', params) + const recordKey = workspaceFilesKeys.record('ws-1', 'a') + const lookupKey = workspaceFilesKeys.lookup('ws-1', 'id', 'a') + const list = { pages: [{ files: [fileRecord('a'), fileRecord('b')], nextCursor: null }], pageParams: [null], - }) - client.setQueryData(searchKey, [fileRecord('a')]) - client.setQueryData(workspaceFilesKeys.record('ws-1', 'a'), fileRecord('a')) - const names = () => [ - ( - client.getQueryData(workspaceFilesKeys.list('ws-1', params)) as { - pages: { files: WorkspaceFileRecord[] }[] - } - ).pages[0].files.map((file) => file.name), - (client.getQueryData(searchKey) as WorkspaceFileRecord[])[0].name, - (client.getQueryData(workspaceFilesKeys.record('ws-1', 'a')) as WorkspaceFileRecord).name, - ] + } + client.setQueryData(listKey, list) + client.setQueryData(recordKey, fileRecord('a')) + client.setQueryData(lookupKey, fileRecord('a')) + const name = (key: readonly unknown[]) => + (client.getQueryData(key) as WorkspaceFileRecord | undefined)?.name const rename = createDeferred<never>() mockRequestJson.mockImplementation((contract) => contract === renameWorkspaceFileContract ? rename.promise : new Promise(() => {}) @@ -620,14 +646,59 @@ describe('useRenameWorkspaceFile', () => { .mutateAsync({ workspaceId: 'ws-1', fileId: 'a', name: 'renamed.md' }) .catch(() => {}) }) - await vi.waitFor(() => - expect(names()).toEqual([['renamed.md', 'b.md'], 'renamed.md', 'renamed.md']) - ) + await vi.waitFor(() => expect(name(recordKey)).toBe('renamed.md')) + expect(name(lookupKey)).toBe('renamed.md') + /** The list keeps the server's copy: rewriting it would leave the file at its old position. */ + expect(client.getQueryData(listKey)).toBe(list) await act(async () => { rename.reject(new Error('Name taken')) await settled }) - expect(names()).toEqual([['a.md', 'b.md'], 'a.md', 'a.md']) + expect(name(recordKey)).toBe('a.md') + expect(name(lookupKey)).toBe('a.md') + } finally { + hook.unmount() + } + }) + + it('refetches lists and searches once a rename lands, so the file takes its new position and leaves a search it no longer matches', async () => { + let renamed = false + mockRequestJson.mockImplementation(async (contract) => { + if (contract === renameWorkspaceFileContract) { + renamed = true + return { success: true, file: fileRecord('a', { name: 'z.md' }) } + } + if (contract === listWorkspaceFilesContract) { + const files = renamed + ? [fileRecord('b'), fileRecord('a', { name: 'z.md' })] + : [fileRecord('a'), fileRecord('b')] + return { success: true, files, nextCursor: null } + } + if (contract === searchWorkspaceFilesContract) { + return { success: true, files: renamed ? [] : [fileRecord('a')] } + } + return new Promise(() => {}) + }) + const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) + const hook = mountHook(() => ({ + rename: useRenameWorkspaceFile(), + list: useWorkspaceFileList('ws-1', params), + search: useWorkspaceFileSearch('ws-1', { q: 'a', limit: 20 }), + })) + try { + await vi.waitFor(() => expect(hook.result().search.data).toHaveLength(1)) + await vi.waitFor(() => expect(hook.result().list.data?.files).toHaveLength(2)) + await act(async () => { + await hook.result().rename.mutateAsync({ workspaceId: 'ws-1', fileId: 'a', name: 'z.md' }) + }) + /** Settled only once the mounted reads refetched: the edit closes onto the server's order. */ + const list = hook.client.getQueryData<{ pages: { files: WorkspaceFileRecord[] }[] }>( + workspaceFilesKeys.list('ws-1', params) + ) + expect(list?.pages[0].files.map((file) => file.name)).toEqual(['b.md', 'z.md']) + expect( + hook.client.getQueryData(workspaceFilesKeys.search('ws-1', { q: 'a', limit: 20 })) + ).toEqual([]) } finally { hook.unmount() } @@ -670,3 +741,120 @@ describe('invalidateWorkspaceFileBrowsers', () => { } }) }) + +describe('useUpdateWorkspaceFileContent cache writes', () => { + it('writes the saved record through every read holding it and refetches only the listings', async () => { + const saved = fileRecord('a', { key: 'workspace/ws-1/a-v2.md', size: 9 }) + mockRequestJson.mockImplementation(async (contract) => + contract === updateWorkspaceFileContentContract + ? { success: true, file: saved } + : new Promise(() => {}) + ) + const hook = mountHook(() => useUpdateWorkspaceFileContent()) + const { client } = hook + const recordKey = workspaceFilesKeys.record('ws-1', 'a') + const searchKey = workspaceFilesKeys.search('ws-1', { q: 'a', limit: 20 }) + const listKey = workspaceFilesKeys.list( + 'ws-1', + workspaceFileListParams({ folderId: null, sortBy: 'size', sortOrder: 'asc' }) + ) + client.setQueryData(recordKey, fileRecord('a')) + client.setQueryData(searchKey, [fileRecord('a')]) + client.setQueryData(listKey, { + pages: [{ files: [fileRecord('a')], nextCursor: null }], + pageParams: [null], + }) + try { + await act(async () => { + await hook.result().mutateAsync({ workspaceId: 'ws-1', fileId: 'a', content: 'x' }) + }) + expect(client.getQueryData(recordKey)).toEqual(saved) + expect(client.getQueryData(searchKey)).toEqual([saved]) + expect(client.getQueryState(recordKey)?.isInvalidated).toBe(false) + expect(client.getQueryState(searchKey)?.isInvalidated).toBe(true) + expect(client.getQueryState(listKey)?.isInvalidated).toBe(true) + } finally { + hook.unmount() + } + }) +}) + +describe('useWorkspaceFileFolderSizes', () => { + it('reads the child folders of every parent in one request', async () => { + const queries: Record<string, unknown>[] = [] + mockRequestJson.mockImplementation(async (_contract, input) => { + queries.push((input as { query: Record<string, unknown> }).query) + return { + success: true, + folders: [ + { folderId: 'f1', fileCount: 1, sizeBytes: 10 }, + { folderId: 'f2', fileCount: 2, sizeBytes: 20 }, + ], + } + }) + const parents = ['folder-b', null, 'folder-b'] + const hook = mountHook(() => useWorkspaceFileFolderSizes('ws-1', parents)) + try { + await vi.waitFor(() => expect(hook.result().get('f2')?.sizeBytes).toBe(20)) + expect(queries).toEqual([{ parentIds: ['folder-b', 'root'] }]) + } finally { + hook.unmount() + } + }) +}) + +describe('useWorkspaceFilePickerOptions', () => { + it('offers the most recent files untyped and the server matches once a query settles', async () => { + mockRequestJson.mockImplementation(async (contract) => + contract === searchWorkspaceFilesContract + ? { success: true, files: [fileRecord('match')] } + : { success: true, files: [fileRecord('recent')], nextCursor: null } + ) + let query = '' + const hook = mountHook(() => useWorkspaceFilePickerOptions('ws-1', query)) + try { + await vi.waitFor(() => + expect(hook.result().files?.map((file) => file.id)).toEqual(['recent']) + ) + query = ' mat ' + hook.rerender() + /** Typed but not settled: still pending, so an empty menu is not read as "no match". */ + expect(hook.result().isPending).toBe(true) + await vi.waitFor(() => expect(hook.result().files?.map((file) => file.id)).toEqual(['match'])) + expect(hook.result().settledQuery).toBe('mat') + expect(hook.result().isPending).toBe(false) + } finally { + hook.unmount() + } + }) +}) + +describe('useWorkspaceImageDimensionsAdapter', () => { + it('re-runs the image reads once its own lookup lands on a cold load, not for other lookups', async () => { + const image = fileRecord('img', { + key: 'workspace/ws-1/1731-ab-img.png', + width: 40, + height: 30, + }) + const lookup = createDeferred<unknown>() + mockRequestJson.mockImplementation(() => lookup.promise) + const src = `/api/files/serve/${encodeURIComponent(image.key)}?context=workspace` + const hook = mountHook(() => useWorkspaceImageDimensionsAdapter('ws-1')) + try { + const cold = hook.result() + expect(cold.getImageDimensions(src)).toBeNull() + act(() => { + hook.client.setQueryData(workspaceFilesKeys.lookup('ws-1', 'id', 'unrelated'), null) + }) + expect(hook.result()).toBe(cold) + await act(async () => { + lookup.resolve({ success: true, files: [image], references: [] }) + await flushMacrotask() + }) + await vi.waitFor(() => expect(hook.result()).not.toBe(cold)) + expect(hook.result().getImageDimensions(src)).toEqual({ width: 40, height: 30 }) + } finally { + hook.unmount() + } + }) +}) diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index a50ad01e257..5442fc9a874 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -2,10 +2,14 @@ import { useCallback, useMemo, useRef, useSyncExternalStore } from 'react' import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' import { backoffWithJitter } from '@sim/utils/retry' import { + hashKey, + type InfiniteData, keepPreviousData, type QueryClient, + type QueryKey, useInfiniteQuery, useIsFetching, useMutation, @@ -37,12 +41,19 @@ import type { UploadProgressEvent } from '@/lib/uploads/client/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' import { extractStorageKey } from '@/lib/uploads/utils/file' +import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { WorkspaceFileSelection } from '@/lib/workspace-files/selection' import type { UserFile } from '@/executor/types' +import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { recentlyDeletedKeys } from '@/hooks/queries/utils/recently-deleted-keys' import { + type CachedWorkspaceFile, + EVERY_WORKSPACE_FILE_READ, findCachedWorkspaceFile, + indexCachedWorkspaceFiles, + invalidateWorkspaceFileListings, invalidateWorkspaceFileReads, + type PatchableWorkspaceFileRead, restoreWorkspaceFileReads, snapshotWorkspaceFileReads, updateCachedWorkspaceFile, @@ -58,11 +69,14 @@ import { type WorkspaceFileListParams, type WorkspaceFileLookupKind, type WorkspaceFileSearchParams, + workspaceFileFolderQuery, + workspaceFileListParams, workspaceFileListQuery, workspaceFilesKeys, } from '@/hooks/queries/utils/workspace-file-keys' import { createWorkspaceFileLookupBatcher } from '@/hooks/queries/utils/workspace-file-lookup-batcher' import { type ImageDimensionsSource, useFileContentSource } from '@/hooks/use-file-content-source' +import { useSearchFilterValue } from '@/hooks/use-search-filter-value' const logger = createLogger('WorkspaceFilesQuery') @@ -108,9 +122,22 @@ function isSameListPlace(previous: unknown, next: WorkspaceFileListParams): bool ) } +/** A list's loaded pages, plus their files in page order. */ +type WorkspaceFileListData = InfiniteData<WorkspaceFileListPage, string | null> & { + files: WorkspaceFileRecord[] +} + +/** Module-level so the selected `files` keep their identity while the pages do. */ +function selectListFiles( + data: InfiniteData<WorkspaceFileListPage, string | null> +): WorkspaceFileListData { + return { ...data, files: data.pages.flatMap((page) => page.files) } +} + /** - * One server-sorted, server-filtered file list, a page at a time (`fetchNextPage` loads the - * next). Build `params` with `workspaceFileListParams`. + * One server-sorted, server-filtered file list, a page at a time: `data.files` holds every + * loaded file, and `loadMore` fetches the next page unless one is already on its way or the list + * is refetching. Build `params` with `workspaceFileListParams`. * * A re-sort, a re-filter, or a new search keeps showing the previous rows until the new first * page lands, as long as the list still shows the same place. Moving to another folder never @@ -121,14 +148,24 @@ export function useWorkspaceFileList( params: WorkspaceFileListParams, options?: { enabled?: boolean } ) { - return useInfiniteQuery({ + const query = useInfiniteQuery({ ...getWorkspaceFileListQueryOptions(workspaceId, params), enabled: Boolean(workspaceId) && (options?.enabled ?? true), + select: selectListFiles, placeholderData: (previous, previousQuery) => { const key = previousQuery?.queryKey return key?.[2] === workspaceId && isSameListPlace(key[3], params) ? previous : undefined }, }) + /** Read when called, not at render, so a caller only re-renders for what it reads itself. */ + const queryRef = useRef(query) + queryRef.current = query + const loadMore = useCallback(() => { + const { isFetching, hasNextPage, fetchNextPage } = queryRef.current + if (!isFetching && hasNextPage) void fetchNextPage() + }, []) + // Adds a property to the hook's own result object; reads none of its tracked fields. + return Object.assign(query, { loadMore }) } async function fetchWorkspaceFileSearch( @@ -138,13 +175,7 @@ async function fetchWorkspaceFileSearch( ): Promise<WorkspaceFileRecord[]> { const data = await requestJson(searchWorkspaceFilesContract, { params: { id: workspaceId }, - query: { - q: params.q, - ...(params.folderId === undefined - ? {} - : { folderId: params.folderId ?? 'root', recursive: params.recursive }), - limit: params.limit, - }, + query: { q: params.q, ...workspaceFileFolderQuery(params), limit: params.limit }, signal, }) return data.files @@ -187,12 +218,12 @@ export interface WorkspaceFileFolderStat { async function fetchWorkspaceFileFolderStats( workspaceId: string, - parentId: string | null, + parentIds: readonly string[], signal?: AbortSignal ): Promise<Map<string, WorkspaceFileFolderStat>> { const data = await requestJson(workspaceFileFolderStatsContract, { params: { id: workspaceId }, - query: { parentId: parentId ?? 'root' }, + query: { parentIds: [...parentIds] }, signal, }) return new Map( @@ -200,40 +231,31 @@ async function fetchWorkspaceFileFolderStats( ) } -function combineFolderSizes( - results: { data: Map<string, WorkspaceFileFolderStat> | undefined }[] -): ReadonlyMap<string, WorkspaceFileFolderStat> { - if (results.length === 1) return results[0].data ?? NO_FOLDER_STATS - const merged = new Map<string, WorkspaceFileFolderStat>() - for (const result of results) { - for (const [folderId, stat] of result.data ?? []) merged.set(folderId, stat) - } - return merged -} - const NO_FOLDER_STATS: ReadonlyMap<string, WorkspaceFileFolderStat> = new Map() /** * The live file count and bytes under each child folder of every parent in `parentIds` (`null` * for the root) — each folder's whole subtree rolled up, what a folder row shows as its size — - * merged into one map by folder id. Several parents because search results span folders. + * in one map by folder id, read in one request. Several parents because search results span + * folders; while a new set of parents loads, the last set's sizes stay. */ export function useWorkspaceFileFolderSizes( workspaceId: string, parentIds: readonly (string | null)[], options?: { enabled?: boolean } ): ReadonlyMap<string, WorkspaceFileFolderStat> { - const enabled = Boolean(workspaceId) && (options?.enabled ?? true) - return useQueries({ - queries: parentIds.map((parentId) => ({ - queryKey: workspaceFilesKeys.folderStat(workspaceId, parentId), - queryFn: ({ signal }: { signal?: AbortSignal }) => - fetchWorkspaceFileFolderStats(workspaceId, parentId, signal), - enabled, - staleTime: WORKSPACE_FILE_FOLDER_STATS_STALE_TIME, - })), - combine: combineFolderSizes, + const parents = useMemo( + () => [...new Set(parentIds.map((parentId) => parentId ?? 'root'))].sort(), + [parentIds] + ) + const { data } = useQuery({ + queryKey: workspaceFilesKeys.folderStat(workspaceId, parents), + queryFn: ({ signal }) => fetchWorkspaceFileFolderStats(workspaceId, parents, signal), + enabled: Boolean(workspaceId) && parents.length > 0 && (options?.enabled ?? true), + staleTime: WORKSPACE_FILE_FOLDER_STATS_STALE_TIME, + placeholderData: keepPreviousData, }) + return data ?? NO_FOLDER_STATS } const workspaceFileLookups = createWorkspaceFileLookupBatcher((workspaceId, body) => @@ -249,14 +271,18 @@ export function getWorkspaceFileLookupQueryOptions( queryClient: QueryClient, workspaceId: string, kind: WorkspaceFileLookupKind, - value: string + value: string, + /** One scan of the cache shared by a batch of lookups of `kind`, for its seeds. */ + cachedIndex?: () => ReadonlyMap<string, CachedWorkspaceFile> ) { const cached = - kind === 'id' - ? () => findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.id === value) - : kind === 'key' - ? () => findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.key === value) - : undefined + kind === 'reference' + ? undefined + : once(() => + cachedIndex + ? cachedIndex().get(value) + : findCachedWorkspaceFile(queryClient, workspaceId, (file) => file[kind] === value) + ) return { queryKey: workspaceFilesKeys.lookup(workspaceId, kind, value), queryFn: ({ signal }: { signal?: AbortSignal }) => @@ -267,6 +293,15 @@ export function getWorkspaceFileLookupQueryOptions( } } +/** `compute`, run on the first call only: a seed read by both `initialData` and its age. */ +function once<T>(compute: () => T): () => T { + let result: { value: T } | undefined + return () => { + result ??= { value: compute() } + return result.value + } +} + /** One file by id, storage key, or reference; `data` is `null` when nothing live answers to it. */ export function useWorkspaceFileLookup( workspaceId: string, @@ -319,9 +354,13 @@ export function useWorkspaceFileLookups( combineLookups(values, results), [values] ) + const cachedIndex = + kind === 'reference' + ? undefined + : once(() => indexCachedWorkspaceFiles(queryClient, workspaceId, kind)) return useQueries({ queries: values.map((value) => ({ - ...getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value), + ...getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value, cachedIndex), enabled, })), combine, @@ -373,16 +412,9 @@ export function useWorkspaceFileSelectionCandidates( })), combine: combineSearches, }) - /** Distinct by id: a file two reads found is one candidate, or a unique name reads as shared. */ + /** Distinct: a file two reads found is one candidate, or a unique name reads as shared. */ const candidates = useMemo( - () => [ - ...new Map( - [...byId.byValue.values(), ...byKey.byValue.values(), ...byName.files].map((file) => [ - file.id, - file, - ]) - ).values(), - ], + () => distinctFiles([...byId.byValue.values(), ...byKey.byValue.values(), byName.files]), [byId.byValue, byKey.byValue, byName.files] ) return { @@ -400,6 +432,92 @@ function combineSearches( } } +/** + * The distinct files across several reads — single records, `null` misses, and file arrays — + * by id, first occurrence's position, last occurrence's copy: one file two reads found is one. + */ +export function distinctFiles( + reads: readonly (WorkspaceFileRecord | readonly WorkspaceFileRecord[] | null | undefined)[] +): WorkspaceFileRecord[] { + const files = new Map<string, WorkspaceFileRecord>() + for (const read of reads) { + if (!read) continue + for (const file of Array.isArray(read) ? read : [read]) files.set(file.id, file) + } + return [...files.values()] +} + +/** How many files a picker offers: its recent list, and its matches (the search's maximum). */ +const WORKSPACE_FILE_PICKER_LIMIT = 50 + +const RECENT_PICKER_FILES = new Map<number, WorkspaceFileListParams>() + +/** The most recently updated live files anywhere, `limit` of them, as one shared list. */ +function recentPickerFiles(limit: number): WorkspaceFileListParams { + let params = RECENT_PICKER_FILES.get(limit) + if (!params) { + params = workspaceFileListParams({ sortBy: 'updatedAt', sortOrder: 'desc', limit }) + RECENT_PICKER_FILES.set(limit, params) + } + return params +} + +export interface WorkspaceFilePickerOptions { + /** The files to offer: the best matches for the settled query, else the most recent. */ + files: WorkspaceFileRecord[] | undefined + /** The query `files` answers, trimmed; empty while browsing recent files. */ + settledQuery: string + /** True until `files` answers what was typed: a menu still empty here is not "no match" yet. */ + isPending: boolean +} + +/** + * What a file picker or mention menu offers, without the workspace's whole list: with nothing + * typed, the `limit` most recently updated files; once `query` settles (debounced), the server's + * best matches for it by name and folder path, at most {@link WORKSPACE_FILE_PICKER_LIMIT}. + */ +export function useWorkspaceFilePickerOptions( + workspaceId: string, + query: string, + options?: { limit?: number; enabled?: boolean } +): WorkspaceFilePickerOptions { + const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + const typed = query.trim() + const settledQuery = useSearchFilterValue(query, SEARCH_DEBOUNCE_MS).trim() + const recent = useWorkspaceFileList( + workspaceId, + recentPickerFiles(options?.limit ?? WORKSPACE_FILE_PICKER_LIMIT), + { enabled } + ) + const matching = useWorkspaceFileSearch( + workspaceId, + { q: settledQuery, limit: WORKSPACE_FILE_PICKER_LIMIT }, + { enabled } + ) + return { + files: settledQuery ? matching.data : recent.data?.pages[0]?.files, + settledQuery, + isPending: + recent.isPending || (typed.length > 0 && (typed !== settledQuery || matching.isFetching)), + } +} + +/** + * The live file at a canonical VFS path (`files/docs/a.md`), or `undefined`. The reference + * lookup also answers by bare name, so only a file whose path is exactly `path` counts. + */ +export function useWorkspaceFileAtPath( + workspaceId: string, + path: string | undefined, + options?: { enabled?: boolean } +): { file: WorkspaceFileRecord | undefined; isFetching: boolean } { + const lookup = useWorkspaceFileLookup(workspaceId, 'reference', path, options) + return { + file: findWorkspaceFileByPath(lookup.data ? [lookup.data] : undefined, path), + isFetching: lookup.isFetching, + } +} + /** The file's record, or `null` once no live file has that id (deleted, or never existed). */ async function fetchWorkspaceFileRecord( workspaceId: string, @@ -428,8 +546,9 @@ export function getWorkspaceFileRecordQueryOptions( workspaceId: string, fileId: string ) { - const cached = () => + const cached = once(() => findCachedWorkspaceFile(queryClient, workspaceId, (file) => file.id === fileId) + ) return { queryKey: workspaceFilesKeys.record(workspaceId, fileId), queryFn: ({ signal }: { signal?: AbortSignal }) => @@ -458,20 +577,17 @@ export function useWorkspaceFileRecord( } /** - * Back the file content source's image-dimension capability with workspace file metadata. Each - * embedded image is resolved by the storage key or file id its `src` carries — from any cached - * read of the file, else one batched lookup — so a stored image reserves its box before it - * downloads without the workspace's whole file list. The adapter takes a new identity whenever - * a lookup of this workspace lands, which re-runs the node views' dimension reads: on a cold - * direct file-view load the first read misses and the second, once the lookup resolves, hits. + * Back the file content source's image-dimension capability with workspace file metadata: each + * embedded image resolves by the storage key or file id its `src` carries — from any cached read + * of the file, else one batched lookup — so it reserves its box before it downloads. The adapter + * takes a new identity when a lookup it asked for lands, re-running the node views' reads (on a + * cold load the first read misses; the one after the lookup resolves hits). * - * Persists the browser's measured dimensions when they're absent or disagree with what's stored — - * an overwrite, so a stale value (left over after a content swap, or a non-EXIF-corrected one) - * self-corrects rather than sticking. The write is fire-and-forget and de-duped (an exact-match - * cache check plus mismatch-only reporting from the caller), so it never storms, never blocks - * render, and never touches the collaborative document. `options.enabled` turns it off for - * callers that supply their own content source (the public share page, whose `workspaceId` is - * a share token that would 404). + * Persists the browser's measured dimensions when absent or different from what's stored — an + * overwrite, so a stale value self-corrects. The write is fire-and-forget and de-duped, never + * blocks render, and never touches the collaborative document. `options.enabled` turns it off + * for callers with their own content source (the public share page, whose `workspaceId` is a + * share token). */ export function useWorkspaceImageDimensionsAdapter( workspaceId: string, @@ -479,21 +595,28 @@ export function useWorkspaceImageDimensionsAdapter( ): ImageDimensionsSource { const queryClient = useQueryClient() const enabled = Boolean(workspaceId) && (options?.enabled ?? true) + /** + * A count of the lookups this adapter asked for that have landed. A counter rather than + * `useIsFetching`: a lookup's fetching-to-settled flip can land between two renders, which + * would leave the reads that missed it never re-run. + */ const versionRef = useRef(0) + /** The lookups this adapter warmed, by query hash; only their landing re-runs the reads. */ + const requestedRef = useRef<Set<string> | null>(null) + requestedRef.current ??= new Set() const subscribe = useCallback( (onChange: () => void) => queryClient.getQueryCache().subscribe((event) => { if ( event.type === 'updated' && event.action.type === 'success' && - event.query.queryKey[1] === 'lookup' && - event.query.queryKey[2] === workspaceId + requestedRef.current?.delete(event.query.queryHash) ) { versionRef.current += 1 onChange() } }), - [queryClient, workspaceId] + [queryClient] ) const version = useSyncExternalStore( subscribe, @@ -513,6 +636,7 @@ export function useWorkspaceImageDimensionsAdapter( if (resolved !== undefined) return resolved ?? undefined const cached = lookup.initialData?.() if (cached) return cached + requestedRef.current?.add(hashKey(lookup.queryKey)) queueMicrotask(() => void queryClient.prefetchQuery(lookup)) return undefined } @@ -532,10 +656,13 @@ export function useWorkspaceImageDimensionsAdapter( if (!record || (record.width === dimensions.width && record.height === dimensions.height)) return // Populate every cached read so this and sibling views reserve space immediately. - updateCachedWorkspaceFile(queryClient, workspaceId, record.id, (entry) => ({ - ...entry, - ...dimensions, - })) + updateCachedWorkspaceFile( + queryClient, + workspaceId, + record.id, + (entry) => ({ ...entry, ...dimensions }), + EVERY_WORKSPACE_FILE_READ + ) void requestJson(updateWorkspaceFileDimensionsContract, { params: { id: workspaceId, fileId: record.id }, // Send the key we measured against; the server rejects the write if the row's content (key) has @@ -576,20 +703,31 @@ class StaleStorageKeyError extends Error { } } -/** The mounted metadata reads of a workspace — whichever one handed the reader its storage key. */ -function activeWorkspaceFileReads(workspaceId: string) { - return { ...workspaceFileMetadataFilter(workspaceId), type: 'active' as const } +/** + * The reads that can have handed a content reader `fileId`'s storage key: its record (which a + * list row seeds) and the lookups that resolved to it. + */ +function fileAddressFilter(workspaceId: string, fileId: string) { + const record = hashKey(workspaceFilesKeys.record(workspaceId, fileId)) + const lookups = workspaceFilesKeys.workspaceLookups(workspaceId) + return { + predicate: (query: { queryHash: string; queryKey: QueryKey; state: { data: unknown } }) => + query.queryHash === record || + (lookups.every((segment, index) => query.queryKey[index] === segment) && + isRecordLike(query.state.data) && + query.state.data.id === fileId), + } } /** - * Re-resolve a workspace's file records after a read found its storage key superseded. + * Re-resolve a file's address after a read found its storage key superseded. * - * The key a content read addresses comes from a cached record — the file's own, or the list, - * search, or lookup the reader opened it from — and nothing invalidates those when a write rotates - * the key: the collaborative relay projects an open document back to markdown every few seconds, - * entirely server-side, so an open tab's key can be replaced many times over without a single - * client-visible event. Recovering on the 404 makes the rotation cost one refetch of the mounted - * reads instead of stranding the reader on a dead key until a full reload. + * The key a content read addresses comes from a cached record — the file's own, seeded from the + * list the reader opened it from, or a lookup that resolved to it — and nothing invalidates those + * when a write rotates the key: the collaborative relay projects an open document back to + * markdown every few seconds, entirely server-side, so an open tab's key can be replaced many + * times over without a single client-visible event. Recovering on the 404 makes the rotation + * cost a refetch of the file's own mounted reads instead of stranding the reader on a dead key. * * Re-reads the RECORD, never the failed query: the refetched record either hands back a new key — * which re-keys the read onto a fresh cache entry that fetches once — or the same one, in which case @@ -601,11 +739,11 @@ function activeWorkspaceFileReads(workspaceId: string) { * on the dead key showing a failure until something unrelated (a window focus, another consumer) * happens to re-resolve the record. */ -function useStaleKeyRecovery(workspaceId: string): (error: unknown) => void { +function useStaleKeyRecovery(workspaceId: string, fileId: string): (error: unknown) => void { const queryClient = useQueryClient() return useCallback( (error: unknown) => { - if (!workspaceId || !(error instanceof StaleStorageKeyError)) return + if (!workspaceId || !fileId || !(error instanceof StaleStorageKeyError)) return // Off this fetch's own cycle (one microtask): a refetch asked for from inside a `queryFn` — where // this catch sits — is dropped by react-query, silently. This is what turned "one extra request" // into "no recovery at all". @@ -613,10 +751,13 @@ function useStaleKeyRecovery(workspaceId: string): (error: unknown) => void { // `cancelRefetch` because a re-resolution has to OBSERVE the rotation: a record read already in // flight was started before it, so it can only hand back the key we already know is dead. void Promise.resolve().then(() => - queryClient.refetchQueries(activeWorkspaceFileReads(workspaceId), { cancelRefetch: true }) + queryClient.refetchQueries( + { ...fileAddressFilter(workspaceId, fileId), type: 'active' }, + { cancelRefetch: true } + ) ) }, - [queryClient, workspaceId] + [queryClient, workspaceId, fileId] ) } @@ -664,7 +805,7 @@ export function useWorkspaceFileContent( } ): WorkspaceFileContentResult { const source = useFileContentSource() - const recoverStaleKey = useStaleKeyRecovery(workspaceId) + const recoverStaleKey = useStaleKeyRecovery(workspaceId, fileId) const query = useQuery({ queryKey: workspaceFilesKeys.content(workspaceId, fileId, raw ? 'raw' : 'text', key), queryFn: async ({ signal }) => { @@ -683,7 +824,7 @@ export function useWorkspaceFileContent( }) return { data: query.data, - ...useStaleKeyRecoveryState(workspaceId, query.isLoading, query.error), + ...useStaleKeyRecoveryState(workspaceId, fileId, query.isLoading, query.error), } } @@ -710,10 +851,11 @@ export interface WorkspaceFileContentResult { */ function useStaleKeyRecoveryState( workspaceId: string, + fileId: string, isLoading: boolean, error: unknown ): { isLoading: boolean; error: Error | null } { - const resolvingRecord = useIsFetching(workspaceFileMetadataFilter(workspaceId)) + const resolvingRecord = useIsFetching(fileAddressFilter(workspaceId, fileId)) const recovering = error instanceof StaleStorageKeyError && resolvingRecord > 0 return { isLoading: isLoading || recovering, @@ -778,7 +920,7 @@ export function useWorkspaceFileBinary( options?: { enabled?: boolean; version?: string | number } ) { const source = useFileContentSource() - const recoverStaleKey = useStaleKeyRecovery(workspaceId) + const recoverStaleKey = useStaleKeyRecovery(workspaceId, fileId) return useQuery({ queryKey: options?.version != null @@ -969,12 +1111,28 @@ export function useUpdateWorkspaceFileContent() { body, }) }, + /** + * The saved record is the response: written through to the file's record and every read that + * holds it in place, since a content write moves no file between folders or out of a search. + * Lists are refetched — a size or update-time sort may move it — and so are folder sizes. + */ + onSuccess: (data, { workspaceId, fileId }) => { + updateCachedWorkspaceFile( + queryClient, + workspaceId, + fileId, + () => data.file, + EVERY_WORKSPACE_FILE_READ + ) + void invalidateWorkspaceFileListings(queryClient, workspaceId, { folderStats: true }) + void queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + }, /** A lost response may follow a committed write; reconcile bytes and versions even after transport errors. */ - onSettled: (_data, _error, variables) => { + onSettled: (_data, error, variables) => { queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.contentFile(variables.workspaceId, variables.fileId), }) - void invalidateAfterFileWrite(queryClient, variables.workspaceId) + if (error) void invalidateAfterFileWrite(queryClient, variables.workspaceId) }, onError: (error) => { logger.error('Failed to update file content:', error) @@ -1023,6 +1181,17 @@ interface RenameFileParams { name: string } +/** What a rename rewrites in place: the reads that address the file by identity. */ +const RENAME_PATCHED_READS: readonly PatchableWorkspaceFileRead[] = ['identityLookup', 'record'] + +/** What a delete drops the file from: every live read but its own record. */ +const DELETE_PATCHED_READS: readonly PatchableWorkspaceFileRead[] = [ + 'list', + 'search', + 'identityLookup', + 'referenceLookup', +] + export function useRenameWorkspaceFile() { const queryClient = useQueryClient() @@ -1032,19 +1201,41 @@ export function useRenameWorkspaceFile() { params: { id: workspaceId, fileId }, body: { name }, }), + /** + * The new name shows at once wherever the file is addressed by identity — its record, its + * id and key lookups. Lists, searches, and name references are not rewritten: the new name + * can move the file in a name-sorted list or out of a search, so they are refetched once the + * rename lands, and the returned promise settles after the mounted ones have. + */ onMutate: async ({ workspaceId, fileId, name }) => { await queryClient.cancelQueries(workspaceFileMetadataFilter(workspaceId)) - const previous = snapshotWorkspaceFileReads(queryClient, workspaceId) - updateCachedWorkspaceFile(queryClient, workspaceId, fileId, (file) => ({ ...file, name })) + const previous = snapshotWorkspaceFileReads(queryClient, workspaceId, RENAME_PATCHED_READS) + updateCachedWorkspaceFile( + queryClient, + workspaceId, + fileId, + (file) => ({ ...file, name }), + RENAME_PATCHED_READS + ) return { previous } }, + onSuccess: (data, { workspaceId, fileId }) => { + updateCachedWorkspaceFile( + queryClient, + workspaceId, + fileId, + () => data.file, + RENAME_PATCHED_READS + ) + }, onError: (error, _variables, context) => { if (context) restoreWorkspaceFileReads(queryClient, context.previous) toast.error(error.message, { duration: 5000 }) }, - onSettled: (_data, _error, variables) => { - void invalidateWorkspaceFileReads(queryClient, variables.workspaceId) - }, + onSettled: (_data, error, { workspaceId }) => + error + ? invalidateWorkspaceFileReads(queryClient, workspaceId) + : invalidateWorkspaceFileListings(queryClient, workspaceId), }) } @@ -1066,8 +1257,8 @@ export function useDeleteWorkspaceFile() { }), onMutate: async ({ workspaceId, fileId }) => { await queryClient.cancelQueries(workspaceFileMetadataFilter(workspaceId)) - const previous = snapshotWorkspaceFileReads(queryClient, workspaceId) - updateCachedWorkspaceFile(queryClient, workspaceId, fileId, () => null) + const previous = snapshotWorkspaceFileReads(queryClient, workspaceId, DELETE_PATCHED_READS) + updateCachedWorkspaceFile(queryClient, workspaceId, fileId, () => null, DELETE_PATCHED_READS) return { previous } }, onError: (error, _variables, context) => { diff --git a/apps/sim/lib/mothership/tools/client/resource-display.ts b/apps/sim/lib/mothership/tools/client/resource-display.ts index b50ca5d864b..830bf4c04a8 100644 --- a/apps/sim/lib/mothership/tools/client/resource-display.ts +++ b/apps/sim/lib/mothership/tools/client/resource-display.ts @@ -19,7 +19,7 @@ import { tableKeys } from '@/hooks/queries/utils/table-keys' import { workflowKeys } from '@/hooks/queries/utils/workflow-keys' import { findCachedWorkspaceFile, - isWorkspaceFileMetadataKey, + isWorkspaceFileRecordKey, } from '@/hooks/queries/utils/workspace-file-cache' import { workspaceFilesKeys } from '@/hooks/queries/utils/workspace-file-keys' import { workspaceKeys } from '@/hooks/queries/workspace' @@ -119,7 +119,7 @@ export function resolveResourceDisplayName( export function isResourceNameQuery(key: QueryKey, workspaceId?: string): boolean { if (key[0] === workspaceKeys.all[0]) return key[1] === 'list' || key[1] === 'adminList' if (key[0] === workspaceFilesKeys.all[0]) { - return isWorkspaceFileMetadataKey(key, workspaceId ?? String(key[2])) + return isWorkspaceFileRecordKey(key, workspaceId) } return ( [ From dde3796db59ae726427d00eacc41ada856bb06e1 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 21:07:20 -0700 Subject: [PATCH 09/15] perf(files): resolve a file chip's candidates by reference alone A reference lookup matches an exact id before a path or name, so the separate id lookup per candidate found nothing the reference did not. Each candidate is now one reference lookup, plus the name search that keeps a shared name ambiguous; the de-duplicator is the shared one. --- .../home/resolve-resource-ref.test.ts | 16 ++++++++++++++++ .../home/resolve-resource-ref.ts | 19 ++++--------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts index fc0b5fb7071..814e1c86275 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts @@ -124,4 +124,20 @@ describe('fetchWorkspaceResourceRef', () => { ) ).toHaveLength(1) }) + + it('asks for each candidate once, as a reference, which already answers an exact id', async () => { + serveWorkspace([file({ id: 'wf_a', name: 'notes.md' })]) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + await expect( + fetchWorkspaceResourceRef(client, 'ws-1', ref({ id: 'wf_a' })) + ).resolves.toMatchObject({ id: 'wf_a' }) + const lookups = apiClientRequestMockFns.mockRequestJson.mock.calls.filter( + ([contract]) => contract === lookupWorkspaceFilesContract + ) + expect(lookups).toHaveLength(1) + const body = (lookups[0][1] as { body: Record<string, unknown> }).body + expect(body.ids ?? []).toEqual([]) + expect(body.references).toEqual(expect.arrayContaining(['wf_a'])) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts index df4f0fbb2d0..9521c7ddcd2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts @@ -5,6 +5,7 @@ import { canonicalWorkspaceFilePath } from '@/lib/mothership/vfs/path-utils' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { findWorkspaceFileByPath } from '@/hooks/queries/utils/find-workspace-file-by-src' import { + distinctFiles, getWorkspaceFileLookupQueryOptions, getWorkspaceFileSearchQueryOptions, } from '@/hooks/queries/workspace-files' @@ -96,13 +97,12 @@ function fileRefCandidates(ref: WorkspaceResourceRef): string[] { /** * The reads that together hold every file a chip's candidates can match under - * {@link resolveWorkspaceResourceRef}'s rules: each candidate as an exact id, as a reference (an - * exact VFS path resolves first), and — for a candidate that could be a bare name — every file - * whose name contains it, so a name shared by two files is still seen as ambiguous. + * {@link resolveWorkspaceResourceRef}'s rules: each candidate as a reference (an exact id + * resolves first, then an exact VFS path), and — for a candidate that could be a bare name — + * every file whose name contains it, so a name shared by two files is still seen as ambiguous. */ function fileRefReads(queryClient: QueryClient, workspaceId: string, ref: WorkspaceResourceRef) { return fileRefCandidates(ref).flatMap((candidate) => [ - getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, 'id', candidate), getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, 'reference', candidate), ...(candidate.includes('/') || candidate.length > V2_SEARCH_MAX_LENGTH ? [] @@ -115,17 +115,6 @@ function fileRefReads(queryClient: QueryClient, workspaceId: string, ref: Worksp ]) } -/** The distinct files several reads returned: one file found by two reads is still one file. */ -function distinctFiles( - reads: readonly (WorkspaceFileRecord | WorkspaceFileRecord[] | null)[] -): WorkspaceFileRecord[] { - const files = new Map<string, WorkspaceFileRecord>() - for (const data of reads) { - for (const file of Array.isArray(data) ? data : data ? [data] : []) files.set(file.id, file) - } - return [...files.values()] -} - /** * Resolves a chip against the workspace's live files, fetching only the handful of files its * candidates can name rather than the workspace's list. `fresh` re-reads them all: a chip may From c94cf78eb21289e6a371627b6231bf77e5604f5c Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 21:32:24 -0700 Subject: [PATCH 10/15] chore(files): tidy the paged file reads' memos, query options, and comments - The table's load-more footer splits its two concerns: the spinner follows the next-page fetch, and the sentinel waits on any fetch (`isFetching`, defaulting to `isLoadingMore` so other tables are unchanged) and re-arms when it settles. The Files list no longer shows the bottom spinner for a background refetch, and reaching the bottom mid-refetch still pages. - A search keeps its previous matches only within the same workspace, and the list's placeholder guard checks the workspace through the key factory. - The picker's pending state stops counting the recent list once a settled query is showing matches; search-replace hands it the raw query instead of debouncing twice. - Plain computations replace memos the query key already compares by value, and the flyout's entries depend on the pinned files and ids rather than the hook's result object. - Drops comments that restate their line. --- .../components/resource/resource.tsx | 10 +++++- .../workspace/[workspaceId]/files/files.tsx | 13 +++----- .../components/file-upload/file-upload.tsx | 2 -- ...e-workflow-resource-replacement-options.ts | 5 +-- .../rail-resource-flyout.tsx | 8 ++--- apps/sim/hooks/queries/workspace-files.ts | 31 +++++++++++++------ apps/sim/lib/workspace-files/file-type-sql.ts | 1 - 7 files changed, 40 insertions(+), 30 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx index 54694858846..9a1cbab363c 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx @@ -231,7 +231,14 @@ interface ResourceTableProps { onRowContextMenu?: (e: React.MouseEvent, rowId: string) => void onLoadMore?: () => void hasMore?: boolean + /** The next page is loading: shows the spinner under the rows. */ isLoadingMore?: boolean + /** + * Any request for the list is in flight, a background refetch included. Paging waits while it + * runs and asks again once it settles, so a sentinel reached mid-refetch is not left idle. + * Defaults to `isLoadingMore`. + */ + isFetching?: boolean pagination?: PaginationConfig /** * Sanctioned overlay slot. Rendered absolutely against the table region @@ -286,6 +293,7 @@ const ResourceTable = memo(function ResourceTable({ onLoadMore, hasMore, isLoadingMore, + isFetching = isLoadingMore, pagination, overlay, emptyState, @@ -298,7 +306,7 @@ const ResourceTable = memo(function ResourceTable({ */ const loadMoreRef = useLoadMoreSentinel({ onLoadMore, - enabled: Boolean(hasMore) && !isLoadingMore, + enabled: Boolean(hasMore) && !isFetching, }) const showEmptyState = rows.length === 0 && Boolean(emptyState) diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index dd2fa70d3bc..074f8c9862f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -639,13 +639,9 @@ function FilesContent() { ) /** Each shown folder's whole-subtree size, rolled up on the server per parent folder. */ - const folderSizeParents = useMemo( - () => - isSearching - ? [...new Set(visibleFolders.map((folder) => folder.parentId ?? null))] - : [currentFolderId], - [isSearching, visibleFolders, currentFolderId] - ) + const folderSizeParents = isSearching + ? visibleFolders.map((folder) => folder.parentId ?? null) + : [currentFolderId] const folderSizes = useWorkspaceFileFolderSizes(workspaceId, folderSizeParents, { enabled: isListOpen, }) @@ -2207,7 +2203,8 @@ function FilesContent() { virtualized onLoadMore={fileList.loadMore} hasMore={fileList.hasNextPage} - isLoadingMore={fileList.isFetching} + isLoadingMore={fileList.isFetchingNextPage} + isFetching={fileList.isFetching} emptyState={ listState === 'empty' ? ( <FilesEmptyState diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx index a5c6007d61d..c8457d034f0 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/file-upload/file-upload.tsx @@ -48,7 +48,6 @@ import { useWorkflowStore } from '@/stores/workflows/workflow/store' const logger = createLogger('FileUpload') -/** How many files the picker lists per page. */ const PICKER_PAGE_SIZE = 100 const EMPTY_PICKER_FILES: WorkspaceFileRecord[] = [] @@ -147,7 +146,6 @@ interface SingleFileSelectorProps { onOpenChange: (open: boolean) => void /** Hears the typed query (and `''` when the menu closes), which the options are searched by. */ onSearchChange: (query: string) => void - /** Another page of options can be loaded. */ hasMore: boolean isLoadingMore: boolean onLoadMore: () => void diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts index 0db3ed1e6ef..7acf2a800a0 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/search-replace/hooks/use-workflow-resource-replacement-options.ts @@ -1,5 +1,4 @@ import { useMemo } from 'react' -import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { WorkflowSearchMatch, WorkflowSearchReplacementOption, @@ -15,7 +14,6 @@ import { useWorkflowSearchSelectorReplacementOptions, useWorkflowSearchTableReplacementOptions, } from '@/hooks/queries/workflow-search-replace' -import { useDebounce } from '@/hooks/use-debounce' interface UseWorkflowResourceReplacementOptionsParams { matches: WorkflowSearchMatch[] @@ -34,13 +32,12 @@ export function useWorkflowResourceReplacementOptions({ query, chosen, }: UseWorkflowResourceReplacementOptionsParams): WorkflowSearchReplacementOption[] { - const fileQuery = useDebounce(query.trim(), SEARCH_DEBOUNCE_MS) const oauthOptions = useWorkflowSearchOAuthReplacementOptions(matches, workspaceId, workflowId) const knowledgeOptions = useWorkflowSearchKnowledgeReplacementOptions(matches, workspaceId) const selectorOptions = useWorkflowSearchSelectorReplacementOptions(matches) const tableOptions = useWorkflowSearchTableReplacementOptions(matches, workspaceId) const fileOptions = useWorkflowSearchFileReplacementOptions(matches, workspaceId, { - query: fileQuery, + query, chosen, }) const mcpServerOptions = useWorkflowSearchMcpServerReplacementOptions(matches, workspaceId) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx index 2f06dc82160..c1de58eb183 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx @@ -121,21 +121,21 @@ function FilesFlyoutLevel({ }) ) const folderStats = useWorkspaceFileFolderSizes(workspaceId, [folderId]) - const pinned = usePinnedWorkspaceFiles(workspaceId, folderId) + const { files: pinnedFiles, pinnedFileIds } = usePinnedWorkspaceFiles(workspaceId, folderId) const pinnedFolderIds = usePinnedIds(workspaceId, 'folder') const entries = useMemo(() => { - const files = distinctFiles([pinned.files, fileList.data?.files]) + const files = distinctFiles([pinnedFiles, fileList.data?.files]) return buildFlyoutEntries({ folders: folders .filter((folder) => (folder.parentId ?? null) === folderId) .map((folder) => ({ ...folder, parentId: null })), items: files.map((file) => ({ ...file, folderId: null })), pinnedFolderIds, - pinnedItemIds: pinned.pinnedFileIds, + pinnedItemIds: pinnedFileIds, hrefForItem: (file) => `/workspace/${workspaceId}/${FILE_META.listSegment}/${file.id}`, }) - }, [fileList.data, folders, folderId, pinned, pinnedFolderIds, workspaceId]) + }, [fileList.data, folders, folderId, pinnedFiles, pinnedFileIds, pinnedFolderIds, workspaceId]) /** The folders that have subfolders, indexed once for the per-folder emptiness check. */ const parentsWithSubfolders = useMemo( diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 5442fc9a874..0bd03986890 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -99,7 +99,6 @@ async function fetchWorkspaceFileListPage( return { files: data.files, nextCursor: data.nextCursor ?? null } } -/** The options behind one paged list. */ function getWorkspaceFileListQueryOptions(workspaceId: string, params: WorkspaceFileListParams) { return { queryKey: workspaceFilesKeys.list(workspaceId, params), @@ -111,6 +110,11 @@ function getWorkspaceFileListQueryOptions(workspaceId: string, params: Workspace } } +/** Whether `queryKey` starts with `prefix`, segment for segment. */ +function hasKeyPrefix(queryKey: QueryKey, prefix: QueryKey): boolean { + return prefix.every((segment, index) => queryKey[index] === segment) +} + /** Whether two lists show the same place: one folder (or the same whole-workspace span). */ function isSameListPlace(previous: unknown, next: WorkspaceFileListParams): boolean { if (typeof previous !== 'object' || previous === null) return false @@ -154,7 +158,11 @@ export function useWorkspaceFileList( select: selectListFiles, placeholderData: (previous, previousQuery) => { const key = previousQuery?.queryKey - return key?.[2] === workspaceId && isSameListPlace(key[3], params) ? previous : undefined + return key && + hasKeyPrefix(key, workspaceFilesKeys.workspaceLists(workspaceId)) && + isSameListPlace(key[3], params) + ? previous + : undefined }, }) /** Read when called, not at render, so a caller only re-renders for what it reads itself. */ @@ -207,7 +215,12 @@ export function useWorkspaceFileSearch( return useQuery({ ...getWorkspaceFileSearchQueryOptions(workspaceId, { ...params, q }), enabled: Boolean(workspaceId) && q.length > 0 && (options?.enabled ?? true), - placeholderData: keepPreviousData, + /** The last search's matches while the next lands — never another workspace's. */ + placeholderData: (previous, previousQuery) => + previousQuery && + hasKeyPrefix(previousQuery.queryKey, workspaceFilesKeys.workspaceSearches(workspaceId)) + ? previous + : undefined, }) } @@ -244,10 +257,7 @@ export function useWorkspaceFileFolderSizes( parentIds: readonly (string | null)[], options?: { enabled?: boolean } ): ReadonlyMap<string, WorkspaceFileFolderStat> { - const parents = useMemo( - () => [...new Set(parentIds.map((parentId) => parentId ?? 'root'))].sort(), - [parentIds] - ) + const parents = [...new Set(parentIds.map((parentId) => parentId ?? 'root'))].sort() const { data } = useQuery({ queryKey: workspaceFilesKeys.folderStat(workspaceId, parents), queryFn: ({ signal }) => fetchWorkspaceFileFolderStats(workspaceId, parents, signal), @@ -497,8 +507,9 @@ export function useWorkspaceFilePickerOptions( return { files: settledQuery ? matching.data : recent.data?.pages[0]?.files, settledQuery, - isPending: - recent.isPending || (typed.length > 0 && (typed !== settledQuery || matching.isFetching)), + isPending: settledQuery + ? typed !== settledQuery || matching.isFetching + : recent.isPending || typed.length > 0, } } @@ -713,7 +724,7 @@ function fileAddressFilter(workspaceId: string, fileId: string) { return { predicate: (query: { queryHash: string; queryKey: QueryKey; state: { data: unknown } }) => query.queryHash === record || - (lookups.every((segment, index) => query.queryKey[index] === segment) && + (hasKeyPrefix(query.queryKey, lookups) && isRecordLike(query.state.data) && query.state.data.id === fileId), } diff --git a/apps/sim/lib/workspace-files/file-type-sql.ts b/apps/sim/lib/workspace-files/file-type-sql.ts index a69193d93ca..20b4d01f7d8 100644 --- a/apps/sim/lib/workspace-files/file-type-sql.ts +++ b/apps/sim/lib/workspace-files/file-type-sql.ts @@ -72,7 +72,6 @@ export function fileKindSql(kind: FileBrowserType, mime: SQL, extension: SQL): S return sql`lower(${mime}) in (${list(mimesOfKind(kind))})` } -/** Whether a file of `size` bytes is in `bucket`. */ export function fileSizeBucketSql(bucket: FileBrowserSize, size: SQL): SQL { if (bucket === 'small') return sql`${size} < ${FILE_BROWSER_SIZE_BOUNDARIES.small}` if (bucket === 'medium') { From adc7f97e94baaabb708f359c6a9fffd1e03ce5ef Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 21:32:25 -0700 Subject: [PATCH 11/15] test(files): tighten the paged file reads' tests - Cover the load-more footer: paging resumes once a background refetch settles, and only a next-page load shows the spinner; a search never keeps another workspace's matches. - The browser-invalidation test mounts a search beside the list and holds both refetches; the stale-key test checks its refetch filter against real cache entries. - The full-table type parity test also covers names the MIME table does not know, replacing the sort-local label check. - The lookup batcher test derives its sizes from the contract limits; drop a route test the contract enum already guarantees, a chip test the following ones cover, and an orphaned comment. --- .../api/workspaces/[id]/files/route.test.ts | 6 -- .../resource/resource-table-paging.test.tsx | 73 +++++++++++++++ .../home/resolve-resource-ref.test.ts | 14 --- .../utils/workspace-file-cache.test.ts | 2 +- .../workspace-file-lookup-batcher.test.ts | 17 +++- .../hooks/queries/workspace-files.test.tsx | 90 +++++++++++-------- .../contexts/workspace/file-query.test.ts | 7 -- .../file-listing.integration.ts | 29 +++--- 8 files changed, 155 insertions(+), 83 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx diff --git a/apps/sim/app/api/workspaces/[id]/files/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/route.test.ts index 252c13f25f5..950a2602c90 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.test.ts @@ -158,11 +158,5 @@ describe('/api/workspaces/[id]/files', () => { } expect(collection).toHaveBeenCalledTimes(2) }) - - it('rejects a kind or size bucket the list does not know', async () => { - expect((await list('?types=spreadsheet')).status).toBe(400) - expect((await list('?sizes=huge')).status).toBe(400) - expect(collection).not.toHaveBeenCalled() - }) }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx new file mode 100644 index 00000000000..88e4c9fbb46 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx @@ -0,0 +1,73 @@ +/** + * @vitest-environment jsdom + * + * The table's load-more footer keeps two concerns apart: its spinner shows only while the next + * page loads, and its sentinel waits out any request for the list — a background refetch + * included — then asks again once it settles, so reaching the bottom mid-refetch never stalls. + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { Resource } from '@/app/workspace/[workspaceId]/components/resource/resource' + +/** Each observed sentinel, reported as on screen the moment it is observed. */ +class OnScreenObserver { + constructor(private readonly report: IntersectionObserverCallback) {} + observe(target: Element) { + this.report( + [{ isIntersecting: true, target } as unknown as IntersectionObserverEntry], + this as unknown as IntersectionObserver + ) + } + disconnect() {} + unobserve() {} + takeRecords() { + return [] + } +} + +const columns = [{ id: 'name', header: 'Name' }] +const rows = [{ id: 'a', cells: { name: { label: 'a.md' } } }] + +describe('Resource.Table paging footer', () => { + let root: Root + let container: HTMLDivElement + + beforeEach(() => { + vi.stubGlobal('IntersectionObserver', OnScreenObserver) + container = document.createElement('div') + root = createRoot(container) + }) + + afterEach(() => { + act(() => root.unmount()) + vi.unstubAllGlobals() + }) + + function render(props: { isLoadingMore: boolean; isFetching: boolean; onLoadMore: () => void }) { + act(() => { + root.render(<Resource.Table columns={columns} rows={rows} hasMore {...props} />) + }) + } + + it('asks for the next page once a background refetch it reached the bottom during settles', () => { + const onLoadMore = vi.fn() + render({ isLoadingMore: false, isFetching: true, onLoadMore }) + expect(onLoadMore).not.toHaveBeenCalled() + + render({ isLoadingMore: false, isFetching: false, onLoadMore }) + expect(onLoadMore).toHaveBeenCalledTimes(1) + }) + + it('shows the bottom spinner for a next page, never for a background refetch', () => { + const onLoadMore = vi.fn() + /** The footer is the sentinel the table mounts below its rows while more pages remain. */ + const footer = () => container.querySelector('.py-3') + render({ isLoadingMore: false, isFetching: true, onLoadMore }) + expect(footer()).not.toBeNull() + expect(footer()?.childElementCount).toBe(0) + + render({ isLoadingMore: true, isFetching: true, onLoadMore }) + expect(footer()?.childElementCount).toBe(1) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts index 814e1c86275..f23601d942d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts @@ -111,20 +111,6 @@ describe('fetchWorkspaceResourceRef', () => { ).resolves.toMatchObject({ type: 'file', id: 'wf_b', path: 'files/b/notes.md' }) }) - it('resolves a unique name with one batched lookup and its name search', async () => { - serveWorkspace([file({ id: 'wf_a', name: 'notes.md' }), file({ id: 'wf_c', name: 'plan.md' })]) - const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) - - await expect(fetchWorkspaceResourceRef(client, 'ws-1', ref())).resolves.toMatchObject({ - id: 'wf_a', - }) - expect( - apiClientRequestMockFns.mockRequestJson.mock.calls.filter( - ([contract]) => contract === lookupWorkspaceFilesContract - ) - ).toHaveLength(1) - }) - it('asks for each candidate once, as a reference, which already answers an exact id', async () => { serveWorkspace([file({ id: 'wf_a', name: 'notes.md' })]) const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) diff --git a/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts b/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts index 46561c44c1a..0a276490a8e 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-cache.test.ts @@ -132,7 +132,7 @@ describe('updateCachedWorkspaceFile', () => { }) describe('cached file seeds', () => { - it('finds the freshest copy across reads, by one scan for a batch', () => { + it('finds the freshest copy across live reads, never one from an archived list', () => { const client = new QueryClient() client.setQueryData(liveKey, pages([file('a', { name: 'old.md' })]), { updatedAt: 1 }) client.setQueryData(workspaceFilesKeys.record('ws-1', 'a'), file('a', { name: 'new.md' }), { diff --git a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts index 351506718b0..2915b68cd43 100644 --- a/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts +++ b/apps/sim/hooks/queries/utils/workspace-file-lookup-batcher.test.ts @@ -1,6 +1,10 @@ import { flushMicrotasks } from '@sim/testing/helpers/async' import { createDeferred } from '@sim/testing/helpers/deferred' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + MAX_WORKSPACE_FILE_LOOKUP_ENTRIES, + MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH, +} from '@/lib/api/contracts/workspace-files' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { createWorkspaceFileLookupBatcher } from '@/hooks/queries/utils/workspace-file-lookup-batcher' @@ -77,7 +81,11 @@ describe('createWorkspaceFileLookupBatcher', () => { }) it('splits a kind past the per-request cap across requests, resolving every value', async () => { - const files = Array.from({ length: 150 }, (_, index) => record(`wf_${index}`)) + const remainder = MAX_WORKSPACE_FILE_LOOKUP_ENTRIES / 2 + const files = Array.from( + { length: MAX_WORKSPACE_FILE_LOOKUP_ENTRIES + remainder }, + (_, index) => record(`wf_${index}`) + ) const fetchLookup = serve(files) const batcher = createWorkspaceFileLookupBatcher(fetchLookup) @@ -85,7 +93,10 @@ describe('createWorkspaceFileLookupBatcher', () => { await vi.runAllTimersAsync() expect(await results).toEqual(files) - expect(fetchLookup.mock.calls.map(([, body]) => body.ids?.length)).toEqual([100, 50]) + expect(fetchLookup.mock.calls.map(([, body]) => body.ids?.length)).toEqual([ + MAX_WORKSPACE_FILE_LOOKUP_ENTRIES, + remainder, + ]) }) it('drops a lookup aborted before its batch is sent, and rejects one aborted after alone', async () => { @@ -120,7 +131,7 @@ describe('createWorkspaceFileLookupBatcher', () => { const batcher = createWorkspaceFileLookupBatcher(fetchLookup) const results = Promise.all([ - batcher.load('ws-1', 'key', 'x'.repeat(1025)), + batcher.load('ws-1', 'key', 'x'.repeat(MAX_WORKSPACE_FILE_LOOKUP_VALUE_LENGTH + 1)), batcher.load('ws-1', 'reference', ''), batcher.load('ws-1', 'id', a.id), ]) diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index 83b4c0d5bfa..e41beb7c93a 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -282,35 +282,23 @@ describe('useWorkspaceFileContent stale storage key', () => { // is what would spin. expect(refetchQueries).toHaveBeenCalledTimes(1) /** Only the reads that can hold this file's key: its record and the lookups that found it. */ - const [filter] = refetchQueries.mock.calls[0] as unknown as [ - { - predicate: (query: { - queryHash: string - queryKey: unknown[] - state: { data: unknown } - }) => boolean - }, - ] - const query = (queryKey: readonly unknown[], data: unknown = null) => ({ - queryKey: [...queryKey], - queryHash: JSON.stringify(queryKey), - state: { data }, - }) - expect(filter.predicate(query(workspaceFilesKeys.record('ws-1', 'file-1')))).toBe(true) - expect( - filter.predicate( - query(workspaceFilesKeys.lookup('ws-1', 'reference', 'doc.md'), fileRecord('file-1')) - ) - ).toBe(true) - expect( - filter.predicate(query(workspaceFilesKeys.lookup('ws-1', 'id', 'other'), fileRecord('other'))) - ).toBe(false) - expect(filter.predicate(query(workspaceFilesKeys.record('ws-1', 'other')))).toBe(false) + const record = workspaceFilesKeys.record('ws-1', 'file-1') + const lookup = workspaceFilesKeys.lookup('ws-1', 'reference', 'doc.md') + queryClient.setQueryData(record, fileRecord('file-1')) + queryClient.setQueryData(lookup, fileRecord('file-1')) + queryClient.setQueryData(workspaceFilesKeys.lookup('ws-1', 'id', 'other'), fileRecord('other')) + queryClient.setQueryData(workspaceFilesKeys.record('ws-1', 'other'), fileRecord('other')) + queryClient.setQueryData(workspaceFilesKeys.search('ws-1', { q: 'doc', limit: 20 }), [ + fileRecord('file-1'), + ]) + const [filter] = refetchQueries.mock.calls[0] expect( - filter.predicate( - query(workspaceFilesKeys.search('ws-1', { q: 'doc', limit: 20 }), [fileRecord('file-1')]) - ) - ).toBe(false) + queryClient + .getQueryCache() + .findAll({ ...filter, type: 'all' }) + .map((query) => query.queryKey) + ).toEqual(expect.arrayContaining([record, lookup])) + expect(queryClient.getQueryCache().findAll({ ...filter, type: 'all' })).toHaveLength(2) unmount() }) }) @@ -709,33 +697,38 @@ describe('invalidateWorkspaceFileBrowsers', () => { it('settles only once the mounted list and search reads have refetched', async () => { let listCalls = 0 let searchCalls = 0 - const refetch = createDeferred<void>() + const listRefetch = createDeferred<void>() + const searchRefetch = createDeferred<void>() mockRequestJson.mockImplementation(async (contract) => { if (contract === listWorkspaceFilesContract) { - if (++listCalls > 1) await refetch.promise + if (++listCalls > 1) await listRefetch.promise return { success: true, files: [fileRecord('a')], nextCursor: null } } if (contract === searchWorkspaceFilesContract) { - searchCalls++ + if (++searchCalls > 1) await searchRefetch.promise return { success: true, files: [fileRecord('a')] } } return { success: true, folders: [] } }) const params = workspaceFileListParams({ folderId: null, sortBy: 'name', sortOrder: 'asc' }) - const hook = mountHook(() => useWorkspaceFileList('ws-1', params)) + const hook = mountHook(() => ({ + list: useWorkspaceFileList('ws-1', params), + search: useWorkspaceFileSearch('ws-1', { q: 'a', limit: 20 }), + })) try { - await vi.waitFor(() => expect(hook.result().data).toBeDefined()) + await vi.waitFor(() => expect(hook.result().list.data).toBeDefined()) + await vi.waitFor(() => expect(hook.result().search.data).toBeDefined()) let done = false const invalidation = invalidateWorkspaceFileBrowsers(hook.client, 'ws-1').then(() => { done = true }) - await vi.waitFor(() => expect(listCalls).toBe(2)) + await vi.waitFor(() => expect([listCalls, searchCalls]).toEqual([2, 2])) + listRefetch.resolve() await flushMacrotask() expect(done).toBe(false) - refetch.resolve() + searchRefetch.resolve() await invalidation expect(done).toBe(true) - expect(searchCalls).toBe(0) } finally { hook.unmount() } @@ -858,3 +851,28 @@ describe('useWorkspaceImageDimensionsAdapter', () => { } }) }) + +describe('useWorkspaceFileSearch placeholder', () => { + it("keeps the last matches while a new query loads, but never another workspace's", async () => { + const pending = createDeferred<unknown>() + mockRequestJson.mockImplementation(async (_contract, input) => { + const { q } = (input as { query: { q: string } }).query + if (q === 'first') return { success: true, files: [fileRecord('first')] } + return pending.promise + }) + let workspaceId = 'ws-1' + let q = 'first' + const hook = mountHook(() => useWorkspaceFileSearch(workspaceId, { q, limit: 20 })) + try { + await vi.waitFor(() => expect(hook.result().data?.[0]?.id).toBe('first')) + q = 'second' + hook.rerender() + expect(hook.result().data?.[0]?.id).toBe('first') + workspaceId = 'ws-2' + hook.rerender() + expect(hook.result().data).toBeUndefined() + } finally { + hook.unmount() + } + }) +}) diff --git a/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts b/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts index 473583ca76f..307860d4e7a 100644 --- a/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/file-query.test.ts @@ -134,10 +134,3 @@ describe('queryWorkspaceFiles', () => { ).rejects.toMatchObject({ code: 'validation' }) }) }) - -/** - * `listWorkspaceFiles` materializes a whole scope, so what it reads per row is - * multiplied by the size of the workspace. These assertions pin the two ways that - * stays bounded: the projection, and the optional row cap its one budgeted caller - * (the workspace layout's server seed) uses to detect an oversized workspace. - */ diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 689c84d1b49..d8ea7d4ba40 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -647,15 +647,6 @@ describe('bounded workspace file reads in PostgreSQL', () => { } ) - it('labels each file as its Type column does', async () => { - const rows = await db.execute<{ id: string; label: string }>(sql` - SELECT id, ${workspaceFileTypeLabelExpression} AS label FROM ${workspaceFiles} - WHERE workspace_id = ${sortWorkspaceId}`) - expect(new Map(rows.map((row) => [row.id, row.label]))).toEqual( - new Map(seeded.map((file) => [file.id, file.label])) - ) - }) - /** * Neither key is indexed, so a page sorts its scope — but only its scope: the folder * predicate must still seek the folder index rather than reading every live file. @@ -700,15 +691,21 @@ describe('bounded workspace file reads in PostgreSQL', () => { /** * Every extension the MIME table knows, stored three ways: declaring nothing (so the * extension decides), declaring the generic type (likewise), and declaring its own type (so - * the declared type decides). The SQL must agree with the JS rules for each. + * the declared type decides) — plus names the table does not know, with and without an + * extension. The SQL must agree with the JS rules for each. */ it('labels and classifies a file of every known extension as the JS rules do', async () => { - const cases = Object.keys(EXTENSION_TO_MIME).flatMap((extension) => - ['', 'application/octet-stream', EXTENSION_TO_MIME[extension]].map((contentType) => ({ - name: `file.${extension}`, - contentType, - })) - ) + const cases = [ + ...Object.keys(EXTENSION_TO_MIME).flatMap((extension) => + ['', 'application/octet-stream', EXTENSION_TO_MIME[extension]].map((contentType) => ({ + name: `file.${extension}`, + contentType, + })) + ), + { name: 'noext', contentType: 'application/x-custom' }, + { name: 'file.foo', contentType: 'application/octet-stream' }, + { name: 'file.foo', contentType: '' }, + ] const ids = new Map<string, { name: string; contentType: string }>() for (const [index, fixture] of cases.entries()) { const file = await insertFile({ From cd527cd6ceea5868981635ed306bfc5adcbe3795 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 21:44:54 -0700 Subject: [PATCH 12/15] test(files): drop a redundant global unstub and the deleted queries test's type baseline --- .../components/resource/resource-table-paging.test.tsx | 1 - scripts/check-test-types.baseline.json | 3 --- 2 files changed, 4 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx index 88e4c9fbb46..07a8bb751f8 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx @@ -41,7 +41,6 @@ describe('Resource.Table paging footer', () => { afterEach(() => { act(() => root.unmount()) - vi.unstubAllGlobals() }) function render(props: { isLoadingMore: boolean; isFetching: boolean; onLoadMore: () => void }) { diff --git a/scripts/check-test-types.baseline.json b/scripts/check-test-types.baseline.json index d0d62062e90..57307784036 100644 --- a/scripts/check-test-types.baseline.json +++ b/scripts/check-test-types.baseline.json @@ -2428,9 +2428,6 @@ "apps/sim/lib/workspace-files/page-document.server.test.ts": { "TS2345": 4 }, - "apps/sim/lib/workspace-files/queries.test.ts": { - "TS2488": 1 - }, "apps/sim/lib/workspace-files/search/admission.test.ts": { "TS2550": 1 }, From 03d31626c9652c6679c6c58e6a8f8bdf64d5479d Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 22:09:41 -0700 Subject: [PATCH 13/15] fix(files): address the paged file reads' first review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A chip whose bare name is longer than a search query is checked for a shared name by its leading part, so a duplicate long name still refuses rather than opening one of the files. - Folder sizes split more parents than one request may name into several requests. - File menus keep the server's ranked matches for a settled query (`matchedQuery`) instead of re-filtering them by substring; other families, and files still answering an older query, are filtered as before. - An empty set filter (`?types=`) reads as no filter rather than switching to the paged list. - After a failed next-page load the table's sentinel waits for the bottom to leave view and return before asking again, rather than retrying in a loop. - Both browsers read the stored type as `normalizeMimeType` does — lowercased, parameters dropped — in JS and SQL alike; the parity test covers parameterized and mixed-case types. - The rail flyout's root keeps folders whose parent is archived or gone, as the whole-tree build does, so their files stay reachable. - Image lookups use the stored id when a document percent-encodes it. - A malformed persisted serve path is left unresolved instead of throwing in search-replace and picker selections (`tryExtractStorageKey`). - A routed file whose record read fails for a reason other than a 404 shows the failure with a retry instead of an empty list shell; only a 404 falls back to the list. - A heading-derived rename steps past numbered names the sibling search did not reach when the server reports them taken. --- .../api/workspaces/[id]/files/route.test.ts | 11 +++ .../components/folders/flyout-entries.test.ts | 23 ++++- .../components/folders/flyout-entries.ts | 19 ++++ .../[workspaceId]/components/folders/index.ts | 2 +- .../resource/resource-table-paging.test.tsx | 37 +++++++- .../components/resource/resource.tsx | 7 ++ .../mention/mention-list.tsx | 10 +- .../mention/mention-matching.test.ts | 15 +++ .../rich-markdown-editor/mention/types.ts | 5 + .../mention/use-markdown-mentions.ts | 4 +- .../workspace/[workspaceId]/files/files.tsx | 44 +++++++-- .../[workspaceId]/files/list-entries.test.ts | 13 +++ .../files/untitled-title.test.ts | 36 ++++++- .../[workspaceId]/files/untitled-title.ts | 29 ++++++ .../add-resource-dropdown.tsx | 2 + .../available-resources.ts | 27 ++++-- .../resource-folder-tree.ts | 9 ++ .../resource-mention-items.test.ts | 16 ++++ .../resource-mention-items.ts | 6 +- .../home/resolve-resource-ref.test.ts | 17 ++++ .../home/resolve-resource-ref.ts | 17 +++- .../rail-resource-flyout.tsx | 5 +- .../hooks/queries/pinned-workspace-files.ts | 4 +- .../queries/workflow-search-replace.test.ts | 9 ++ .../hooks/queries/workflow-search-replace.ts | 8 +- .../hooks/queries/workspace-files.test.tsx | 93 ++++++++++++++++++- apps/sim/hooks/queries/workspace-files.ts | 76 +++++++++++---- apps/sim/hooks/use-load-more-sentinel.ts | 23 ++++- apps/sim/lib/api/contracts/primitives.ts | 31 ++++--- apps/sim/lib/api/contracts/workspace-files.ts | 5 +- apps/sim/lib/uploads/utils/file.ts | 12 +++ .../file-listing.integration.ts | 15 ++- apps/sim/lib/workspace-files/browser-query.ts | 3 +- apps/sim/lib/workspace-files/browser.test.ts | 17 ++++ apps/sim/lib/workspace-files/browser.ts | 12 ++- apps/sim/lib/workspace-files/file-type-sql.ts | 10 +- apps/sim/lib/workspace-files/query-scope.ts | 5 +- packages/testing/src/mocks/file-utils.mock.ts | 10 ++ 38 files changed, 602 insertions(+), 85 deletions(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts create mode 100644 apps/sim/lib/workspace-files/browser.test.ts diff --git a/apps/sim/app/api/workspaces/[id]/files/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/route.test.ts index 950a2602c90..406d8ffb2b0 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.test.ts @@ -109,6 +109,17 @@ describe('/api/workspaces/[id]/files', () => { }) }) + it('reads an empty set filter as no filter, keeping the whole-scope listing', async () => { + collection.mockResolvedValueOnce({ files: [FILE], nextKeys: null }) + const response = await list('?scope=active&types=&sizes=&uploadedBy=') + + expect(response.status).toBe(200) + expect(collection.mock.calls[0][0].input).toEqual({ + workspaceId: WORKSPACE_ID, + scope: 'active', + }) + }) + it('resumes from its own cursor and refuses it under another folder', async () => { collection.mockResolvedValue({ files: [FILE], nextKeys: ['notes.md', FILE.id] }) diff --git a/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.test.ts b/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.test.ts index 78529616a73..1b0ed88cb63 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { buildFlyoutEntries } from '@/app/workspace/[workspaceId]/components/folders/flyout-entries' +import { + buildFlyoutEntries, + flyoutLevelFolders, +} from '@/app/workspace/[workspaceId]/components/folders/flyout-entries' function folder(id: string, name: string, parentId: string | null, updatedAt: string) { return { id, name, parentId, updatedAt: new Date(updatedAt) } @@ -49,3 +52,21 @@ describe('buildFlyoutEntries', () => { expect(entries.map((entry) => entry.id)).toEqual(['root']) }) }) + +describe('flyoutLevelFolders', () => { + const folders = [ + folder('docs', 'Docs', null, '2026-01-01'), + folder('sub', 'Sub', 'docs', '2026-01-01'), + folder('orphan', 'Orphan', 'archived-parent', '2026-01-01'), + ] + + it('lists the root children and the folders whose parent is gone at the root', () => { + expect(flyoutLevelFolders(folders, null).map((entry) => entry.id)).toEqual(['docs', 'orphan']) + }) + + it("lists only a folder's own children one level down", () => { + expect(flyoutLevelFolders(folders, 'docs').map((entry) => [entry.id, entry.parentId])).toEqual([ + ['sub', null], + ]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.ts b/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.ts index d9ba6729ef6..3208993b6b0 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/folders/flyout-entries.ts @@ -32,6 +32,25 @@ export interface BuildFlyoutEntriesParams<Item extends FlyoutItemSource> { hrefForItem: (item: Item) => string } +/** + * The folders one lazily built flyout level lists, as root folders of that level: `parentId`'s + * children, and at the root (`null`) also every folder whose parent is not among `folders` — an + * archived or missing parent — which {@link buildFlyoutEntries} surfaces at the root of a + * whole tree, so its contents stay reachable one level at a time too. + */ +export function flyoutLevelFolders<Folder extends FlyoutFolderSource>( + folders: readonly Folder[], + parentId: string | null +): Folder[] { + const folderIds = new Set(folders.map((folder) => folder.id)) + return folders + .filter((folder) => { + const parent = folder.parentId ?? null + return parentId === null ? parent === null || !folderIds.has(parent) : parent === parentId + }) + .map((folder) => ({ ...folder, parentId: null })) +} + function flyoutSortTime(value: Date | string): number { const time = value instanceof Date ? value.getTime() : Date.parse(value) return Number.isNaN(time) ? 0 : time diff --git a/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts b/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts index 36b951ba62a..8813496a37a 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/folders/index.ts @@ -1,6 +1,6 @@ export { readRowDragPayload, writeRowDragPayload } from './drag-payload' export type { BuildFlyoutEntriesParams, FlyoutEntry } from './flyout-entries' -export { buildFlyoutEntries } from './flyout-entries' +export { buildFlyoutEntries, flyoutLevelFolders } from './flyout-entries' export type { BreadcrumbFolder, FolderBreadcrumbItemsOptions } from './folder-breadcrumbs' export { breadcrumbFolderChain, folderBreadcrumbItems } from './folder-breadcrumbs' export { FolderContextMenu } from './folder-context-menu' diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx index 07a8bb751f8..795c063fd00 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource-table-paging.test.tsx @@ -10,16 +10,28 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { Resource } from '@/app/workspace/[workspaceId]/components/resource/resource' +/** The live observers, so a test can move the sentinel out of view and back. */ +const observers = new Set<OnScreenObserver>() + /** Each observed sentinel, reported as on screen the moment it is observed. */ class OnScreenObserver { - constructor(private readonly report: IntersectionObserverCallback) {} + private target: Element | null = null + constructor(private readonly report: IntersectionObserverCallback) { + observers.add(this) + } observe(target: Element) { + this.target = target + this.emit(true) + } + emit(isIntersecting: boolean) { this.report( - [{ isIntersecting: true, target } as unknown as IntersectionObserverEntry], + [{ isIntersecting, target: this.target } as unknown as IntersectionObserverEntry], this as unknown as IntersectionObserver ) } - disconnect() {} + disconnect() { + observers.delete(this) + } unobserve() {} takeRecords() { return [] @@ -43,7 +55,12 @@ describe('Resource.Table paging footer', () => { act(() => root.unmount()) }) - function render(props: { isLoadingMore: boolean; isFetching: boolean; onLoadMore: () => void }) { + function render(props: { + isLoadingMore: boolean + isFetching: boolean + loadMoreFailed?: boolean + onLoadMore: () => void + }) { act(() => { root.render(<Resource.Table columns={columns} rows={rows} hasMore {...props} />) }) @@ -69,4 +86,16 @@ describe('Resource.Table paging footer', () => { render({ isLoadingMore: true, isFetching: true, onLoadMore }) expect(footer()?.childElementCount).toBe(1) }) + + it('does not retry a failed page on its own, only once the bottom is scrolled back to', () => { + const onLoadMore = vi.fn() + render({ isLoadingMore: true, isFetching: true, onLoadMore }) + render({ isLoadingMore: false, isFetching: false, loadMoreFailed: true, onLoadMore }) + expect(onLoadMore).not.toHaveBeenCalled() + + const sentinel = () => [...observers][0] + act(() => sentinel().emit(false)) + act(() => sentinel().emit(true)) + expect(onLoadMore).toHaveBeenCalledTimes(1) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx index 9a1cbab363c..5a3293a5594 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/resource/resource.tsx @@ -239,6 +239,11 @@ interface ResourceTableProps { * Defaults to `isLoadingMore`. */ isFetching?: boolean + /** + * The last next-page load failed. Paging then waits for the sentinel to leave view and come + * back before asking again, rather than retrying a failing request in a loop. + */ + loadMoreFailed?: boolean pagination?: PaginationConfig /** * Sanctioned overlay slot. Rendered absolutely against the table region @@ -294,6 +299,7 @@ const ResourceTable = memo(function ResourceTable({ hasMore, isLoadingMore, isFetching = isLoadingMore, + loadMoreFailed = false, pagination, overlay, emptyState, @@ -307,6 +313,7 @@ const ResourceTable = memo(function ResourceTable({ const loadMoreRef = useLoadMoreSentinel({ onLoadMore, enabled: Boolean(hasMore) && !isFetching, + waitForReentry: loadMoreFailed, }) const showEmptyState = rows.length === 0 && Boolean(emptyState) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx index 618560c3f2f..ff73e3dae8b 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx @@ -8,6 +8,14 @@ import { import type { MentionStore } from './mention-store' import type { MentionItem } from './types' +/** + * Whether `item` belongs under the typed `query` (lowercased, trimmed): its label contains it, or + * it is one of the server's ranked matches for exactly that query (`matchedQuery`). + */ +export function mentionItemMatches(item: MentionItem, query: string): boolean { + return item.matchedQuery?.toLowerCase() === query || item.label.toLowerCase().includes(query) +} + export type MentionListHandle = SuggestionKeyDownHandler interface MentionListProps { @@ -65,7 +73,7 @@ export const MentionList = forwardRef<MentionListHandle, MentionListProps>(funct const byGroup = new Map<string, MentionItem[]>() let shown = 0 for (const item of rawItems) { - if (q && !item.label.toLowerCase().includes(q)) continue + if (q && !mentionItemMatches(item, q)) continue if (q && shown >= MAX_WHEN_FILTERED) break const bucket = byGroup.get(item.group) if (!q && bucket && bucket.length >= MAX_PER_GROUP) continue diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts new file mode 100644 index 00000000000..a23b164e7b9 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest' +import { mentionItemMatches } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list' + +describe('mentionItemMatches', () => { + const file = { kind: 'file' as const, id: 'wf_a', label: 'Quarterly-Report.md', group: 'Files' } + + it("keeps a file the server matched for the query though its name doesn't contain it", () => { + expect(mentionItemMatches({ ...file, matchedQuery: 'qrep' }, 'qrep')).toBe(true) + }) + + it('filters by substring an item no search matched for this query', () => { + expect(mentionItemMatches({ ...file, matchedQuery: 'older' }, 'qrep')).toBe(false) + expect(mentionItemMatches(file, 'report')).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/types.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/types.ts index a26eb56869b..9c539af4f42 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/types.ts @@ -26,4 +26,9 @@ export interface MentionItem { group: string /** Optional per-item icon (emcn category icon or a brand block icon). */ icon?: ComponentType<{ className?: string }> + /** + * The query this item is one of the server's ranked matches for (files only). The menu keeps + * it listed for that query rather than re-filtering it by substring. + */ + matchedQuery?: string } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts index 5ae376ffad9..bb9e6f7ecbd 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts @@ -29,7 +29,7 @@ export function useMarkdownMentions( const ws = active ? workspaceId : undefined const wsStr = ws ?? '' - const { files } = useWorkspaceFilePickerOptions(wsStr, options.fileQuery ?? '', { + const { files, matchedQuery } = useWorkspaceFilePickerOptions(wsStr, options.fileQuery ?? '', { enabled: active, }) const folders = useWorkspaceFileFolders(wsStr, 'active') @@ -61,6 +61,7 @@ export function useMarkdownMentions( label: file.name, group: 'Files', icon: mentionIcon('file', file.id, file.name), + ...(matchedQuery ? { matchedQuery } : {}), }) for (const folder of folders.data ?? []) items.push({ @@ -108,6 +109,7 @@ export function useMarkdownMentions( }, [ active, files, + matchedQuery, folders.data, tables.data, knowledgeBases.data, diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 074f8c9862f..08cf1f6be66 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Avatar, Button, + Chip, ChipCombobox, ChipConfirmModal, Columns2, @@ -26,6 +27,7 @@ import { useQueryClient } from '@tanstack/react-query' import { useParams, useRouter } from 'next/navigation' import { useQueryStates } from 'nuqs' import { usePostHog } from 'posthog-js/react' +import { EmptyState } from '@/components/empty-state/empty-state' import { getDocumentIcon } from '@/components/icons/document-icons' import { useLimitUpgradeToast } from '@/lib/billing/client' import { captureEvent } from '@/lib/posthog/client' @@ -130,6 +132,7 @@ import { DEFAULT_UNTITLED_NAME, deriveMarkdownFileName, isUntitledName, + renameToUniqueMarkdownName, uniqueMarkdownName, } from '@/app/workspace/[workspaceId]/files/untitled-title' import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' @@ -150,6 +153,7 @@ import { import { getWorkspaceFileRecordQueryOptions, getWorkspaceFileSearchQueryOptions, + isWorkspaceFileNameTaken, useCreateWorkspaceFile, useDeleteWorkspaceFile, useRenameWorkspaceFile, @@ -157,6 +161,7 @@ import { useWorkspaceFileFolderSizes, useWorkspaceFileList, useWorkspaceFileRecord, + workspaceFileRecordState, } from '@/hooks/queries/workspace-files' import { useContextMenu } from '@/hooks/use-context-menu' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' @@ -472,9 +477,9 @@ function FilesContent() { enabled: fileIdFromRoute !== null, }) const selectedFile = fileIdFromRoute ? selectedFileRecord.data : null + const routedFile = fileIdFromRoute ? workspaceFileRecordState(selectedFileRecord) : 'none' /** A route naming no live file falls back to the list, which is then what is shown. */ - const isRoutedFileMissing = - fileIdFromRoute !== null && (selectedFileRecord.data === null || selectedFileRecord.isError) + const isRoutedFileMissing = routedFile === 'missing' /** A remembered sort or filter is applied before the list is read, so it is read once. */ const isListOpen = (fileIdFromRoute === null || isRoutedFileMissing) && isListPreferenceReady @@ -595,9 +600,15 @@ function FilesContent() { const siblingNames = new Set( similar.filter((file) => file.id !== currentFile.id).map((file) => file.name) ) - const name = uniqueMarkdownName(derived, siblingNames) - if (name === currentFile.name) return - await renameFile.mutateAsync({ workspaceId, fileId: currentFile.id, name }) + if (uniqueMarkdownName(derived, siblingNames) === currentFile.name) return + /** The search reaches at most its limit of siblings; the server catches any it missed. */ + await renameToUniqueMarkdownName( + derived, + siblingNames, + (name) => + renameFile.mutateAsync({ workspaceId, fileId: currentFile.id, name, quiet: true }), + isWorkspaceFileNameTaken + ) } catch (err) { logger.error('Failed to auto-name file from heading:', err) } @@ -1515,7 +1526,8 @@ function FilesContent() { * 'editor' instead of waiting forever). */ const appliedModeFileIdRef = useRef<string | null>(null) - const routedFileResolved = selectedFile != null || isRoutedFileMissing + const routedFileResolved = + routedFile === 'found' || routedFile === 'missing' || routedFile === 'failed' useEffect(() => { if (fileIdFromRoute === appliedModeFileIdRef.current) return const isJustCreated = @@ -2106,6 +2118,25 @@ function FilesContent() { ) } + if (routedFile === 'failed') { + return ( + <Resource> + <Resource.Header icon={FILES_HEADER.rootIcon} breadcrumbs={loadingBreadcrumbs} /> + <div className='flex flex-1 items-center justify-center bg-[var(--bg)]'> + <EmptyState + title='Could not load this file' + description={getErrorMessage(selectedFileRecord.error, 'Try again in a moment.')} + action={ + <Chip variant='border' onClick={() => void selectedFileRecord.refetch()}> + Try again + </Chip> + } + /> + </div> + </Resource> + ) + } + if (selectedFile) { return ( <> @@ -2205,6 +2236,7 @@ function FilesContent() { hasMore={fileList.hasNextPage} isLoadingMore={fileList.isFetchingNextPage} isFetching={fileList.isFetching} + loadMoreFailed={fileList.isFetchNextPageError} emptyState={ listState === 'empty' ? ( <FilesEmptyState diff --git a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts index cd6cf3a3a8a..2706ccc15b0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/list-entries.test.ts @@ -82,6 +82,19 @@ describe('mergeFileListEntries', () => { ).toEqual(['pinned20', 'Fpin', 'far90', 'F5', 'a10']) }) + it('orders pinned rows by the active sort, whatever order the pins arrive in', () => { + const pinnedFiles = [file('p30', 30), file('p10', 10), file('p20', 20)] + const order = (pins: Row[]) => + merge({ + files: [], + pinnedFiles: pins, + hasMore: true, + pinned: pins.map((row) => row.id), + }) + expect(order(pinnedFiles)).toEqual(['p10', 'p20', 'p30']) + expect(order([...pinnedFiles].reverse())).toEqual(['p10', 'p20', 'p30']) + }) + it('places a folder tying on the column by name, and rows without a value last either way', () => { const owned = (id: string, owner: string | null): Row & { owner: string | null } => ({ ...file(id, 0), diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts index 16738629d31..11a24da73d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.test.ts @@ -1,5 +1,10 @@ -import { describe, expect, it } from 'vitest' -import { deriveMarkdownFileName, isUntitledName, uniqueMarkdownName } from './untitled-title' +import { describe, expect, it, vi } from 'vitest' +import { + deriveMarkdownFileName, + isUntitledName, + renameToUniqueMarkdownName, + uniqueMarkdownName, +} from './untitled-title' describe('isUntitledName', () => { it.each([ @@ -37,3 +42,30 @@ describe('uniqueMarkdownName', () => { ) }) }) + +describe('renameToUniqueMarkdownName', () => { + const taken = new Error('taken') + const isTaken = (error: unknown) => error === taken + + it('steps past numbered names the sibling read did not reach, as the server reports them', async () => { + const occupied = new Set(['Plan (1).md', 'Plan (2).md']) + const rename = vi.fn(async (candidate: string) => { + if (occupied.has(candidate)) throw taken + }) + await expect( + renameToUniqueMarkdownName('Plan.md', new Set(['Plan.md']), rename, isTaken) + ).resolves.toBe('Plan (3).md') + expect(rename.mock.calls.map(([candidate]) => candidate)).toEqual([ + 'Plan (1).md', + 'Plan (2).md', + 'Plan (3).md', + ]) + }) + + it('rethrows a failure that is not a taken name', async () => { + const failure = new Error('offline') + await expect( + renameToUniqueMarkdownName('Plan.md', new Set(), async () => Promise.reject(failure), isTaken) + ).rejects.toBe(failure) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts index b9e4daadd45..ca8ec6c6b40 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/untitled-title.ts @@ -55,3 +55,32 @@ export function uniqueMarkdownName(name: string, existingNames: ReadonlySet<stri } return candidate } + +/** How many taken names a heading-derived rename steps past before giving up. */ +const MAX_UNIQUE_NAME_ATTEMPTS = 20 + +/** + * Renames to `name`, or its first free numbered variant: starts from the variant `knownNames` + * leaves free and, when the server reports that one taken too (`isTaken` — a sibling the caller's + * read did not reach), steps to the next. Resolves the name it settled on, or `null` after + * {@link MAX_UNIQUE_NAME_ATTEMPTS} taken names; any other failure rejects. + */ +export async function renameToUniqueMarkdownName( + name: string, + knownNames: ReadonlySet<string>, + rename: (candidate: string) => Promise<unknown>, + isTaken: (error: unknown) => boolean +): Promise<string | null> { + const taken = new Set(knownNames) + for (let attempt = 0; attempt < MAX_UNIQUE_NAME_ATTEMPTS; attempt++) { + const candidate = uniqueMarkdownName(name, taken) + try { + await rename(candidate) + return candidate + } catch (error) { + if (!isTaken(error)) throw error + taken.add(candidate) + } + } + return null +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx index e9f74322f38..eeafefe1ea1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/add-resource-dropdown.tsx @@ -35,6 +35,7 @@ import { } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/organization-resource-inventory' import { buildResourceFolderTree, + isServerMatchFor, type ResourceTreeNode, } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' import { resourceFromItem } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-from-item' @@ -329,6 +330,7 @@ function ResourceMenuSearch({ items .filter( (item) => + isServerMatchFor(item, q) || item.name.toLowerCase().includes(q) || (typeof item.workspaceName === 'string' && item.workspaceName.toLowerCase().includes(q)) ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts index 3297e3a50d7..194939e45f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/available-resources.ts @@ -83,8 +83,10 @@ interface UseAvailableResourcesOptions { /** * The text the menu is filtering by, as typed. Files are not listed whole — a workspace can * hold far more than a menu can show — so without a query the file family holds the most - * recently updated files, and with one, the files whose names match it anywhere in the - * workspace (searched once the typing settles). Other families are filtered by the caller. + * recently updated files, and once the typing settles, the server's ranked matches for it + * anywhere in the workspace (by name, word, fuzzy match, or folder path). Those carry the + * query as `matchedQuery`, and a caller keeps them as they are (`isServerMatchFor`) while it + * filters the other families — and files still answering an older query — by substring. */ fileSearch?: string } @@ -142,11 +144,14 @@ export function useAvailableResources( const { data: tests, isPending: testsPending } = useWorkflowTests(workspaceId, { enabled: enabled && testsEnabled && !excludeTypes?.includes('test'), }) - const { files, isPending: filesPending } = useWorkspaceFilePickerOptions( - workspaceId, - options?.fileSearch ?? '', - { limit: RECENT_FILE_LIMIT, enabled } - ) + const { + files, + matchedQuery: fileMatchedQuery, + isPending: filesPending, + } = useWorkspaceFilePickerOptions(workspaceId, options?.fileSearch ?? '', { + limit: RECENT_FILE_LIMIT, + enabled, + }) const { data: knowledgeBases, isPending: knowledgeBasesPending } = useKnowledgeBasesQuery( workspaceId, { enabled: enabled && Boolean(workspaceId) } @@ -249,7 +254,12 @@ export function useAvailableResources( }, { type: 'file' as const, - items: (files ?? []).map((f) => ({ id: f.id, name: f.name, folderId: f.folderId ?? null })), + items: (files ?? []).map((f) => ({ + id: f.id, + name: f.name, + folderId: f.folderId ?? null, + ...(fileMatchedQuery ? { matchedQuery: fileMatchedQuery } : {}), + })), }, { type: 'filefolder' as const, @@ -339,6 +349,7 @@ export function useAvailableResources( fileFolders, tables, files, + fileMatchedQuery, dashboardData, tests, knowledgeBases, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree.ts b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree.ts index ad2a58ca289..fa60e0c99fb 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree.ts @@ -1,5 +1,14 @@ export type AvailableItem = { id: string; name: string; [key: string]: unknown } +/** + * Whether `item` is one of the server's matches for `query` (lowercased and trimmed): a file + * search ranks by name, word, fuzzy, and folder path, so its matches stay listed for the query + * they answer instead of being re-filtered by substring. See `matchedQuery` on file items. + */ +export function isServerMatchFor(item: AvailableItem, query: string): boolean { + return typeof item.matchedQuery === 'string' && item.matchedQuery.toLowerCase() === query +} + export type ResourceTreeNode = | { kind: 'item'; id: string; item: AvailableItem } | { kind: 'folder'; id: string; name: string; children: ResourceTreeNode[] } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts new file mode 100644 index 00000000000..caccc90e1b7 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' +import { resourceMentionMatches } from '@/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items' + +describe('resourceMentionMatches', () => { + it("keeps a file the server ranked for the query, though its name doesn't contain it", () => { + const file = { id: 'wf_a', name: 'Quarterly-Report.md', matchedQuery: 'QRep' } + expect(resourceMentionMatches(file, 'qrep')).toBe(true) + }) + + it('still filters by substring what no search matched for this query', () => { + expect( + resourceMentionMatches({ id: 'wf_a', name: 'notes.md', matchedQuery: 'old' }, 'qrep') + ).toBe(false) + expect(resourceMentionMatches({ id: 't1', name: 'Quarterly table' }, 'quarter')).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts index 4a7e45c129f..d56684458ba 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/plus-menu-dropdown/resource-mention-items.ts @@ -3,7 +3,10 @@ import type { TerminalTabState } from '@sim/terminal-protocol' import { browserTabTitle } from '@/lib/browser-agent/tab-label' import { terminalResourceId } from '@/lib/terminal/resource-id' import { terminalTabTitle } from '@/lib/terminal/tab-label' -import type { AvailableItem } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' +import { + type AvailableItem, + isServerMatchFor, +} from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/add-resource-dropdown/resource-folder-tree' import type { MothershipResourceType } from '@/app/workspace/[workspaceId]/home/types' export interface ResourceMentionGroup { @@ -38,6 +41,7 @@ export function resourceMentionMatches(item: AvailableItem, query: string): bool const normalized = query.toLowerCase().trim() if (!normalized) return true return ( + isServerMatchFor(item, normalized) || item.name.toLowerCase().includes(normalized) || (typeof item.workspaceName === 'string' && item.workspaceName.toLowerCase().includes(normalized)) || diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts index f23601d942d..f259527ae1b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.test.ts @@ -111,6 +111,23 @@ describe('fetchWorkspaceResourceRef', () => { ).resolves.toMatchObject({ type: 'file', id: 'wf_b', path: 'files/b/notes.md' }) }) + it('still refuses a name two files share when it is longer than a search query may be', async () => { + const long = `${'quarterly report '.repeat(15)}final.md` + serveWorkspace([ + file({ id: 'wf_a', name: long, folderPath: 'a' }), + file({ id: 'wf_b', name: long, folderPath: 'b' }), + ]) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + + await expect( + fetchWorkspaceResourceRef(client, 'ws-1', ref({ title: long })) + ).resolves.toBeNull() + for (const [contract, input] of apiClientRequestMockFns.mockRequestJson.mock.calls) { + if (contract !== searchWorkspaceFilesContract) continue + expect((input as { query: { q: string } }).query.q.length).toBeLessThanOrEqual(200) + } + }) + it('asks for each candidate once, as a reference, which already answers an exact id', async () => { serveWorkspace([file({ id: 'wf_a', name: 'notes.md' })]) const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts index 9521c7ddcd2..1a21d462826 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/resolve-resource-ref.ts @@ -81,6 +81,17 @@ export function resolveWorkspaceResourceRef( } } +/** + * The longest leading part of `name` a search query can carry, never splitting a surrogate pair: + * a lone surrogate would match nothing and hide the name's duplicates. + */ +function searchablePrefix(name: string): string { + if (name.length <= V2_SEARCH_MAX_LENGTH) return name + const cut = name.slice(0, V2_SEARCH_MAX_LENGTH) + const last = cut.charCodeAt(cut.length - 1) + return (last >= 0xd800 && last <= 0xdbff ? cut.slice(0, -1) : cut).trimEnd() +} + /** The most files a chip's name candidate is checked against for ambiguity. */ const NAME_CANDIDATE_LIMIT = 50 @@ -100,15 +111,17 @@ function fileRefCandidates(ref: WorkspaceResourceRef): string[] { * {@link resolveWorkspaceResourceRef}'s rules: each candidate as a reference (an exact id * resolves first, then an exact VFS path), and — for a candidate that could be a bare name — * every file whose name contains it, so a name shared by two files is still seen as ambiguous. + * A name longer than a search query may be is searched by its leading part, which every file of + * that exact name contains and ranks first. */ function fileRefReads(queryClient: QueryClient, workspaceId: string, ref: WorkspaceResourceRef) { return fileRefCandidates(ref).flatMap((candidate) => [ getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, 'reference', candidate), - ...(candidate.includes('/') || candidate.length > V2_SEARCH_MAX_LENGTH + ...(candidate.includes('/') ? [] : [ getWorkspaceFileSearchQueryOptions(workspaceId, { - q: candidate, + q: searchablePrefix(candidate), limit: NAME_CANDIDATE_LIMIT, }), ]), diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx index c1de58eb183..031f5b6675b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/rail-resource-flyout/rail-resource-flyout.tsx @@ -17,6 +17,7 @@ import { useParams } from 'next/navigation' import { buildFlyoutEntries, FOLDERED_RESOURCE_HEADERS, + flyoutLevelFolders, folderedResourceListHref, } from '@/app/workspace/[workspaceId]/components/folders' import { CollapsedResourceFlyout } from '@/app/workspace/[workspaceId]/w/components/sidebar/components/collapsed-sidebar-menu' @@ -127,9 +128,7 @@ function FilesFlyoutLevel({ const entries = useMemo(() => { const files = distinctFiles([pinnedFiles, fileList.data?.files]) return buildFlyoutEntries({ - folders: folders - .filter((folder) => (folder.parentId ?? null) === folderId) - .map((folder) => ({ ...folder, parentId: null })), + folders: flyoutLevelFolders(folders, folderId), items: files.map((file) => ({ ...file, folderId: null })), pinnedFolderIds, pinnedItemIds: pinnedFileIds, diff --git a/apps/sim/hooks/queries/pinned-workspace-files.ts b/apps/sim/hooks/queries/pinned-workspace-files.ts index 4f8cf6f35fe..34eb94176f1 100644 --- a/apps/sim/hooks/queries/pinned-workspace-files.ts +++ b/apps/sim/hooks/queries/pinned-workspace-files.ts @@ -5,7 +5,9 @@ import { useWorkspaceFileLookups } from '@/hooks/queries/workspace-files' /** * The pinned files in one folder (`null` for the root), looked up by id so a folder's pins can - * lead it before any page of its list has reached them. In pin order. + * lead it before any page of its list has reached them. In no particular order: a list orders + * its pinned rows among themselves by its active sort, as the Files page always has + * (`mergeFileListEntries`, `buildFlyoutEntries`). */ export function usePinnedWorkspaceFiles( workspaceId: string, diff --git a/apps/sim/hooks/queries/workflow-search-replace.test.ts b/apps/sim/hooks/queries/workflow-search-replace.test.ts index ca29129fe38..d52b1da3ea7 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.test.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.test.ts @@ -85,4 +85,13 @@ describe('workflowSearchFileLookups', () => { ], }) }) + + it('leaves a serve path with malformed percent-encoding unresolved instead of throwing', () => { + expect(workflowSearchFileLookups(['/api/files/serve/workspace%2Fws-1%2F%E0%A4%A.png'])).toEqual( + { + references: [], + keyValues: [], + } + ) + }) }) diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index 57da58d5318..87d2a486a77 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -30,7 +30,7 @@ import { type SelectorKey, } from '@/lib/selectors/manifest' import type { SelectorOption, SelectorScope } from '@/lib/selectors/types' -import { extractStorageKey } from '@/lib/uploads/utils/file' +import { tryExtractStorageKey } from '@/lib/uploads/utils/file' import { getWorkflowSearchMatchResourceGroupKey } from '@/lib/workflows/search-replace/resources' import type { WorkflowSearchMatch, @@ -357,7 +357,9 @@ export function workflowSearchFileLookups(rawValues: readonly string[]): { const references = new Set<string>() const keyValues = new Set<string>() for (const rawValue of rawValues) { - const key = extractStorageKey(rawValue) + const key = tryExtractStorageKey(rawValue) + // A serve path with malformed percent-encoding names nothing a lookup can find. + if (key === undefined) continue if (key !== rawValue) { keyValues.add(key) continue @@ -392,7 +394,7 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor () => fileMatches.map((match) => { const file = - byKey.byValue.get(extractStorageKey(match.rawValue)) ?? + byKey.byValue.get(tryExtractStorageKey(match.rawValue) ?? '') ?? byReference.byValue.get(match.rawValue) return { data: settled diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index e41beb7c93a..94d1c9dc5d3 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -22,6 +22,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ApiClientError } from '@/lib/api/client/errors' import { listWorkspaceFilesContract, + MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS, readWorkspaceFileContract, renameWorkspaceFileContract, searchWorkspaceFilesContract, @@ -46,6 +47,7 @@ import { useWorkspaceFileSearch, useWorkspaceImageDimensionsAdapter, type WorkspaceFileContentResult, + workspaceFileRecordState, } from '@/hooks/queries/workspace-files' const mockRequestJson = apiClientRequestMockFns.mockRequestJson @@ -796,6 +798,38 @@ describe('useWorkspaceFileFolderSizes', () => { }) }) +describe('useWorkspaceFileFolderSizes across many parents', () => { + it('splits parents past the per-request cap into several requests and merges them', async () => { + const queries: string[][] = [] + mockRequestJson.mockImplementation(async (_contract, input) => { + const { parentIds } = (input as { query: { parentIds: string[] } }).query + queries.push(parentIds) + return { + success: true, + folders: parentIds.map((parentId) => ({ + folderId: `child-of-${parentId}`, + fileCount: 1, + sizeBytes: 1, + })), + } + }) + const parents = Array.from( + { length: MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS + 1 }, + (_, index) => `p${String(index).padStart(3, '0')}` + ) + const hook = mountHook(() => useWorkspaceFileFolderSizes('ws-1', parents)) + try { + await vi.waitFor(() => expect(hook.result().size).toBe(parents.length)) + expect(queries.map((chunk) => chunk.length)).toEqual([ + MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS, + 1, + ]) + } finally { + hook.unmount() + } + }) +}) + describe('useWorkspaceFilePickerOptions', () => { it('offers the most recent files untyped and the server matches once a query settles', async () => { mockRequestJson.mockImplementation(async (contract) => @@ -814,12 +848,35 @@ describe('useWorkspaceFilePickerOptions', () => { /** Typed but not settled: still pending, so an empty menu is not read as "no match". */ expect(hook.result().isPending).toBe(true) await vi.waitFor(() => expect(hook.result().files?.map((file) => file.id)).toEqual(['match'])) - expect(hook.result().settledQuery).toBe('mat') + expect(hook.result().matchedQuery).toBe('mat') expect(hook.result().isPending).toBe(false) } finally { hook.unmount() } }) + + it('claims no matched query while the last query’s matches stand in for the next', async () => { + const next = createDeferred<unknown>() + mockRequestJson.mockImplementation(async (contract, input) => { + if (contract !== searchWorkspaceFilesContract) { + return { success: true, files: [], nextCursor: null } + } + const { q } = (input as { query: { q: string } }).query + return q === 'mat' ? { success: true, files: [fileRecord('match')] } : next.promise + }) + let query = 'mat' + const hook = mountHook(() => useWorkspaceFilePickerOptions('ws-1', query)) + try { + await vi.waitFor(() => expect(hook.result().matchedQuery).toBe('mat')) + query = 'other' + hook.rerender() + await vi.waitFor(() => expect(mockRequestJson).toHaveBeenCalledTimes(3)) + expect(hook.result().files?.map((file) => file.id)).toEqual(['match']) + expect(hook.result().matchedQuery).toBe('') + } finally { + hook.unmount() + } + }) }) describe('useWorkspaceImageDimensionsAdapter', () => { @@ -876,3 +933,37 @@ describe('useWorkspaceFileSearch placeholder', () => { } }) }) + +describe('useWorkspaceImageDimensionsAdapter encoded ids', () => { + it('looks an image up by its stored id when the document percent-encodes it', async () => { + const image = fileRecord('wf_img', { width: 40, height: 30 }) + const bodies: unknown[] = [] + mockRequestJson.mockImplementation(async (_contract, input) => { + bodies.push((input as { body: unknown }).body) + return { success: true, files: [image], references: [] } + }) + const src = '/api/files/view/wf%5Fimg' + const hook = mountHook(() => useWorkspaceImageDimensionsAdapter('ws-1')) + try { + expect(hook.result().getImageDimensions(src)).toBeNull() + await vi.waitFor(() => + expect(hook.result().getImageDimensions(src)).toEqual({ width: 40, height: 30 }) + ) + expect(bodies).toEqual([expect.objectContaining({ ids: ['wf_img'] })]) + } finally { + hook.unmount() + } + }) +}) + +describe('workspaceFileRecordState', () => { + it('reads only a 404 (a null record) as missing, and any other failure as failed', () => { + expect(workspaceFileRecordState({ data: null, isError: false })).toBe('missing') + expect(workspaceFileRecordState({ data: undefined, isError: true })).toBe('failed') + }) + + it('keeps a record in hand found though its refetch failed, and waits on a first read', () => { + expect(workspaceFileRecordState({ data: fileRecord('a'), isError: true })).toBe('found') + expect(workspaceFileRecordState({ data: undefined, isError: false })).toBe('loading') + }) +}) diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 0bd03986890..8ecef8b49b6 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -26,6 +26,7 @@ import { deleteWorkspaceFileContract, listWorkspaceFilesContract, lookupWorkspaceFilesContract, + MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS, readWorkspaceFileContract, renameWorkspaceFileContract, restoreWorkspaceFileContract, @@ -39,8 +40,8 @@ import { uploadWorkspaceFileSession } from '@/lib/uploads/client/session-upload' import { FileTextSizeError, readFileText } from '@/lib/uploads/client/text-content' import type { UploadProgressEvent } from '@/lib/uploads/client/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import { extractEmbeddedFileRef } from '@/lib/uploads/utils/embedded-image-ref' -import { extractStorageKey } from '@/lib/uploads/utils/file' +import { extractEmbeddedFileRef, storedFileId } from '@/lib/uploads/utils/embedded-image-ref' +import { tryExtractStorageKey } from '@/lib/uploads/utils/file' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { WorkspaceFileSelection } from '@/lib/workspace-files/selection' import type { UserFile } from '@/executor/types' @@ -229,18 +230,32 @@ export interface WorkspaceFileFolderStat { sizeBytes: number } +/** + * The rollups for `parentIds`, one request per {@link MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS} + * of them (a search can span more parents than one request may name), merged. + */ async function fetchWorkspaceFileFolderStats( workspaceId: string, parentIds: readonly string[], signal?: AbortSignal ): Promise<Map<string, WorkspaceFileFolderStat>> { - const data = await requestJson(workspaceFileFolderStatsContract, { - params: { id: workspaceId }, - query: { parentIds: [...parentIds] }, - signal, - }) + const chunks: string[][] = [] + for (let start = 0; start < parentIds.length; start += MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS) { + chunks.push(parentIds.slice(start, start + MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS)) + } + const responses = await Promise.all( + chunks.map((chunk) => + requestJson(workspaceFileFolderStatsContract, { + params: { id: workspaceId }, + query: { parentIds: chunk }, + signal, + }) + ) + ) return new Map( - data.folders.map(({ folderId, fileCount, sizeBytes }) => [folderId, { fileCount, sizeBytes }]) + responses.flatMap((data) => + data.folders.map(({ folderId, fileCount, sizeBytes }) => [folderId, { fileCount, sizeBytes }]) + ) ) } @@ -404,8 +419,11 @@ export function useWorkspaceFileSelectionCandidates( for (const selection of selections) { if (selection.id) ids.add(selection.id) else if (selection.key) keys.add(selection.key) - else if (selection.path) keys.add(extractStorageKey(selection.path)) - else if (selection.name.trim()) names.add(selection.name.trim()) + else if (selection.path) { + // A persisted path with malformed percent-encoding names no file; it stays unresolved. + const key = tryExtractStorageKey(selection.path) + if (key) keys.add(key) + } else if (selection.name.trim()) names.add(selection.name.trim()) } return { ids: [...ids], keys: [...keys], names: [...names] } }, [selections]) @@ -475,8 +493,12 @@ function recentPickerFiles(limit: number): WorkspaceFileListParams { export interface WorkspaceFilePickerOptions { /** The files to offer: the best matches for the settled query, else the most recent. */ files: WorkspaceFileRecord[] | undefined - /** The query `files` answers, trimmed; empty while browsing recent files. */ - settledQuery: string + /** + * The query `files` are the server's matches for, trimmed — ranked by name, word, fuzzy, and + * folder path, so a caller keeps them rather than re-filtering by substring. Empty while + * `files` are the recent files, or a previous query's matches held while the next loads. + */ + matchedQuery: string /** True until `files` answers what was typed: a menu still empty here is not "no match" yet. */ isPending: boolean } @@ -506,7 +528,7 @@ export function useWorkspaceFilePickerOptions( ) return { files: settledQuery ? matching.data : recent.data?.pages[0]?.files, - settledQuery, + matchedQuery: settledQuery && matching.data && !matching.isPlaceholderData ? settledQuery : '', isPending: settledQuery ? typed !== settledQuery || matching.isFetching : recent.isPending || typed.length > 0, @@ -571,6 +593,20 @@ export function getWorkspaceFileRecordQueryOptions( } } +/** + * What a {@link useWorkspaceFileRecord} read says about its file: `found` (a record is in hand, + * even if a refetch then failed), `missing` (no live file has the id — a 404 reads as `null`), + * `failed` (any other error, with no record to show), or still `loading`. + */ +export function workspaceFileRecordState(record: { + data: WorkspaceFileRecord | null | undefined + isError: boolean +}): 'loading' | 'found' | 'missing' | 'failed' { + if (record.data) return 'found' + if (record.data === null) return 'missing' + return record.isError ? 'failed' : 'loading' +} + /** * One live file's record by id. `data` is `null` once the file is gone — a 404 is an answer, not * a failure, so a file deleted while open does not keep showing the record it was opened from. @@ -640,8 +676,9 @@ export function useWorkspaceImageDimensionsAdapter( const findRecord = (src: string | undefined): WorkspaceFileRecord | undefined => { const ref = enabled && src ? extractEmbeddedFileRef(src) : null if (!ref) return undefined + // The document's spelling of an id may be percent-encoded; the lookup takes the stored id. const [kind, value] = - 'key' in ref ? (['key', ref.key] as const) : (['id', ref.fileId] as const) + 'key' in ref ? (['key', ref.key] as const) : (['id', storedFileId(ref.fileId)] as const) const lookup = getWorkspaceFileLookupQueryOptions(queryClient, workspaceId, kind, value) const resolved = queryClient.getQueryData<WorkspaceFileRecord | null>(lookup.queryKey) if (resolved !== undefined) return resolved ?? undefined @@ -1190,6 +1227,13 @@ interface RenameFileParams { workspaceId: string fileId: string name: string + /** Leave a failure to the caller instead of toasting it: a rename the caller retries. */ + quiet?: boolean +} + +/** Whether a rename failed because a sibling already has the name (the server's 409). */ +export function isWorkspaceFileNameTaken(error: unknown): boolean { + return isApiClientError(error) && error.status === 409 } /** What a rename rewrites in place: the reads that address the file by identity. */ @@ -1239,9 +1283,9 @@ export function useRenameWorkspaceFile() { RENAME_PATCHED_READS ) }, - onError: (error, _variables, context) => { + onError: (error, variables, context) => { if (context) restoreWorkspaceFileReads(queryClient, context.previous) - toast.error(error.message, { duration: 5000 }) + if (!variables.quiet) toast.error(error.message, { duration: 5000 }) }, onSettled: (_data, error, { workspaceId }) => error diff --git a/apps/sim/hooks/use-load-more-sentinel.ts b/apps/sim/hooks/use-load-more-sentinel.ts index 9436bbc94b1..e348a183bbd 100644 --- a/apps/sim/hooks/use-load-more-sentinel.ts +++ b/apps/sim/hooks/use-load-more-sentinel.ts @@ -7,26 +7,41 @@ interface UseLoadMoreSentinelOptions { onLoadMore: (() => void) | undefined /** Whether reaching the sentinel should load more right now. */ enabled: boolean + /** + * Ask only once the sentinel comes back into view, not because it already is — after a failed + * load, so a rebuilt observer does not retry a failing request on its own, in a loop. + */ + waitForReentry?: boolean } /** * A callback ref for the element at the end of a paged list: `onLoadMore` runs when it scrolls * within 200px of view, while `enabled`. Pass `enabled: false` during a fetch, so the - * observer is rebuilt when the fetch ends and fires again if the sentinel is still in view. + * observer is rebuilt when the fetch ends and fires again if the sentinel is still in view — + * unless `waitForReentry`, when it fires only after the sentinel leaves view and returns. */ -export function useLoadMoreSentinel({ onLoadMore, enabled }: UseLoadMoreSentinelOptions) { +export function useLoadMoreSentinel({ + onLoadMore, + enabled, + waitForReentry = false, +}: UseLoadMoreSentinelOptions) { return useCallback( (sentinel: HTMLDivElement | null) => { if (!sentinel || !onLoadMore || !enabled) return + let armed = !waitForReentry const observer = new IntersectionObserver( ([entry]) => { - if (entry?.isIntersecting) onLoadMore() + if (!entry?.isIntersecting) { + armed = true + return + } + if (armed) onLoadMore() }, { rootMargin: '200px' } ) observer.observe(sentinel) return () => observer.disconnect() }, - [onLoadMore, enabled] + [onLoadMore, enabled, waitForReentry] ) } diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index ee1be2dca98..0be51427bbc 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -609,26 +609,29 @@ export type RetentionOverrides = z.output<typeof retentionOverridesSchema> * A list-valued query param, sent repeated (`a=x&a=y`, how `requestJson` serializes an array) or * comma-separated (`a=x,y`): entries trimmed, empty ones dropped. `canonical` also de-duplicates * and sorts them, for a filter the query treats as a set — so `x,y` and `y,x` are one filter, for - * the query and for a cursor bound to it. + * the query and for a cursor bound to it — and reads an empty set (`a=`) as no filter at all. */ export function queryListSchema<T extends z.ZodType<string>>( member: T, options: { max?: number; canonical?: boolean } = {} ) { - const entries = z.array(member) - return z.preprocess( - (raw) => { - if (typeof raw !== 'string' && !Array.isArray(raw)) return raw - const values = (Array.isArray(raw) ? raw : [raw]) - .flatMap((entry) => (typeof entry === 'string' ? entry.split(',') : [entry])) - .map((entry) => (typeof entry === 'string' ? entry.trim() : entry)) - .filter((entry) => entry !== '') - return options.canonical ? [...new Set(values)].sort() : values - }, + const entries = options.max === undefined - ? entries - : entries.max(options.max, `Cannot list more than ${options.max} values`) - ) + ? z.array(member) + : z.array(member).max(options.max, `Cannot list more than ${options.max} values`) + const split = (raw: unknown) => { + if (typeof raw !== 'string' && !Array.isArray(raw)) return raw + return (Array.isArray(raw) ? raw : [raw]) + .flatMap((entry) => (typeof entry === 'string' ? entry.split(',') : [entry])) + .map((entry) => (typeof entry === 'string' ? entry.trim() : entry)) + .filter((entry) => entry !== '') + } + if (!options.canonical) return z.preprocess(split, entries) + return z.preprocess((raw) => { + const values = split(raw) + if (!Array.isArray(values)) return values + return values.length === 0 ? undefined : [...new Set(values)].sort() + }, entries.optional()) } export const booleanQueryFlagSchema = z.preprocess( diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 97903d57f3b..40d2da5a727 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -350,6 +350,9 @@ export const lookupWorkspaceFilesContract = defineRouteContract({ response: { mode: 'json', schema: lookupWorkspaceFilesResponseSchema }, }) +/** The most parents one folder-stats request may name. */ +export const MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS = 100 + const workspaceFileFolderStatsQuerySchema = z.object({ /** The folder whose child folders to report: `root` (default) or a folder id. */ parentId: workspaceFileFolderFilterSchema.prefault('root'), @@ -358,7 +361,7 @@ const workspaceFileFolderStatsQuerySchema = z.object({ * whose folders sit under different parents; replaces `parentId` when sent. */ parentIds: queryListSchema(z.union([z.literal('root'), folderIdSchema]), { - max: 100, + max: MAX_WORKSPACE_FILE_FOLDER_STATS_PARENTS, canonical: true, }).optional(), }) diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index be43f05152d..a40309452fd 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -686,6 +686,18 @@ export function extractStorageKey(filePath: string): string { return pathWithoutQuery } +/** + * {@link extractStorageKey} for a path read back from storage or a document, which may be + * malformed: `undefined` when its percent-encoding cannot be decoded, rather than throwing. + */ +export function tryExtractStorageKey(filePath: string): string | undefined { + try { + return extractStorageKey(filePath) + } catch { + return undefined + } +} + /** * Whether a URL targets the internal file-serve endpoint (`/api/files/serve/`). * diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index d8ea7d4ba40..995e14b1825 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -13,12 +13,7 @@ import { inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import type { CursorKey } from '@/lib/api/list-query' import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager' -import { - EXTENSION_TO_MIME, - getFileExtension, - MIME_TYPE_MAPPING, - resolveEffectiveMimeType, -} from '@/lib/uploads/utils/file' +import { EXTENSION_TO_MIME, getFileExtension, MIME_TYPE_MAPPING } from '@/lib/uploads/utils/file' import { isSupportedExtension } from '@/lib/uploads/utils/validation' import { getWorkspaceFileFolderStats, @@ -26,7 +21,7 @@ import { lookupWorkspaceFiles, searchWorkspaceFiles, } from '@/lib/workspace-files/application/list-workspace-files' -import { fileBrowserTypeLabel } from '@/lib/workspace-files/browser' +import { fileBrowserMimeType, fileBrowserTypeLabel } from '@/lib/workspace-files/browser' import { workspaceFileKindCondition, workspaceFileTypeLabelExpression, @@ -703,6 +698,10 @@ describe('bounded workspace file reads in PostgreSQL', () => { })) ), { name: 'noext', contentType: 'application/x-custom' }, + { name: 'file.bin', contentType: 'Image/PNG; charset=binary' }, + { name: 'file.txt', contentType: ' Text/Markdown ; charset=utf-8' }, + { name: 'file.mp3', contentType: 'AUDIO/MPEG' }, + { name: 'file.dat', contentType: 'Application/Octet-Stream; x=1' }, { name: 'file.foo', contentType: 'application/octet-stream' }, { name: 'file.foo', contentType: '' }, ] @@ -727,7 +726,7 @@ describe('bounded workspace file reads in PostgreSQL', () => { for (const row of rows) { const fixture = ids.get(String(row.id)) if (!fixture) throw new Error('Unexpected row') - const mime = resolveEffectiveMimeType(fixture.contentType, fixture.name) + const mime = fileBrowserMimeType(fixture.contentType, fixture.name) expect({ id: row.id, label: row.label, ...pick(row) }).toEqual({ id: row.id, label: fileBrowserTypeLabel(fixture.contentType, fixture.name), diff --git a/apps/sim/lib/workspace-files/browser-query.ts b/apps/sim/lib/workspace-files/browser-query.ts index da74ec5aef7..fd440c332b7 100644 --- a/apps/sim/lib/workspace-files/browser-query.ts +++ b/apps/sim/lib/workspace-files/browser-query.ts @@ -24,6 +24,7 @@ import type { FileBrowserSort, } from '@/lib/workspace-files/browser' import { + declaredMimeSql, effectiveMimeSql, fileExtensionSql, fileKindSql, @@ -123,7 +124,7 @@ export async function queryFileBrowserItems( join owned_files files on files.parent_id = tree.id group by tree.root_id ), effective_files as ( - select *, ${effectiveMimeSql(sql`lower(btrim(split_part(content_type, ';', 1)))`, sql`extension`)} as mime + select *, ${effectiveMimeSql(declaredMimeSql(sql`content_type`), sql`extension`)} as mime from owned_files ), items as ( diff --git a/apps/sim/lib/workspace-files/browser.test.ts b/apps/sim/lib/workspace-files/browser.test.ts new file mode 100644 index 00000000000..90b835a958d --- /dev/null +++ b/apps/sim/lib/workspace-files/browser.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { fileBrowserMimeType, fileBrowserTypeLabel } from '@/lib/workspace-files/browser' + +describe('fileBrowserTypeLabel', () => { + it('reads a stored type with parameters or mixed case as its bare media type', () => { + expect(fileBrowserMimeType('Image/PNG; charset=binary', 'file.bin')).toBe('image/png') + expect(fileBrowserTypeLabel(' Text/Markdown ; charset=utf-8', 'notes.txt')).toBe( + fileBrowserTypeLabel('text/markdown', 'notes.txt') + ) + }) + + it('falls back to the extension when the stored type is generic, however it is spelled', () => { + expect(fileBrowserMimeType('Application/Octet-Stream; x=1', 'song.mp3')).toBe( + fileBrowserMimeType('', 'song.mp3') + ) + }) +}) diff --git a/apps/sim/lib/workspace-files/browser.ts b/apps/sim/lib/workspace-files/browser.ts index 21646d01abf..3b40a6e4239 100644 --- a/apps/sim/lib/workspace-files/browser.ts +++ b/apps/sim/lib/workspace-files/browser.ts @@ -1,4 +1,5 @@ import { getFileExtension, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' +import { normalizeMimeType } from '@/lib/uploads/utils/mime' export const FILE_BROWSER_SORTS = ['name', 'size', 'type', 'created', 'owner', 'updated'] as const export type FileBrowserSort = (typeof FILE_BROWSER_SORTS)[number] @@ -65,13 +66,22 @@ export interface FileBrowserFilters { creatorIds?: readonly string[] } +/** + * The MIME type the file browsers classify a file by: its stored type read as + * `normalizeMimeType` reads it (lowercased, parameters dropped), else its extension's — the + * rule `declaredMimeSql` and `effectiveMimeSql` spell for SQL. + */ +export function fileBrowserMimeType(storedType: string, filename: string): string { + return resolveEffectiveMimeType(normalizeMimeType(storedType), filename) +} + /** * A file's Type column label: a specific label for its effective MIME type, else its media * family (Audio, Video, Image), else its extension uppercased, else the stored type. The SQL * twin is `fileTypeLabelSql`, which the browsers' Type sorts order by. */ export function fileBrowserTypeLabel(storedType: string, filename: string): string { - const mimeType = resolveEffectiveMimeType(storedType, filename) + const mimeType = fileBrowserMimeType(storedType, filename) const label = FILE_BROWSER_MIME_LABELS[mimeType] if (label) return label if (mimeType.startsWith('audio/')) return 'Audio' diff --git a/apps/sim/lib/workspace-files/file-type-sql.ts b/apps/sim/lib/workspace-files/file-type-sql.ts index 20b4d01f7d8..6f8f4315bd1 100644 --- a/apps/sim/lib/workspace-files/file-type-sql.ts +++ b/apps/sim/lib/workspace-files/file-type-sql.ts @@ -47,10 +47,14 @@ export function fileExtensionSql(name: SQLWrapper): SQL<string> { return sql<string>`(case when strpos(${name}, '.') > 0 then lower(regexp_replace(${name}, '^.*[.]', '')) else '' end)` } +/** The stored type as `normalizeMimeType` reads it: lowercased, parameters dropped, trimmed. */ +export function declaredMimeSql(contentType: SQLWrapper): SQL<string> { + return sql<string>`lower(btrim(split_part(coalesce(${contentType}, ''), ';', 1)))` +} + /** - * The effective MIME type: `declared` when it names a format, else the extension's own. The - * caller decides how `declared` is normalized — the Files page trims the stored type, as - * `resolveEffectiveMimeType` does; the Project browser also lowercases it and drops parameters. + * The effective MIME type: `declared` (from {@link declaredMimeSql}) when it names a format, + * else the extension's own — `resolveEffectiveMimeType` over `normalizeMimeType`'s reading. */ export function effectiveMimeSql(declared: SQL, extension: SQL): SQL<string> { return sql<string>`(case when ${declared} not in ('', ${list([...GENERIC_MIME_TYPES])}) then ${declared} diff --git a/apps/sim/lib/workspace-files/query-scope.ts b/apps/sim/lib/workspace-files/query-scope.ts index 9233ad7974e..e8109468dfd 100644 --- a/apps/sim/lib/workspace-files/query-scope.ts +++ b/apps/sim/lib/workspace-files/query-scope.ts @@ -4,6 +4,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import type { FolderIdScope } from '@/lib/folders/scope' import type { FileBrowserSize, FileBrowserType } from '@/lib/workspace-files/browser' import { + declaredMimeSql, effectiveMimeSql, fileExtensionSql, fileKindSql, @@ -78,9 +79,9 @@ export function workspaceFileFolderScopeCondition( const workspaceFileExtension = fileExtensionSql(workspaceFiles.originalName) -/** The MIME type `resolveEffectiveMimeType` gives the file: the trimmed stored type, else the extension's. */ +/** The file's effective MIME type: its normalized stored type, else its extension's. */ const workspaceFileEffectiveMime = effectiveMimeSql( - sql`btrim(${workspaceFiles.contentType})`, + declaredMimeSql(workspaceFiles.contentType), workspaceFileExtension ) diff --git a/packages/testing/src/mocks/file-utils.mock.ts b/packages/testing/src/mocks/file-utils.mock.ts index 46b634c04c1..86e5c298109 100644 --- a/packages/testing/src/mocks/file-utils.mock.ts +++ b/packages/testing/src/mocks/file-utils.mock.ts @@ -374,6 +374,14 @@ function extractStorageKey(filePath: string): string { return pathWithoutQuery } +function tryExtractStorageKey(filePath: string): string | undefined { + try { + return extractStorageKey(filePath) + } catch { + return undefined + } +} + function isInternalFileUrl(fileUrl: string): boolean { if (typeof fileUrl !== 'string') return false let path = fileUrl @@ -570,6 +578,7 @@ export const fileUtilsMockFns = { } ), mockExtractStorageKey: vi.fn(extractStorageKey), + mockTryExtractStorageKey: vi.fn(tryExtractStorageKey), mockIsInternalFileUrl: vi.fn(isInternalFileUrl), mockInferContextFromKey: vi.fn(inferContextFromKey), mockTryInferContextFromKey: vi.fn(tryInferContextFromKey), @@ -732,6 +741,7 @@ export const fileUtilsMock = { formatFileSize: fileUtilsMockFns.mockFormatFileSize, validateKnowledgeBaseFile: fileUtilsMockFns.mockValidateKnowledgeBaseFile, extractStorageKey: fileUtilsMockFns.mockExtractStorageKey, + tryExtractStorageKey: fileUtilsMockFns.mockTryExtractStorageKey, isInternalFileUrl: fileUtilsMockFns.mockIsInternalFileUrl, inferContextFromKey: fileUtilsMockFns.mockInferContextFromKey, tryInferContextFromKey: fileUtilsMockFns.mockTryInferContextFromKey, From 6cd7537d967404839f78e32b1e063b83dd0cf07c Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 22:32:15 -0700 Subject: [PATCH 14/15] fix(files): address the paged file reads' second review round - Only a serve URL is read as a storage key with a query: a stored name or path holding `?` or `#` is looked up by reference (and by key when it has a `/`), in search-replace and picker selections alike (`storageKeyOfStoredValue`). - The `@` menu reads as loading, not "No results", while the file search behind its query is settling: the mention store carries the picker's pending state alongside the items. - The Audio and Video filters match any `audio/*` or `video/*` type, as the Type label does, in SQL and JS for both browsers (`fileKindSql`, `fileBrowserKinds`); the parity test covers types the MIME table does not list. --- .../mention/mention-list.tsx | 16 +++++++++++--- .../mention/mention-matching.test.ts | 17 +++++++++++++- .../mention/mention-store.ts | 22 +++++++++++++------ .../mention/use-editor-mentions.ts | 6 ++--- .../mention/use-markdown-mentions.ts | 18 +++++++++++---- .../queries/workflow-search-replace.test.ts | 7 ++++++ .../hooks/queries/workflow-search-replace.ts | 19 ++++++++-------- apps/sim/hooks/queries/workspace-files.ts | 4 ++-- apps/sim/lib/uploads/utils/file.test.ts | 21 ++++++++++++++++++ apps/sim/lib/uploads/utils/file.ts | 10 +++++---- .../file-listing.integration.ts | 18 ++++++++------- apps/sim/lib/workspace-files/browser.test.ts | 18 ++++++++++++++- apps/sim/lib/workspace-files/browser.ts | 16 ++++++++++++++ apps/sim/lib/workspace-files/file-type-sql.ts | 10 ++------- packages/testing/src/mocks/file-utils.mock.ts | 9 ++++---- 15 files changed, 156 insertions(+), 55 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx index ff73e3dae8b..bcada83cea1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list.tsx @@ -5,7 +5,7 @@ import { type SuggestionKeyDownHandler, useSuggestionKeyboard, } from '../menus/use-suggestion-keyboard' -import type { MentionStore } from './mention-store' +import type { MentionMenuData, MentionStore } from './mention-store' import type { MentionItem } from './types' /** @@ -16,6 +16,15 @@ export function mentionItemMatches(item: MentionItem, query: string): boolean { return item.matchedQuery?.toLowerCase() === query || item.label.toLowerCase().includes(query) } +/** + * What the menu says with nothing to list: still loading while it has no data yet or its file + * search is settling — the other families alone can fill it, so an empty match is not yet + * "no results" until the files answering the query have landed. + */ +export function mentionEmptyLabel(menu: MentionMenuData): string { + return menu.items.length === 0 || menu.filesPending ? 'Loading…' : 'No results' +} + export type MentionListHandle = SuggestionKeyDownHandler interface MentionListProps { @@ -58,7 +67,8 @@ export const MentionList = forwardRef<MentionListHandle, MentionListProps>(funct { query, command, store, editor }, ref ) { - const rawItems = useSyncExternalStore(store.subscribe, store.getSnapshot, store.getSnapshot) + const menu = useSyncExternalStore(store.subscribe, store.getSnapshot, store.getSnapshot) + const rawItems = menu.items const containerRef = useRef<HTMLDivElement>(null) /** @@ -109,7 +119,7 @@ export const MentionList = forwardRef<MentionListHandle, MentionListProps>(funct command={command} ariaLabel='Mentions' idPrefix='mention' - emptyLabel={rawItems.length === 0 ? 'Loading…' : 'No results'} + emptyLabel={mentionEmptyLabel(menu)} itemKey={(item) => `${item.kind}:${item.id}`} renderItem={(item) => { const Icon = item.icon diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts index a23b164e7b9..a5f412489d2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-matching.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest' -import { mentionItemMatches } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list' +import { + mentionEmptyLabel, + mentionItemMatches, +} from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-list' describe('mentionItemMatches', () => { const file = { kind: 'file' as const, id: 'wf_a', label: 'Quarterly-Report.md', group: 'Files' } @@ -13,3 +16,15 @@ describe('mentionItemMatches', () => { expect(mentionItemMatches(file, 'report')).toBe(true) }) }) + +describe('mentionEmptyLabel', () => { + const file = { kind: 'file' as const, id: 'wf_a', label: 'a.md', group: 'Files' } + + it('reads as loading, not "No results", while the file search behind a query is settling', () => { + expect(mentionEmptyLabel({ items: [file], filesPending: true })).toBe('Loading…') + }) + + it('says "No results" once the files for the query have landed', () => { + expect(mentionEmptyLabel({ items: [file], filesPending: false })).toBe('No results') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-store.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-store.ts index f3ac59d24b3..1eb63fac9cb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-store.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-store.ts @@ -3,29 +3,37 @@ import type { MentionItem } from './types' /** * A tiny external store bridging React Query data (host component) into the `@` menu list, which is * rendered by TipTap's `ReactRenderer` as a detached root with no access to the app's React context - * providers. The host pushes the latest items via {@link MentionStore.set}; the list subscribes with + * providers. The host pushes the latest items (and whether the file search is settling) via {@link MentionStore.set}; the list subscribes with * `useSyncExternalStore` and re-renders when async data lands — so the menu populates live even if it * was opened before the data finished loading. One store instance lives per editor (in extension * storage). */ export interface MentionStore { - getSnapshot: () => MentionItem[] + getSnapshot: () => MentionMenuData subscribe: (listener: () => void) => () => void - set: (items: MentionItem[]) => void + set: (data: MentionMenuData) => void } +/** The menu's items, and whether the file search behind them is still settling. */ +export interface MentionMenuData { + items: MentionItem[] + filesPending: boolean +} + +const EMPTY_MENU: MentionMenuData = { items: [], filesPending: false } + export function createMentionStore(): MentionStore { - let items: MentionItem[] = [] + let data = EMPTY_MENU const listeners = new Set<() => void>() return { - getSnapshot: () => items, + getSnapshot: () => data, subscribe: (listener) => { listeners.add(listener) return () => listeners.delete(listener) }, set: (next) => { - if (next === items) return - items = next + if (next === data) return + data = next for (const listener of listeners) listener() }, } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts index e2b37ff309c..888e8e64faf 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-editor-mentions.ts @@ -22,7 +22,7 @@ export function useEditorMentions( ): void { const [active, setActive] = useState(false) const [query, setQuery] = useState('') - const items = useMarkdownMentions(workspaceId, { enabled: active, fileQuery: query }) + const menu = useMarkdownMentions(workspaceId, { enabled: active, fileQuery: query }) const navigable = options?.navigable ?? false const disableTagging = options?.disableTagging ?? false @@ -40,6 +40,6 @@ export function useEditorMentions( }, [editor, workspaceId, navigable, disableTagging]) useEffect(() => { - editor?.storage.mentionMenu.store.set(items) - }, [editor, items]) + editor?.storage.mentionMenu.store.set(menu) + }, [editor, menu]) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts index bb9e6f7ecbd..1542e8672b8 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/use-markdown-mentions.ts @@ -1,4 +1,5 @@ import { useMemo } from 'react' +import type { MentionMenuData } from '@/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-store' import { listIntegrations } from '@/blocks/integration-matcher' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useSkills } from '@/hooks/queries/skills' @@ -18,10 +19,12 @@ import type { MentionItem } from './types' * Files are the one family not listed whole: the menu offers the most recently updated, and once * `fileQuery` (the text after the `@`) settles, the files whose names match it anywhere. */ +const NO_MENTION_ITEMS: MentionItem[] = [] + export function useMarkdownMentions( workspaceId: string | undefined, options: { enabled: boolean; fileQuery?: string } -): MentionItem[] { +): MentionMenuData { const active = options.enabled && Boolean(workspaceId) // When inactive, `ws` is undefined and `wsStr` is '' so every query stays disabled until the first // `@`: the hooks that expose an `enabled` option get it explicitly; the rest (which take no options) @@ -29,7 +32,11 @@ export function useMarkdownMentions( const ws = active ? workspaceId : undefined const wsStr = ws ?? '' - const { files, matchedQuery } = useWorkspaceFilePickerOptions(wsStr, options.fileQuery ?? '', { + const { + files, + matchedQuery, + isPending: filesPending, + } = useWorkspaceFilePickerOptions(wsStr, options.fileQuery ?? '', { enabled: active, }) const folders = useWorkspaceFileFolders(wsStr, 'active') @@ -50,8 +57,8 @@ export function useMarkdownMentions( })) }, [active]) - return useMemo(() => { - if (!active) return [] + const items = useMemo(() => { + if (!active) return NO_MENTION_ITEMS const items: MentionItem[] = [] for (const file of files ?? []) @@ -117,4 +124,7 @@ export function useMarkdownMentions( skills.data, integrationItems, ]) + /** Files still answering an older query (or none yet) keep the menu from reading as empty. */ + const pending = active && filesPending + return useMemo(() => ({ items, filesPending: pending }), [items, pending]) } diff --git a/apps/sim/hooks/queries/workflow-search-replace.test.ts b/apps/sim/hooks/queries/workflow-search-replace.test.ts index d52b1da3ea7..d134349f20a 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.test.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.test.ts @@ -94,4 +94,11 @@ describe('workflowSearchFileLookups', () => { } ) }) + + it('looks a name or path holding `?` or `#` up by reference, never as a serve URL', () => { + expect(workflowSearchFileLookups(['what?.md', 'docs/q&a #1?.md'])).toEqual({ + references: ['what?.md', 'docs/q&a #1?.md'], + keyValues: ['docs/q&a #1?.md'], + }) + }) }) diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index 87d2a486a77..97ea8e409bd 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -30,7 +30,7 @@ import { type SelectorKey, } from '@/lib/selectors/manifest' import type { SelectorOption, SelectorScope } from '@/lib/selectors/types' -import { tryExtractStorageKey } from '@/lib/uploads/utils/file' +import { isInternalFileUrl, storageKeyOfStoredValue } from '@/lib/uploads/utils/file' import { getWorkflowSearchMatchResourceGroupKey } from '@/lib/workflows/search-replace/resources' import type { WorkflowSearchMatch, @@ -346,9 +346,10 @@ export function useWorkflowSearchTableDetails( } /** - * What to look a stored file value up by. It is an id, a storage key, a serve path, or a name: a - * reference lookup answers an id (its first rule), a path, and a name; a storage key — every one - * has a `/` — and a serve path also need the key lookup, which a plain name or id never would. + * What to look a stored file value up by. It is an id, a storage key, a serve URL, or a name or + * path: a serve URL is looked up by the key it carries (nothing, when its encoding is malformed); + * anything else by reference, which answers an id (its first rule), a path, and a name — and also + * by key when it has a `/`, as every storage key does. Only a serve URL reads `?` as a query. */ export function workflowSearchFileLookups(rawValues: readonly string[]): { references: string[] @@ -357,11 +358,9 @@ export function workflowSearchFileLookups(rawValues: readonly string[]): { const references = new Set<string>() const keyValues = new Set<string>() for (const rawValue of rawValues) { - const key = tryExtractStorageKey(rawValue) - // A serve path with malformed percent-encoding names nothing a lookup can find. - if (key === undefined) continue - if (key !== rawValue) { - keyValues.add(key) + if (isInternalFileUrl(rawValue)) { + const key = storageKeyOfStoredValue(rawValue) + if (key) keyValues.add(key) continue } references.add(rawValue) @@ -394,7 +393,7 @@ export function useWorkflowSearchFileDetails(matches: WorkflowSearchMatch[], wor () => fileMatches.map((match) => { const file = - byKey.byValue.get(tryExtractStorageKey(match.rawValue) ?? '') ?? + byKey.byValue.get(storageKeyOfStoredValue(match.rawValue) ?? '') ?? byReference.byValue.get(match.rawValue) return { data: settled diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 8ecef8b49b6..98f7b1fd8c8 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -41,7 +41,7 @@ import { FileTextSizeError, readFileText } from '@/lib/uploads/client/text-conte import type { UploadProgressEvent } from '@/lib/uploads/client/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef, storedFileId } from '@/lib/uploads/utils/embedded-image-ref' -import { tryExtractStorageKey } from '@/lib/uploads/utils/file' +import { storageKeyOfStoredValue } from '@/lib/uploads/utils/file' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import type { WorkspaceFileSelection } from '@/lib/workspace-files/selection' import type { UserFile } from '@/executor/types' @@ -421,7 +421,7 @@ export function useWorkspaceFileSelectionCandidates( else if (selection.key) keys.add(selection.key) else if (selection.path) { // A persisted path with malformed percent-encoding names no file; it stays unresolved. - const key = tryExtractStorageKey(selection.path) + const key = storageKeyOfStoredValue(selection.path) if (key) keys.add(key) } else if (selection.name.trim()) names.add(selection.name.trim()) } diff --git a/apps/sim/lib/uploads/utils/file.test.ts b/apps/sim/lib/uploads/utils/file.test.ts index 286d067b7f2..d0c220a4f0d 100644 --- a/apps/sim/lib/uploads/utils/file.test.ts +++ b/apps/sim/lib/uploads/utils/file.test.ts @@ -10,6 +10,7 @@ import { resolveEffectiveMimeType, resolveFileType, resolveTrustedFileContext, + storageKeyOfStoredValue, } from '@/lib/uploads/utils/file' const logger = createLogger('FileUtilsTest') @@ -184,3 +185,23 @@ describe('observedMediaType', () => { expect(observedMediaType('IMAGE/PNG', 'photo.png')).toBe('image/png') }) }) + +describe('storageKeyOfStoredValue', () => { + it('reads a serve URL as the key it carries, its query dropped', () => { + expect( + storageKeyOfStoredValue('/api/files/serve/workspace%2Fws-1%2Fa.png?context=workspace') + ).toBe('workspace/ws-1/a.png') + expect(storageKeyOfStoredValue('https://sim.ai/api/files/serve/s3/workspace%2Fb.md')).toBe( + 'workspace/b.md' + ) + }) + + it('keeps any other value whole, `?` and `#` included', () => { + expect(storageKeyOfStoredValue('workspace/ws-1/what?.md')).toBe('workspace/ws-1/what?.md') + expect(storageKeyOfStoredValue('notes #1?.md')).toBe('notes #1?.md') + }) + + it('names nothing for a serve URL whose encoding is malformed, instead of throwing', () => { + expect(storageKeyOfStoredValue('/api/files/serve/workspace%E0%A4%A.png')).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index a40309452fd..acddf49d950 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -687,12 +687,14 @@ export function extractStorageKey(filePath: string): string { } /** - * {@link extractStorageKey} for a path read back from storage or a document, which may be - * malformed: `undefined` when its percent-encoding cannot be decoded, rather than throwing. + * The storage key a stored file value names: a serve URL's key (see {@link isInternalFileUrl}), + * else the value itself — a name or path may hold `?` or `#`, which only a URL reads as its query. + * `undefined` for a serve URL whose percent-encoding cannot be decoded, rather than throwing. */ -export function tryExtractStorageKey(filePath: string): string | undefined { +export function storageKeyOfStoredValue(value: string): string | undefined { + if (!isInternalFileUrl(value)) return value try { - return extractStorageKey(filePath) + return extractStorageKey(value) } catch { return undefined } diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index 995e14b1825..bdb93803fb5 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -13,15 +13,14 @@ import { inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import type { CursorKey } from '@/lib/api/list-query' import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager' -import { EXTENSION_TO_MIME, getFileExtension, MIME_TYPE_MAPPING } from '@/lib/uploads/utils/file' -import { isSupportedExtension } from '@/lib/uploads/utils/validation' +import { EXTENSION_TO_MIME } from '@/lib/uploads/utils/file' import { getWorkspaceFileFolderStats, listWorkspaceFileCollection, lookupWorkspaceFiles, searchWorkspaceFiles, } from '@/lib/workspace-files/application/list-workspace-files' -import { fileBrowserMimeType, fileBrowserTypeLabel } from '@/lib/workspace-files/browser' +import { fileBrowserKinds, fileBrowserTypeLabel } from '@/lib/workspace-files/browser' import { workspaceFileKindCondition, workspaceFileTypeLabelExpression, @@ -702,6 +701,9 @@ describe('bounded workspace file reads in PostgreSQL', () => { { name: 'file.txt', contentType: ' Text/Markdown ; charset=utf-8' }, { name: 'file.mp3', contentType: 'AUDIO/MPEG' }, { name: 'file.dat', contentType: 'Application/Octet-Stream; x=1' }, + { name: 'clip.bin', contentType: 'video/ogg' }, + { name: 'song.bin', contentType: 'Audio/X-Unlisted; rate=44100' }, + { name: 'clip.vid', contentType: 'video/x-unlisted' }, { name: 'file.foo', contentType: 'application/octet-stream' }, { name: 'file.foo', contentType: '' }, ] @@ -726,14 +728,14 @@ describe('bounded workspace file reads in PostgreSQL', () => { for (const row of rows) { const fixture = ids.get(String(row.id)) if (!fixture) throw new Error('Unexpected row') - const mime = fileBrowserMimeType(fixture.contentType, fixture.name) + const kinds = fileBrowserKinds(fixture.contentType, fixture.name) expect({ id: row.id, label: row.label, ...pick(row) }).toEqual({ id: row.id, label: fileBrowserTypeLabel(fixture.contentType, fixture.name), - document: isSupportedExtension(getFileExtension(fixture.name)), - image: mime.startsWith('image/'), - audio: MIME_TYPE_MAPPING[mime.toLowerCase()] === 'audio', - video: MIME_TYPE_MAPPING[mime.toLowerCase()] === 'video', + document: kinds.has('document'), + image: kinds.has('image'), + audio: kinds.has('audio'), + video: kinds.has('video'), }) } }) diff --git a/apps/sim/lib/workspace-files/browser.test.ts b/apps/sim/lib/workspace-files/browser.test.ts index 90b835a958d..e4fe9a3d532 100644 --- a/apps/sim/lib/workspace-files/browser.test.ts +++ b/apps/sim/lib/workspace-files/browser.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { fileBrowserMimeType, fileBrowserTypeLabel } from '@/lib/workspace-files/browser' +import { + fileBrowserKinds, + fileBrowserMimeType, + fileBrowserTypeLabel, +} from '@/lib/workspace-files/browser' describe('fileBrowserTypeLabel', () => { it('reads a stored type with parameters or mixed case as its bare media type', () => { @@ -15,3 +19,15 @@ describe('fileBrowserTypeLabel', () => { ) }) }) + +describe('fileBrowserKinds', () => { + it('counts any audio/* or video/* type as Audio or Video, as its Type label does', () => { + expect([...fileBrowserKinds('video/x-unlisted', 'clip.bin')]).toEqual(['video']) + expect(fileBrowserTypeLabel('video/x-unlisted', 'clip.bin')).toBe('Video') + expect([...fileBrowserKinds('Audio/X-Unlisted; rate=44100', 'song.bin')]).toEqual(['audio']) + }) + + it('reads a document by its extension, whatever its stored type', () => { + expect(fileBrowserKinds('application/octet-stream', 'notes.md').has('document')).toBe(true) + }) +}) diff --git a/apps/sim/lib/workspace-files/browser.ts b/apps/sim/lib/workspace-files/browser.ts index 3b40a6e4239..05eb3095bbb 100644 --- a/apps/sim/lib/workspace-files/browser.ts +++ b/apps/sim/lib/workspace-files/browser.ts @@ -1,5 +1,6 @@ import { getFileExtension, resolveEffectiveMimeType } from '@/lib/uploads/utils/file' import { normalizeMimeType } from '@/lib/uploads/utils/mime' +import { isSupportedExtension } from '@/lib/uploads/utils/validation' export const FILE_BROWSER_SORTS = ['name', 'size', 'type', 'created', 'owner', 'updated'] as const export type FileBrowserSort = (typeof FILE_BROWSER_SORTS)[number] @@ -75,6 +76,21 @@ export function fileBrowserMimeType(storedType: string, filename: string): strin return resolveEffectiveMimeType(normalizeMimeType(storedType), filename) } +/** + * The kinds a file is, for the browsers' Type filter: `document` by its extension, and a media + * kind by its MIME type's family — any `image/*`, `audio/*`, or `video/*`, as its Type label reads + * it. The SQL twin is `fileKindSql`. + */ +export function fileBrowserKinds(storedType: string, filename: string): Set<FileBrowserType> { + const mimeType = fileBrowserMimeType(storedType, filename) + const kinds = new Set<FileBrowserType>() + if (isSupportedExtension(getFileExtension(filename))) kinds.add('document') + for (const kind of ['image', 'audio', 'video'] as const) { + if (mimeType.startsWith(`${kind}/`)) kinds.add(kind) + } + return kinds +} + /** * A file's Type column label: a specific label for its effective MIME type, else its media * family (Audio, Video, Image), else its extension uppercased, else the stored type. The SQL diff --git a/apps/sim/lib/workspace-files/file-type-sql.ts b/apps/sim/lib/workspace-files/file-type-sql.ts index 6f8f4315bd1..71b3e640a37 100644 --- a/apps/sim/lib/workspace-files/file-type-sql.ts +++ b/apps/sim/lib/workspace-files/file-type-sql.ts @@ -2,7 +2,6 @@ import { type SQL, type SQLWrapper, sql } from 'drizzle-orm' import { EXTENSION_TO_MIME, GENERIC_MIME_TYPES, - MIME_TYPE_MAPPING, resolveEffectiveMimeType, } from '@/lib/uploads/utils/file' import { SUPPORTED_DOCUMENT_EXTENSIONS } from '@/lib/uploads/utils/validation' @@ -31,10 +30,6 @@ const EFFECTIVE_EXTENSION_MIMES = JSON.stringify( const LABELS = JSON.stringify(FILE_BROWSER_MIME_LABELS) const GENERIC_MIME = resolveEffectiveMimeType(null, 'file') -function mimesOfKind(kind: 'audio' | 'video'): string[] { - return Object.keys(MIME_TYPE_MAPPING).filter((mime) => MIME_TYPE_MAPPING[mime] === kind) -} - function list(values: readonly string[]): SQL { return sql.join( values.map((value) => sql`${value}`), @@ -69,11 +64,10 @@ export function fileTypeLabelSql(mime: SQL, extension: SQL, contentType: SQLWrap else coalesce(${contentType}, 'File') end)` } -/** Whether a file of effective MIME type `mime` is of `kind`. */ +/** Whether a file of effective MIME type `mime` is of `kind` (see `fileBrowserKinds`). */ export function fileKindSql(kind: FileBrowserType, mime: SQL, extension: SQL): SQL { if (kind === 'document') return sql`${extension} in (${list(SUPPORTED_DOCUMENT_EXTENSIONS)})` - if (kind === 'image') return sql`${mime} like 'image/%'` - return sql`lower(${mime}) in (${list(mimesOfKind(kind))})` + return sql`${mime} like ${`${kind}/%`}` } export function fileSizeBucketSql(bucket: FileBrowserSize, size: SQL): SQL { diff --git a/packages/testing/src/mocks/file-utils.mock.ts b/packages/testing/src/mocks/file-utils.mock.ts index 86e5c298109..614cdb971b9 100644 --- a/packages/testing/src/mocks/file-utils.mock.ts +++ b/packages/testing/src/mocks/file-utils.mock.ts @@ -374,9 +374,10 @@ function extractStorageKey(filePath: string): string { return pathWithoutQuery } -function tryExtractStorageKey(filePath: string): string | undefined { +function storageKeyOfStoredValue(value: string): string | undefined { + if (!isInternalFileUrl(value)) return value try { - return extractStorageKey(filePath) + return extractStorageKey(value) } catch { return undefined } @@ -578,7 +579,7 @@ export const fileUtilsMockFns = { } ), mockExtractStorageKey: vi.fn(extractStorageKey), - mockTryExtractStorageKey: vi.fn(tryExtractStorageKey), + mockStorageKeyOfStoredValue: vi.fn(storageKeyOfStoredValue), mockIsInternalFileUrl: vi.fn(isInternalFileUrl), mockInferContextFromKey: vi.fn(inferContextFromKey), mockTryInferContextFromKey: vi.fn(tryInferContextFromKey), @@ -741,7 +742,7 @@ export const fileUtilsMock = { formatFileSize: fileUtilsMockFns.mockFormatFileSize, validateKnowledgeBaseFile: fileUtilsMockFns.mockValidateKnowledgeBaseFile, extractStorageKey: fileUtilsMockFns.mockExtractStorageKey, - tryExtractStorageKey: fileUtilsMockFns.mockTryExtractStorageKey, + storageKeyOfStoredValue: fileUtilsMockFns.mockStorageKeyOfStoredValue, isInternalFileUrl: fileUtilsMockFns.mockIsInternalFileUrl, inferContextFromKey: fileUtilsMockFns.mockInferContextFromKey, tryInferContextFromKey: fileUtilsMockFns.mockTryInferContextFromKey, From fcd000a2c2789deb252cb2bbf46c18a747aa0048 Mon Sep 17 00:00:00 2001 From: Waleed Latif <walif6@gmail.com> Date: Wed, 28 Oct 2026 22:48:36 -0700 Subject: [PATCH 15/15] fix(files): address the paged file reads' third review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Type labels and extension MIME types read their tables by own key only, so a type or extension such as `constructor` or `__proto__` never resolves to an `Object.prototype` member; the parity test covers them. - A serve URL's fragment, like its query, is no part of its storage key. - Workflow search-replace offers only files answering what is typed now — the recent files with nothing typed, else the settled matches for the current query — never a previous query's (`currentFiles`). --- apps/sim/hooks/queries/workflow-search-replace.ts | 13 +++++++++---- apps/sim/hooks/queries/workspace-files.test.tsx | 3 +++ apps/sim/hooks/queries/workspace-files.ts | 15 +++++++++++++-- apps/sim/lib/uploads/utils/file.test.ts | 9 +++++++++ apps/sim/lib/uploads/utils/file.ts | 11 ++++++++--- .../__integration__/file-listing.integration.ts | 4 ++++ apps/sim/lib/workspace-files/browser.test.ts | 9 +++++++++ apps/sim/lib/workspace-files/browser.ts | 4 ++-- packages/testing/src/mocks/file-utils.mock.ts | 3 ++- 9 files changed, 59 insertions(+), 12 deletions(-) diff --git a/apps/sim/hooks/queries/workflow-search-replace.ts b/apps/sim/hooks/queries/workflow-search-replace.ts index 97ea8e409bd..7d0e6b1b3f2 100644 --- a/apps/sim/hooks/queries/workflow-search-replace.ts +++ b/apps/sim/hooks/queries/workflow-search-replace.ts @@ -712,10 +712,15 @@ export function useWorkflowSearchFileReplacementOptions( [matches] ) const enabled = Boolean(workspaceId && fileGroups.length > 0) - const { files: offered } = useWorkspaceFilePickerOptions(workspaceId ?? '', options.query, { - limit: RECENT_REPLACEMENT_FILE_LIMIT, - enabled, - }) + /** Never a previous query's matches: a choice offered must answer what is typed now. */ + const { currentFiles: offered } = useWorkspaceFilePickerOptions( + workspaceId ?? '', + options.query, + { + limit: RECENT_REPLACEMENT_FILE_LIMIT, + enabled, + } + ) const chosenKeys = useMemo( () => options.chosen.flatMap((value) => { diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index 94d1c9dc5d3..95bf6b293b4 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -850,6 +850,7 @@ describe('useWorkspaceFilePickerOptions', () => { await vi.waitFor(() => expect(hook.result().files?.map((file) => file.id)).toEqual(['match'])) expect(hook.result().matchedQuery).toBe('mat') expect(hook.result().isPending).toBe(false) + expect(hook.result().currentFiles?.map((file) => file.id)).toEqual(['match']) } finally { hook.unmount() } @@ -873,6 +874,8 @@ describe('useWorkspaceFilePickerOptions', () => { await vi.waitFor(() => expect(mockRequestJson).toHaveBeenCalledTimes(3)) expect(hook.result().files?.map((file) => file.id)).toEqual(['match']) expect(hook.result().matchedQuery).toBe('') + /** What is offered as-is never holds the previous query's matches. */ + expect(hook.result().currentFiles).toBeUndefined() } finally { hook.unmount() } diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 98f7b1fd8c8..3fd234a473b 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -499,6 +499,13 @@ export interface WorkspaceFilePickerOptions { * `files` are the recent files, or a previous query's matches held while the next loads. */ matchedQuery: string + /** + * `files` only while they answer exactly what is typed now: the recent files with nothing + * typed, the server's matches once they are in for the current query — never a previous + * query's matches, nor the recent files while a query settles. For a caller that offers the + * files as they are rather than filtering them. + */ + currentFiles: WorkspaceFileRecord[] | undefined /** True until `files` answers what was typed: a menu still empty here is not "no match" yet. */ isPending: boolean } @@ -526,9 +533,13 @@ export function useWorkspaceFilePickerOptions( { q: settledQuery, limit: WORKSPACE_FILE_PICKER_LIMIT }, { enabled } ) + const files = settledQuery ? matching.data : recent.data?.pages[0]?.files + const matchedQuery = + settledQuery && matching.data && !matching.isPlaceholderData ? settledQuery : '' return { - files: settledQuery ? matching.data : recent.data?.pages[0]?.files, - matchedQuery: settledQuery && matching.data && !matching.isPlaceholderData ? settledQuery : '', + files, + matchedQuery, + currentFiles: !typed || matchedQuery === typed ? files : undefined, isPending: settledQuery ? typed !== settledQuery || matching.isFetching : recent.isPending || typed.length > 0, diff --git a/apps/sim/lib/uploads/utils/file.test.ts b/apps/sim/lib/uploads/utils/file.test.ts index d0c220a4f0d..9021ffa59fd 100644 --- a/apps/sim/lib/uploads/utils/file.test.ts +++ b/apps/sim/lib/uploads/utils/file.test.ts @@ -196,6 +196,15 @@ describe('storageKeyOfStoredValue', () => { ) }) + it("drops a serve URL's fragment as well as its query", () => { + expect(storageKeyOfStoredValue('/api/files/serve/workspace%2Fws-1%2Fa.png#page=2')).toBe( + 'workspace/ws-1/a.png' + ) + expect( + storageKeyOfStoredValue('/api/files/serve/workspace%2Fws-1%2Fa.png?context=workspace#x') + ).toBe('workspace/ws-1/a.png') + }) + it('keeps any other value whole, `?` and `#` included', () => { expect(storageKeyOfStoredValue('workspace/ws-1/what?.md')).toBe('workspace/ws-1/what?.md') expect(storageKeyOfStoredValue('notes #1?.md')).toBe('notes #1?.md') diff --git a/apps/sim/lib/uploads/utils/file.ts b/apps/sim/lib/uploads/utils/file.ts index acddf49d950..cf7bc616d84 100644 --- a/apps/sim/lib/uploads/utils/file.ts +++ b/apps/sim/lib/uploads/utils/file.ts @@ -413,7 +413,9 @@ function identifiesFormat(declared: string | undefined): declared is string { * Get MIME type from file extension (fallback if not provided) */ export function getMimeTypeFromExtension(extension: string): string { - return EXTENSION_TO_MIME[extension.toLowerCase()] || GENERIC_MIME_TYPE + const key = extension.toLowerCase() + // Own keys only: an extension such as `constructor` must not read an `Object.prototype` member. + return (Object.hasOwn(EXTENSION_TO_MIME, key) && EXTENSION_TO_MIME[key]) || GENERIC_MIME_TYPE } /** @@ -435,7 +437,9 @@ export function resolveEffectiveMimeType( if (identifiesFormat(declared)) return declared const extension = getFileExtension(filename) - return DUAL_CONTAINER_MIME[extension] ?? getMimeTypeFromExtension(extension) + return Object.hasOwn(DUAL_CONTAINER_MIME, extension) + ? DUAL_CONTAINER_MIME[extension] + : getMimeTypeFromExtension(extension) } const MEDIA_FALLBACK_MIME = { audio: 'audio/mpeg', video: 'video/mp4' } as const @@ -694,7 +698,8 @@ export function extractStorageKey(filePath: string): string { export function storageKeyOfStoredValue(value: string): string | undefined { if (!isInternalFileUrl(value)) return value try { - return extractStorageKey(value) + // The key is the path alone: a URL's query and fragment are no part of it. + return extractStorageKey(value.split(/[?#]/, 1)[0]) } catch { return undefined } diff --git a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts index bdb93803fb5..3c47a1870de 100644 --- a/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts +++ b/apps/sim/lib/workspace-files/__integration__/file-listing.integration.ts @@ -704,6 +704,10 @@ describe('bounded workspace file reads in PostgreSQL', () => { { name: 'clip.bin', contentType: 'video/ogg' }, { name: 'song.bin', contentType: 'Audio/X-Unlisted; rate=44100' }, { name: 'clip.vid', contentType: 'video/x-unlisted' }, + { name: 'file.constructor', contentType: '' }, + { name: 'file.__proto__', contentType: 'application/octet-stream' }, + { name: 'noext', contentType: 'constructor' }, + { name: 'noext', contentType: 'toString' }, { name: 'file.foo', contentType: 'application/octet-stream' }, { name: 'file.foo', contentType: '' }, ] diff --git a/apps/sim/lib/workspace-files/browser.test.ts b/apps/sim/lib/workspace-files/browser.test.ts index e4fe9a3d532..12735c98f20 100644 --- a/apps/sim/lib/workspace-files/browser.test.ts +++ b/apps/sim/lib/workspace-files/browser.test.ts @@ -31,3 +31,12 @@ describe('fileBrowserKinds', () => { expect(fileBrowserKinds('application/octet-stream', 'notes.md').has('document')).toBe(true) }) }) + +describe('table lookups by name', () => { + it('never reads an Object.prototype member as a label or a MIME type', () => { + expect(fileBrowserTypeLabel('constructor', 'noext')).toBe('constructor') + expect(fileBrowserTypeLabel('toString', 'noext')).toBe('toString') + expect(fileBrowserMimeType('', 'file.constructor')).toBe(fileBrowserMimeType('', 'file.zzz')) + expect(fileBrowserTypeLabel('', 'file.__proto__')).toBe('__PROTO__') + }) +}) diff --git a/apps/sim/lib/workspace-files/browser.ts b/apps/sim/lib/workspace-files/browser.ts index 05eb3095bbb..557b0a47f77 100644 --- a/apps/sim/lib/workspace-files/browser.ts +++ b/apps/sim/lib/workspace-files/browser.ts @@ -98,8 +98,8 @@ export function fileBrowserKinds(storedType: string, filename: string): Set<File */ export function fileBrowserTypeLabel(storedType: string, filename: string): string { const mimeType = fileBrowserMimeType(storedType, filename) - const label = FILE_BROWSER_MIME_LABELS[mimeType] - if (label) return label + // Own keys only, as the SQL twin's `jsonb ->>` reads them: `constructor` is no label. + if (Object.hasOwn(FILE_BROWSER_MIME_LABELS, mimeType)) return FILE_BROWSER_MIME_LABELS[mimeType] if (mimeType.startsWith('audio/')) return 'Audio' if (mimeType.startsWith('video/')) return 'Video' if (mimeType.startsWith('image/')) return 'Image' diff --git a/packages/testing/src/mocks/file-utils.mock.ts b/packages/testing/src/mocks/file-utils.mock.ts index 614cdb971b9..5304bd52f50 100644 --- a/packages/testing/src/mocks/file-utils.mock.ts +++ b/packages/testing/src/mocks/file-utils.mock.ts @@ -377,7 +377,8 @@ function extractStorageKey(filePath: string): string { function storageKeyOfStoredValue(value: string): string | undefined { if (!isInternalFileUrl(value)) return value try { - return extractStorageKey(value) + // The key is the path alone: a URL's query and fragment are no part of it. + return extractStorageKey(value.split(/[?#]/, 1)[0]) } catch { return undefined }