From dd78cc3dfed44dd97f34fc650b1889bc5fae7404 Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Tue, 6 Oct 2026 20:34:13 -0500 Subject: [PATCH 01/18] feat: Initial Base of tools for Archives, decompilation, and logging --- packages/ui/package.json | 6 + .../ui/src/helpers/toolbox/decompile/impl.ts | 119 +++ .../ui/src/helpers/toolbox/decompile/index.ts | 41 + packages/ui/src/helpers/toolbox/file/index.ts | 253 +++++++ .../helpers/toolbox/messenger/implement.ts | 705 ++++++++++++++++++ .../ui/src/helpers/toolbox/messenger/index.ts | 216 ++++++ packages/ui/src/helpers/toolbox/misc/index.ts | 353 +++++++++ .../ui/src/helpers/toolbox/misc/logging.ts | 313 ++++++++ .../toolbox/workers/decompile/index.ts | 8 + .../ui/src/helpers/toolbox/workers/index.ts | 81 ++ .../ui/src/helpers/toolbox/workers/zip/api.ts | 176 +++++ .../src/helpers/toolbox/workers/zip/index.ts | 319 ++++++++ packages/ui/src/helpers/toolbox/zip/impl.ts | 314 ++++++++ packages/ui/src/helpers/toolbox/zip/index.ts | 376 ++++++++++ packages/ui/src/utils/file-extensions.ts | 16 +- pnpm-lock.yaml | 91 +++ 16 files changed, 3384 insertions(+), 3 deletions(-) create mode 100644 packages/ui/src/helpers/toolbox/decompile/impl.ts create mode 100644 packages/ui/src/helpers/toolbox/decompile/index.ts create mode 100644 packages/ui/src/helpers/toolbox/file/index.ts create mode 100644 packages/ui/src/helpers/toolbox/messenger/implement.ts create mode 100644 packages/ui/src/helpers/toolbox/messenger/index.ts create mode 100644 packages/ui/src/helpers/toolbox/misc/index.ts create mode 100644 packages/ui/src/helpers/toolbox/misc/logging.ts create mode 100644 packages/ui/src/helpers/toolbox/workers/decompile/index.ts create mode 100644 packages/ui/src/helpers/toolbox/workers/index.ts create mode 100644 packages/ui/src/helpers/toolbox/workers/zip/api.ts create mode 100644 packages/ui/src/helpers/toolbox/workers/zip/index.ts create mode 100644 packages/ui/src/helpers/toolbox/zip/impl.ts create mode 100644 packages/ui/src/helpers/toolbox/zip/index.ts diff --git a/packages/ui/package.json b/packages/ui/package.json index 22c6b79d50d..61ec8786be6 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -67,6 +67,9 @@ "@modrinth/assets": "workspace:*", "@modrinth/blog": "workspace:*", "@modrinth/utils": "workspace:*", + "@run-slicer/cfr": "0.1.8-0.152", + "@run-slicer/procyon": "0.1.2-0.6.0", + "@run-slicer/vf": "0.6.4-1.12.0", "@tanstack/vue-query": "5.101.4", "@tresjs/cientos": "^4.3.0", "@tresjs/core": "^4.3.4", @@ -79,6 +82,7 @@ "@xterm/addon-fit": "^0.11.0", "@xterm/addon-search": "^0.16.0", "@xterm/xterm": "^6.0.0", + "@zip.js/zip.js": "^2.8.34", "ace-builds": "^1.43.5", "apexcharts": "^4.0.0", "colord": "^2.9.3", @@ -88,6 +92,7 @@ "dompurify": "^3.1.7", "es-toolkit": "^1.44.0", "fabric": "^7.4.0", + "file-type": "^22.1.1", "flatpickr": "^4.6.13", "fuse.js": "^6.6.2", "highlight.js": "^11.9.0", @@ -96,6 +101,7 @@ "jszip": "^3.10.1", "lru-cache": "^11.2.4", "markdown-it": "^13.0.2", + "mime": "^4.1.0", "motion-v": "^2.2.1", "obscenity": "^0.4.6", "overlayscrollbars": "^2.15.1", diff --git a/packages/ui/src/helpers/toolbox/decompile/impl.ts b/packages/ui/src/helpers/toolbox/decompile/impl.ts new file mode 100644 index 00000000000..6310ab0ae14 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/decompile/impl.ts @@ -0,0 +1,119 @@ + +import { + type DecompileInput, type Decompiler, + type DecompilerResult, + type DecompilerTypes, DEFAULT_DECOMPILE_OPTIONS +} from "#ui/helpers/toolbox/decompile/index.ts"; +import type { FileTypeInfo } from "#ui/helpers/toolbox/file"; +import type { Logger } from "#ui/helpers/toolbox/misc/logging.ts"; +import type { ZipFileContent, ZipHandler } from "#ui/helpers/toolbox/zip"; + + +export class DecompilerResultImpl implements DecompilerResult { + wasDecompiled: boolean; + error?: string | undefined; + source?: string | undefined; + fileType?: FileTypeInfo | undefined; + asString: string; + + constructor(source?: string, fileType?: FileTypeInfo, error?: string,) { + this.error = error; + this.source = source; + this.fileType = fileType; + this.wasDecompiled = source != null; + this.asString = this.source ?? this.error ?? "UNKNOWN DUE TO AN ODD ERROR?"; + } +} + +async function getDecompiler(logger: Logger, type: DecompilerTypes): Promise { + let decompiler: Decompiler | undefined = undefined; + let possibleErr: any; + try { + if (type == "cfr") { + decompiler = await import("@run-slicer/cfr"); + } else if (type == "procyon") { + decompiler = await import("@run-slicer/procyon"); + } /*else if (type == 'krakatau') { + const { KrakClient } = await import('krakatau-web'); + const client = new KrakClient(); + client.decompile() + decompiler = { + decompile(names: string, config: { + source?: (name: string) => Promise; + options?: Record; + resources?: string[] + } | undefined): Promise> { + + return Promise.resolve(undefined); + } + } + }*/ + } catch (err: any) { + possibleErr = err; + } + + try { + if (decompiler == undefined) { + if (type == "vineflower") logger.warn(`Unable to get desired decompiler '${type}' due to an error: `, { err: possibleErr }) + decompiler = await import("@run-slicer/vf"); + } + } catch (err) { + const outerErr = new Error(`Unable to get vineflower decompiler due to an error'`); + outerErr.cause = err; + throw outerErr + } + + return decompiler; +} + +export async function decompileFileContent(logger: Logger, zipHandler: ZipHandler | undefined, {targetFile, resources, options}: DecompileInput): Promise { + let error: string | undefined = undefined; + let source: string | undefined = undefined; + + if (options == null) options = DEFAULT_DECOMPILE_OPTIONS; + + const rootKey = (targetFile as any).rootKey ? (targetFile as ZipFileContent).rootKey : undefined; + + try { + // const pathParts = targetFile.fullPath.split('.jar'); \n const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); \n + const pathParts = targetFile.fullPath.split('.jar'); + const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); + const buffer = targetFile.content; + const decompiled = await (await getDecompiler(logger, options.decompiler!)) + .decompile(withoutClassSuffix, { + source: async (name) => { + if (name === withoutClassSuffix) { + return new Uint8Array(buffer); + } else if (rootKey && name.includes(withoutClassSuffix)) { + // TODO: THIS DOES NOT PROPERLY HANDLE WHICH JAR WE NEED TO LOOK AT SO IF DUPLICATE FILES EXIST, WE MAY GET THE WRONG ONE!!!! + const buffer = await zipHandler?.readNodeData({ path: name + ".class", root: rootKey, relative: true }); + if (buffer) return new Uint8Array(buffer); + } + + return null; + }, + resources: [ + ...(resources?.map(details => details.fullPath.slice(0, -6)) ?? []), + ], + options: options.options + }); + + if (typeof decompiled === 'string') { + source = decompiled ?? undefined; + } else if (!decompiled[withoutClassSuffix]) { + console.error("Decompilation failed for", targetFile.fullPath, decompiled); + error = "// [Decompilation output either is kotlin or their is issues just a fyi!] \n\n"; + + // for (const [key, value] of Object.entries(decompiled)) { \n error = `${error} \n\n Class: ${key} \n\n ${value}` \n } \n + for (const [key, value] of Object.entries(decompiled)) { + error = `${error} \n\n Class: ${key} \n\n ${value}` + } + } else { + source = decompiled[withoutClassSuffix] ?? undefined; + } + } catch (err: any) { + error = `[${err?.message ?? "Decompilation has due to an unknown error being thrown!"}]`; + } + + return new DecompilerResultImpl(source, { mime: "text/x-java-source", extension: "java" }, error); +} diff --git a/packages/ui/src/helpers/toolbox/decompile/index.ts b/packages/ui/src/helpers/toolbox/decompile/index.ts new file mode 100644 index 00000000000..d5f3ad96445 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/decompile/index.ts @@ -0,0 +1,41 @@ +import type { FileContent, FileDetails, FileTypeInfo } from "#ui/helpers/toolbox/file"; + +export const ALL_DECOMPILER_TYPES = ['cfr', 'vineflower', 'procyon', 'krakatau'] as const; + +export type DecompilerTypes = typeof ALL_DECOMPILER_TYPES[number]; + +export type DecompileInput = { + targetFile: FileContent, resources?: FileDetails[], options?: DecompileOptions +}; + +export type DecompileOptions = { + decompiler?: DecompilerTypes, options?: Record +} + +export const DEFAULT_DECOMPILE_OPTIONS = { + decompiler: 'vineflower', + options: { + 'remove-synthetic': '0', + 'remove-bridge': '0', + 'kt-enable': '0' // TODO: MAYBE ADJUST STUFF IDK + } +} as const; + +export type DecompileHandler = (input: DecompileInput) => Promise; + +export interface DecompilerResult { + wasDecompiled: boolean, + error?: string, + source?: string, + fileType?: FileTypeInfo + asString: string, +} + +export interface Decompiler { + // TODO: SOME DECOMPILERS DO NOT TAKE IN AN ARRAY AT ALL + decompile(names: string/* | string[]*/, config?: { + source?: (name: string) => Promise, + options?: Record, + resources?: string[] + }): Promise> +} diff --git a/packages/ui/src/helpers/toolbox/file/index.ts b/packages/ui/src/helpers/toolbox/file/index.ts new file mode 100644 index 00000000000..2db29462930 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/file/index.ts @@ -0,0 +1,253 @@ +import { fileTypeFromBuffer } from "file-type"; +import yaml from "js-yaml" +import mime from "mime"; +import toml from 'smol-toml' + +export interface FileDetails { + fullPath: string, + name: string, + url?: string +} + +export interface URLFileDetails extends FileDetails { + url: string +} + +export interface FileContent extends FileDetails { + content: ArrayBuffer; +} + +export interface HashedFileContent extends FileContent { + sha512: string, +} + +export interface FileContentPromise extends FileDetails { + content(): Promise; +} + +export async function resolveContent(content: T): Promise> { + return { + ...content, + fullPath: content.fullPath, + name: content.name, + content: await content.content(), + url: content.url + } +} + +export function fileFrom Promise), P extends {fullPath?: string, name: string, content: T, url?: string}>(params: Partial

& {fullPath?: string, name: string, content: T, url?: string}): T extends Promise ? FileContentPromise & P : FileContent & P { + return { + ...params, + fullPath: params.fullPath ?? params.name, + name: params.name, + content: typeof params.content === "function" ? params.content : params.content + } as any; +} + + +export function prettyifyJson(data: string, indentSize?: number) { + return JSON.stringify(JSON.parse(data), null, indentSize ?? 2); +} + +export interface FileTypeInfo { + mime: string; + extension?: string; + method?: 'content-analysis' | 'extension-lookup'; +} + +type PromiseMaybe = T extends undefined ? R : Promise + +export function getFileTypeInfo(path: string, data?: T): PromiseMaybe { + const promiseLike: Promise | undefined = data ? getFileTypeFromData(data) : undefined; + + if (promiseLike != null) { + return promiseLike.then((result) => result ?? getFileTypeFromExt(path)) as PromiseMaybe; + } else { + return getFileTypeFromExt(path) as PromiseMaybe; + } +} + +function getFileTypeFromExt(path: string): FileTypeInfo | null { + const typeFromPath = mime.getType(path); + + if (typeFromPath != null) { + return { + mime: typeFromPath, + extension: path.split('.').pop()?.split(/\#|\?/)[0], + method: 'extension-lookup' + }; + } + + return null; +} + +async function getFileTypeFromData(data: ArrayBuffer | Blob): Promise { + const buffer = (data as Partial).arrayBuffer + ? await (data as Blob).arrayBuffer() + : data as ArrayBuffer; + + if (buffer != null) { + const typeFromContent = await fileTypeFromBuffer(buffer.slice(0, 4100)); + + if (typeFromContent != null) { + return { + mime: typeFromContent.mime, + extension: typeFromContent.ext, + method: 'content-analysis' + }; + } + } + return null; +} + +export async function hashString(str: string, algorithm: AlgorithmIdentifier = "SHA-512") { + return hashBuffer(new TextEncoder().encode(str).buffer, algorithm); +} + +export async function hashBlob(blob: Blob, algorithm: AlgorithmIdentifier = "SHA-512") { + return hashBuffer(await blob.arrayBuffer(), algorithm); +} + + +export async function hashArrays(bytes: Uint8Array[], errorNameGetter?: () => string): Promise { + try { + // Concatenate all byte arrays into a single Buffer/Uint8Array + const totalLength = bytes.reduce((acc, curr) => acc + curr.length, 0); + const combinedBytes = new Uint8Array(totalLength); + + let offset = 0; + for (const buffer of bytes) { + combinedBytes.set(buffer, offset); + offset += bytes.length; + } + + // Digest via Web Crypto API (returns ArrayBuffer) + return await hashBuffer(combinedBytes.buffer, "SHA-512"); + } catch (e) { + console.warn(`Unable to hash the following '${errorNameGetter?.() ?? "UNKNOWN"}' due the following exception: `, e); + return null; + } +} + +export async function hashBuffer(data: ArrayBuffer, algorithm: AlgorithmIdentifier = "SHA-512") { + const hashBuffer = await crypto.subtle.digest(algorithm, data); + + return Array.from(new Uint8Array(hashBuffer)) + .map(b => b.toString(16).padStart(2, '0')) + .join(''); +} + +const MIME_TO_LANGUAGE: Record = { + // JavaScript & TypeScript + 'text/javascript': 'javascript', + 'application/javascript': 'javascript', + 'text/typescript': 'typescript', + 'application/typescript': 'typescript', + 'text/jsx': 'javascript', + 'text/tsx': 'typescript', + + // Web Formats + 'text/html': 'xml', + 'text/css': 'css', + 'application/json': 'json', + 'text/xml': 'xml', + 'application/xml': 'xml', + + // Scripting & Languages + 'text/x-python': 'python', + 'text/x-script.python': 'python', + 'text/x-sh': 'bash', + 'application/x-sh': 'bash', + 'text/x-c': 'c', + 'text/x-c++': 'cpp', + 'text/x-java-source': 'java', + 'text/x-rust': 'rust', + 'text/x-go': 'go', + 'text/x-kotlin': 'kotlin', + 'text/markdown': 'markdown', + 'text/x-yaml': 'yaml', + 'application/x-yaml': 'yaml', + + // Shell & Batch + 'application/x-bat': 'dos', + 'application/bat': 'dos', + 'text/x-bat': 'dos', + 'text/x-shellscript': 'bash', + + // Configs & Markup + 'application/toml': 'toml', + 'text/x-ini': 'ini', + 'text/x-properties': 'properties', +}; + +const EXTENSION_OVERRIDES: Record = { + // Minecraft Modding Specifics + 'mcfunction': 'mcfunction', // Uses custom grammar if registered, otherwise falls to fallback + 'mcmeta': 'json', + 'bbmodel': 'json', // Blockbench 3D model files are JSON inside + 'accesswidener': 'properties', + 'accesstransformer': 'properties', + 'lang': 'properties', // Legacy translation files + 'nbt': 'plaintext', // Binary format + + // Build & Config Scripts + 'gradle': 'gradle', + 'kts': 'kotlin', + 'bat': 'dos', + 'cmd': 'dos', + 'sh': 'bash', + 'toml': 'toml', + 'cfg': 'ini', + 'conf': 'ini', + 'properties': 'properties', + 'env': 'ini', + 'log': 'plaintext', + + // Special files without extensions (e.g. "gradlew", "Dockerfile") + 'gradlew': 'bash', + 'dockerfile': 'dockerfile', + 'gitignore': 'plaintext' +}; + +export function getLanguageFromFile(info: FileTypeInfo): string | undefined { + return MIME_TO_LANGUAGE[info.mime] ?? (info.extension ? EXTENSION_OVERRIDES[info.extension] : undefined); +} + +const javaManifestParser = { + parse(text: string) { + const manifest = {} as Record; + let currentKey: string | null = null; + + for (const line of text.replace(/\r\n/g, '\n').split('\n')) { + if (line.startsWith(' ') && currentKey) { + // Continuation line (Java Manifests wrap at 72 chars) + manifest[currentKey] += line.substring(1); + } else if (line.includes(':')) { + const [key, ...valueParts] = line.split(':'); + currentKey = key.trim(); + manifest[currentKey] = valueParts.join(':').trim(); + } + } + return manifest; + } +} + +export function parseInfoFile(fileName: string, ext: string, text: string) { + try { + if (ext === 'json' || ext === 'mcmeta' || fileName === 'mcmod.info') { + return JSON.parse(text); + } else if (ext === 'yaml' || ext === 'yml') { + return yaml.load(text); + } else if (ext === 'toml') { + return toml.parse(text); + } else if (fileName.toUpperCase().endsWith('MANIFEST.MF')) { + return javaManifestParser.parse(text); + } else { + throw new Error(`Unrecognized file type: ${fileName}`); + } + } catch (e: any) { + (window as any)?.["toolbox"]?.app?.error("Metadata File Parser", `Error processing ${fileName}: ${e.message}`, { err: e }) + } + + return null; +} diff --git a/packages/ui/src/helpers/toolbox/messenger/implement.ts b/packages/ui/src/helpers/toolbox/messenger/implement.ts new file mode 100644 index 00000000000..57fa4733866 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/messenger/implement.ts @@ -0,0 +1,705 @@ +import { type EventSource,EventStream } from "#ui/helpers/toolbox/misc"; +import { type CounterCallbackData, logFactory, type Logger, type LoggerCallbackData, type LoggerFactory } from '#ui/helpers/toolbox/misc/logging'; + +import { + type ChannelHolder, + type ClientCompoundMessenger, + type ClientMessenger, + type CompoundMessengerOptions, + type EntryOptions, + type InputMessageTypes, +type InstanceConfig, + type InstanceKey, + isWorker, type Message, + type MessageSchema, + type MessageTypes, + type Messenger, + type MessengerOptions, + type MessengerRegistry, + type MessengerType, + type MessengerVariants, + type OutputMessageTypes, + type ServerInputHandler, + type ServerMessenger, + type TypeId} from '.'; + +// #region MessengerType Types + +type InternalMessenger = M & C & { terminate(): void; onTerminate: EventSource; }; + +interface MessengerTypeImpl extends MessengerType { + registry: MessengerRegistryImpl; + + instances?: (() => InternalMessenger) | Map>; + wrapWith: (channelHolder: () => ChannelHolder, key: InstanceKey) => InternalMessenger; + + getOrCreateWith: (baseMessenger?: () => ChannelHolder, instanceKey?: string) => M extends ClientMessenger ? M & ChannelHolder & H : M & ChannelHolder; + workerFactory?: () => Worker; + + getInstances: () => M[]; +} + +type WrapperFunc = (type: MessengerTypeImpl, key: InstanceKey, holder: C) => M & C & InternalMessenger; + +function typeFor( + id: string, instanceConfig: InstanceConfig, closeCallback?: (messenger: M) => void) { + return { + id: id, + instances: instanceConfig.count == 1 ? undefined : new Map(), + getInstances() { + if (this.instances == null) return []; + return (this.instances as Partial>).values ? (this.instances as Map).values().toArray() : [(this.instances as unknown as M)]; + }, + getOrCreateWith(holder, instanceKey) { + if ((instanceConfig.count == 1) && this.instances != null) { + return (this.instances as () => M)(); + } + + if (holder == null) holder = this.workerFactory; + + if (instanceKey) { + if (holder == null) throw new Error(`Unable to create messenger for '${this.id}' as the entry used was found to be null! Did you setup the main thread worker factory or pass in the port to the call!`) + + const wrappedMessenger = this.wrapWith(holder, instanceKey); + + return wrappedMessenger; + } + + if (holder == null) throw new Error(`Unable to create messenger for '${this.id}' as the entry used was found to be null! Did you setup the main thread worker factory or pass in the port to the call!`) + + let wrappedMessenger: InternalMessenger; + + if (instanceConfig.count == 1) { + wrappedMessenger = this.wrapWith(holder, "primary"); + + wrappedMessenger.onTerminate.subscribe((forceClose) => { + if (forceClose || instanceConfig.timeout != null) this.instances = undefined as any; + }) + + this.instances = () => wrappedMessenger; + } else { + const instances = (this.instances as Map>); + + // TODO: ADD TIME BASED HASH? + const key = `temp-${instances.size}`; + + wrappedMessenger = this.wrapWith(holder, key); + + (this.instances as Map).set(wrappedMessenger.key!, wrappedMessenger); + } + + wrappedMessenger.postMessage?.({ type: "set-key", key: wrappedMessenger.key} as SetOffThreadKeyMessage); + + return wrappedMessenger; + }, + close(forceClosed) { + if (instanceConfig.count == 1) { + if (this.instances != undefined) { + const instance = (this.instances as () => InternalMessenger)(); + closeCallback?.(instance); + instance.postMessage({type: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) + } + } else { + const map = (this.instances as Map>); + map.values().forEach(entry => { + closeCallback?.(entry) + entry.postMessage({type: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) + }) + map.clear(); + } + } + /* wrapWith: wrapWith, */ + } as MessengerTypeImpl; +} + +// #endregion + + +// #region MessengerRegistry Types + +interface MessengerRegistryImpl extends MessengerRegistry { + setupMessengerType(id: string, wrapWith: WrapperFunc>, extra?: EntryOptions>): MessengerType>; + + serverMessenger?: ServerHandlerImpl, + types: Map>; + logger: LoggerFactory; + onLogFactoryChange: EventSource<(logger: LoggerFactory) => void>; +} + +export function createRegistryInternal(): MessengerRegistry { + return { + types: new Map(), + logger: logFactory(), + compoundMessenger: function >(id: string, extra: CompoundMessengerOptions & EntryOptions>): MessengerType> { + function wrapWith(type: MessengerTypeImpl>, key: string, channelHolder: C) { + return setupAsCompoundClient(type, "client", key, channelHolder, extra) + }; + + // @ts-expect-error + return this.setupMessengerType(id, wrapWith, extra); + }, + simpleMessenger: function (id: string, extra: MessengerOptions & EntryOptions>): MessengerType> { + function wrapWith(type: MessengerTypeImpl>, key: string, channelHolder: C) { + return setupAsClient(type, "client", key, channelHolder, extra) + }; + + // @ts-expect-error + return this.setupMessengerType(id, wrapWith, extra); + }, + setupMessengerType: function (id: string, wrapWith: WrapperFunc>, extra: MessengerOptions & EntryOptions): MessengerType> { + const instanceConfig: InstanceConfig = { + timeout: extra?.timeout ?? undefined, + shared: extra?.shared ?? true, + count: extra.count ?? 1, + }; + const type = typeFor>(id, instanceConfig); + + type.wrapWith = (holder, key) => wrapWith(type, key, extra.inputBypassHandler ? {} as any : holder()); + + type.registry = this; + type.instanceConfig = instanceConfig; + + this.types.set(id, type); + + return type; + }, + getType: function (id: string): MessengerType | undefined { + if (!this.types.has(id)) throw new Error(`MessengerRegistry did not contain entry '${id}'`); + return this.types.get(id)!; + }, + setLogFactory(logger: LoggerFactory): void { + this.logger = logger; + }, + getLogFactory(): LoggerFactory { + return this.logger; + }, + closeAll(forceClosed): void { + this.types.values().forEach(async type => { + type.close(forceClosed); + }) + }, + getMessenger(type: MessengerType) { + if (this.serverMessenger) { + return this.serverMessenger.getMessenger(type); + } else { + return (type as MessengerTypeImpl).getOrCreateWith(); + } + }, + getActiveMessengers() { + return this.types.values().flatMap((type) => (type as MessengerTypeImpl).getInstances()).toArray(); + }, + setWorkerFactoryFor(type, worker) { + if (!isWorker()) (type as MessengerTypeImpl).workerFactory = worker; + } + } as MessengerRegistryImpl; +} + +// #endregion + + +// --- + +// #region Base Messenger Types + +type OnTerminateCallback = (forceClose: boolean) => void + +//#region Impl handling + +function handleInput(messenger: Messenger & ChannelHolder, taskHolder: TaskHolder, input: any, transfer?: Transferable[]) { + const task = taskHolder.startTask(); + + messenger.postMessage({ + type: "input", + data: input, + key: task.transactionKey, + } as MessengerInputMessage, transfer ?? []); + + return task.promise; +} + +interface RootMessageHandler = Message> extends Messenger { + handleRootMessage(msg: MSG, ports?: readonly MessagePort[]): Promise; +} + +type TaskHolder = { + hasTask: (transactionKey: string) => boolean, + startTask: () => ({ transactionKey: string, promise: Promise }), + endTask: (transactionKey: string, value: any, isError?: boolean) => boolean; +} + +function createTaskHolder(): TaskHolder{ + const outstandingMessages: Map> = new Map(); + let index = 0; + return { + hasTask: (key) => outstandingMessages.has(key), + startTask: () => { + const key = `${index++}`; + + const promiseHolder = Promise.withResolvers(); + + outstandingMessages.set(key, promiseHolder); + + return {transactionKey: key, promise: promiseHolder.promise }; + }, + endTask: (key, value, isError) => { + const promise = outstandingMessages.get(key); + if (promise == null) { + return false; + } + + outstandingMessages.delete(key); + + if (isError) { + promise.reject(value); + } else { + promise.resolve(value); + } + + return true; + } + } +} + +interface ClientHandlerImpl extends RootMessageHandler, ClientMessenger { + taskHolder: TaskHolder; +} + +const FORCED_OUTPUT_KEY = "INTERNAL_FORCED_OUTPUT"; + +function clientMessageHandler( + messenger: InternalMessenger, + options: MessengerOptions, +): ClientHandlerImpl & InternalMessenger, C> & (H extends Function ? {} : H) { + const taskHolder = createTaskHolder(); + + const sendMessage: ClientMessenger["sendMessage"] = options.inputBypassHandler ? async (input) => { + const output = (await options.inputBypassHandler!(messenger as any, input)); + messenger.terminate(); + messenger.isAlive = false; + messenger.onTerminate.stream.sink(true) + return Array.isArray(output) ? output[0] : output; + } : (input) => handleInput(messenger, taskHolder, input, options.inputTransferHandler?.(input)); + + const handler = options.handlerConstructor?.(messenger as any) ?? ({} as any) + + const fullHandler = Object.assign(messenger, { + taskHolder: taskHolder, + async handleRootMessage(msg: ClientboundMessages){ + if (msg.type == "operation-error" || msg.type == "output") { + handleOutputMessages(this, msg, messenger.logger, taskHolder, options.forcedOutputHandler) + } else if (messenger.variant == "client") { + if (msg.type == "logging") { // MAIN THREAD ONLY + const factory = messenger.logger.factory; + if (msg.logMsg.type == 'callback') { + factory.log(...msg.logMsg.data) + } else if (msg.logMsg.type == 'counter') { + factory.counter(...msg.logMsg.data) + } else { + throw new Error(`Unknown type for logging '${(msg.logMsg as any).types}'!`) + } + + } else if (msg.type == "setup-root") { // MAIN THREAD ONLY + const registry = messenger.type.registry; + const newMessenger = await registry.getMessenger(registry.getType(msg.id)!) as InternalMessenger; + + const channel = new MessageChannel(); + + messenger.postMessage({ type: "setup-port", id: newMessenger.type.id, key: newMessenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port1]) + newMessenger.postMessage({ type: "setup-port", id: messenger.type.id, key: messenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port2]) + } else if (msg.type == "close-root") { + messenger.terminate(); + messenger.isAlive = false; + messenger.onTerminate.stream.sink(true) + } else { + return false; + } + } else { + return false; + } + + return true; + }, + sendMessage: sendMessage, + handler: handler, + } as ClientHandlerImpl & InternalMessenger); + + if (typeof handler === "object") Object.assign(fullHandler, handler); + + return fullHandler as any; +} + +function handleOutputMessages(messenger: ClientHandlerImpl, msg: MessengerOperationError | MessengerOutputMessage, logger: Logger, taskHolder: TaskHolder, forcedOutputHandler?: (messenger: ClientHandlerImpl, msg: O) => boolean) { + if (msg.type == "operation-error") { + if (taskHolder.hasTask(msg.key)) { + logger.error("An error occurred when attempting to complete a task!", {err: msg.error, obj: msg.info}) + } else { + logger.error("An error occurred when attempting to task no longer present!", {err: msg.error, obj: msg.info}) + } + + taskHolder.endTask(msg.key, msg.error, true); + } else { // MAIN THREAD OR PORT + if (taskHolder.hasTask(msg.key)) { + logger.debug("Completed a task for given data!", {obj: msg}) + } else { + logger.error("Unable to Complete a task for given data as its not here anymore!!", {obj: msg}) + } + + if (msg.key == FORCED_OUTPUT_KEY) { + if (forcedOutputHandler == null || !forcedOutputHandler(messenger, msg.data)) { + logger.error("Unable to Complete a forced output for given data as its not supported for this messenger!", {obj: msg}) + } + + return; + } + + taskHolder.endTask(msg.key, msg.data, false); + } +} + +interface ServerHandlerImpl extends RootMessageHandler, ServerMessenger, Messenger { + handleServerInput(sender: InternalMessenger, msg: MessengerInputMessage): Promise; + attemptTimeoutWait(): void; + startShutDown(): void; + setupPort(id: TypeId, transactionKey: string, key: InstanceKey, port: MessagePort): void; + closePort(fullPortKey: string): void; +} + +function serverMessageHandler( + messenger: InternalMessenger, + onInput: ServerInputHandler +): ServerHandlerImpl & InternalMessenger { + let currentCloseFuncId: ReturnType | undefined = undefined; + const currentInputKeys: Set = new Set(); + + const peerPorts: Map = new Map(); + const waitingForPeerPort: Map, promiseHolder: PromiseWithResolvers}> = new Map(); + + const impl: ServerHandlerImpl & InternalMessenger = Object.assign(messenger, { + async handleRootMessage(msg, ports) { + if (msg.type == "input") { + this.handleServerInput(this, msg) + } else if (this.variant == "server") { + if (msg.type == "set-key") { + this.key = msg.key; + this.logger = this.type.registry.getLogFactory().loggerFor(debugKeyFrom(this)); + } else if(msg.type == "setup-port") { + this.setupPort(msg.id, msg.transactionKey, msg.key, ports![0]) + } else if(msg.type == "force-close-root") { + if (this.instanceData.timeout != null || msg.forceClose) { + this.startShutDown() + } else { + this.logger.warn("Close is either not supported on this messenger or it was not a force close!") + } + } else { + return false; + } + } else { + return false; + } + + return true; + }, + async handleServerInput(sender: InternalMessenger, msg: MessengerInputMessage) { + const transactionKey = `${fullInstanceKeyFrom(sender)}/${msg.key}`; + + currentInputKeys.add(transactionKey); + + const logger = this.logger; + + try { + logger.debug(`Attempting to process '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}' `, { obj: msg }) + + const result = await onInput(this, msg.data, sender); + const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; + + logger.debug(`Finished processing '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) + + sender.postMessage(...([{type: "output", data: tuple[0], key: msg.key} as MessengerOutputMessage, tuple[1]])) + } catch (err) { + logger.debug(`An error occurred when processing '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) + //logger.debug("Error has occurred on off-thread worker: ", { err: err, obj: msg}) + sender.postMessage({ type: "operation-error", error: err, info: msg, key: msg.key} as MessengerOperationError) + } + + currentInputKeys.delete(transactionKey); + + if (!this.instanceData.shared || (this.instanceData.timeout != null && currentInputKeys.size == 0)) { + this.attemptTimeoutWait() + } + }, + attemptTimeoutWait() { + if (!this.isAlive) return; + const closeFuncId = setTimeout(async () => { + //if (currentCloseFuncId != closeFuncId) return; + + if (!this.instanceData.shared || (this.instanceData.timeout != null && currentInputKeys.size == 0)) { + this.startShutDown(); + } else { + currentCloseFuncId = undefined; + } + }, this.instanceData.timeout ?? 100); + clearTimeout(currentCloseFuncId); + currentCloseFuncId = closeFuncId; + }, + startShutDown() { + if (!this.isAlive) return; + + this.isAlive = false; + + peerPorts.values().forEach((messenger) => messenger.postMessage({ type: "close-port", id: messenger.type.id, key: messenger.key } as CloseMessengerPortMessage)) + + setTimeout(() => { + this.postMessage({ type: "close-root", id: this.type.id, key: this.key } as CloseMessengerRootMessage) + }, 100) + }, + setupPort(id, transactionKey, key, port) { + const fullTransactionKey = `${id}:${transactionKey}`; + + const data = waitingForPeerPort.get(fullTransactionKey)!; + waitingForPeerPort.delete(fullTransactionKey); + + const portMessenger = ((data?.type ?? this.type.registry.getType(id)) as MessengerTypeImpl)!.getOrCreateWith(() => port as Messenger & MessagePort, key) as InternalMessenger & MessagePort; + + portMessenger.variant = "port" + + portMessenger.logger.debug(`Port '${fullInstanceKeyFrom(portMessenger)}' is being setup on '${fullInstanceKeyFrom(this)}'`) + peerPorts.set(fullInstanceKeyFrom(portMessenger), portMessenger); + + const baseHandler = portMessenger.handleRootMessage.bind(portMessenger); + const portClientHandler = portClientMessageHandler(this); + + portMessenger.handleRootMessage = async (msg, ports) => { + if ((msg.type == "close-port" || msg.type == "output" || msg.type == "operation-error") && await portClientHandler(msg as ClientboundPortMessages, ports)) { + return true; + } else if (msg.type == "input") { + this.handleServerInput(portMessenger, msg as MessengerInputMessage) + } + + return baseHandler(msg, ports); + } + + data?.promiseHolder?.resolve(portMessenger); + }, + closePort(fullPortKey) { + const portChannelHolder = peerPorts.get(fullPortKey); + + if (portChannelHolder != null) { + peerPorts.delete(fullPortKey) + portChannelHolder.close(); + this.logger.debug(`Port '${fullInstanceKeyFrom(portChannelHolder)}' is closed on '${fullInstanceKeyFrom(this)}'`); + } else { + this.logger.error(`Unable to close a given port as '${fullPortKey}' was not found on '${fullInstanceKeyFrom(this)}'`) + } + }, + getMessenger(type) { + const transactionKey = `${peerPorts.size}`; + + const holder = Promise.withResolvers(); + + waitingForPeerPort.set(`${type.id}:${transactionKey}`, { type: type, promiseHolder: holder}); + + this.postMessage({ type: "setup-root", id: type.id, transactionKey: transactionKey} as StartMessengerRootMessage) + + return holder.promise as any; + }, + getActiveMessengers() { + return peerPorts.values().toArray() as Messenger[]; + }, + async sendOutputMessage(sender, msg) { + const result = await msg; + const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; + + (sender as InternalMessenger).postMessage(...([{type: "output", data: tuple[0], key: FORCED_OUTPUT_KEY} as MessengerOutputMessage, tuple[1]])) + } + } as ServerHandlerImpl & InternalMessenger); + + // TODO: ADD WARNING TO PREVENT DOUBLE SERVER STUFF + (impl.type.registry as MessengerRegistryImpl).serverMessenger = impl; + + return impl; +} + +function portClientMessageHandler(rootServerHandler: ServerHandlerImpl): RootMessageHandler["handleRootMessage"] { + return async (msg) => { + if (msg.type == "close-port") { + + rootServerHandler.closePort(fullInstanceKey(msg.id, msg.key)); + return true; + } + return false; + } +} + +//#endregion + +// #region Base Messenger Constructors + +export function setupHolderAsImpl( + holder: C, + type: MessengerTypeImpl, + variant: MessengerVariants, + key: InstanceKey | undefined, + rootHandlerSetup: (holder: C & InternalMessenger) => C & InternalMessenger & R +): C & Messenger & R { + const infoHolder = holder as (C & InternalMessenger); + + if (key) infoHolder.key = key; + + infoHolder.type = type as any; + infoHolder.instanceData = type.instanceConfig; + infoHolder.variant = variant; + infoHolder.isAlive = true; + + const messenger = rootHandlerSetup(holder as C & InternalMessenger); + + if ((messenger as MessagePort).close) messenger.terminate = (messenger as MessagePort).close; + + const registry = type.registry; + + if (variant == "server" && registry.getLogFactory() == null) { + registry.setLogFactory(logFactory({ + log: (level, loggerName, message, extra) => holder.postMessage({ type: "logging", logMsg: { type: 'callback', data: [level, loggerName, message, extra]}} as MessengerLogMessage), + counter: (level, loggerName, msg) => holder.postMessage({ type: "logging", logMsg: { type: 'counter', data: [level, loggerName, msg]}} as MessengerLogMessage) + })) + } + + messenger.logger = registry.getLogFactory().loggerFor(debugKey(type.id, key, variant)) + + messenger.onTerminate = new EventStream((callbacks) => (forceClose) => callbacks.forEach((callback) => callback(forceClose))).source; + + messenger.onmessage = function (e: MessageEvent>) { + const msg = e.data; + + if (msg.type != "logging") messenger.logger.debug(`Message '${msg.type}' received on '${debugKeyFrom(messenger)}'`, {obj: msg}); + + messenger.handleRootMessage(msg, e.ports); + }; + + (messenger as Worker).onerror = (err: ErrorEvent) => { + messenger.logger.error("An error occurred when attempting to complete a task and may be fatal error!", {err: err.error}) + }; + + return messenger; +} + +function setupAsCompoundClient>( + type: MessengerTypeImpl, + variant: MessengerVariants, + key: InstanceKey, + holder: ChannelHolder, + options: CompoundMessengerOptions, +): ClientCompoundMessenger { + // @ts-expect-error + const extendOptions: MessengerOptions, OutputMessageTypes, H> = { + ...options, + inputTransferHandler: options?.inputTransferHandler == null ? undefined : (msg) => { + const copied = { + ...msg, + resolve: (resolver: any) => resolver[msg.type]?.(msg) ?? [] + } + return options.inputTransferHandler!(copied as any); + } + }; + // @ts-expect-error + return setupAsClient, OutputMessageTypes, H>(type, variant, key, holder, extendOptions) as ClientCompoundMessenger; +} + +function setupAsClient( + type: MessengerTypeImpl, + variant: MessengerVariants, + key: InstanceKey, + holder: C, + options: MessengerOptions, +): ClientHandlerImpl { + return setupHolderAsImpl>(holder, type, variant, key, (impl) => clientMessageHandler(impl, options)); +} + +export function setupAsServerInternal( + type: MessengerType, + holder: ChannelHolder, + inputHandler: ServerInputHandler, +): ServerMessenger { + return setupHolderAsImpl(holder, type as MessengerTypeImpl, "server", undefined, (impl) => serverMessageHandler(impl, inputHandler)); +} + + +// #endregion + + +// #region Root Messenger Message types + +type ClientboundMessages = StartMessengerRootMessage | MessengerOutputMessage | MessengerOperationError | MessengerLogMessage | CloseMessengerRootMessage; +type ServerboundMessages = MessengerInputMessage | SetupMessengerPortMessage | CloseMessengerPortMessage | SetOffThreadKeyMessage | ForceCloseMessengerRootMessage; + +type ClientboundPortMessages = MessengerOperationError | MessengerOutputMessage | CloseMessengerPortMessage; + +interface MessengerTarget extends MessengerTypeTarget { + id: TypeId; + key: InstanceKey; +} + +interface MessengerTypeTarget { + id: TypeId; +} + +type ConversationId = string; + +interface ConversationTarget { + key: ConversationId; +} + +interface SetOffThreadKeyMessage extends Message<"set-key"> { + key: InstanceKey +} + +interface StartMessengerRootMessage extends Message<"setup-root">, MessengerTypeTarget { + transactionKey: string, +} + +interface SetupMessengerPortMessage extends Message<"setup-port">, MessengerTarget { + transactionKey: string, +} + +interface CloseMessengerRootMessage extends Message<"close-root">, MessengerTarget {} + +interface ForceCloseMessengerRootMessage extends Message<"force-close-root"> { + forceClose: boolean +} + +interface CloseMessengerPortMessage extends Message<"close-port">, MessengerTarget {} + +export interface MessengerInputMessage extends Message<"input"> { + key: string; + data: T; +} + +interface MessengerLogMessage extends Message<"logging"> { + logMsg: {type: "callback", data: LoggerCallbackData} | { type: "counter", data: CounterCallbackData }; +} + +interface MessengerOutputMessage extends Message<"output">, ConversationTarget { + data: T; +} + +interface MessengerOperationError extends Message<"operation-error">, ConversationTarget { + error: any; + info?: any; +} + +// #endregion + +function fullInstanceKeyFrom(messenger: Messenger) { + return fullInstanceKey(messenger.type.id, messenger.key); +} + +function fullInstanceKey(id: TypeId, key: InstanceKey | undefined) { + return `${id}:${key ?? "UNSET"}` +} + +function debugKeyFrom(messenger: Messenger) { + return debugKey(messenger.type.id, messenger.key, messenger.variant); +} + +function debugKey(id: TypeId, key: InstanceKey | undefined, variant: MessengerVariants) { + return `${id}:${key ?? "UNSET"}#${variant}` +} diff --git a/packages/ui/src/helpers/toolbox/messenger/index.ts b/packages/ui/src/helpers/toolbox/messenger/index.ts new file mode 100644 index 00000000000..af315acb1d7 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/messenger/index.ts @@ -0,0 +1,216 @@ +import { createRegistryInternal, setupAsServerInternal } from "#ui/helpers/toolbox/messenger/implement"; +import type { Logger, LoggerFactory } from "#ui/helpers/toolbox/misc/logging"; + +export type TypeId = string; + +/* @ts-ignore */ +export interface MessengerType { + id: TypeId; + registry: MessengerRegistry; + instanceConfig: InstanceConfig; + + close(forceClosed?: boolean): void; +} + +export interface MessengerAccessor { + getMessenger: (type: MessengerType) => (M extends ClientMessenger ? Promise : Promise) | undefined; + getActiveMessengers: () => Messenger[]; +} + +export interface MessengerRegistry extends MessengerAccessor { + compoundMessenger>(id: string, extra?: CompoundMessengerOptions & EntryOptions>): MessengerType>; + + simpleMessenger(id: string, extra?: MessengerOptions & EntryOptions>): MessengerType>; + + getType(id: string): MessengerType | undefined; + + setWorkerFactoryFor(type: MessengerType, factory: () => Worker): void; + + setLogFactory(factory: LoggerFactory): void; + getLogFactory(): LoggerFactory; + + closeAll(forceClosed?: boolean): void; +} + +export type EntryOptions = { closeCallback?: (messenger: M) => void} & InstanceConfig; + + +//-- + +export interface Message { type: K } + +// #region Compound Message Types + +export type MessageSchema = Record; + +export type MessageTypes = + T extends Record + ? { [E in K]: {type: E, input: T[E]["input"] /* & Message */ /* & O */, output: T[E]["output"] /* & Message */ /* & I */}; } + : never; + +export type InputMessageTypes> = P[Extract] extends { type: infer K extends string, input: infer I } ? I & Message : never; +export type OutputMessageTypes> = P[Extract] extends { type: infer K extends string, output: infer O } ? O & Message : never; +export type TypeMessageTypes> = P[Extract] extends { type: infer T } ? T : never; + +export type InputMessageFor, P = MessageTypes> = P[K] extends { input: infer I } ? I & Message : never; +export type OutputMessageFor, P = MessageTypes> = P[K] extends { output: infer O } ? O & Message : never; + +export type InputFor, P = MessageTypes> = P[K] extends { input: infer I } ? I : never; +export type OutputFor, P = MessageTypes> = P[K] extends { output: infer O } ? O : never; + +// #endregion + +// TODO: ATTEMPT TO ADD LOAD BALANCING FOR WHEN INSTANCES ARE SHARED AND COUNT IS GREATER THAN 1 +export interface InstanceConfig { + timeout?: number; + shared: boolean; + count: number; +} + +export interface InstanceData { + timeout?: number; + shared: boolean; +} + +export type InstanceTypes = "singleton-shared" | "temporary-exclusive" | "temporary-shared"; + +export type InstanceKey = string; + +export type MessengerVariants = "server" | "port" | "client"; + +export type ChannelHolder = Worker | MessagePort; + +export interface Messenger { + type: MessengerType; + key?: InstanceKey; + + instanceData: InstanceData; + variant: MessengerVariants; + + isAlive: boolean; + logger: Logger; +} + +export interface ServerMessenger extends Messenger, MessengerAccessor { + /** + * @experimental TBD if this should be exposed better + */ + sendOutputMessage(sender: Messenger, msg: Promise>): Promise; +} + +export interface ClientMessenger extends Messenger { + type: MessengerType>, + + sendMessage: (input: I) => Promise; + + handler: H; +} + + +// @ts-expect-error - No worries as we are stating the the input will always have a tied to output but TS can't seem to resolve it for some reason +export interface ClientCompoundMessenger> extends ClientMessenger, OutputMessageTypes, H>/* , CompoundInputHandler */ { + type: MessengerType>, + + sendMessage: , K extends Extract>(input: I_M) => Promise> + + handler: H; +} + +export type MessengerHandlerConstructor = (messenger: ClientMessenger) => H + +export type CompoundHandlerConstructor> = (messenger: ClientCompoundMessenger) => H + +//-- + +//-- + + +export type PossibleTransferable = T | [T, Transferable[]]; + +export type ClientInputUnpacker = (input: I) => Transferable[] | undefined; + + +export interface CompoundInputClientMessageExt, P = MessageTypes> extends Message { + resolve(handlers: InputResolverMap): Transferable[] | undefined; +} + +export type InputResolverMap> = { [E in Extract]?: ((msg: InputMessageFor) => Transferable[]) | undefined} + +export type CompoundInputClientMessage, P = MessageTypes> = CompoundInputClientMessageExt & InputMessageFor + +export type ClientCompoundInputUnpacker> + = , K extends Extract>(input: I_M) => Transferable[] | undefined; + +// -- + +export interface MessengerOptions { + handlerConstructor?: MessengerHandlerConstructor, + inputBypassHandler?: ServerInputHandler, + inputTransferHandler?: ClientInputUnpacker; + forcedOutputHandler?: (messenger: ClientMessenger, output: O) => boolean; +} + +// @ts-expect-error - No worries as we are stating the the input will always have a tied to output but TS can't seem to resolve it for some reason +export interface CompoundMessengerOptions> extends MessengerOptions, OutputMessageTypes, H> { + handlerConstructor?: CompoundHandlerConstructor + inputBypassHandler?: ServerCompoundInputHandler, + inputTransferHandler?: ClientCompoundInputUnpacker; + forcedOutputHandler?: (messenger: ClientCompoundMessenger, output: OutputMessageTypes) => boolean; +} + +export function createRegistry(): MessengerRegistry { + return createRegistryInternal(); +} + + +// WORKER SECTION + +export type ServerInputHandler = (messenger: ServerMessenger, input: I, sender?: Messenger) => Promise>; + + +export type ServerCompoundInputHandler> + = , K extends Extract>(messenger: ServerMessenger>, input: I_M, sender?: Messenger) => Promise>> + + +export interface CompoundInputServerMessageExt, P = MessageTypes> extends Message { + resolve(handlers: OutputResolverMap): Promise>; +} + +export type OutputResolverMap> = { [E in Extract]: (msg: InputMessageFor) => Promise>>} + +export type CompoundInputServerMessage, P = MessageTypes> = CompoundInputServerMessageExt & InputMessageFor; + + +export function setupAsCompoundServer>( + type: MessengerType, + holder: ChannelHolder, + inputHandler: ServerCompoundInputHandler +): ServerMessenger { + // @ts-expect-error + return setupAsServer, OutputMessageTypes>(type, holder, async (messenger, msg, sender) => { + const copiedMsg = { + ...msg, + resolve: (resolvers: any) => { + const result = (resolvers[msg.type] as ((msg: any) => any) | undefined)!(msg) as PossibleTransferable; + + return (Array.isArray(result)) ? [Object.assign(result[0], { type: msg.type }), result[1]] : Object.assign(result, { type: msg.type }) + } + } as any; + return inputHandler(messenger, copiedMsg, sender); + }); +} + +export function setupAsServer( + type: MessengerType, + holder: ChannelHolder, + inputHandler: ServerInputHandler, +): ServerMessenger { + return setupAsServerInternal(type, holder, inputHandler); +} + +export function isWorker() { + //@ts-expect-error TS2304: Cannot find name WorkerGlobalScope + return typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope; +} + +// -- diff --git a/packages/ui/src/helpers/toolbox/misc/index.ts b/packages/ui/src/helpers/toolbox/misc/index.ts new file mode 100644 index 00000000000..662bb298343 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/misc/index.ts @@ -0,0 +1,353 @@ +// #region Event Stream + +export class EventStream { + protected sinkFactory: (callbacks: T[]) => T; + protected subscribers: T[] = []; + public source: EventSource; + public sink: T; + private owner?: any; + + constructor(sinkFactory: (callbacks: T[]) => T, owner?: any) { + this.sinkFactory = sinkFactory; + this.sink = this.regenerateSink(); + this.source = new EventSource(this); + this.owner = owner; + } + + addSubscriber(subscriber: T): T { + const boundFunc = this.owner ? subscriber.bind(this.owner) : subscriber; + this.subscribers.push(boundFunc); + this.regenerateSink(); + return boundFunc; + } + + removeSubscriber(subscriber: T): void { + this.subscribers = this.subscribers.filter((entry) => entry != subscriber); + this.regenerateSink(); + } + + protected regenerateSink(): T { + const sink = this.sinkFactory(this.subscribers); + return this.sink = this.owner ? sink.bind(this.owner) : sink; + } + +} + +export class EventSource { + + stream: EventStream; + + constructor(stream: EventStream) { + this.stream = stream; + } + + public subscribe(subscriber: T): EventSubscription { + return new EventSubscription(this, this.stream.addSubscriber(subscriber)); + } +} + +export class EventSubscription { + protected source: EventSource; + protected subscriber: T; + + constructor(source: EventSource, subscriber: T) { + this.source = source; + this.subscriber = subscriber; + } + + public cancel(): void { + this.source.stream.removeSubscriber(this.subscriber); + } +} + +// #endregion + + +// #region Observable Types + +export type Consumer = (value: T) => void; + +export class Observable { + static of(startingValue: T) { + let value = startingValue; + return new Observable(() => value, (newValue) => value = newValue); + } + + static and(...observables: Observable[]){ + const evalFunc = () => observables.every((o) => o.get()); + + let value = evalFunc(); + const wrapper = new Observable(() => value); + + const onChange = (_: boolean) => { + const newValue = evalFunc(); + if (value != newValue) { + value = newValue; + wrapper.onChangeEvent.sink(wrapper.get()); + } + }; + observables.forEach((observable) => observable.onChange().subscribe(onChange)); + + return wrapper; + } + + static or(...observables: Observable[]){ + const evalFunc = () => observables.find((o) => o.get())?.get() ?? false; + + let value = evalFunc(); + const wrapper = new Observable(() => value); + + const onChange = (_: boolean) => { + const newValue = evalFunc(); + if (value != newValue) { + value = newValue; + wrapper.onChangeEvent.sink(wrapper.get()); + } + } + observables.forEach((observable) => observable.onChange().subscribe(onChange)); + + return wrapper; + } + + get: () => T; + set: (value: T) => void; + protected onChangeEvent: EventStream>; + + constructor (getCallback: () => T, setCallback?: (value: T) => T, runChangeCallbackOnSet?: boolean) { + this.get = getCallback; + this.set = (value) => { + if (setCallback != null) setCallback(value); + if (runChangeCallbackOnSet ?? true) this.onChangeEvent.sink(value); + }; + this.onChangeEvent = new EventStream>((callbacks) => (value: T) => callbacks.forEach((callback) => callback(value))); + } + + onChange(): EventSource> { + return this.onChangeEvent.source; + } + + ifNotNull(func: Consumer>) { + const value = this.get(); + if (value != null) func(value); + } + + mapTo(convertTo: (t: T) => R, convertFrom?: (value: R) => T): Observable { + const mappedObs = new Observable(() => convertTo(this.get()), convertFrom ? (value) => { + this.set(convertFrom(value)) + return value; + } : undefined); + + this.onChange().subscribe((value) => mappedObs.onChangeEvent.sink(convertTo(value))); + + return mappedObs; + } +} + +export class KeyedObservable extends Observable { + key: string; + + constructor (key: string, getCallback: (key: string) => T, setCallback?: (value: T) => T, runChangeCallbackOnSet?: boolean) { + super(() => getCallback(this.key), setCallback, runChangeCallbackOnSet); + this.key = key; + } +} + +// #endregion + + +// #region Misc Util Methods + +export function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export function areArraysEqual(a: readonly any[], b: readonly any[]) { + if (a === b) return true; + if (a == null || b == null || a.length !== b.length) return false; + + for (let i = 0; i < a.length; ++i) { + if (a[i] !== b[i]) return false; + } + + return true; +} + +// #endregion + + +// #region Lazy Values + +export interface Supplier { + get: () => Promise; +} + +export interface LazyValue extends Supplier { + isSet(): boolean; + reset(): void; + lazyMap(mapper: (value: T) => Promise): LazyValue; + map(mapper: (value: T) => Promise): Supplier +} + +export interface LazyMutableValue extends LazyValue { + updateGetter(getter: () => Promise): this; +} + +export function lazyMutableValue(getter: () => Promise): LazyMutableValue { + return Object.assign(lazyValue(getter), { + updateGetter(getter: () => Promise): LazyMutableValue { + this.reset(); + this._getter = getter; + return this; + }, + } as LazyMutableValueImpl) +} + +export function lazyValue(getter: () => Promise): LazyValue { + return { + _getter: getter, + _value: undefined, + _isSet: false, + + isSet(): boolean { + return this._isSet; + }, + + async get(): Promise { + if (!this._isSet) { + this._value = await this._getter(); + this._isSet = true; + } + return this._value!; + }, + + reset() { + this._value = undefined; + this._isSet = false; + this._onResetEvent.stream.sink(); + }, + + lazyMap(mapper: (value: T) => R): LazyValue { + const value = lazyValue(async () => await mapper(await getter())); + this._onResetEvent.subscribe(() => value.reset()); + return value; + }, + + _onResetEvent: new EventStream<() => void>((callbacks) => () => { + for (const callback of callbacks) { + callback(); + } + }).source + } as LazyValueImpl +} + +export interface LazyValueImpl extends LazyValue { + _value: T | undefined; + _isSet: boolean; + _getter: () => Promise; + _onResetEvent: EventSource<() => void>; +} + +export interface LazyMutableValueImpl extends LazyValueImpl, LazyMutableValue { +} + +// #endregion + +// #region Tree Node + +export interface TreeNodeBase> { + parent?: N; + children: Map; + name: string; + path: string; + + sort(compare: ((a: N, b: N) => number)): this; +} + +export class TreeNode> implements TreeNodeBase { + parent?: N; + children: Map = new Map(); + name: string; + path: string; + + constructor(parent: N | undefined, name: string, path: string) { + this.parent = parent; + this.name = name; + this.path = path; + } + + sort(compare: ((a: N, b: N) => number)): this { + this.children = new Map(Array.from(this.children.entries()).sort((entry1, entry2) => compare(entry1[1], entry2[1]))); + for (const value of this.children.values()) value.sort(compare); + return this; + } +} + +// #endregion + +export function wrapMap( + source: Map, + allowedKeys: Iterable, + mapper: (value: VIn, key: K) => VOut +): ReadonlyMap { + return new MappedFilteredReadonlyMap(source, allowedKeys, mapper); +} + +export class MappedFilteredReadonlyMap implements ReadonlyMap { + private readonly allowedKeys: Set; + + constructor( + private readonly sourceMap: ReadonlyMap, + allowedKeys: Iterable, + private readonly mapper: (value: VIn, key: K) => VOut + ) { + this.allowedKeys = new Set(allowedKeys); + } + + has(key: K): boolean { + return this.allowedKeys.has(key) && this.sourceMap.has(key); + } + + get(key: K): VOut | undefined { + if (!this.has(key)) return undefined; + const rawVal = this.sourceMap.get(key); + return rawVal !== undefined ? this.mapper(rawVal, key) : undefined; + } + + get size(): number { + let count = 0; + for (const key of this.allowedKeys) { + if (this.sourceMap.has(key)) count++; + } + return count; + } + + *entries(): MapIterator<[K, VOut]> { + for (const [key, value] of this.sourceMap.entries()) { + if (this.allowedKeys.has(key)) { + yield [key, this.mapper(value, key)]; + } + } + } + + *keys(): MapIterator { + for (const key of this.allowedKeys) { + if (this.sourceMap.has(key)) yield key; + } + } + + *values(): MapIterator { + for (const [_, value] of this.entries()) { + yield value; + } + } + + forEach(callbackfn: (value: VOut, key: K, map: ReadonlyMap) => void, thisArg?: any): void { + for (const [key, value] of this.entries()) { + callbackfn.call(thisArg, value, key, this as ReadonlyMap); + } + } + + [Symbol.iterator](): MapIterator<[K, VOut]> { + return this.entries(); + } +} diff --git a/packages/ui/src/helpers/toolbox/misc/logging.ts b/packages/ui/src/helpers/toolbox/misc/logging.ts new file mode 100644 index 00000000000..edd672253e7 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/misc/logging.ts @@ -0,0 +1,313 @@ +import type { EventSource } from '#ui/helpers/toolbox/misc'; +import { EventStream } from '#ui/helpers/toolbox/misc'; + +export type LogLevel = "debug" | "success" | "info" | "error" | "warning" | "neutral"; + +export function consoleOutFor(type: LogLevel) { + if (type == 'error') return console.error.bind(console); + else if (type == 'warning') return console.warn.bind(console); + else if (type == 'debug') return console.debug.bind(console); + else return console.info.bind(console); +} + +export type ExtraData = {err?: Error, silence?: boolean, obj?: any | Array, ext?: any} + +export type LoggerCallbackData = [level: LogLevel, loggerName: string, message : string, extra?: ExtraData]; +export type CounterCallbackData = [level: 'info' | 'debug', loggerName: string, message: {id: string, size?: number, amount?: number}]; + +export type LoggerCallback = (this: LoggerFactory, ...data: LoggerCallbackData) => void | Promise | boolean; +export type CounterCallback = (this: LoggerFactory, ...data: CounterCallbackData) => void | Promise | boolean; + +export type LoggerEntryFormatter = (this: LoggerFactory, ...data: LoggerCallbackData) => string[]; + +export interface Counter { + startCounting(id: string, size: number): void; + incrementCount(id: string, amount: number): void; +} + +export interface Timer { + startTimer(id: string): void; + + endTimer(id: string): {start: Date, end: Date, totalTime: number} | undefined; +} + +export function createTimer(): Timer { + const map = new Map(); + + return { + startTimer: (id) => { + map.delete(id); + + const startDate = new Date(Date.now()); + + map.set(id, startDate); + }, + endTimer: (id) => { + const startDate = map.get(id); + + if (!startDate) return undefined; + + const endDate = new Date(Date.now()); + + return { + start: startDate, + end: endDate, + totalTime: endDate.getTime() - startDate.getTime() + } + } + } +} + +export function createCounter(): Counter { + const map = new Map(); + return { + startCounting(id, size) { + map.set(id, { count: 0, size: size }) + }, + incrementCount(id, amount) { + const state = map.get(id); + if (state) { + state.count += amount; + } + } + } +} + +export interface LoggerFactory { + log: LoggerCallback; + formatter: LoggerEntryFormatter; + counter: CounterCallback; + + onLog: EventSource, + onCounter: EventSource, + + loggerFor(name: string): Logger; + + startTimer(loggerName: string, id: string): void; + + endTimer(loggerName: string, id: string): {start: Date, end: Date, totalTime: number} | undefined; + + startCounting(loggerName: string, id: string, size: number): void; + + incrementCount(loggerName: string, id: string, amount: number): void; + + info(loggerName: string, message: string, extra?: ExtraData): void; + + success(loggerName: string, message: string, extra?: ExtraData): void; + + warn(loggerName: string, message: string, extra?: ExtraData): void; + + error(loggerName: string, message: string, extra?: ExtraData): void; + + debug(loggerName: string, message: string, extra?: ExtraData): void; +} + +export class LoggerFactoryDelegate implements LoggerFactory { + factory: LoggerFactory; + + constructor(factory: LoggerFactory) { + this.log = factory.log; + this.formatter = factory.formatter; + this.counter = factory.counter; + this.onLog = factory.onLog; + this.onCounter = factory.onCounter; + this.loggerFor = factory.loggerFor; + this.startTimer = factory.startTimer; + this.startCounting = factory.startCounting; + this.incrementCount = factory.incrementCount; + this.endTimer = factory.endTimer; + this.info = factory.info; + this.success = factory.success; + this.warn = factory.warn; + this.error = factory.error; + this.debug = factory.debug; + this.factory = factory; + } + + log: LoggerCallback; + + formatter: LoggerEntryFormatter; + + counter: CounterCallback; + + onLog: EventSource; + + onCounter: EventSource; + + loggerFor: (name: string) => Logger; + + startTimer: (loggerName: string, id: string) => void; + + endTimer: (loggerName: string, id: string) => {start: Date, end: Date, totalTime: number} | undefined; + + startCounting: (loggerName: string, id: string, size: number) => void; + + incrementCount: (loggerName: string, id: string, amount: number) => void; + + info: (loggerName: string, message: string, extra?: ExtraData) => void; + + success: (loggerName: string, message: string, extra?: ExtraData) => void; + + warn: (loggerName: string, message: string, extra?: ExtraData) => void; + + error: (loggerName: string, message: string, extra?: ExtraData) => void; + + debug: (loggerName: string, message: string, extra?: ExtraData) => void; +} + +const defaultCallback: LoggerCallback = function (level, loggerName, message, extra) { + const console = consoleOutFor(level); + const entries: Array = this.formatter(level, loggerName, message, extra); + if (extra?.err) entries.push(extra.err); + console(...entries); + const debugInfo = extra?.obj; + if (!debugInfo) return; + // TODO: MAYBE USE SPREAD INSTEAD? + if (Array.isArray(debugInfo)) { + debugInfo.forEach(entry => console(entry)) + } else { + console(debugInfo) + } +} + +const defaultCounter: CounterCallback = function (level, loggerName, message) { + const key = `[${loggerName}]: ${message.id}` + if (message.size) { + consoleOutFor(level)(`[${loggerName}]: ${message.id} | Size ${message.size} `) + console.countReset(key); + } else if (message.amount) { + for (let i = 0; i < message.amount; i++) { + console.count(key) + } + } + +} + +const defaultFormatter: LoggerEntryFormatter = (_, loggerName, message) => [`[${loggerName}]:`, `${message}`]; + +export type FactoryOptions = { + log?: LoggerCallback, formatter?: LoggerEntryFormatter, counter?: CounterCallback +} + +// TODO: ADD ABILITY TO ADJUST WHAT LEVELS OR MESSAGES ARE LOGGED VIA EVENT? +export function logFactory({log, formatter, counter}: FactoryOptions = {}): LoggerFactory { + const map = new Map>(); + const factory: LoggerFactory = { + formatter: formatter ?? defaultFormatter, + info(loggerName: string, message: string, extra: ExtraData) { + this.log('info', loggerName, message, extra) + }, + success(loggerName: string, message: string, extra: ExtraData) { + this.log('success', loggerName, message, extra) + }, + warn(loggerName: string, message: string, extra: ExtraData) { + this.log('warning', loggerName, message, extra) + }, + error(loggerName: string, message: string, extra: ExtraData) { + this.log('error', loggerName, message, extra) + }, + debug(loggerName: string, message: string, extra: ExtraData) { + this.log('debug', loggerName, message,extra) + }, + startTimer(loggerName: string, id: string) { + const loggerEntries = (map.has(loggerName) ? map : map.set(loggerName, new Map())) + .get(loggerName)!; + + loggerEntries.delete(id); + + const startDate = new Date(Date.now()); + + loggerEntries.set(id, startDate); + }, + endTimer(loggerName: string, id: string) { + const loggerEntries = (map.has(loggerName) ? map : map.set(loggerName, new Map())) + .get(loggerName)!; + + const startDate = loggerEntries.get(id); + + if (!startDate) return undefined; + + const endDate = new Date(Date.now()); + + const timeData = { + start: startDate, + end: endDate, + totalTime: endDate.getTime() - startDate.getTime() + }; + + this.debug(loggerName, `'${id}' took ${timeData.totalTime} ms to complete!`, { obj: timeData }) + + return timeData; + }, + startCounting(loggerName: string, id: string, size: number) { + this.counter('info', loggerName, {id: id, size}) + }, + incrementCount(loggerName: string, id: string, amount: number) { + this.counter('info', loggerName, {id: id, amount}) + }, + loggerFor(name: string): Logger { + return { + factory: this, + name, + startTimer(id: string) { this.factory.startTimer(this.name, id); }, + endTimer(id: string) { return this.factory.endTimer(this.name, id); }, + startCounting(id: string, size: number) { return this.factory.startCounting(this.name, id, size); }, + incrementCount(id: string, amount: number) { return this.factory.incrementCount(this.name, id, amount); }, + log(level, message, extra) { this.factory.log(level, name, message, extra ?? {}) }, + info(data, extra) { this.factory.info(this.name, data, extra) }, + success(data, extra) { this.factory.success(this.name, data, extra) }, + warn(data, extra) { this.factory.warn(this.name, data, extra) }, + error(data, extra) { this.factory.error(this.name, data, extra) }, + debug(data, extra) { this.factory.debug(this.name, data, extra) }, + } + } + } as LoggerFactory; + + const onLogEvent = new EventStream((callbacks) => { + const boundCallbacks = callbacks.map((callback) => callback.bind(factory)); + return (level, loggerName, message, extra) => { + for (const callback of boundCallbacks) callback.bind(factory)(level, loggerName, message, extra); + } + }, factory); + + const finalCallback = (log ?? defaultCallback).bind(factory); + factory.log = (level, loggerName, message, extra) => { + finalCallback(level, loggerName, message, extra); + // @ts-expect-error The 'this' context of type 'void' is not assignable to method's 'this' of type 'LoggerFactory'. + onLogEvent.sink(level, loggerName, message, extra); + } + + const onCounterEvent = new EventStream((callbacks) => { + const boundCallbacks = callbacks.map((callback) => callback.bind(factory)); + return (level, loggerName, message) => { + for (const callback of boundCallbacks) callback.bind(factory)(level, loggerName, message); + } + }, factory); + + const finalCounter = (counter ?? defaultCounter).bind(factory); + + factory.onCounter = onCounterEvent.source; + factory.counter = (level, loggerName, message) => { + finalCounter(level, loggerName, message); + // @ts-expect-error The 'this' context of type 'void' is not assignable to method's 'this' of type 'LoggerFactory'. + onCounterEvent.sink(level, loggerName, message); + } + return factory; +} + +export interface Logger extends Timer, Counter { + factory: LoggerFactory, + name: string, + + log(level: LogLevel, message: string, extra?: ExtraData): void; + + info(message: string, extra?: ExtraData): void; + + success(message: string, extra?: ExtraData): void; + + warn(message: string, extra?: ExtraData): void; + + error(message: string, extra?: ExtraData): void; + + debug(message: string, extra?: ExtraData): void; +} diff --git a/packages/ui/src/helpers/toolbox/workers/decompile/index.ts b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts new file mode 100644 index 00000000000..274cc93e403 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts @@ -0,0 +1,8 @@ +import type {DecompileInput, DecompilerResult} from "#ui/helpers/toolbox/decompile"; +import { decompileFileContent } from "#ui/helpers/toolbox/decompile/impl"; +import { setupAsServer } from "#ui/helpers/toolbox/messenger"; +import {decompileMessengerType, zipHandlerType} from "#ui/helpers/toolbox/workers"; + +setupAsServer(decompileMessengerType, self as unknown as Worker, async (messenger, msg) => { + return decompileFileContent(messenger.logger, (await messenger.getMessenger(zipHandlerType))?.handler, msg); +}) diff --git a/packages/ui/src/helpers/toolbox/workers/index.ts b/packages/ui/src/helpers/toolbox/workers/index.ts new file mode 100644 index 00000000000..f39a4e10087 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/workers/index.ts @@ -0,0 +1,81 @@ +import { useRouter } from "vue-router"; + +import type {DecompileHandler, DecompileInput, DecompilerResult} from "#ui/helpers/toolbox/decompile"; +import { + type ClientCompoundMessenger, type ClientMessenger, + createRegistry, + isWorker, + type MessengerType +} from '#ui/helpers/toolbox/messenger'; +// import { AtlasMessenger, handlerConstructor as atlasHandlerConstructor } from "./atlas/common"; +import { handlerConstructor as zipHandlerConstructor,type ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; +import type { DelegateFileZipTreeNode, ZipHandler } from "#ui/helpers/toolbox/zip"; + +import DecompileWorker from "./decompile?worker"; +import ZipWorker from "./zip?worker"; + +export const messengerRegistry = createRegistry(); + +export type ZipHandlerMessenger = ClientCompoundMessenger; +export const zipHandlerType: MessengerType = messengerRegistry.compoundMessenger("zip-handler", { + handlerConstructor: zipHandlerConstructor, + inputBypassHandler: import.meta.env.DEV ? (await import('./zip')).getMessageHandler() : undefined, + inputTransferHandler: ((input) => { + return input.resolve({ + node_write: (msg) => [msg.buffer], + read_zip_tree: (msg) => [msg.buffer], + }); + }), + forcedOutputHandler: (messenger, msg) => { + if (msg.type == "update_node_data") { + const data = msg as ZipMessages['update_node_data']['output']; + + const root = messenger.handler.zipArchivesByHash.get(data.root); + + if (root != null) { + const node = (data.relative ? root.relativeFileEntries : root.fileEntries).get(data.path); + + if (node) { + for (const lazy of [node.sha1, node.sha256, node.sha512, node.fileTypeInfo]) { + lazy.reset() + } + (node as unknown as Partial).onDataUpdate?.() + } + } + } + return false; + }, + closeCallback: (messenger) => { + return messenger.handler.closeAllZipTrees(); + }, + shared: true, + count: 1, +}); + +// export const atlasMessengerType: MessengerType = messengerRegistry.simpleMessenger("atlas-builder", { +// handlerConstructor: atlasHandlerConstructor, +// inputBypassHandler: import.meta.env.DEV ? (await import('./atlas/implementation')).convertArchiveFilesToAtlas : undefined, +// instanceType: "temporary-exclusive" +// }); +// + +type DecompileMessenger = ClientMessenger; +export const decompileMessengerType: MessengerType = messengerRegistry.simpleMessenger("decompiler-vf", { + inputBypassHandler: import.meta.env.DEV ? async (messenger, input) => { + return (await import('../decompile/impl.ts')).decompileFileContent(messenger.logger, (await messengerRegistry.getMessenger(zipHandlerType)), input) + } : undefined, + handlerConstructor: (messenger) => messenger.sendMessage, + inputTransferHandler: (file) => [file.targetFile.content], + shared: true, + timeout: 30000, + count: 1 +}); +messengerRegistry.setWorkerFactoryFor(decompileMessengerType, () => new DecompileWorker()); +messengerRegistry.setWorkerFactoryFor(zipHandlerType, () => new ZipWorker()); + +if (!isWorker()) { + // TODO: IMPLEMENT CORRECT CLEANUP I.E. WHEN PROPERLY CHANGING ROOT PAGES? + useRouter().beforeEach((to, from, next) => { + if (to.params.project != from.params.project) messengerRegistry.closeAll(true); + }) +} diff --git a/packages/ui/src/helpers/toolbox/workers/zip/api.ts b/packages/ui/src/helpers/toolbox/workers/zip/api.ts new file mode 100644 index 00000000000..f1941520026 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/workers/zip/api.ts @@ -0,0 +1,176 @@ +import type { ZipWriterAddDataOptions } from "@zip.js/zip.js"; + +import type { DecompilerResult } from "#ui/helpers/toolbox/decompile"; +import type { FileDetails, HashedFileContent } from "#ui/helpers/toolbox/file"; +import type { ZipHandlerMessenger } from "#ui/helpers/toolbox/workers"; +import { + type FileMatcher, type FileZipTreeNode, getAsTarget, type LazyDataAccess, type NodeTarget, + reconstituteRoot, type RootNodeTarget, type RootZipTreeNode, type Sha512, type SourceOptions, + type ZipHandler +} from "#ui/helpers/toolbox/zip"; + +export interface DataInput { + buffer: ArrayBuffer +} + +export type ZipMessages = { + lazy_read: { + input: { key: keyof LazyDataAccess } & NodeTarget, + output: { value: Awaited> } + }, + source_read: { + input: NodeTarget & { options?: SourceOptions } + output: DecompilerResult + } + node_read: { + input: NodeTarget, + output: { buffer: ArrayBuffer } + }, + node_write: { + input: NodeTarget & { nodeType?: "zip" | "file", options?: ZipWriterAddDataOptions } & DataInput, + output: { wasWritten: boolean } + }, + read_zip_tree: { + input: { name?: string, url?: string, hash?: string } & DataInput, + output: { node: RootZipTreeNode } + }, + get_zip_tree: { + input: { value: string | Sha512 }, + output: { node: RootZipTreeNode | undefined } + }, + collect_files: { + input: RootNodeTarget & { matcher?: FileMatcher }, + output: { files: FileDetails[] } + }, + collect_files_hashed: { + input: RootNodeTarget & { matcher?: FileMatcher, deduplicate?: boolean}, + output: { files: HashedFileContent[] } + }, + close_zip_tree: { + input: RootNodeTarget, + output: { err?: Error, wasClosed: boolean } + }, + close_all_zip_trees: { + input: {}, + output: { err?: Error, wasClosed: boolean } + }, + update_node_data: { + input: NodeTarget, + output: NodeTarget + } +} + +export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler = (messenger: ZipHandlerMessenger) => { + const hashToUrls = new Map(); + return { + zipArchivesByHash: new Map(), + urlToHash: new Map(), + readLazyData: async function (target: FileZipTreeNode | NodeTarget, key: K): Promise>> { + return (await messenger.sendMessage({ type: "lazy_read", ...getAsTarget(target), key: key })).value as Awaited>; + }, + readSource: async function (target: FileZipTreeNode | NodeTarget, options?: SourceOptions): Promise { + return (await messenger.sendMessage({ type: 'source_read', ...getAsTarget(target), options: options})) + }, + readNodeData: async (target: FileZipTreeNode | NodeTarget) => { + return (await messenger.sendMessage({ type: "node_read", ...getAsTarget(target)})).buffer + }, + getOrReadZipTree: async function (buffer, {name, url, hash} = {}) { + let node: RootZipTreeNode | undefined; + if (url) node = await this.getZipTree(url, true); + if (node == null && hash) node = await this.getZipTree(hash, false); + const data = await buffer(); + return data ? this.readZipTree(data, {name: name, url: url, hash: hash}) : undefined; + }, + readZipTree: async function (buffer, {name, url, hash} = {}) { + const filteredURL = url?.split("?")[0]; + + if (filteredURL) { + const node = await this.getZipTree(filteredURL, true, true); + if (node) return node; + } + + if (hash) { + const node = await this.getZipTree(hash, false, true); + if (node) return node; + } + + const node = reconstituteRoot(messenger.handler, (await messenger.sendMessage({ type: "read_zip_tree", buffer, name: name, url: url, hash: hash })).node); + + if (filteredURL) { + this.urlToHash.set(filteredURL, node.key); + const urls = (hashToUrls.has(node.key)) + ? hashToUrls.get(node.key)! + : (() => { const array = new Array(); hashToUrls.set(node.key, array); return array; })(); + + urls.push(filteredURL); + } + + this.zipArchivesByHash.set(node.key, node); + + return node; + }, + writeNodeData: async (target, buffer, type, options) => { + return (await messenger.sendMessage({type: "node_write", ...getAsTarget(target), buffer, nodeType: type, options})).wasWritten; + }, + collectFiles: async (root, matcher) => { + return (await messenger.sendMessage({ type: "collect_files", matcher: matcher, root: (root as RootZipTreeNode).key ?? root})).files + }, + collectHashedFiles: async (root, matcher, deduplicate)=> { + return (await messenger.sendMessage({ type: "collect_files_hashed", matcher: matcher, root: (root as RootZipTreeNode).key ?? root, deduplicate: deduplicate})).files + }, + closeZipTree: async function (root){ + const hash: Sha512 = (root as RootZipTreeNode).key ?? root; + const value = await messenger.sendMessage({ type: "close_zip_tree", root: (root as RootZipTreeNode).key ?? root}); + + const urls = hashToUrls.get(hash); + if (urls) for (const url of urls) this.urlToHash.delete(url); + hashToUrls.delete(hash); + + return value; + }, + closeAllZipTrees: async function () { + const value = await messenger.sendMessage({ type: "close_all_zip_trees" }); + + this.zipArchivesByHash.clear(); + this.urlToHash.clear(); + hashToUrls.clear(); + + return value; + }, + getZipTree: async function (value, isUrl, localOnly) { + if (this.zipArchivesByHash.has(value)) { + return this.zipArchivesByHash.get(value); + } + + if (isUrl) { + const filteredURL = value.split("?")[0]; + if (this.urlToHash.has(filteredURL)) { + const hash = this.urlToHash.get(value)!; + const node = this.zipArchivesByHash.get(hash); + + if (node) return node; + } + } + + if (localOnly) return undefined; + + const rawNode = (await messenger.sendMessage({ type: "get_zip_tree", value: value })).node; + + if (rawNode == undefined) return undefined; + + const node = reconstituteRoot(messenger.handler, rawNode); + + if (isUrl) { + const filteredURL = value.split("?")[0]; + this.urlToHash.set(filteredURL, node.key); + if (hashToUrls.has(node.key)) hashToUrls.set(node.key, []) + hashToUrls.get(node.key)?.push(filteredURL); + } + + this.zipArchivesByHash.set(node.key, node); + + return node; + } + } +}; + diff --git a/packages/ui/src/helpers/toolbox/workers/zip/index.ts b/packages/ui/src/helpers/toolbox/workers/zip/index.ts new file mode 100644 index 00000000000..fe43ba49b96 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/workers/zip/index.ts @@ -0,0 +1,319 @@ + +import { + type FileContentPromise, type FileDetails, type FileTypeInfo, + type HashedFileContent, resolveContent +} from "#ui/helpers/toolbox/file"; +import { type MessageTypes, type ServerCompoundInputHandler, setupAsCompoundServer } from "#ui/helpers/toolbox/messenger"; +import { zipHandlerType } from "#ui/helpers/toolbox/workers"; +import type { ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; +import type { + LazyDataAccess, + NodeTarget, + RootZipTreeNode, + Sha512, + StringMatcher, + ZipTreeNode } from "#ui/helpers/toolbox/zip"; +import { readZipTree } from "#ui/helpers/toolbox/zip/impl"; + +const zipArchivesByHash = new Map(); + +const urlToHash = new Map(); +const hashToUrls = new Map(); + +function getNodeFrom(target: NodeTarget) { + if (!zipArchivesByHash || zipArchivesByHash.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); + + const root = zipArchivesByHash.get(target.root); + if (root == undefined) throw new Error(`Unable to get node '${target.root}:${target.path}' as their is no active root with the hash given namespace hash!`); + + const node = (target.relative ? root.relativeFileEntries : root.fileEntries).get(target.path); + if (node == undefined) throw new Error(`Unable to get node '${target.root}:${target.path}' as their is no active node with the given path!`); + + return node; +} + +function getRootFrom(target: Sha512) { + if (!zipArchivesByHash || zipArchivesByHash.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); + + const root = zipArchivesByHash.get(target); + if (root == undefined) throw new Error(`Unable to get root '${target}' as their is no active root with the hash given hash!`); + + return root; +} + +function getZipNode(value: string) { + if (zipArchivesByHash.has(value)) { + return zipArchivesByHash.get(value); + } + + if (urlToHash.has(value)) { + const hash = urlToHash.get(value)!; + const node = zipArchivesByHash.get(hash); + + if (node) return node; + } + + return undefined; +} + +export function getMessageHandler

>(): ServerCompoundInputHandler { + return messageHandler as unknown as ServerCompoundInputHandler; +} + +async function isValidFor(path: string, fileType: () => Promise, pathMatcher?: StringMatcher, mimeMatcher?: StringMatcher) { + if (pathMatcher) { + const negate = pathMatcher.negate; + if (pathMatcher.regex && (negate ? path.match(pathMatcher.regex) : !path.match(pathMatcher.regex))) return false; + if (pathMatcher.startsWith && (negate ? path.startsWith(pathMatcher.startsWith) : !path.startsWith(pathMatcher.startsWith))) return false; + if (pathMatcher.includes && (negate ? path.includes(pathMatcher.includes) : !path.startsWith(pathMatcher.includes))) return false; + if (pathMatcher.endsWith && (negate ? path.endsWith(pathMatcher.endsWith) : !path.endsWith(pathMatcher.endsWith))) return false; + } + + if (mimeMatcher) { + const info = await fileType(); + if (info == null) return false; + const mime = info.mime; + const negate = mimeMatcher.negate; + if (mimeMatcher.regex && (negate ? mime.match(mimeMatcher.regex) : !mime.match(mimeMatcher.regex))) return false; + if (mimeMatcher.startsWith && (negate ? mime.startsWith(mimeMatcher.startsWith) : !mime.startsWith(mimeMatcher.startsWith))) return false; + if (mimeMatcher.includes && (negate ? mime.includes(mimeMatcher.includes) : !mime.includes(mimeMatcher.includes))) return false; + if (mimeMatcher.endsWith && (negate ? mime.endsWith(mimeMatcher.endsWith) : !mime.endsWith(mimeMatcher.endsWith))) return false; + } + + return true; +} + +export const messageHandler: ServerCompoundInputHandler = function (messenger, msg, sender) { + return msg.resolve({ + lazy_read: async (msg) => { + const node = getNodeFrom(msg); + + return { + key: msg.key, + value: await (node as LazyDataAccess)[msg.key].get() + }; + }, + source_read: (msg) => { + return getNodeFrom(msg).asText(msg.options); + }, + node_read: async (msg) => { + const node = getNodeFrom(msg); + + const buffer = await node.getBuffer(); + return { + buffer: buffer + }; + }, + node_write: async (msg) => { + const node = getNodeFrom(msg); + await node.setBuffer(async () => msg.buffer, msg.nodeType, msg.options); + return { + wasWritten: true + }; + }, + read_zip_tree: async (msg) => { + let tree: RootZipTreeNode | undefined = undefined; + + if (msg.hash) tree = getZipNode(msg.hash); + if (tree == null && msg.url) tree = getZipNode(msg.url); + if (tree == null) { + tree = await readZipTree(messenger.logger, msg.buffer, { name: msg.name, logger: messenger.logger }); + + zipArchivesByHash.set(tree.key, tree); + + if (msg.url) { + urlToHash.set(msg.url, tree.key); + + const urls = (hashToUrls.has(tree.key)) + ? hashToUrls.get(tree.key)! + : (() => { const array = new Array(); hashToUrls.set(tree.key, array); return array; })(); + + urls.push(msg.url); + } + } + + const copiedTree = copy(tree); + + return { + node: copiedTree + }; + }, + get_zip_tree: async (msg) => { + const node = getZipNode(msg.value); + const copiedTree = node ? copy(node) : undefined; + + return { + node: copiedTree + }; + }, + collect_files: async (msg) => { + const root = getRootFrom(msg.root); + + const mimeMatcher = msg.matcher?.mime; + const pathMatcher = msg.matcher?.path; + + const entries = root.fileEntries; + + if (mimeMatcher || pathMatcher) { + const files: FileDetails[] = []; + let index = 0; + const size = entries.size; + + const promises: Promise[] = []; + + for (const file of entries.values()) { + // TODO: ADD ABILITY TO TELL WORKERS TO NOT DEBUG LOG THOUGH A MESSAGE? + messenger.logger.debug(`[File Collector]: Looking at ${index}/${size} '${file.path}'`); + + index += 1; + + promises.push( + isValidFor(file.path, async () => file.fileTypeInfo.get(), pathMatcher, mimeMatcher) + .then((value) => { if (value) files.push({ fullPath: file.path, name: file.name }); }) + ) + } + + await Promise.all(promises); + + return { + files: files, + }; + } else { + return { + files: entries.values().map(entry => { return { fullPath: entry.path, name: entry.name }; }).toArray(), + }; + } + }, + collect_files_hashed: async (msg) => { + const root = getRootFrom(msg.root); + + const mimeMatcher = msg.matcher?.mime; + const pathMatcher = msg.matcher?.path; + + const entries = root.fileEntries; + + const hashes = new Set(); + + let files: HashedFileContent[]; + + if (mimeMatcher || pathMatcher || msg.deduplicate) { + files = []; + let index = 0; + const baseFiles: (FileContentPromise & { sha512: () => Promise }) [] = []; + + const filterPromises: Promise[] = []; + + for (const file of entries.values()) { + // TODO: ADD ABILITY TO TELL WORKERS TO NOT DEBUG LOG THOUGH A MESSAGE? + //this.logger().debug(`[File Collector]: Looking at ${index}/${size} '${file.path}'`); + + index += 1; + + filterPromises.push( + isValidFor(file.path, async () => file.fileTypeInfo.get(), pathMatcher, mimeMatcher) + .then((value) => { + if (value) baseFiles.push({ fullPath: file.path, name: file.name, content: () => file.getBuffer(), sha512: () => file.sha512.get()}); + }) + ) + } + + await Promise.all(filterPromises); + + const baseFileContent = await Promise.all(baseFiles.map((filePromise) => resolveContent(filePromise))); + + const duplicateCheckPromises: Promise[] = []; + + async function isDuplicateFile(file: { sha512: () => Promise}) { + const sha512 = await file.sha512(); + + if (msg.deduplicate) { + if (hashes.has(sha512)) return { result: true, sha512: sha512 }; + + hashes.add(sha512) + } + + return {result: false, sha512: sha512 }; + } + + for (const file of baseFileContent) { + duplicateCheckPromises.push( + isDuplicateFile(file) + .then((value) => {if (!value.result) files.push({ ...file, sha512: value.sha512 });}) + ) + } + + if (duplicateCheckPromises.length > 0) await Promise.all(duplicateCheckPromises); + } else { + files = await Promise.all( + entries.values().map(async file => { + const buffer = await file.getBuffer(); + const sha512 = await file.sha512.get(); + + return { fullPath: file.path, name: file.name, content: buffer, sha512: sha512}; + }).toArray() + ); + } + + return { files: files }; + }, + close_zip_tree: async (msg) => { + zipArchivesByHash.delete(msg.root); + + const urls = hashToUrls.get(msg.root); + if (urls) for (const url of urls) urlToHash.delete(url); + hashToUrls.delete(msg.root); + + return { + err: undefined, + wasClosed: true, + } + }, + close_all_zip_trees: async () => { + zipArchivesByHash.clear(); + urlToHash.clear(); + hashToUrls.clear(); + + return { + err: undefined, + wasClosed: true, + } + }, + update_node_data: async (target) => { + if (sender != null) { + const connectedMessengers = messenger.getActiveMessengers(); + + for (const connectedMessenger of connectedMessengers) { + messenger.sendOutputMessage(connectedMessenger, Promise.resolve(target)) + } + } + + return target; + } + }) +}; + +export function copy(node: RootZipTreeNode): RootZipTreeNode { + const cloneNode = { children: new Map(), key: node.key } as RootZipTreeNode; + return copyNodeData(cloneNode, node); +} + +function copyNodeData(clone: N, oldNode: N): N { + for (const entry of oldNode.children.values()) { + // Required to strip the functions that can not cross the message barrier + const cloneMetaData = { + ...entry.metadata, + getData: undefined, + arrayBuffer: undefined, + }; + + const entryClone = { name: entry.name, path: entry.path, type: entry.type, metadata: cloneMetaData, children: new Map() } as any as ZipTreeNode + + clone.children.set(entry.name, entryClone) + + copyNodeData(entryClone, entry); + } + + return clone; +} + +setupAsCompoundServer(zipHandlerType, self as unknown as Worker, getMessageHandler()); diff --git a/packages/ui/src/helpers/toolbox/zip/impl.ts b/packages/ui/src/helpers/toolbox/zip/impl.ts new file mode 100644 index 00000000000..0b0a2ba45dc --- /dev/null +++ b/packages/ui/src/helpers/toolbox/zip/impl.ts @@ -0,0 +1,314 @@ +import { BlobReader, type EntryMetaData, type FileEntry, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, type ZipWriterAddDataOptions } from "@zip.js/zip.js"; + +import { + type DecompilerResult, DEFAULT_DECOMPILE_OPTIONS +} from "#ui/helpers/toolbox/decompile"; +import { type FileTypeInfo, getFileTypeInfo, hashBuffer } from "#ui/helpers/toolbox/file"; +import { +type LazyMutableValue, lazyMutableValue, + type LazyMutableValueImpl, + type LazyValue} from '#ui/helpers/toolbox/misc'; +import type { Logger } from '#ui/helpers/toolbox/misc/logging'; +import {decompileMessengerType, messengerRegistry} from "#ui/helpers/toolbox/workers"; +import { + type ArrayBufferSource, + type EntryData, + fileOrNull, + type FileZipTreeNode, + RootZipTreeNode, + type Sha512, type SourceOptions, + type TypedArrayBuffer, + type TypedBlob, + type ZipFileContent, + type ZipNodeTypes, + ZipTreeNode +} from "#ui/helpers/toolbox/zip"; + +/** + * Method used to generate the zip tree of the given blob returning the root node of the entire zip file + */ +export async function readZipTree(logger: Logger, buffer: ArrayBuffer, options: ReadExtraOptions = {}): Promise { + const root = new RootZipTreeNodeImpl(logger, await hashBuffer(buffer, "SHA-512")); + return readZipDataToNode(root, root, buffer, options); +} + +export async function writeZipTree(node: ZipTreeNodeImpl): Promise { + const zip = new ZipWriter(new Uint8ArrayWriter()); + + if (node.type == "directory") { + const children = node.children + for (const node of Object.values(children)) { + const blob = await node.blob(); + const reader = blob != null ? new BlobReader(blob) : null; + if (reader) await zip.add(node.path, reader, node.metadata); + } + } + + return (await zip.close()).buffer; +} + +const zipArchiveExtensionRegex = /\.(ear|war|jar|zip|mrpack)$/; + +export type ReadExtraOptions = { name?: string, logger?: Logger }; + +/** + * Reads a given buffer and apply its content to the baseNode passed with the primary + */ +async function readZipDataToNode(rootNode: RootZipTreeNodeImpl, baseNode: N, buffer: ArrayBuffer, {name, logger}: ReadExtraOptions): Promise { + const zip = new ZipReader(new Uint8ArrayReader(new Uint8Array(buffer))); + + logger?.debug(`Setting up to read ${name} archive.`) + + const handleEntry = async (entry: EntryData) => { + const segments = entry.filename.split('/').filter(Boolean); // Remove empty strings from trailing slashes + + let parentNode: ZipTreeNode = baseNode; + + let path = ""; + + for (const [index, segment] of segments.entries()) { + path = path.length == 0 ? segment : `${path}/${segment}`; + + let segmentNode = parentNode.children.get(segment); + + if (!segmentNode) { + const fileEntry = (index === segments.length - 1 && !entry.directory) + ? fileOrNull(entry) + : null; + + if (fileEntry && fileEntry.filename == path) { + segmentNode = parentNode.addFile( + segment, + undefined, + zipArchiveExtensionRegex.test(segment) ? "zip" : "file", + fileEntry + ); + } else { + segmentNode = parentNode.addDirectory(segment, entry); + } + } + + if (segmentNode.type === 'directory') { + // Set next parent node to add the given next segment node + parentNode = segmentNode; + } else if(segmentNode.type === 'zip' && !segmentNode.metadata.encrypted) { + // FYI: This technically will not work when using onprogress as technically the + // given arraybuffer method is not ready yet meaning this is here just for + // legacy or needing to go back to waiting for all entries is better. + const arrayBuffer = await fileOrNull(segmentNode)?.arrayBuffer(); + if (arrayBuffer) readZipDataToNode(rootNode, segmentNode, arrayBuffer, { name: segmentNode.name, logger: logger }); + } + } + } + + logger?.debug(`Reading entries ${name} archive.`) + + const entries = await zip.getEntries({ + onprogress: (_progress, _total, entry) => handleEntry(entry as EntryData) + }) + + const zipFiles: Promise[] = []; + + for (const entry of entries) { + const fileNode = rootNode.fileEntries.get( + (baseNode.type != "root" ? baseNode.path + "/" : "") + entry.filename.split('/').filter(Boolean).join('/') + ); + + if (fileNode) { + fileNode.setBuffer( + fileOrNull(entry)!.arrayBuffer.bind(entry), + fileNode.type, + entry as ZipWriterAddDataOptions + ); + + if (!fileNode.isSetup) fileNode.isSetup = true; + + if (fileNode.type == "zip") zipFiles.push(readZipDataToNode(rootNode, fileNode, await fileNode.getBuffer(), {name: fileNode.name, logger: logger})) + } + } + + logger?.debug(`Entries from ${name} archive have been read${zipFiles.length > 0 ? ', waiting for given inner Zip Files' : '.'}`) + + // TODO: MAKE INNER JARS ASYNC I.E. UPDATE LATER TYPE DEAL AFTER THEY ARE DONE LOADING.... + if (zipFiles.length > 0) await Promise.all(zipFiles); + + return baseNode; +} + +class ZipTreeNodeImpl extends ZipTreeNode { + protected override createFileNode(root: RootZipTreeNodeImpl, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { + return new FileZipTreeNodeImpl(root, parent, name, path, buffer, type, metadata); + } + + protected override createNode(root: RootZipTreeNodeImpl, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { + return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) + } +} + +/** + * Base Node type for a given Archive Tree acting as the holder for the base children + * combined with extra info like maps for all entries and all file entries within the archive + */ +class RootZipTreeNodeImpl extends RootZipTreeNode { + constructor(logger: Logger, key: Sha512) { + super(key); + this.logger = logger; + } + + protected override createFileNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { + return new FileZipTreeNodeImpl(root as RootZipTreeNodeImpl, parent as ZipTreeNodeImpl, name, path, buffer, type, metadata); + } + + protected override createNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { + return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) + } +} + +class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { + + isSetup: boolean; + private arrayBuffer: LazyMutableValue; + + declare type: "zip" | "file"; + declare metadata: FileEntry; + declare root: RootZipTreeNode; + + fileTypeInfo: LazyValue; + + asText: (options?: SourceOptions) => Promise; + + sha1: LazyValue; + + sha256: LazyValue; + + sha512: LazyValue; + + constructor(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata?: FileEntry) { + super(root, parent, name, path, type, metadata); + + if (buffer) { + this.isSetup = true; + this.arrayBuffer = lazyMutableValue(() => typeof buffer === 'function' ? buffer() : Promise.resolve(buffer)); + } else { + this.isSetup = false; + this.arrayBuffer = lazyMutableValue( + () => { + return new Promise((resolve, reject) => { + const id = setInterval(async () => { + if (this.isSetup) { + clearInterval(id); + return this.getBuffer().then(resolve, reject) + } else { + root.logger?.debug(`Waiting for '${this.path}' buffer to be set`) + } + }, 100) + }) + } + ); + } + + this.fileTypeInfo = this.arrayBuffer.lazyMap(async (buffer) => await getFileTypeInfo(this.path, buffer)); + + const map = new Map, LazyValue>(); + + (this.arrayBuffer as LazyMutableValueImpl)._onResetEvent.subscribe(() => map.clear()); + + this.asText = async (options?: SourceOptions) => { + let finalOptions: SourceOptions | undefined = undefined; + if (options == null || !map.has(options)) { + const type = await this.fileTypeInfo.get(); + let isValid = false; + + if (type != null) { + if (this.name.endsWith(".class") || type.mime == 'application/java-vm' || type.extension == 'class' || options?.forcedType == 'class') { + finalOptions = options ?? DEFAULT_DECOMPILE_OPTIONS; + if (!map.has(finalOptions)) { + const lazyValue = this.arrayBuffer.lazyMap(async (buffer) => { + let error: string | undefined = undefined; + + const decompilier = (await messengerRegistry?.getMessenger(decompileMessengerType))!; + + if (buffer != undefined) { + return decompilier({ + targetFile: { fullPath: this.path, name: this.name, content: buffer, rootKey: this.root.key } as ZipFileContent, + resources: this.parent?.children.values().map(entry => { return { fullPath: entry.path, name: entry.name } }).toArray(), + options: options + }); + } + + error = "Unable to decompile file as Blob was found to be undefined!"; + + return { error, wasDecompiled: false, asString: error } as DecompilerResult + }); + map.set(options ?? DEFAULT_DECOMPILE_OPTIONS, lazyValue); + return lazyValue.get(); + } + isValid = true; + } else if (type.mime.startsWith('text') || options?.forcedType == 'text') { + finalOptions = options ?? { forcedType: 'text' } + if (!map.has(finalOptions)) { + const lazyValue = this.arrayBuffer.lazyMap(async (buffer) => { + const str = new TextDecoder().decode(buffer); + return { source: str, wasDecompiled: true, asString: str } as DecompilerResult + }); + map.set({ forcedType: 'text' }, lazyValue); + return lazyValue.get(); + } + isValid = true; + } + } + + if (!isValid) { + const msg = "Current file was found not to be either a class or text, no sources could be created"; + return Promise.resolve( + { source: msg, wasDecompiled: false, asString: msg } as DecompilerResult + ) + } + } else { + finalOptions = options; + } + + return map.get(finalOptions!)!.get(); + } + + this.sha1 = this.arrayBuffer.lazyMap(async (buffer) => hashBuffer(buffer, "SHA-1")) + + this.sha256 = this.arrayBuffer.lazyMap(async (buffer) => hashBuffer(buffer, "SHA-256")) + + this.sha512 = this.arrayBuffer.lazyMap(async (buffer) => hashBuffer(buffer, "SHA-512")) + } + + async setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions) { + this.arrayBuffer.updateGetter(() => typeof buffer === 'function' ? buffer() : Promise.resolve(buffer)); + this.type = type ?? "file"; + this.metadata = { + ...options, + ...this.metadata + } as FileEntry; + + return this; + } + + getBuffer(): Promise { + return this.arrayBuffer.get(); + } + + async getFileType(): Promise { + return this.fileTypeInfo.get(); + } + + async getTypedBlob(): Promise { + const buffer = await this.getBuffer(); + const fileType = await this.fileTypeInfo.get(); + const typedBlob = new Blob([buffer], { type: fileType?.mime }) as TypedBlob; + typedBlob.fileType = fileType; + return typedBlob; + } + + async getTypedBuffer(): Promise { + const buffer = await this.getBuffer(); + const typedBuffer = buffer as TypedArrayBuffer; + typedBuffer.fileType = await this.fileTypeInfo.get(); + return typedBuffer; + } +} diff --git a/packages/ui/src/helpers/toolbox/zip/index.ts b/packages/ui/src/helpers/toolbox/zip/index.ts new file mode 100644 index 00000000000..5f0034a1cd9 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/zip/index.ts @@ -0,0 +1,376 @@ +import type { DirectoryEntry, EntryMetaData, FileEntry, ZipWriterAddDataOptions } from "@zip.js/zip.js"; + +import { + type DecompileOptions, + type DecompilerResult, + DEFAULT_DECOMPILE_OPTIONS +} from "#ui/helpers/toolbox/decompile"; +import type { FileDetails, FileTypeInfo, HashedFileContent } from "#ui/helpers/toolbox/file"; +import { + type LazyValue, + lazyValue, + TreeNode +} from "#ui/helpers/toolbox/misc"; +import type { Logger } from "#ui/helpers/toolbox/misc/logging"; + +// TODO: SWITCH TO ZipWriterAddDataOptions instead of EntryMetaData ??? +export type EntryData = EntryMetaData & { directory: boolean }; + +export type ZipNodeTypes = "zip" | "file" | "directory" | "root"; + +export const nodeTypeToOrdinal = { + "zip": 2, + "directory": 1, + "file": 0 +} as Record + +export function sortZipNodes(node1: ZipTreeNode, node2: ZipTreeNode) { + const order = nodeTypeToOrdinal[node1.type] - nodeTypeToOrdinal[node2.type]; + + return order == 0 ? node1.name.localeCompare(node2.name, "en") : -order +} + +export type ArrayBufferSource = (() => Promise) | Promise | ArrayBuffer; + +export interface ZipFileContent extends FileDetails { + rootKey: Sha512; + content: ArrayBuffer; +} + +export class ZipTreeNode extends TreeNode { + root?: RootZipTreeNode; + type: ZipNodeTypes; + metadata: EntryData + + constructor(root: RootZipTreeNode | undefined, parent: ZipTreeNode | undefined, name: string, path: string, type: ZipNodeTypes, metadata?: EntryMetaData) { + super(parent, name, path); + this.root = root; + this.type = type; + this.metadata = { + ...(metadata ?? {}), + directory: type == "directory" + } as EntryData; + } + + remove() { + this.children.delete(this.name); + + this.root?.removeNodeFromRoot(this) + }; + + addFile(name: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { + return this.addNode(name, (root, parent, name, path) => this.createFileNode(root, parent, name, path, buffer, type, metadata)); + }; + + protected createFileNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { + const node = new DelegateFileZipTreeNode(root, parent, name, path, type, metadata); + if (buffer != null) node.setBuffer(buffer); + return node; + } + + addDirectory(name: string, metadata?: EntryMetaData): ZipTreeNode { + return this.addNode(name, (root, parent, name, path) => this.createNode(root, parent, name, path, metadata)); + }; + + protected createNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { + return new ZipTreeNode(root, parent, name, path, "directory", metadata) + } + + addNode(name: string, nodeBuilder: (root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string) => N): N { + const root = this.root ?? this as unknown as RootZipTreeNode; + this.children.get(name)?.remove(); + const node = nodeBuilder(root, this, name, (this.type != "root" ? this.path + "/" : "") + name); + this.children.set(name, node); + root.addNodeToRoot(node); + return node; + }; + + asFile(): FileZipTreeNode | undefined{ + return (this.type == "zip" || this.type == "file") ? this as unknown as FileZipTreeNode : undefined; + } +} + +export type Sha512 = string; + +/** + * Root tree node — extends TreeNode with a flat entries list. + */ +export class RootZipTreeNode extends ZipTreeNode { + key: Sha512; + entries = new Map(); + fileEntries = new Map(); + relativeFileEntries = new Map(); + + logger?: Logger; + + declare type: "root"; + + declare root: undefined; + declare parent: undefined; + + constructor(key: Sha512) { + super(undefined, undefined, "", "", "root"); + this.key = key; + } + + /** + * @internal Only used to add a node to the given reference maps + */ + addNodeToRoot(node: ZipTreeNode) { + this.entries.set(node.path, node) + const fileNode = node.asFile(); + + if (fileNode) { + this.fileEntries.set(node.path, fileNode); + + const parts = node.path.split(".jar/"); + this.relativeFileEntries.set((parts.length > 1) ? parts[parts.length - 1] : node.path, fileNode) + } + + return node; + } + + /** + * @internal Only used to remove a node from the given reference maps + */ + removeNodeFromRoot(node: ZipTreeNode) { + this.entries.delete(node.path) + this.fileEntries.delete(node.path) + + const parts = node.path.split(".jar/"); + if (parts.length > 1) this.relativeFileEntries.delete(parts[parts.length - 1]) + + return node; + } +} + +export type SourceOptions = DecompileOptions & { forcedType?: "class" | "text" | undefined; }; + +export interface SourceHandler { + asText: (options?: SourceOptions) => Promise, +} + +export interface LazyDataAccess { + fileTypeInfo: LazyValue, + sha1: LazyValue, + sha256: LazyValue, + sha512: LazyValue +} + +export interface FileZipTreeNode extends ZipTreeNode, LazyDataAccess, SourceHandler { + type: "zip" | "file"; + + metadata: FileEntry + isSetup: boolean; + + getBuffer(): Promise + setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise; + + getFileType(): Promise + + getTypedBlob(): Promise + + getTypedBuffer(): Promise +} + + +export interface TypedArrayBuffer extends ArrayBuffer { + fileType: FileTypeInfo | null; +} + +export interface TypedBlob extends Blob { + fileType: FileTypeInfo | null; +} + +export function fileOrNull(data: ZipTreeNode|EntryData): FileEntry | null { + const castedData = (((data as Partial).metadata ?? data) as EntryData); + return !castedData.directory ? castedData as FileEntry : null; +} + +export function dirOrNull(data: ZipTreeNode|EntryData): DirectoryEntry | null { + const castedData = (((data as Partial).metadata ?? data) as EntryData); + return castedData.directory ? castedData as DirectoryEntry : null; +} + +export type StringMatcher = { startsWith?: string, includes?: string, endsWith?: string, regex?: RegExp, negate?: boolean}; +export type FileMatcher = { + mime?: StringMatcher, + path?: StringMatcher +} + +export interface RootNodeTarget { + root: Sha512 +} + +export interface NodeTarget extends RootNodeTarget { + path: string; + relative?: boolean +} + +export function getAsTarget(target: FileZipTreeNode | NodeTarget) { + return (target as FileZipTreeNode).type != undefined ? {path: target.path, root: (target.root as RootZipTreeNode).key} : target as NodeTarget; +} + +export interface ZipHandler { + zipArchivesByHash: Map; + urlToHash: Map; + readLazyData(node: FileZipTreeNode | NodeTarget, key: K): Promise>>; + readSource(node: FileZipTreeNode | NodeTarget, options?: SourceOptions): Promise; + readNodeData(node: FileZipTreeNode | NodeTarget): Promise; + writeNodeData(node: FileZipTreeNode | NodeTarget, buffer: ArrayBuffer, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise; + collectFiles(root: RootZipTreeNode | Sha512, matcher?: FileMatcher): Promise; + collectHashedFiles(root: RootZipTreeNode | Sha512, matcher?: FileMatcher, deduplicate?: boolean): Promise; + getOrReadZipTree(buffer: () => Promise, details?: { name?: string, url?: string, hash?: string }): Promise; + readZipTree(buffer: ArrayBuffer, details?: { name?: string, url?: string, hash?: string }): Promise; + getZipTree(value: string | Sha512, isUrl: boolean, localOnly?: boolean): Promise; + closeZipTree(root: RootZipTreeNode | Sha512, name?: string): Promise<{ err?: Error, wasClosed: boolean }>; + closeAllZipTrees(): Promise<{ err?: Error, wasClosed: boolean }>; +} + +export class DelegateFileZipTreeNode extends ZipTreeNode implements FileZipTreeNode { + declare type: "zip" | "file"; + declare metadata: FileEntry; + declare handler: ZipHandler | undefined; + + isSetup: boolean; + + constructor(root: RootZipTreeNode | undefined, parent: ZipTreeNode | undefined, name: string, path: string, type: "zip" | "file", metadata: FileEntry) { + super(root, parent, name, path, type, metadata); + + this.handler = undefined; + + this.fileTypeInfo = lazyValue(() => { throw new Error("Unable to get value 'fileTypeInfo' as it was not setup yet") }); + this.asText = () => { throw new Error("Unable to get value 'asText' as it was not setup yet") }; + this.sha1 = lazyValue(() => { throw new Error("Unable to get value 'sha1' as it was not setup yet") }); + this.sha256 = lazyValue(() => { throw new Error("Unable to get value 'sha256' as it was not setup yet") }); + this.sha512 = lazyValue(() => { throw new Error("Unable to get value 'sha512' as it was not setup yet") }); + + this.isSetup = false; + } + + onDataUpdate?: () => void; + + finishNodeSetup(handler: ZipHandler) { + this.handler = handler; + + this.fileTypeInfo = lazyValue(() => this.getHandler().readLazyData(this, "fileTypeInfo")); + this.sha1 = lazyValue(() => this.getHandler().readLazyData(this, "sha1")); + this.sha256 = lazyValue(() => this.getHandler().readLazyData(this, "sha256")); + this.sha512 = lazyValue(() => this.getHandler().readLazyData(this, "sha512")); + + const map = new Map, LazyValue>(); + + this.onDataUpdate = () => map.clear() + + this.asText = async (options?: SourceOptions) => { + let finalOptions: SourceOptions | undefined = undefined; + if (options == null || !map.has(options)) { + const type = await this.fileTypeInfo.get(); + let isValid = false; + if (type != null) { + if (this.name.endsWith(".class") || type.mime == 'application/java-vm' || type.extension == 'class' || options?.forcedType == 'class') { + finalOptions = options ?? DEFAULT_DECOMPILE_OPTIONS; + if (!map.has(finalOptions)) { + const value = lazyValue(() => this.getHandler().readSource(this, finalOptions)) + map.set(options ?? DEFAULT_DECOMPILE_OPTIONS, value); + return value.get(); + } + isValid = true; + } else if (type.mime.startsWith('text') || options?.forcedType == 'text') { + finalOptions = options ?? { forcedType: 'text' } + if (!map.has(finalOptions)) { + const value = lazyValue(() => this.getHandler().readSource(this, finalOptions)) + map.set(options ?? DEFAULT_DECOMPILE_OPTIONS, value); + return value.get(); + } + isValid = true; + } + } + + if (!isValid) { + const msg = "Current file was found not to be either a class or text, no sources could be created"; + return Promise.resolve( + { source: msg, wasDecompiled: false, asString: msg } as DecompilerResult + ) + } + } else { + finalOptions = options; + } + + return map.get(finalOptions!)!.get(); + } + + this.isSetup = true; + } + + getHandler() { + return this.handler!; + } + + fileTypeInfo: LazyValue; + asText: (options?: SourceOptions) => Promise; + sha1: LazyValue; + sha256: LazyValue; + sha512: LazyValue; + + getBuffer(): Promise { + return this.getHandler().readNodeData(this); + } + + async setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise { + const bufferPromise = typeof buffer === 'function' ? buffer() : Promise.resolve(buffer); + await this.getHandler().writeNodeData(this, await bufferPromise, type, options); + return this; + } + + getFileType(): Promise { + return this.fileTypeInfo.get(); + } + + async getTypedBlob(): Promise { + const buffer = await this.getBuffer(); + const fileType = await this.getFileType(); + const typedBlob = new Blob([buffer], { type: fileType?.mime }) as TypedBlob; + typedBlob.fileType = fileType; + return typedBlob; + } + + async getTypedBuffer(): Promise { + const buffer = await this.getBuffer(); + const typedBuffer = buffer as TypedArrayBuffer; + typedBuffer.fileType = await this.getFileType(); + return typedBuffer; + } +} + +export function reconstituteRoot(handler: ZipHandler, root: RootZipTreeNode): RootZipTreeNode { + root.entries = new Map(); + root.fileEntries = new Map(); + + return reconstituteNodes(handler, root, root, root) +} + +function reconstituteNodes(handler: ZipHandler, root: RootZipTreeNode, parent: N, node: N): N { + let reconstitutedNode: ZipTreeNode; + if (node.type == "root") { + reconstitutedNode = Object.setPrototypeOf(node, RootZipTreeNode.prototype); + } else if (node.type == "directory") { + reconstitutedNode = Object.setPrototypeOf(node, ZipTreeNode.prototype); + } else { + reconstitutedNode = Object.setPrototypeOf(node, DelegateFileZipTreeNode.prototype); + root.fileEntries.set(reconstitutedNode.path, reconstitutedNode as FileZipTreeNode); + } + + node.children.forEach((entry) => reconstituteNodes(handler, root, node, entry)) + + if (reconstitutedNode.type != "root") { + root.entries.set(reconstitutedNode.path, reconstitutedNode); + + reconstitutedNode.root = root; + reconstitutedNode.parent = parent; + } + + if (node.type == "file") (node as unknown as DelegateFileZipTreeNode).finishNodeSetup(handler) + + return reconstitutedNode as N; +} diff --git a/packages/ui/src/utils/file-extensions.ts b/packages/ui/src/utils/file-extensions.ts index 16e40221704..a8cdf3864e8 100644 --- a/packages/ui/src/utils/file-extensions.ts +++ b/packages/ui/src/utils/file-extensions.ts @@ -1,4 +1,6 @@ // File extension constants +import {getFileTypeInfo, getLanguageFromFile} from "#ui/helpers/toolbox/file"; + export const FILE_CODE_EXTENSIONS = [ 'json', 'json5', @@ -93,15 +95,23 @@ export function isEditableFile(ext: string): boolean { * Check if a file can be opened in the file editor */ export function canOpenInFileEditor(filename: string): boolean { - const ext = getFileExtension(filename) + const fileInfo = getFileTypeInfo(filename); + if (fileInfo != null && (getLanguageFromFile(fileInfo) || fileInfo.mime.startsWith("text/"))) return true; + const ext = getFileExtension(filename); return !filename.includes('.') || isEditableFile(ext) || isImageFile(ext) } /** * Get Ace editor language mode for a file extension */ -export function getEditorLanguage(ext: string): string { - const lowered = ext.toLowerCase() +export function getEditorLanguage(filename: string): string { + const fileInfo = getFileTypeInfo(filename); + if (fileInfo != null) { + const language = getLanguageFromFile(fileInfo); + if (language) return language; + } + const lowered = getFileExtension(filename); + switch (lowered) { // Code files case 'json': diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd1c684424a..77d942555ca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -844,6 +844,15 @@ importers: '@modrinth/utils': specifier: workspace:* version: link:../utils + '@run-slicer/cfr': + specifier: 0.1.8-0.152 + version: 0.1.8-0.152 + '@run-slicer/procyon': + specifier: 0.1.2-0.6.0 + version: 0.1.2-0.6.0 + '@run-slicer/vf': + specifier: 0.6.4-1.12.0 + version: 0.6.4-1.12.0 '@tanstack/vue-query': specifier: 5.101.4 version: 5.101.4(vue@3.5.43(typescript@5.9.3)) @@ -880,6 +889,9 @@ importers: '@xterm/xterm': specifier: ^6.0.0 version: 6.0.0 + '@zip.js/zip.js': + specifier: ^2.8.34 + version: 2.8.34 ace-builds: specifier: ^1.43.5 version: 1.43.6 @@ -907,6 +919,9 @@ importers: fabric: specifier: ^7.4.0 version: 7.4.0(supports-color@10.2.2) + file-type: + specifier: ^22.1.1 + version: 22.1.1(supports-color@10.2.2) flatpickr: specifier: ^4.6.13 version: 4.6.13 @@ -931,6 +946,9 @@ importers: markdown-it: specifier: ^13.0.2 version: 13.0.2 + mime: + specifier: ^4.1.0 + version: 4.1.0 motion-v: specifier: ^2.2.1 version: 2.2.1(@vueuse/core@11.3.0(vue@3.5.43(typescript@5.9.3)))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(vue@3.5.43(typescript@5.9.3)) @@ -1324,6 +1342,9 @@ packages: commander: optional: true + '@borewit/text-codec@0.2.2': + resolution: {integrity: sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==} + '@capsizecss/unpack@4.0.0': resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==} engines: {node: '>=18'} @@ -3871,6 +3892,15 @@ packages: cpu: [x64] os: [win32] + '@run-slicer/cfr@0.1.8-0.152': + resolution: {integrity: sha512-FN5I41CF0QcscV5HCcbq5iJHqIueNtROvLtfXohNt9pNJUxo2WFncg0a7jPinRjBi7VgNxjJmWE7sP++vYVFRw==} + + '@run-slicer/procyon@0.1.2-0.6.0': + resolution: {integrity: sha512-xPra0FuATHnjLYLXFUlmRYkfmIukCrl5Zv6iM8ajSTYFDgK85HVNpBAo23NQbp+cMU6QpvOEviwE8nRwGuS07Q==} + + '@run-slicer/vf@0.6.4-1.12.0': + resolution: {integrity: sha512-fKKqDHVCOKed3jcbrX4aA9Za7YY+/7tq2tVlkbSPQ9NVMaGeBhqyRE/gnLc30ZJ3zKP/UYIh69FaUkrF/dmLuQ==} + '@rushstack/node-core-library@5.23.1': resolution: {integrity: sha512-wlKmIKIYCKuCASbITvOxLZXepPbwXvrv7S6ig6XNWFchSyhL/E2txmVXspHY49Wu2dzf7nI27a2k/yV5BA3EiA==} peerDependencies: @@ -4327,6 +4357,13 @@ packages: peerDependencies: '@testing-library/dom': '>=7.21.4' + '@tokenizer/inflate@0.4.1': + resolution: {integrity: sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==} + engines: {node: '>=18'} + + '@tokenizer/token@0.3.0': + resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + '@tresjs/cientos@4.3.1': resolution: {integrity: sha512-3qp6lEtMrFdhxDuASP1Sz/hEi8+xcEpM6Vd6uDJysCh4uRAzyJLlBSbPoR7gVjN12wrhwJIF1AfYEFz/Vhz5ZQ==} peerDependencies: @@ -6448,6 +6485,10 @@ packages: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} + file-type@22.1.1: + resolution: {integrity: sha512-sEB0oStmeLGtpmSAD88RDNuLsvhv4iC3u0lE25CUvfCULvj49xoQ3l3K525Vn+ZzsZ6DOLajQCV6RmhwwhqiJg==} + engines: {node: '>=22'} + file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} @@ -9230,6 +9271,10 @@ packages: '@types/node': optional: true + strtok3@10.3.5: + resolution: {integrity: sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==} + engines: {node: '>=18'} + structured-clone-es@2.0.1: resolution: {integrity: sha512-10ZL5r77LhknxlP1FBiCW+VdnuWOEFLdSS2SKtjyEV+L4qP1hUEIMIZW94LC3jKxRmT/Dj7KkD1mqh/IY6lhKQ==} @@ -9412,6 +9457,10 @@ packages: token-stream@1.0.0: resolution: {integrity: sha512-VSsyNPPW74RpHwR8Fc21uubwHY7wMDeJLys2IX5zJNih+OnAnaifKHo+1LHT7DAdloQ7apeaaWg8l7qnf/TnEg==} + token-types@6.1.2: + resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==} + engines: {node: '>=14.16'} + totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -9535,6 +9584,10 @@ packages: ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + uint8array-extras@1.6.0: + resolution: {integrity: sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==} + engines: {node: '>=18'} + ultrahtml@1.6.0: resolution: {integrity: sha512-R9fBn90VTJrqqLDwyMph+HGne8eqY1iPfYhPzZrvKpIfwkWZbcYlfpsb8B9dTvBfpy1/hqAD7Wi8EKfP9e8zdw==} @@ -11000,6 +11053,8 @@ snapshots: cac: 6.7.14 citty: 0.2.2 + '@borewit/text-codec@0.2.2': {} + '@capsizecss/unpack@4.0.0': dependencies: fontkitten: 1.0.2 @@ -13261,6 +13316,12 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.63.6': optional: true + '@run-slicer/cfr@0.1.8-0.152': {} + + '@run-slicer/procyon@0.1.2-0.6.0': {} + + '@run-slicer/vf@0.6.4-1.12.0': {} + '@rushstack/node-core-library@5.23.1(@types/node@24.12.2)': dependencies: ajv: 8.18.0 @@ -13723,6 +13784,15 @@ snapshots: dependencies: '@testing-library/dom': 10.4.1 + '@tokenizer/inflate@0.4.1(supports-color@10.2.2)': + dependencies: + debug: 4.4.3(supports-color@10.2.2) + token-types: 6.1.2 + transitivePeerDependencies: + - supports-color + + '@tokenizer/token@0.3.0': {} + '@tresjs/cientos@4.3.1(@tresjs/core@4.3.6(three@0.172.0)(typescript@5.9.3)(vue@3.5.43(typescript@5.9.3)))(@types/three@0.172.0)(react@19.2.8)(three@0.172.0)(typescript@5.9.3)(vue@3.5.43(typescript@5.9.3))': dependencies: '@tresjs/core': 4.3.6(three@0.172.0)(typescript@5.9.3)(vue@3.5.43(typescript@5.9.3)) @@ -16392,6 +16462,15 @@ snapshots: dependencies: flat-cache: 4.0.1 + file-type@22.1.1(supports-color@10.2.2): + dependencies: + '@tokenizer/inflate': 0.4.1(supports-color@10.2.2) + strtok3: 10.3.5 + token-types: 6.1.2 + uint8array-extras: 1.6.0 + transitivePeerDependencies: + - supports-color + file-uri-to-path@1.0.0: {} fill-range@7.1.1: @@ -20028,6 +20107,10 @@ snapshots: optionalDependencies: '@types/node': 24.12.2 + strtok3@10.3.5: + dependencies: + '@tokenizer/token': 0.3.0 + structured-clone-es@2.0.1: {} style-mod@4.1.3: {} @@ -20261,6 +20344,12 @@ snapshots: token-stream@1.0.0: {} + token-types@6.1.2: + dependencies: + '@borewit/text-codec': 0.2.2 + '@tokenizer/token': 0.3.0 + ieee754: 1.2.1 + totalist@3.0.1: {} tough-cookie@5.1.2: @@ -20358,6 +20447,8 @@ snapshots: ufo@1.6.4: {} + uint8array-extras@1.6.0: {} + ultrahtml@1.6.0: {} ultrahtml@1.7.0: {} From 3e543813f7372c6d874cae29634bac96c1d6a170 Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Thu, 8 Oct 2026 20:20:30 -0500 Subject: [PATCH 02/18] refactor: Simplify Messenger Worker API and resolve bugs with such - Fix issues with zip handling not properly finishing zip nodes on the main thread - Resolve issues with decompiling inner classes not working due to incorrect split --- .../ui/src/helpers/toolbox/decompile/impl.ts | 2 +- .../ui/src/helpers/toolbox/decompile/index.ts | 11 +- .../helpers/toolbox/messenger/implement.ts | 243 +++++++++--------- .../ui/src/helpers/toolbox/messenger/index.ts | 167 +++++------- .../toolbox/workers/decompile/index.ts | 14 +- .../ui/src/helpers/toolbox/workers/index.ts | 65 ++--- .../ui/src/helpers/toolbox/workers/zip/api.ts | 113 ++++---- .../src/helpers/toolbox/workers/zip/index.ts | 18 +- packages/ui/src/helpers/toolbox/zip/impl.ts | 4 +- packages/ui/src/helpers/toolbox/zip/index.ts | 2 +- 10 files changed, 318 insertions(+), 321 deletions(-) diff --git a/packages/ui/src/helpers/toolbox/decompile/impl.ts b/packages/ui/src/helpers/toolbox/decompile/impl.ts index 6310ab0ae14..3ec806a76c5 100644 --- a/packages/ui/src/helpers/toolbox/decompile/impl.ts +++ b/packages/ui/src/helpers/toolbox/decompile/impl.ts @@ -76,7 +76,7 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle try { // const pathParts = targetFile.fullPath.split('.jar'); \n const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); \n - const pathParts = targetFile.fullPath.split('.jar'); + const pathParts = targetFile.fullPath.split('.jar/'); const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); const buffer = targetFile.content; const decompiled = await (await getDecompiler(logger, options.decompiler!)) diff --git a/packages/ui/src/helpers/toolbox/decompile/index.ts b/packages/ui/src/helpers/toolbox/decompile/index.ts index d5f3ad96445..e75b9b8da6f 100644 --- a/packages/ui/src/helpers/toolbox/decompile/index.ts +++ b/packages/ui/src/helpers/toolbox/decompile/index.ts @@ -1,4 +1,5 @@ import type { FileContent, FileDetails, FileTypeInfo } from "#ui/helpers/toolbox/file"; +import type {MessageTypes} from "#ui/helpers/toolbox/messenger"; export const ALL_DECOMPILER_TYPES = ['cfr', 'vineflower', 'procyon', 'krakatau'] as const; @@ -21,7 +22,15 @@ export const DEFAULT_DECOMPILE_OPTIONS = { } } as const; -export type DecompileHandler = (input: DecompileInput) => Promise; +export type DecompilerMessages = MessageTypes< + { + main: {input: DecompileInput, output: DecompilerResult} + } +> + +export type DecompileHandler = { + decompile(input: DecompileInput): Promise; +}; export interface DecompilerResult { wasDecompiled: boolean, diff --git a/packages/ui/src/helpers/toolbox/messenger/implement.ts b/packages/ui/src/helpers/toolbox/messenger/implement.ts index 57fa4733866..8c43f057d90 100644 --- a/packages/ui/src/helpers/toolbox/messenger/implement.ts +++ b/packages/ui/src/helpers/toolbox/messenger/implement.ts @@ -1,27 +1,32 @@ -import { type EventSource,EventStream } from "#ui/helpers/toolbox/misc"; -import { type CounterCallbackData, logFactory, type Logger, type LoggerCallbackData, type LoggerFactory } from '#ui/helpers/toolbox/misc/logging'; +import {type EventSource, EventStream} from "#ui/helpers/toolbox/misc"; +import { + type CounterCallbackData, + logFactory, + type Logger, + type LoggerCallbackData, + type LoggerFactory +} from '#ui/helpers/toolbox/misc/logging'; import { type ChannelHolder, - type ClientCompoundMessenger, type ClientMessenger, - type CompoundMessengerOptions, - type EntryOptions, - type InputMessageTypes, -type InstanceConfig, + type MessengerOptions, + type EntryOptions, type InputMessage, + type InstanceConfig, type InstanceKey, - isWorker, type Message, - type MessageSchema, + isWorker, + type Message, type MessageTypes, type Messenger, - type MessengerOptions, type MessengerRegistry, type MessengerType, - type MessengerVariants, - type OutputMessageTypes, - type ServerInputHandler, + type MessengerVariants, type OutputMessage, + type OutputResolverMap, + type PossibleTransferable, type ServerInputHandler, type ServerMessenger, - type TypeId} from '.'; + type TypeId, type AsString, type InputResolverMap, type MessengerOptionsBase, + type ValueBase, +} from '.'; // #region MessengerType Types @@ -33,13 +38,13 @@ interface MessengerTypeImpl extends MessengerType { instances?: (() => InternalMessenger) | Map>; wrapWith: (channelHolder: () => ChannelHolder, key: InstanceKey) => InternalMessenger; - getOrCreateWith: (baseMessenger?: () => ChannelHolder, instanceKey?: string) => M extends ClientMessenger ? M & ChannelHolder & H : M & ChannelHolder; + getOrCreateWith: (baseMessenger?: () => ChannelHolder, instanceKey?: string) => M extends ClientMessenger ? M & ChannelHolder & H : M & ChannelHolder; workerFactory?: () => Worker; getInstances: () => M[]; } -type WrapperFunc = (type: MessengerTypeImpl, key: InstanceKey, holder: C) => M & C & InternalMessenger; +type WrapperFunc = (type: MessengerTypeImpl, key: InstanceKey, holder: C) => InternalMessenger; function typeFor( id: string, instanceConfig: InstanceConfig, closeCallback?: (messenger: M) => void) { @@ -60,9 +65,7 @@ function typeFor( if (instanceKey) { if (holder == null) throw new Error(`Unable to create messenger for '${this.id}' as the entry used was found to be null! Did you setup the main thread worker factory or pass in the port to the call!`) - const wrappedMessenger = this.wrapWith(holder, instanceKey); - - return wrappedMessenger; + return this.wrapWith(holder, instanceKey); } if (holder == null) throw new Error(`Unable to create messenger for '${this.id}' as the entry used was found to be null! Did you setup the main thread worker factory or pass in the port to the call!`) @@ -88,7 +91,7 @@ function typeFor( (this.instances as Map).set(wrappedMessenger.key!, wrappedMessenger); } - wrappedMessenger.postMessage?.({ type: "set-key", key: wrappedMessenger.key} as SetOffThreadKeyMessage); + wrappedMessenger.postMessage?.({ channel: "set-key", key: wrappedMessenger.key} as SetOffThreadKeyMessage); return wrappedMessenger; }, @@ -97,13 +100,13 @@ function typeFor( if (this.instances != undefined) { const instance = (this.instances as () => InternalMessenger)(); closeCallback?.(instance); - instance.postMessage({type: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) + instance.postMessage({channel: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) } } else { const map = (this.instances as Map>); map.values().forEach(entry => { closeCallback?.(entry) - entry.postMessage({type: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) + entry.postMessage({channel: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) }) map.clear(); } @@ -118,8 +121,6 @@ function typeFor( // #region MessengerRegistry Types interface MessengerRegistryImpl extends MessengerRegistry { - setupMessengerType(id: string, wrapWith: WrapperFunc>, extra?: EntryOptions>): MessengerType>; - serverMessenger?: ServerHandlerImpl, types: Map>; logger: LoggerFactory; @@ -130,31 +131,15 @@ export function createRegistryInternal(): MessengerRegistry { return { types: new Map(), logger: logFactory(), - compoundMessenger: function >(id: string, extra: CompoundMessengerOptions & EntryOptions>): MessengerType> { - function wrapWith(type: MessengerTypeImpl>, key: string, channelHolder: C) { - return setupAsCompoundClient(type, "client", key, channelHolder, extra) - }; - - // @ts-expect-error - return this.setupMessengerType(id, wrapWith, extra); - }, - simpleMessenger: function (id: string, extra: MessengerOptions & EntryOptions>): MessengerType> { - function wrapWith(type: MessengerTypeImpl>, key: string, channelHolder: C) { - return setupAsClient(type, "client", key, channelHolder, extra) - }; - - // @ts-expect-error - return this.setupMessengerType(id, wrapWith, extra); - }, - setupMessengerType: function (id: string, wrapWith: WrapperFunc>, extra: MessengerOptions & EntryOptions): MessengerType> { + of: function (id: string, extra: MessengerOptions & EntryOptions>): MessengerType> { const instanceConfig: InstanceConfig = { timeout: extra?.timeout ?? undefined, shared: extra?.shared ?? true, count: extra.count ?? 1, }; - const type = typeFor>(id, instanceConfig); + const type = typeFor>(id, instanceConfig); - type.wrapWith = (holder, key) => wrapWith(type, key, extra.inputBypassHandler ? {} as any : holder()); + type.wrapWith = (holder, key) => setupAsClient(type, "client", key, extra.inputBypassHandler != null ? {} as any : holder(), extra); type.registry = this; type.instanceConfig = instanceConfig; @@ -209,7 +194,7 @@ function handleInput(messenger: Messenger & ChannelHolder, taskHolder: TaskHolde const task = taskHolder.startTask(); messenger.postMessage({ - type: "input", + channel: "input", data: input, key: task.transactionKey, } as MessengerInputMessage, transfer ?? []); @@ -260,19 +245,19 @@ function createTaskHolder(): TaskHolder{ } } -interface ClientHandlerImpl extends RootMessageHandler, ClientMessenger { +interface ClientHandlerImpl extends RootMessageHandler, ClientMessenger { taskHolder: TaskHolder; } const FORCED_OUTPUT_KEY = "INTERNAL_FORCED_OUTPUT"; -function clientMessageHandler( +function clientMessageHandler( messenger: InternalMessenger, - options: MessengerOptions, -): ClientHandlerImpl & InternalMessenger, C> & (H extends Function ? {} : H) { + options: MessengerOptionsBase, +): ClientHandlerImpl & InternalMessenger, C> & (H extends Function ? {} : H) { const taskHolder = createTaskHolder(); - const sendMessage: ClientMessenger["sendMessage"] = options.inputBypassHandler ? async (input) => { + const sendMessage: ClientMessenger["sendMessage"] = options.inputBypassHandler ? async (input) => { const output = (await options.inputBypassHandler!(messenger as any, input)); messenger.terminate(); messenger.isAlive = false; @@ -280,15 +265,13 @@ function clientMessageHandler( return Array.isArray(output) ? output[0] : output; } : (input) => handleInput(messenger, taskHolder, input, options.inputTransferHandler?.(input)); - const handler = options.handlerConstructor?.(messenger as any) ?? ({} as any) - const fullHandler = Object.assign(messenger, { taskHolder: taskHolder, async handleRootMessage(msg: ClientboundMessages){ - if (msg.type == "operation-error" || msg.type == "output") { - handleOutputMessages(this, msg, messenger.logger, taskHolder, options.forcedOutputHandler) + if (msg.channel == "operation-error" || msg.channel == "output") { + handleOutputMessages(this, msg, messenger.logger, taskHolder, options.independentOutputHandler) } else if (messenger.variant == "client") { - if (msg.type == "logging") { // MAIN THREAD ONLY + if (msg.channel == "logging") { // MAIN THREAD ONLY const factory = messenger.logger.factory; if (msg.logMsg.type == 'callback') { factory.log(...msg.logMsg.data) @@ -298,15 +281,15 @@ function clientMessageHandler( throw new Error(`Unknown type for logging '${(msg.logMsg as any).types}'!`) } - } else if (msg.type == "setup-root") { // MAIN THREAD ONLY + } else if (msg.channel == "setup-root") { // MAIN THREAD ONLY const registry = messenger.type.registry; const newMessenger = await registry.getMessenger(registry.getType(msg.id)!) as InternalMessenger; const channel = new MessageChannel(); - messenger.postMessage({ type: "setup-port", id: newMessenger.type.id, key: newMessenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port1]) - newMessenger.postMessage({ type: "setup-port", id: messenger.type.id, key: messenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port2]) - } else if (msg.type == "close-root") { + messenger.postMessage({ channel: "setup-port", id: newMessenger.type.id, key: newMessenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port1]) + newMessenger.postMessage({ channel: "setup-port", id: messenger.type.id, key: messenger.key, transactionKey: msg.transactionKey } as SetupMessengerPortMessage, [channel.port2]) + } else if (msg.channel == "close-root") { messenger.terminate(); messenger.isAlive = false; messenger.onTerminate.stream.sink(true) @@ -320,16 +303,19 @@ function clientMessageHandler( return true; }, sendMessage: sendMessage, - handler: handler, - } as ClientHandlerImpl & InternalMessenger); + } as ClientHandlerImpl & InternalMessenger); + + const handler = options.handlerConstructor?.(fullHandler as any) ?? (null as any) + + fullHandler.handler = handler; if (typeof handler === "object") Object.assign(fullHandler, handler); return fullHandler as any; } -function handleOutputMessages(messenger: ClientHandlerImpl, msg: MessengerOperationError | MessengerOutputMessage, logger: Logger, taskHolder: TaskHolder, forcedOutputHandler?: (messenger: ClientHandlerImpl, msg: O) => boolean) { - if (msg.type == "operation-error") { +function handleOutputMessages(messenger: ClientHandlerImpl, msg: MessengerOperationError | MessengerOutputMessage, logger: Logger, taskHolder: TaskHolder, forcedOutputHandler?: (messenger: ClientHandlerImpl, msg: OutputMessage) => boolean) { + if (msg.channel == "operation-error") { if (taskHolder.hasTask(msg.key)) { logger.error("An error occurred when attempting to complete a task!", {err: msg.error, obj: msg.info}) } else { @@ -356,7 +342,7 @@ function handleOutputMessages(messenger: ClientHandlerImpl, ms } } -interface ServerHandlerImpl extends RootMessageHandler, ServerMessenger, Messenger { +interface ServerHandlerImpl extends RootMessageHandler, ServerMessenger, Messenger { handleServerInput(sender: InternalMessenger, msg: MessengerInputMessage): Promise; attemptTimeoutWait(): void; startShutDown(): void; @@ -364,10 +350,10 @@ interface ServerHandlerImpl extends RootMessageHandler( +function serverMessageHandler( messenger: InternalMessenger, - onInput: ServerInputHandler -): ServerHandlerImpl & InternalMessenger { + onInput: ServerInputHandler +): ServerHandlerImpl> & InternalMessenger { let currentCloseFuncId: ReturnType | undefined = undefined; const currentInputKeys: Set = new Set(); @@ -376,15 +362,15 @@ function serverMessageHandler( const impl: ServerHandlerImpl & InternalMessenger = Object.assign(messenger, { async handleRootMessage(msg, ports) { - if (msg.type == "input") { + if (msg.channel == "input") { this.handleServerInput(this, msg) } else if (this.variant == "server") { - if (msg.type == "set-key") { + if (msg.channel == "set-key") { this.key = msg.key; this.logger = this.type.registry.getLogFactory().loggerFor(debugKeyFrom(this)); - } else if(msg.type == "setup-port") { + } else if(msg.channel == "setup-port") { this.setupPort(msg.id, msg.transactionKey, msg.key, ports![0]) - } else if(msg.type == "force-close-root") { + } else if(msg.channel == "force-close-root") { if (this.instanceData.timeout != null || msg.forceClose) { this.startShutDown() } else { @@ -407,18 +393,18 @@ function serverMessageHandler( const logger = this.logger; try { - logger.debug(`Attempting to process '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}' `, { obj: msg }) + logger.debug(`Attempting to process '${msg.channel}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}' `, { obj: msg }) const result = await onInput(this, msg.data, sender); const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; - logger.debug(`Finished processing '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) + logger.debug(`Finished processing '${msg.channel}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) - sender.postMessage(...([{type: "output", data: tuple[0], key: msg.key} as MessengerOutputMessage, tuple[1]])) + sender.postMessage(...([{channel: "output", data: tuple[0], key: msg.key} as MessengerOutputMessage, tuple[1]])) } catch (err) { - logger.debug(`An error occurred when processing '${msg.type}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) + logger.debug(`An error occurred when processing '${msg.channel}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) //logger.debug("Error has occurred on off-thread worker: ", { err: err, obj: msg}) - sender.postMessage({ type: "operation-error", error: err, info: msg, key: msg.key} as MessengerOperationError) + sender.postMessage({ channel: "operation-error", error: err, info: msg, key: msg.key} as MessengerOperationError) } currentInputKeys.delete(transactionKey); @@ -446,10 +432,10 @@ function serverMessageHandler( this.isAlive = false; - peerPorts.values().forEach((messenger) => messenger.postMessage({ type: "close-port", id: messenger.type.id, key: messenger.key } as CloseMessengerPortMessage)) + peerPorts.values().forEach((messenger) => messenger.postMessage({ channel: "close-port", id: messenger.type.id, key: messenger.key } as CloseMessengerPortMessage)) setTimeout(() => { - this.postMessage({ type: "close-root", id: this.type.id, key: this.key } as CloseMessengerRootMessage) + this.postMessage({ channel: "close-root", id: this.type.id, key: this.key } as CloseMessengerRootMessage) }, 100) }, setupPort(id, transactionKey, key, port) { @@ -458,7 +444,8 @@ function serverMessageHandler( const data = waitingForPeerPort.get(fullTransactionKey)!; waitingForPeerPort.delete(fullTransactionKey); - const portMessenger = ((data?.type ?? this.type.registry.getType(id)) as MessengerTypeImpl)!.getOrCreateWith(() => port as Messenger & MessagePort, key) as InternalMessenger & MessagePort; + const portMessenger = ((data?.type ?? this.type.registry.getType(id)) as MessengerTypeImpl)! + .getOrCreateWith(() => port as Messenger & MessagePort, key) as InternalMessenger & MessagePort; portMessenger.variant = "port" @@ -469,9 +456,9 @@ function serverMessageHandler( const portClientHandler = portClientMessageHandler(this); portMessenger.handleRootMessage = async (msg, ports) => { - if ((msg.type == "close-port" || msg.type == "output" || msg.type == "operation-error") && await portClientHandler(msg as ClientboundPortMessages, ports)) { + if ((msg.channel == "close-port" || msg.channel == "output" || msg.channel == "operation-error") && await portClientHandler(msg as ClientboundPortMessages, ports)) { return true; - } else if (msg.type == "input") { + } else if (msg.channel == "input") { this.handleServerInput(portMessenger, msg as MessengerInputMessage) } @@ -498,7 +485,7 @@ function serverMessageHandler( waitingForPeerPort.set(`${type.id}:${transactionKey}`, { type: type, promiseHolder: holder}); - this.postMessage({ type: "setup-root", id: type.id, transactionKey: transactionKey} as StartMessengerRootMessage) + this.postMessage({ channel: "setup-root", id: type.id, transactionKey: transactionKey} as StartMessengerRootMessage) return holder.promise as any; }, @@ -509,9 +496,9 @@ function serverMessageHandler( const result = await msg; const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; - (sender as InternalMessenger).postMessage(...([{type: "output", data: tuple[0], key: FORCED_OUTPUT_KEY} as MessengerOutputMessage, tuple[1]])) + (sender as InternalMessenger).postMessage(...([{channel: "output", data: tuple[0], key: FORCED_OUTPUT_KEY} as MessengerOutputMessage, tuple[1]])) } - } as ServerHandlerImpl & InternalMessenger); + } as ServerHandlerImpl> & InternalMessenger); // TODO: ADD WARNING TO PREVENT DOUBLE SERVER STUFF (impl.type.registry as MessengerRegistryImpl).serverMessenger = impl; @@ -521,7 +508,7 @@ function serverMessageHandler( function portClientMessageHandler(rootServerHandler: ServerHandlerImpl): RootMessageHandler["handleRootMessage"] { return async (msg) => { - if (msg.type == "close-port") { + if (msg.channel == "close-port") { rootServerHandler.closePort(fullInstanceKey(msg.id, msg.key)); return true; @@ -540,7 +527,7 @@ export function setupHolderAsImpl C & InternalMessenger & R -): C & Messenger & R { +): InternalMessenger & R { const infoHolder = holder as (C & InternalMessenger); if (key) infoHolder.key = key; @@ -552,14 +539,20 @@ export function setupHolderAsImpl).close; + + if (close != null) { + messenger.terminate = close.bind(messenger); + } else if (messenger.terminate == null) { + messenger.terminate = () => {}; + } const registry = type.registry; if (variant == "server" && registry.getLogFactory() == null) { registry.setLogFactory(logFactory({ - log: (level, loggerName, message, extra) => holder.postMessage({ type: "logging", logMsg: { type: 'callback', data: [level, loggerName, message, extra]}} as MessengerLogMessage), - counter: (level, loggerName, msg) => holder.postMessage({ type: "logging", logMsg: { type: 'counter', data: [level, loggerName, msg]}} as MessengerLogMessage) + log: (level, loggerName, message, extra) => holder.postMessage({ channel: "logging", logMsg: { type: 'callback', data: [level, loggerName, message, extra]}} as MessengerLogMessage), + counter: (level, loggerName, msg) => holder.postMessage({ channel: "logging", logMsg: { type: 'counter', data: [level, loggerName, msg]}} as MessengerLogMessage) })) } @@ -567,10 +560,10 @@ export function setupHolderAsImpl((callbacks) => (forceClose) => callbacks.forEach((callback) => callback(forceClose))).source; - messenger.onmessage = function (e: MessageEvent>) { + messenger.onmessage = function (e: MessageEvent>) { const msg = e.data; - if (msg.type != "logging") messenger.logger.debug(`Message '${msg.type}' received on '${debugKeyFrom(messenger)}'`, {obj: msg}); + if (msg.channel != "logging") messenger.logger.debug(`Message '${msg.channel}' received on '${debugKeyFrom(messenger)}'`, {obj: msg}); messenger.handleRootMessage(msg, e.ports); }; @@ -582,44 +575,48 @@ export function setupHolderAsImpl>( - type: MessengerTypeImpl, - variant: MessengerVariants, - key: InstanceKey, - holder: ChannelHolder, - options: CompoundMessengerOptions, -): ClientCompoundMessenger { - // @ts-expect-error - const extendOptions: MessengerOptions, OutputMessageTypes, H> = { - ...options, - inputTransferHandler: options?.inputTransferHandler == null ? undefined : (msg) => { - const copied = { - ...msg, - resolve: (resolver: any) => resolver[msg.type]?.(msg) ?? [] - } - return options.inputTransferHandler!(copied as any); - } - }; - // @ts-expect-error - return setupAsClient, OutputMessageTypes, H>(type, variant, key, holder, extendOptions) as ClientCompoundMessenger; -} +type ServerInputHandlerBase + = , K extends AsString>( + messenger: ServerMessenger>, input: I_M, sender?: Messenger +) => Promise>> -function setupAsClient( +function setupAsClient( type: MessengerTypeImpl, variant: MessengerVariants, key: InstanceKey, holder: C, - options: MessengerOptions, -): ClientHandlerImpl { - return setupHolderAsImpl>(holder, type, variant, key, (impl) => clientMessageHandler(impl, options)); -} - -export function setupAsServerInternal( - type: MessengerType, - holder: ChannelHolder, - inputHandler: ServerInputHandler, -): ServerMessenger { - return setupHolderAsImpl(holder, type as MessengerTypeImpl, "server", undefined, (impl) => serverMessageHandler(impl, inputHandler)); + options: MessengerOptions, +): InternalMessenger, C> { + return setupHolderAsImpl>(holder, type, variant, key, (impl) => clientMessageHandler(impl, + { + ...options, + inputTransferHandler: options?.inputTransferHandler == null ? undefined : (msg) => { + const copied = { + ...msg, + resolve: (resolver: InputResolverMap) => resolver[msg.channel]?.(msg) ?? [] + } + return options.inputTransferHandler!(copied); + }, + inputBypassHandler: options.inputBypassHandler != null ? ( + async (messenger, msg, sender) => { + const msgWithResolver = Object.assign( + {resolve: (resolvers: OutputResolverMap) => (resolvers[msg.channel])(msg)[0]}, + msg + ) + return (await options.inputBypassHandler!(messenger, msgWithResolver, sender))[0]; + } + ) as ServerInputHandlerBase : undefined, + } + )); +} + +export function setupAsServerInternal( + type: MessengerType, + holder: ChannelHolder, + inputHandler: ServerInputHandler, +): ServerMessenger> { + return setupHolderAsImpl(holder, type as MessengerTypeImpl, "server", undefined, + (impl) => serverMessageHandler(impl, inputHandler)); } diff --git a/packages/ui/src/helpers/toolbox/messenger/index.ts b/packages/ui/src/helpers/toolbox/messenger/index.ts index af315acb1d7..c833c3fddd0 100644 --- a/packages/ui/src/helpers/toolbox/messenger/index.ts +++ b/packages/ui/src/helpers/toolbox/messenger/index.ts @@ -13,14 +13,12 @@ export interface MessengerType { } export interface MessengerAccessor { - getMessenger: (type: MessengerType) => (M extends ClientMessenger ? Promise : Promise) | undefined; + getMessenger: (type: MessengerType) => (M extends ClientMessenger ? Promise : Promise) | undefined; getActiveMessengers: () => Messenger[]; } export interface MessengerRegistry extends MessengerAccessor { - compoundMessenger>(id: string, extra?: CompoundMessengerOptions & EntryOptions>): MessengerType>; - - simpleMessenger(id: string, extra?: MessengerOptions & EntryOptions>): MessengerType>; + of(id: string, extra?: MessengerOptions & EntryOptions>): MessengerType>; getType(id: string): MessengerType | undefined; @@ -37,26 +35,33 @@ export type EntryOptions = { closeCallback?: (messenger: M) //-- -export interface Message { type: K } +export type ValueBase = { [key: string]: any } + +export interface Message { channel: K } + +type IOType = { input: I, output: O } + +type MessageIOType = IOType & Message -// #region Compound Message Types +export type MessageSchema = Record; -export type MessageSchema = Record; +export type MessageTypes = + T extends Record + ? { [E in AsString]: T[E] extends IOType ? MessageIOType : never; }// & { '_type_flag_fix': MessageIOType<'_type_flag_fix', {}, {}>} + : never; -export type MessageTypes = - T extends Record - ? { [E in K]: {type: E, input: T[E]["input"] /* & Message */ /* & O */, output: T[E]["output"] /* & Message */ /* & I */}; } - : never; +export type AsString = Extract -export type InputMessageTypes> = P[Extract] extends { type: infer K extends string, input: infer I } ? I & Message : never; -export type OutputMessageTypes> = P[Extract] extends { type: infer K extends string, output: infer O } ? O & Message : never; -export type TypeMessageTypes> = P[Extract] extends { type: infer T } ? T : never; +export type InputMessage = AsString> = + T[K] extends MessageIOType ? I & Message : never; -export type InputMessageFor, P = MessageTypes> = P[K] extends { input: infer I } ? I & Message : never; -export type OutputMessageFor, P = MessageTypes> = P[K] extends { output: infer O } ? O & Message : never; +export type OutputMessage = AsString> = + T[K] extends MessageIOType ? O & Message : never; -export type InputFor, P = MessageTypes> = P[K] extends { input: infer I } ? I : never; -export type OutputFor, P = MessageTypes> = P[K] extends { output: infer O } ? O : never; +export type OutputBase> = + T[K] extends MessageIOType ? O : never; + +export type MessageChannels = keyof T; // #endregion @@ -72,8 +77,6 @@ export interface InstanceData { shared: boolean; } -export type InstanceTypes = "singleton-shared" | "temporary-exclusive" | "temporary-shared"; - export type InstanceKey = string; export type MessengerVariants = "server" | "port" | "client"; @@ -91,71 +94,52 @@ export interface Messenger { logger: Logger; } -export interface ServerMessenger extends Messenger, MessengerAccessor { +export interface ServerMessenger extends Messenger, MessengerAccessor { /** * @experimental TBD if this should be exposed better */ sendOutputMessage(sender: Messenger, msg: Promise>): Promise; } -export interface ClientMessenger extends Messenger { - type: MessengerType>, - - sendMessage: (input: I) => Promise; - - handler: H; -} - +export interface ClientMessenger extends Messenger { -// @ts-expect-error - No worries as we are stating the the input will always have a tied to output but TS can't seem to resolve it for some reason -export interface ClientCompoundMessenger> extends ClientMessenger, OutputMessageTypes, H>/* , CompoundInputHandler */ { - type: MessengerType>, + type: MessengerType>, - sendMessage: , K extends Extract>(input: I_M) => Promise> + sendMessage: , K extends AsString>(input: I_M) => Promise> handler: H; } - -export type MessengerHandlerConstructor = (messenger: ClientMessenger) => H - -export type CompoundHandlerConstructor> = (messenger: ClientCompoundMessenger) => H +export type MessengerHandlerConstructor = (messenger: ClientMessenger) => H //-- //-- - export type PossibleTransferable = T | [T, Transferable[]]; -export type ClientInputUnpacker = (input: I) => Transferable[] | undefined; - - -export interface CompoundInputClientMessageExt, P = MessageTypes> extends Message { - resolve(handlers: InputResolverMap): Transferable[] | undefined; +export type InputResolverMap = { + [E in AsString]?: ((msg: InputMessage) => Transferable[]) | undefined } -export type InputResolverMap> = { [E in Extract]?: ((msg: InputMessageFor) => Transferable[]) | undefined} - -export type CompoundInputClientMessage, P = MessageTypes> = CompoundInputClientMessageExt & InputMessageFor - -export type ClientCompoundInputUnpacker> - = , K extends Extract>(input: I_M) => Transferable[] | undefined; +export type ClientInputUnpacker = + >(input: I_M) => Transferable[] | undefined; // -- -export interface MessengerOptions { - handlerConstructor?: MessengerHandlerConstructor, - inputBypassHandler?: ServerInputHandler, - inputTransferHandler?: ClientInputUnpacker; - forcedOutputHandler?: (messenger: ClientMessenger, output: O) => boolean; +export type ClientOutputHandler = (messenger: ClientMessenger, output: OutputMessage) => boolean; + +export interface MessengerOptions { + handlerConstructor?: MessengerHandlerConstructor + inputBypassHandler?: ServerInputHandler, + inputTransferHandler?: ClientInputUnpacker; + independentOutputHandler?: ClientOutputHandler; } -// @ts-expect-error - No worries as we are stating the the input will always have a tied to output but TS can't seem to resolve it for some reason -export interface CompoundMessengerOptions> extends MessengerOptions, OutputMessageTypes, H> { - handlerConstructor?: CompoundHandlerConstructor - inputBypassHandler?: ServerCompoundInputHandler, - inputTransferHandler?: ClientCompoundInputUnpacker; - forcedOutputHandler?: (messenger: ClientCompoundMessenger, output: OutputMessageTypes) => boolean; +export interface MessengerOptionsBase { + handlerConstructor?: MessengerHandlerConstructor + inputBypassHandler?: ServerInputHandler, + inputTransferHandler?: ClientInputUnpacker; + independentOutputHandler?: ClientOutputHandler; } export function createRegistry(): MessengerRegistry { @@ -165,47 +149,22 @@ export function createRegistry(): MessengerRegistry { // WORKER SECTION -export type ServerInputHandler = (messenger: ServerMessenger, input: I, sender?: Messenger) => Promise>; - - -export type ServerCompoundInputHandler> - = , K extends Extract>(messenger: ServerMessenger>, input: I_M, sender?: Messenger) => Promise>> - - -export interface CompoundInputServerMessageExt, P = MessageTypes> extends Message { - resolve(handlers: OutputResolverMap): Promise>; -} - -export type OutputResolverMap> = { [E in Extract]: (msg: InputMessageFor) => Promise>>} - -export type CompoundInputServerMessage, P = MessageTypes> = CompoundInputServerMessageExt & InputMessageFor; +export type ServerSimpleInputHandler = + (messenger: ServerMessenger, input: I, sender?: Messenger) => Promise>; +export type ServerInputHandler = + , I_M extends InputMessage>( + messenger: ServerMessenger>, input: I_M, sender?: Messenger + ) => Promise>> -export function setupAsCompoundServer>( - type: MessengerType, - holder: ChannelHolder, - inputHandler: ServerCompoundInputHandler -): ServerMessenger { - // @ts-expect-error - return setupAsServer, OutputMessageTypes>(type, holder, async (messenger, msg, sender) => { - const copiedMsg = { - ...msg, - resolve: (resolvers: any) => { - const result = (resolvers[msg.type] as ((msg: any) => any) | undefined)!(msg) as PossibleTransferable; - - return (Array.isArray(result)) ? [Object.assign(result[0], { type: msg.type }), result[1]] : Object.assign(result, { type: msg.type }) - } - } as any; - return inputHandler(messenger, copiedMsg, sender); - }); +export type OutputResolverMap = { + [E in AsString]: (msg: InputMessage) => Promise>> } -export function setupAsServer( - type: MessengerType, - holder: ChannelHolder, - inputHandler: ServerInputHandler, -): ServerMessenger { - return setupAsServerInternal(type, holder, inputHandler); +export function setupAsServer( + type: MessengerType, holder: ChannelHolder, inputHandler: ServerInputHandler +): ServerMessenger> { + return setupAsServerInternal(type, holder, inputHandler); } export function isWorker() { @@ -213,4 +172,18 @@ export function isWorker() { return typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope; } +export async function resolveOutput, I_M extends InputMessage>( + msg: I_M, resolvers: OutputResolverMap +): Promise>> { + const result = await (resolvers[msg.channel]!(msg)) + + return (Array.isArray(result) ? [Object.assign(result[0], { channel: msg.channel }), result[1]] : Object.assign(result, { channel: msg.channel })) as PossibleTransferable> +} + +export function resolveInputTransfers>( + msg: I_M, resolvers: InputResolverMap +): Transferable[] | undefined { + return resolvers[msg.channel]?.(msg); +} + // -- diff --git a/packages/ui/src/helpers/toolbox/workers/decompile/index.ts b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts index 274cc93e403..ffc5572704a 100644 --- a/packages/ui/src/helpers/toolbox/workers/decompile/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts @@ -1,8 +1,14 @@ -import type {DecompileInput, DecompilerResult} from "#ui/helpers/toolbox/decompile"; +import type { + DecompilerMessages, +} from "#ui/helpers/toolbox/decompile"; import { decompileFileContent } from "#ui/helpers/toolbox/decompile/impl"; -import { setupAsServer } from "#ui/helpers/toolbox/messenger"; +import {resolveOutput, setupAsServer} from "#ui/helpers/toolbox/messenger"; import {decompileMessengerType, zipHandlerType} from "#ui/helpers/toolbox/workers"; -setupAsServer(decompileMessengerType, self as unknown as Worker, async (messenger, msg) => { - return decompileFileContent(messenger.logger, (await messenger.getMessenger(zipHandlerType))?.handler, msg); +setupAsServer(decompileMessengerType, self as unknown as Worker, async (messenger, msg) => { + return resolveOutput(msg, { + main: async (msg) => { + return decompileFileContent(messenger.logger, (await messenger.getMessenger(zipHandlerType))?.handler, msg); + } + }) }) diff --git a/packages/ui/src/helpers/toolbox/workers/index.ts b/packages/ui/src/helpers/toolbox/workers/index.ts index f39a4e10087..00588303e55 100644 --- a/packages/ui/src/helpers/toolbox/workers/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/index.ts @@ -1,11 +1,13 @@ -import { useRouter } from "vue-router"; - -import type {DecompileHandler, DecompileInput, DecompilerResult} from "#ui/helpers/toolbox/decompile"; +import type { + DecompileHandler, + DecompilerMessages, +} from "#ui/helpers/toolbox/decompile"; import { - type ClientCompoundMessenger, type ClientMessenger, + type ClientMessenger, createRegistry, - isWorker, - type MessengerType + type InputResolverMap, + type MessengerType, + resolveInputTransfers, } from '#ui/helpers/toolbox/messenger'; // import { AtlasMessenger, handlerConstructor as atlasHandlerConstructor } from "./atlas/common"; import { handlerConstructor as zipHandlerConstructor,type ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; @@ -16,18 +18,18 @@ import ZipWorker from "./zip?worker"; export const messengerRegistry = createRegistry(); -export type ZipHandlerMessenger = ClientCompoundMessenger; -export const zipHandlerType: MessengerType = messengerRegistry.compoundMessenger("zip-handler", { +export type ZipHandlerMessenger = ClientMessenger; + +export const zipHandlerType: MessengerType = messengerRegistry.of("zip-handler", { handlerConstructor: zipHandlerConstructor, - inputBypassHandler: import.meta.env.DEV ? (await import('./zip')).getMessageHandler() : undefined, - inputTransferHandler: ((input) => { - return input.resolve({ + inputTransferHandler: (input) => { + return resolveInputTransfers(input, { node_write: (msg) => [msg.buffer], read_zip_tree: (msg) => [msg.buffer], - }); - }), - forcedOutputHandler: (messenger, msg) => { - if (msg.type == "update_node_data") { + } as InputResolverMap); + }, + independentOutputHandler: (messenger, msg) => { + if (msg.channel == "update_node_data") { const data = msg as ZipMessages['update_node_data']['output']; const root = messenger.handler.zipArchivesByHash.get(data.root); @@ -59,23 +61,28 @@ export const zipHandlerType: MessengerType = messengerRegis // }); // -type DecompileMessenger = ClientMessenger; -export const decompileMessengerType: MessengerType = messengerRegistry.simpleMessenger("decompiler-vf", { - inputBypassHandler: import.meta.env.DEV ? async (messenger, input) => { - return (await import('../decompile/impl.ts')).decompileFileContent(messenger.logger, (await messengerRegistry.getMessenger(zipHandlerType)), input) - } : undefined, - handlerConstructor: (messenger) => messenger.sendMessage, - inputTransferHandler: (file) => [file.targetFile.content], +type DecompileMessenger = ClientMessenger; + +export const decompileMessengerType: MessengerType = messengerRegistry.of("decompiler-vf", { + handlerConstructor: (messenger) => { + return { + decompile: async (input) => { + return await messenger.sendMessage({ + ...input, + channel: 'main' + }) + } + }; + }, + inputTransferHandler: (input) => { + return resolveInputTransfers(input, { + main: (msg) => [msg.targetFile.content], + } as InputResolverMap) + }, shared: true, timeout: 30000, count: 1 }); + messengerRegistry.setWorkerFactoryFor(decompileMessengerType, () => new DecompileWorker()); messengerRegistry.setWorkerFactoryFor(zipHandlerType, () => new ZipWorker()); - -if (!isWorker()) { - // TODO: IMPLEMENT CORRECT CLEANUP I.E. WHEN PROPERLY CHANGING ROOT PAGES? - useRouter().beforeEach((to, from, next) => { - if (to.params.project != from.params.project) messengerRegistry.closeAll(true); - }) -} diff --git a/packages/ui/src/helpers/toolbox/workers/zip/api.ts b/packages/ui/src/helpers/toolbox/workers/zip/api.ts index f1941520026..b12f1fa4d7d 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/api.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/api.ts @@ -8,57 +8,60 @@ import { reconstituteRoot, type RootNodeTarget, type RootZipTreeNode, type Sha512, type SourceOptions, type ZipHandler } from "#ui/helpers/toolbox/zip"; +import type {MessageTypes} from "#ui/helpers/toolbox/messenger"; export interface DataInput { buffer: ArrayBuffer } -export type ZipMessages = { - lazy_read: { - input: { key: keyof LazyDataAccess } & NodeTarget, - output: { value: Awaited> } - }, - source_read: { - input: NodeTarget & { options?: SourceOptions } - output: DecompilerResult - } - node_read: { - input: NodeTarget, - output: { buffer: ArrayBuffer } - }, - node_write: { - input: NodeTarget & { nodeType?: "zip" | "file", options?: ZipWriterAddDataOptions } & DataInput, - output: { wasWritten: boolean } - }, - read_zip_tree: { - input: { name?: string, url?: string, hash?: string } & DataInput, - output: { node: RootZipTreeNode } - }, - get_zip_tree: { - input: { value: string | Sha512 }, - output: { node: RootZipTreeNode | undefined } - }, - collect_files: { - input: RootNodeTarget & { matcher?: FileMatcher }, - output: { files: FileDetails[] } - }, - collect_files_hashed: { - input: RootNodeTarget & { matcher?: FileMatcher, deduplicate?: boolean}, - output: { files: HashedFileContent[] } - }, - close_zip_tree: { - input: RootNodeTarget, - output: { err?: Error, wasClosed: boolean } - }, - close_all_zip_trees: { - input: {}, - output: { err?: Error, wasClosed: boolean } - }, - update_node_data: { - input: NodeTarget, - output: NodeTarget +export type ZipMessages = MessageTypes< + { + lazy_read: { + input: { key: keyof LazyDataAccess } & NodeTarget, + output: { value: Awaited> } + }, + source_read: { + input: NodeTarget & { options?: SourceOptions } + output: DecompilerResult + } + node_read: { + input: NodeTarget, + output: { buffer: ArrayBuffer } + }, + node_write: { + input: NodeTarget & { nodeType?: "zip" | "file", options?: ZipWriterAddDataOptions } & DataInput, + output: { wasWritten: boolean } + }, + read_zip_tree: { + input: { name?: string, url?: string, hash?: string } & DataInput, + output: { node: RootZipTreeNode } + }, + get_zip_tree: { + input: { value: string | Sha512 }, + output: { node: RootZipTreeNode | undefined } + }, + collect_files: { + input: RootNodeTarget & { matcher?: FileMatcher }, + output: { files: FileDetails[] } + }, + collect_files_hashed: { + input: RootNodeTarget & { matcher?: FileMatcher, deduplicate?: boolean}, + output: { files: HashedFileContent[] } + }, + close_zip_tree: { + input: RootNodeTarget, + output: { err?: Error, wasClosed: boolean } + }, + close_all_zip_trees: { + input: { }, + output: { err?: Error, wasClosed: boolean } + }, + update_node_data: { + input: NodeTarget, + output: NodeTarget + } } -} +> export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler = (messenger: ZipHandlerMessenger) => { const hashToUrls = new Map(); @@ -66,13 +69,13 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler zipArchivesByHash: new Map(), urlToHash: new Map(), readLazyData: async function (target: FileZipTreeNode | NodeTarget, key: K): Promise>> { - return (await messenger.sendMessage({ type: "lazy_read", ...getAsTarget(target), key: key })).value as Awaited>; + return (await messenger.sendMessage({ channel: "lazy_read", ...getAsTarget(target), key: key })).value as Awaited>; }, readSource: async function (target: FileZipTreeNode | NodeTarget, options?: SourceOptions): Promise { - return (await messenger.sendMessage({ type: 'source_read', ...getAsTarget(target), options: options})) + return (await messenger.sendMessage({ channel: 'source_read', ...getAsTarget(target), options: options})) }, readNodeData: async (target: FileZipTreeNode | NodeTarget) => { - return (await messenger.sendMessage({ type: "node_read", ...getAsTarget(target)})).buffer + return (await messenger.sendMessage({ channel: "node_read", ...getAsTarget(target)})).buffer }, getOrReadZipTree: async function (buffer, {name, url, hash} = {}) { let node: RootZipTreeNode | undefined; @@ -94,7 +97,7 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler if (node) return node; } - const node = reconstituteRoot(messenger.handler, (await messenger.sendMessage({ type: "read_zip_tree", buffer, name: name, url: url, hash: hash })).node); + const node = reconstituteRoot(messenger.handler, (await messenger.sendMessage({ channel: "read_zip_tree", buffer, name: name, url: url, hash: hash })).node); if (filteredURL) { this.urlToHash.set(filteredURL, node.key); @@ -110,17 +113,17 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler return node; }, writeNodeData: async (target, buffer, type, options) => { - return (await messenger.sendMessage({type: "node_write", ...getAsTarget(target), buffer, nodeType: type, options})).wasWritten; + return (await messenger.sendMessage({ channel: "node_write", ...getAsTarget(target), buffer, nodeType: type, options})).wasWritten; }, collectFiles: async (root, matcher) => { - return (await messenger.sendMessage({ type: "collect_files", matcher: matcher, root: (root as RootZipTreeNode).key ?? root})).files + return (await messenger.sendMessage({ channel: "collect_files", matcher: matcher, root: (root as RootZipTreeNode).key ?? root})).files }, collectHashedFiles: async (root, matcher, deduplicate)=> { - return (await messenger.sendMessage({ type: "collect_files_hashed", matcher: matcher, root: (root as RootZipTreeNode).key ?? root, deduplicate: deduplicate})).files + return (await messenger.sendMessage({ channel: "collect_files_hashed", matcher: matcher, root: (root as RootZipTreeNode).key ?? root, deduplicate: deduplicate})).files }, closeZipTree: async function (root){ const hash: Sha512 = (root as RootZipTreeNode).key ?? root; - const value = await messenger.sendMessage({ type: "close_zip_tree", root: (root as RootZipTreeNode).key ?? root}); + const value = await messenger.sendMessage({ channel: "close_zip_tree", root: (root as RootZipTreeNode).key ?? root}); const urls = hashToUrls.get(hash); if (urls) for (const url of urls) this.urlToHash.delete(url); @@ -129,7 +132,7 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler return value; }, closeAllZipTrees: async function () { - const value = await messenger.sendMessage({ type: "close_all_zip_trees" }); + const value = await messenger.sendMessage({ channel: "close_all_zip_trees" }); this.zipArchivesByHash.clear(); this.urlToHash.clear(); @@ -154,7 +157,7 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler if (localOnly) return undefined; - const rawNode = (await messenger.sendMessage({ type: "get_zip_tree", value: value })).node; + const rawNode = (await messenger.sendMessage({ channel: "get_zip_tree", value: value })).node; if (rawNode == undefined) return undefined; diff --git a/packages/ui/src/helpers/toolbox/workers/zip/index.ts b/packages/ui/src/helpers/toolbox/workers/zip/index.ts index fe43ba49b96..474467207a7 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/index.ts @@ -3,7 +3,12 @@ import { type FileContentPromise, type FileDetails, type FileTypeInfo, type HashedFileContent, resolveContent } from "#ui/helpers/toolbox/file"; -import { type MessageTypes, type ServerCompoundInputHandler, setupAsCompoundServer } from "#ui/helpers/toolbox/messenger"; +import { + isWorker, + resolveOutput, + type ServerInputHandler, + setupAsServer +} from "#ui/helpers/toolbox/messenger"; import { zipHandlerType } from "#ui/helpers/toolbox/workers"; import type { ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; import type { @@ -56,10 +61,6 @@ function getZipNode(value: string) { return undefined; } -export function getMessageHandler

>(): ServerCompoundInputHandler { - return messageHandler as unknown as ServerCompoundInputHandler; -} - async function isValidFor(path: string, fileType: () => Promise, pathMatcher?: StringMatcher, mimeMatcher?: StringMatcher) { if (pathMatcher) { const negate = pathMatcher.negate; @@ -83,8 +84,8 @@ async function isValidFor(path: string, fileType: () => Promise = function (messenger, msg, sender) { - return msg.resolve({ +export const messageHandler: ServerInputHandler = function (messenger, msg, sender) { + return resolveOutput(msg, { lazy_read: async (msg) => { const node = getNodeFrom(msg); @@ -316,4 +317,5 @@ function copyNodeData(clone: N, oldNode: N): N { return clone; } -setupAsCompoundServer(zipHandlerType, self as unknown as Worker, getMessageHandler()); +/** In development the handler is also loaded on the main thread as a bypass, where no server is set up. */ +if (isWorker()) setupAsServer(zipHandlerType, self as unknown as Worker, messageHandler); diff --git a/packages/ui/src/helpers/toolbox/zip/impl.ts b/packages/ui/src/helpers/toolbox/zip/impl.ts index 0b0a2ba45dc..e605f87ba68 100644 --- a/packages/ui/src/helpers/toolbox/zip/impl.ts +++ b/packages/ui/src/helpers/toolbox/zip/impl.ts @@ -226,10 +226,10 @@ class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { const lazyValue = this.arrayBuffer.lazyMap(async (buffer) => { let error: string | undefined = undefined; - const decompilier = (await messengerRegistry?.getMessenger(decompileMessengerType))!; + const decompiler = (await messengerRegistry?.getMessenger(decompileMessengerType))!; if (buffer != undefined) { - return decompilier({ + return await decompiler.decompile({ targetFile: { fullPath: this.path, name: this.name, content: buffer, rootKey: this.root.key } as ZipFileContent, resources: this.parent?.children.values().map(entry => { return { fullPath: entry.path, name: entry.name } }).toArray(), options: options diff --git a/packages/ui/src/helpers/toolbox/zip/index.ts b/packages/ui/src/helpers/toolbox/zip/index.ts index 5f0034a1cd9..a11d4b12ce9 100644 --- a/packages/ui/src/helpers/toolbox/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/zip/index.ts @@ -370,7 +370,7 @@ function reconstituteNodes(handler: ZipHandler, root: Roo reconstitutedNode.parent = parent; } - if (node.type == "file") (node as unknown as DelegateFileZipTreeNode).finishNodeSetup(handler) + if (node.type == "file" || node.type == "zip") (node as unknown as DelegateFileZipTreeNode).finishNodeSetup(handler) return reconstitutedNode as N; } From 4e2b0687b4ac3ac7d77458c519855daff610eaf1 Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Mon, 12 Oct 2026 22:29:57 -0500 Subject: [PATCH 03/18] refactor: Rework Messenger API types, Setup more async zip handling, - Add Range Reader that attempts to read data via range requests to specifically get the Central Directory first and slowly load the rest of the zip data async - Switch from using hash for archive keys to random UUID v7 - Further refinement of messenger API with more network based terminology and better type handling --- packages/ui/package.json | 1 + .../ui/src/helpers/toolbox/decompile/impl.ts | 12 +- .../ui/src/helpers/toolbox/decompile/index.ts | 16 +- packages/ui/src/helpers/toolbox/file/index.ts | 15 +- .../helpers/toolbox/messenger/implement.ts | 301 +++++---- .../ui/src/helpers/toolbox/messenger/index.ts | 230 +++---- .../toolbox/workers/decompile/index.ts | 8 +- .../ui/src/helpers/toolbox/workers/index.ts | 65 +- .../ui/src/helpers/toolbox/workers/zip/api.ts | 208 ++++--- .../src/helpers/toolbox/workers/zip/index.ts | 219 ++++--- packages/ui/src/helpers/toolbox/zip/impl.ts | 149 +++-- packages/ui/src/helpers/toolbox/zip/index.ts | 163 +++-- .../helpers/toolbox/zip/range-cache-reader.ts | 586 ++++++++++++++++++ 13 files changed, 1378 insertions(+), 595 deletions(-) create mode 100644 packages/ui/src/helpers/toolbox/zip/range-cache-reader.ts diff --git a/packages/ui/package.json b/packages/ui/package.json index 61ec8786be6..688664a67e8 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -108,6 +108,7 @@ "postprocessing": "^6.37.6", "qrcode.vue": "^3.4.1", "three": "^0.172.0", + "uuid": "^14.0.2", "vue-i18n": "^10.0.0", "vue-select": "4.0.0-beta.6", "vue-typed-virtual-list": "^1.0.10", diff --git a/packages/ui/src/helpers/toolbox/decompile/impl.ts b/packages/ui/src/helpers/toolbox/decompile/impl.ts index 3ec806a76c5..7e376f7c860 100644 --- a/packages/ui/src/helpers/toolbox/decompile/impl.ts +++ b/packages/ui/src/helpers/toolbox/decompile/impl.ts @@ -72,7 +72,7 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle if (options == null) options = DEFAULT_DECOMPILE_OPTIONS; - const rootKey = (targetFile as any).rootKey ? (targetFile as ZipFileContent).rootKey : undefined; + const rootKey = (targetFile as Partial).root ?? undefined; try { // const pathParts = targetFile.fullPath.split('.jar'); \n const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); \n @@ -84,9 +84,9 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle source: async (name) => { if (name === withoutClassSuffix) { return new Uint8Array(buffer); - } else if (rootKey && name.includes(withoutClassSuffix)) { + } else if (rootKey != null && name.includes(withoutClassSuffix)) { // TODO: THIS DOES NOT PROPERLY HANDLE WHICH JAR WE NEED TO LOOK AT SO IF DUPLICATE FILES EXIST, WE MAY GET THE WRONG ONE!!!! - const buffer = await zipHandler?.readNodeData({ path: name + ".class", root: rootKey, relative: true }); + const buffer = await zipHandler?.readNodeData({ path: "/" + name + ".class", root: rootKey, relative: true }); if (buffer) return new Uint8Array(buffer); } @@ -98,9 +98,11 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle options: options.options }); + const unnormalizedPath = withoutClassSuffix.slice(1); + if (typeof decompiled === 'string') { source = decompiled ?? undefined; - } else if (!decompiled[withoutClassSuffix]) { + } else if (!decompiled[unnormalizedPath]) { console.error("Decompilation failed for", targetFile.fullPath, decompiled); error = "// [Decompilation output either is kotlin or their is issues just a fyi!] \n\n"; @@ -109,7 +111,7 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle error = `${error} \n\n Class: ${key} \n\n ${value}` } } else { - source = decompiled[withoutClassSuffix] ?? undefined; + source = decompiled[unnormalizedPath] ?? undefined; } } catch (err: any) { error = `[${err?.message ?? "Decompilation has due to an unknown error being thrown!"}]`; diff --git a/packages/ui/src/helpers/toolbox/decompile/index.ts b/packages/ui/src/helpers/toolbox/decompile/index.ts index e75b9b8da6f..84eea6680dc 100644 --- a/packages/ui/src/helpers/toolbox/decompile/index.ts +++ b/packages/ui/src/helpers/toolbox/decompile/index.ts @@ -5,12 +5,16 @@ export const ALL_DECOMPILER_TYPES = ['cfr', 'vineflower', 'procyon', 'krakatau'] export type DecompilerTypes = typeof ALL_DECOMPILER_TYPES[number]; -export type DecompileInput = { - targetFile: FileContent, resources?: FileDetails[], options?: DecompileOptions -}; +export interface DecompileInput { + targetFile: FileContent, + transferBuffer?: boolean, + resources?: FileDetails[], + options?: DecompileOptions +} -export type DecompileOptions = { - decompiler?: DecompilerTypes, options?: Record +export interface DecompileOptions { + decompiler?: DecompilerTypes, + options?: Record } export const DEFAULT_DECOMPILE_OPTIONS = { @@ -24,7 +28,7 @@ export const DEFAULT_DECOMPILE_OPTIONS = { export type DecompilerMessages = MessageTypes< { - main: {input: DecompileInput, output: DecompilerResult} + main: { serverbound: DecompileInput, clientbound: DecompilerResult } } > diff --git a/packages/ui/src/helpers/toolbox/file/index.ts b/packages/ui/src/helpers/toolbox/file/index.ts index 2db29462930..01519c10308 100644 --- a/packages/ui/src/helpers/toolbox/file/index.ts +++ b/packages/ui/src/helpers/toolbox/file/index.ts @@ -55,16 +55,15 @@ export interface FileTypeInfo { method?: 'content-analysis' | 'extension-lookup'; } -type PromiseMaybe = T extends undefined ? R : Promise +export type AsyncMaybe = T extends D ? R : Promise -export function getFileTypeInfo(path: string, data?: T): PromiseMaybe { - const promiseLike: Promise | undefined = data ? getFileTypeFromData(data) : undefined; +export function getFileTypeInfo(path: string, data?: T): AsyncMaybe { + const promiseLike = data ? getFileTypeFromData(data) : undefined; - if (promiseLike != null) { - return promiseLike.then((result) => result ?? getFileTypeFromExt(path)) as PromiseMaybe; - } else { - return getFileTypeFromExt(path) as PromiseMaybe; - } + return ((promiseLike != null) + ? promiseLike.then((result) => result ?? getFileTypeFromExt(path)) + : getFileTypeFromExt(path) + ) as AsyncMaybe } function getFileTypeFromExt(path: string): FileTypeInfo | null { diff --git a/packages/ui/src/helpers/toolbox/messenger/implement.ts b/packages/ui/src/helpers/toolbox/messenger/implement.ts index 8c43f057d90..959a7d79b20 100644 --- a/packages/ui/src/helpers/toolbox/messenger/implement.ts +++ b/packages/ui/src/helpers/toolbox/messenger/implement.ts @@ -8,56 +8,57 @@ import { } from '#ui/helpers/toolbox/misc/logging'; import { - type ChannelHolder, type ClientMessenger, type MessengerOptions, - type EntryOptions, type InputMessage, + type ServerboundMessage, type InstanceConfig, type InstanceKey, isWorker, - type Message, + type Channel, type MessageTypes, type Messenger, type MessengerRegistry, type MessengerType, - type MessengerVariants, type OutputMessage, - type OutputResolverMap, - type PossibleTransferable, type ServerInputHandler, + type MessengerVariants, + type ClientboundMessage, + type ServerMessageHandler, type ServerMessenger, - type TypeId, type AsString, type InputResolverMap, type MessengerOptionsBase, - type ValueBase, + type TypeId, + type ServerboundChannels, + type ServerHandlerResolver, + type MaybeTransferable, } from '.'; // #region MessengerType Types -type InternalMessenger = M & C & { terminate(): void; onTerminate: EventSource; }; +type ChannelHolder = Worker | MessagePort; -interface MessengerTypeImpl extends MessengerType { +type InternalMessenger = Messenger & C & { terminate(): void; onTerminate: EventSource; }; + +interface MessengerTypeImpl extends MessengerType { registry: MessengerRegistryImpl; - instances?: (() => InternalMessenger) | Map>; - wrapWith: (channelHolder: () => ChannelHolder, key: InstanceKey) => InternalMessenger; + instances?: (() => InternalMessenger) | Map>; + wrapWith: (channelHolder: () => ChannelHolder, key: InstanceKey) => InternalMessenger; - getOrCreateWith: (baseMessenger?: () => ChannelHolder, instanceKey?: string) => M extends ClientMessenger ? M & ChannelHolder & H : M & ChannelHolder; + getOrCreateWith: (baseMessenger?: () => ChannelHolder, instanceKey?: string) => Messenger & ChannelHolder; workerFactory?: () => Worker; - getInstances: () => M[]; + getInstances: () => Messenger[]; } -type WrapperFunc = (type: MessengerTypeImpl, key: InstanceKey, holder: C) => InternalMessenger; - -function typeFor( - id: string, instanceConfig: InstanceConfig, closeCallback?: (messenger: M) => void) { +function typeFor( + id: string, instanceConfig: InstanceConfig, closeCallback?: (messenger: Messenger) => void) { return { id: id, instances: instanceConfig.count == 1 ? undefined : new Map(), getInstances() { if (this.instances == null) return []; - return (this.instances as Partial>).values ? (this.instances as Map).values().toArray() : [(this.instances as unknown as M)]; + return (this.instances as Partial>>).values ? (this.instances as Map>).values().toArray() : [(this.instances as unknown as Messenger)]; }, getOrCreateWith(holder, instanceKey) { if ((instanceConfig.count == 1) && this.instances != null) { - return (this.instances as () => M)(); + return (this.instances as () => Messenger)(); } if (holder == null) holder = this.workerFactory; @@ -70,7 +71,7 @@ function typeFor( if (holder == null) throw new Error(`Unable to create messenger for '${this.id}' as the entry used was found to be null! Did you setup the main thread worker factory or pass in the port to the call!`) - let wrappedMessenger: InternalMessenger; + let wrappedMessenger: InternalMessenger; if (instanceConfig.count == 1) { wrappedMessenger = this.wrapWith(holder, "primary"); @@ -81,14 +82,14 @@ function typeFor( this.instances = () => wrappedMessenger; } else { - const instances = (this.instances as Map>); + const instances = (this.instances as Map>); // TODO: ADD TIME BASED HASH? const key = `temp-${instances.size}`; wrappedMessenger = this.wrapWith(holder, key); - (this.instances as Map).set(wrappedMessenger.key!, wrappedMessenger); + (this.instances as Map>).set(wrappedMessenger.key!, wrappedMessenger); } wrappedMessenger.postMessage?.({ channel: "set-key", key: wrappedMessenger.key} as SetOffThreadKeyMessage); @@ -98,12 +99,12 @@ function typeFor( close(forceClosed) { if (instanceConfig.count == 1) { if (this.instances != undefined) { - const instance = (this.instances as () => InternalMessenger)(); + const instance = (this.instances as () => InternalMessenger)(); closeCallback?.(instance); instance.postMessage({channel: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) } } else { - const map = (this.instances as Map>); + const map = (this.instances as Map>); map.values().forEach(entry => { closeCallback?.(entry) entry.postMessage({channel: "force-close-root", forceClose: forceClosed } as ForceCloseMessengerRootMessage) @@ -112,16 +113,13 @@ function typeFor( } } /* wrapWith: wrapWith, */ - } as MessengerTypeImpl; + } as MessengerTypeImpl; } -// #endregion - - -// #region MessengerRegistry Types +// --- interface MessengerRegistryImpl extends MessengerRegistry { - serverMessenger?: ServerHandlerImpl, + serverMessenger?: ServerHandlerImpl, types: Map>; logger: LoggerFactory; onLogFactoryChange: EventSource<(logger: LoggerFactory) => void>; @@ -131,15 +129,15 @@ export function createRegistryInternal(): MessengerRegistry { return { types: new Map(), logger: logFactory(), - of: function (id: string, extra: MessengerOptions & EntryOptions>): MessengerType> { + of: function (id: string, extra: MessengerOptions & InstanceConfig): MessengerType { const instanceConfig: InstanceConfig = { timeout: extra?.timeout ?? undefined, shared: extra?.shared ?? true, count: extra.count ?? 1, }; - const type = typeFor>(id, instanceConfig); + const type = typeFor(id, instanceConfig); - type.wrapWith = (holder, key) => setupAsClient(type, "client", key, extra.inputBypassHandler != null ? {} as any : holder(), extra); + type.wrapWith = (holder, key) => setupAsClient(type, "client", key, extra.bypassServerMessageHandler != null ? {} as any : holder(), extra); type.registry = this; type.instanceConfig = instanceConfig; @@ -148,7 +146,7 @@ export function createRegistryInternal(): MessengerRegistry { return type; }, - getType: function (id: string): MessengerType | undefined { + getType: function (id: string): MessengerType | undefined { if (!this.types.has(id)) throw new Error(`MessengerRegistry did not contain entry '${id}'`); return this.types.get(id)!; }, @@ -163,52 +161,38 @@ export function createRegistryInternal(): MessengerRegistry { type.close(forceClosed); }) }, - getMessenger(type: MessengerType) { + getMessenger(type: MessengerType) { if (this.serverMessenger) { return this.serverMessenger.getMessenger(type); } else { - return (type as MessengerTypeImpl).getOrCreateWith(); + return (type as MessengerTypeImpl).getOrCreateWith(); } }, getActiveMessengers() { - return this.types.values().flatMap((type) => (type as MessengerTypeImpl).getInstances()).toArray(); + return this.types.values().flatMap((type) => (type as MessengerTypeImpl).getInstances()).toArray(); }, - setWorkerFactoryFor(type, worker) { - if (!isWorker()) (type as MessengerTypeImpl).workerFactory = worker; + forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void) { + for (const connection of this.getActiveMessengers()) { + if (connection !== messenger) callback(connection); + } + }, + setWorkerFactoryFor(type, worker) { + if (!isWorker()) (type as MessengerTypeImpl).workerFactory = worker; } } as MessengerRegistryImpl; } -// #endregion - - // --- -// #region Base Messenger Types - type OnTerminateCallback = (forceClose: boolean) => void -//#region Impl handling - -function handleInput(messenger: Messenger & ChannelHolder, taskHolder: TaskHolder, input: any, transfer?: Transferable[]) { - const task = taskHolder.startTask(); - - messenger.postMessage({ - channel: "input", - data: input, - key: task.transactionKey, - } as MessengerInputMessage, transfer ?? []); - - return task.promise; -} - -interface RootMessageHandler = Message> extends Messenger { +interface RootMessageHandler = Channel> extends Messenger { handleRootMessage(msg: MSG, ports?: readonly MessagePort[]): Promise; } type TaskHolder = { hasTask: (transactionKey: string) => boolean, - startTask: () => ({ transactionKey: string, promise: Promise }), + startTask: () => ({ transactionKey: string, promise: Promise }), endTask: (transactionKey: string, value: any, isError?: boolean) => boolean; } @@ -217,10 +201,10 @@ function createTaskHolder(): TaskHolder{ let index = 0; return { hasTask: (key) => outstandingMessages.has(key), - startTask: () => { + startTask: () => { const key = `${index++}`; - const promiseHolder = Promise.withResolvers(); + const promiseHolder = Promise.withResolvers(); outstandingMessages.set(key, promiseHolder); @@ -245,7 +229,7 @@ function createTaskHolder(): TaskHolder{ } } -interface ClientHandlerImpl extends RootMessageHandler, ClientMessenger { +interface ClientHandlerImpl extends RootMessageHandler, ClientMessenger { taskHolder: TaskHolder; } @@ -253,23 +237,33 @@ const FORCED_OUTPUT_KEY = "INTERNAL_FORCED_OUTPUT"; function clientMessageHandler( messenger: InternalMessenger, - options: MessengerOptionsBase, -): ClientHandlerImpl & InternalMessenger, C> & (H extends Function ? {} : H) { + options: MessengerOptions, +): ClientHandlerImpl & InternalMessenger & (H extends object ? H : {}) { const taskHolder = createTaskHolder(); - const sendMessage: ClientMessenger["sendMessage"] = options.inputBypassHandler ? async (input) => { - const output = (await options.inputBypassHandler!(messenger as any, input)); - messenger.terminate(); - messenger.isAlive = false; - messenger.onTerminate.stream.sink(true) - return Array.isArray(output) ? output[0] : output; - } : (input) => handleInput(messenger, taskHolder, input, options.inputTransferHandler?.(input)); + async function handleInput, M extends ServerboundMessage>(msg: M) { + if (options.bypassServerMessageHandler != null) { + const output = await options.bypassServerMessageHandler!(fullHandler as Messenger as ServerMessenger, msg, serverHandlerResolver()); + messenger.terminate(); + messenger.isAlive = false; + messenger.onTerminate.stream.sink(true) + return (Array.isArray(output) ? output[0] : output) + } else { + const task = taskHolder.startTask extends never ? void : ClientboundMessage>(); + const wrappedMsg: MessengerServerboundMessage = { + channel: "input", data: msg, key: task.transactionKey, + }; + fullHandler.postMessage(wrappedMsg, options.serverMessageTransferHandler?.(msg, (msg, resolvers) => (resolvers[msg.channel]?.(msg) ?? [])) ?? []); + + return await task.promise; + } + } const fullHandler = Object.assign(messenger, { taskHolder: taskHolder, - async handleRootMessage(msg: ClientboundMessages){ + async handleRootMessage(msg: ClientboundMessengerChannels){ if (msg.channel == "operation-error" || msg.channel == "output") { - handleOutputMessages(this, msg, messenger.logger, taskHolder, options.independentOutputHandler) + handleOutputMessages(this, msg, messenger.logger, taskHolder, options.independentClientMessageHandler) } else if (messenger.variant == "client") { if (msg.channel == "logging") { // MAIN THREAD ONLY const factory = messenger.logger.factory; @@ -278,12 +272,12 @@ function clientMessageHandler).type}'!`) } } else if (msg.channel == "setup-root") { // MAIN THREAD ONLY const registry = messenger.type.registry; - const newMessenger = await registry.getMessenger(registry.getType(msg.id)!) as InternalMessenger; + const newMessenger = await registry.getMessenger(registry.getType(msg.id)!) as InternalMessenger & ClientMessenger; const channel = new MessageChannel(); @@ -302,19 +296,17 @@ function clientMessageHandler & InternalMessenger); + sendMessage: handleInput, + } as ClientHandlerImpl & InternalMessenger & (H extends object ? H : {})); - const handler = options.handlerConstructor?.(fullHandler as any) ?? (null as any) + fullHandler.handler = options.handlerConstructor(fullHandler); - fullHandler.handler = handler; + if (typeof fullHandler.handler == 'object') Object.assign(fullHandler, fullHandler.handler); - if (typeof handler === "object") Object.assign(fullHandler, handler); - - return fullHandler as any; + return fullHandler; } -function handleOutputMessages(messenger: ClientHandlerImpl, msg: MessengerOperationError | MessengerOutputMessage, logger: Logger, taskHolder: TaskHolder, forcedOutputHandler?: (messenger: ClientHandlerImpl, msg: OutputMessage) => boolean) { +function handleOutputMessages(messenger: ClientHandlerImpl, msg: MessengerOperationError | MessengerOutputMessage, logger: Logger, taskHolder: TaskHolder, forcedOutputHandler?: (messenger: ClientHandlerImpl, msg: ClientboundMessage) => boolean) { if (msg.channel == "operation-error") { if (taskHolder.hasTask(msg.key)) { logger.error("An error occurred when attempting to complete a task!", {err: msg.error, obj: msg.info}) @@ -342,8 +334,8 @@ function handleOutputMessages(messenger: ClientHandle } } -interface ServerHandlerImpl extends RootMessageHandler, ServerMessenger, Messenger { - handleServerInput(sender: InternalMessenger, msg: MessengerInputMessage): Promise; +interface ServerHandlerImpl extends RootMessageHandler, ServerMessenger, Messenger { + handleServerInput, M extends ServerboundMessage>(sender: InternalMessenger, msg: MessengerServerboundMessage): Promise; attemptTimeoutWait(): void; startShutDown(): void; setupPort(id: TypeId, transactionKey: string, key: InstanceKey, port: MessagePort): void; @@ -352,15 +344,15 @@ interface ServerHandlerImpl extends RootMessage function serverMessageHandler( messenger: InternalMessenger, - onInput: ServerInputHandler -): ServerHandlerImpl> & InternalMessenger { + onInput: ServerMessageHandler +): ServerHandlerImpl & InternalMessenger { let currentCloseFuncId: ReturnType | undefined = undefined; const currentInputKeys: Set = new Set(); - const peerPorts: Map = new Map(); + const peerPorts: Map & MessagePort> = new Map(); const waitingForPeerPort: Map, promiseHolder: PromiseWithResolvers}> = new Map(); - const impl: ServerHandlerImpl & InternalMessenger = Object.assign(messenger, { + const impl: ServerHandlerImpl & InternalMessenger = Object.assign(messenger, { async handleRootMessage(msg, ports) { if (msg.channel == "input") { this.handleServerInput(this, msg) @@ -385,7 +377,7 @@ function serverMessageHandler( return true; }, - async handleServerInput(sender: InternalMessenger, msg: MessengerInputMessage) { + async handleServerInput, M extends ServerboundMessage>(sender: InternalMessenger, msg: MessengerServerboundMessage) { const transactionKey = `${fullInstanceKeyFrom(sender)}/${msg.key}`; currentInputKeys.add(transactionKey); @@ -395,7 +387,7 @@ function serverMessageHandler( try { logger.debug(`Attempting to process '${msg.channel}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}' `, { obj: msg }) - const result = await onInput(this, msg.data, sender); + const result = await onInput(this, msg.data, serverHandlerResolver(), sender); const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; logger.debug(`Finished processing '${msg.channel}' on '${fullInstanceKeyFrom(this)}' for '${debugKeyFrom(sender)}'!`, { obj: msg }) @@ -444,8 +436,8 @@ function serverMessageHandler( const data = waitingForPeerPort.get(fullTransactionKey)!; waitingForPeerPort.delete(fullTransactionKey); - const portMessenger = ((data?.type ?? this.type.registry.getType(id)) as MessengerTypeImpl)! - .getOrCreateWith(() => port as Messenger & MessagePort, key) as InternalMessenger & MessagePort; + const portMessenger = ((data?.type ?? this.type.registry.getType(id)) as MessengerTypeImpl) + .getOrCreateWith(() => port as Messenger & MessagePort, key) as InternalMessenger & RootMessageHandler & MessagePort; portMessenger.variant = "port" @@ -459,7 +451,7 @@ function serverMessageHandler( if ((msg.channel == "close-port" || msg.channel == "output" || msg.channel == "operation-error") && await portClientHandler(msg as ClientboundPortMessages, ports)) { return true; } else if (msg.channel == "input") { - this.handleServerInput(portMessenger, msg as MessengerInputMessage) + this.handleServerInput(portMessenger, msg as MessengerServerboundMessage) } return baseHandler(msg, ports); @@ -490,15 +482,20 @@ function serverMessageHandler( return holder.promise as any; }, getActiveMessengers() { - return peerPorts.values().toArray() as Messenger[]; + return peerPorts.values().toArray() as Messenger[]; }, - async sendOutputMessage(sender, msg) { - const result = await msg; + forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void) { + for (const connection of this.getActiveMessengers()) { + if (connection !== messenger) callback(connection); + } + }, + async sendClientMessage(sender, msg) { + const result = await msg(); const tuple = (Array.isArray(result) ? result : [result, new Array()]) as [any, Transferable[]]; (sender as InternalMessenger).postMessage(...([{channel: "output", data: tuple[0], key: FORCED_OUTPUT_KEY} as MessengerOutputMessage, tuple[1]])) } - } as ServerHandlerImpl> & InternalMessenger); + } as ServerHandlerImpl & InternalMessenger); // TODO: ADD WARNING TO PREVENT DOUBLE SERVER STUFF (impl.type.registry as MessengerRegistryImpl).serverMessenger = impl; @@ -506,7 +503,7 @@ function serverMessageHandler( return impl; } -function portClientMessageHandler(rootServerHandler: ServerHandlerImpl): RootMessageHandler["handleRootMessage"] { +function portClientMessageHandler(rootServerHandler: ServerHandlerImpl): RootMessageHandler["handleRootMessage"] { return async (msg) => { if (msg.channel == "close-port") { @@ -517,18 +514,16 @@ function portClientMessageHandler(rootServerHandler: ServerHandlerImpl): RootMes } } -//#endregion - -// #region Base Messenger Constructors +// --- -export function setupHolderAsImpl( +export function setupHolderAsImpl( holder: C, - type: MessengerTypeImpl, + type: MessengerTypeImpl, variant: MessengerVariants, key: InstanceKey | undefined, - rootHandlerSetup: (holder: C & InternalMessenger) => C & InternalMessenger & R -): InternalMessenger & R { - const infoHolder = holder as (C & InternalMessenger); + rootHandlerSetup: (holder: C & InternalMessenger) => C & InternalMessenger & R +): InternalMessenger & R { + const infoHolder = holder as (C & InternalMessenger); if (key) infoHolder.key = key; @@ -537,7 +532,7 @@ export function setupHolderAsImpl); const close = (messenger as Partial).close; @@ -560,7 +555,7 @@ export function setupHolderAsImpl((callbacks) => (forceClose) => callbacks.forEach((callback) => callback(forceClose))).source; - messenger.onmessage = function (e: MessageEvent>) { + messenger.onmessage = function (e: MessageEvent>) { const msg = e.data; if (msg.channel != "logging") messenger.logger.debug(`Message '${msg.channel}' received on '${debugKeyFrom(messenger)}'`, {obj: msg}); @@ -575,60 +570,32 @@ export function setupHolderAsImpl - = , K extends AsString>( - messenger: ServerMessenger>, input: I_M, sender?: Messenger -) => Promise>> - function setupAsClient( - type: MessengerTypeImpl, + type: MessengerTypeImpl, variant: MessengerVariants, key: InstanceKey, holder: C, options: MessengerOptions, -): InternalMessenger, C> { - return setupHolderAsImpl>(holder, type, variant, key, (impl) => clientMessageHandler(impl, - { - ...options, - inputTransferHandler: options?.inputTransferHandler == null ? undefined : (msg) => { - const copied = { - ...msg, - resolve: (resolver: InputResolverMap) => resolver[msg.channel]?.(msg) ?? [] - } - return options.inputTransferHandler!(copied); - }, - inputBypassHandler: options.inputBypassHandler != null ? ( - async (messenger, msg, sender) => { - const msgWithResolver = Object.assign( - {resolve: (resolvers: OutputResolverMap) => (resolvers[msg.channel])(msg)[0]}, - msg - ) - return (await options.inputBypassHandler!(messenger, msgWithResolver, sender))[0]; - } - ) as ServerInputHandlerBase : undefined, - } - )); +): InternalMessenger { + return setupHolderAsImpl>(holder, type, variant, key, (impl) => clientMessageHandler(impl, options)); } export function setupAsServerInternal( - type: MessengerType, + type: MessengerType, holder: ChannelHolder, - inputHandler: ServerInputHandler, -): ServerMessenger> { - return setupHolderAsImpl(holder, type as MessengerTypeImpl, "server", undefined, + inputHandler: ServerMessageHandler, +): ServerMessenger { + return setupHolderAsImpl>(holder, type as MessengerTypeImpl, "server", undefined, (impl) => serverMessageHandler(impl, inputHandler)); } -// #endregion - - -// #region Root Messenger Message types +// --- -type ClientboundMessages = StartMessengerRootMessage | MessengerOutputMessage | MessengerOperationError | MessengerLogMessage | CloseMessengerRootMessage; -type ServerboundMessages = MessengerInputMessage | SetupMessengerPortMessage | CloseMessengerPortMessage | SetOffThreadKeyMessage | ForceCloseMessengerRootMessage; +type ClientboundMessengerChannels = StartMessengerRootMessage | MessengerOutputMessage | MessengerOperationError | MessengerLogMessage | CloseMessengerRootMessage; +type ServerboundMessengerChannels = MessengerServerboundMessage | SetupMessengerPortMessage | CloseMessengerPortMessage | SetOffThreadKeyMessage | ForceCloseMessengerRootMessage; -type ClientboundPortMessages = MessengerOperationError | MessengerOutputMessage | CloseMessengerPortMessage; +type ClientboundPortMessages = MessengerOperationError | MessengerOutputMessage | CloseMessengerPortMessage; interface MessengerTarget extends MessengerTypeTarget { id: TypeId; @@ -645,45 +612,45 @@ interface ConversationTarget { key: ConversationId; } -interface SetOffThreadKeyMessage extends Message<"set-key"> { +interface SetOffThreadKeyMessage extends Channel<"set-key"> { key: InstanceKey } -interface StartMessengerRootMessage extends Message<"setup-root">, MessengerTypeTarget { +interface StartMessengerRootMessage extends Channel<"setup-root">, MessengerTypeTarget { transactionKey: string, } -interface SetupMessengerPortMessage extends Message<"setup-port">, MessengerTarget { +interface SetupMessengerPortMessage extends Channel<"setup-port">, MessengerTarget { transactionKey: string, } -interface CloseMessengerRootMessage extends Message<"close-root">, MessengerTarget {} +interface CloseMessengerRootMessage extends Channel<"close-root">, MessengerTarget {} -interface ForceCloseMessengerRootMessage extends Message<"force-close-root"> { +interface ForceCloseMessengerRootMessage extends Channel<"force-close-root"> { forceClose: boolean } -interface CloseMessengerPortMessage extends Message<"close-port">, MessengerTarget {} +interface CloseMessengerPortMessage extends Channel<"close-port">, MessengerTarget {} -export interface MessengerInputMessage extends Message<"input"> { +export interface MessengerServerboundMessage extends Channel<"input"> { key: string; data: T; } -interface MessengerLogMessage extends Message<"logging"> { +interface MessengerLogMessage extends Channel<"logging"> { logMsg: {type: "callback", data: LoggerCallbackData} | { type: "counter", data: CounterCallbackData }; } -interface MessengerOutputMessage extends Message<"output">, ConversationTarget { +interface MessengerOutputMessage extends Channel<"output">, ConversationTarget { data: T; } -interface MessengerOperationError extends Message<"operation-error">, ConversationTarget { +interface MessengerOperationError extends Channel<"operation-error">, ConversationTarget { error: any; info?: any; } -// #endregion +// --- function fullInstanceKeyFrom(messenger: Messenger) { return fullInstanceKey(messenger.type.id, messenger.key); @@ -700,3 +667,15 @@ function debugKeyFrom(messenger: Messenger) { function debugKey(id: TypeId, key: InstanceKey | undefined, variant: MessengerVariants) { return `${id}:${key ?? "UNSET"}#${variant}` } + +function serverHandlerResolver(): ServerHandlerResolver { + return async (msg, resolvers) => { + const result = await resolvers[msg.channel]!(msg); + + return ( + typeof result === 'undefined' + ? undefined as void + : (Array.isArray(result) ? [Object.assign(result[0], { channel: msg.channel }), result[1]] : Object.assign(result, { channel: msg.channel })) + ) as ClientboundMessage extends never ? void : MaybeTransferable> + } +} diff --git a/packages/ui/src/helpers/toolbox/messenger/index.ts b/packages/ui/src/helpers/toolbox/messenger/index.ts index c833c3fddd0..7ce5992f78b 100644 --- a/packages/ui/src/helpers/toolbox/messenger/index.ts +++ b/packages/ui/src/helpers/toolbox/messenger/index.ts @@ -1,77 +1,117 @@ import { createRegistryInternal, setupAsServerInternal } from "#ui/helpers/toolbox/messenger/implement"; import type { Logger, LoggerFactory } from "#ui/helpers/toolbox/misc/logging"; -export type TypeId = string; +/** + * Messenger is an API used to set up communication between main thread and workers to in a manor + * that allows for instantiation via a given registry on the main thread rather than on the workers + * allowing for control better control. This means we can control the pool of workers and setup + * worker <-> worker communication for existing workers allowing sharing of workers for variours + * tasks. + * + * Communcation happens on a given Channel type either with a Serverbound (To the worker) Message + * and/or Clientbound (To the Main thread) Message. + */ -/* @ts-ignore */ -export interface MessengerType { - id: TypeId; - registry: MessengerRegistry; - instanceConfig: InstanceConfig; +//-- - close(forceClosed?: boolean): void; -} +export type ValueBase = { [key: string]: any } | undefined -export interface MessengerAccessor { - getMessenger: (type: MessengerType) => (M extends ClientMessenger ? Promise : Promise) | undefined; - getActiveMessengers: () => Messenger[]; -} +export type AsString = Extract; -export interface MessengerRegistry extends MessengerAccessor { - of(id: string, extra?: MessengerOptions & EntryOptions>): MessengerType>; +export interface ServerboundType { serverbound: S } +export interface ClientboundType { clientbound: C } - getType(id: string): MessengerType | undefined; +export type ClientServerBoundType = ServerboundType & ClientboundType +export type ClientOrServerBoundType = ServerboundType | ClientboundType | ClientServerBoundType - setWorkerFactoryFor(type: MessengerType, factory: () => Worker): void; +export interface Channel { channel: K } - setLogFactory(factory: LoggerFactory): void; - getLogFactory(): LoggerFactory; +export type ClientServerBoundChannel = Channel & ClientServerBoundType - closeAll(forceClosed?: boolean): void; -} +export type ServerboundChannel = Channel & ServerboundType +export type ClientboundChannel = Channel & ClientboundType -export type EntryOptions = { closeCallback?: (messenger: M) => void} & InstanceConfig; +export type MessageSchema = Record; +export type TypeToChannel = + T extends ClientServerBoundType ? ClientServerBoundChannel : ( + T extends ClientboundType ? ClientboundChannel : ( + T extends ServerboundType ? ServerboundChannel : + { type_error: 'NOT_VALID_TYPE' } + ) + ) -//-- +export type MessageTypes = + T extends Record + ? { [E in AsString]: TypeToChannel } + : { type_error: 'INVALID_SCHEMA_TYPE' }; -export type ValueBase = { [key: string]: any } +export type ServerboundChannels + = AsString as ServerboundMessage extends never ? never : E]: T[E] }>; -export interface Message { channel: K } +export type ClientboundChannels + = AsString as ClientboundMessage extends never ? never : E]: T[E] }>; -type IOType = { input: I, output: O } +export type ClientServerBoundChannels + = AsString as ServerboundMessage & ClientboundMessage extends never ? never : E]: T[E] }>; -type MessageIOType = IOType & Message +export type ServerboundMessage = AsString> = + T[K] extends ServerboundChannel ? S & Channel : never; -export type MessageSchema = Record; +export type ClientboundMessage = AsString> = + T[K] extends ClientboundChannel ? C & Channel : never; -export type MessageTypes = - T extends Record - ? { [E in AsString]: T[E] extends IOType ? MessageIOType : never; }// & { '_type_flag_fix': MessageIOType<'_type_flag_fix', {}, {}>} - : never; +export type ClientboundBase> = + T[K] extends ClientServerBoundChannel ? O : never; -export type AsString = Extract +// --- -export type InputMessage = AsString> = - T[K] extends MessageIOType ? I & Message : never; +export type TypeId = string; -export type OutputMessage = AsString> = - T[K] extends MessageIOType ? O & Message : never; +export interface MessengerType { + id: TypeId; + registry: MessengerRegistry; + instanceConfig: InstanceConfig; -export type OutputBase> = - T[K] extends MessageIOType ? O : never; + close(forceClosed?: boolean): void; +} -export type MessageChannels = keyof T; +export interface MessengerAccessor { + getMessenger: (type: MessengerType) => Promise & H> | undefined; + getActiveMessengers(): Messenger[]; + forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void): void +} -// #endregion +export interface MessengerOptions { + handlerConstructor: MessengerHandlerConstructor + bypassServerMessageHandler?: ServerMessageHandler, + serverMessageTransferHandler?: ServerMessageTransferHandler; + independentClientMessageHandler?: ClientMessageHandler; + closeCallback?: (messenger: ClientMessenger) => void; +} // TODO: ATTEMPT TO ADD LOAD BALANCING FOR WHEN INSTANCES ARE SHARED AND COUNT IS GREATER THAN 1 export interface InstanceConfig { - timeout?: number; - shared: boolean; + timeout?: number; + shared: boolean; count: number; } +export interface MessengerRegistry extends MessengerAccessor { + of(id: string, extra?: MessengerOptions & InstanceConfig): MessengerType; + + getType(id: string): MessengerType | undefined; + + setWorkerFactoryFor(type: MessengerType, factory: () => Worker): void; + + setLogFactory(factory: LoggerFactory): void; + getLogFactory(): LoggerFactory; + + closeAll(forceClosed?: boolean): void; +} + +//-- + export interface InstanceData { timeout?: number; shared: boolean; @@ -81,10 +121,8 @@ export type InstanceKey = string; export type MessengerVariants = "server" | "port" | "client"; -export type ChannelHolder = Worker | MessagePort; - -export interface Messenger { - type: MessengerType; +export interface Messenger { + type: MessengerType; key?: InstanceKey; instanceData: InstanceData; @@ -94,96 +132,58 @@ export interface Messenger { logger: Logger; } -export interface ServerMessenger extends Messenger, MessengerAccessor { - /** - * @experimental TBD if this should be exposed better - */ - sendOutputMessage(sender: Messenger, msg: Promise>): Promise; +export interface ServerMessenger extends Messenger, MessengerAccessor { + sendClientMessage(sender: Messenger, msg: () => Promise>>): Promise; } -export interface ClientMessenger extends Messenger { - - type: MessengerType>, +export interface ClientMessenger extends Messenger { + type: MessengerType, - sendMessage: , K extends AsString>(input: I_M) => Promise> + sendMessage: , M extends ServerboundMessage>(msg: M) + => Promise extends never ? void : ClientboundMessage> handler: H; } -export type MessengerHandlerConstructor = (messenger: ClientMessenger) => H - -//-- - -//-- - -export type PossibleTransferable = T | [T, Transferable[]]; - -export type InputResolverMap = { - [E in AsString]?: ((msg: InputMessage) => Transferable[]) | undefined -} - -export type ClientInputUnpacker = - >(input: I_M) => Transferable[] | undefined; // -- -export type ClientOutputHandler = (messenger: ClientMessenger, output: OutputMessage) => boolean; +export type MessengerHandlerConstructor = (messenger: ClientMessenger) => H -export interface MessengerOptions { - handlerConstructor?: MessengerHandlerConstructor - inputBypassHandler?: ServerInputHandler, - inputTransferHandler?: ClientInputUnpacker; - independentOutputHandler?: ClientOutputHandler; -} +export type ClientMessageHandler = (messenger: ClientMessenger, output: ClientboundMessage) => boolean; -export interface MessengerOptionsBase { - handlerConstructor?: MessengerHandlerConstructor - inputBypassHandler?: ServerInputHandler, - inputTransferHandler?: ClientInputUnpacker; - independentOutputHandler?: ClientOutputHandler; -} +export type ServerMessageHandler = + , M extends ServerboundMessage>( + messenger: ServerMessenger, msg: M, resolver: ServerHandlerResolver, sender?: Messenger + ) => Promise extends never ? void : MaybeTransferable>> + +export type ServerHandlerResolver = + , M extends ServerboundMessage>( + msg: M, handlers: ServerMessageHandlers + ) => Promise extends never ? void : MaybeTransferable>> -export function createRegistry(): MessengerRegistry { - return createRegistryInternal(); +export type ServerMessageHandlers = { + [E in ServerboundChannels]: (msg: ServerboundMessage) + => Promise extends never ? void : MaybeTransferable>>; } +export type MaybeTransferable = T | [T, Transferable[]]; -// WORKER SECTION +export type ServerMessageTransferHandler = + , M extends ServerboundMessage>( + msg: M, resolver: (msg: M, handlers: ServerTransferHandlers) => Transferable[] + ) => Transferable[] -export type ServerSimpleInputHandler = - (messenger: ServerMessenger, input: I, sender?: Messenger) => Promise>; +export type ServerTransferHandlers = { + [E in ServerboundChannels]?: (msg: ServerboundMessage) => Transferable[] +} -export type ServerInputHandler = - , I_M extends InputMessage>( - messenger: ServerMessenger>, input: I_M, sender?: Messenger - ) => Promise>> +// -- -export type OutputResolverMap = { - [E in AsString]: (msg: InputMessage) => Promise>> -} +export const createRegistry = createRegistryInternal; -export function setupAsServer( - type: MessengerType, holder: ChannelHolder, inputHandler: ServerInputHandler -): ServerMessenger> { - return setupAsServerInternal(type, holder, inputHandler); -} +export const setupAsServer = setupAsServerInternal; export function isWorker() { //@ts-expect-error TS2304: Cannot find name WorkerGlobalScope return typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope; } - -export async function resolveOutput, I_M extends InputMessage>( - msg: I_M, resolvers: OutputResolverMap -): Promise>> { - const result = await (resolvers[msg.channel]!(msg)) - - return (Array.isArray(result) ? [Object.assign(result[0], { channel: msg.channel }), result[1]] : Object.assign(result, { channel: msg.channel })) as PossibleTransferable> -} - -export function resolveInputTransfers>( - msg: I_M, resolvers: InputResolverMap -): Transferable[] | undefined { - return resolvers[msg.channel]?.(msg); -} - -// -- diff --git a/packages/ui/src/helpers/toolbox/workers/decompile/index.ts b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts index ffc5572704a..50c38b05ba5 100644 --- a/packages/ui/src/helpers/toolbox/workers/decompile/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/decompile/index.ts @@ -2,11 +2,11 @@ import type { DecompilerMessages, } from "#ui/helpers/toolbox/decompile"; import { decompileFileContent } from "#ui/helpers/toolbox/decompile/impl"; -import {resolveOutput, setupAsServer} from "#ui/helpers/toolbox/messenger"; -import {decompileMessengerType, zipHandlerType} from "#ui/helpers/toolbox/workers"; +import { setupAsServer } from "#ui/helpers/toolbox/messenger"; +import { decompileMessengerType, zipHandlerType } from "#ui/helpers/toolbox/workers"; -setupAsServer(decompileMessengerType, self as unknown as Worker, async (messenger, msg) => { - return resolveOutput(msg, { +setupAsServer(decompileMessengerType, self as unknown as Worker, async (messenger, msg, resolver) => { + return resolver(msg, { main: async (msg) => { return decompileFileContent(messenger.logger, (await messenger.getMessenger(zipHandlerType))?.handler, msg); } diff --git a/packages/ui/src/helpers/toolbox/workers/index.ts b/packages/ui/src/helpers/toolbox/workers/index.ts index 00588303e55..4580ee78347 100644 --- a/packages/ui/src/helpers/toolbox/workers/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/index.ts @@ -1,13 +1,9 @@ import type { - DecompileHandler, - DecompilerMessages, + DecompileHandler, DecompileInput, + DecompilerMessages, DecompilerResult, } from "#ui/helpers/toolbox/decompile"; import { - type ClientMessenger, - createRegistry, - type InputResolverMap, - type MessengerType, - resolveInputTransfers, + createRegistry, type MessengerType, } from '#ui/helpers/toolbox/messenger'; // import { AtlasMessenger, handlerConstructor as atlasHandlerConstructor } from "./atlas/common"; import { handlerConstructor as zipHandlerConstructor,type ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; @@ -18,36 +14,22 @@ import ZipWorker from "./zip?worker"; export const messengerRegistry = createRegistry(); -export type ZipHandlerMessenger = ClientMessenger; - -export const zipHandlerType: MessengerType = messengerRegistry.of("zip-handler", { +export const zipHandlerType = messengerRegistry.of("zip-handler", { handlerConstructor: zipHandlerConstructor, - inputTransferHandler: (input) => { - return resolveInputTransfers(input, { + serverMessageTransferHandler(msg, resolver){ + return resolver(msg, { node_write: (msg) => [msg.buffer], - read_zip_tree: (msg) => [msg.buffer], - } as InputResolverMap); + read_zip_tree: (msg) => msg.data_type == 'buffer' ? [msg.buffer] : [], + }); }, - independentOutputHandler: (messenger, msg) => { - if (msg.channel == "update_node_data") { - const data = msg as ZipMessages['update_node_data']['output']; - - const root = messenger.handler.zipArchivesByHash.get(data.root); - - if (root != null) { - const node = (data.relative ? root.relativeFileEntries : root.fileEntries).get(data.path); - - if (node) { - for (const lazy of [node.sha1, node.sha256, node.sha512, node.fileTypeInfo]) { - lazy.reset() - } - (node as unknown as Partial).onDataUpdate?.() - } - } + independentClientMessageHandler(messenger, msg){ + if (msg.channel === "archive_update") { + messenger.handler.archiveUpdate(msg as ZipMessages['archive_update']['clientbound']) + return true; } return false; }, - closeCallback: (messenger) => { + closeCallback(messenger){ return messenger.handler.closeAllZipTrees(); }, shared: true, @@ -61,23 +43,22 @@ export const zipHandlerType: MessengerType = messengerRegis // }); // -type DecompileMessenger = ClientMessenger; - -export const decompileMessengerType: MessengerType = messengerRegistry.of("decompiler-vf", { - handlerConstructor: (messenger) => { +export const decompileMessengerType: MessengerType = messengerRegistry.of("decompiler-vf", { + serverMessageTransferHandler(msg, resolver) { + return resolver(msg, { + main: (msg) => msg.transferBuffer ? [msg.targetFile.content] : [] + }); + }, + handlerConstructor(messenger) { return { decompile: async (input) => { return await messenger.sendMessage({ ...input, - channel: 'main' + channel: 'main', + transferBuffer: input.transferBuffer ?? false }) } - }; - }, - inputTransferHandler: (input) => { - return resolveInputTransfers(input, { - main: (msg) => [msg.targetFile.content], - } as InputResolverMap) + } as DecompileHandler; }, shared: true, timeout: 30000, diff --git a/packages/ui/src/helpers/toolbox/workers/zip/api.ts b/packages/ui/src/helpers/toolbox/workers/zip/api.ts index b12f1fa4d7d..fc8dd0bc2c8 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/api.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/api.ts @@ -2,72 +2,78 @@ import type { ZipWriterAddDataOptions } from "@zip.js/zip.js"; import type { DecompilerResult } from "#ui/helpers/toolbox/decompile"; import type { FileDetails, HashedFileContent } from "#ui/helpers/toolbox/file"; -import type { ZipHandlerMessenger } from "#ui/helpers/toolbox/workers"; import { + type ArchiveUpdateTypes, DelegateFileZipTreeNode, type FileMatcher, type FileZipTreeNode, getAsTarget, type LazyDataAccess, type NodeTarget, - reconstituteRoot, type RootNodeTarget, type RootZipTreeNode, type Sha512, type SourceOptions, - type ZipHandler + reconstituteNodes, + reconstituteRoot, type RootNodeTarget, type RootZipTreeNode, type SourceOptions, + type UUID_V7, type ZipDataSource, + type ZipHandler, type ZipNode, ZipTreeNode } from "#ui/helpers/toolbox/zip"; -import type {MessageTypes} from "#ui/helpers/toolbox/messenger"; - -export interface DataInput { - buffer: ArrayBuffer -} +import type { + ClientMessenger, + MessageTypes, + MessengerHandlerConstructor +} from "#ui/helpers/toolbox/messenger"; +import {EventStream} from "#ui/helpers/toolbox/misc"; +import type {Sha512} from "#ui/helpers/toolbox/workers/zip/index.ts"; export type ZipMessages = MessageTypes< { lazy_read: { - input: { key: keyof LazyDataAccess } & NodeTarget, - output: { value: Awaited> } + serverbound: { key: keyof LazyDataAccess } & NodeTarget, + clientbound: { value: Awaited> } }, source_read: { - input: NodeTarget & { options?: SourceOptions } - output: DecompilerResult + serverbound: NodeTarget & { options?: SourceOptions } + clientbound: DecompilerResult } node_read: { - input: NodeTarget, - output: { buffer: ArrayBuffer } + serverbound: NodeTarget, + clientbound: { buffer: ArrayBuffer } }, node_write: { - input: NodeTarget & { nodeType?: "zip" | "file", options?: ZipWriterAddDataOptions } & DataInput, - output: { wasWritten: boolean } + serverbound: NodeTarget & { options?: ZipWriterAddDataOptions } & { buffer: ArrayBuffer }, + clientbound: { wasWritten: boolean } }, read_zip_tree: { - input: { name?: string, url?: string, hash?: string } & DataInput, - output: { node: RootZipTreeNode } + serverbound: ZipDataSource, + clientbound: { node: RootZipTreeNode, isFullyLoaded?: boolean} }, get_zip_tree: { - input: { value: string | Sha512 }, - output: { node: RootZipTreeNode | undefined } + serverbound: { value: string | UUID_V7 }, + clientbound: { node: RootZipTreeNode | undefined } }, collect_files: { - input: RootNodeTarget & { matcher?: FileMatcher }, - output: { files: FileDetails[] } + serverbound: RootNodeTarget & { matcher?: FileMatcher }, + clientbound: { files: FileDetails[] } }, collect_files_hashed: { - input: RootNodeTarget & { matcher?: FileMatcher, deduplicate?: boolean}, - output: { files: HashedFileContent[] } + serverbound: RootNodeTarget & { matcher?: FileMatcher, deduplicate?: boolean}, + clientbound: { files: HashedFileContent[] } }, close_zip_tree: { - input: RootNodeTarget, - output: { err?: Error, wasClosed: boolean } + serverbound: RootNodeTarget, + clientbound: { err?: Error, wasClosed: boolean } }, close_all_zip_trees: { - input: { }, - output: { err?: Error, wasClosed: boolean } + serverbound: { }, + clientbound: { err?: Error, wasClosed: boolean } }, - update_node_data: { - input: NodeTarget, - output: NodeTarget + archive_update: { + //serverbound: {} + clientbound: ArchiveUpdateTypes } } > -export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler = (messenger: ZipHandlerMessenger) => { - const hashToUrls = new Map(); +export const handlerConstructor: MessengerHandlerConstructor = (messenger) => { + const uuidToUrls = new Map(); + const hashToUUID = new Map(); + const uuidToOnFinishedLoadingHolder = new Map>(); return { - zipArchivesByHash: new Map(), - urlToHash: new Map(), + zipByUUID: new Map(), + urlToUUID: new Map(), readLazyData: async function (target: FileZipTreeNode | NodeTarget, key: K): Promise>> { return (await messenger.sendMessage({ channel: "lazy_read", ...getAsTarget(target), key: key })).value as Awaited>; }, @@ -77,43 +83,53 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler readNodeData: async (target: FileZipTreeNode | NodeTarget) => { return (await messenger.sendMessage({ channel: "node_read", ...getAsTarget(target)})).buffer }, - getOrReadZipTree: async function (buffer, {name, url, hash} = {}) { + getOrReadZipTree: async function (source: ZipDataSource) { + const { url, sha512Hash } = source; let node: RootZipTreeNode | undefined; - if (url) node = await this.getZipTree(url, true); - if (node == null && hash) node = await this.getZipTree(hash, false); - const data = await buffer(); - return data ? this.readZipTree(data, {name: name, url: url, hash: hash}) : undefined; + const filteredURL = url?.split("?")[0]; + if (filteredURL) node = await this.getZipTree(filteredURL, true); + if (node == null && sha512Hash) node = await this.getZipTree(sha512Hash, false); + if (node == null) node = await this.readZipTree(source); + return node; }, - readZipTree: async function (buffer, {name, url, hash} = {}) { - const filteredURL = url?.split("?")[0]; + readZipTree: async function (source: ZipDataSource) { + const { data_type, name, url, sha512Hash, buffer, rangeOptions } = source as ZipDataSource; + const msg = (await messenger.sendMessage({ + channel: "read_zip_tree", + data_type, + name, + url, + sha512Hash, + buffer: buffer != null ? await (typeof buffer === 'function' ? buffer() : Promise.resolve(buffer)) : undefined, + rangeOptions: rangeOptions, + })); + const node = reconstituteRoot(messenger.handler, msg.node); - if (filteredURL) { - const node = await this.getZipTree(filteredURL, true, true); - if (node) return node; - } + const holder = Promise.withResolvers(); - if (hash) { - const node = await this.getZipTree(hash, false, true); - if (node) return node; - } + node.onFinishedLoading = msg.isFullyLoaded ? Promise.resolve() : holder.promise; - const node = reconstituteRoot(messenger.handler, (await messenger.sendMessage({ channel: "read_zip_tree", buffer, name: name, url: url, hash: hash })).node); + const filteredURL = url?.split("?")[0]; if (filteredURL) { - this.urlToHash.set(filteredURL, node.key); - const urls = (hashToUrls.has(node.key)) - ? hashToUrls.get(node.key)! - : (() => { const array = new Array(); hashToUrls.set(node.key, array); return array; })(); + this.urlToUUID.set(filteredURL, node.key); + const urls = (uuidToUrls.has(node.key)) + ? uuidToUrls.get(node.key)! + : (() => { const array = new Array(); uuidToUrls.set(node.key, array); return array; })(); urls.push(filteredURL); } - this.zipArchivesByHash.set(node.key, node); + if (sha512Hash != null) { + hashToUUID.set(sha512Hash, node.key) + } + + this.zipByUUID.set(node.key, node); return node; }, - writeNodeData: async (target, buffer, type, options) => { - return (await messenger.sendMessage({ channel: "node_write", ...getAsTarget(target), buffer, nodeType: type, options})).wasWritten; + writeNodeData: async (target, buffer, options) => { + return (await messenger.sendMessage({ channel: "node_write", ...getAsTarget(target), buffer, options})).wasWritten; }, collectFiles: async (root, matcher) => { return (await messenger.sendMessage({ channel: "collect_files", matcher: matcher, root: (root as RootZipTreeNode).key ?? root})).files @@ -122,34 +138,34 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler return (await messenger.sendMessage({ channel: "collect_files_hashed", matcher: matcher, root: (root as RootZipTreeNode).key ?? root, deduplicate: deduplicate})).files }, closeZipTree: async function (root){ - const hash: Sha512 = (root as RootZipTreeNode).key ?? root; + const hash: UUID_V7 = (root as RootZipTreeNode).key ?? root; const value = await messenger.sendMessage({ channel: "close_zip_tree", root: (root as RootZipTreeNode).key ?? root}); - const urls = hashToUrls.get(hash); - if (urls) for (const url of urls) this.urlToHash.delete(url); - hashToUrls.delete(hash); + const urls = uuidToUrls.get(hash); + if (urls) for (const url of urls) this.urlToUUID.delete(url); + uuidToUrls.delete(hash); return value; }, closeAllZipTrees: async function () { const value = await messenger.sendMessage({ channel: "close_all_zip_trees" }); - this.zipArchivesByHash.clear(); - this.urlToHash.clear(); - hashToUrls.clear(); + this.zipByUUID.clear(); + this.urlToUUID.clear(); + uuidToUrls.clear(); return value; }, getZipTree: async function (value, isUrl, localOnly) { - if (this.zipArchivesByHash.has(value)) { - return this.zipArchivesByHash.get(value); + if (this.zipByUUID.has(value)) { + return this.zipByUUID.get(value); } if (isUrl) { const filteredURL = value.split("?")[0]; - if (this.urlToHash.has(filteredURL)) { - const hash = this.urlToHash.get(value)!; - const node = this.zipArchivesByHash.get(hash); + if (this.urlToUUID.has(filteredURL)) { + const hash = this.urlToUUID.get(value)!; + const node = this.zipByUUID.get(hash); if (node) return node; } @@ -165,15 +181,57 @@ export const handlerConstructor: (messenger: ZipHandlerMessenger) => ZipHandler if (isUrl) { const filteredURL = value.split("?")[0]; - this.urlToHash.set(filteredURL, node.key); - if (hashToUrls.has(node.key)) hashToUrls.set(node.key, []) - hashToUrls.get(node.key)?.push(filteredURL); + this.urlToUUID.set(filteredURL, node.key); + if (uuidToUrls.has(node.key)) uuidToUrls.set(node.key, []) + uuidToUrls.get(node.key)?.push(filteredURL); } - this.zipArchivesByHash.set(node.key, node); + this.zipByUUID.set(node.key, node); return node; - } + }, + archiveUpdate: function (msg) { + if (msg.type == 'updated_node_data') { + const nodeTarget = msg.node; + const root = messenger.handler.zipByUUID.get(nodeTarget.root); + + if (root != null) { + const node = (nodeTarget.relative ? root.relativeFileEntries : root.fileEntries).get(nodeTarget.path); + + if (node) { + for (const lazy of [node.sha1, node.sha256, node.sha512, node.fileTypeInfo]) { + lazy.reset() + } + (node as unknown as Partial).onDataUpdate?.() + } + } + } else if (msg.type == 'node_loaded'){ + const root = messenger.handler.zipByUUID.get(msg.root.root); + + if (root != null) { + const parentPath = msg.node.path.split("/").slice(0, -1).join("/"); + const parent = root.entries.get(parentPath) + + if (parent != null) { + reconstituteNodes(this, root, parent, msg.node) + } + } + } else if (msg.type == 'root_loaded') { + const root = messenger.handler.zipByUUID.get(msg.root.root); + + if (root != null) { + uuidToOnFinishedLoadingHolder.get(root.key)?.resolve() + uuidToOnFinishedLoadingHolder.delete(root.key); + } + } + + this.onArchiveUpdate.stream.sink(msg); + }, + onArchiveUpdate: new EventStream<(msg: ArchiveUpdateTypes) => void>((callbacks) => (msg) => { + for(const callback of callbacks) { + callback(msg); + } + }).source } }; diff --git a/packages/ui/src/helpers/toolbox/workers/zip/index.ts b/packages/ui/src/helpers/toolbox/workers/zip/index.ts index 474467207a7..f52d4557c18 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/index.ts @@ -1,34 +1,39 @@ import { - type FileContentPromise, type FileDetails, type FileTypeInfo, + type FileContentPromise, type FileDetails, type FileTypeInfo, hashBuffer, type HashedFileContent, resolveContent } from "#ui/helpers/toolbox/file"; import { isWorker, - resolveOutput, - type ServerInputHandler, + type ServerMessageHandler, setupAsServer } from "#ui/helpers/toolbox/messenger"; import { zipHandlerType } from "#ui/helpers/toolbox/workers"; import type { ZipMessages } from "#ui/helpers/toolbox/workers/zip/api"; -import type { - LazyDataAccess, - NodeTarget, - RootZipTreeNode, - Sha512, - StringMatcher, - ZipTreeNode } from "#ui/helpers/toolbox/zip"; -import { readZipTree } from "#ui/helpers/toolbox/zip/impl"; +import { + arrayBufferFromSource, + type LazyDataAccess, + type NodeTarget, type RootNodeTarget, + type RootZipTreeNode, + type StringMatcher, + type UUID_V7, + type ZipNode, +} from "#ui/helpers/toolbox/zip"; +import {readZipTreeFromBuffer, readZipTreeFromURL} from "#ui/helpers/toolbox/zip/impl"; + +export type Sha512 = string; + +const zipByUUID = new Map(); -const zipArchivesByHash = new Map(); +const hashToUUID = new Map(); -const urlToHash = new Map(); -const hashToUrls = new Map(); +const urlToUUID = new Map(); +const UUIDToUrls = new Map(); function getNodeFrom(target: NodeTarget) { - if (!zipArchivesByHash || zipArchivesByHash.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); + if (!zipByUUID || zipByUUID.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); - const root = zipArchivesByHash.get(target.root); + const root = zipByUUID.get(target.root); if (root == undefined) throw new Error(`Unable to get node '${target.root}:${target.path}' as their is no active root with the hash given namespace hash!`); const node = (target.relative ? root.relativeFileEntries : root.fileEntries).get(target.path); @@ -37,23 +42,23 @@ function getNodeFrom(target: NodeTarget) { return node; } -function getRootFrom(target: Sha512) { - if (!zipArchivesByHash || zipArchivesByHash.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); +function getRootFrom(target: UUID_V7) { + if (!zipByUUID || zipByUUID.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); - const root = zipArchivesByHash.get(target); + const root = zipByUUID.get(target); if (root == undefined) throw new Error(`Unable to get root '${target}' as their is no active root with the hash given hash!`); return root; } function getZipNode(value: string) { - if (zipArchivesByHash.has(value)) { - return zipArchivesByHash.get(value); + if (zipByUUID.has(value)) { + return zipByUUID.get(value); } - if (urlToHash.has(value)) { - const hash = urlToHash.get(value)!; - const node = zipArchivesByHash.get(hash); + if (urlToUUID.has(value)) { + const hash = urlToUUID.get(value)!; + const node = zipByUUID.get(hash); if (node) return node; } @@ -84,8 +89,8 @@ async function isValidFor(path: string, fileType: () => Promise = function (messenger, msg, sender) { - return resolveOutput(msg, { +export const messageHandler: ServerMessageHandler = (messenger, msg, resolver, sender)=> { + return resolver(msg, { lazy_read: async (msg) => { const node = getNodeFrom(msg); @@ -101,47 +106,127 @@ export const messageHandler: ServerInputHandler = function (messeng const node = getNodeFrom(msg); const buffer = await node.getBuffer(); + console.log(buffer); return { buffer: buffer }; }, node_write: async (msg) => { const node = getNodeFrom(msg); - await node.setBuffer(async () => msg.buffer, msg.nodeType, msg.options); + await node.setBuffer(async () => msg.buffer, msg.options); + messenger.forActiveMessengersUnless(sender, (connection) => { + messenger.sendClientMessage(connection, async () => { + return { + channel: "archive_update", + type: "updated_node_data", + node: { + root: node.root?.key!, + path: node.path, + } + } + }) + }) return { wasWritten: true }; }, read_zip_tree: async (msg) => { let tree: RootZipTreeNode | undefined = undefined; + let isFullyLoaded = sender != null; - if (msg.hash) tree = getZipNode(msg.hash); + if (msg.sha512Hash) tree = getZipNode(msg.sha512Hash); if (tree == null && msg.url) tree = getZipNode(msg.url); if (tree == null) { - tree = await readZipTree(messenger.logger, msg.buffer, { name: msg.name, logger: messenger.logger }); + if (msg.data_type == 'buffer') { + const buffer = await arrayBufferFromSource(msg.buffer); + const hash = await hashBuffer(buffer) + + tree = await readZipTreeFromBuffer(messenger.logger, buffer, { + name: msg.name, + logger: messenger.logger, + innerArchiveReadCallback: sender != null ? (root, node) => { + messenger.forActiveMessengersUnless(sender, (connection) => { + messenger.sendClientMessage(connection, async () => { + return { + channel: 'archive_update', + type: 'node_loaded', + root: { root: root.key } as RootNodeTarget, + node: copyChildNode(node) + } + }) + }) + } : undefined + }); + + hashToUUID.set(hash, tree.key) + } else { + tree = await readZipTreeFromURL(messenger.logger, msg.url, { + name: msg.name, + logger: messenger.logger, + innerArchiveReadCallback: (root, node) => { + messenger.forActiveMessengersUnless(sender, (connection) => { + messenger.sendClientMessage(connection, async () => { + return { + channel: 'archive_update', + type: 'node_loaded', + root: { root: root.key } as RootNodeTarget, + node: copyChildNode(node) + } + }) + }) + }, + rangeOptions: { + ...msg.rangeOptions, + onProgress(current, total) { + messenger.forActiveMessengersUnless(sender, (connection) => { + messenger.sendClientMessage(connection, async () => { + return { + channel: 'archive_update', + type: 'archive_download', + root: { root: tree!.key } as RootNodeTarget, + current: current, + total: total + } + }) + }) + } + }, + }); + + if (msg.sha512Hash) zipByUUID.set(msg.sha512Hash, tree); + } - zipArchivesByHash.set(tree.key, tree); + tree!.onFinishedLoading.then(() => { + isFullyLoaded = true; + messenger.forActiveMessengersUnless(sender, (connection) => { + messenger.sendClientMessage(connection, async () => { + return { + channel: 'archive_update', + type: 'root_loaded', + root: {root: tree!.key} as RootNodeTarget + } + }) + }) + }) + + zipByUUID.set(tree.key, tree); if (msg.url) { - urlToHash.set(msg.url, tree.key); + urlToUUID.set(msg.url, tree.key); - const urls = (hashToUrls.has(tree.key)) - ? hashToUrls.get(tree.key)! - : (() => { const array = new Array(); hashToUrls.set(tree.key, array); return array; })(); + const urls = (UUIDToUrls.has(tree.key)) + ? UUIDToUrls.get(tree.key)! + : (() => { const array = new Array(); UUIDToUrls.set(tree.key, array); return array; })(); urls.push(msg.url); } } - const copiedTree = copy(tree); - - return { - node: copiedTree - }; + return { node: copyRoot(tree), isFullyLoaded: isFullyLoaded }; }, get_zip_tree: async (msg) => { const node = getZipNode(msg.value); - const copiedTree = node ? copy(node) : undefined; + const copiedTree = node ? copyRoot(node) : undefined; return { node: copiedTree @@ -258,11 +343,11 @@ export const messageHandler: ServerInputHandler = function (messeng return { files: files }; }, close_zip_tree: async (msg) => { - zipArchivesByHash.delete(msg.root); + zipByUUID.delete(msg.root); - const urls = hashToUrls.get(msg.root); - if (urls) for (const url of urls) urlToHash.delete(url); - hashToUrls.delete(msg.root); + const urls = UUIDToUrls.get(msg.root); + if (urls) for (const url of urls) urlToUUID.delete(url); + UUIDToUrls.delete(msg.root); return { err: undefined, @@ -270,52 +355,44 @@ export const messageHandler: ServerInputHandler = function (messeng } }, close_all_zip_trees: async () => { - zipArchivesByHash.clear(); - urlToHash.clear(); - hashToUrls.clear(); + zipByUUID.clear(); + urlToUUID.clear(); + UUIDToUrls.clear(); return { err: undefined, wasClosed: true, } }, - update_node_data: async (target) => { - if (sender != null) { - const connectedMessengers = messenger.getActiveMessengers(); - - for (const connectedMessenger of connectedMessengers) { - messenger.sendOutputMessage(connectedMessenger, Promise.resolve(target)) - } - } - - return target; - } }) }; -export function copy(node: RootZipTreeNode): RootZipTreeNode { +export function copyRoot(node: RootZipTreeNode): RootZipTreeNode { const cloneNode = { children: new Map(), key: node.key } as RootZipTreeNode; + return copyNodeData(cloneNode, node); } -function copyNodeData(clone: N, oldNode: N): N { - for (const entry of oldNode.children.values()) { - // Required to strip the functions that can not cross the message barrier - const cloneMetaData = { - ...entry.metadata, - getData: undefined, - arrayBuffer: undefined, - }; +export function copyChildNode(node: ZipNode): ZipNode { + // Required to strip the functions that can not cross the message barrier + const cloneMetaData = { + ...node.metadata, + getData: undefined, + arrayBuffer: undefined, + }; - const entryClone = { name: entry.name, path: entry.path, type: entry.type, metadata: cloneMetaData, children: new Map() } as any as ZipTreeNode + const cloneNode = { name: node.name, path: node.path, type: node.type, metadata: cloneMetaData, children: new Map() } as any as ZipNode - clone.children.set(entry.name, entryClone) + return copyNodeData(cloneNode, node); +} - copyNodeData(entryClone, entry); +function copyNodeData(clone: N, oldNode: N): N { + for (const entry of oldNode.children.values()) { + clone.children.set(entry.name, copyChildNode(entry)) } return clone; } /** In development the handler is also loaded on the main thread as a bypass, where no server is set up. */ -if (isWorker()) setupAsServer(zipHandlerType, self as unknown as Worker, messageHandler); +if (isWorker()) setupAsServer(zipHandlerType, self as unknown as Worker, messageHandler); diff --git a/packages/ui/src/helpers/toolbox/zip/impl.ts b/packages/ui/src/helpers/toolbox/zip/impl.ts index e605f87ba68..0086b499ae4 100644 --- a/packages/ui/src/helpers/toolbox/zip/impl.ts +++ b/packages/ui/src/helpers/toolbox/zip/impl.ts @@ -16,20 +16,37 @@ import { fileOrNull, type FileZipTreeNode, RootZipTreeNode, - type Sha512, type SourceOptions, + type SourceOptions, type TypedArrayBuffer, type TypedBlob, type ZipFileContent, - type ZipNodeTypes, - ZipTreeNode + type ZipNode, + ZipTreeNode, + type UUID_V7, arrayBufferFromSource } from "#ui/helpers/toolbox/zip"; +import { v7 as uuidV7 } from 'uuid'; +import { + RangeCacheReader, + type RangeCacheReaderOptions +} from "#ui/helpers/toolbox/zip/range-cache-reader.ts"; /** * Method used to generate the zip tree of the given blob returning the root node of the entire zip file */ -export async function readZipTree(logger: Logger, buffer: ArrayBuffer, options: ReadExtraOptions = {}): Promise { - const root = new RootZipTreeNodeImpl(logger, await hashBuffer(buffer, "SHA-512")); - return readZipDataToNode(root, root, buffer, options); +export async function readZipTreeFromBuffer(logger: Logger, buffer: ArrayBuffer, options: ReadExtraOptions = {}): Promise { + const onFinishedLoadingHolder = Promise.withResolvers(); + const root = new RootZipTreeNodeImpl(logger, uuidV7(), onFinishedLoadingHolder); + const result = await readZipDataToNode(root, root, zipReaderFromBuffer(buffer), options); + if (options.innerArchiveReadCallback == null) onFinishedLoadingHolder.resolve() + return result; +} + +export async function readZipTreeFromURL(logger: Logger, url: string, options: ReadExtraOptions & {rangeOptions?: RangeCacheReaderOptions} = {}): Promise { + const onFinishedLoadingHolder = Promise.withResolvers(); + const root = new RootZipTreeNodeImpl(logger, uuidV7(), onFinishedLoadingHolder); + const result = await readZipDataToNode(root, root, zipReaderFromURL(url, options.rangeOptions), options); + if (options.innerArchiveReadCallback == null) onFinishedLoadingHolder.resolve() + return result; } export async function writeZipTree(node: ZipTreeNodeImpl): Promise { @@ -47,22 +64,31 @@ export async function writeZipTree(node: ZipTreeNodeImpl): Promise void }; + +function zipReaderFromBuffer(buffer: ArrayBuffer): ZipReader> { + return new ZipReader(new Uint8ArrayReader(new Uint8Array(buffer))); +} + +function zipReaderFromURL(url: string, options: RangeCacheReaderOptions = {}): ZipReader { + return new ZipReader(new RangeCacheReader(url, options)); +} /** * Reads a given buffer and apply its content to the baseNode passed with the primary */ -async function readZipDataToNode(rootNode: RootZipTreeNodeImpl, baseNode: N, buffer: ArrayBuffer, {name, logger}: ReadExtraOptions): Promise { - const zip = new ZipReader(new Uint8ArrayReader(new Uint8Array(buffer))); - +async function readZipDataToNode(rootNode: RootZipTreeNodeImpl, baseNode: N, reader: ZipReader, options: ReadExtraOptions): Promise { + const { name, logger, innerArchiveReadCallback } = options; logger?.debug(`Setting up to read ${name} archive.`) - const handleEntry = async (entry: EntryData) => { + const zipFiles: Promise[] = []; + + async function handleEntry(entry: EntryData){ const segments = entry.filename.split('/').filter(Boolean); // Remove empty strings from trailing slashes - let parentNode: ZipTreeNode = baseNode; + let parentNode: ZipNode = baseNode; let path = ""; @@ -77,12 +103,7 @@ async function readZipDataToNode(rootNode: RootZipTreeNod : null; if (fileEntry && fileEntry.filename == path) { - segmentNode = parentNode.addFile( - segment, - undefined, - zipArchiveExtensionRegex.test(segment) ? "zip" : "file", - fileEntry - ); + segmentNode = parentNode.addFile(segment, undefined, fileEntry); } else { segmentNode = parentNode.addDirectory(segment, entry); } @@ -91,57 +112,71 @@ async function readZipDataToNode(rootNode: RootZipTreeNod if (segmentNode.type === 'directory') { // Set next parent node to add the given next segment node parentNode = segmentNode; - } else if(segmentNode.type === 'zip' && !segmentNode.metadata.encrypted) { + } else if(!segmentNode.metadata.encrypted) { // FYI: This technically will not work when using onprogress as technically the // given arraybuffer method is not ready yet meaning this is here just for // legacy or needing to go back to waiting for all entries is better. - const arrayBuffer = await fileOrNull(segmentNode)?.arrayBuffer(); - if (arrayBuffer) readZipDataToNode(rootNode, segmentNode, arrayBuffer, { name: segmentNode.name, logger: logger }); + const fileEntry = fileOrNull(segmentNode) ?? undefined; + const fileNode = segmentNode.asFile(); + + if (fileNode == null || fileEntry == null) continue; + + const bufferSetPromise = handleFileNode(fileEntry, fileNode); + + if (bufferSetPromise != null) zipFiles.push(bufferSetPromise) } } } + function handleFileNode(fileEntry: FileEntry, fileNode: FileZipTreeNode): Promise | undefined { + const isZipArchive = zipArchiveExtensionRegex.test(fileNode.name); + + const bufferSetPromise = fileEntry.arrayBuffer() + .then(async (buffer) => { + await fileNode.setBuffer(buffer); + if (isZipArchive) await readZipDataToNode(rootNode, fileNode, zipReaderFromBuffer(buffer), options); + if (innerArchiveReadCallback) innerArchiveReadCallback(rootNode, fileNode); + return fileNode; + }) + + return isZipArchive ? bufferSetPromise : undefined; + } + logger?.debug(`Reading entries ${name} archive.`) - const entries = await zip.getEntries({ + const entries = await reader.getEntries({ onprogress: (_progress, _total, entry) => handleEntry(entry as EntryData) }) - const zipFiles: Promise[] = []; - for (const entry of entries) { const fileNode = rootNode.fileEntries.get( - (baseNode.type != "root" ? baseNode.path + "/" : "") + entry.filename.split('/').filter(Boolean).join('/') + [baseNode.path].concat(entry.filename.split('/')).join('/') ); + if (fileNode == null) continue; - if (fileNode) { - fileNode.setBuffer( - fileOrNull(entry)!.arrayBuffer.bind(entry), - fileNode.type, - entry as ZipWriterAddDataOptions - ); - - if (!fileNode.isSetup) fileNode.isSetup = true; + // TODO: SOMETHING HAS GONE WRONG IF WE HAVE A FILE NODE BUT NO FILE ENTRY + const fileEntry = fileOrNull(entry); + if (fileEntry == null) continue; - if (fileNode.type == "zip") zipFiles.push(readZipDataToNode(rootNode, fileNode, await fileNode.getBuffer(), {name: fileNode.name, logger: logger})) - } + const bufferSetPromise = handleFileNode(fileEntry, fileNode); + if (bufferSetPromise != null) zipFiles.push(bufferSetPromise) } logger?.debug(`Entries from ${name} archive have been read${zipFiles.length > 0 ? ', waiting for given inner Zip Files' : '.'}`) // TODO: MAKE INNER JARS ASYNC I.E. UPDATE LATER TYPE DEAL AFTER THEY ARE DONE LOADING.... - if (zipFiles.length > 0) await Promise.all(zipFiles); + if (zipFiles.length > 0 && innerArchiveReadCallback == null) await Promise.all(zipFiles); return baseNode; } class ZipTreeNodeImpl extends ZipTreeNode { - protected override createFileNode(root: RootZipTreeNodeImpl, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { - return new FileZipTreeNodeImpl(root, parent, name, path, buffer, type, metadata); + protected override createFileNode(root: RootZipTreeNodeImpl, parent: ZipNode, name: string, path: string, buffer: ArrayBufferSource | undefined, metadata: FileEntry): FileZipTreeNode { + return new FileZipTreeNodeImpl(root, parent, name, path, buffer, "file", metadata); } - protected override createNode(root: RootZipTreeNodeImpl, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { - return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) + protected override createNode(root: RootZipTreeNodeImpl, parent: ZipNode, name: string, path: string, metadata?: EntryMetaData): ZipNode { + return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) as ZipNode } } @@ -150,17 +185,21 @@ class ZipTreeNodeImpl extends ZipTreeNode { * combined with extra info like maps for all entries and all file entries within the archive */ class RootZipTreeNodeImpl extends RootZipTreeNode { - constructor(logger: Logger, key: Sha512) { - super(key); + loadingInnerZipCount: number = -1; + onFinishedLoadingHolder: PromiseWithResolvers + + constructor(logger: Logger, key: UUID_V7, onFinishedLoadingHolder: PromiseWithResolvers) { + super(key, onFinishedLoadingHolder.promise); this.logger = logger; + this.onFinishedLoadingHolder = onFinishedLoadingHolder; } - protected override createFileNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { - return new FileZipTreeNodeImpl(root as RootZipTreeNodeImpl, parent as ZipTreeNodeImpl, name, path, buffer, type, metadata); + protected override createFileNode(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, buffer: ArrayBufferSource | undefined, metadata: FileEntry): FileZipTreeNode { + return new FileZipTreeNodeImpl(root as RootZipTreeNodeImpl, parent as ZipNode, name, path, buffer, "file", metadata); } - protected override createNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { - return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) + protected override createNode(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, metadata?: EntryMetaData): ZipNode { + return new ZipTreeNodeImpl(root, parent, name, path, "directory", metadata) as ZipNode } } @@ -169,7 +208,7 @@ class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { isSetup: boolean; private arrayBuffer: LazyMutableValue; - declare type: "zip" | "file"; + declare type: "file"; declare metadata: FileEntry; declare root: RootZipTreeNode; @@ -183,12 +222,12 @@ class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { sha512: LazyValue; - constructor(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata?: FileEntry) { + constructor(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "file", metadata?: FileEntry) { super(root, parent, name, path, type, metadata); if (buffer) { this.isSetup = true; - this.arrayBuffer = lazyMutableValue(() => typeof buffer === 'function' ? buffer() : Promise.resolve(buffer)); + this.arrayBuffer = lazyMutableValue(() => arrayBufferFromSource(buffer)); } else { this.isSetup = false; this.arrayBuffer = lazyMutableValue( @@ -230,7 +269,7 @@ class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { if (buffer != undefined) { return await decompiler.decompile({ - targetFile: { fullPath: this.path, name: this.name, content: buffer, rootKey: this.root.key } as ZipFileContent, + targetFile: { fullPath: this.path, name: this.name, content: buffer, root: this.root.key } as ZipFileContent, resources: this.parent?.children.values().map(entry => { return { fullPath: entry.path, name: entry.name } }).toArray(), options: options }); @@ -278,14 +317,16 @@ class FileZipTreeNodeImpl extends ZipTreeNodeImpl implements FileZipTreeNode { this.sha512 = this.arrayBuffer.lazyMap(async (buffer) => hashBuffer(buffer, "SHA-512")) } - async setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions) { - this.arrayBuffer.updateGetter(() => typeof buffer === 'function' ? buffer() : Promise.resolve(buffer)); - this.type = type ?? "file"; + async setBuffer(buffer: ArrayBufferSource, options?: ZipWriterAddDataOptions) { + this.arrayBuffer.updateGetter(() => arrayBufferFromSource(buffer)); + this.type = "file"; this.metadata = { ...options, ...this.metadata } as FileEntry; + this.isSetup = true; + return this; } diff --git a/packages/ui/src/helpers/toolbox/zip/index.ts b/packages/ui/src/helpers/toolbox/zip/index.ts index a11d4b12ce9..308024faa17 100644 --- a/packages/ui/src/helpers/toolbox/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/zip/index.ts @@ -5,44 +5,83 @@ import { type DecompilerResult, DEFAULT_DECOMPILE_OPTIONS } from "#ui/helpers/toolbox/decompile"; -import type { FileDetails, FileTypeInfo, HashedFileContent } from "#ui/helpers/toolbox/file"; +import type { + AsyncMaybe, + FileDetails, + FileTypeInfo, + HashedFileContent +} from "#ui/helpers/toolbox/file"; import { type LazyValue, lazyValue, TreeNode } from "#ui/helpers/toolbox/misc"; import type { Logger } from "#ui/helpers/toolbox/misc/logging"; +import type { EventSource } from "#ui/helpers/toolbox/misc" +import type {RangeCacheReaderOptions} from "#ui/helpers/toolbox/zip/range-cache-reader.ts"; // TODO: SWITCH TO ZipWriterAddDataOptions instead of EntryMetaData ??? export type EntryData = EntryMetaData & { directory: boolean }; -export type ZipNodeTypes = "zip" | "file" | "directory" | "root"; +export type ZipNodeType = { + 'file': FileZipTreeNode, + 'directory': ZipTreeNode & { type: 'directory'}, +} + +export type ZipNodeTypes = keyof ZipNodeType; +export type ZipNode = ZipNodeType[ZipNodeTypes]; export const nodeTypeToOrdinal = { - "zip": 2, "directory": 1, "file": 0 } as Record -export function sortZipNodes(node1: ZipTreeNode, node2: ZipTreeNode) { - const order = nodeTypeToOrdinal[node1.type] - nodeTypeToOrdinal[node2.type]; +export const zipExtensionRegex = /\.(ear|war|jar|zip|mrpack)$/; +export const zipMimeTypes = new Set([ + 'application/x-zip-compressed', + 'application/zip', + 'application/zip-compressed', + 'application/java-archive' +]) + +export function isZipArchive(node: ZipNode, verifyWithData?: BL): AsyncMaybe { + return (node.type == 'file' + ? (verifyWithData + ? node.getFileType() + .then((type) => type != null ? zipMimeTypes.has(type.mime) : zipExtensionRegex.test(node.name)) + : zipExtensionRegex.test(node.name) + ) + : false + ) as AsyncMaybe +} + +function nodeToOrdinal(node: ZipNode) { + return isZipArchive(node) ? 2 : (node.type == 'directory' ? 1 : 0); +} + +export function sortZipNodes(node1: ZipNode, node2: ZipNode) { + const order = nodeToOrdinal(node1) - nodeToOrdinal(node2); return order == 0 ? node1.name.localeCompare(node2.name, "en") : -order } export type ArrayBufferSource = (() => Promise) | Promise | ArrayBuffer; +export function arrayBufferFromSource(source: ArrayBufferSource): Promise { + return typeof source === 'function' ? source() : Promise.resolve(source); +} + export interface ZipFileContent extends FileDetails { - rootKey: Sha512; + root: UUID_V7; content: ArrayBuffer; } -export class ZipTreeNode extends TreeNode { +export class ZipTreeNode extends TreeNode { root?: RootZipTreeNode; type: ZipNodeTypes; metadata: EntryData - constructor(root: RootZipTreeNode | undefined, parent: ZipTreeNode | undefined, name: string, path: string, type: ZipNodeTypes, metadata?: EntryMetaData) { + constructor(root: RootZipTreeNode | undefined, parent: ZipNode | undefined, name: string, path: string, type: ZipNodeTypes, metadata?: EntryMetaData) { super(parent, name, path); this.root = root; this.type = type; @@ -55,68 +94,71 @@ export class ZipTreeNode extends TreeNode { remove() { this.children.delete(this.name); - this.root?.removeNodeFromRoot(this) + this.root?.removeNodeFromRoot(this as ZipNode) }; - addFile(name: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { - return this.addNode(name, (root, parent, name, path) => this.createFileNode(root, parent, name, path, buffer, type, metadata)); + addFile(name: string, buffer: ArrayBufferSource | undefined, metadata: FileEntry): FileZipTreeNode { + return this.addNode(name, (root, parent, name, path) => this.createFileNode(root, parent, name, path, buffer, metadata)); }; - protected createFileNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, buffer: ArrayBufferSource | undefined, type: "zip" | "file", metadata: FileEntry): FileZipTreeNode { - const node = new DelegateFileZipTreeNode(root, parent, name, path, type, metadata); + protected createFileNode(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, buffer: ArrayBufferSource | undefined, metadata: FileEntry): FileZipTreeNode { + const node = new DelegateFileZipTreeNode(root, parent, name, path, metadata); if (buffer != null) node.setBuffer(buffer); return node; } - addDirectory(name: string, metadata?: EntryMetaData): ZipTreeNode { + addDirectory(name: string, metadata?: EntryMetaData): ZipNode { return this.addNode(name, (root, parent, name, path) => this.createNode(root, parent, name, path, metadata)); }; - protected createNode(root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string, metadata?: EntryMetaData): ZipTreeNode { - return new ZipTreeNode(root, parent, name, path, "directory", metadata) + protected createNode(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, metadata?: EntryMetaData): ZipNode { + return new ZipTreeNode(root, parent, name, path, "directory", metadata) as ZipNode } - addNode(name: string, nodeBuilder: (root: RootZipTreeNode, parent: ZipTreeNode, name: string, path: string) => N): N { + addNode(name: string, nodeBuilder: (root: RootZipTreeNode, parent: ZipNode, name: string, path: string) => N): N { const root = this.root ?? this as unknown as RootZipTreeNode; this.children.get(name)?.remove(); - const node = nodeBuilder(root, this, name, (this.type != "root" ? this.path + "/" : "") + name); + const node = nodeBuilder(root, this as ZipNode, name, this.path + "/" + name); this.children.set(name, node); root.addNodeToRoot(node); return node; }; asFile(): FileZipTreeNode | undefined{ - return (this.type == "zip" || this.type == "file") ? this as unknown as FileZipTreeNode : undefined; + return this.type == "file" ? this as unknown as FileZipTreeNode : undefined; } } -export type Sha512 = string; +export type UUID_V7 = string; /** * Root tree node — extends TreeNode with a flat entries list. */ export class RootZipTreeNode extends ZipTreeNode { - key: Sha512; - entries = new Map(); + key: UUID_V7; + entries = new Map(); fileEntries = new Map(); relativeFileEntries = new Map(); logger?: Logger; - declare type: "root"; + onFinishedLoading: Promise + + declare type: "directory"; declare root: undefined; declare parent: undefined; - constructor(key: Sha512) { - super(undefined, undefined, "", "", "root"); + constructor(key: UUID_V7, onFinishedLoading: Promise) { + super(undefined, undefined, "", "", "directory"); this.key = key; + this.onFinishedLoading = onFinishedLoading; } /** * @internal Only used to add a node to the given reference maps */ - addNodeToRoot(node: ZipTreeNode) { + addNodeToRoot(node: ZipNode) { this.entries.set(node.path, node) const fileNode = node.asFile(); @@ -133,7 +175,7 @@ export class RootZipTreeNode extends ZipTreeNode { /** * @internal Only used to remove a node from the given reference maps */ - removeNodeFromRoot(node: ZipTreeNode) { + removeNodeFromRoot(node: ZipNode) { this.entries.delete(node.path) this.fileEntries.delete(node.path) @@ -158,13 +200,13 @@ export interface LazyDataAccess { } export interface FileZipTreeNode extends ZipTreeNode, LazyDataAccess, SourceHandler { - type: "zip" | "file"; + type: "file"; metadata: FileEntry isSetup: boolean; getBuffer(): Promise - setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise; + setBuffer(buffer: ArrayBufferSource, options?: ZipWriterAddDataOptions): Promise; getFileType(): Promise @@ -182,13 +224,13 @@ export interface TypedBlob extends Blob { fileType: FileTypeInfo | null; } -export function fileOrNull(data: ZipTreeNode|EntryData): FileEntry | null { - const castedData = (((data as Partial).metadata ?? data) as EntryData); +export function fileOrNull(data: ZipNode|EntryData): FileEntry | null { + const castedData = (((data as Partial).metadata ?? data) as EntryData); return !castedData.directory ? castedData as FileEntry : null; } -export function dirOrNull(data: ZipTreeNode|EntryData): DirectoryEntry | null { - const castedData = (((data as Partial).metadata ?? data) as EntryData); +export function dirOrNull(data: ZipNode|EntryData): DirectoryEntry | null { + const castedData = (((data as Partial).metadata ?? data) as EntryData); return castedData.directory ? castedData as DirectoryEntry : null; } @@ -199,7 +241,7 @@ export type FileMatcher = { } export interface RootNodeTarget { - root: Sha512 + root: UUID_V7 } export interface NodeTarget extends RootNodeTarget { @@ -211,31 +253,45 @@ export function getAsTarget(target: FileZipTreeNode | NodeTarget) { return (target as FileZipTreeNode).type != undefined ? {path: target.path, root: (target.root as RootZipTreeNode).key} : target as NodeTarget; } +export type ZipBufferSource = { data_type: 'buffer', name?: string, url?: string, sha512Hash?: string, buffer: ArrayBufferSource, rangeOptions?: undefined }; +export type ZipURLSource = { data_type: 'url', name?: string, url: string, sha512Hash?: string, buffer?: undefined, rangeOptions?: RangeCacheReaderOptions,} +export type ZipDataSource = ZipBufferSource | ZipURLSource; + +type Typed = { type: K } & T; + +export type ArchiveUpdateTypes = + Typed<'node_loaded', { root: RootNodeTarget, node: ZipNode, }> | + Typed<'updated_node_data', { node: NodeTarget }> | + Typed<'root_loaded', { root: RootNodeTarget }> | + Typed<'archive_download', { root: RootNodeTarget, current: number, total: number }>; + export interface ZipHandler { - zipArchivesByHash: Map; - urlToHash: Map; + zipByUUID: Map; + urlToUUID: Map; readLazyData(node: FileZipTreeNode | NodeTarget, key: K): Promise>>; readSource(node: FileZipTreeNode | NodeTarget, options?: SourceOptions): Promise; readNodeData(node: FileZipTreeNode | NodeTarget): Promise; - writeNodeData(node: FileZipTreeNode | NodeTarget, buffer: ArrayBuffer, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise; - collectFiles(root: RootZipTreeNode | Sha512, matcher?: FileMatcher): Promise; - collectHashedFiles(root: RootZipTreeNode | Sha512, matcher?: FileMatcher, deduplicate?: boolean): Promise; - getOrReadZipTree(buffer: () => Promise, details?: { name?: string, url?: string, hash?: string }): Promise; - readZipTree(buffer: ArrayBuffer, details?: { name?: string, url?: string, hash?: string }): Promise; - getZipTree(value: string | Sha512, isUrl: boolean, localOnly?: boolean): Promise; - closeZipTree(root: RootZipTreeNode | Sha512, name?: string): Promise<{ err?: Error, wasClosed: boolean }>; + writeNodeData(node: FileZipTreeNode | NodeTarget, buffer: ArrayBuffer, options?: ZipWriterAddDataOptions): Promise; + collectFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher): Promise; + collectHashedFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher, deduplicate?: boolean): Promise; + getOrReadZipTree(source: ZipDataSource): Promise; + readZipTree(source: ZipDataSource): Promise; + getZipTree(value: string | UUID_V7, isUrl: boolean, localOnly?: boolean): Promise; + closeZipTree(root: RootZipTreeNode | UUID_V7, name?: string): Promise<{ err?: Error, wasClosed: boolean }>; closeAllZipTrees(): Promise<{ err?: Error, wasClosed: boolean }>; + archiveUpdate(msg: ArchiveUpdateTypes): void; + onArchiveUpdate: EventSource<(msg: ArchiveUpdateTypes) => void>; } export class DelegateFileZipTreeNode extends ZipTreeNode implements FileZipTreeNode { - declare type: "zip" | "file"; + declare type: "file"; declare metadata: FileEntry; declare handler: ZipHandler | undefined; isSetup: boolean; - constructor(root: RootZipTreeNode | undefined, parent: ZipTreeNode | undefined, name: string, path: string, type: "zip" | "file", metadata: FileEntry) { - super(root, parent, name, path, type, metadata); + constructor(root: RootZipTreeNode | undefined, parent: ZipNode | undefined, name: string, path: string, metadata: FileEntry) { + super(root, parent, name, path, "file", metadata); this.handler = undefined; @@ -317,9 +373,8 @@ export class DelegateFileZipTreeNode extends ZipTreeNode implements FileZipTreeN return this.getHandler().readNodeData(this); } - async setBuffer(buffer: ArrayBufferSource, type?: "zip" | "file", options?: ZipWriterAddDataOptions): Promise { - const bufferPromise = typeof buffer === 'function' ? buffer() : Promise.resolve(buffer); - await this.getHandler().writeNodeData(this, await bufferPromise, type, options); + async setBuffer(buffer: ArrayBufferSource, options?: ZipWriterAddDataOptions): Promise { + await this.getHandler().writeNodeData(this, await arrayBufferFromSource(buffer), options); return this; } @@ -350,9 +405,9 @@ export function reconstituteRoot(handler: ZipHandler, root: RootZipTreeNode): Ro return reconstituteNodes(handler, root, root, root) } -function reconstituteNodes(handler: ZipHandler, root: RootZipTreeNode, parent: N, node: N): N { - let reconstitutedNode: ZipTreeNode; - if (node.type == "root") { +export function reconstituteNodes(handler: ZipHandler, root: RootZipTreeNode, parent: N, node: N): N { + let reconstitutedNode: ZipNode; + if (node instanceof RootZipTreeNode) { reconstitutedNode = Object.setPrototypeOf(node, RootZipTreeNode.prototype); } else if (node.type == "directory") { reconstitutedNode = Object.setPrototypeOf(node, ZipTreeNode.prototype); @@ -363,14 +418,14 @@ function reconstituteNodes(handler: ZipHandler, root: Roo node.children.forEach((entry) => reconstituteNodes(handler, root, node, entry)) - if (reconstitutedNode.type != "root") { + if (!(reconstitutedNode instanceof RootZipTreeNode)) { root.entries.set(reconstitutedNode.path, reconstitutedNode); reconstitutedNode.root = root; reconstitutedNode.parent = parent; } - if (node.type == "file" || node.type == "zip") (node as unknown as DelegateFileZipTreeNode).finishNodeSetup(handler) + if (node.type == "file") (node as unknown as DelegateFileZipTreeNode).finishNodeSetup(handler) return reconstitutedNode as N; } diff --git a/packages/ui/src/helpers/toolbox/zip/range-cache-reader.ts b/packages/ui/src/helpers/toolbox/zip/range-cache-reader.ts new file mode 100644 index 00000000000..68d1e408a3d --- /dev/null +++ b/packages/ui/src/helpers/toolbox/zip/range-cache-reader.ts @@ -0,0 +1,586 @@ +import { Reader } from "@zip.js/zip.js"; + +/** + * A zip.js Reader for remote archives that: + * 1. Fetches the tail (EOCD + central directory) first, so getEntries() resolves after 1–2 requests, + * 2. Serves on-demand reads (entry.getData) with priority, + * 3. Downloads the rest of the file in the background. + * + * Overall the given zip is cached in fixed-size blocks, any tiny reads + * (30-byte local headers, names, extra fields) become a few block-sized requests, and only unique + * data is saved to the cache. + */ + +// --------------------------------------------------------------------------------------------- +// Block cache +// --------------------------------------------------------------------------------------------- + +/** + * Fixed-size block store. Block i covers [i*blockSize, min((i+1)*blockSize, fileSize)). + * Blocks are views into response buffers: inserting never copies, reading copies only when a read + * spans more than one block. + */ +export class BlockCache { + readonly blockCount: number; + private readonly blocks: (Uint8Array | undefined)[]; + private cached = 0; + + constructor( + readonly fileSize: number, + readonly blockSize: number, + ) { + this.blockCount = Math.ceil(fileSize / blockSize); + this.blocks = new Array(this.blockCount); + } + + /** Total number of bytes cached. */ + get size(): number { + return this.cached; + } + + blockOf(offset: number): number { + return Math.floor(offset / this.blockSize); + } + + blockStart(i: number): number { + return i * this.blockSize; + } + + blockEnd(i: number): number { + return Math.min((i + 1) * this.blockSize, this.fileSize); + } + + has(i: number): boolean { + return this.blocks[i] !== undefined; + } + + /** + * Stores `bytes`, which start at the beginning of block `first` and cover whole blocks (the last + * one may end at EOF). Already-cached blocks are left as they are. + */ + insertBlocks(first: number, bytes: Uint8Array): void { + let pos = 0; + for (let i = first; pos < bytes.length; i++) { + const len = this.blockEnd(i) - this.blockStart(i); + if (pos + len > bytes.length) throw new RangeError(`Partial block ${i}: ${bytes.length - pos} of ${len} bytes`); + if (!this.blocks[i]) { + this.blocks[i] = bytes.subarray(pos, pos + len); + this.cached += len; + } + pos += len; + } + } + + /** Indices of blocks in [first, last] that are not cached. */ + missing(first: number, last: number): number[] { + const out: number[] = []; + for (let i = first; i <= last; i++) if (!this.blocks[i]) out.push(i); + return out; + } + + /** Bytes [start, end) if fully cached, else undefined. Zero-copy within a single block. */ + read(start: number, end: number): Uint8Array | undefined { + if (end <= start) return new Uint8Array(0); + const first = this.blockOf(start); + const last = this.blockOf(end - 1); + if (first === last) { + const b = this.blocks[first]; + if (!b) return undefined; + const base = this.blockStart(first); + return b.subarray(start - base, end - base); + } + for (let i = first; i <= last; i++) if (!this.blocks[i]) return undefined; + const out = new Uint8Array(end - start); + for (let i = first; i <= last; i++) { + const b = this.blocks[i]!; + const base = this.blockStart(i); + const from = Math.max(start, base) - base; + const to = Math.min(end, base + b.length) - base; + out.set(b.subarray(from, to), base + from - start); + } + return out; + } + + /** All blocks in order, for building a Blob without concatenating. Undefined if incomplete. */ + allBlocks(): Uint8Array[] | undefined { + return this.cached === this.fileSize ? (this.blocks as Uint8Array[]) : undefined; + } +} + +// --------------------------------------------------------------------------------------------- +// Request limiter with priority (foreground reads before background fill) +// --------------------------------------------------------------------------------------------- + +class Limiter { + private active = 0; + private readonly high: (() => void)[] = []; + private readonly low: (() => void)[] = []; + + constructor(private readonly max: number) {} + + async run(priority: "high" | "low", task: () => Promise): Promise { + if (this.active >= this.max) { + await new Promise((resolve) => (priority === "high" ? this.high : this.low).push(resolve)); + } else { + this.active++; + } + try { + return await task(); + } finally { + const next = this.high.shift() ?? this.low.shift(); + if (next) next(); // hand the slot over directly + else this.active--; + } + } +} + +// --------------------------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------------------------- + +export class RangeReaderError extends Error { + override name = "RangeReaderError"; +} +/** The remote file changed between requests (If-Range mismatch, validator change, size change). */ +export class FileChangedError extends RangeReaderError { + override name = "FileChangedError"; +} + +// --------------------------------------------------------------------------------------------- +// Central directory location (EOCD / Zip64) +// --------------------------------------------------------------------------------------------- + +export interface CentralDirectoryLocation { + offset: number; + size: number; + entryCount: number; + zip64: boolean; +} + +const SIG_EOCD = 0x06054b50; +const SIG_ZIP64_LOCATOR = 0x07064b50; +const SIG_ZIP64_EOCD = 0x06064b50; +const EOCD_MIN = 22; +const MAX_COMMENT = 0xffff; +const ZIP64_LOCATOR_SIZE = 20; +const ZIP64_EOCD_MIN = 56; + +function u64(view: DataView, at: number): number { + const v = view.getBigUint64(at, true); + if (v > BigInt(Number.MAX_SAFE_INTEGER)) throw new RangeReaderError("Zip64 value exceeds 2^53"); + return Number(v); +} + +/** Finds the EOCD inside `tail` (the last bytes of the file, starting at absolute `tailStart`). */ +function findEocd(tail: Uint8Array, tailStart: number, fileSize: number) { + const view = new DataView(tail.buffer, tail.byteOffset, tail.byteLength); + for (let i = tail.length - EOCD_MIN; i >= 0; i--) { + if (view.getUint32(i, true) !== SIG_EOCD) continue; + const commentLength = view.getUint16(i + 20, true); + // The comment must run exactly to EOF; guards against the signature appearing inside the comment. + if (tailStart + i + EOCD_MIN + commentLength !== fileSize) continue; + return { + absOffset: tailStart + i, + entryCount: view.getUint16(i + 10, true), + cdSize: view.getUint32(i + 12, true), + cdOffset: view.getUint32(i + 16, true), + }; + } + return undefined; +} + +// --------------------------------------------------------------------------------------------- +// Reader +// --------------------------------------------------------------------------------------------- + +export interface RangeCacheReaderOptions { + /** + * Known total size in bytes. Skips the initial HEAD, so the tree needs a single request. + */ + size?: number; + /** + * Cache/request granularity. Every request covers whole blocks, so this is the minimum request + * size. Default 128 KiB. + */ + blockSize?: number; + /** + * Bytes requested from the end of the file first. Default 256 KiB (covers EOCD + typical zip CD). + */ + tailSize?: number; + /** + * Max size of one background-fill request. Default 4 MiB. + */ + backgroundRequestSize?: number; + /** + * Max size of one on-demand request (long missing runs are split). Default 8 MiB. + */ + maxRequestSize?: number; + /** + * Max concurrent requests. On-demand reads are queued ahead of background fill. Default 4. + */ + maxConcurrentRequests?: number; + /** + * Send If-Range so a changed file is detected server-side. Default true. If-Range is not a + * CORS-safelisted header, so it triggers a preflight. The Last-Modified/ETag comparison on + * every response still catches a replaced file. + */ + validate?: boolean; + /** + * Extra fetch options (credentials, mode, ...). Headers given here are merged. + */ + fetchInit?: RequestInit; + /** + * Called after each response is cached. Throttle on your side if it drives UI. + */ + onProgress?: (cached: number, total: number) => void; +} + +interface Validator { + header: "etag" | "last-modified"; + value: string; +} + +export class RangeCacheReader extends Reader { + readonly url: string; + private readonly opts: Required> & + Pick; + + private cache!: BlockCache; + /** Pending block downloads, by block index, for de-duplication. */ + private readonly pending = new Map>(); + private readonly limiter: Limiter; + private readonly controllers = new Set(); + private aborted: { reason: unknown } | undefined; + private validator: Validator | undefined; + private initPromise: Promise | undefined; + private backgroundPromise: Promise | undefined; + + // Foreground reads pause the background fill between requests. + private foregroundCount = 0; + private foregroundIdle: Promise = Promise.resolve(); + private releaseIdle: () => void = () => {}; + + /** False if the server ignored Range and the whole file was downloaded in one response. */ + rangesSupported = true; + /** Location of the central directory, available after init(). */ + centralDirectory!: CentralDirectoryLocation; + + constructor(url: string, options: RangeCacheReaderOptions = {}) { + super(url); + this.url = url; + this.opts = { + size: options.size, + blockSize: options.blockSize ?? 128 * 1024, + // Never smaller than the EOCD search window + Zip64 locator, or locating the CD costs extra requests. + tailSize: Math.max(options.tailSize ?? 256 * 1024, EOCD_MIN + MAX_COMMENT + ZIP64_LOCATOR_SIZE), + backgroundRequestSize: options.backgroundRequestSize ?? 4 * 1024 * 1024, + maxRequestSize: options.maxRequestSize ?? 8 * 1024 * 1024, + maxConcurrentRequests: options.maxConcurrentRequests ?? 4, + validate: options.validate ?? true, + fetchInit: options.fetchInit, + onProgress: options.onProgress, + }; + this.limiter = new Limiter(this.opts.maxConcurrentRequests); + } + + /** Idempotent: zip.js calls init() itself, and callers may call it earlier. */ + override init(): Promise { + this.initPromise ??= this.doInit(); + return this.initPromise; + } + + private async doInit(): Promise { + // 1. Total size, via HEAD unless given. Content-Length/Last-Modified are CORS-safelisted. + let size = this.opts.size; + if (size === undefined) { + const head = await this.exchange({}, "HEAD"); + const len = Number(head.headers.get("content-length") ?? NaN); + if (head.ok && Number.isSafeInteger(len) && len >= 0) { + size = len; + this.validator = pickValidator(head.headers); + } + } + if (size === undefined) return this.initFromFullDownload(); // HEAD unusable: just download + + this.size = size; + this.cache = new BlockCache(size, this.opts.blockSize); + + // 2. Tail, aligned down to a block boundary, via an explicit range (never `bytes=-N`). + const firstBlock = this.cache.blockOf(Math.max(0, size - this.opts.tailSize)); + const tailStart = this.cache.blockStart(firstBlock); + const headers: Record = { Range: `bytes=${tailStart}-${size - 1}` }; + if (this.opts.validate && this.validator) headers["If-Range"] = this.validator.value; + const res = await this.exchange(headers); + + // 200 = server ignored Range, or the file changed since HEAD. Either way the body is the + // current, complete file, so use it. + if (res.status === 200) return this.initFromBytes(res.body!); + if (res.status !== 206) throw new RangeReaderError(`Unexpected status ${res.status} for ${this.url}`); + + this.validator ??= pickValidator(res.headers); + this.assertSameFile(res.headers); + this.assertRange(res.headers, tailStart, size, res.body!.length); + this.cache.insertBlocks(firstBlock, res.body!); + this.report(); + + this.centralDirectory = await this.locateCentralDirectory(); + // Make sure the whole CD is cached so getEntries() never waits on the network again. + const cd = this.centralDirectory; + await this.ensure(cd.offset, cd.offset + cd.size, "high"); + } + + private async initFromFullDownload(): Promise { + const res = await this.exchange({}); + if (!res.ok) throw new RangeReaderError(`Unexpected status ${res.status} for ${this.url}`); + return this.initFromBytes(res.body!); + } + + /** Uses a complete-file body as the whole cache; no further requests are made. */ + private async initFromBytes(bytes: Uint8Array): Promise { + this.rangesSupported = false; + this.size = bytes.length; + this.cache = new BlockCache(bytes.length, this.opts.blockSize); + this.cache.insertBlocks(0, bytes); + this.report(); + this.centralDirectory = await this.locateCentralDirectory(); + } + + private async locateCentralDirectory(): Promise { + const searchLen = Math.min(this.size, EOCD_MIN + MAX_COMMENT); + const tailStart = this.size - searchLen; + const tail = await this.bytes(tailStart, this.size); + const eocd = findEocd(tail, tailStart, this.size); + if (!eocd) throw new RangeReaderError("End of central directory record not found (not a zip?)"); + + const needsZip64 = eocd.cdOffset === 0xffffffff || eocd.cdSize === 0xffffffff || eocd.entryCount === 0xffff; + const locatorAt = eocd.absOffset - ZIP64_LOCATOR_SIZE; + if (locatorAt >= 0) { + const loc = await this.bytes(locatorAt, eocd.absOffset); + const lv = new DataView(loc.buffer, loc.byteOffset, loc.byteLength); + if (lv.getUint32(0, true) === SIG_ZIP64_LOCATOR) { + const z64At = u64(lv, 8); + const rec = await this.bytes(z64At, z64At + ZIP64_EOCD_MIN); + const rv = new DataView(rec.buffer, rec.byteOffset, rec.byteLength); + if (rv.getUint32(0, true) !== SIG_ZIP64_EOCD) throw new RangeReaderError("Bad Zip64 EOCD record"); + return { entryCount: u64(rv, 32), size: u64(rv, 40), offset: u64(rv, 48), zip64: true }; + } + } + if (needsZip64) throw new RangeReaderError("Zip64 markers present but no Zip64 EOCD locator"); + return { offset: eocd.cdOffset, size: eocd.cdSize, entryCount: eocd.entryCount, zip64: false }; + } + + override async readUint8Array(index: number, length: number): Promise { + await this.init(); + const end = Math.min(index + length, this.size); // zip.js contract: truncate at EOF + const hit = this.cache.read(index, end); + if (hit) return hit; + + this.beginForeground(); + try { + return await this.bytes(index, end); + } finally { + this.endForeground(); + } + } + + /** + * Starts (or returns) the background download of everything not yet cached. + * Resolves when the whole file is cached. Safe to call multiple times. + */ + fillBackground(): Promise { + this.backgroundPromise ??= (async () => { + await this.init(); + const step = Math.max(1, Math.floor(this.opts.backgroundRequestSize / this.opts.blockSize)); + for (let b = 0; b < this.cache.blockCount; b += step) { + await this.foregroundIdle; // yield to on-demand reads + this.throwIfAborted(); + const last = Math.min(b + step, this.cache.blockCount) - 1; + await this.ensureBlocks(b, last, "low", step); + } + })(); + return this.backgroundPromise; + } + + /** Whole file as a Blob once fully cached (starts the background fill if needed). No big copy. */ + async toBlob(type = "application/java-archive"): Promise { + await this.fillBackground(); + return new Blob(this.cache.allBlocks()! as Uint8Array[], { type }); + } + + /** Cancels all in-flight requests; pending and future reads reject. */ + abort(reason: unknown = new DOMException("Aborted", "AbortError")): void { + this.aborted = { reason }; + for (const c of this.controllers) c.abort(reason); + } + + get cachedBytes(): number { + return this.cache?.size ?? 0; + } + + // ------------------------------------------------------------------------------------------- + // Internals + // ------------------------------------------------------------------------------------------- + + private async bytes(start: number, end: number): Promise { + await this.ensure(start, end, "high"); + return this.cache.read(start, end)!; + } + + private ensure(start: number, end: number, priority: "high" | "low"): Promise { + if (end <= start) return Promise.resolve(); + const maxBlocks = Math.max(1, Math.floor(this.opts.maxRequestSize / this.opts.blockSize)); + return this.ensureBlocks(this.cache.blockOf(start), this.cache.blockOf(end - 1), priority, maxBlocks); + } + + /** + * Ensures blocks [first, last] are cached. Blocks already downloading are awaited; the rest are + * grouped into contiguous runs of at most `maxBlocks` and fetched, one request per run. + */ + private async ensureBlocks(first: number, last: number, priority: "high" | "low", maxBlocks: number): Promise { + this.throwIfAborted(); + const waits: Promise[] = []; + let runStart = -1; + let runEnd = -1; // inclusive + const flush = () => { + for (let s = runStart; s <= runEnd; s += maxBlocks) { + waits.push(this.fetchBlocks(s, Math.min(s + maxBlocks - 1, runEnd), priority)); + } + runStart = runEnd = -1; + }; + for (const i of this.cache.missing(first, last)) { + const p = this.pending.get(i); + if (p) { + if (runStart !== -1) flush(); + if (!waits.includes(p)) waits.push(p); + continue; + } + if (runStart !== -1 && i !== runEnd + 1) flush(); + if (runStart === -1) runStart = i; + runEnd = i; + } + if (runStart !== -1) flush(); + await Promise.all(waits); + } + + /** Fetches blocks [first, last] in one request and caches them. */ + private fetchBlocks(first: number, last: number, priority: "high" | "low"): Promise { + if (!this.rangesSupported) { + return Promise.reject(new RangeReaderError("Range requests unsupported and bytes not cached")); + } + const start = this.cache.blockStart(first); + const end = this.cache.blockEnd(last); + const promise = this.limiter + .run(priority, async () => { + this.throwIfAborted(); + const headers: Record = { Range: `bytes=${start}-${end - 1}` }; + if (this.opts.validate && this.validator) headers["If-Range"] = this.validator.value; + // Don't download a full body we'd throw away. + const res = await this.exchange(headers, "GET", (status) => status !== 206); + + if (res.status === 200) { + throw new FileChangedError(`Server returned the full file for a range request; ${this.url} changed?`); + } + if (res.status !== 206) throw new RangeReaderError(`Unexpected status ${res.status}`); + // Backstop for servers/CDNs that ignore If-Range: compare the validator ourselves. + this.assertSameFile(res.headers); + this.assertRange(res.headers, start, end, res.body!.length); + this.cache.insertBlocks(first, res.body!); + this.report(); + }) + .finally(() => { + for (let i = first; i <= last; i++) if (this.pending.get(i) === promise) this.pending.delete(i); + }); + for (let i = first; i <= last; i++) this.pending.set(i, promise); + return promise; + } + + /** + * Performs one request and reads its body. Each request gets its own AbortController (tracked so + * abort() reaches it) instead of sharing one signal, so listeners never pile up. + */ + private async exchange( + extraHeaders: Record, + method = "GET", + skipBody: (status: number) => boolean = () => false, + ): Promise<{ status: number; ok: boolean; headers: Headers; body?: Uint8Array }> { + this.throwIfAborted(); + const init = this.opts.fetchInit ?? {}; + const headers = new Headers(init.headers); + for (const [k, v] of Object.entries(extraHeaders)) headers.set(k, v); + const controller = new AbortController(); + this.controllers.add(controller); + try { + const res = await fetch(this.url, { ...init, method, headers, signal: controller.signal }); + let body: Uint8Array | undefined; + if (method === "HEAD" || skipBody(res.status)) await res.body?.cancel(); + else body = new Uint8Array(await res.arrayBuffer()); + return { status: res.status, ok: res.ok, headers: res.headers, body }; + } finally { + this.controllers.delete(controller); + } + } + + /** Throws FileChangedError if the response's validator differs from the one we started with. */ + private assertSameFile(headers: Headers): void { + if (!this.validator) return; + const now = headers.get(this.validator.header); + if (now !== null && now !== this.validator.value) { + throw new FileChangedError(`${this.validator.header} changed (${this.validator.value} → ${now}); ${this.url} was replaced`); + } + } + + /** + * Verifies a 206 body covers [start, end). Content-Range is checked when readable (cross-origin it + * is only readable with Access-Control-Expose-Headers); otherwise the exact body length is required. + */ + private assertRange(headers: Headers, start: number, end: number, bodyLength: number): void { + const range = parseContentRange(headers.get("content-range")); + if (range) { + if (range.total !== this.size) throw new FileChangedError(`File size changed (${this.size} → ${range.total})`); + if (range.start !== start || range.end !== end) { + throw new RangeReaderError(`Server returned bytes ${range.start}-${range.end - 1}, wanted ${start}-${end - 1}`); + } + } + if (bodyLength !== end - start) throw new RangeReaderError(`Got ${bodyLength} bytes, wanted ${end - start}`); + } + + private beginForeground(): void { + if (this.foregroundCount++ === 0) { + this.foregroundIdle = new Promise((resolve) => (this.releaseIdle = resolve)); + } + } + + private endForeground(): void { + if (--this.foregroundCount === 0) this.releaseIdle(); + } + + private throwIfAborted(): void { + if (this.aborted) throw this.aborted.reason; + } + + private report(): void { + this.opts.onProgress?.(this.cache.size, this.size); + } +} + +// --------------------------------------------------------------------------------------------- +// Header helpers +// --------------------------------------------------------------------------------------------- + +/** Parses `bytes start-end/total` into a half-open range. */ +function parseContentRange(value: string | null): { start: number; end: number; total: number } | undefined { + const m = value?.match(/^bytes (\d+)-(\d+)\/(\d+)$/); + if (!m) return undefined; + return { start: Number(m[1]), end: Number(m[2]) + 1, total: Number(m[3]) }; +} + +/** + * If-Range requires a strong ETag; weak ones are ignored by servers, so fall back to Last-Modified. + * Cross-origin, ETag is only readable with Access-Control-Expose-Headers; Last-Modified always is. + */ +function pickValidator(headers: Headers): Validator | undefined { + const etag = headers.get("etag"); + if (etag && !etag.startsWith("W/")) return { header: "etag", value: etag }; + const lm = headers.get("last-modified"); + return lm ? { header: "last-modified", value: lm } : undefined; +} From 0a6569a8527d4b86ed7934aff5bad15bbf029e1b Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Tue, 13 Oct 2026 20:29:58 -0500 Subject: [PATCH 04/18] fix: Resolve issues with async node updating - Fix node merging for existing nodes not merging children properly and instead deleting them - Hash JAR from URL request and update UUID for proper hash or set hash if not found --- .../helpers/toolbox/messenger/implement.ts | 23 +-- .../ui/src/helpers/toolbox/messenger/index.ts | 3 +- .../ui/src/helpers/toolbox/workers/zip/api.ts | 91 +++++++-- .../src/helpers/toolbox/workers/zip/index.ts | 174 ++++++++++-------- packages/ui/src/helpers/toolbox/zip/impl.ts | 122 ++++++++---- packages/ui/src/helpers/toolbox/zip/index.ts | 13 +- 6 files changed, 281 insertions(+), 145 deletions(-) diff --git a/packages/ui/src/helpers/toolbox/messenger/implement.ts b/packages/ui/src/helpers/toolbox/messenger/implement.ts index 959a7d79b20..fa00f8fe41e 100644 --- a/packages/ui/src/helpers/toolbox/messenger/implement.ts +++ b/packages/ui/src/helpers/toolbox/messenger/implement.ts @@ -171,11 +171,6 @@ export function createRegistryInternal(): MessengerRegistry { getActiveMessengers() { return this.types.values().flatMap((type) => (type as MessengerTypeImpl).getInstances()).toArray(); }, - forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void) { - for (const connection of this.getActiveMessengers()) { - if (connection !== messenger) callback(connection); - } - }, setWorkerFactoryFor(type, worker) { if (!isWorker()) (type as MessengerTypeImpl).workerFactory = worker; } @@ -316,12 +311,6 @@ function handleOutputMessages(messenger: ClientHandle taskHolder.endTask(msg.key, msg.error, true); } else { // MAIN THREAD OR PORT - if (taskHolder.hasTask(msg.key)) { - logger.debug("Completed a task for given data!", {obj: msg}) - } else { - logger.error("Unable to Complete a task for given data as its not here anymore!!", {obj: msg}) - } - if (msg.key == FORCED_OUTPUT_KEY) { if (forcedOutputHandler == null || !forcedOutputHandler(messenger, msg.data)) { logger.error("Unable to Complete a forced output for given data as its not supported for this messenger!", {obj: msg}) @@ -330,6 +319,12 @@ function handleOutputMessages(messenger: ClientHandle return; } + if (taskHolder.hasTask(msg.key)) { + logger.debug("Completed a task for given data!", {obj: msg}) + } else { + logger.error("Unable to Complete a task for given data as its not here anymore!!", {obj: msg}) + } + taskHolder.endTask(msg.key, msg.data, false); } } @@ -484,9 +479,9 @@ function serverMessageHandler( getActiveMessengers() { return peerPorts.values().toArray() as Messenger[]; }, - forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void) { - for (const connection of this.getActiveMessengers()) { - if (connection !== messenger) callback(connection); + forConnections(callback: (connection: Messenger) => void, except?: Messenger) { + for (const connection of [this, ...this.getActiveMessengers()]) { + if (connection !== except) callback(connection); } }, async sendClientMessage(sender, msg) { diff --git a/packages/ui/src/helpers/toolbox/messenger/index.ts b/packages/ui/src/helpers/toolbox/messenger/index.ts index 7ce5992f78b..21b9094d23f 100644 --- a/packages/ui/src/helpers/toolbox/messenger/index.ts +++ b/packages/ui/src/helpers/toolbox/messenger/index.ts @@ -79,7 +79,6 @@ export interface MessengerType { getMessenger: (type: MessengerType) => Promise & H> | undefined; getActiveMessengers(): Messenger[]; - forActiveMessengersUnless(messenger: Messenger | undefined, callback: (connection: Messenger) => void): void } export interface MessengerOptions { @@ -134,6 +133,8 @@ export interface Messenger { export interface ServerMessenger extends Messenger, MessengerAccessor { sendClientMessage(sender: Messenger, msg: () => Promise>>): Promise; + + forConnections(callback: (connection: Messenger) => void, except?: Messenger | undefined): void } export interface ClientMessenger extends Messenger { diff --git a/packages/ui/src/helpers/toolbox/workers/zip/api.ts b/packages/ui/src/helpers/toolbox/workers/zip/api.ts index fc8dd0bc2c8..7fd073a690a 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/api.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/api.ts @@ -42,7 +42,7 @@ export type ZipMessages = MessageTypes< }, get_zip_tree: { serverbound: { value: string | UUID_V7 }, - clientbound: { node: RootZipTreeNode | undefined } + clientbound: { node: RootZipTreeNode | undefined, isFullyLoaded?: boolean } }, collect_files: { serverbound: RootNodeTarget & { matcher?: FileMatcher }, @@ -67,10 +67,56 @@ export type ZipMessages = MessageTypes< } > +/** + * Adds a node read in the background (e.g. a nested archive's tree) to the client's copy of the + * tree. A node the client already has (e.g. the nested archive's file) gets the loaded children. + */ +function mergeLoadedNode(handler: ZipHandler, root: RootZipTreeNode, loaded: ZipNode) { + const existing = root.entries.get(loaded.path); + if (existing != null) { + for (const child of loaded.children.values()) { + existing.children.set(child.name, reconstituteNodes(handler, root, existing, child)); + } + return; + } + + const parentPath = loaded.path.split("/").slice(0, -1).join("/"); + const parent = parentPath === "" ? root : root.entries.get(parentPath); + if (parent == null) return; + + parent.children.set(loaded.name, reconstituteNodes(handler, root, parent, loaded)); +} + export const handlerConstructor: MessengerHandlerConstructor = (messenger) => { const uuidToUrls = new Map(); const hashToUUID = new Map(); const uuidToOnFinishedLoadingHolder = new Map>(); + /** Roots whose loading finished, which may be reported before the tree itself is returned. */ + const finishedRoots = new Set(); + /** Hashes of roots, which may also be reported before the tree itself is returned. */ + const rootHashes = new Map(); + + function forgetRoot(key: UUID_V7) { + finishedRoots.delete(key); + uuidToOnFinishedLoadingHolder.delete(key); + rootHashes.delete(key); + for (const [hash, uuid] of hashToUUID) if (uuid === key) hashToUUID.delete(hash); + } + + function trackLoading(node: RootZipTreeNode, isFullyLoaded: boolean | undefined) { + node.sha512Hash ??= rootHashes.get(node.key); + if (node.sha512Hash != null) hashToUUID.set(node.sha512Hash, node.key); + + if (isFullyLoaded || finishedRoots.has(node.key)) { + node.onFinishedLoading = Promise.resolve(); + return; + } + + const holder = uuidToOnFinishedLoadingHolder.get(node.key) ?? Promise.withResolvers(); + uuidToOnFinishedLoadingHolder.set(node.key, holder); + node.onFinishedLoading = holder.promise; + } + return { zipByUUID: new Map(), urlToUUID: new Map(), @@ -105,9 +151,7 @@ export const handlerConstructor: MessengerHandlerConstructor(); - - node.onFinishedLoading = msg.isFullyLoaded ? Promise.resolve() : holder.promise; + trackLoading(node, msg.isFullyLoaded); const filteredURL = url?.split("?")[0]; @@ -120,7 +164,7 @@ export const handlerConstructor: MessengerHandlerConstructor Promise void }> | undefined)?.close?.(); +} + +/** + * Maps the archive's actual hash to it, dropping the hash it was opened with when that turns out to + * be wrong, and lets every connection know. + */ +function onArchiveHashed(messenger: ServerMessenger, root: RootZipTreeNode, sha512Hash: Sha512, expectedHash?: Sha512) { + root.sha512Hash = sha512Hash; + hashToUUID.set(sha512Hash, root.key); + + const mismatched = expectedHash != null && expectedHash !== sha512Hash; + if (mismatched) { + messenger.logger.warn(`Archive '${root.key}' was opened with SHA-512 '${expectedHash}' but its contents hash to '${sha512Hash}'`); + if (hashToUUID.get(expectedHash) === root.key) hashToUUID.delete(expectedHash); + } + + broadcastArchiveUpdate(messenger, { + type: 'root_hashed', + root: { root: root.key }, + sha512Hash: sha512Hash, + expectedHash: mismatched ? expectedHash : undefined, + }); +} + +/** + * Sends an archive update to every connection, i.e. the main thread (the server itself) and any + * peer worker, except the one given. + */ +function broadcastArchiveUpdate(messenger: ServerMessenger, update: ArchiveUpdateTypes, except?: Messenger) { + messenger.forConnections((connection) => { + messenger.sendClientMessage(connection, async () => ({ channel: 'archive_update', ...update })) + }, except); +} + export const messageHandler: ServerMessageHandler = (messenger, msg, resolver, sender)=> { return resolver(msg, { lazy_read: async (msg) => { @@ -106,7 +155,6 @@ export const messageHandler: ServerMessageHandler = (messenger, msg const node = getNodeFrom(msg); const buffer = await node.getBuffer(); - console.log(buffer); return { buffer: buffer }; @@ -114,100 +162,66 @@ export const messageHandler: ServerMessageHandler = (messenger, msg node_write: async (msg) => { const node = getNodeFrom(msg); await node.setBuffer(async () => msg.buffer, msg.options); - messenger.forActiveMessengersUnless(sender, (connection) => { - messenger.sendClientMessage(connection, async () => { - return { - channel: "archive_update", - type: "updated_node_data", - node: { - root: node.root?.key!, - path: node.path, - } - } - }) - }) + broadcastArchiveUpdate(messenger, { + type: "updated_node_data", + node: { + root: node.root?.key!, + path: node.path, + } + }, sender) return { wasWritten: true }; }, read_zip_tree: async (msg) => { let tree: RootZipTreeNode | undefined = undefined; - let isFullyLoaded = sender != null; if (msg.sha512Hash) tree = getZipNode(msg.sha512Hash); if (tree == null && msg.url) tree = getZipNode(msg.url); if (tree == null) { + const key = uuidV7(); + const readOptions: ReadExtraOptions = { + key: key, + name: msg.name, + logger: messenger.logger, + innerArchiveReadCallback: (_root: RootZipTreeNode, node: ZipNode) => broadcastArchiveUpdate(messenger, { + type: 'node_loaded', + root: { root: key }, + node: copyChildNode(node) + }), + }; + if (msg.data_type == 'buffer') { const buffer = await arrayBufferFromSource(msg.buffer); const hash = await hashBuffer(buffer) - tree = await readZipTreeFromBuffer(messenger.logger, buffer, { - name: msg.name, - logger: messenger.logger, - innerArchiveReadCallback: sender != null ? (root, node) => { - messenger.forActiveMessengersUnless(sender, (connection) => { - messenger.sendClientMessage(connection, async () => { - return { - channel: 'archive_update', - type: 'node_loaded', - root: { root: root.key } as RootNodeTarget, - node: copyChildNode(node) - } - }) - }) - } : undefined - }); + tree = await readZipTreeFromBuffer(messenger.logger, buffer, readOptions); - hashToUUID.set(hash, tree.key) + onArchiveHashed(messenger, tree, hash, msg.sha512Hash); } else { - tree = await readZipTreeFromURL(messenger.logger, msg.url, { - name: msg.name, - logger: messenger.logger, - innerArchiveReadCallback: (root, node) => { - messenger.forActiveMessengersUnless(sender, (connection) => { - messenger.sendClientMessage(connection, async () => { - return { - channel: 'archive_update', - type: 'node_loaded', - root: { root: root.key } as RootNodeTarget, - node: copyChildNode(node) - } - }) - }) - }, + const url = msg.url; + tree = await readZipTreeFromURL(messenger.logger, url, { + ...readOptions, + archiveHashedCallback: (root, hash) => onArchiveHashed(messenger, root, hash, msg.sha512Hash), rangeOptions: { ...msg.rangeOptions, onProgress(current, total) { - messenger.forActiveMessengersUnless(sender, (connection) => { - messenger.sendClientMessage(connection, async () => { - return { - channel: 'archive_update', - type: 'archive_download', - root: { root: tree!.key } as RootNodeTarget, - current: current, - total: total - } - }) + broadcastArchiveUpdate(messenger, { + type: 'archive_download', + root: key, + url: url, + current: current, + total: total }) } }, }); - if (msg.sha512Hash) zipByUUID.set(msg.sha512Hash, tree); + // Trusted until the download finishes and the archive is hashed, unless that already happened + if (msg.sha512Hash && tree.sha512Hash == null) hashToUUID.set(msg.sha512Hash, tree.key); } - tree!.onFinishedLoading.then(() => { - isFullyLoaded = true; - messenger.forActiveMessengersUnless(sender, (connection) => { - messenger.sendClientMessage(connection, async () => { - return { - channel: 'archive_update', - type: 'root_loaded', - root: {root: tree!.key} as RootNodeTarget - } - }) - }) - }) + tree.onFinishedLoading.then(() => broadcastArchiveUpdate(messenger, { type: 'root_loaded', root: { root: key } })) zipByUUID.set(tree.key, tree); @@ -222,14 +236,15 @@ export const messageHandler: ServerMessageHandler = (messenger, msg } } - return { node: copyRoot(tree), isFullyLoaded: isFullyLoaded }; + return { node: copyRoot(tree), isFullyLoaded: (tree as Partial<{ isFullyLoaded: boolean }>).isFullyLoaded ?? true }; }, get_zip_tree: async (msg) => { const node = getZipNode(msg.value); const copiedTree = node ? copyRoot(node) : undefined; return { - node: copiedTree + node: copiedTree, + isFullyLoaded: (node as Partial<{ isFullyLoaded: boolean }> | undefined)?.isFullyLoaded ?? true }; }, collect_files: async (msg) => { @@ -343,21 +358,26 @@ export const messageHandler: ServerMessageHandler = (messenger, msg return { files: files }; }, close_zip_tree: async (msg) => { + closeRoot(zipByUUID.get(msg.root)); zipByUUID.delete(msg.root); const urls = UUIDToUrls.get(msg.root); if (urls) for (const url of urls) urlToUUID.delete(url); UUIDToUrls.delete(msg.root); + for (const [hash, key] of hashToUUID) if (key === msg.root) hashToUUID.delete(hash); + return { err: undefined, wasClosed: true, } }, close_all_zip_trees: async () => { + for (const root of zipByUUID.values()) closeRoot(root); zipByUUID.clear(); urlToUUID.clear(); UUIDToUrls.clear(); + hashToUUID.clear(); return { err: undefined, @@ -368,7 +388,15 @@ export const messageHandler: ServerMessageHandler = (messenger, msg }; export function copyRoot(node: RootZipTreeNode): RootZipTreeNode { - const cloneNode = { children: new Map(), key: node.key } as RootZipTreeNode; + const cloneNode = { + children: new Map(), + key: node.key, + name: node.name, + path: node.path, + type: node.type, + metadata: { directory: true }, + sha512Hash: node.sha512Hash, + } as unknown as RootZipTreeNode; return copyNodeData(cloneNode, node); } diff --git a/packages/ui/src/helpers/toolbox/zip/impl.ts b/packages/ui/src/helpers/toolbox/zip/impl.ts index 0086b499ae4..fedca47e161 100644 --- a/packages/ui/src/helpers/toolbox/zip/impl.ts +++ b/packages/ui/src/helpers/toolbox/zip/impl.ts @@ -34,18 +34,31 @@ import { * Method used to generate the zip tree of the given blob returning the root node of the entire zip file */ export async function readZipTreeFromBuffer(logger: Logger, buffer: ArrayBuffer, options: ReadExtraOptions = {}): Promise { - const onFinishedLoadingHolder = Promise.withResolvers(); - const root = new RootZipTreeNodeImpl(logger, uuidV7(), onFinishedLoadingHolder); - const result = await readZipDataToNode(root, root, zipReaderFromBuffer(buffer), options); - if (options.innerArchiveReadCallback == null) onFinishedLoadingHolder.resolve() + const root = new RootZipTreeNodeImpl(logger, options.key ?? uuidV7(), Promise.withResolvers()); + const result = await readZipDataToNode(root, root, zipReaderFromBuffer(buffer), options); + root.markEntriesRead(); return result; } export async function readZipTreeFromURL(logger: Logger, url: string, options: ReadExtraOptions & {rangeOptions?: RangeCacheReaderOptions} = {}): Promise { - const onFinishedLoadingHolder = Promise.withResolvers(); - const root = new RootZipTreeNodeImpl(logger, uuidV7(), onFinishedLoadingHolder); - const result = await readZipDataToNode(root, root, zipReaderFromURL(url, options.rangeOptions), options); - if (options.innerArchiveReadCallback == null) onFinishedLoadingHolder.resolve() + const root = new RootZipTreeNodeImpl(logger, options.key ?? uuidV7(), Promise.withResolvers()); + const rangeReader = new RangeCacheReader(url, options.rangeOptions); + root.closeCallbacks.push(() => rangeReader.abort()); + + const result = await readZipDataToNode(root, root, new ZipReader(rangeReader), options); + root.markEntriesRead(); + + // Files are fetched on demand ahead of this, but caching the rest makes later reads instant + rangeReader.toBlob() + .then(async (blob) => { + if (root.isClosed) return; + root.sha512Hash = await hashBuffer(await blob.arrayBuffer(), "SHA-512"); + if (!root.isClosed) options.archiveHashedCallback?.(root, root.sha512Hash); + }) + .catch((err) => { + if (!root.isClosed) logger.warn(`Unable to fetch the rest of '${url}' in the background`, { err: err }); + }); + return result; } @@ -66,16 +79,24 @@ export async function writeZipTree(node: ZipTreeNodeImpl): Promise void }; +export type ReadExtraOptions = { + name?: string, + logger?: Logger, + /** Key for the root, for callers that need to refer to it before the read finishes. */ + key?: UUID_V7, + /** + * Called once a nested archive's tree has been read. When given, nested archives are read in + * the background instead of being waited on, with `onFinishedLoading` resolving once all are. + */ + innerArchiveReadCallback?: (root: RootZipTreeNode, node: FileZipTreeNode) => void + /** Called with the SHA-512 of an archive read from a URL, once all of it has been downloaded. */ + archiveHashedCallback?: (root: RootZipTreeNode, sha512Hash: string) => void +}; function zipReaderFromBuffer(buffer: ArrayBuffer): ZipReader> { return new ZipReader(new Uint8ArrayReader(new Uint8Array(buffer))); } -function zipReaderFromURL(url: string, options: RangeCacheReaderOptions = {}): ZipReader { - return new ZipReader(new RangeCacheReader(url, options)); -} - /** * Reads a given buffer and apply its content to the baseNode passed with the primary */ @@ -109,37 +130,36 @@ async function readZipDataToNode(rootNode: RootZipTreeNodeImp } } - if (segmentNode.type === 'directory') { - // Set next parent node to add the given next segment node - parentNode = segmentNode; - } else if(!segmentNode.metadata.encrypted) { - // FYI: This technically will not work when using onprogress as technically the - // given arraybuffer method is not ready yet meaning this is here just for - // legacy or needing to go back to waiting for all entries is better. - const fileEntry = fileOrNull(segmentNode) ?? undefined; - const fileNode = segmentNode.asFile(); - - if (fileNode == null || fileEntry == null) continue; - - const bufferSetPromise = handleFileNode(fileEntry, fileNode); - - if (bufferSetPromise != null) zipFiles.push(bufferSetPromise) - } + // Entry data isn't readable until all entries are, so buffers are set up afterwards + if (segmentNode.type === 'directory') parentNode = segmentNode; } } + /** + * Files are read lazily, which with a ranged reader only fetches their bytes once asked for. + * Nested archives are read right away to build their trees. + */ function handleFileNode(fileEntry: FileEntry, fileNode: FileZipTreeNode): Promise | undefined { - const isZipArchive = zipArchiveExtensionRegex.test(fileNode.name); + if (fileEntry.encrypted || !zipArchiveExtensionRegex.test(fileNode.name)) { + fileNode.setBuffer(() => fileEntry.arrayBuffer()); + return undefined; + } - const bufferSetPromise = fileEntry.arrayBuffer() + const archivePromise = fileEntry.arrayBuffer() .then(async (buffer) => { await fileNode.setBuffer(buffer); - if (isZipArchive) await readZipDataToNode(rootNode, fileNode, zipReaderFromBuffer(buffer), options); - if (innerArchiveReadCallback) innerArchiveReadCallback(rootNode, fileNode); + await readZipDataToNode(rootNode, fileNode, zipReaderFromBuffer(buffer), options); + innerArchiveReadCallback?.(rootNode, fileNode); return fileNode; }) + .catch((err) => { + logger?.warn(`Unable to read inner archive '${fileNode.path}'`, { err: err }); + return fileNode; + }); - return isZipArchive ? bufferSetPromise : undefined; + rootNode.trackInnerArchive(archivePromise); + + return archivePromise; } logger?.debug(`Reading entries ${name} archive.`) @@ -185,8 +205,13 @@ class ZipTreeNodeImpl extends ZipTreeNode { * combined with extra info like maps for all entries and all file entries within the archive */ class RootZipTreeNodeImpl extends RootZipTreeNode { - loadingInnerZipCount: number = -1; + loadingInnerZipCount: number = 0; onFinishedLoadingHolder: PromiseWithResolvers + isFullyLoaded = false; + isClosed = false; + /** Stops work still running for this archive (e.g. background downloads) once it is closed. */ + closeCallbacks: (() => void)[] = []; + private entriesRead = false; constructor(logger: Logger, key: UUID_V7, onFinishedLoadingHolder: PromiseWithResolvers) { super(key, onFinishedLoadingHolder.promise); @@ -194,6 +219,33 @@ class RootZipTreeNodeImpl extends RootZipTreeNode { this.onFinishedLoadingHolder = onFinishedLoadingHolder; } + /** Counts a nested archive (at any depth) as loading until the given read settles. */ + trackInnerArchive(read: Promise) { + this.loadingInnerZipCount += 1; + read.finally(() => { + this.loadingInnerZipCount -= 1; + this.resolveIfLoaded(); + }); + } + + close() { + if (this.isClosed) return; + this.isClosed = true; + for (const callback of this.closeCallbacks) callback(); + } + + /** Marks the top level entries as read, after which the root finishes once no nested archive is loading. */ + markEntriesRead() { + this.entriesRead = true; + this.resolveIfLoaded(); + } + + private resolveIfLoaded() { + if (this.isFullyLoaded || !this.entriesRead || this.loadingInnerZipCount > 0) return; + this.isFullyLoaded = true; + this.onFinishedLoadingHolder.resolve(); + } + protected override createFileNode(root: RootZipTreeNode, parent: ZipNode, name: string, path: string, buffer: ArrayBufferSource | undefined, metadata: FileEntry): FileZipTreeNode { return new FileZipTreeNodeImpl(root as RootZipTreeNodeImpl, parent as ZipNode, name, path, buffer, "file", metadata); } diff --git a/packages/ui/src/helpers/toolbox/zip/index.ts b/packages/ui/src/helpers/toolbox/zip/index.ts index 308024faa17..11b1477fc45 100644 --- a/packages/ui/src/helpers/toolbox/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/zip/index.ts @@ -144,6 +144,9 @@ export class RootZipTreeNode extends ZipTreeNode { onFinishedLoading: Promise + /** SHA-512 of the whole archive, once known (for a URL, only after all of it is downloaded). */ + sha512Hash?: string; + declare type: "directory"; declare root: undefined; @@ -263,7 +266,13 @@ export type ArchiveUpdateTypes = Typed<'node_loaded', { root: RootNodeTarget, node: ZipNode, }> | Typed<'updated_node_data', { node: NodeTarget }> | Typed<'root_loaded', { root: RootNodeTarget }> | - Typed<'archive_download', { root: RootNodeTarget, current: number, total: number }>; + /** + * The archive's own SHA-512 was computed from its downloaded bytes. `expectedHash` is the hash it + * was opened with when that didn't match, which is no longer mapped to the archive. + */ + Typed<'root_hashed', { root: RootNodeTarget, sha512Hash: string, expectedHash?: string }> | + /** Bytes of the archive cached so far, keyed by `url` too as it starts before the root is returned. */ + Typed<'archive_download', { root?: UUID_V7, url?: string, sha512Hash?: string, current: number, total: number }>; export interface ZipHandler { zipByUUID: Map; @@ -274,7 +283,7 @@ export interface ZipHandler { writeNodeData(node: FileZipTreeNode | NodeTarget, buffer: ArrayBuffer, options?: ZipWriterAddDataOptions): Promise; collectFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher): Promise; collectHashedFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher, deduplicate?: boolean): Promise; - getOrReadZipTree(source: ZipDataSource): Promise; + getOrReadZipTree(source: ZipDataSource): Promise; readZipTree(source: ZipDataSource): Promise; getZipTree(value: string | UUID_V7, isUrl: boolean, localOnly?: boolean): Promise; closeZipTree(root: RootZipTreeNode | UUID_V7, name?: string): Promise<{ err?: Error, wasClosed: boolean }>; From 4a164722d287454c9821414557d564d4d6feee98 Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Fri, 16 Oct 2026 22:11:29 -0500 Subject: [PATCH 05/18] fix: add UUID to lock file --- pnpm-lock.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 77d942555ca..1b834319387 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -967,6 +967,9 @@ importers: three: specifier: ^0.172.0 version: 0.172.0 + uuid: + specifier: ^14.0.2 + version: 14.0.3 vue-i18n: specifier: ^10.0.0 version: 10.0.8(vue@3.5.43(typescript@5.9.3)) @@ -9946,6 +9949,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@14.0.3: + resolution: {integrity: sha512-Sc5u1VVln93y1Xnz4R+CunOZc7R9LUcTmhj7Rc7Eg3HVcf7rVn2FOoa812fe0W8p3jD7xeoGSYbfqCGGH2B6xw==} + hasBin: true + validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} @@ -20758,6 +20765,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@14.0.3: {} + validate-npm-package-license@3.0.4: dependencies: spdx-correct: 3.2.0 From 826c208a94a5f408c138133021abfde03a09d52a Mon Sep 17 00:00:00 2001 From: Blodhgarm Date: Fri, 16 Oct 2026 22:49:07 -0500 Subject: [PATCH 06/18] feat: Initial implementation of Archive Audit tool - Resolve issues with WASM loading and running for decompiles - Resolve issues with node lookup being strict and not waiting for root to be fully loaded before attempting to return node - Load archives lazy with the concept that we read Central Directory first to build Archive structure as quick as possible --- apps/frontend/nuxt.config.ts | 4 + .../archive-viewer/ArchiveProjectDetails.vue | 321 +++++++++ apps/frontend/src/locales/en-US/index.json | 111 +++ apps/frontend/src/pages/[type]/[project].vue | 9 + .../src/pages/[type]/[project]/files.vue | 16 + apps/frontend/src/pages/archive.vue | 282 ++++++++ packages/ui/package.json | 2 + .../ui/src/components/floating/Tooltip.vue | 2 - .../ui/src/helpers/toolbox/decompile/impl.ts | 145 ++-- .../helpers/toolbox/decompile/run-slicer.d.ts | 30 + packages/ui/src/helpers/toolbox/file/index.ts | 2 +- .../ui/src/helpers/toolbox/misc/logging.ts | 3 +- .../ui/src/helpers/toolbox/workers/zip/api.ts | 15 +- .../src/helpers/toolbox/workers/zip/index.ts | 29 +- packages/ui/src/helpers/toolbox/zip/index.ts | 43 +- packages/ui/src/layouts/index.ts | 2 + .../components/ArchiveBrowser.vue | 42 ++ .../components/ArchiveWorkbench.vue | 170 +++++ .../archive-viewer/components/HashValue.vue | 54 ++ .../components/panels/ArchiveDetailsPanel.vue | 202 ++++++ .../components/panels/DetailsList.vue | 39 ++ .../components/panels/FileDetailsPanel.vue | 191 ++++++ .../components/panels/LogsPanel.vue | 132 ++++ .../composables/archive-file-manager.ts | 413 +++++++++++ .../composables/archive-loader.ts | 122 ++++ .../archive-viewer/composables/node-values.ts | 44 ++ .../composables/toolbox-logs.ts | 51 ++ .../layouts/shared/archive-viewer/index.ts | 4 + .../layouts/shared/archive-viewer/layout.vue | 158 +++++ .../providers/archive-viewer.ts | 26 + .../layouts/shared/archive-viewer/types.ts | 33 + .../files-tab/components/FileActionBar.vue | 25 +- .../files-tab/components/FileBrowserPanel.vue | 160 +---- .../components/FileEmptyWorkspace.vue | 26 +- .../files-tab/components/FileListing.vue | 182 +++++ .../files-tab/components/FileNavbar.vue | 39 +- .../shared/files-tab/components/FileRow.vue | 3 +- .../files-tab/components/FileSearch.vue | 3 +- .../files-tab/components/FileSideBar.vue | 33 +- .../shared/files-tab/components/FileTree.vue | 30 +- .../files-tab/components/FileWorkbench.vue | 69 ++ .../components/editor/FileEditor.vue | 4 +- .../files-tab/components/tabs/FileTab.vue | 41 +- .../components/tabs/FileTabPanel.vue | 83 ++- .../files-tab/components/tabs/FileTabs.vue | 32 +- .../components/tabs/FileTabsActions.vue | 18 +- .../files-tab/composables/directory-view.ts | 118 ++++ .../files-tab/composables/entry-menu.ts | 21 +- .../files-tab/composables/file-selection.ts | 63 +- .../shared/files-tab/composables/file-tabs.ts | 238 +++++-- .../ui/src/layouts/shared/files-tab/index.ts | 1 + .../src/layouts/shared/files-tab/layout.vue | 147 ++-- .../files-tab/providers/file-browser-ui.ts | 35 +- .../files-tab/providers/file-manager.ts | 9 + .../ui/src/layouts/shared/files-tab/types.ts | 5 - .../ui/src/layouts/shared/files-tab/utils.ts | 7 +- .../workbench/components/WorkbenchDock.vue | 75 ++ .../workbench/components/WorkbenchPanel.vue | 24 + .../workbench/components/WorkbenchTab.vue | 70 ++ .../workbench/components/WorkbenchToggles.vue | 110 +++ .../workbench/composables/workbench-layout.ts | 639 ++++++++++++++++++ .../workbench/composables/workbench-menus.ts | 155 +++++ .../ui/src/layouts/shared/workbench/index.ts | 3 + .../src/layouts/shared/workbench/layout.vue | 171 +++++ .../shared/workbench/providers/workbench.ts | 59 ++ .../ui/src/layouts/shared/workbench/types.ts | 63 ++ packages/ui/src/locales/en-US/index.json | 267 ++++++++ packages/ui/src/styles/dockview-theme.css | 28 + pnpm-lock.yaml | 12 + 69 files changed, 5313 insertions(+), 452 deletions(-) create mode 100644 apps/frontend/src/components/ui/archive-viewer/ArchiveProjectDetails.vue create mode 100644 apps/frontend/src/pages/[type]/[project]/files.vue create mode 100644 apps/frontend/src/pages/archive.vue create mode 100644 packages/ui/src/helpers/toolbox/decompile/run-slicer.d.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/ArchiveBrowser.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/ArchiveWorkbench.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/HashValue.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/panels/ArchiveDetailsPanel.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/panels/DetailsList.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/panels/FileDetailsPanel.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/components/panels/LogsPanel.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/composables/archive-file-manager.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/composables/archive-loader.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/composables/node-values.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/composables/toolbox-logs.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/index.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/layout.vue create mode 100644 packages/ui/src/layouts/shared/archive-viewer/providers/archive-viewer.ts create mode 100644 packages/ui/src/layouts/shared/archive-viewer/types.ts create mode 100644 packages/ui/src/layouts/shared/files-tab/components/FileListing.vue create mode 100644 packages/ui/src/layouts/shared/files-tab/components/FileWorkbench.vue create mode 100644 packages/ui/src/layouts/shared/files-tab/composables/directory-view.ts create mode 100644 packages/ui/src/layouts/shared/workbench/components/WorkbenchDock.vue create mode 100644 packages/ui/src/layouts/shared/workbench/components/WorkbenchPanel.vue create mode 100644 packages/ui/src/layouts/shared/workbench/components/WorkbenchTab.vue create mode 100644 packages/ui/src/layouts/shared/workbench/components/WorkbenchToggles.vue create mode 100644 packages/ui/src/layouts/shared/workbench/composables/workbench-layout.ts create mode 100644 packages/ui/src/layouts/shared/workbench/composables/workbench-menus.ts create mode 100644 packages/ui/src/layouts/shared/workbench/index.ts create mode 100644 packages/ui/src/layouts/shared/workbench/layout.vue create mode 100644 packages/ui/src/layouts/shared/workbench/providers/workbench.ts create mode 100644 packages/ui/src/layouts/shared/workbench/types.ts create mode 100644 packages/ui/src/styles/dockview-theme.css diff --git a/apps/frontend/nuxt.config.ts b/apps/frontend/nuxt.config.ts index d8fff056805..aaabd24299e 100644 --- a/apps/frontend/nuxt.config.ts +++ b/apps/frontend/nuxt.config.ts @@ -110,6 +110,10 @@ export default defineNuxtConfig({ external: ['cloudflare:workers'], }, }, + worker: { + // The toolbox workers (archives, decompilation) lazily import their dependencies + format: 'es', + }, }, hooks: { async 'build:before'() { diff --git a/apps/frontend/src/components/ui/archive-viewer/ArchiveProjectDetails.vue b/apps/frontend/src/components/ui/archive-viewer/ArchiveProjectDetails.vue new file mode 100644 index 00000000000..649ec80d0e4 --- /dev/null +++ b/apps/frontend/src/components/ui/archive-viewer/ArchiveProjectDetails.vue @@ -0,0 +1,321 @@ + + + diff --git a/apps/frontend/src/locales/en-US/index.json b/apps/frontend/src/locales/en-US/index.json index c41acab6d6d..78ee0bf1e9c 100644 --- a/apps/frontend/src/locales/en-US/index.json +++ b/apps/frontend/src/locales/en-US/index.json @@ -731,6 +731,114 @@ "app-marketing.show-other-packages": { "message": "Show other packages" }, + "archive.go-back": { + "message": "Go back" + }, + "archive.load-failed": { + "message": "Unable to load the project" + }, + "archive.load-failed-description": { + "message": "The project or its versions could not be fetched." + }, + "archive.loading-versions": { + "message": "Loading versions..." + }, + "archive.nothing-to-show": { + "message": "No archive to show" + }, + "archive.nothing-to-show-description": { + "message": "Open a project with version files, or a file link from the Modrinth CDN." + }, + "archive.open-in-slicer": { + "message": "Open in Slicer" + }, + "archive.project-details.channel.alpha": { + "message": "Alpha" + }, + "archive.project-details.channel.beta": { + "message": "Beta" + }, + "archive.project-details.channel.release": { + "message": "Release" + }, + "archive.project-details.downloads": { + "message": "Downloads" + }, + "archive.project-details.file": { + "message": "File" + }, + "archive.project-details.file-type": { + "message": "File type" + }, + "archive.project-details.hash-check": { + "message": "Download matches" + }, + "archive.project-details.hash-match": { + "message": "Matches listed hash" + }, + "archive.project-details.hash-mismatch": { + "message": "Does not match listed hash" + }, + "archive.project-details.hash-pending": { + "message": "Checked once fully downloaded" + }, + "archive.project-details.id": { + "message": "ID" + }, + "archive.project-details.more-game-versions": { + "message": "+{count} more" + }, + "archive.project-details.name": { + "message": "Name" + }, + "archive.project-details.open-project": { + "message": "Project page" + }, + "archive.project-details.primary": { + "message": "Primary" + }, + "archive.project-details.published": { + "message": "Published" + }, + "archive.project-details.sha512": { + "message": "Listed SHA-512" + }, + "archive.project-details.size": { + "message": "Size" + }, + "archive.project-details.tech-review": { + "message": "Tech review" + }, + "archive.project-details.version": { + "message": "Version" + }, + "archive.project-details.version-status.archived": { + "message": "Archived" + }, + "archive.project-details.version-status.draft": { + "message": "Draft" + }, + "archive.project-details.version-status.listed": { + "message": "Listed" + }, + "archive.project-details.version-status.scheduled": { + "message": "Scheduled" + }, + "archive.project-details.version-status.unknown": { + "message": "Unknown" + }, + "archive.project-details.version-status.unlisted": { + "message": "Unlisted" + }, + "archive.search-versions": { + "message": "Search versions..." + }, + "archive.select-version": { + "message": "Select a version" + }, + "archive.title": { + "message": "Archive viewer" + }, "auth.authorize.action.authorize": { "message": "Authorize" }, @@ -4109,6 +4217,9 @@ "project.error.project-not-found": { "message": "Project not found" }, + "project.files.title": { + "message": "Files" + }, "project.gallery.title": { "message": "Gallery" }, diff --git a/apps/frontend/src/pages/[type]/[project].vue b/apps/frontend/src/pages/[type]/[project].vue index 5c358eb82d7..54307e59e52 100644 --- a/apps/frontend/src/pages/[type]/[project].vue +++ b/apps/frontend/src/pages/[type]/[project].vue @@ -813,6 +813,10 @@ const messages = defineMessages({ id: 'project.moderation.title', defaultMessage: 'Moderation', }, + filesTab: { + id: 'project.files.title', + defaultMessage: 'Files', + }, noCollectionsFound: { id: 'project.collections.none-found', defaultMessage: 'No collections found.', @@ -2490,6 +2494,11 @@ const navLinks = computed(() => { href: withInstallContextQuery(`${projectUrl}/moderation`), shown: !!currentMember.value, }, + { + label: formatMessage(messages.filesTab), + href: `/archive?project=${project.value.id}`, + shown: isStaff(auth.value.user) && hasVersions.value, + }, ] }) diff --git a/apps/frontend/src/pages/[type]/[project]/files.vue b/apps/frontend/src/pages/[type]/[project]/files.vue new file mode 100644 index 00000000000..49477f7fd5f --- /dev/null +++ b/apps/frontend/src/pages/[type]/[project]/files.vue @@ -0,0 +1,16 @@ + + + diff --git a/apps/frontend/src/pages/archive.vue b/apps/frontend/src/pages/archive.vue new file mode 100644 index 00000000000..6b93baa8ece --- /dev/null +++ b/apps/frontend/src/pages/archive.vue @@ -0,0 +1,282 @@ + + + diff --git a/packages/ui/package.json b/packages/ui/package.json index 688664a67e8..0ef49285d04 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -42,6 +42,7 @@ "@storybook/vue3-vite": "^10.1.10", "@stripe/stripe-js": "^7.3.1", "@tailwindcss/vite": "^4.1.18", + "@types/js-yaml": "^4.0.9", "@vitejs/plugin-vue": "^5.2.1", "eslint-plugin-storybook": "^10.1.10", "storybook": "^10.1.10", @@ -107,6 +108,7 @@ "overlayscrollbars": "^2.15.1", "postprocessing": "^6.37.6", "qrcode.vue": "^3.4.1", + "smol-toml": "^1.9.0", "three": "^0.172.0", "uuid": "^14.0.2", "vue-i18n": "^10.0.0", diff --git a/packages/ui/src/components/floating/Tooltip.vue b/packages/ui/src/components/floating/Tooltip.vue index fffe07c5bf6..8ea7f916973 100644 --- a/packages/ui/src/components/floating/Tooltip.vue +++ b/packages/ui/src/components/floating/Tooltip.vue @@ -228,9 +228,7 @@ function onEnter() { } function onTooltipEnter() { - console.log('WWWWWWWWWWWWWWWWWWWW') if ((props.hoverable || props.pinned) && props.reference) { - console.log('WEEEEEEEE') preventHide(props.reference) } } diff --git a/packages/ui/src/helpers/toolbox/decompile/impl.ts b/packages/ui/src/helpers/toolbox/decompile/impl.ts index 7e376f7c860..92a5cb659eb 100644 --- a/packages/ui/src/helpers/toolbox/decompile/impl.ts +++ b/packages/ui/src/helpers/toolbox/decompile/impl.ts @@ -25,45 +25,104 @@ export class DecompilerResultImpl implements DecompilerResult { } } -async function getDecompiler(logger: Logger, type: DecompilerTypes): Promise { - let decompiler: Decompiler | undefined = undefined; - let possibleErr: any; +type DecompileFunction = Decompiler["decompile"]; + +/** + * Loads a run-slicer decompiler's WASM build. Their own entry points take any defined `process` + * (which Nuxt shims, even in workers) to mean Node and then fail to locate the module, so it's + * fetched here and handed over as bytes instead. Its URL is imported only when a decompiler is + * first loaded (on the client), so the server build never deals with the module. + */ +async function loadWasmDecompiler( + runtime: () => Promise, + wasmUrlModule: () => Promise<{ default: string }>, + options?: { noAutoImports?: boolean, takesArray?: boolean }, +): Promise { + const [{ load }, bytes] = await Promise.all([ + runtime(), + wasmUrlModule().then(({ default: wasmUrl }) => fetch(wasmUrl)).then((response) => { + if (!response.ok) throw new Error(`Unable to fetch '${response.url}' (${response.status})`); + return response.arrayBuffer(); + }), + ]); + const { exports } = await load(bytes, { noAutoImports: options?.noAutoImports }); + return options?.takesArray + ? (name, config) => exports.decompile([name], config) + : (name, config) => exports.decompile(name, config); +} + +/** Falls back to the decompiler's JavaScript build when its WASM one can't be used. */ +async function withJsFallback(logger: Logger, type: DecompilerTypes, wasm: () => Promise, js: () => Promise) { try { - if (type == "cfr") { - decompiler = await import("@run-slicer/cfr"); - } else if (type == "procyon") { - decompiler = await import("@run-slicer/procyon"); - } /*else if (type == 'krakatau') { - const { KrakClient } = await import('krakatau-web'); - const client = new KrakClient(); - client.decompile() - decompiler = { - decompile(names: string, config: { - source?: (name: string) => Promise; - options?: Record; - resources?: string[] - } | undefined): Promise> { - - return Promise.resolve(undefined); - } - } - }*/ - } catch (err: any) { - possibleErr = err; + return await wasm(); + } catch (err) { + logger.warn(`Unable to load the WASM build of '${type}', falling back to its JavaScript build`, { err }); + return js(); } +} + +const DECOMPILER_LOADERS: Partial Promise>> = { + cfr: (logger) => withJsFallback( + logger, + "cfr", + () => loadWasmDecompiler( + () => import("@run-slicer/cfr/cfr.wasm-runtime.js"), + () => import("@run-slicer/cfr/cfr.wasm?url"), + { takesArray: true }, + ), + async () => { + const { decompile } = await import("@run-slicer/cfr/cfr.runtime.js"); + return (name, config) => decompile([name], config); + }, + ), + vineflower: (logger) => withJsFallback( + logger, + "vineflower", + () => loadWasmDecompiler( + () => import("@run-slicer/vf/vf.wasm-runtime.js"), + () => import("@run-slicer/vf/vf.wasm?url"), + { takesArray: true }, + ), + async () => { + const { decompile } = await import("@run-slicer/vf/vf.runtime.js"); + return (name, config) => decompile([name], config); + }, + ), + procyon: () => loadWasmDecompiler( + () => import("@run-slicer/procyon/procyon.wasm-runtime.js"), + () => import("@run-slicer/procyon/procyon.wasm?url"), + { noAutoImports: true }, + ), +}; + +const loadedDecompilers = new Map>(); + +function loadDecompiler(logger: Logger, type: DecompilerTypes) { + let loaded = loadedDecompilers.get(type); + if (loaded == null) { + const loader = DECOMPILER_LOADERS[type]; + if (loader == null) return Promise.reject(new Error(`The '${type}' decompiler is not available`)); + loaded = loader(logger); + loaded.catch(() => loadedDecompilers.delete(type)); + loadedDecompilers.set(type, loaded); + } + return loaded; +} +/** The requested decompiler, or Vineflower when that one can't be loaded. */ +async function getDecompiler(logger: Logger, type: DecompilerTypes): Promise { try { - if (decompiler == undefined) { - if (type == "vineflower") logger.warn(`Unable to get desired decompiler '${type}' due to an error: `, { err: possibleErr }) - decompiler = await import("@run-slicer/vf"); - } + return { decompile: await loadDecompiler(logger, type) }; } catch (err) { - const outerErr = new Error(`Unable to get vineflower decompiler due to an error'`); - outerErr.cause = err; - throw outerErr + if (type === "vineflower") throw new Error(`Unable to get vineflower decompiler due to an error`, { cause: err }); + logger.warn(`Unable to get desired decompiler '${type}' due to an error, using vineflower instead`, { err }); } - return decompiler; + try { + return { decompile: await loadDecompiler(logger, "vineflower") }; + } catch (err) { + throw new Error(`Unable to get vineflower decompiler due to an error`, { cause: err }); + } } export async function decompileFileContent(logger: Logger, zipHandler: ZipHandler | undefined, {targetFile, resources, options}: DecompileInput): Promise { @@ -76,18 +135,24 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle try { // const pathParts = targetFile.fullPath.split('.jar'); \n const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); \n - const pathParts = targetFile.fullPath.split('.jar/'); + const pathParts = targetFile.fullPath.split('.jar'); const withoutClassSuffix = pathParts[pathParts.length - 1].slice(0, -6); + const unnormalizedPath = withoutClassSuffix.slice(1); + const buffer = targetFile.content; const decompiled = await (await getDecompiler(logger, options.decompiler!)) - .decompile(withoutClassSuffix, { + .decompile(unnormalizedPath, { source: async (name) => { - if (name === withoutClassSuffix) { + if (name === unnormalizedPath) { return new Uint8Array(buffer); - } else if (rootKey != null && name.includes(withoutClassSuffix)) { + } else if (rootKey != null && zipHandler != null && name.includes(unnormalizedPath)) { // TODO: THIS DOES NOT PROPERLY HANDLE WHICH JAR WE NEED TO LOOK AT SO IF DUPLICATE FILES EXIST, WE MAY GET THE WRONG ONE!!!! - const buffer = await zipHandler?.readNodeData({ path: "/" + name + ".class", root: rootKey, relative: true }); - if (buffer) return new Uint8Array(buffer); + try { + const buffer = await zipHandler.findNodeData({ path: "/" + name + ".class", root: rootKey, relative: true }); + if (buffer) return new Uint8Array(buffer); + } catch (err) { + logger.warn(`Unable to read '${name}' for decompiling '${targetFile.fullPath}'`, { err }); + } } return null; @@ -98,12 +163,10 @@ export async function decompileFileContent(logger: Logger, zipHandler: ZipHandle options: options.options }); - const unnormalizedPath = withoutClassSuffix.slice(1); - if (typeof decompiled === 'string') { source = decompiled ?? undefined; } else if (!decompiled[unnormalizedPath]) { - console.error("Decompilation failed for", targetFile.fullPath, decompiled); + console.error("Decompilation failed for: ", targetFile.fullPath, unnormalizedPath, decompiled); error = "// [Decompilation output either is kotlin or their is issues just a fyi!] \n\n"; // for (const [key, value] of Object.entries(decompiled)) { \n error = `${error} \n\n Class: ${key} \n\n ${value}` \n } \n diff --git a/packages/ui/src/helpers/toolbox/decompile/run-slicer.d.ts b/packages/ui/src/helpers/toolbox/decompile/run-slicer.d.ts new file mode 100644 index 00000000000..b05975dbed8 --- /dev/null +++ b/packages/ui/src/helpers/toolbox/decompile/run-slicer.d.ts @@ -0,0 +1,30 @@ +/** + * The WASM runtimes of the run-slicer decompilers, which ship without types. `load` takes the + * module's URL or its bytes. + */ +interface RunSlicerWasmRuntime { + load( + module: string | BufferSource, + options?: { noAutoImports?: boolean }, + ): Promise<{ exports: { decompile: (...args: any[]) => Promise } }> +} + +declare module '@run-slicer/cfr/cfr.wasm-runtime.js' { + export const load: RunSlicerWasmRuntime['load'] +} + +declare module '@run-slicer/vf/vf.wasm-runtime.js' { + export const load: RunSlicerWasmRuntime['load'] +} + +declare module '@run-slicer/procyon/procyon.wasm-runtime.js' { + export const load: RunSlicerWasmRuntime['load'] +} + +declare module '@run-slicer/cfr/cfr.runtime.js' { + export const decompile: (names: string[], config?: object) => Promise> +} + +declare module '@run-slicer/vf/vf.runtime.js' { + export const decompile: (names: string[], config?: object) => Promise> +} diff --git a/packages/ui/src/helpers/toolbox/file/index.ts b/packages/ui/src/helpers/toolbox/file/index.ts index 01519c10308..4bf1b8421f9 100644 --- a/packages/ui/src/helpers/toolbox/file/index.ts +++ b/packages/ui/src/helpers/toolbox/file/index.ts @@ -1,7 +1,7 @@ import { fileTypeFromBuffer } from "file-type"; import yaml from "js-yaml" import mime from "mime"; -import toml from 'smol-toml' +import * as toml from 'smol-toml' export interface FileDetails { fullPath: string, diff --git a/packages/ui/src/helpers/toolbox/misc/logging.ts b/packages/ui/src/helpers/toolbox/misc/logging.ts index edd672253e7..b977b00c100 100644 --- a/packages/ui/src/helpers/toolbox/misc/logging.ts +++ b/packages/ui/src/helpers/toolbox/misc/logging.ts @@ -271,7 +271,8 @@ export function logFactory({log, formatter, counter}: FactoryOptions = {}): Logg }, factory); const finalCallback = (log ?? defaultCallback).bind(factory); - factory.log = (level, loggerName, message, extra) => { + factory.onLog = onLogEvent.source; + factory.log =(level, loggerName, message, extra) => { finalCallback(level, loggerName, message, extra); // @ts-expect-error The 'this' context of type 'void' is not assignable to method's 'this' of type 'LoggerFactory'. onLogEvent.sink(level, loggerName, message, extra); diff --git a/packages/ui/src/helpers/toolbox/workers/zip/api.ts b/packages/ui/src/helpers/toolbox/workers/zip/api.ts index 7fd073a690a..664f88f872a 100644 --- a/packages/ui/src/helpers/toolbox/workers/zip/api.ts +++ b/packages/ui/src/helpers/toolbox/workers/zip/api.ts @@ -29,8 +29,8 @@ export type ZipMessages = MessageTypes< clientbound: DecompilerResult } node_read: { - serverbound: NodeTarget, - clientbound: { buffer: ArrayBuffer } + serverbound: NodeTarget & { optional?: boolean }, + clientbound: { buffer: ArrayBuffer | undefined } }, node_write: { serverbound: NodeTarget & { options?: ZipWriterAddDataOptions } & { buffer: ArrayBuffer }, @@ -127,8 +127,13 @@ export const handlerConstructor: MessengerHandlerConstructor { - return (await messenger.sendMessage({ channel: "node_read", ...getAsTarget(target)})).buffer - }, + const { buffer } = await messenger.sendMessage({ channel: "node_read", ...getAsTarget(target) }); + if (buffer == null) throw new Error(`No data was read for '${getAsTarget(target).path}'`); + return buffer; + }, + findNodeData: async (target: NodeTarget) => { + return (await messenger.sendMessage({ channel: "node_read", ...target, optional: true })).buffer + }, getOrReadZipTree: async function (source: ZipDataSource) { const { url, sha512Hash } = source; let node: RootZipTreeNode | undefined; @@ -249,7 +254,7 @@ export const handlerConstructor: MessengerHandlerConstructor(); const urlToUUID = new Map(); const UUIDToUrls = new Map(); -function getNodeFrom(target: NodeTarget) { +/** + * Finds the targeted file. Nested archives are read in the background, so a file not known yet is + * looked for again once the archive finishes loading, before it's deemed missing. + */ +async function findNodeFrom(target: NodeTarget) { if (!zipByUUID || zipByUUID.size == 0) throw new Error(`Unable to locate node as no known root tree is within the worker!`); const root = zipByUUID.get(target.root); if (root == undefined) throw new Error(`Unable to get node '${target.root}:${target.path}' as their is no active root with the hash given namespace hash!`); - const node = (target.relative ? root.relativeFileEntries : root.fileEntries).get(target.path); + const node = root.findFileEntry(target.path, target.relative); + if (node != undefined || (root as Partial<{ isFullyLoaded: boolean }>).isFullyLoaded !== false) return node; + + await root.onFinishedLoading; + return root.findFileEntry(target.path, target.relative); +} + +async function getNodeFrom(target: NodeTarget) { + const node = await findNodeFrom(target); if (node == undefined) throw new Error(`Unable to get node '${target.root}:${target.path}' as their is no active node with the given path!`); return node; @@ -141,26 +153,25 @@ function broadcastArchiveUpdate(messenger: ServerMessenger, update: export const messageHandler: ServerMessageHandler = (messenger, msg, resolver, sender)=> { return resolver(msg, { lazy_read: async (msg) => { - const node = getNodeFrom(msg); + const node = await getNodeFrom(msg); return { key: msg.key, value: await (node as LazyDataAccess)[msg.key].get() }; }, - source_read: (msg) => { - return getNodeFrom(msg).asText(msg.options); + source_read: async (msg) => { + return (await getNodeFrom(msg)).asText(msg.options); }, node_read: async (msg) => { - const node = getNodeFrom(msg); + const node = msg.optional ? await findNodeFrom(msg) : await getNodeFrom(msg); - const buffer = await node.getBuffer(); return { - buffer: buffer + buffer: node ? await node.getBuffer() : undefined }; }, node_write: async (msg) => { - const node = getNodeFrom(msg); + const node = await getNodeFrom(msg); await node.setBuffer(async () => msg.buffer, msg.options); broadcastArchiveUpdate(messenger, { type: "updated_node_data", diff --git a/packages/ui/src/helpers/toolbox/zip/index.ts b/packages/ui/src/helpers/toolbox/zip/index.ts index 11b1477fc45..e736e11a0d1 100644 --- a/packages/ui/src/helpers/toolbox/zip/index.ts +++ b/packages/ui/src/helpers/toolbox/zip/index.ts @@ -127,6 +127,14 @@ export class ZipTreeNode extends TreeNode { asFile(): FileZipTreeNode | undefined{ return this.type == "file" ? this as unknown as FileZipTreeNode : undefined; } + + /** The archive this node is stored in: the root, or the nested archive (e.g. jar-in-jar) holding it. */ + getArchive(): RootZipTreeNode | FileZipTreeNode { + for (let parent = this.parent; parent != null; parent = parent.parent) { + if (isZipArchive(parent)) return parent as FileZipTreeNode; + } + return this.root ?? this as unknown as RootZipTreeNode; + } } export type UUID_V7 = string; @@ -167,14 +175,29 @@ export class RootZipTreeNode extends ZipTreeNode { if (fileNode) { this.fileEntries.set(node.path, fileNode); - - const parts = node.path.split(".jar/"); - this.relativeFileEntries.set((parts.length > 1) ? parts[parts.length - 1] : node.path, fileNode) + this.relativeFileEntries.set(RootZipTreeNode.relativePathOf(node.path), fileNode) } return node; } + /** + * The path of a file within its innermost archive, with a leading slash, e.g. `/a/B.class` for + * both `/a/B.class` and `/META-INF/jars/c.jar/a/B.class`. + */ + static relativePathOf(path: string) { + const parts = path.split(".jar/"); + const relative = parts[parts.length - 1]; + return relative.startsWith("/") ? relative : "/" + relative; + } + + /** Finds a file by its full path, or by its path within its innermost archive when `relative`. */ + findFileEntry(path: string, relative?: boolean) { + return relative + ? this.relativeFileEntries.get(RootZipTreeNode.relativePathOf(path)) + : this.fileEntries.get(path); + } + /** * @internal Only used to remove a node from the given reference maps */ @@ -182,8 +205,8 @@ export class RootZipTreeNode extends ZipTreeNode { this.entries.delete(node.path) this.fileEntries.delete(node.path) - const parts = node.path.split(".jar/"); - if (parts.length > 1) this.relativeFileEntries.delete(parts[parts.length - 1]) + const relativePath = RootZipTreeNode.relativePathOf(node.path); + if (this.relativeFileEntries.get(relativePath) === node) this.relativeFileEntries.delete(relativePath) return node; } @@ -280,6 +303,11 @@ export interface ZipHandler { readLazyData(node: FileZipTreeNode | NodeTarget, key: K): Promise>>; readSource(node: FileZipTreeNode | NodeTarget, options?: SourceOptions): Promise; readNodeData(node: FileZipTreeNode | NodeTarget): Promise; + /** + * Reads the node's data, waiting for the archive to finish loading when the node isn't known + * yet, and resolves to `undefined` when the archive turns out not to hold it. + */ + findNodeData(node: NodeTarget): Promise; writeNodeData(node: FileZipTreeNode | NodeTarget, buffer: ArrayBuffer, options?: ZipWriterAddDataOptions): Promise; collectFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher): Promise; collectHashedFiles(root: RootZipTreeNode | UUID_V7, matcher?: FileMatcher, deduplicate?: boolean): Promise; @@ -408,8 +436,10 @@ export class DelegateFileZipTreeNode extends ZipTreeNode implements FileZipTreeN } export function reconstituteRoot(handler: ZipHandler, root: RootZipTreeNode): RootZipTreeNode { + Object.setPrototypeOf(root, RootZipTreeNode.prototype); root.entries = new Map(); root.fileEntries = new Map(); + root.relativeFileEntries = new Map(); return reconstituteNodes(handler, root, root, root) } @@ -422,13 +452,12 @@ export function reconstituteNodes(handler: ZipHandler, root: reconstitutedNode = Object.setPrototypeOf(node, ZipTreeNode.prototype); } else { reconstitutedNode = Object.setPrototypeOf(node, DelegateFileZipTreeNode.prototype); - root.fileEntries.set(reconstitutedNode.path, reconstitutedNode as FileZipTreeNode); } node.children.forEach((entry) => reconstituteNodes(handler, root, node, entry)) if (!(reconstitutedNode instanceof RootZipTreeNode)) { - root.entries.set(reconstitutedNode.path, reconstitutedNode); + root.addNodeToRoot(reconstitutedNode); reconstitutedNode.root = root; reconstitutedNode.parent = parent; diff --git a/packages/ui/src/layouts/index.ts b/packages/ui/src/layouts/index.ts index e2cc22785ee..2a69dd3dee0 100644 --- a/packages/ui/src/layouts/index.ts +++ b/packages/ui/src/layouts/index.ts @@ -1,4 +1,5 @@ export * from './shared/appearance-settings' +export * from './shared/archive-viewer' export * from './shared/browse-tab' export * from './shared/console' export * from './shared/content-tab' @@ -6,4 +7,5 @@ export * from './shared/files-tab' export * from './shared/installation-settings' export * from './shared/server-settings' export * from './shared/user-profile' +export * from './shared/workbench' export * from './wrapped' diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveBrowser.vue b/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveBrowser.vue new file mode 100644 index 00000000000..900eb06b383 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveBrowser.vue @@ -0,0 +1,42 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveWorkbench.vue b/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveWorkbench.vue new file mode 100644 index 00000000000..bcc6f36c7c0 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/ArchiveWorkbench.vue @@ -0,0 +1,170 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/HashValue.vue b/packages/ui/src/layouts/shared/archive-viewer/components/HashValue.vue new file mode 100644 index 00000000000..e22cbf375b3 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/HashValue.vue @@ -0,0 +1,54 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/panels/ArchiveDetailsPanel.vue b/packages/ui/src/layouts/shared/archive-viewer/components/panels/ArchiveDetailsPanel.vue new file mode 100644 index 00000000000..9d56c2e08bb --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/panels/ArchiveDetailsPanel.vue @@ -0,0 +1,202 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/panels/DetailsList.vue b/packages/ui/src/layouts/shared/archive-viewer/components/panels/DetailsList.vue new file mode 100644 index 00000000000..694afaa7f37 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/panels/DetailsList.vue @@ -0,0 +1,39 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/panels/FileDetailsPanel.vue b/packages/ui/src/layouts/shared/archive-viewer/components/panels/FileDetailsPanel.vue new file mode 100644 index 00000000000..f1d925485ec --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/panels/FileDetailsPanel.vue @@ -0,0 +1,191 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/panels/LogsPanel.vue b/packages/ui/src/layouts/shared/archive-viewer/components/panels/LogsPanel.vue new file mode 100644 index 00000000000..f318bce1c1a --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/components/panels/LogsPanel.vue @@ -0,0 +1,132 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/composables/archive-file-manager.ts b/packages/ui/src/layouts/shared/archive-viewer/composables/archive-file-manager.ts new file mode 100644 index 00000000000..fef7bdb84cf --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/composables/archive-file-manager.ts @@ -0,0 +1,413 @@ +import { ClipboardCopyIcon, DownloadIcon } from '@modrinth/assets' +import { computed, markRaw, onScopeDispose, type Ref, ref, shallowRef } from 'vue' + +import { useFormatBytes } from '#ui/composables/format-bytes' +import { defineMessages, useVIntl } from '#ui/composables/i18n' +import { + type FileZipTreeNode, isZipArchive, + type LazyDataAccess, + type RootZipTreeNode, + type ZipHandler, type ZipNode, +} from '#ui/helpers/toolbox/zip' +import type { + DirectoryResult, + DirectoryTree, + FileInfo, + FileItemResultFrom, + FileManagerContext, + FileResult, + FileTypes, +} from '#ui/layouts/shared/files-tab/providers/file-manager' +import type { + FileEntryAction, + FileEntryDetail, + FileItem, +} from '#ui/layouts/shared/files-tab/types' +import { + infoFrom, + isSameInfo, + normalizeDirectoryPath, + parentInfoFrom, +} from '#ui/layouts/shared/files-tab/utils' +import { injectNotificationManager } from '#ui/providers/web-notifications' +import { canOpenInFileEditor, getFileExtension } from '#ui/utils/file-extensions' + +import type { ArchiveSource } from '../types' + +const messages = defineMessages({ + readOnly: { + id: 'archive-viewer.read-only', + defaultMessage: 'Archives are read-only', + }, + missingEntry: { + id: 'archive-viewer.missing-entry', + defaultMessage: '"{path}" does not exist within this archive', + }, + detectedType: { + id: 'archive-viewer.detail.detected-type', + defaultMessage: 'Detected type', + }, + compressedSize: { + id: 'archive-viewer.detail.compressed-size', + defaultMessage: 'Compressed size', + }, + sha1: { + id: 'archive-viewer.detail.sha1', + defaultMessage: 'SHA-1', + }, + downloadArchive: { + id: 'archive-viewer.action.download-archive', + defaultMessage: 'Download archive', + }, + copySha512: { + id: 'archive-viewer.action.copy-sha512', + defaultMessage: 'Copy SHA-512', + }, + copiedSha512: { + id: 'archive-viewer.action.copied-sha512', + defaultMessage: 'Copied SHA-512', + }, +}) + +export interface ArchiveFileManagerOptions { + root: RootZipTreeNode + handler: ZipHandler + source: ArchiveSource + /** Path within the archive to open first, e.g. `com/example/Mod.class`. */ + initialPath?: string | null +} + +interface DirectoryState { + result: DirectoryResult + isLoading: Ref + loadError: Ref +} + +function isClassFile(name: string) { + return getFileExtension(name) === 'class' +} + +/** Nested archives (e.g. jar-in-jar) are file nodes with children, which are browsed as folders. */ +function isArchive(node: ZipNode) { + return node.type === 'file' && isZipArchive(node) +} + +function toSeconds(date: Date | undefined) { + return date ? Math.floor(new Date(date).getTime() / 1000) : 0 +} + +function toError(error: unknown) { + return error instanceof Error ? error : new Error(String(error)) +} + +/** + * A read-only {@link FileManagerContext} over an archive read by the zip worker, for the files + * layout to browse. The tree grows while nested archives are read in the background, class files + * open decompiled, and file contents are only ever read off the main thread. + */ +export function useArchiveFileManager({ + root, + handler, + source, + initialPath, +}: ArchiveFileManagerOptions) { + const { formatMessage } = useVIntl() + const formatBytes = useFormatBytes() + const { addNotification } = injectNotificationManager() + + /** Bumped whenever nodes are added to the tree, which listings and search depend on. */ + const treeVersion = ref(0) + const fullyLoaded = ref(false) + /** The archive's own SHA-512, once it has been downloaded in full and hashed. */ + const sha512Hash = ref(root.sha512Hash ?? null) + let treeUpdated = Promise.withResolvers() + + function nodeAt(path: string): ZipNode | undefined { + const normalized = normalizeDirectoryPath(path) + return normalized === '/' ? root : root.entries.get(normalized) + } + + function fileNodeAt(path: string): FileZipTreeNode | undefined { + return nodeAt(path)?.asFile() + } + + /** Whether the entry is a nested archive whose own entries haven't been read yet. */ + function isPendingArchive(node: ZipNode) { + return !fullyLoaded.value && isArchive(node) && node.children.size === 0 + } + + function toItem(node: ZipNode): FileItem { + const modified = toSeconds(node.metadata.lastModDate) + const browsable = node.type === 'directory' || isArchive(node) + return { + name: node.name, + type: browsable ? 'directory' : 'file', + path: normalizeDirectoryPath(node.path), + modified, + created: toSeconds(node.metadata.creationDate) || modified, + size: node.type === 'directory' ? null : node.metadata.uncompressedSize, + count: browsable ? node.children.size : null, + readOnly: true, + } + } + + function missingEntryError(path: string) { + return new Error(formatMessage(messages.missingEntry, { path })) + } + + const directoryStates = new Map() + const fileResults = new Map() + + /** Entries may only show up once the nested archive holding them is read. */ + function syncDirectoryState(path: string, state: DirectoryState) { + const node = nodeAt(path) + state.isLoading.value = node ? isPendingArchive(node) : !fullyLoaded.value + state.loadError.value = node || !fullyLoaded.value ? null : missingEntryError(path) + } + + function directoryResult(info: FileInfo<'directory'>): DirectoryResult { + const path = normalizeDirectoryPath(info.path) + let state = directoryStates.get(path) + if (!state) { + const isLoading = ref(false) + const loadError = ref(null) + const data = computed(() => { + void treeVersion.value + const node = nodeAt(path) + return node && (node.type === 'directory' || isArchive(node)) + ? [...node.children.values()].map(toItem) + : [] + }) + state = { result: { ...info, data, isLoading, loadError }, isLoading, loadError } + syncDirectoryState(path, state) + directoryStates.set(path, state) + } + return state.result + } + + /** Resolves the file at `path`, waiting for the nested archive holding it to be read. */ + async function waitForFileNode(path: string): Promise { + for (;;) { + const node = fileNodeAt(path) + if (node) return node + if (fullyLoaded.value) throw missingEntryError(path) + await treeUpdated.promise + } + } + + /** Class files are decompiled, everything else is read as is. */ + async function readContent(path: string): Promise { + const node = await waitForFileNode(path) + const result = await node.asText() + return result.wasDecompiled + ? new TextEncoder().encode(result.asString).buffer + : await node.getBuffer() + } + + function fileResult(info: FileInfo<'file'>): FileResult { + const path = normalizeDirectoryPath(info.path) + let result = fileResults.get(path) + if (!result) { + const data = shallowRef(null) + const isLoading = ref(true) + const loadError = ref(null) + readContent(path) + .then( + (buffer) => (data.value = markRaw(buffer)), + (error) => (loadError.value = toError(error)), + ) + .finally(() => (isLoading.value = false)) + result = { ...info, data: computed(() => data.value), isLoading, loadError } + fileResults.set(path, result) + } + return result + } + + const directoryTree = { + get: (info: FileInfo): FileItemResultFrom => + (info.type === 'file' + ? fileResult(info as FileInfo<'file'>) + : directoryResult(info as FileInfo<'directory'>)) as FileItemResultFrom, + /** Class files aren't prefetched, so hovering over a package doesn't queue up decompiles. */ + prefetch: (info: FileInfo) => { + if (info.type === 'file' && isClassFile(info.name)) return + directoryTree.get(info) + }, + expandedEntries: ref([]), + loadedEntries: computed(() => { + void treeVersion.value + return [...root.entries.values()].map(toItem) + }), + } satisfies DirectoryTree + + /** The entry at a path within the archive, or `null` when there is none (yet). */ + function locate(path: string): FileInfo | null { + const node = nodeAt(path) + if (!node) return null + return node === root ? infoFrom('/') : infoFrom(toItem(node)) + } + + const currentLocation = ref((initialPath ? locate(initialPath) : null) ?? infoFrom('/')) + const currentDirectory = computed>(() => + currentLocation.value.type === 'file' + ? parentInfoFrom(currentLocation.value) + : (currentLocation.value as FileInfo<'directory'>), + ) + + /** A requested path within a nested archive that hasn't been read yet, opened once it is. */ + let pendingPath = initialPath && !locate(initialPath) ? initialPath : null + + function navigateTo(file: FileInfo) { + if (file.path !== '/') pendingPath = null + currentLocation.value = file + } + + /** Opens a path within the archive, or once it shows up if a nested archive is still being read. */ + function open(path: string) { + const target = locate(path) + if (target) { + pendingPath = null + if (!isSameInfo(target, currentLocation.value)) navigateTo(target) + } else if (!fullyLoaded.value) { + pendingPath = path + } + } + + function onTreeChanged() { + treeVersion.value++ + for (const [path, state] of directoryStates) syncDirectoryState(path, state) + + if (pendingPath) { + const target = locate(pendingPath) + if (target) { + pendingPath = null + navigateTo(target) + } else if (fullyLoaded.value) { + pendingPath = null + } + } + + treeUpdated.resolve() + treeUpdated = Promise.withResolvers() + } + + const updates = handler.onArchiveUpdate.subscribe((update) => { + if (update.type === 'node_loaded' && update.root.root === root.key) { + onTreeChanged() + } else if (update.type === 'updated_node_data' && update.node.root === root.key) { + fileResults.delete(normalizeDirectoryPath(update.node.path)) + } else if (update.type === 'root_hashed' && update.root.root === root.key) { + sha512Hash.value = update.sha512Hash + } + }) + onScopeDispose(() => updates.cancel()) + + root.onFinishedLoading.then(() => { + fullyLoaded.value = true + onTreeChanged() + }) + + function saveBuffer(buffer: ArrayBuffer, name: string) { + const url = URL.createObjectURL(new Blob([buffer])) + const anchor = document.createElement('a') + anchor.href = url + anchor.download = name + anchor.click() + setTimeout(() => URL.revokeObjectURL(url), 0) + } + + async function downloadEntry(file: FileInfo) { + const node = fileNodeAt(file.path) + if (!node) return + saveBuffer(await node.getBuffer(), node.name) + } + + async function readLazy(entry: FileInfo, key: K): Promise | null> { + const node = fileNodeAt(entry.path) + return node ? (await node[key].get()) as ReturnType : null + } + + const isFileNode = (entry: FileInfo) => fileNodeAt(entry.path) != null + + const entryDetails: FileEntryDetail[] = [ + { + id: 'archive-detected-type', + label: formatMessage(messages.detectedType), + shown: isFileNode, + value: async (entry) => (await readLazy(entry, 'fileTypeInfo'))?.mime ?? null, + }, + { + id: 'archive-compressed-size', + label: formatMessage(messages.compressedSize), + shown: isFileNode, + value: (entry) => { + const node = fileNodeAt(entry.path) + return node ? formatBytes(node.metadata.compressedSize) : null + }, + }, + { + id: 'archive-sha1', + label: formatMessage(messages.sha1), + shown: isFileNode, + value: async (entry) => await readLazy(entry, 'sha1'), + }, + ] + + const entryActions: FileEntryAction[] = [ + { + id: 'archive-download-nested', + label: formatMessage(messages.downloadArchive), + icon: DownloadIcon, + shown: (entry) => { + const node = nodeAt(entry.path); + return node != null && isZipArchive(node); + }, + action: downloadEntry, + }, + { + id: 'archive-copy-sha512', + label: formatMessage(messages.copySha512), + icon: ClipboardCopyIcon, + shown: isFileNode, + action: async (entry) => { + const hash = await readLazy(entry, 'sha512') + if (!hash) return + await navigator.clipboard.writeText(hash) + addNotification({ title: formatMessage(messages.copiedSha512), type: 'success' }) + }, + }, + ] + + const context: FileManagerContext = { + workspaceId: `archive:${source.sha512Hash ?? source.url.split('?')[0]}`, + currentFile: computed(() => currentLocation.value), + currentDirectory, + directoryTree, + loading: computed(() => directoryResult(currentDirectory.value).isLoading.value), + error: computed(() => directoryResult(currentDirectory.value).loadError.value), + navigateTo, + + createItem: async () => null, + renameItem: async () => null, + moveItem: async () => null, + deleteItem: async () => false, + writeFile: async () => {}, + uploadFiles: () => {}, + downloadFile: downloadEntry, + + isRefreshing: ref(false), + refresh: async () => fileResults.clear(), + + readOnly: true, + isReadOnly: () => true, + readOnlyReason: computed(() => formatMessage(messages.readOnly)), + basePath: computed(() => source.name), + + canOpenInEditor: (file) => isClassFile(file.name) || canOpenInFileEditor(file.name), + editorLanguage: (file) => (isClassFile(file.name) ? 'java' : undefined), + entryActions, + entryDetails, + } + + return { context, location: currentLocation, open, fullyLoaded, treeVersion, sha512Hash, nodeAt } +} diff --git a/packages/ui/src/layouts/shared/archive-viewer/composables/archive-loader.ts b/packages/ui/src/layouts/shared/archive-viewer/composables/archive-loader.ts new file mode 100644 index 00000000000..fb272a0f9dc --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/composables/archive-loader.ts @@ -0,0 +1,122 @@ +import { markRaw, type MaybeRefOrGetter, onScopeDispose, ref, shallowRef, toValue, watch } from 'vue' + +import type { EventSubscription } from '#ui/helpers/toolbox/misc' +import type { ArchiveUpdateTypes, RootZipTreeNode, ZipHandler } from '#ui/helpers/toolbox/zip' + +import type { ArchiveLoadProgress, ArchiveLoadStage, ArchiveSource } from '../types' +import { startToolboxLogCapture } from './toolbox-logs' + +export interface ArchiveLoaderOptions { + /** Keeps archives open in the worker once they are left, so reopening them skips reading them. */ + retain?: boolean +} + +/** The toolbox is loaded on demand, as it pulls in the workers, zip.js and the decompilers. */ +async function getZipHandler(): Promise { + await startToolboxLogCapture() + const { messengerRegistry, zipHandlerType } = await import('#ui/helpers/toolbox/workers') + return (await messengerRegistry.getMessenger(zipHandlerType)!).handler +} + +/** + * Opens an archive by URL within the zip worker, reopening it whenever the source's URL changes. + * Only the archive's index is fetched before its tree is shown; file contents are fetched with + * range requests as they are opened, while nested archives are read in the background. + */ +export function useArchiveLoader( + source: MaybeRefOrGetter, + options: ArchiveLoaderOptions = {}, +) { + const stage = ref('idle') + /** Bytes of the archive fetched so far, which keeps growing as files are opened. */ + const progress = ref({ loaded: 0, total: null }) + const error = ref(null) + const root = shallowRef(null) + const handler = shallowRef(null) + /** Whether every nested archive has been read too. */ + const fullyLoaded = ref(false) + + let progressSubscription: EventSubscription<(update: ArchiveUpdateTypes) => void> | null = null + /** Identifies the latest load, so an earlier one finishing late is ignored. */ + let generation = 0 + + function release(node: RootZipTreeNode | null) { + if (options.retain || !node || !handler.value) return + handler.value.closeZipTree(node).catch(() => undefined) + } + + function trackProgress(zip: ZipHandler, url: string) { + progressSubscription?.cancel() + progressSubscription = zip.onArchiveUpdate.subscribe((update) => { + if (update.type === 'archive_download' && update.url === url) { + progress.value = { loaded: update.current, total: update.total } + } + }) + } + + async function load(target: ArchiveSource | null) { + const current = ++generation + release(root.value) + root.value = null + error.value = null + fullyLoaded.value = false + progressSubscription?.cancel() + progressSubscription = null + + if (!target || typeof window === 'undefined') { + stage.value = 'idle' + return + } + + stage.value = 'reading' + progress.value = { loaded: 0, total: target.size ?? null } + + try { + const zip = await getZipHandler() + if (current !== generation) return + handler.value = zip + trackProgress(zip, target.url) + + const node = await zip.getOrReadZipTree({ + data_type: 'url', + name: target.name, + url: target.url, + sha512Hash: target.sha512Hash, + rangeOptions: { size: target.size }, + }) + if (current !== generation) return + + root.value = markRaw(node) + stage.value = 'ready' + node.onFinishedLoading.then(() => { + if (current === generation) fullyLoaded.value = true + }) + } catch (e) { + if (current !== generation) return + error.value = e instanceof Error ? e : new Error(String(e)) + stage.value = 'error' + } + } + + watch( + () => toValue(source)?.url, + () => load(toValue(source)), + { immediate: true }, + ) + + onScopeDispose(() => { + generation++ + progressSubscription?.cancel() + release(root.value) + }) + + return { + stage, + progress, + error, + root, + handler, + fullyLoaded, + retry: () => load(toValue(source)), + } +} diff --git a/packages/ui/src/layouts/shared/archive-viewer/composables/node-values.ts b/packages/ui/src/layouts/shared/archive-viewer/composables/node-values.ts new file mode 100644 index 00000000000..f7336c184e7 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/composables/node-values.ts @@ -0,0 +1,44 @@ +import { type Ref, ref, watch } from 'vue' + +import type { FileZipTreeNode, LazyDataAccess } from '#ui/helpers/toolbox/zip' + +export interface NodeLazyValue { + value: Ref + loading: Ref +} + +/** + * A value computed by the zip worker for a file node (e.g. a hash), loaded whenever the node + * changes. Results for a node that's no longer current are dropped. + */ +export function useNodeLazyValue( + node: () => FileZipTreeNode | undefined, + key: K, +): NodeLazyValue>> { + type Value = Awaited> + const value = ref(null) as Ref + const loading = ref(false) + + watch( + node, + async (current, _, onCleanup) => { + let stale = false + onCleanup(() => (stale = true)) + + value.value = null + if (!current) { + loading.value = false + return + } + + loading.value = true + const result = await (current[key].get() as Promise).catch(() => null) + if (stale) return + value.value = result + loading.value = false + }, + { immediate: true }, + ) + + return { value, loading } +} diff --git a/packages/ui/src/layouts/shared/archive-viewer/composables/toolbox-logs.ts b/packages/ui/src/layouts/shared/archive-viewer/composables/toolbox-logs.ts new file mode 100644 index 00000000000..9b90331bafc --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/composables/toolbox-logs.ts @@ -0,0 +1,51 @@ +import { shallowRef } from 'vue' + +import type { LogLevel } from '#ui/helpers/toolbox/misc/logging' + +export interface ToolboxLogEntry { + id: number + time: Date + level: LogLevel + logger: string + message: string + error?: string +} + +const MAX_ENTRIES = 1000 + +const entries = shallowRef([]) +let nextId = 0 +let capture: Promise | null = null + +function errorText(error: unknown) { + if (error == null) return undefined + return error instanceof Error ? (error.stack ?? error.message) : String(error) +} + +/** + * Starts keeping the toolbox's logs, including those its workers forward to the main thread. + * Only the latest entries are kept. Safe to call more than once. + */ +export function startToolboxLogCapture(): Promise { + capture ??= import('#ui/helpers/toolbox/workers').then(({ messengerRegistry }) => { + messengerRegistry.getLogFactory().onLog.subscribe((level, logger, message, extra) => { + const entry: ToolboxLogEntry = { + id: nextId++, + time: new Date(), + level, + logger, + message, + error: errorText(extra?.err), + } + entries.value = [...entries.value.slice(-(MAX_ENTRIES - 1)), entry] + }) + }) + return capture +} + +export function useToolboxLogs() { + return { + entries, + clear: () => (entries.value = []), + } +} diff --git a/packages/ui/src/layouts/shared/archive-viewer/index.ts b/packages/ui/src/layouts/shared/archive-viewer/index.ts new file mode 100644 index 00000000000..96d0428d4c4 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/index.ts @@ -0,0 +1,4 @@ +export { default as ArchiveHashValue } from './components/HashValue.vue' +export { default as ArchiveViewerLayout } from './layout.vue' +export { type ArchiveViewerContext, injectArchiveViewer } from './providers/archive-viewer' +export * from './types' diff --git a/packages/ui/src/layouts/shared/archive-viewer/layout.vue b/packages/ui/src/layouts/shared/archive-viewer/layout.vue new file mode 100644 index 00000000000..3a7a2858105 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/layout.vue @@ -0,0 +1,158 @@ + + + diff --git a/packages/ui/src/layouts/shared/archive-viewer/providers/archive-viewer.ts b/packages/ui/src/layouts/shared/archive-viewer/providers/archive-viewer.ts new file mode 100644 index 00000000000..116da5d7610 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/providers/archive-viewer.ts @@ -0,0 +1,26 @@ +import type { Ref } from 'vue' + +import type { RootZipTreeNode, ZipHandler, ZipNode } from '#ui/helpers/toolbox/zip' +import { createContext } from '#ui/providers/create-context' + +import type { ArchiveSource } from '../types' + +/** The archive open in the archive viewer, for its panels. */ +export interface ArchiveViewerContext { + root: RootZipTreeNode + handler: ZipHandler + source: ArchiveSource + /** Whether every nested archive has been read. */ + fullyLoaded: Readonly> + /** Bumped whenever nodes are added to the tree. */ + treeVersion: Readonly> + /** The archive's own SHA-512, once it has been downloaded in full and hashed. */ + sha512Hash: Readonly> + /** The node at a path of the files layout (`/` being the archive's root). */ + nodeAt: (path: string) => ZipNode | undefined +} + +export const [injectArchiveViewer, provideArchiveViewer] = createContext( + 'ArchiveWorkbench', + 'archiveViewer', +) diff --git a/packages/ui/src/layouts/shared/archive-viewer/types.ts b/packages/ui/src/layouts/shared/archive-viewer/types.ts new file mode 100644 index 00000000000..e4fc12dc3b9 --- /dev/null +++ b/packages/ui/src/layouts/shared/archive-viewer/types.ts @@ -0,0 +1,33 @@ +/** An archive (e.g. a mod's `.jar` or a modpack's `.mrpack`) that can be opened in the archive viewer. */ +export interface ArchiveSource { + /** Where the archive is downloaded from, usually a Modrinth CDN link. */ + url: string + /** The archive's file name, shown as the root of its paths. */ + name: string + /** Size in bytes, which saves a request when reading the archive by ranges. */ + size?: number + /** The archive's SHA-512 hash, so an archive already open under another URL is reused. */ + sha512Hash?: string +} + +/** `reading` covers fetching the archive's index; its tree is `ready` before nested archives are read. */ +export type ArchiveLoadStage = 'idle' | 'reading' | 'ready' | 'error' + +/** A labelled value listed in one of the archive viewer's detail panels. */ +export interface ArchiveDetailRow { + id: string + label: string + value: string | null | undefined + /** Shows a spinner while the value is computed, e.g. a hash. */ + loading?: boolean + /** Shows the value in a monospace button that copies it. */ + copyable?: boolean + /** Shows the value shortened, in full on hover, in a button that copies it. */ + hash?: boolean + href?: string +} + +export interface ArchiveLoadProgress { + loaded: number + total: number | null +} diff --git a/packages/ui/src/layouts/shared/files-tab/components/FileActionBar.vue b/packages/ui/src/layouts/shared/files-tab/components/FileActionBar.vue index 2c5b08db119..6731ba34aef 100644 --- a/packages/ui/src/layouts/shared/files-tab/components/FileActionBar.vue +++ b/packages/ui/src/layouts/shared/files-tab/components/FileActionBar.vue @@ -4,12 +4,16 @@ :class="[hasNav ? 'gap-4' : 'gap-3']" :aria-label="formatMessage(messages.fileNavigation)" > -

+
-
+
{{ formatMessage(messages.expand) }}
-
+
+ @@ -76,7 +83,7 @@ (), { diff --git a/packages/ui/src/layouts/shared/files-tab/components/FileBrowserPanel.vue b/packages/ui/src/layouts/shared/files-tab/components/FileBrowserPanel.vue index ca3f0de830e..b76313620de 100644 --- a/packages/ui/src/layouts/shared/files-tab/components/FileBrowserPanel.vue +++ b/packages/ui/src/layouts/shared/files-tab/components/FileBrowserPanel.vue @@ -19,7 +19,7 @@ :sidebar-open="sidebarOpen" :active-location="ui.activeLocation.value" :is-editor-find-open="ui.fileEditorApi.value?.isFindOpen.value ?? false" - :search-query="ui.searchQuery.value" + :search-query="view.searchQuery.value" :show-refresh-button="ui.showRefreshButton.value" :show-install-from-url="ui.showInstallFromUrl" :base-id="ui.baseId" @@ -29,12 +29,13 @@ :can-go-back="ui.fileTabs.canGoBack.value" :can-go-forward="ui.fileTabs.canGoForward.value" :sidebar-toggleable="ui.advancedView.value" + :compact="ui.compactNavigation.value" @back="ui.fileTabs.back" @forward="ui.fileTabs.forward" @navigate="ui.navigateToSegment" @navigate-home="() => ui.navigateToSegment(0)" @prefetch-home="ui.handleHomePrefetch" - @update:search-query="(value) => (ui.searchQuery.value = value)" + @update:search-query="(value) => (view.searchQuery.value = value)" @create="ui.showCreateModal" @upload="ui.initiateFileUpload" @upload-zip="() => {}" @@ -62,128 +63,38 @@
- -
- - - -
-
- -
-
-
-
- -

- {{ formatMessage(messages.emptyFolderTitle) }} -

-

- {{ formatMessage(messages.emptyFolderDescription) }} -

-
-
- - -
diff --git a/packages/ui/src/layouts/shared/files-tab/components/FileNavbar.vue b/packages/ui/src/layouts/shared/files-tab/components/FileNavbar.vue index 75b4464a9d9..617dfae15bb 100644 --- a/packages/ui/src/layouts/shared/files-tab/components/FileNavbar.vue +++ b/packages/ui/src/layouts/shared/files-tab/components/FileNavbar.vue @@ -3,12 +3,14 @@
@@ -72,14 +82,11 @@ import { type Component, computed, onBeforeUnmount, ref, useId, type VNode } fro import { IconButton } from '#ui/components/base/buttons' import { useVIntl } from '#ui/composables/i18n' -import type { HashType } from '#ui/helpers/toolbox/file' +import { HASH_TYPES, type HashType } from '#ui/helpers/toolbox/file' import type { ArchiveDetailRow } from '../../types' import type { ArchiveCounts, ArchiveReadStatus } from '../../types' -import { - DEFAULT_ARCHIVE_HASH_TYPES, - injectArchiveDetailsSettings, -} from '../../providers/details-settings' +import { type DetailVisibility, useArchiveDetailSettings } from '../../settings' import DetailsCounts from './counts.vue' import DetailsToggle from './details-toggle.vue' import HashList from './hash-list.vue' @@ -89,12 +96,13 @@ import StatGrid from './stat-grid.vue' /** * One described item of the details stack: what it is and its name (with how far it's been read, - * for an archive), then its details and sizes. In-depth details and hashes are tucked behind a - * pull tab along its bottom, open or closed for every item sharing its `detailsKey`; contents - * counts show as the viewer's settings say. Hosts can use it for items of their own (e.g. the - * project an archive was published by), with an `icon` slot in place of the icon, `actions` - * beside the name, their own content (the default slot) before the details and more of it - * (the `details` slot) behind the pull tab. + * for an archive), then its details. Each row, size, the contents counts and each hash shows + * always, only behind the pull tab along the section's bottom, or never, as the row (or the + * viewer's settings) say; the pull tab is open or closed for every item sharing its + * `detailsKey`. Hosts can use it for items of their own (e.g. the project an archive was + * published by), with an `icon` slot in place of the icon, `actions` beside the name, their own + * content (the default slot) before the details and more of it (the `details` slot) behind the + * pull tab. */ const props = defineProps<{ kind: string @@ -104,12 +112,14 @@ const props = defineProps<{ /** Where the item sits in the archive, shown on hover and copied from a button. */ path?: string status?: ArchiveReadStatus + /** Details shown as their `visibility` says, always by default. */ rows?: ArchiveDetailRow[] - /** In-depth details, folded away by default. */ + /** In-depth details, only shown behind the pull tab. */ more?: ArchiveDetailRow[] + /** Sizes, shown as their `visibility` says, always by default. */ sizes?: ArchiveDetailRow[] counts?: ArchiveCounts - /** Computes the item's hash of a type, for those the viewer's settings list. */ + /** Computes the item's hash of a type, for those the viewer's settings show. */ hash?: (type: HashType) => Promise /** Hash types the item can have, when it can't have every kind. */ hashTypes?: HashType[] @@ -127,25 +137,49 @@ const slots = defineSlots<{ const { formatMessage } = useVIntl() const headingId = useId() const detailsId = useId() -const settings = injectArchiveDetailsSettings(null) +const settings = useArchiveDetailSettings() const open = useLocalStorage(`archive-viewer-details:open:${props.detailsKey}`, false, { initOnMounted: true, }) -const countsVisibility = computed(() => settings?.contentCounts.value ?? 'details') +const countsVisibility = computed(() => settings.visibility('contentCounts')) -const shownHashTypes = computed(() => { - const selected = settings?.hashTypes.value ?? DEFAULT_ARCHIVE_HASH_TYPES - return props.hashTypes ? selected.filter((type) => props.hashTypes!.includes(type)) : selected +/** Rows split by where they show; ones never shown are left out. */ +function byVisibility(rows: ArchiveDetailRow[] | undefined) { + const split: Record, ArchiveDetailRow[]> = { + always: [], + details: [], + } + for (const row of rows ?? []) { + const visibility = row.visibility ?? 'always' + if (visibility !== 'never') split[visibility].push(row) + } + return split +} + +const shownRows = computed(() => byVisibility(props.rows)) +const shownSizes = computed(() => byVisibility(props.sizes)) +const detailRows = computed(() => [...shownRows.value.details, ...(props.more ?? [])]) + +/** The item's hash types split by where the settings show them, in the order they're listed. */ +const hashTypesShown = computed(() => { + const split: Record, HashType[]> = { always: [], details: [] } + for (const type of HASH_TYPES) { + if (props.hashTypes && !props.hashTypes.includes(type)) continue + const visibility = settings.hashVisibility(type) + if (visibility !== 'never') split[visibility].push(type) + } + return split }) const hasDetails = computed( () => !!slots.details || - !!props.more?.length || + detailRows.value.length > 0 || + shownSizes.value.details.length > 0 || (!!props.counts && countsVisibility.value === 'details') || - (!!props.hash && shownHashTypes.value.length > 0), + (!!props.hash && hashTypesShown.value.details.length > 0), ) const copied = ref(false) diff --git a/packages/ui/src/layouts/shared/archive-viewer/components/details-section/list.vue b/packages/ui/src/layouts/shared/archive-viewer/components/details-section/list.vue index 4b25f4b8b19..60013ff6cca 100644 --- a/packages/ui/src/layouts/shared/archive-viewer/components/details-section/list.vue +++ b/packages/ui/src/layouts/shared/archive-viewer/components/details-section/list.vue @@ -3,12 +3,12 @@