diff --git a/services/apps/connectors_worker/src/bin/dry-run-credly-badge-poll.ts b/services/apps/connectors_worker/src/bin/dry-run-credly-badge-poll.ts index f601fdb603..60a314a1ed 100644 --- a/services/apps/connectors_worker/src/bin/dry-run-credly-badge-poll.ts +++ b/services/apps/connectors_worker/src/bin/dry-run-credly-badge-poll.ts @@ -125,6 +125,9 @@ async function main(): Promise { saveBadges: async () => undefined, listDueExpiries: async () => [], markExpiriesEmitted: async () => 0, + nextRelease: async () => null, + saveReleaseCursor: async () => 0, + completeRelease: async () => 0, } const outcome = await runBadgePoll({ diff --git a/services/apps/connectors_worker/src/bin/resolve-credly-templates.ts b/services/apps/connectors_worker/src/bin/resolve-credly-templates.ts index 1ecc749bc4..a17e3ae9e1 100644 --- a/services/apps/connectors_worker/src/bin/resolve-credly-templates.ts +++ b/services/apps/connectors_worker/src/bin/resolve-credly-templates.ts @@ -10,9 +10,11 @@ import { mintCredlyAccessToken, } from '@crowd/connectors/src/connectors/credly/oauth' import { + listActiveCredlyTemplateMappings, listCredlyTemplateMappings, listHeldCredlyTemplateMappings, } from '@crowd/data-access-layer/src/credly' +import { queueCredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases' import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { getServiceLogger } from '@crowd/logging' @@ -20,11 +22,13 @@ import { SnowflakeClient } from '@crowd/snowflake' import { activeHeld, buildHeldQueueReport, postHeldQueueReport } from '../credlyHeldQueueReport' import { + CREDLY_HISTORY_START, applyDecisions, buildPlan, loadCatalogue, loadCredlyTemplates, loadSubprojectSegments, + releasesToQueue, } from '../credlyTemplateResolve' const log = getServiceLogger() @@ -32,7 +36,7 @@ const HELD_QUEUE_PRINTED = 20 function usage(): never { log.error( - 'Usage: resolve-credly-templates --integration-id [--apply [--notify | --digest]] [--out ]', + 'Usage: resolve-credly-templates --integration-id [--apply [--notify | --digest]] [--out ] | --release ', ) process.exit(1) } @@ -54,6 +58,7 @@ function takeSwitch(argv: string[], flag: string): boolean { } interface Args { + release?: string integrationId: string apply: boolean notify: 'changes' | 'digest' | null @@ -62,6 +67,11 @@ interface Args { function parseArgs(rawArgv: string[]): Args { const argv = rawArgv.filter((arg) => arg !== '--') + const release = takeFlag(argv, '--release') + if (release) { + if (argv.length > 0) usage() + return { release, integrationId: '', apply: false, notify: null, outFile: '' } + } const integrationId = takeFlag(argv, '--integration-id') const apply = takeSwitch(argv, '--apply') const digest = takeSwitch(argv, '--digest') @@ -75,10 +85,28 @@ function parseArgs(rawArgv: string[]): Args { } async function main(): Promise { - const { integrationId, apply, notify, outFile } = parseArgs(process.argv.slice(2)) + const { release, integrationId, apply, notify, outFile } = parseArgs(process.argv.slice(2)) const db = await getDbConnection(WRITE_DB_CONFIG()) const qx = pgpQx(db) + if (release) { + const active = await listActiveCredlyTemplateMappings(qx) + if (!active.some((mapping) => mapping.badgeTemplateId === release)) { + log.error( + { badgeTemplateId: release }, + 'credly template is not mapped to a live integration, nothing to release', + ) + process.exit(1) + } + await queueCredlyTemplateRelease(qx, { + badgeTemplateId: release, + floor: CREDLY_HISTORY_START, + top: new Date().toISOString(), + }) + log.info({ badgeTemplateId: release }, 'credly template history queued for release') + return + } + const credential = await getCredential(qx, integrationId) assertCredentialKind(credential, 'credly-oauth-client') const { token } = await mintCredlyAccessToken(credential) @@ -153,7 +181,13 @@ async function main(): Promise { 'the credly listing changed while paging: templates not seen are left as they are', ) } - const result = await applyDecisions(qx, decisions, listing.complete ? missing : [], log) + const result = await applyDecisions( + qx, + decisions, + listing.complete ? missing : [], + log, + releasesToQueue(plan, templates, new Date()), + ) const heldQueue = await listHeldCredlyTemplateMappings(qx) log.info( { diff --git a/services/apps/connectors_worker/src/credlyTemplateResolve.test.ts b/services/apps/connectors_worker/src/credlyTemplateResolve.test.ts index aac6c86824..0be27b0db1 100644 --- a/services/apps/connectors_worker/src/credlyTemplateResolve.test.ts +++ b/services/apps/connectors_worker/src/credlyTemplateResolve.test.ts @@ -5,10 +5,12 @@ import type { ResolvedTemplate } from '@crowd/connectors/src/connectors/credly/m import type { ICredlyStoredTemplateMapping } from '@crowd/data-access-layer/src/credly' import { + CREDLY_HISTORY_START, buildPlan, loadCatalogue, loadCredlyTemplates, loadSubprojectSegments, + releasesToQueue, } from './credlyTemplateResolve' function decision(id: string, overrides: Partial = {}): ResolvedTemplate { @@ -221,8 +223,8 @@ describe('loadCredlyTemplates', () => { expect(listing.complete).toBe(true) expect(listing.templates).toEqual([ - { id: 'a', name: 'A', badgeCount: 3, url: null }, - { id: 'b', name: 'B', badgeCount: null, url: null }, + { id: 'a', name: 'A', badgeCount: 3, url: null, createdAt: null }, + { id: 'b', name: 'B', badgeCount: null, url: null, createdAt: null }, ]) }) @@ -234,3 +236,86 @@ describe('loadCredlyTemplates', () => { expect((await loadCredlyTemplates(http, 'org', log)).complete).toBe(false) }) }) + +describe('releasesToQueue', () => { + const now = new Date('2026-11-02T12:00:00.000Z') + const plan = ( + entries: { + badgeTemplateId: string + kind: string + before: string | null + after: string | null + }[], + ) => + ({ + entries: entries.map((e) => ({ ...e, badgeTemplateName: e.badgeTemplateId })), + counts: {}, + }) as never + + it('queues the templates that reached a project, new or released, from their creation date', () => { + const releases = releasesToQueue( + plan([ + { + badgeTemplateId: 'lfc131', + kind: 'changed', + before: 'held:no_project', + after: 'mapped:green-software', + }, + { badgeTemplateId: 'old', kind: 'changed', before: 'held:no_match', after: 'mapped:cncf' }, + { badgeTemplateId: 'moved', kind: 'changed', before: 'mapped:k8s', after: 'mapped:cncf' }, + { + badgeTemplateId: 'locked', + kind: 'locked', + before: 'held:no_match', + after: 'mapped:cncf', + }, + { badgeTemplateId: 'excluded', kind: 'changed', before: 'held:test', after: 'excluded' }, + { + badgeTemplateId: 'unexcluded', + kind: 'changed', + before: 'excluded', + after: 'mapped:finops', + }, + { badgeTemplateId: 'fresh', kind: 'new', before: null, after: 'mapped:k8s' }, + ]), + [ + { id: 'lfc131', name: 'LFC131', createdAt: '2022-10-25T19:41:32.416Z' }, + { id: 'old', name: 'Old', createdAt: '2018-04-19T00:00:00.000Z' }, + { id: 'unexcluded', name: 'Unexcluded', createdAt: '2026-11-02T12:00:00.000Z' }, + { id: 'fresh', name: 'Fresh', createdAt: '2026-11-01T09:00:00.000Z' }, + ], + now, + ) + + expect(releases).toEqual([ + { + badgeTemplateId: 'lfc131', + floor: '2022-10-25T19:41:32.416Z', + top: '2026-11-02T12:00:00.000Z', + }, + { badgeTemplateId: 'old', floor: CREDLY_HISTORY_START, top: '2026-11-02T12:00:00.000Z' }, + { + badgeTemplateId: 'unexcluded', + floor: '2026-11-02T11:59:59.000Z', + top: '2026-11-02T12:00:00.000Z', + }, + { + badgeTemplateId: 'fresh', + floor: '2026-11-01T09:00:00.000Z', + top: '2026-11-02T12:00:00.000Z', + }, + ]) + }) + + it('falls back to the history start when the template has no creation date', () => { + const releases = releasesToQueue( + plan([ + { badgeTemplateId: 'x', kind: 'changed', before: 'held:no_match', after: 'mapped:k8s' }, + ]), + [{ id: 'x', name: 'X', createdAt: null }], + now, + ) + + expect(releases[0].floor).toBe(CREDLY_HISTORY_START) + }) +}) diff --git a/services/apps/connectors_worker/src/credlyTemplateResolve.ts b/services/apps/connectors_worker/src/credlyTemplateResolve.ts index a39294a000..4d57dc4f15 100644 --- a/services/apps/connectors_worker/src/credlyTemplateResolve.ts +++ b/services/apps/connectors_worker/src/credlyTemplateResolve.ts @@ -15,6 +15,8 @@ import { findOrCreateCredlyIntegration, upsertCredlyTemplateDecision, } from '@crowd/data-access-layer/src/credly' +import type { ICredlyTemplateReleaseInput } from '@crowd/data-access-layer/src/credly/releases' +import { queueCredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases' import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { listActiveSubprojectSegments } from '@crowd/data-access-layer/src/segments' import type { Logger } from '@crowd/logging' @@ -77,6 +79,7 @@ export async function loadCredlyTemplates( name: template.name, badgeCount: typeof count === 'number' && Number.isFinite(count) ? count : null, url: template.url ?? null, + createdAt: template.created_at ?? null, }) } if (result.data.length === 0 || page >= result.metadata.total_pages) break @@ -183,19 +186,55 @@ export interface ApplyResult { outcomes: Record integrationsCreated: number deactivated: number + releasesQueued: number } +// Credly moved to its current platform in March 2019; nothing older carries a state_updated_at. +export const CREDLY_HISTORY_START = '2019-03-01T00:00:00.000Z' + +export function releaseFloor(createdAt: string | null | undefined, top: Date): string { + const createdMs = Date.parse(createdAt ?? '') + const startMs = Date.parse(CREDLY_HISTORY_START) + const floorMs = Number.isFinite(createdMs) ? Math.max(createdMs, startMs) : startMs + return new Date(Math.min(floorMs, top.getTime() - 1000)).toISOString() +} + +export function releasesToQueue( + plan: ResolvePlan, + templates: TemplateToResolve[], + now: Date, +): ICredlyTemplateReleaseInput[] { + const createdAtById = new Map(templates.map((template) => [template.id, template.createdAt])) + return plan.entries + .filter( + (entry) => + (entry.kind === 'changed' || entry.kind === 'new') && + !entry.before?.startsWith('mapped:') && + entry.after?.startsWith('mapped:'), + ) + .map((entry) => ({ + badgeTemplateId: entry.badgeTemplateId, + floor: releaseFloor(createdAtById.get(entry.badgeTemplateId), now), + top: now.toISOString(), + })) +} + +// A release is queued in the same transaction as the decision that frees the template: a run that +// dies in between leaves the template held, and the next run decides it again. export async function applyDecisions( qx: QueryExecutor, decisions: ResolvedTemplate[], missing: PlanEntry[], log: Logger, + releases: ICredlyTemplateReleaseInput[] = [], ): Promise { const result: ApplyResult = { outcomes: { inserted: 0, updated: 0, unchanged: 0, locked: 0 }, integrationsCreated: 0, deactivated: 0, + releasesQueued: 0, } + const releaseById = new Map(releases.map((release) => [release.badgeTemplateId, release])) const integrationBySegment = new Map() for (const decision of decisions) { let integrationId: string | null = null @@ -208,7 +247,15 @@ export async function applyDecisions( integrationBySegment.set(decision.segmentId, integrationId) } } - const outcome = await upsertCredlyTemplateDecision(qx, { ...decision, integrationId }) + const release = releaseById.get(decision.badgeTemplateId) + const outcome = await qx.tx(async (tx) => { + const stored = await upsertCredlyTemplateDecision(tx, { ...decision, integrationId }) + if (release && (stored === 'inserted' || stored === 'updated')) { + await queueCredlyTemplateRelease(tx, release) + result.releasesQueued += 1 + } + return stored + }) result.outcomes[outcome] += 1 if (outcome === 'locked') { log.warn( diff --git a/services/libs/connectors/src/connectors/credly/apiClient.ts b/services/libs/connectors/src/connectors/credly/apiClient.ts index d9d12d5b0d..7c1644c7b2 100644 --- a/services/libs/connectors/src/connectors/credly/apiClient.ts +++ b/services/libs/connectors/src/connectors/credly/apiClient.ts @@ -78,6 +78,7 @@ export const credlyBadgeTemplateListingSchema = z type_category: z.string().nullable().optional(), global_activity_url: z.string().nullable().optional(), reporting_tags: z.array(z.string()).nullable().optional(), + created_at: z.string().nullable().optional(), }) .passthrough() diff --git a/services/libs/connectors/src/connectors/credly/deps.test.ts b/services/libs/connectors/src/connectors/credly/deps.test.ts index 206781434b..cb6a40fd19 100644 --- a/services/libs/connectors/src/connectors/credly/deps.test.ts +++ b/services/libs/connectors/src/connectors/credly/deps.test.ts @@ -94,6 +94,18 @@ describe('createCredlyDeps', () => { expect(mockedStale).toHaveBeenCalledTimes(1) }) + it('reads the mapping again when a fresh routing is asked for', async () => { + const deps = createCredlyDeps({ qx, integrationId: 'anchor' }) + + const first = await deps.loadTemplateRouting() + const fresh = await deps.loadTemplateRouting(true) + const cached = await deps.loadTemplateRouting() + + expect(fresh).not.toBe(first) + expect(cached).toBe(fresh) + expect(mockedActive).toHaveBeenCalledTimes(2) + }) + it('locks a template through the mapping table', async () => { vi.mocked(lockCredlyTemplateMapping).mockResolvedValueOnce(true) const deps = createCredlyDeps({ qx, integrationId: 'anchor' }) diff --git a/services/libs/connectors/src/connectors/credly/deps.ts b/services/libs/connectors/src/connectors/credly/deps.ts index 1380909ffd..baa6424ea5 100644 --- a/services/libs/connectors/src/connectors/credly/deps.ts +++ b/services/libs/connectors/src/connectors/credly/deps.ts @@ -14,6 +14,15 @@ import { markCredlyExpiriesEmitted, upsertCredlyBadges, } from '@crowd/data-access-layer/src/credly/badges' +import type { + ICredlyTemplateRelease, + ICredlyTemplateReleaseCursor, +} from '@crowd/data-access-layer/src/credly/releases' +import { + completeCredlyTemplateRelease, + nextPendingCredlyTemplateRelease, + saveCredlyTemplateReleaseCursor, +} from '@crowd/data-access-layer/src/credly/releases' import type { DepsFactoryInput } from '../../types' @@ -29,19 +38,26 @@ export interface CredlyTemplateRouting { } export interface CredlyDeps { - loadTemplateRouting(): Promise + loadTemplateRouting(fresh?: boolean): Promise lockTemplate(badgeTemplateId: string, target: CredlyTemplateTarget): Promise findBadges(badgeIds: string[]): Promise saveBadges(badges: ICredlyBadgeInput[]): Promise listDueExpiries(now: Date, limit: number, after?: ICredlyExpiryCursor): Promise markExpiriesEmitted(badgeIds: string[]): Promise + nextRelease(): Promise + saveReleaseCursor( + release: ICredlyTemplateRelease, + cursor: ICredlyTemplateReleaseCursor, + ): Promise + completeRelease(release: ICredlyTemplateRelease): Promise } export function createCredlyDeps({ qx }: DepsFactoryInput): CredlyDeps { let routing: Promise | null = null return { - loadTemplateRouting() { + loadTemplateRouting(fresh = false) { + if (fresh) routing = null routing ??= Promise.all([ listActiveCredlyTemplateMappings(qx), listStaleCredlyTemplateMappings(qx), @@ -71,5 +87,14 @@ export function createCredlyDeps({ qx }: DepsFactoryInput): CredlyDeps { markExpiriesEmitted(badgeIds) { return markCredlyExpiriesEmitted(qx, badgeIds) }, + nextRelease() { + return nextPendingCredlyTemplateRelease(qx) + }, + saveReleaseCursor(release, cursor) { + return saveCredlyTemplateReleaseCursor(qx, release, cursor) + }, + completeRelease(release) { + return completeCredlyTemplateRelease(qx, release) + }, } } diff --git a/services/libs/connectors/src/connectors/credly/mapping/resolver.ts b/services/libs/connectors/src/connectors/credly/mapping/resolver.ts index bd1f32b183..73f7074d1a 100644 --- a/services/libs/connectors/src/connectors/credly/mapping/resolver.ts +++ b/services/libs/connectors/src/connectors/credly/mapping/resolver.ts @@ -21,6 +21,7 @@ export interface TemplateToResolve { name: string badgeCount?: number | null url?: string | null + createdAt?: string | null } export interface ResolvedTemplate extends Required< diff --git a/services/libs/connectors/src/connectors/credly/syncs/badgePoll.test.ts b/services/libs/connectors/src/connectors/credly/syncs/badgePoll.test.ts index 9708c66eb7..97c7c8c3ad 100644 --- a/services/libs/connectors/src/connectors/credly/syncs/badgePoll.test.ts +++ b/services/libs/connectors/src/connectors/credly/syncs/badgePoll.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ICredlyBadgeInput, ICredlyBadgeRow } from '@crowd/data-access-layer/src/credly/badges' +import type { ICredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases' import { CredlyActivityType } from '@crowd/integrations/src/integrations/credly/types' import type { Logger } from '@crowd/logging' @@ -102,7 +103,9 @@ function makeCtx( routing: CredlyTemplateRouting = DEFAULT_ROUTING, emitEnabled = true, stored: ICredlyBadgeRow[] = [], + releases: ICredlyTemplateRelease[] = [], ) { + const queue = [...releases] const emitted: unknown[][] = [] const emits: { records: unknown[]; segmentId?: string; integrationId?: string }[] = [] const watermarks: Record[] = [] @@ -116,6 +119,15 @@ function makeCtx( }), listDueExpiries: vi.fn(async () => []), markExpiriesEmitted: vi.fn(async () => 0), + nextRelease: vi.fn(async () => queue[0] ?? null), + saveReleaseCursor: vi.fn(async (release, cursor) => { + queue[0] = { ...release, ...cursor } + return 1 + }), + completeRelease: vi.fn(async () => { + queue.shift() + return 1 + }), } const ctx: SyncContext = { channel: { channelId: ORGANIZATION_ID, channelName: ORGANIZATION_ID }, @@ -1261,3 +1273,258 @@ describe('runBadgePoll backfill', () => { ) }) }) + +describe('runBadgePoll releases', () => { + const FLOOR = '2026-10-10T00:00:00.000Z' + const TOP = '2026-10-20T00:00:00.000Z' + const MID = new Date(Date.parse(TOP) - MAX_WINDOW_MS).toISOString() + + function release(badgeTemplateId: string, overrides: Partial = {}) { + return { + badgeTemplateId, + floor: FLOOR, + top: TOP, + since: null, + until: null, + page: 1, + queuedAt: '2026-11-02T10:00:00.000Z', + completedAt: null, + ...overrides, + } + } + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(NOW) + mockedFetchBadgesPage.mockReset() + mockedFetchBadge.mockReset() + }) + + afterEach(() => { + vi.useRealTimers() + vi.clearAllMocks() + }) + + it('re-reads a released template with the template filter, window by window, then completes it', async () => { + mockedFetchBadgesPage.mockResolvedValue(page([badge({ id: 'old-1' })])) + const { ctx, emitted, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1')], + ) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: true }) + expect(requestedParams().map((p) => [p.window.min, p.window.max, p.badgeTemplateId])).toEqual([ + [MID, TOP, 'template-1'], + [FLOOR, MID, 'template-1'], + ]) + expect(emitted).toHaveLength(2) + expect(deps.completeRelease).toHaveBeenCalledTimes(1) + expect(LOG.info).toHaveBeenCalledWith( + { badgeTemplateId: 'template-1' }, + 'credly template history released', + ) + expect(LOG.info).toHaveBeenCalledWith( + expect.objectContaining({ event: 'credly_badge_poll_run', releasePages: 2, releases: 1 }), + expect.any(String), + ) + }) + + it('halves a failing release window and walks its upper half before moving down', async () => { + const err = new ProviderUnavailableError('provider returned status 500', { status: 500 }) + mockedFetchBadgesPage.mockRejectedValueOnce(err).mockResolvedValue(page([])) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1')], + ) + + await runBadgePoll(ctx) + + const half = new Date(Date.parse(MID) + MAX_WINDOW_MS / 2).toISOString() + expect(requestedParams().map((p) => p.window)).toEqual([ + { min: MID, max: TOP }, + { min: MID, max: half }, + { min: half, max: TOP }, + { min: FLOOR, max: MID }, + ]) + expect(deps.completeRelease).toHaveBeenCalledTimes(1) + }) + + it('reloads the routing for a release queued after the run started instead of skipping it', async () => { + mockedFetchBadgesPage.mockResolvedValue(page([badge({ id: 'old-1' })])) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1')], + ) + vi.mocked(deps.loadTemplateRouting).mockResolvedValueOnce({ + targets: new Map(), + staleTemplateIds: [], + }) + + await runBadgePoll(ctx) + + expect(vi.mocked(deps.loadTemplateRouting).mock.calls).toEqual([[false], [true]]) + expect(mockedFetchBadgesPage).toHaveBeenCalledTimes(2) + expect(deps.completeRelease).toHaveBeenCalledTimes(1) + expect(LOG.warn).not.toHaveBeenCalled() + }) + + it('completes a release whose template is not mapped without calling credly', async () => { + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('tpl-unmapped')], + ) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: true }) + expect(mockedFetchBadgesPage).not.toHaveBeenCalled() + expect(deps.completeRelease).toHaveBeenCalledTimes(1) + expect(LOG.warn).toHaveBeenCalledWith( + { badgeTemplateId: 'tpl-unmapped' }, + 'credly release skipped: template not mapped', + ) + }) + + it('stops skipping unmapped releases at the cap, leaving the run incomplete', async () => { + const releases = Array.from({ length: MAX_PAGES_PER_RUN + 1 }, (_, i) => + release(`tpl-unmapped-${i}`), + ) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + releases, + ) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: false }) + expect(mockedFetchBadgesPage).not.toHaveBeenCalled() + expect(deps.completeRelease).toHaveBeenCalledTimes(MAX_PAGES_PER_RUN) + }) + + it('stops at the page cap mid-release with the cursor saved, and stays incomplete', async () => { + mockedFetchBadgesPage.mockResolvedValue( + page([badge({ id: 'b', state_updated_at: MID })], 1, 1000), + ) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1')], + ) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: false }) + expect(mockedFetchBadgesPage).toHaveBeenCalledTimes(MAX_PAGES_PER_RUN) + expect(deps.completeRelease).not.toHaveBeenCalled() + expect(vi.mocked(deps.saveReleaseCursor).mock.lastCall?.[1]).toEqual({ + floor: FLOOR, + top: TOP, + since: MID, + until: TOP, + page: MAX_PAGES_PER_RUN + 1, + }) + }) + + it('serves a release before the global history, which then goes on without the filter', async () => { + mockedFetchBadgesPage.mockResolvedValue(page([])) + const { ctx } = makeCtx( + { since: HORIZON, backfill: { floor: FLOOR, top: TOP } }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1', { floor: MID })], + ) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: true }) + expect(requestedParams().map((p) => [p.window.min, p.badgeTemplateId])).toEqual([ + [MID, 'template-1'], + [MID, undefined], + [FLOOR, undefined], + ]) + }) + + it('drops a walk whose release was queued again underneath and reads the queue afresh', async () => { + mockedFetchBadgesPage.mockResolvedValue(page([])) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1')], + ) + vi.mocked(deps.saveReleaseCursor).mockResolvedValue(0) + vi.mocked(deps.nextRelease).mockResolvedValueOnce(release('template-1')).mockResolvedValue(null) + + const outcome = await runBadgePoll(ctx) + + expect(outcome).toEqual({ complete: true }) + expect(mockedFetchBadgesPage).not.toHaveBeenCalled() + expect(deps.nextRelease).toHaveBeenCalledTimes(2) + }) + + it('completes a release whose bounds cannot be read, with a warning', async () => { + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + true, + [], + [release('template-1', { floor: TOP, top: FLOOR })], + ) + + await runBadgePoll(ctx) + + expect(mockedFetchBadgesPage).not.toHaveBeenCalled() + expect(deps.completeRelease).toHaveBeenCalledTimes(1) + expect(LOG.warn).toHaveBeenCalledWith( + { badgeTemplateId: 'template-1' }, + 'credly release skipped: bounds unreadable', + ) + }) + + it('leaves the release queue alone during a shadow run', async () => { + mockedFetchBadgesPage.mockResolvedValue(page([])) + const { ctx, deps } = makeCtx( + { since: HORIZON }, + () => true, + DEFAULT_ROUTING, + false, + [], + [release('template-1')], + ) + + await runBadgePoll(ctx) + + expect(deps.nextRelease).not.toHaveBeenCalled() + expect(mockedFetchBadgesPage).not.toHaveBeenCalled() + }) +}) diff --git a/services/libs/connectors/src/connectors/credly/syncs/badgePoll.ts b/services/libs/connectors/src/connectors/credly/syncs/badgePoll.ts index 061e9b9c1d..6721c83842 100644 --- a/services/libs/connectors/src/connectors/credly/syncs/badgePoll.ts +++ b/services/libs/connectors/src/connectors/credly/syncs/badgePoll.ts @@ -1,4 +1,5 @@ import type { ICredlyBadgeInput, ICredlyBadgeRow } from '@crowd/data-access-layer/src/credly/badges' +import type { ICredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases' import { ConnectorError, ProviderUnavailableError } from '../../../http/errors' import { assertCredentialKind } from '../../../types' @@ -134,6 +135,25 @@ function afterBackfillDrain(cursor: CredlyBackfillCursor): CredlyBackfillCursor return { ...cursor, top, since: null, until: null, page: 1 } } +function afterReleaseDrain(cursor: CredlyBackfillCursor): CredlyBackfillCursor | undefined { + const next = afterBackfillDrain(cursor) + return next && !next.since ? openBackfillWindow(next) : next +} + +function releaseCursor(release: ICredlyTemplateRelease): CredlyBackfillCursor | undefined { + const iso = (value: string | null) => { + const parsable = parsableIso(value) + return parsable === null ? null : toIsoTimestamp(parsable) + } + return readBackfill({ + floor: iso(release.floor), + top: iso(release.top), + since: iso(release.since), + until: iso(release.until), + page: release.page, + }) +} + function withBackfill( watermark: CredlyBadgePollWatermark, backfill: CredlyBackfillCursor | undefined, @@ -227,13 +247,17 @@ export async function runBadgePoll(ctx: SyncContext): Promise 0) { - ctx.log.warn( - { badgeTemplateIds: routing.staleTemplateIds }, - 'credly template mappings point to a deleted or moved integration, their badges are held', - ) + const loadRouting = async (fresh: boolean) => { + const loaded = await ctx.deps.loadTemplateRouting(fresh) + if (loaded.staleTemplateIds.length > 0) { + ctx.log.warn( + { badgeTemplateIds: loaded.staleTemplateIds }, + 'credly template mappings point to a deleted or moved integration, their badges are held', + ) + } + return loaded } + let routing = await loadRouting(false) const startedAtMs = Date.now() const horizonMs = startedAtMs - WINDOW_LAG_MS @@ -263,6 +287,9 @@ export async function runBadgePoll(ctx: SyncContext): Promise() @@ -410,11 +437,17 @@ export async function runBadgePoll(ctx: SyncContext): Promise } | { halved: Window }> => { try { const page = await fetchBadgesPage( ctx.http, - { organizationId, window: { min: window.since, max: window.until }, page: window.page }, + { + organizationId, + window: { min: window.since, max: window.until }, + page: window.page, + badgeTemplateId, + }, ctx.log, ) counters.pagesFetched += 1 @@ -463,7 +496,76 @@ export async function runBadgePoll(ctx: SyncContext): Promise { + ctx.log.warn({ badgeTemplateId: release.badgeTemplateId }, reason) + counters.releasesSkipped += 1 + await ctx.deps.completeRelease(release) + return ctx.deps.nextRelease() + } + const advanceRelease = async (release: ICredlyTemplateRelease, cursor: CredlyBackfillCursor) => + (await ctx.deps.saveReleaseCursor(release, cursor)) + ? { ...release, ...cursor } + : ctx.deps.nextRelease() + let release = ctx.emitEnabled && !watermark.until ? await ctx.deps.nextRelease() : null + while ( + release && + counters.pagesFetched < MAX_PAGES_PER_RUN && + counters.releasesSkipped < MAX_PAGES_PER_RUN && + ctx.hasRunBudget() + ) { + const templateId = release.badgeTemplateId + if (!routing.targets.has(templateId)) routing = await loadRouting(true) + if (!routing.targets.has(templateId)) { + release = await skipRelease(release, 'credly release skipped: template not mapped') + continue + } + let cursor = releaseCursor(release) + if (!cursor) { + release = await skipRelease(release, 'credly release skipped: bounds unreadable') + continue + } + if (!cursor.since || !cursor.until) { + cursor = openBackfillWindow(cursor) + if ((await ctx.deps.saveReleaseCursor(release, cursor)) === 0) { + release = await ctx.deps.nextRelease() + continue + } + } + const window: Window = { since: cursor.since, until: cursor.until, page: cursor.page } + + const result = await fetchOrHalve(window, templateId) + if ('halved' in result) { + release = await advanceRelease(release, { ...cursor, ...result.halved }) + continue + } + counters.releasePages += 1 + + const badges = await withReplacements(result.page.data) + if (!badges) break + await emitPage(badges) + + let next: CredlyBackfillCursor | undefined + if (drainedPage(window, result.page)) { + next = afterReleaseDrain(cursor) + } else { + next = { ...cursor, ...nextPage(window, result.page.data) } + } + if (!next) { + if (await ctx.deps.completeRelease(release)) { + counters.releases += 1 + ctx.log.info({ badgeTemplateId: templateId }, 'credly template history released') + } + release = await ctx.deps.nextRelease() + } else { + release = await advanceRelease(release, next) + } + } + const releasePending = release !== null + while ( + !releasePending && watermark.backfill && !watermark.until && counters.pagesFetched < MAX_PAGES_PER_RUN && @@ -508,5 +610,5 @@ export async function runBadgePoll(ctx: SyncContext): Promise