feat: re-read the history of a credly template released from hold

services/apps/connectors_worker/src/bin/dry-run-credly-badge-poll.ts

@@ -125,6 +125,9 @@ async function main(): Promise<void> {
     saveBadges: async () => undefined,
     listDueExpiries: async () => [],
     markExpiriesEmitted: async () => 0,
+    nextRelease: async () => null,
+    saveReleaseCursor: async () => 0,
+    completeRelease: async () => 0,
   }
 
   const outcome = await runBadgePoll({

services/apps/connectors_worker/src/bin/resolve-credly-templates.ts

@@ -10,29 +10,33 @@ import {
   mintCredlyAccessToken,
 } from '@crowd/connectors/src/connectors/credly/oauth'
 import {
+  listActiveCredlyTemplateMappings,
   listCredlyTemplateMappings,
   listHeldCredlyTemplateMappings,
 } from '@crowd/data-access-layer/src/credly'
+import { queueCredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases'
 import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { getServiceLogger } from '@crowd/logging'
 import { SnowflakeClient } from '@crowd/snowflake'
 
 import { activeHeld, buildHeldQueueReport, postHeldQueueReport } from '../credlyHeldQueueReport'
 import {
+  CREDLY_HISTORY_START,
   applyDecisions,
   buildPlan,
   loadCatalogue,
   loadCredlyTemplates,
   loadSubprojectSegments,
+  releasesToQueue,
 } from '../credlyTemplateResolve'
 
 const log = getServiceLogger()
 const HELD_QUEUE_PRINTED = 20
 
 function usage(): never {
   log.error(
-    'Usage: resolve-credly-templates --integration-id <uuid> [--apply [--notify | --digest]] [--out <file>]',
+    'Usage: resolve-credly-templates --integration-id <uuid> [--apply [--notify | --digest]] [--out <file>] | --release <badge template id>',
   )
   process.exit(1)
 }
@@ -54,6 +58,7 @@ function takeSwitch(argv: string[], flag: string): boolean {
 }
 
 interface Args {
+  release?: string
   integrationId: string
   apply: boolean
   notify: 'changes' | 'digest' | null
@@ -62,6 +67,11 @@ interface Args {
 
 function parseArgs(rawArgv: string[]): Args {
   const argv = rawArgv.filter((arg) => arg !== '--')
+  const release = takeFlag(argv, '--release')
+  if (release) {
+    if (argv.length > 0) usage()
+    return { release, integrationId: '', apply: false, notify: null, outFile: '' }
+  }
   const integrationId = takeFlag(argv, '--integration-id')
   const apply = takeSwitch(argv, '--apply')
   const digest = takeSwitch(argv, '--digest')
@@ -75,10 +85,28 @@ function parseArgs(rawArgv: string[]): Args {
 }
 
 async function main(): Promise<void> {
-  const { integrationId, apply, notify, outFile } = parseArgs(process.argv.slice(2))
+  const { release, integrationId, apply, notify, outFile } = parseArgs(process.argv.slice(2))
   const db = await getDbConnection(WRITE_DB_CONFIG())
   const qx = pgpQx(db)
 
+  if (release) {
+    const active = await listActiveCredlyTemplateMappings(qx)
+    if (!active.some((mapping) => mapping.badgeTemplateId === release)) {
+      log.error(
+        { badgeTemplateId: release },
+        'credly template is not mapped to a live integration, nothing to release',
+      )
+      process.exit(1)
+    }
+    await queueCredlyTemplateRelease(qx, {
+      badgeTemplateId: release,
+      floor: CREDLY_HISTORY_START,
+      top: new Date().toISOString(),
+    })
+    log.info({ badgeTemplateId: release }, 'credly template history queued for release')
+    return
+  }
+
   const credential = await getCredential(qx, integrationId)
   assertCredentialKind(credential, 'credly-oauth-client')
   const { token } = await mintCredlyAccessToken(credential)
@@ -153,7 +181,13 @@ async function main(): Promise<void> {
         'the credly listing changed while paging: templates not seen are left as they are',
       )
     }
-    const result = await applyDecisions(qx, decisions, listing.complete ? missing : [], log)
+    const result = await applyDecisions(
+      qx,
+      decisions,
+      listing.complete ? missing : [],
+      log,
+      releasesToQueue(plan, templates, new Date()),
+    )
     const heldQueue = await listHeldCredlyTemplateMappings(qx)
     log.info(
       {

services/apps/connectors_worker/src/credlyTemplateResolve.test.ts

@@ -5,10 +5,12 @@ import type { ResolvedTemplate } from '@crowd/connectors/src/connectors/credly/m
 import type { ICredlyStoredTemplateMapping } from '@crowd/data-access-layer/src/credly'
 
 import {
+  CREDLY_HISTORY_START,
   buildPlan,
   loadCatalogue,
   loadCredlyTemplates,
   loadSubprojectSegments,
+  releasesToQueue,
 } from './credlyTemplateResolve'
 
 function decision(id: string, overrides: Partial<ResolvedTemplate> = {}): ResolvedTemplate {
@@ -221,8 +223,8 @@ describe('loadCredlyTemplates', () => {
 
     expect(listing.complete).toBe(true)
     expect(listing.templates).toEqual([
-      { id: 'a', name: 'A', badgeCount: 3, url: null },
-      { id: 'b', name: 'B', badgeCount: null, url: null },
+      { id: 'a', name: 'A', badgeCount: 3, url: null, createdAt: null },
+      { id: 'b', name: 'B', badgeCount: null, url: null, createdAt: null },
     ])
   })
 
@@ -234,3 +236,86 @@ describe('loadCredlyTemplates', () => {
     expect((await loadCredlyTemplates(http, 'org', log)).complete).toBe(false)
   })
 })
+
+describe('releasesToQueue', () => {
+  const now = new Date('2026-11-02T12:00:00.000Z')
+  const plan = (
+    entries: {
+      badgeTemplateId: string
+      kind: string
+      before: string | null
+      after: string | null
+    }[],
+  ) =>
+    ({
+      entries: entries.map((e) => ({ ...e, badgeTemplateName: e.badgeTemplateId })),
+      counts: {},
+    }) as never
+
+  it('queues the templates that reached a project, new or released, from their creation date', () => {
+    const releases = releasesToQueue(
+      plan([
+        {
+          badgeTemplateId: 'lfc131',
+          kind: 'changed',
+          before: 'held:no_project',
+          after: 'mapped:green-software',
+        },
+        { badgeTemplateId: 'old', kind: 'changed', before: 'held:no_match', after: 'mapped:cncf' },
+        { badgeTemplateId: 'moved', kind: 'changed', before: 'mapped:k8s', after: 'mapped:cncf' },
+        {
+          badgeTemplateId: 'locked',
+          kind: 'locked',
+          before: 'held:no_match',
+          after: 'mapped:cncf',
+        },
+        { badgeTemplateId: 'excluded', kind: 'changed', before: 'held:test', after: 'excluded' },
+        {
+          badgeTemplateId: 'unexcluded',
+          kind: 'changed',
+          before: 'excluded',
+          after: 'mapped:finops',
+        },
+        { badgeTemplateId: 'fresh', kind: 'new', before: null, after: 'mapped:k8s' },
+      ]),
+      [
+        { id: 'lfc131', name: 'LFC131', createdAt: '2022-10-25T19:41:32.416Z' },
+        { id: 'old', name: 'Old', createdAt: '2018-04-19T00:00:00.000Z' },
+        { id: 'unexcluded', name: 'Unexcluded', createdAt: '2026-11-02T12:00:00.000Z' },
+        { id: 'fresh', name: 'Fresh', createdAt: '2026-11-01T09:00:00.000Z' },
+      ],
+      now,
+    )
+
+    expect(releases).toEqual([
+      {
+        badgeTemplateId: 'lfc131',
+        floor: '2022-10-25T19:41:32.416Z',
+        top: '2026-11-02T12:00:00.000Z',
+      },
+      { badgeTemplateId: 'old', floor: CREDLY_HISTORY_START, top: '2026-11-02T12:00:00.000Z' },
+      {
+        badgeTemplateId: 'unexcluded',
+        floor: '2026-11-02T11:59:59.000Z',
+        top: '2026-11-02T12:00:00.000Z',
+      },
+      {
+        badgeTemplateId: 'fresh',
+        floor: '2026-11-01T09:00:00.000Z',
+        top: '2026-11-02T12:00:00.000Z',
+      },
+    ])
+  })
+
+  it('falls back to the history start when the template has no creation date', () => {
+    const releases = releasesToQueue(
+      plan([
+        { badgeTemplateId: 'x', kind: 'changed', before: 'held:no_match', after: 'mapped:k8s' },
+      ]),
+      [{ id: 'x', name: 'X', createdAt: null }],
+      now,
+    )
+
+    expect(releases[0].floor).toBe(CREDLY_HISTORY_START)
+  })
+})

services/apps/connectors_worker/src/credlyTemplateResolve.ts

@@ -15,6 +15,8 @@ import {
   findOrCreateCredlyIntegration,
   upsertCredlyTemplateDecision,
 } from '@crowd/data-access-layer/src/credly'
+import type { ICredlyTemplateReleaseInput } from '@crowd/data-access-layer/src/credly/releases'
+import { queueCredlyTemplateRelease } from '@crowd/data-access-layer/src/credly/releases'
 import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
 import { listActiveSubprojectSegments } from '@crowd/data-access-layer/src/segments'
 import type { Logger } from '@crowd/logging'
@@ -77,6 +79,7 @@ export async function loadCredlyTemplates(
         name: template.name,
         badgeCount: typeof count === 'number' && Number.isFinite(count) ? count : null,
         url: template.url ?? null,
+        createdAt: template.created_at ?? null,
       })
     }
     if (result.data.length === 0 || page >= result.metadata.total_pages) break
@@ -183,19 +186,55 @@ export interface ApplyResult {
   outcomes: Record<CredlyDecisionOutcome, number>
   integrationsCreated: number
   deactivated: number
+  releasesQueued: number
 }
 
+// Credly moved to its current platform in March 2019; nothing older carries a state_updated_at.
+export const CREDLY_HISTORY_START = '2019-03-01T00:00:00.000Z'
+
+export function releaseFloor(createdAt: string | null | undefined, top: Date): string {
+  const createdMs = Date.parse(createdAt ?? '')
+  const startMs = Date.parse(CREDLY_HISTORY_START)
+  const floorMs = Number.isFinite(createdMs) ? Math.max(createdMs, startMs) : startMs
+  return new Date(Math.min(floorMs, top.getTime() - 1000)).toISOString()
+}
+
+export function releasesToQueue(
+  plan: ResolvePlan,
+  templates: TemplateToResolve[],
+  now: Date,
+): ICredlyTemplateReleaseInput[] {
+  const createdAtById = new Map(templates.map((template) => [template.id, template.createdAt]))
+  return plan.entries
+    .filter(
+      (entry) =>
+        (entry.kind === 'changed' || entry.kind === 'new') &&
+        !entry.before?.startsWith('mapped:') &&
+        entry.after?.startsWith('mapped:'),
+    )
+    .map((entry) => ({
+      badgeTemplateId: entry.badgeTemplateId,
+      floor: releaseFloor(createdAtById.get(entry.badgeTemplateId), now),
+      top: now.toISOString(),
+    }))
+}
+
+// A release is queued in the same transaction as the decision that frees the template: a run that
+// dies in between leaves the template held, and the next run decides it again.
 export async function applyDecisions(
   qx: QueryExecutor,
   decisions: ResolvedTemplate[],
   missing: PlanEntry[],
   log: Logger,
+  releases: ICredlyTemplateReleaseInput[] = [],
 ): Promise<ApplyResult> {
   const result: ApplyResult = {
     outcomes: { inserted: 0, updated: 0, unchanged: 0, locked: 0 },
     integrationsCreated: 0,
     deactivated: 0,
+    releasesQueued: 0,
   }
+  const releaseById = new Map(releases.map((release) => [release.badgeTemplateId, release]))
   const integrationBySegment = new Map<string, string>()
   for (const decision of decisions) {
     let integrationId: string | null = null
@@ -208,7 +247,15 @@ export async function applyDecisions(
         integrationBySegment.set(decision.segmentId, integrationId)
       }
     }
-    const outcome = await upsertCredlyTemplateDecision(qx, { ...decision, integrationId })
+    const release = releaseById.get(decision.badgeTemplateId)
+    const outcome = await qx.tx(async (tx) => {
+      const stored = await upsertCredlyTemplateDecision(tx, { ...decision, integrationId })
+      if (release && (stored === 'inserted' || stored === 'updated')) {
+        await queueCredlyTemplateRelease(tx, release)
+        result.releasesQueued += 1
+      }
+      return stored
+    })
     result.outcomes[outcome] += 1
     if (outcome === 'locked') {
       log.warn(

services/libs/connectors/src/connectors/credly/apiClient.ts

@@ -78,6 +78,7 @@ export const credlyBadgeTemplateListingSchema = z
     type_category: z.string().nullable().optional(),
     global_activity_url: z.string().nullable().optional(),
     reporting_tags: z.array(z.string()).nullable().optional(),
+    created_at: z.string().nullable().optional(),
   })
   .passthrough()
 

services/libs/connectors/src/connectors/credly/deps.test.ts

@@ -94,6 +94,18 @@ describe('createCredlyDeps', () => {
     expect(mockedStale).toHaveBeenCalledTimes(1)
   })
 
+  it('reads the mapping again when a fresh routing is asked for', async () => {
+    const deps = createCredlyDeps({ qx, integrationId: 'anchor' })
+
+    const first = await deps.loadTemplateRouting()
+    const fresh = await deps.loadTemplateRouting(true)
+    const cached = await deps.loadTemplateRouting()
+
+    expect(fresh).not.toBe(first)
+    expect(cached).toBe(fresh)
+    expect(mockedActive).toHaveBeenCalledTimes(2)
+  })
+
   it('locks a template through the mapping table', async () => {
     vi.mocked(lockCredlyTemplateMapping).mockResolvedValueOnce(true)
     const deps = createCredlyDeps({ qx, integrationId: 'anchor' })

services/libs/connectors/src/connectors/credly/deps.ts

@@ -14,6 +14,15 @@ import {
   markCredlyExpiriesEmitted,
   upsertCredlyBadges,
 } from '@crowd/data-access-layer/src/credly/badges'
+import type {
+  ICredlyTemplateRelease,
+  ICredlyTemplateReleaseCursor,
+} from '@crowd/data-access-layer/src/credly/releases'
+import {
+  completeCredlyTemplateRelease,
+  nextPendingCredlyTemplateRelease,
+  saveCredlyTemplateReleaseCursor,
+} from '@crowd/data-access-layer/src/credly/releases'
 
 import type { DepsFactoryInput } from '../../types'
 
@@ -29,19 +38,26 @@ export interface CredlyTemplateRouting {
 }
 
 export interface CredlyDeps {
-  loadTemplateRouting(): Promise<CredlyTemplateRouting>
+  loadTemplateRouting(fresh?: boolean): Promise<CredlyTemplateRouting>
   lockTemplate(badgeTemplateId: string, target: CredlyTemplateTarget): Promise<boolean>
   findBadges(badgeIds: string[]): Promise<ICredlyBadgeRow[]>
   saveBadges(badges: ICredlyBadgeInput[]): Promise<void>
   listDueExpiries(now: Date, limit: number, after?: ICredlyExpiryCursor): Promise<ICredlyBadgeRow[]>
   markExpiriesEmitted(badgeIds: string[]): Promise<number>
+  nextRelease(): Promise<ICredlyTemplateRelease | null>
+  saveReleaseCursor(
+    release: ICredlyTemplateRelease,
+    cursor: ICredlyTemplateReleaseCursor,
+  ): Promise<number>
+  completeRelease(release: ICredlyTemplateRelease): Promise<number>
 }
 
 export function createCredlyDeps({ qx }: DepsFactoryInput): CredlyDeps {
   let routing: Promise<CredlyTemplateRouting> | null = null
 
   return {
-    loadTemplateRouting() {
+    loadTemplateRouting(fresh = false) {
+      if (fresh) routing = null
       routing ??= Promise.all([
         listActiveCredlyTemplateMappings(qx),
         listStaleCredlyTemplateMappings(qx),
@@ -71,5 +87,14 @@ export function createCredlyDeps({ qx }: DepsFactoryInput): CredlyDeps {
     markExpiriesEmitted(badgeIds) {
       return markCredlyExpiriesEmitted(qx, badgeIds)
     },
+    nextRelease() {
+      return nextPendingCredlyTemplateRelease(qx)
+    },
+    saveReleaseCursor(release, cursor) {
+      return saveCredlyTemplateReleaseCursor(qx, release, cursor)
+    },
+    completeRelease(release) {
+      return completeCredlyTemplateRelease(qx, release)
+    },
   }
 }