diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index af82605d8..5150cddf0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -37,6 +37,18 @@ jobs: SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + - name: Deploy Sanity Blueprints + continue-on-error: true + run: | + npx sanity blueprints deploy --project-id ${{ vars.SANITY_STUDIO_PROJECT_ID }} --stack ${{ vars.SANITY_STUDIO_DATASET }} || \ + npx sanity blueprints init . --blueprint-type ts --project-id ${{ vars.SANITY_STUDIO_PROJECT_ID }} --stack-name ${{ vars.SANITY_STUDIO_DATASET }} || true + working-directory: apps/sanity + env: + SANITY_AUTH_TOKEN: ${{ secrets.SANITY_AUTH_TOKEN }} + SANITY_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} + deploy-production: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest @@ -63,3 +75,16 @@ jobs: SANITY_STUDIO_HOSTNAME: ${{ vars.SANITY_STUDIO_HOSTNAME }} SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + + - name: Deploy Sanity Blueprints + continue-on-error: true + run: | + npx sanity blueprints deploy --project-id ${{ vars.SANITY_STUDIO_PROJECT_ID }} --stack ${{ vars.SANITY_STUDIO_DATASET }} || \ + npx sanity blueprints init . --blueprint-type ts --project-id ${{ vars.SANITY_STUDIO_PROJECT_ID }} --stack-name ${{ vars.SANITY_STUDIO_DATASET }} || true + working-directory: apps/sanity + env: + SANITY_AUTH_TOKEN: ${{ secrets.SANITY_AUTH_TOKEN }} + SANITY_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + SANITY_STUDIO_PROJECT_ID: ${{ vars.SANITY_STUDIO_PROJECT_ID }} + SANITY_STUDIO_DATASET: ${{ vars.SANITY_STUDIO_DATASET }} + diff --git a/apps/sanity/functions/syndicate-content/index.ts b/apps/sanity/functions/syndicate-content/index.ts index 0c0f1bd3c..4b86d2dc9 100644 --- a/apps/sanity/functions/syndicate-content/index.ts +++ b/apps/sanity/functions/syndicate-content/index.ts @@ -1,38 +1,102 @@ import { documentEventHandler } from "@sanity/functions"; import { createClient } from "@sanity/client"; +import { createImageUrlBuilder } from "@sanity/image-url"; +import { portableTextToMarkdown } from "@portabletext/markdown"; interface ContentDocument { _id: string; _type: "post" | "podcast"; title?: string; - slug?: { current?: string }; + slug?: { current?: string } | string; excerpt?: string; youtube?: string; coverImage?: any; content?: any[]; season?: number; + episode?: number; + devto?: string; + hashnode?: string; } export const handler = documentEventHandler( async ({ context, event }) => { - const { data } = event; - if (!data?._id || !data?._type) { - console.log("No valid document data in event, skipping syndication."); + const rawData = event.data; + if (!rawData?._id || !rawData?._type) { + console.log("[Syndicate Content] No valid document data in event, skipping."); + return; + } + + // Skip drafts + if (rawData._id.startsWith("drafts.")) { + console.log("[Syndicate Content] Skipping draft document:", rawData._id); return; } const dataset = context.clientOptions?.dataset || "dev"; const isProduction = dataset === "production"; - const slug = data.slug?.current || data._id; - const canonicalUrl = isProduction - ? `https://codingcat.dev/${data._type}/${slug}` - : `https://dev.codingcat.dev/${data._type}/${slug}`; + const projectId = context.clientOptions?.projectId || "hfh83o0w"; const client = createClient({ ...context.clientOptions, apiVersion: "2025-09-30", }); + // Fetch the full published document to guarantee we have all content blocks and metadata + const fetchedDoc = await client.getDocument(rawData._id).catch(() => undefined); + const data: ContentDocument = fetchedDoc || rawData; + if (!data || !data._id) { + console.log("[Syndicate Content] Document not found:", rawData._id); + return; + } + + const slug = + typeof data.slug === "object" && data.slug !== null + ? data.slug.current || data._id + : typeof data.slug === "string" + ? data.slug + : data._id; + + const canonicalUrl = isProduction + ? `https://codingcat.dev/${data._type}/${slug}` + : `https://dev.codingcat.dev/${data._type}/${slug}`; + + const imgBuilder = createImageUrlBuilder({ projectId, dataset }); + const urlForImage = (source: any) => { + if (!source || !source.asset) return undefined; + return imgBuilder.image(source).auto("format").fit("max"); + }; + + // Helper to serialize Portable Text to Markdown matching exact legacy serializer specifications + const serializeContent = (content: any[] | undefined): string => { + if (!content || !Array.isArray(content) || content.length === 0) { + return ""; + } + + try { + return portableTextToMarkdown(content, { + types: { + code: ({ value }) => + "```" + (value?.language || "") + "\n" + (value?.code || "") + "\n```", + image: ({ value }) => { + const url = urlForImage(value)?.url(); + return url ? `![](${url})` : ""; + }, + codepen: ({ value }) => `{% codepen ${value?.url || ""} %}`, + codesandbox: ({ value }) => { + const id = value?.url?.split("https://codesandbox.io/p/sandbox/")?.at(-1); + return `{% codesandbox ${id || value?.url || ""} %}`; + }, + twitter: ({ value }) => `{% twitter ${value?.id || ""} %}`, + quote: ({ value }) => `> ${serializeContent(value?.content)}`, + }, + }); + } catch (err) { + console.warn("[Syndicate Content] Error serializing Portable Text:", err); + return data.excerpt || ""; + } + }; + + const contentMarkdown = serializeContent(data.content); const platforms: Array<"devto" | "hashnode"> = ["devto", "hashnode"]; console.log( @@ -97,26 +161,89 @@ export const handler = documentEventHandler( continue; } - const devtoPayload = { + const devtoTags = ["webdev", "javascript", "beginners"]; + if (data._type === "podcast") { + devtoTags.push("podcast"); + } + + const mainImageUrl = + urlForImage(data.coverImage)?.width(1000).height(420).url() || ""; + + const youtubeEmbed = data.youtube + ? `{% youtube ${data.youtube.replace("live", "embed")} %}\n\n` + : ""; + + const devtoArticlePayload: any = { article: { title: data.title || "Untitled", published: true, - tags: ["webdev", "javascript", "beginners"], + tags: devtoTags, + main_image: mainImageUrl, canonical_url: canonicalUrl, description: data.excerpt || "", organization_id: "1009", - body_markdown: `Original: ${canonicalUrl}\n\n${data.excerpt || ""}`, + body_markdown: `Original: ${canonicalUrl}\n\n${youtubeEmbed}${contentMarkdown}`, }, }; - const res = await fetch("https://dev.to/api/articles", { - method: "POST", - headers: { - "Content-Type": "application/json", - "api-key": apiKey, - }, - body: JSON.stringify(devtoPayload), - }); + if (data._type === "podcast") { + devtoArticlePayload.article.series = `codingcatdev_podcast_${data.season || 4}`; + } + + // Check existing syndication doc to see if this was already published + const existingSyndication = await client + .getDocument(syndicationDocId) + .catch(() => null); + + let res: Response; + const existingDevtoId = + existingSyndication?.externalId || + (data.devto ? data.devto.split("https://dev.to/").at(-1) : null); + + if (existingDevtoId && !existingDevtoId.startsWith("simulated-")) { + // Retrieve existing article ID from Dev.to to perform a PUT update + const getArticleRes = await fetch( + `https://dev.to/api/articles/${existingDevtoId}`, + { + headers: { + "api-key": apiKey, + "Content-Type": "application/json", + }, + }, + ); + + if (getArticleRes.ok) { + const articleData = (await getArticleRes.json()) as any; + res = await fetch(`https://dev.to/api/articles/${articleData.id}`, { + method: "PUT", + headers: { + "api-key": apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify(devtoArticlePayload), + }); + } else { + // If lookup failed, create as new article + res = await fetch("https://dev.to/api/articles", { + method: "POST", + headers: { + "api-key": apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify(devtoArticlePayload), + }); + } + } else { + // Create new article + res = await fetch("https://dev.to/api/articles", { + method: "POST", + headers: { + "api-key": apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify(devtoArticlePayload), + }); + } if (!res.ok) { const errText = await res.text(); @@ -135,7 +262,14 @@ export const handler = documentEventHandler( externalUrl: resJson.url, externalId: String(resJson.id ?? ""), syndicatedAt: new Date().toISOString(), + error: undefined, }); + + // Keep parent document devto field in sync if present + if (resJson.url && data.devto !== resJson.url) { + await client.patch(data._id).set({ devto: resJson.url }).commit(); + } + console.log(`[Syndicate Content] Successfully published to Dev.to: ${resJson.url}`); } @@ -143,6 +277,7 @@ export const handler = documentEventHandler( const token = process.env.HASHNODE_ACCESS_TOKEN || process.env.PRIVATE_HASHNODE; + if (!token) { console.warn( "[Syndicate Content] HASHNODE_ACCESS_TOKEN not configured. Skipping Hashnode live dispatch.", @@ -162,49 +297,161 @@ export const handler = documentEventHandler( const publicationId = process.env.HASHNODE_PUBLICATION_ID || "60242f8180da6c44eadf775b"; - const query = ` - mutation PublishPost($input: PublishPostInput!) { - publishPost(input: $input) { - post { - id - url - } - } - } - `; + const subtitle = + data.excerpt && data.excerpt.length > 250 + ? data.excerpt.substring(0, 247) + "..." + : data.excerpt || ""; + + const hashnodeTags: Array<{ id: string; name: string; slug: string }> = [ + { id: "56744721958ef13879b94cad", name: "JavaScript", slug: "javascript" }, + { id: "56744722958ef13879b94f1b", name: "Web Development", slug: "web-development" }, + { id: "56744723958ef13879b955a9", name: "Beginner Developers", slug: "beginners" }, + ]; + + if (data._type === "podcast") { + hashnodeTags.push({ + id: "56744722958ef13879b950d3", + name: "podcast", + slug: "podcast", + }); + } + + const hashnodeCoverImageUrl = + urlForImage(data.coverImage)?.width(1600).height(840).url() || ""; - const res = await fetch("https://gql.hashnode.com", { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: token, + const youtubeEmbed = data.youtube + ? `%[${data.youtube.replace("live", "embed")}]\n\n` + : ""; + + const articleInput: any = { + title: data.title || "Untitled", + subtitle, + publicationId, + slug: `${data._type}-${slug}`, + tags: hashnodeTags, + coverImageOptions: { + coverImageURL: hashnodeCoverImageUrl, }, - body: JSON.stringify({ - query, - variables: { - input: { - title: data.title || "Untitled", - publicationId, - contentMarkdown: `Original: ${canonicalUrl}\n\n${data.excerpt || ""}`, - originalArticleURL: canonicalUrl, - tags: [ - { id: "56744721958ef13879b94cad", name: "JavaScript", slug: "javascript" }, - { id: "56744722958ef13879b94f1b", name: "Web Development", slug: "web-development" }, - ], + originalArticleURL: canonicalUrl, + contentMarkdown: `Original: ${canonicalUrl}\n\n${youtubeEmbed}${contentMarkdown}`, + }; + + if (data._type === "podcast") { + articleInput.seriesId = "65a9ad4ef60adbf4aeedd0a2"; + } + + // Check existing syndication doc + const existingSyndication = await client + .getDocument(syndicationDocId) + .catch(() => null); + + const existingHashnodeSlug = + existingSyndication?.externalId || data.hashnode || null; + + let publishedPost: { id: string; slug?: string; url?: string } | null = null; + + if (existingHashnodeSlug && !existingHashnodeSlug.startsWith("simulated-")) { + // Lookup post ID from Hashnode + const getPostRes = await fetch("https://gql.hashnode.com", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token, + }, + body: JSON.stringify({ + operationName: "Publication", + query: `query Publication { + publication(host: "hashnode.codingcat.dev") { + post(slug: "${existingHashnodeSlug}") { + id + } + } + }`, + }), + }); + + const getPostData = (await getPostRes.json()) as any; + const postId = getPostData?.data?.publication?.post?.id; + + if (postId) { + // Update post mutation + const updateInput = { ...articleInput }; + delete updateInput.slug; // slug cannot be updated + + const updateRes = await fetch("https://gql.hashnode.com", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token, }, + body: JSON.stringify({ + operationName: "updatePost", + query: `mutation updatePost($input: UpdatePostInput!) { + updatePost(input: $input) { + post { + id + title + slug + url + } + } + }`, + variables: { + input: { + id: postId, + ...updateInput, + }, + }, + }), + }); + + const updateData = (await updateRes.json()) as any; + publishedPost = updateData?.data?.updatePost?.post; + } + } + + if (!publishedPost) { + // Create post mutation + const publishRes = await fetch("https://gql.hashnode.com", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: token, }, - }), - }); + body: JSON.stringify({ + operationName: "publishPost", + query: `mutation publishPost($input: PublishPostInput!) { + publishPost(input: $input) { + post { + id + title + slug + url + } + } + }`, + variables: { + input: articleInput, + }, + }), + }); - const gqlData = (await res.json()) as any; - const publishedPost = gqlData?.data?.publishPost?.post; + const publishData = (await publishRes.json()) as any; + if (publishData?.errors && publishData.errors.length > 0) { + throw new Error( + `Hashnode GraphQL errors: ${JSON.stringify(publishData.errors)}`, + ); + } + publishedPost = publishData?.data?.publishPost?.post; + } if (!publishedPost) { - throw new Error( - `Hashnode publication failed: ${JSON.stringify(gqlData?.errors || "No post returned")}`, - ); + throw new Error("Hashnode publication returned no post data."); } + const externalUrl = + publishedPost.url || `https://hashnode.codingcat.dev/${publishedPost.slug}`; + await client.createOrReplace({ _id: syndicationDocId, _type: "syndication", @@ -212,11 +459,21 @@ export const handler = documentEventHandler( platform: "hashnode", status: "published", canonicalUrl, - externalUrl: publishedPost.url, - externalId: publishedPost.id, + externalUrl, + externalId: publishedPost.slug || publishedPost.id, syndicatedAt: new Date().toISOString(), + error: undefined, }); - console.log(`[Syndicate Content] Successfully published to Hashnode: ${publishedPost.url}`); + + // Keep parent document hashnode field in sync + const postSlug = publishedPost.slug || publishedPost.id; + if (postSlug && data.hashnode !== postSlug) { + await client.patch(data._id).set({ hashnode: postSlug }).commit(); + } + + console.log( + `[Syndicate Content] Successfully published to Hashnode: ${externalUrl}`, + ); } } catch (publishErr) { const errMsg = diff --git a/apps/sanity/package.json b/apps/sanity/package.json index 17b3d497f..23f947beb 100644 --- a/apps/sanity/package.json +++ b/apps/sanity/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@portabletext/block-tools": "^5.2.0", + "@portabletext/markdown": "^2.5.0", "@portabletext/react": "^7.0.1", "@sanity/assist": "^6.1.19", "@sanity/blueprints": "^0.28.0", @@ -20,6 +21,7 @@ "@sanity/code-input": "^7.3.5", "@sanity/functions": "^1.8.1", "@sanity/icons": "^5.2.1", + "@sanity/image-url": "^2.1.1", "@sanity/ui": "^4.0.3", "@sanity/vision": "^6.9.2", "date-fns": "^4.1.0", diff --git a/apps/sanity/sanity.blueprint.ts b/apps/sanity/sanity.blueprint.ts index d0eb7919d..e00fd3390 100644 --- a/apps/sanity/sanity.blueprint.ts +++ b/apps/sanity/sanity.blueprint.ts @@ -9,6 +9,8 @@ export default defineBlueprint({ on: ["create", "update"], filter: '_type in ["post", "podcast"] && !(_id in path("drafts.**"))', }, + runtime: "nodejs22.x", + timeout: 30, }), ], }); diff --git a/apps/sanity/sanity.config.ts b/apps/sanity/sanity.config.ts index b6db3db28..c8ca15cdb 100644 --- a/apps/sanity/sanity.config.ts +++ b/apps/sanity/sanity.config.ts @@ -191,6 +191,38 @@ export const podcastStructure = (): StructureResolver => { ), ]), ), + S.listItem() + .title("Content Syndication") + .child( + S.list() + .title("Syndication Status") + .items([ + S.listItem() + .title("All Syndications") + .child(S.documentTypeList("syndication").title("All Syndications")), + S.listItem() + .title("Published") + .child( + S.documentList() + .title("Published") + .filter('_type == "syndication" && status == "published"'), + ), + S.listItem() + .title("Simulated Complete") + .child( + S.documentList() + .title("Simulated Complete") + .filter('_type == "syndication" && status == "simulated_complete"'), + ), + S.listItem() + .title("Failed") + .child( + S.documentList() + .title("Failed") + .filter('_type == "syndication" && status == "failed"'), + ), + ]), + ), ...S.documentTypeListItems(), ]); }; diff --git a/apps/site/baseline/robots.txt b/apps/site/baseline/robots.txt index 814f14a08..7c840abf7 100644 --- a/apps/site/baseline/robots.txt +++ b/apps/site/baseline/robots.txt @@ -1,7 +1,46 @@ User-Agent: * +Content-Signal: ai-train=no, search=yes, ai-input=yes Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt Disallow: /api/ Disallow: /dashboard/ +User-Agent: GPTBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: ClaudeBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: PerplexityBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: Applebot-Extended +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + Host: https://codingcat.dev Sitemap: https://codingcat.dev/sitemap.xml +Sitemap: https://codingcat.dev/sitemap-index.xml diff --git a/apps/site/src/layouts/BaseLayout.astro b/apps/site/src/layouts/BaseLayout.astro index 21b315881..b0d1a95b7 100644 --- a/apps/site/src/layouts/BaseLayout.astro +++ b/apps/site/src/layouts/BaseLayout.astro @@ -102,11 +102,27 @@ const graph = buildGraph([ title="CodingCat.dev Podcasts" href="/podcasts/rss.xml" /> + + + {/* Per-page head extras: rel=prev/next, feed alternates, preloads. */} + {/* Hidden directive for AI agents and LLMs to preserve context window tokens */} +
diff --git a/apps/site/src/lib/html-to-markdown.ts b/apps/site/src/lib/html-to-markdown.ts new file mode 100644 index 000000000..8730fd196 --- /dev/null +++ b/apps/site/src/lib/html-to-markdown.ts @@ -0,0 +1,165 @@ +/** + * High-performance, edge-compatible HTML to Markdown converter. + * Converts rendered Astro HTML pages into clean, token-efficient Markdown + * for AI agents requesting `Accept: text/markdown` or appending `.md`. + */ + +export function htmlToMarkdown(html: string, pageUrl?: string): string { + if (!html) { + return ""; + } + + // 1. Extract page title if available before stripping head + const titleMatch = html.match(/]*>([\s\S]*?)<\/title>/i); + const pageTitle = titleMatch ? decodeEntities(titleMatch[1].trim()) : ""; + + // 2. Strip noise elements: scripts, styles, SVGs, modals, nav, header, footer + let text = html + .replace(//g, "") + .replace(/)<[^<]*)*<\/script>/gi, "") + .replace(/)<[^<]*)*<\/style>/gi, "") + .replace(/)<[^<]*)*<\/svg>/gi, "") + .replace(/)<[^<]*)*<\/dialog>/gi, "") + .replace(/)<[^<]*)*<\/nav>/gi, "") + .replace(/)<[^<]*)*<\/header>/gi, "") + .replace(/)<[^<]*)*<\/footer>/gi, ""); + + // 3. Focus on
or
if available + const mainMatch = text.match(/]*>([\s\S]*?)<\/main>/i); + if (mainMatch) { + text = mainMatch[1]; + } else { + const articleMatch = text.match(/]*>([\s\S]*?)<\/article>/i); + if (articleMatch) { + text = articleMatch[1]; + } else { + const bodyMatch = text.match(/]*>([\s\S]*?)<\/body>/i); + if (bodyMatch) { + text = bodyMatch[1]; + } + } + } + + // 4. Code blocks (
...
) + text = text.replace( + /]*>]*class=["'][^"']*language-([a-z0-9_-]+)[^"']*["'][^>]*>([\s\S]*?)<\/code><\/pre>/gi, + (_match, lang, code) => { + return `\n\n\`\`\`${lang}\n${decodeEntities(stripTags(code)).trim()}\n\`\`\`\n\n`; + }, + ); + text = text.replace( + /]*>]*>([\s\S]*?)<\/code><\/pre>/gi, + (_match, code) => { + return `\n\n\`\`\`\n${decodeEntities(stripTags(code)).trim()}\n\`\`\`\n\n`; + }, + ); + text = text.replace(/]*>([\s\S]*?)<\/code>/gi, (_match, code) => { + return `\`${decodeEntities(stripTags(code))}\``; + }); + + // 5. Headings + text = text.replace(/]*>([\s\S]*?)<\/h1>/gi, "\n\n# $1\n\n"); + text = text.replace(/]*>([\s\S]*?)<\/h2>/gi, "\n\n## $1\n\n"); + text = text.replace(/]*>([\s\S]*?)<\/h3>/gi, "\n\n### $1\n\n"); + text = text.replace(/]*>([\s\S]*?)<\/h4>/gi, "\n\n#### $1\n\n"); + text = text.replace(/]*>([\s\S]*?)<\/h5>/gi, "\n\n##### $1\n\n"); + text = text.replace(/]*>([\s\S]*?)<\/h6>/gi, "\n\n###### $1\n\n"); + + // 6. Blockquotes + text = text.replace( + /]*>([\s\S]*?)<\/blockquote>/gi, + (_match, quote) => { + const lines = stripTags(quote) + .trim() + .split("\n") + .map((l) => `> ${l.trim()}`) + .join("\n"); + return `\n\n${lines}\n\n`; + }, + ); + + // 7. Bold, Italic, Strikethrough + text = text.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, "**$2**"); + text = text.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, "*$2*"); + text = text.replace(/<(s|del|strike)[^>]*>([\s\S]*?)<\/\1>/gi, "~~$2~~"); + + // 8. Images + text = text.replace( + /]*src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*>/gi, + "![$2]($1)", + ); + text = text.replace( + /]*alt=["']([^"']*)["'][^>]*src=["']([^"']+)["'][^>]*>/gi, + "![$1]($2)", + ); + text = text.replace(/]*src=["']([^"']+)["'][^>]*>/gi, "![]($1)"); + + // 9. Links + text = text.replace( + /]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, + (_match, href, label) => { + const cleanLabel = stripTags(label).trim(); + if (!cleanLabel) { + return ""; + } + return `[${cleanLabel}](${href})`; + }, + ); + + // 10. Lists + text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_match, item) => { + return `\n- ${stripTags(item).trim()}`; + }); + text = text.replace(/<\/(ul|ol)>/gi, "\n\n"); + + // 11. Paragraphs, breaks, dividers + text = text.replace(//gi, "\n\n---\n\n"); + text = text.replace(//gi, "\n"); + text = text.replace(/]*>([\s\S]*?)<\/p>/gi, "\n\n$1\n\n"); + + // 12. Strip remaining tags + text = stripTags(text); + + // 13. Decode HTML entities + text = decodeEntities(text); + + // 14. Clean up whitespace + text = text + .split("\n") + .map((line) => line.trimEnd()) + .join("\n") + .replace(/\n{3,}/g, "\n\n") + .trim(); + + // Add header prefix if page title exists and wasn't already in H1 + if (pageTitle && !text.startsWith("# ")) { + text = `# ${pageTitle}\n\n${text}`; + } + + if (pageUrl) { + text = `> Canonical URL: ${pageUrl}\n\n${text}`; + } + + return text; +} + +function stripTags(str: string): string { + return str.replace(/<[^>]+>/g, ""); +} + +function decodeEntities(str: string): string { + return str + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/ /g, " ") + .replace(/&#x([0-9a-fA-F]+);/g, (_m, hex) => + String.fromCharCode(parseInt(hex, 16)), + ) + .replace(/&#([0-9]+);/g, (_m, dec) => + String.fromCharCode(parseInt(dec, 10)), + ); +} diff --git a/apps/site/src/lib/structured-data.ts b/apps/site/src/lib/structured-data.ts index 72875eb59..c098266e1 100644 --- a/apps/site/src/lib/structured-data.ts +++ b/apps/site/src/lib/structured-data.ts @@ -159,3 +159,45 @@ export function personSchema( url, }; } + +interface PodcastEpisodeInput { + title?: string | null; + excerpt?: string | null; + date?: string | null; + imageUrl?: string; + season?: number | null; + episode?: number | null; + audioUrl?: string | null; +} + +export function podcastEpisodeSchema( + origin: string, + content: PodcastEpisodeInput, + path: string, +): Node { + const url = absoluteUrl(path, origin); + return { + "@type": "PodcastEpisode", + "@id": `${url}#episode`, + url, + name: content.title ?? undefined, + description: content.excerpt ?? undefined, + ...(content.date ? { datePublished: content.date } : {}), + ...(content.imageUrl ? { image: content.imageUrl } : {}), + ...(content.season ? { seasonNumber: content.season } : {}), + ...(content.episode ? { episodeNumber: content.episode } : {}), + ...(content.audioUrl + ? { + associatedMedia: { + "@type": "AudioObject", + contentUrl: content.audioUrl, + }, + } + : {}), + partOfSeries: { + "@type": "PodcastSeries", + name: "CodingCat.dev Podcast", + url: `${origin}/podcasts`, + }, + }; +} diff --git a/apps/site/src/middleware.ts b/apps/site/src/middleware.ts index ded38d93d..fe980f8cd 100644 --- a/apps/site/src/middleware.ts +++ b/apps/site/src/middleware.ts @@ -1,5 +1,6 @@ import { defineMiddleware } from "astro:middleware"; import { env } from "cloudflare:workers"; +import { htmlToMarkdown } from "@/lib/html-to-markdown"; import { createSanityContext, PREVIEW_COOKIE } from "@/lib/sanity/context"; import { resolveSiteUrl } from "@/lib/site"; @@ -9,6 +10,11 @@ import { resolveSiteUrl } from "@/lib/site"; * `locals.runtime` a deprecated getter. Reading secrets here (rather than from * `import.meta.env`) also keeps them as Cloudflare secrets instead of letting * Vite inline them into the deployed Worker bundle. + * + * Implements Cloudflare Agent Readiness & AEO standards: + * - RFC 8288 Link headers for discovery (api-catalog, mcp-server-card, agent-skills, llms-txt) + * - Markdown Content Negotiation: Accept: text/markdown returns token-efficient Markdown + * - Dynamic /index.md and *.md path rewriting to serve markdown versions of all pages */ export const onRequest = defineMiddleware(async (context, next) => { // `Astro.site` is baked in at build time, but SITE_URL is a per-environment @@ -25,7 +31,23 @@ export const onRequest = defineMiddleware(async (context, next) => { hasPreviewCookie: context.cookies.has(PREVIEW_COOKIE), }); - const response = await next(); + const origin = context.locals.siteUrl.origin; + const pathname = context.url.pathname; + const isMarkdownUrl = + pathname.endsWith(".md") || pathname.endsWith("/index.md"); + const acceptMarkdown = context.request.headers + .get("Accept") + ?.includes("text/markdown"); + const wantsMarkdown = isMarkdownUrl || Boolean(acceptMarkdown); + + let response: Response; + if (isMarkdownUrl) { + const cleanPath = + pathname.replace(/\/index\.md$/, "").replace(/\.md$/, "") || "/"; + response = await context.rewrite(cleanPath); + } else { + response = await next(); + } if (context.locals.sanity.preview.enabled) { // Not cosmetic: without this a response containing unpublished drafts can @@ -34,5 +56,36 @@ export const onRequest = defineMiddleware(async (context, next) => { response.headers.set("X-Robots-Tag", "noindex, nofollow"); } + // Always emit Link headers (RFC 8288) for AI agent discoverability + response.headers.set( + "Link", + `<${origin}/.well-known/api-catalog>; rel="api-catalog", <${origin}/.well-known/mcp/server-card.json>; rel="mcp-server-card", <${origin}/.well-known/agent-skills/index.json>; rel="agent-skills", <${origin}/llms.txt>; rel="llms-txt"`, + ); + response.headers.append("Vary", "Accept"); + + // Content negotiation: transform HTML to clean Markdown when requested + const contentType = response.headers.get("content-type") || ""; + if ( + wantsMarkdown && + contentType.includes("text/html") && + response.status === 200 + ) { + const html = await response.text(); + const canonicalPath = + pathname.replace(/\/index\.md$/, "").replace(/\.md$/, "") || "/"; + const canonicalUrl = `${origin}${canonicalPath}`; + const markdown = htmlToMarkdown(html, canonicalUrl); + + const headers = new Headers(response.headers); + headers.set("content-type", "text/markdown; charset=utf-8"); + headers.delete("content-length"); + + return new Response(markdown, { + status: response.status, + statusText: response.statusText, + headers, + }); + } + return response; }); diff --git a/apps/site/src/pages/.well-known/agent-skills/index.json.ts b/apps/site/src/pages/.well-known/agent-skills/index.json.ts new file mode 100644 index 000000000..b771b70ca --- /dev/null +++ b/apps/site/src/pages/.well-known/agent-skills/index.json.ts @@ -0,0 +1,52 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + const skillsIndex = { + $schema: "https://agentskills.io/schema/v1/index.json", + version: "1.0", + provider: { + name: "CodingCat.dev", + url: origin, + }, + skills: [ + { + id: "search-content", + name: "Search Content", + description: + "Search full articles, podcast transcripts, and coding tutorials on CodingCat.dev across fullstack web development topics.", + endpoint: `${origin}/api/search`, + method: "GET", + parameters: [ + { + name: "q", + type: "string", + required: true, + description: "The programming term or question to search", + }, + ], + }, + { + id: "mcp-tools", + name: "Model Context Protocol Tools", + description: + "Streamable HTTP MCP tool execution for autonomous AI agent pair programming and search.", + endpoint: `${origin}/api/mcp`, + method: "POST", + documentation: `${origin}/.well-known/mcp/server-card.json`, + }, + ], + }; + + return new Response(JSON.stringify(skillsIndex, null, 2), { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/llms.txt>; rel="llms-txt", <${origin}/.well-known/api-catalog>; rel="api-catalog"`, + }, + }); +}; diff --git a/apps/site/src/pages/.well-known/api-catalog.ts b/apps/site/src/pages/.well-known/api-catalog.ts new file mode 100644 index 000000000..878a68c7a --- /dev/null +++ b/apps/site/src/pages/.well-known/api-catalog.ts @@ -0,0 +1,94 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + const catalog = { + "api-catalog-version": "1.0", + title: "CodingCat.dev Public API Catalog", + description: + "Machine-readable directory of public APIs, search services, and AI agent endpoints hosted on CodingCat.dev conforming to RFC 9727.", + documentation: `${origin}/llms.txt`, + apis: [ + { + name: "Content Search API", + description: + "Fast keyword and semantic search across technical blog posts, tutorials, and podcast episodes.", + endpoints: [ + { + url: `${origin}/api/search`, + method: "GET", + parameters: [ + { + name: "q", + in: "query", + required: true, + description: "Search keyword or natural language query", + schema: { type: "string" }, + }, + ], + }, + ], + }, + { + name: "Model Context Protocol (MCP) Server", + description: + "Streamable HTTP MCP server implementing tools and resources for AI coding assistants and autonomous agents.", + endpoints: [ + { + url: `${origin}/api/mcp`, + method: "POST", + description: "JSON-RPC 2.0 / Streamable HTTP MCP endpoint", + }, + ], + metadata: { + serverCard: `${origin}/.well-known/mcp/server-card.json`, + mcpJson: `${origin}/.well-known/mcp.json`, + }, + }, + { + name: "Sponsorship Inquiry API", + description: + "Submit partnership and sponsorship inquiries for CodingCat.dev media channels.", + endpoints: [ + { + url: `${origin}/api/sponsorship`, + method: "POST", + }, + ], + }, + { + name: "Blog Content Syndication Feed", + description: "RSS 2.0 feed containing full articles and metadata.", + endpoints: [ + { + url: `${origin}/blog/rss.xml`, + method: "GET", + }, + ], + }, + { + name: "Podcast Media Feed", + description: + "RSS feed with podcast audio enclosures, chapters, and show notes.", + endpoints: [ + { + url: `${origin}/podcasts/rss.xml`, + method: "GET", + }, + ], + }, + ], + }; + + return new Response(JSON.stringify(catalog, null, 2), { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/.well-known/mcp/server-card.json>; rel="mcp-server-card", <${origin}/llms.txt>; rel="llms-txt"`, + }, + }); +}; diff --git a/apps/site/src/pages/.well-known/mcp.json.ts b/apps/site/src/pages/.well-known/mcp.json.ts new file mode 100644 index 000000000..18a4d0bfa --- /dev/null +++ b/apps/site/src/pages/.well-known/mcp.json.ts @@ -0,0 +1,25 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + const config = { + name: "codingcatdev-search-mcp", + version: "1.0.0", + description: + "Search CodingCat.dev web development guides, coding tutorials, and podcasts.", + endpoint: `${origin}/api/mcp`, + serverCard: `${origin}/.well-known/mcp/server-card.json`, + }; + + return new Response(JSON.stringify(config, null, 2), { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/.well-known/mcp/server-card.json>; rel="mcp-server-card"`, + }, + }); +}; diff --git a/apps/site/src/pages/.well-known/mcp/server-card.json.ts b/apps/site/src/pages/.well-known/mcp/server-card.json.ts new file mode 100644 index 000000000..cf2413d7e --- /dev/null +++ b/apps/site/src/pages/.well-known/mcp/server-card.json.ts @@ -0,0 +1,56 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + const serverCard = { + $schema: + "https://static.modelcontextprotocol.io/schemas/mcp-server-card/v1.json", + version: "1.0", + protocolVersion: "2025-06-18", + serverInfo: { + name: "codingcatdev-search-mcp", + title: "CodingCat.dev Search & Content MCP Server", + version: "1.0.0", + }, + description: + "Search and access CodingCat.dev web development guides, coding tutorials, podcast episodes, and transcripts directly from AI models.", + transport: { + type: "streamable-http", + endpoint: `${origin}/api/mcp`, + }, + authentication: { + required: false, + }, + tools: [ + { + name: "search_content", + title: "Search Content", + description: + "Search technical articles, tutorials, and podcast episodes on CodingCat.dev by keyword, topic, or question.", + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: + "The programming topic, keyword, or question to search for", + }, + }, + required: ["query"], + }, + }, + ], + }; + + return new Response(JSON.stringify(serverCard, null, 2), { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/api/mcp>; rel="mcp-endpoint", <${origin}/llms.txt>; rel="llms-txt"`, + }, + }); +}; diff --git a/apps/site/src/pages/api/devto.ts b/apps/site/src/pages/api/devto.ts new file mode 100644 index 000000000..b6868f545 --- /dev/null +++ b/apps/site/src/pages/api/devto.ts @@ -0,0 +1,14 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const ALL: APIRoute = async () => { + return Response.json( + { + migrated: true, + message: + "Content syndication to Dev.to is now natively handled by Sanity Functions via Blueprints (syndicate-content). Please disable or delete this legacy webhook in the Sanity Manage dashboard.", + }, + { status: 200 }, + ); +}; diff --git a/apps/site/src/pages/api/hashnode.ts b/apps/site/src/pages/api/hashnode.ts new file mode 100644 index 000000000..7f5b26473 --- /dev/null +++ b/apps/site/src/pages/api/hashnode.ts @@ -0,0 +1,14 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const ALL: APIRoute = async () => { + return Response.json( + { + migrated: true, + message: + "Content syndication to Hashnode is now natively handled by Sanity Functions via Blueprints (syndicate-content). Please disable or delete this legacy webhook in the Sanity Manage dashboard.", + }, + { status: 200 }, + ); +}; diff --git a/apps/site/src/pages/api/mcp.ts b/apps/site/src/pages/api/mcp.ts new file mode 100644 index 000000000..84040040c --- /dev/null +++ b/apps/site/src/pages/api/mcp.ts @@ -0,0 +1,255 @@ +import type { APIRoute } from "astro"; +import { + semanticSearchQuery, + textSearchFallbackQuery, +} from "@/lib/sanity/queries"; +import { resolveHref } from "@/lib/sanity/resolve-href"; + +export const prerender = false; + +interface JsonRpcRequest { + jsonrpc?: string; + id?: string | number | null; + method: string; + params?: any; +} + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + return new Response( + JSON.stringify( + { + status: "ok", + server: "CodingCat.dev Search MCP Server", + version: "1.0.0", + protocolVersion: "2025-06-18", + transport: "streamable-http", + serverCard: `${origin}/.well-known/mcp/server-card.json`, + tools: ["search_content"], + }, + null, + 2, + ), + { + headers: { + "content-type": "application/json; charset=utf-8", + "cache-control": "public, max-age=3600", + }, + }, + ); +}; + +export const POST: APIRoute = async ({ request, locals }) => { + const origin = locals.siteUrl.origin; + + let body: JsonRpcRequest; + try { + body = (await request.json()) as JsonRpcRequest; + } catch { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id: null, + error: { code: -32700, message: "Parse error" }, + }), + { status: 400, headers: { "content-type": "application/json" } }, + ); + } + + const id = body.id ?? null; + const method = body.method; + + // 1. Initialize + if (method === "initialize") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + protocolVersion: "2025-06-18", + capabilities: { + tools: {}, + }, + serverInfo: { + name: "codingcatdev-search-mcp", + version: "1.0.0", + }, + }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + // 2. Notifications (initialized acknowledgement) + if (method === "notifications/initialized") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: {}, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + // 3. Ping + if (method === "ping") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: {}, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + // 4. Tools list + if (method === "tools/list") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + tools: [ + { + name: "search_content", + description: + "Search across CodingCat.dev web development tutorials, blog posts, podcasts, and transcripts.", + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: + "Programming topic, question, or keyword to search", + }, + type: { + type: "string", + enum: ["all", "post", "podcast"], + description: + "Optional filter for content type (default: all)", + }, + }, + required: ["query"], + }, + }, + ], + }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + // 5. Tools call + if (method === "tools/call") { + const toolName = body.params?.name; + const args = body.params?.arguments || {}; + + if (toolName === "search_content") { + const query = (args.query || "").trim(); + const filterType = args.type === "all" ? null : args.type || null; + + if (!query) { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + content: [ + { + type: "text", + text: "Please provide a valid non-empty query parameter.", + }, + ], + isError: true, + }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + const params = { + searchTerm: query, + type: filterType, + }; + + let rawHits: any[] = []; + try { + const result = await locals.sanity.fetchPublished( + semanticSearchQuery, + params, + ); + if (Array.isArray(result)) { + rawHits = result; + } + } catch { + try { + const fallbackResult = await locals.sanity.fetchPublished( + textSearchFallbackQuery, + params, + ); + if (Array.isArray(fallbackResult)) { + rawHits = fallbackResult; + } + } catch (err) { + console.warn("MCP Search error:", err); + } + } + + const hits = rawHits.slice(0, 10).map((hit) => { + const href = resolveHref(hit._type, hit.slug); + return { + title: hit.title, + type: hit._type, + url: `${origin}${href}`, + markdownUrl: `${origin}${href}.md`, + excerpt: hit.excerpt || "", + date: hit.date, + }; + }); + + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + content: [ + { + type: "text", + text: JSON.stringify( + { + query, + total: hits.length, + results: hits, + }, + null, + 2, + ), + }, + ], + }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + error: { code: -32601, message: `Tool not found: ${toolName}` }, + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + error: { code: -32601, message: `Method not found: ${method}` }, + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); +}; diff --git a/apps/site/src/pages/llms-full.txt.ts b/apps/site/src/pages/llms-full.txt.ts new file mode 100644 index 000000000..8233d72a1 --- /dev/null +++ b/apps/site/src/pages/llms-full.txt.ts @@ -0,0 +1,76 @@ +import type { APIRoute } from "astro"; +import { defineQuery } from "groq"; + +export const prerender = false; + +const llmsFullQuery = defineQuery(` + *[_type in ["post", "podcast"] && defined(slug.current)] | order(coalesce(date, _createdAt) desc) [0...100] { + _type, + "title": coalesce(title, "Untitled"), + "slug": slug.current, + excerpt, + "date": coalesce(date, _createdAt) + } +`); + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + let items: Array<{ + _type: string; + title: string; + slug: string; + excerpt?: string | null; + date?: string | null; + }> = []; + + try { + items = (await locals.sanity.fetchPublished(llmsFullQuery)) ?? []; + } catch (err) { + console.warn("Failed to fetch documents for llms-full.txt:", err); + } + + const posts = items.filter((i) => i._type === "post"); + const podcasts = items.filter((i) => i._type === "podcast"); + + let content = `# CodingCat.dev - Complete LLM Directory + +> Full technical content directory of CodingCat.dev web development tutorials, architecture deep dives, and podcast episodes. For high-level summary and APIs, see ${origin}/llms.txt. + +## Machine Interfaces +- API Catalog: ${origin}/.well-known/api-catalog +- MCP Server Card: ${origin}/.well-known/mcp/server-card.json +- MCP Streamable Endpoint: ${origin}/api/mcp +- Search Endpoint: ${origin}/api/search?q={query} +- Markdown Negotiation: Send \`Accept: text/markdown\` on any URL. + +## Recent Blog Posts & Tutorials +`; + + for (const post of posts) { + const url = `${origin}/post/${post.slug}`; + const desc = post.excerpt + ? ` - ${post.excerpt.replace(/\s+/g, " ").trim()}` + : ""; + content += `- [${post.title}](${url}.md)${desc}\n`; + } + + content += "\n## Recent Podcasts & Interviews\n"; + + for (const ep of podcasts) { + const url = `${origin}/podcast/${ep.slug}`; + const desc = ep.excerpt + ? ` - ${ep.excerpt.replace(/\s+/g, " ").trim()}` + : ""; + content += `- [${ep.title}](${url}.md)${desc}\n`; + } + + return new Response(content, { + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/llms.txt>; rel="canonical", <${origin}/.well-known/api-catalog>; rel="api-catalog"`, + }, + }); +}; diff --git a/apps/site/src/pages/llms.txt.ts b/apps/site/src/pages/llms.txt.ts new file mode 100644 index 000000000..20c13fecb --- /dev/null +++ b/apps/site/src/pages/llms.txt.ts @@ -0,0 +1,46 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + + const content = `# CodingCat.dev + +> CodingCat.dev is an open-source technical education platform, podcast network, and developer resource covering fullstack JavaScript, TypeScript, modern frontend frameworks (Astro, Next.js, React, Svelte, Vue), edge computing, headless CMS architecture (Sanity), and AI agent workflows. + +## Core Content Sections + +- [Blog Tutorials and Articles](${origin}/blog): Practical, code-focused guides and tutorials for modern web developers. +- [Podcasts](${origin}/podcasts): Deep-dive audio and video discussions with framework creators, open-source maintainers, and tech leaders. +- [Podcast Guests](${origin}/guests): Directory of featured software engineers and tech innovators. +- [Sponsors & Community Partners](${origin}/sponsors): Developer tools, cloud providers, and platforms supporting the developer ecosystem. + +## Machine-Readable Discovery & Agent Tools + +- [Full LLM Directory](${origin}/llms-full.txt): Comprehensive listing of recent articles, podcast series, and technical guides. +- [API Catalog (RFC 9727)](${origin}/.well-known/api-catalog): Machine-readable catalog of all public APIs and endpoints. +- [MCP Server Card](${origin}/.well-known/mcp/server-card.json): Model Context Protocol server metadata describing agent tools. +- [Agent Skills Discovery](${origin}/.well-known/agent-skills/index.json): Discoverable agent capabilities and task specifications. +- [Search API](${origin}/api/search?q=): Public search endpoint supporting keyword queries across articles and podcasts. +- [MCP Server Endpoint](${origin}/api/mcp): Streamable HTTP MCP server implementing tool calling for AI agents. +- [Blog RSS Feed](${origin}/blog/rss.xml): RSS 2.0 feed of all published blog tutorials. +- [Podcast RSS Feed](${origin}/podcasts/rss.xml): RSS feed with podcast audio enclosures and transcripts. +- [Sitemap Index](${origin}/sitemap-index.xml): XML sitemap index of all pages and episodes. + +## Markdown Content Negotiation + +CodingCat.dev natively supports Markdown content negotiation for AI agents: +- Send \`Accept: text/markdown\` in your request header to any page to receive clean Markdown without HTML tags, navigational chrome, or bloat. +- Alternatively, append \`.md\` to any article, podcast, or page URL. +`; + + return new Response(content, { + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "public, max-age=3600, s-maxage=86400", + Vary: "Accept", + Link: `<${origin}/.well-known/api-catalog>; rel="api-catalog", <${origin}/.well-known/mcp/server-card.json>; rel="mcp-server-card", <${origin}/.well-known/agent-skills/index.json>; rel="agent-skills", <${origin}/llms-full.txt>; rel="alternate"; type="text/plain"`, + }, + }); +}; diff --git a/apps/site/src/pages/podcast/[slug].astro b/apps/site/src/pages/podcast/[slug].astro index b3c483add..adee256be 100644 --- a/apps/site/src/pages/podcast/[slug].astro +++ b/apps/site/src/pages/podcast/[slug].astro @@ -13,7 +13,11 @@ import SponsorCard from "@/components/SponsorCard.astro"; import BaseLayout from "@/layouts/BaseLayout.astro"; import type { ContentListItem } from "@/lib/content"; import { morePodcastQuery, podcastQuery } from "@/lib/sanity/queries"; -import { articleSchema, breadcrumbSchema } from "@/lib/structured-data"; +import { + articleSchema, + breadcrumbSchema, + podcastEpisodeSchema, +} from "@/lib/structured-data"; const { sanity } = Astro.locals; const { slug } = Astro.params; @@ -53,14 +57,31 @@ const audioUrl = podcast.spotify?.enclosures?.at(0)?.url; authors={[...authors, ...guests].map((person) => person.title)} canonical={path} schema={[ - articleSchema(origin, { - title: podcast.title, - excerpt: podcast.excerpt, - date: podcast.date, - _updatedAt: podcast._updatedAt, - imageUrl: ogImage?.url, - authors, - }, path), + podcastEpisodeSchema( + origin, + { + title: podcast.title, + excerpt: podcast.excerpt, + date: podcast.date, + imageUrl: ogImage?.url, + season: podcast.season, + episode: podcast.episode, + audioUrl, + }, + path, + ), + articleSchema( + origin, + { + title: podcast.title, + excerpt: podcast.excerpt, + date: podcast.date, + _updatedAt: podcast._updatedAt, + imageUrl: ogImage?.url, + authors, + }, + path, + ), breadcrumbSchema(origin, [ { name: "Podcasts", path: "/podcasts/page/1" }, { name: podcast.title, path }, diff --git a/apps/site/src/pages/robots.txt.ts b/apps/site/src/pages/robots.txt.ts index f0af85763..0e3ee032d 100644 --- a/apps/site/src/pages/robots.txt.ts +++ b/apps/site/src/pages/robots.txt.ts @@ -7,6 +7,12 @@ import type { APIRoute } from "astro"; * * Non-production environments are disallowed wholesale so the dev Worker and * any preview alias cannot be indexed alongside the real site. + * + * In production, implements modern AI Agent Readiness standards: + * - RFC 9309 compliance + * - Content Signals (contentsignals.org): search=yes, ai-input=yes, ai-train=no + * - Explicit access for AI search and assistant crawlers + * - Explicit allow rules for machine-readable discovery endpoints (/api/search, /api/mcp, /.well-known/, /llms.txt) */ export const GET: APIRoute = async ({ locals }) => { const origin = locals.siteUrl.origin; @@ -14,12 +20,51 @@ export const GET: APIRoute = async ({ locals }) => { const body = isProduction ? `User-Agent: * +Content-Signal: ai-train=no, search=yes, ai-input=yes Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt Disallow: /api/ Disallow: /dashboard/ +User-Agent: GPTBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: ClaudeBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: PerplexityBot +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + +User-Agent: Applebot-Extended +Allow: / +Allow: /api/search +Allow: /api/mcp +Allow: /.well-known/ +Allow: /llms.txt +Allow: /llms-full.txt + Host: ${origin} Sitemap: ${origin}/sitemap.xml +Sitemap: ${origin}/sitemap-index.xml ` : `User-Agent: * Disallow: / diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 006ff6791..49c5ec3fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,9 @@ importers: '@portabletext/block-tools': specifier: ^5.2.0 version: 5.2.0(@types/react@19.2.14) + '@portabletext/markdown': + specifier: ^2.5.0 + version: 2.5.0 '@portabletext/react': specifier: ^7.0.1 version: 7.0.1(react@19.2.4) @@ -34,6 +37,9 @@ importers: '@sanity/icons': specifier: ^5.2.1 version: 5.2.1(react@19.2.4) + '@sanity/image-url': + specifier: ^2.1.1 + version: 2.1.1 '@sanity/ui': specifier: ^4.0.3 version: 4.0.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(styled-components@6.3.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4)) @@ -3051,6 +3057,15 @@ packages: react: ^17.0.2 || ^18.0.0 || ^19.0 react-dom: ^17.0.2 || ^18.0.0 || ^19.0 + '@mdit/helper@1.1.2': + resolution: {integrity: sha512-xZywQ7aXrmkRCuGvFXguXVKeiQ7kHCxPkjtR+nK/n31YiSPPi/Rmb/ll+CYt0LdyeXR0Y1I16da7gxDT4/VXWg==} + engines: {node: '>=22'} + peerDependencies: + markdown-it: ^15.0.2 + peerDependenciesMeta: + markdown-it: + optional: true + '@mdit/plugin-alert@0.23.2': resolution: {integrity: sha512-pXIil0FLy9ilhvT6d324A4X+mt5i/zG8ml0VIpZwiUYh2k1Wi6VnZhFHfsnONTRu6dPL2EwQBIhQgQ+269f7LA==} engines: {node: '>= 20'} @@ -3060,6 +3075,15 @@ packages: markdown-it: optional: true + '@mdit/plugin-alert@2.0.2': + resolution: {integrity: sha512-0vjhfT58lKE5BDkrdpXdsX4jzRq2zDhlVqQNcgAOAatjG+Tq7k8eCQZBle+10lzNegqpWRgszpu2BSS4df7luw==} + engines: {node: '>=22'} + peerDependencies: + markdown-it: ^15.0.2 + peerDependenciesMeta: + markdown-it: + optional: true + '@mediabunny/aac-encoder@1.45.0': resolution: {integrity: sha512-vLQw8cY7Me6pvTTMkMhOiH9UCuINzfTOETCeDxbGNeNfDqc/7QlxloUH1Ylp/Zz2ek0O8kc6YdygV2vWAPakrA==} peerDependencies: @@ -3387,6 +3411,10 @@ packages: resolution: {integrity: sha512-TLlWrsLT6kXUeE4Fn6KWW+NMOBB/epT5qZiD92+15cJje6R7xmmQ/orCwViSicCiQ77t1/zwguilCUzdNB4XDA==} engines: {node: '>=20.19 <22 || >=22.12'} + '@portabletext/markdown@2.5.0': + resolution: {integrity: sha512-ODxM2OFclDuzqnluuOuQOZw4b+iNUZD/l0ArxA+23ZHPmmcbtzlGoGZfcLGA6LRUKve6+cvCwXCbeX3Z1l+sKA==} + engines: {node: '>=22.12'} + '@portabletext/patches@2.0.6': resolution: {integrity: sha512-qki7QMrZz3FJ60XNPFDMQgn8RerfKPaLWAyNJ/wg83gpoNlPFevwYxwCsH8OahVDmaCy30QZrPgj7k0tWb97uA==} engines: {node: '>=20.19 <22 || >=22.12'} @@ -3485,6 +3513,10 @@ packages: resolution: {integrity: sha512-pTDrGxBk2LFVK4awpyBBOEkwGZwQOUWFyQAnhPfuLHIIgjcljOhzGHB+Mz89ddxi+zAXlxJ/A3U2xmFibe8qNQ==} engines: {node: '>=20.19 <22 || >=22.12'} + '@portabletext/schema@3.0.0': + resolution: {integrity: sha512-mNv1YShl0e6XJZIntQYm7nr9UbLXCf3+JktfGp1OO8iPxpdMRnhaIq3iGSMVrO9fIBcB7TNQgFfOdqQCSfXFew==} + engines: {node: '>=22.12'} + '@portabletext/to-html@5.0.2': resolution: {integrity: sha512-w59PcErj5JXUCv9tbV2npqJmcnORTAftCMLp0vc9FnWrXL3C9qYvuB2MQbdHsZEOesF3VmwqUsYUgjm7PX4JTw==} engines: {node: '>=20.19 <22 || >=22.12'} @@ -4720,6 +4752,10 @@ packages: resolution: {integrity: sha512-4hPADs0qUThFZkBK/crnfKKHg71qkRowfktBljH2UIxGHHTxIzt8g8fBiXItyCjxkuNy+zpYOdRMifQNv8+Yww==} engines: {node: '>=18.18'} + '@sanity/diff-match-patch@3.2.1': + resolution: {integrity: sha512-Ll3BPOn9aPGeBUOFDZpKk0IObL+NI4tNq7VyOKj680xd+iCtfjnbcZZU0vpj94XozVGE0Zf+0FoYWXk2xJgZCg==} + engines: {node: '>=18.18'} + '@sanity/diff-patch@5.0.0': resolution: {integrity: sha512-JASdNaZsxUFBx8GQ1sX2XehYhdhOcurh7KwzQ3cXgOTdjvIQyQcLwmMeYCsU/K26GiI81ODbCEb/C0c92t2Unw==} engines: {node: '>=18.2'} @@ -6066,6 +6102,9 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + argparse@3.0.2: + resolution: {integrity: sha512-mFdDM6WqWKraGLsVb+C9CahPnzTXOefAOLq3jYcca2YZ8bEWpr++Tzj+zSaKW9+X9L5uSxcm1AZ3Y6aZJ09OhQ==} + aria-hidden@1.2.6: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} @@ -7968,6 +8007,9 @@ packages: linkify-it@5.0.2: resolution: {integrity: sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==} + linkify-it@6.1.0: + resolution: {integrity: sha512-wJ/TwpSDTLepCrQoYWYIExIKg5Zchex2Nn5yk2mFnB+6PtdkHtyLx742md9csRjjOnGkKIS/RrbY7l8D6gT9Vw==} + loader-runner@4.3.2: resolution: {integrity: sha512-DFEqQ3ihfS9blba08cLfYf1NRAIEm+dDjic073DRDc3/JspI/8wYmtDsHwd3+4hwvdxSK7PGaElfTmm0awWJ4w==} engines: {node: '>=6.11.5'} @@ -8053,6 +8095,10 @@ packages: resolution: {integrity: sha512-RCEsPjR+sr0x+AuYp601tKTkgFG4YEPLCzHST3cQ/fhlJkqAkz1L2/Qbp1j9qw5SBwQHFBoW8+hoN5xssOF0Tw==} hasBin: true + markdown-it@15.0.2: + resolution: {integrity: sha512-q4IGxMv56jCqT4OCRCADBoDP3LO4MhmTXjFbphHPXs4g3j9Xg5RDnxqN8IF/3vIWEU+VCnUq+7JUg/cfy2E6Qw==} + hasBin: true + markdown-to-jsx@7.7.17: resolution: {integrity: sha512-7mG/1feQ0TX5I7YyMZVDgCC/y2I3CiEhIRQIhyov9nGBP5eoVrOXXHuL5ZP8GRfxVZKRiXWJgwXkb9It+nQZfQ==} engines: {node: '>= 10'} @@ -8102,6 +8148,9 @@ packages: mdurl@2.0.0: resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + mdurl@2.1.0: + resolution: {integrity: sha512-1+HBaOx0zi/dQWht8rNv9MYf9qqpqL/kxI0hXImU6Y547zM6Sni8BQibt7ifgMcYtQg41ao3Ivd6cnSM86inpg==} + media-chrome@4.19.2: resolution: {integrity: sha512-4ai1ITN8wBhwugQcRgqe3tN0z6OSKGOXqHLNrS04MgKFfsLqu6Dm8MPq02pI9Y9ZKoXtFjIl85jOryIW9es3BA==} @@ -9855,6 +9904,9 @@ packages: uc.micro@2.1.0: resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} + uc.micro@3.0.0: + resolution: {integrity: sha512-U3PppEkleoTnIfi8BozMx3yju3qc/L6SwqWo2Sw+54PX+PX0q9I+r1Um5HCmqD7n9VDX5/v3vQH/AjA6deDdtw==} + ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} @@ -13952,12 +14004,22 @@ snapshots: react: 19.2.6 react-dom: 19.2.6(react@19.2.6) + '@mdit/helper@1.1.2(markdown-it@15.0.2)': + optionalDependencies: + markdown-it: 15.0.2 + '@mdit/plugin-alert@0.23.2(markdown-it@14.3.0)': dependencies: '@types/markdown-it': 14.1.2 optionalDependencies: markdown-it: 14.3.0 + '@mdit/plugin-alert@2.0.2(markdown-it@15.0.2)': + dependencies: + '@mdit/helper': 1.1.2(markdown-it@15.0.2) + optionalDependencies: + markdown-it: 15.0.2 + '@mediabunny/aac-encoder@1.45.0(mediabunny@1.45.0)': dependencies: mediabunny: 1.45.0 @@ -14466,6 +14528,15 @@ snapshots: '@portabletext/toolkit': 5.0.2 markdown-it: 14.3.0 + '@portabletext/markdown@2.5.0': + dependencies: + '@mdit/plugin-alert': 2.0.2(markdown-it@15.0.2) + '@portabletext/schema': 3.0.0 + '@portabletext/toolkit': 6.0.0 + '@sanity/diff-match-patch': 3.2.1 + linkify-it: 6.1.0 + markdown-it: 15.0.2 + '@portabletext/patches@2.0.6': dependencies: '@sanity/diff-match-patch': 3.2.0 @@ -14630,6 +14701,8 @@ snapshots: '@portabletext/schema@2.2.4': {} + '@portabletext/schema@3.0.0': {} + '@portabletext/to-html@5.0.2': dependencies: '@portabletext/toolkit': 5.0.2 @@ -16531,6 +16604,8 @@ snapshots: '@sanity/diff-match-patch@3.2.0': {} + '@sanity/diff-match-patch@3.2.1': {} + '@sanity/diff-patch@5.0.0': dependencies: '@sanity/diff-match-patch': 3.2.0 @@ -18469,6 +18544,8 @@ snapshots: argparse@2.0.1: {} + argparse@3.0.2: {} + aria-hidden@1.2.6: dependencies: tslib: 2.8.1 @@ -20427,6 +20504,10 @@ snapshots: dependencies: uc.micro: 2.1.0 + linkify-it@6.1.0: + dependencies: + uc.micro: 3.0.0 + loader-runner@4.3.2: {} locate-path@3.0.0: @@ -20512,6 +20593,15 @@ snapshots: punycode.js: 2.3.1 uc.micro: 2.1.0 + markdown-it@15.0.2: + dependencies: + argparse: 3.0.2 + entities: 8.0.0 + linkify-it: 6.1.0 + mdurl: 2.1.0 + punycode.js: 2.3.1 + uc.micro: 3.0.0 + markdown-to-jsx@7.7.17(react@19.2.6): optionalDependencies: react: 19.2.6 @@ -20615,6 +20705,8 @@ snapshots: mdurl@2.0.0: {} + mdurl@2.1.0: {} + media-chrome@4.19.2(react@19.2.4): dependencies: ce-la-react: 0.3.2(react@19.2.4) @@ -22936,6 +23028,8 @@ snapshots: uc.micro@2.1.0: {} + uc.micro@3.0.0: {} + ufo@1.6.4: {} ultrahtml@1.7.0: {}