feat: Allow markdown content via 'md' suffix

docs/Setup/Configuration.md

@@ -123,7 +123,7 @@ Browser spellchecking uses the `<html lang="en">` value in
 | EnableTagDiscoveryPanel                       | boolean        | The default value is `true`. Enables the Tag Discovery Panel, which helps users discover topics by browsing popular tags.                                                        |
 | EnableBrokenLinkChecker                       | boolean        | The default value is `true`. Checks all external links of published blog posts once a day and lists broken ones on the "Broken links" admin page.                               |
 | ShowCodeBlockLanguage                         | boolean        | The default value is `true`. Shows the language of a fenced code block (e.g. `csharp`) in the header of the code block.                                                        |
-| EnableMarkdownForLlms                         | boolean        | The default value is `false`. Serves published blog posts as markdown to clients (like LLMs) that ask for `text/markdown`. See [Markdown for LLMs](#markdown-for-llms).        |
+| EnableMarkdownForLlms                         | boolean        | The default value is `false`. Serves published blog posts as markdown to clients (like LLMs) that ask for `text/markdown` or append `.md` to the url, and serves `/llms.txt`. See [Markdown for LLMs](#markdown-for-llms).        |
 
 ### Markdown for LLMs
 
@@ -153,4 +153,7 @@ The content...
 - Markdown is only served if `text/markdown` is weighted at least as high as HTML in the `Accept` header (`text/html`, `text/*` or `*/*`), so `text/markdown, text/html;q=0.9` returns markdown, while `text/html, text/markdown;q=0.9` and a regular browser request return HTML.
 - Only published blog posts are served. Unpublished, scheduled and unknown blog posts return `404` with a markdown body.
 - Blog post responses carry `Vary: Accept` so caches and CDNs keep the HTML and markdown variants apart.
-- If the setting is `false` or missing, the `Accept` header is ignored and HTML is always returned.
+- Appending `.md` to a blog post url (`/blogPost/{id}.md` or `/blogPost/{id}/{slug}.md`) returns markdown regardless of the `Accept` header, for agents that can't set headers.
+- HTML blog post pages advertise the markdown version via a `Link: </blogPost/{id}/{slug}.md>; rel="alternate"; type="text/markdown"` header and a `<link rel="alternate" type="text/markdown">` tag.
+- `/llms.txt` (see [llmstxt.org](https://llmstxt.org)) lists the blog name, introduction and all published blog posts with links to their markdown version.
+- If the setting is `false` or missing, the `Accept` header is ignored and HTML is always returned, `/blogPost/{id}/{slug}.md` renders the HTML page (the slug is ignored) while `/blogPost/{id}.md` shows the not found page, no `Link` header or `<link>` tag is emitted and `/llms.txt` returns `404`.

src/LinkDotNet.Blog.Domain/BlogPost.cs

@@ -36,18 +36,20 @@ public sealed partial class BlogPost : Entity
 
     public int ReadingTimeInMinutes { get; private set; }
 
-    public string Slug => GenerateSlug();
+    public string Slug => CreateSlug(Title);
 
     public string? AuthorName { get; private set; }
 
-    private string GenerateSlug()
+    public static string CreateSlug(string title)
     {
-        if (string.IsNullOrWhiteSpace(Title))
+        ArgumentNullException.ThrowIfNull(title);
+
+        if (string.IsNullOrWhiteSpace(title))
         {
-            return Title;
+            return title;
         }
 
-        var normalizedTitle = Title.Normalize(NormalizationForm.FormD);
+        var normalizedTitle = title.Normalize(NormalizationForm.FormD);
         var stringBuilder = new StringBuilder();
 
         foreach (var c in normalizedTitle.Where(c => CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark))

src/LinkDotNet.Blog.Web/Controller/LlmsTxtController.cs

@@ -0,0 +1,91 @@
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using LinkDotNet.Blog.Domain;
+using LinkDotNet.Blog.Infrastructure.Persistence;
+using LinkDotNet.Blog.Web.Features;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
+using Microsoft.Extensions.Options;
+
+namespace LinkDotNet.Blog.Web.Controller;
+
+[Route("llms.txt")]
+[EnableRateLimiting("ip")]
+public sealed partial class LlmsTxtController : ControllerBase
+{
+    private readonly IOptions<ApplicationConfiguration> applicationConfiguration;
+    private readonly IOptions<Introduction> introductionConfiguration;
+    private readonly IRepository<BlogPost> blogPostRepository;
+
+    public LlmsTxtController(
+        IOptions<ApplicationConfiguration> applicationConfiguration,
+        IOptions<Introduction> introductionConfiguration,
+        IRepository<BlogPost> blogPostRepository)
+    {
+        this.applicationConfiguration = applicationConfiguration;
+        this.introductionConfiguration = introductionConfiguration;
+        this.blogPostRepository = blogPostRepository;
+    }
+
+    [ResponseCache(Duration = 1200)]
+    [HttpGet]
+    public async Task<IActionResult> GetLlmsTxt()
+    {
+        if (!applicationConfiguration.Value.EnableMarkdownForLlms)
+        {
+            return NotFound();
+        }
+
+        var baseUrl = $"{Request.Scheme}://{Request.Host}{Request.PathBase}";
+        var sb = new StringBuilder();
+        sb.Append("# ").Append(applicationConfiguration.Value.BlogName).Append("\n\n");
+        var description = MarkdownConverter.ToPlainString(introductionConfiguration.Value.Description);
+        if (description is not null)
+        {
+            sb.Append("> ").Append(EscapeMarkdown(description)).Append("\n\n");
+        }
+
+        sb.Append("## Blog posts\n\n");
+        var blogPosts = await blogPostRepository.GetAllByProjectionAsync(
+            b => new LlmsTxtEntry { Id = b.Id, Title = b.Title, ShortDescription = b.ShortDescription },
+            f => f.IsPublished,
+            b => b.UpdatedDate);
+        foreach (var blogPost in blogPosts)
+        {
+            sb.Append("- [").Append(EscapeMarkdown(blogPost.Title)).Append("](").Append(baseUrl).Append("/blogPost/").Append(blogPost.Id);
+            var slug = BlogPost.CreateSlug(blogPost.Title);
+            if (!string.IsNullOrEmpty(slug))
+            {
+                sb.Append('/').Append(slug);
+            }
+
+            sb.Append(".md)");
+            var shortDescription = MarkdownConverter.ToPlainString(blogPost.ShortDescription);
+            if (shortDescription is not null)
+            {
+                sb.Append(": ").Append(EscapeMarkdown(shortDescription));
+            }
+
+            sb.Append('\n');
+        }
+
+        return Content(sb.ToString(), "text/plain; charset=utf-8", Encoding.UTF8);
+    }
+
+    private static string EscapeMarkdown(string text)
+        => MarkdownSpecialCharacters().Replace(text.ReplaceLineEndings(" "), @"\$0");
+
+    [GeneratedRegex(@"[\\`*_\[\]<>&]")]
+    private static partial Regex MarkdownSpecialCharacters();
+
+    // Member-init instead of a positional record: RavenDB rejects constructor projections
+    private sealed class LlmsTxtEntry
+    {
+        public required string Id { get; init; }
+
+        public required string Title { get; init; }
+
+        public required string ShortDescription { get; init; }
+    }
+}

src/LinkDotNet.Blog.Web/Features/Components/OgData.razor

@@ -19,6 +19,7 @@
     {
         <meta name="description" property="og:description" content="@Description" />
     }
+    @ChildContent
 </HeadContent>
 @code {
 
@@ -37,6 +38,10 @@
     [Parameter]
     public string? CanonicalRelativeUrl { get; set; }
 
+    // HeadContent is last-one-wins, so pages add extra head tags through here instead of their own HeadContent
+    [Parameter]
+    public RenderFragment? ChildContent { get; set; }
+
     [Parameter]
     public string Robots { get; set; } = "max-image-preview:large";
 

src/LinkDotNet.Blog.Web/Features/ShowBlogPost/BlogPostMarkdownMiddleware.cs

@@ -14,6 +14,7 @@ namespace LinkDotNet.Blog.Web.Features.ShowBlogPost;
 public sealed class BlogPostMarkdownMiddleware
 {
     private const string MarkdownMediaType = "text/markdown";
+    private const string MarkdownSuffix = ".md";
     private static readonly string[] HtmlMediaTypes = ["text/html", "text/*", "*/*"];
 
     private readonly RequestDelegate next;
@@ -41,15 +42,17 @@ public async Task InvokeAsync(
         }
 
         context.Response.Headers.Append(HeaderNames.Vary, HeaderNames.Accept);
-        if (!PrefersMarkdown(context.Request.GetTypedHeaders().Accept))
+        if (!IsMarkdownUrl(context.Request) && !PrefersMarkdown(context.Request.GetTypedHeaders().Accept))
         {
+            var markdownUrl = context.Request.PathBase.Add(context.Request.Path).ToUriComponent().TrimEnd('/') + MarkdownSuffix;
+            context.Response.Headers.Append(HeaderNames.Link, $"<{markdownUrl}>; rel=\"alternate\"; type=\"{MarkdownMediaType}\"");
             await next(context);
             return;
         }
 
         // A content type keeps UseStatusCodePagesWithReExecute from swapping the 404 for the HTML NotFound page
         context.Response.ContentType = $"{MarkdownMediaType}; charset=utf-8";
-        var blogPost = await blogPostRepository.GetByIdAsync(context.Request.RouteValues["blogPostId"]?.ToString() ?? string.Empty);
+        var blogPost = await blogPostRepository.GetByIdAsync(GetBlogPostId(context.Request));
         if (blogPost is not { IsPublished: true })
         {
             context.Response.StatusCode = StatusCodes.Status404NotFound;
@@ -78,6 +81,17 @@ public static bool PrefersMarkdown(IList<MediaTypeHeaderValue> accept)
             .Select(a => (double?)(a.Quality ?? 1))
             .Max();
 
+    private static bool IsMarkdownUrl(HttpRequest request)
+        => request.Path.Value?.EndsWith(MarkdownSuffix, StringComparison.OrdinalIgnoreCase) == true;
+
+    private static string GetBlogPostId(HttpRequest request)
+    {
+        var blogPostId = request.RouteValues["blogPostId"]?.ToString() ?? string.Empty;
+        return request.RouteValues["slug"] is null && IsMarkdownUrl(request)
+            ? blogPostId[..^MarkdownSuffix.Length]
+            : blogPostId;
+    }
+
     private static bool IsBlogPostPage(HttpContext context)
         => context.GetEndpoint()?.Metadata.GetMetadata<ComponentTypeMetadata>()?.Type == typeof(ShowBlogPostPage);
 

src/LinkDotNet.Blog.Web/Features/ShowBlogPost/ShowBlogPostPage.razor

@@ -32,7 +32,12 @@ else if (BlogPost is not null)
 	        AbsolutePreviewImageUrl="@OgDataImage"
 	        Description="@(Markdown.ToPlainText(BlogPost.ShortDescription))"
 	        Keywords="@BlogPost.TagsAsString"
-	        CanonicalRelativeUrl="@BlogPostCanoncialUrl" />
+	        CanonicalRelativeUrl="@BlogPostCanoncialUrl">
+		@if (AppConfiguration.Value.EnableMarkdownForLlms)
+		{
+			<link rel="alternate" type="text/markdown" href="@($"{NavigationManager.BaseUri}{BlogPostCanoncialUrl}.md")" />
+		}
+	</OgData>
 
 	@* Rendered in the body on purpose: HeadOutlet does not emit script tags,
 	   so a JSON-LD block placed in HeadContent never reaches the page. *@

tests/LinkDotNet.Blog.IntegrationTests/BlogPostMarkdownTests.cs

@@ -46,6 +46,78 @@ public async Task ShouldServeMarkdownWithAndWithoutSlug(string urlTemplate)
         body.ShouldEndWith("---\n\n# My Blog Post\n\nHello");
     }
 
+    [Theory]
+    [InlineData("/blogPost/{0}.md")]
+    [InlineData("/blogPost/{0}/my-blog-post.md")]
+    [InlineData("/blogPost/{0}/a-wrong-slug.MD")]
+    [InlineData("/blogPost/{0}/.md")]
+    public async Task ShouldServeMarkdownForMdSuffixRegardlessOfAccept(string urlTemplate)
+    {
+        var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle("My Blog Post").WithContent("Hello").Build());
+
+        using var response = await GetAsync(client, string.Format(urlTemplate, blogPost.Id), "text/html");
+
+        response.StatusCode.ShouldBe(HttpStatusCode.OK);
+        response.Content.Headers.ContentType!.ToString().ShouldBe(MarkdownContentType);
+        var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
+        body.ShouldEndWith("# My Blog Post\n\nHello");
+    }
+
+    [Fact]
+    public async Task ShouldReturnNotFoundForMdSuffixOfUnpublishedBlogPost()
+    {
+        var blogPost = await SeedAsync(factory, new BlogPostBuilder().IsPublished(false).Build());
+
+        using var response = await GetAsync(client, $"/blogPost/{blogPost.Id}.md", null);
+
+        await ShouldBeMarkdownNotFoundAsync(response);
+    }
+
+    [Theory]
+    [InlineData("/blogPost/{0}", "/blogPost/{0}.md")]
+    [InlineData("/blogPost/{0}/some-slug/", "/blogPost/{0}/some-slug.md")]
+    public async Task ShouldAdvertiseMarkdownVersionOnHtmlResponse(string urlTemplate, string linkTemplate)
+    {
+        var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle("Linked Post").Build());
+
+        using var response = await GetAsync(client, string.Format(urlTemplate, blogPost.Id), null);
+
+        await ShouldBeHtmlBlogPostAsync(response, "Linked Post");
+        response.Headers.GetValues("Link").ShouldContain($"<{string.Format(linkTemplate, blogPost.Id)}>; rel=\"alternate\"; type=\"text/markdown\"");
+        var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
+        body.ShouldContain($"<link rel=\"alternate\" type=\"text/markdown\" href=\"http://localhost/blogPost/{blogPost.Id}/linked-post.md\" />");
+    }
+
+    [Fact]
+    public async Task ShouldListPublishedBlogPostsInLlmsTxt()
+    {
+        var published = await SeedAsync(factory, new BlogPostBuilder().WithTitle("Listed Post").WithShortDescription("**Short**\nsummary").Build());
+        await SeedAsync(factory, new BlogPostBuilder().WithTitle("Hidden Draft").IsPublished(false).Build());
+
+        using var response = await GetAsync(client, "/llms.txt", null);
+
+        response.StatusCode.ShouldBe(HttpStatusCode.OK);
+        response.Content.Headers.ContentType!.ToString().ShouldBe("text/plain; charset=utf-8");
+        var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
+        body.ShouldStartWith("# ");
+        body.ShouldContain($"- [Listed Post](http://localhost/blogPost/{published.Id}/listed-post.md): Short summary\n");
+        body.ShouldNotContain("Hidden Draft");
+    }
+
+    [Theory]
+    [InlineData("Why ] matters", @"Why \] matters", "why-matters")]
+    [InlineData("Generic <T> & [more]", @"Generic \<T\> \& \[more\]", "generic-t-more")]
+    [InlineData("Under_score *star* `tick` \\ slash", @"Under\_score \*star\* \`tick\` \\ slash", "underscore-star-tick-slash")]
+    public async Task ShouldEscapeMarkdownInLlmsTxt(string title, string escapedTitle, string slug)
+    {
+        var blogPost = await SeedAsync(factory, new BlogPostBuilder().WithTitle(title).WithShortDescription("`List<T>`").Build());
+
+        using var response = await GetAsync(client, "/llms.txt", null);
+
+        var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
+        body.ShouldContain($"- [{escapedTitle}](http://localhost/blogPost/{blogPost.Id}/{slug}.md): List\\<T\\>\n");
+    }
+
     [Theory]
     [InlineData("text/markdown")]
     [InlineData("TEXT/MARKDOWN")]
@@ -208,6 +280,15 @@ public async Task ShouldServeHtmlWhenDisabled(string? enableMarkdownForLlms)
 
         await ShouldBeHtmlBlogPostAsync(response, "Disabled Post");
         response.Headers.Vary.ShouldNotContain("Accept");
+        response.Headers.Contains("Link").ShouldBeFalse();
+        var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken);
+        body.ShouldNotContain("text/markdown");
+
+        using var mdSuffixResponse = await GetAsync(disabledClient, $"/blogPost/{blogPost.Id}/disabled-post.md", null);
+        await ShouldBeHtmlBlogPostAsync(mdSuffixResponse, "Disabled Post");
+
+        using var llmsTxtResponse = await GetAsync(disabledClient, "/llms.txt", null);
+        llmsTxtResponse.StatusCode.ShouldBe(HttpStatusCode.NotFound);
     }
 
     public void Dispose() => client.Dispose();