Support complete syntax highlight styles and token backgrounds

Assets/HighlightStyles/README.md

@@ -0,0 +1,11 @@
+# Syntax highlight captures
+
+Native iPhone 17 Pro Simulator captures from a temporary UIKit host using Runestone `TextView`, rendered against framework commit `3d7ff92e158ddb37f0661b99a1301a111e2917c1` on iOS 26.5.
+
+Each screen compares two views in the same build. The upper view uses the individual `Theme` callbacks; the lower view returns a complete `HighlightStyle`. These are API comparisons, not screenshots of two different framework revisions, and are visual evidence rather than automated appearance regression tests.
+
+- `token-backgrounds.png`: Alabaster BG colors with HTML captures. The complete style adds yellow comment, green string, and blue tag backgrounds.
+- `nested-style-reset.png`: A synthetic JavaScript query captures `(program) @emphasis` and `(string) @plain`. The outer capture supplies bold, italic, purple text, and shadow. Returning `HighlightStyle()` for strings restores the default foreground and regular font and removes the shadow. The individual callbacks retain the enclosing style.
+- `font-weight-compatibility.png`: Tomorrow colors with JavaScript captures. Keywords and functions use a semibold font (weight 600) through both APIs, demonstrating compatibility with existing custom-font callbacks.
+
+The sample code, fonts, and view layout are identical between each pair of panels. The screenshots were captured directly from Simulator without compositing or retouching.

Sources/Runestone/Documentation.docc/CreatingATheme.md

@@ -75,3 +75,36 @@ The list below contains highlight names that are commonly used across a suite of
 - `type.builtin`
 - `variable`
 - `variable.builtin`
+
+
+## Complete Highlight Styles
+
+Implement ``Theme/highlightStyle(for:)`` when a capture should specify its complete
+appearance instead of adding individual attributes to enclosing captures:
+
+```swift
+func highlightStyle(for highlightName: String) -> HighlightStyle? {
+    switch highlightName {
+    case "comment":
+        return HighlightStyle(textColor: .secondaryLabel,
+                              font: .italicSystemFont(ofSize: font.pointSize))
+    case "comment.special":
+        return HighlightStyle(backgroundColor: .systemYellow)
+    default:
+        return nil
+    }
+}
+```
+
+Here `comment.special` uses the theme's default foreground and font, clears any
+inherited shadow, and adds a yellow background. A supplied font preserves its exact
+weight and traits. `HighlightStyle()` restores the default text appearance, including
+clearing a token background and shadow.
+
+Returning `nil` preserves the existing individual callbacks for that capture, including
+resolving its font from `font(for:)` or the theme's default font. It does not inherit a
+complete outer capture's custom font. Existing themes need no changes because the
+protocol supplies this default implementation.
+The highlight names, query configuration, and capture ordering remain unchanged.
+Backgrounds are painted behind search highlights, marked text, and glyphs. A
+`StringSyntaxHighlighter` result carries them as `.backgroundColor` attributes.

Sources/Runestone/TextView/Appearance/HighlightStyle.swift

@@ -0,0 +1,29 @@
+import UIKit
+
+/// A complete syntax highlight style applied to a captured range.
+///
+/// Unlike the individual styling callbacks on ``Theme``, a complete style replaces
+/// styling from enclosing captures. This allows an inner capture to restore a regular
+/// font or clear a background or shadow. It does not change unrelated attributes such
+/// as paragraph layout or kerning.
+public struct HighlightStyle {
+    /// Foreground color, or `nil` to use ``Theme/textColor``.
+    public var textColor: UIColor?
+    /// Font including its weight and traits, or `nil` to use ``Theme/font``.
+    public var font: UIFont?
+    /// Token background color, or `nil` to clear the background.
+    public var backgroundColor: UIColor?
+    /// Text shadow, or `nil` to clear the shadow.
+    public var shadow: NSShadow?
+
+    /// Creates a complete style. An empty style restores the theme's default text appearance.
+    public init(textColor: UIColor? = nil,
+                font: UIFont? = nil,
+                backgroundColor: UIColor? = nil,
+                shadow: NSShadow? = nil) {
+        self.textColor = textColor
+        self.font = font
+        self.backgroundColor = backgroundColor
+        self.shadow = shadow
+    }
+}

Sources/Runestone/TextView/Appearance/Theme.swift

@@ -35,6 +35,15 @@ public protocol Theme: AnyObject {
     /// Corner radius of the background of marked text. Text will be marked when writing certain languages, for example Chinese and Japanese.
     /// A value of zero or less means that the background will not have rounded corners. Defaults to 0.
     var markedTextBackgroundCornerRadius: CGFloat { get }
+    /// Complete style for text matching a highlight name.
+    ///
+    /// Return a style to replace the foreground color, font, background color, and shadow
+    /// inherited from enclosing captures. A missing color or font uses the theme's defaults;
+    /// a missing background or shadow removes that attribute. The font includes all its traits.
+    ///
+    /// Return `nil` to use the existing individual styling callbacks for this capture.
+    /// The default implementation returns `nil`.
+    func highlightStyle(for highlightName: String) -> HighlightStyle?
     /// Color of text matching the capture sequence.
     ///
     /// See <doc:CreatingATheme> for more information on higlight names.
@@ -65,6 +74,10 @@ public protocol Theme: AnyObject {
 }
 
 public extension Theme {
+    func highlightStyle(for highlightName: String) -> HighlightStyle? {
+        nil
+    }
+
     var gutterHairlineWidth: CGFloat {
         hairlineLength
     }

Sources/Runestone/TextView/LineController/LineFragmentRenderer.swift

@@ -33,6 +33,7 @@ final class LineFragmentRenderer {
     }
 
     func draw(to context: CGContext, inCanvasOfSize canvasSize: CGSize) {
+        drawTokenBackgrounds(to: context)
         drawHighlightedRanges(to: context, inCanvasOfSize: canvasSize)
         drawMarkedRange(to: context)
         drawInvisibleCharacters()
@@ -98,6 +99,41 @@ final class LineFragmentRenderer {
 }
 
 private extension LineFragmentRenderer {
+    private func drawTokenBackgrounds(to context: CGContext) {
+        // Paint token backgrounds separately from glyphs to keep overlays visible.
+        // Runs are visual segments, so their glyph positions also handle mixed writing
+        // directions and avoid spanning unrelated text across a bidirectional boundary.
+        context.saveGState()
+        defer { context.restoreGState() }
+        for run in CTLineGetGlyphRuns(lineFragment.line) as? [CTRun] ?? [] {
+            let attributes = CTRunGetAttributes(run) as NSDictionary
+            guard let color = attributes[NSAttributedString.Key.tokenBackgroundColor] as? UIColor else {
+                continue
+            }
+            let count = CTRunGetGlyphCount(run)
+            guard count > 0 else {
+                continue
+            }
+            var positions = [CGPoint](repeating: .zero, count: count)
+            var advances = [CGSize](repeating: .zero, count: count)
+            CTRunGetPositions(run, CFRange(location: 0, length: 0), &positions)
+            CTRunGetAdvances(run, CFRange(location: 0, length: 0), &advances)
+            var minX = CGFloat.greatestFiniteMagnitude
+            var maxX = -CGFloat.greatestFiniteMagnitude
+            for index in 0 ..< count {
+                let start = positions[index].x
+                let end = start + advances[index].width
+                minX = min(minX, min(start, end))
+                maxX = max(maxX, max(start, end))
+            }
+            guard maxX > minX else {
+                continue
+            }
+            context.setFillColor(color.cgColor)
+            context.fill(CGRect(x: minX, y: 0, width: maxX - minX, height: lineFragment.scaledSize.height))
+        }
+    }
+
     private func horizontalBounds(of run: CTRun) -> ClosedRange<CGFloat>? {
         let count = CTRunGetGlyphCount(run)
         guard count > 0 else {

Sources/Runestone/TextView/LineController/LineSyntaxHighlighter.swift

@@ -4,6 +4,7 @@ import Foundation
 extension NSAttributedString.Key {
     static let isBold = NSAttributedString.Key("runestone_isBold")
     static let isItalic = NSAttributedString.Key("runestone_isItalic")
+    static let tokenBackgroundColor = NSAttributedString.Key("runestone_tokenBackgroundColor")
 }
 
 struct LineSyntaxHiglighterSetAttributesResult {

Sources/Runestone/TextView/LineController/LineTypesetter.swift

@@ -79,7 +79,7 @@ final class LineTypesetter {
     func prepareToTypeset(_ attributedString: NSAttributedString) {
         self.attributedString = attributedString
         stringLength = CFAttributedStringGetLength(attributedString)
-        typesetter = CTTypesetterCreateWithAttributedString(attributedString)
+        typesetter = CTTypesetterCreateWithAttributedString(stringForTypesetting(attributedString))
     }
 
     @discardableResult
@@ -112,6 +112,24 @@ final class LineTypesetter {
 }
 
 private extension LineTypesetter {
+    private func stringForTypesetting(_ string: NSAttributedString) -> NSAttributedString {
+        // Keep token backgrounds out of CTLineDraw so they cannot cover search or
+        // marked-text highlights. The renderer paints these backgrounds first.
+        // Preserve the original attributes for clients of StringSyntaxHighlighter.
+        var copy: NSMutableAttributedString?
+        string.enumerateAttribute(.backgroundColor, in: NSRange(location: 0, length: string.length)) { value, range, _ in
+            guard let value else {
+                return
+            }
+            if copy == nil {
+                copy = NSMutableAttributedString(attributedString: string)
+            }
+            copy?.removeAttribute(.backgroundColor, range: range)
+            copy?.addAttribute(.tokenBackgroundColor, value: value, range: range)
+        }
+        return copy ?? string
+    }
+
     private func updateState(from typesetResult: TypesetResult) {
         lineFragments.append(contentsOf: typesetResult.lineFragments)
         for (id, index) in typesetResult.lineFragmentsMap {

Sources/Runestone/TextView/SyntaxHighlighting/Internal/TreeSitter/TreeSitterSyntaxHighlightToken.swift

@@ -1,16 +1,18 @@
 import UIKit
 
 final class TreeSitterSyntaxHighlightToken {
+    let completeStyle: HighlightStyle?
     let range: NSRange
     let textColor: UIColor?
     let shadow: NSShadow?
     let font: UIFont?
     let fontTraits: FontTraits
     var isEmpty: Bool {
-        range.length == 0 || (textColor == nil && font == nil && shadow == nil)
+        range.length == 0 || (completeStyle == nil && textColor == nil && font == nil && shadow == nil && fontTraits.isEmpty)
     }
 
-    init(range: NSRange, textColor: UIColor?, shadow: NSShadow?, font: UIFont?, fontTraits: FontTraits) {
+    init(range: NSRange, textColor: UIColor?, shadow: NSShadow?, font: UIFont?, fontTraits: FontTraits, completeStyle: HighlightStyle? = nil) {
+        self.completeStyle = completeStyle
         self.range = range
         self.textColor = textColor
         self.shadow = shadow

Sources/Runestone/TextView/SyntaxHighlighting/Internal/TreeSitter/TreeSitterSyntaxHighlighter.swift

@@ -84,44 +84,73 @@ private extension TreeSitterSyntaxHighlighter {
     private func setAttributes(for tokens: [TreeSitterSyntaxHighlightToken], on attributedString: NSMutableAttributedString) {
         attributedString.beginEditing()
         for token in tokens {
-            var attributes: [NSAttributedString.Key: Any] = [:]
-            if let foregroundColor = token.textColor {
-                attributes[.foregroundColor] = foregroundColor
-            }
-            if let shadow = token.shadow {
-                attributes[.shadow] = shadow
-            }
-            if token.fontTraits.contains(.bold) {
-                attributedString.addAttribute(.isBold, value: true, range: token.range)
-            }
-            if token.fontTraits.contains(.italic) {
-                attributedString.addAttribute(.isItalic, value: true, range: token.range)
-            }
-            var symbolicTraits: UIFontDescriptor.SymbolicTraits = []
-            if let isBold = attributedString.attribute(.isBold, at: token.range.location, effectiveRange: nil) as? Bool, isBold {
-                symbolicTraits.insert(.traitBold)
-            }
-            if let isItalic = attributedString.attribute(.isItalic, at: token.range.location, effectiveRange: nil) as? Bool, isItalic {
-                symbolicTraits.insert(.traitItalic)
-            }
-            let currentFont = attributedString.attribute(.font, at: token.range.location, effectiveRange: nil) as? UIFont
-            let baseFont = token.font ?? theme.font
-            let newFont: UIFont
-            if !symbolicTraits.isEmpty {
-                newFont = baseFont.withSymbolicTraits(symbolicTraits) ?? baseFont
-            } else {
-                newFont = baseFont
-            }
-            if newFont != currentFont {
-                attributes[.font] = newFont
-            }
-            if !attributes.isEmpty {
-                attributedString.addAttributes(attributes, range: token.range)
+            if let style = token.completeStyle {
+                apply(style, to: attributedString, in: token.range)
+                continue
             }
+            applyIndividualAttributes(for: token, to: attributedString)
         }
         attributedString.endEditing()
     }
 
+    private func applyIndividualAttributes(for token: TreeSitterSyntaxHighlightToken, to attributedString: NSMutableAttributedString) {
+        var attributes: [NSAttributedString.Key: Any] = [:]
+        if let foregroundColor = token.textColor {
+            attributes[.foregroundColor] = foregroundColor
+        }
+        if let shadow = token.shadow {
+            attributes[.shadow] = shadow
+        }
+        if token.fontTraits.contains(.bold) {
+            attributedString.addAttribute(.isBold, value: true, range: token.range)
+        }
+        if token.fontTraits.contains(.italic) {
+            attributedString.addAttribute(.isItalic, value: true, range: token.range)
+        }
+        var symbolicTraits: UIFontDescriptor.SymbolicTraits = []
+        if let isBold = attributedString.attribute(.isBold, at: token.range.location, effectiveRange: nil) as? Bool, isBold {
+            symbolicTraits.insert(.traitBold)
+        }
+        if let isItalic = attributedString.attribute(.isItalic, at: token.range.location, effectiveRange: nil) as? Bool, isItalic {
+            symbolicTraits.insert(.traitItalic)
+        }
+        let currentFont = attributedString.attribute(.font, at: token.range.location, effectiveRange: nil) as? UIFont
+        let baseFont = token.font ?? theme.font
+        let newFont: UIFont
+        if !symbolicTraits.isEmpty {
+            newFont = baseFont.withSymbolicTraits(symbolicTraits) ?? baseFont
+        } else {
+            newFont = baseFont
+        }
+        if newFont != currentFont {
+            attributes[.font] = newFont
+        }
+        if !attributes.isEmpty {
+            attributedString.addAttributes(attributes, range: token.range)
+        }
+    }
+
+    private func apply(_ style: HighlightStyle, to attributedString: NSMutableAttributedString, in range: NSRange) {
+        // Complete styles must also reset our bookkeeping for subsequent captures that
+        // use the individual callbacks, otherwise an outer trait can reappear later.
+        let font = style.font ?? theme.font
+        let traits = font.fontDescriptor.symbolicTraits
+        attributedString.addAttributes([
+            .foregroundColor: style.textColor ?? theme.textColor,
+            .font: font,
+            .isBold: traits.contains(.traitBold),
+            .isItalic: traits.contains(.traitItalic)
+        ], range: range)
+        attributedString.removeAttribute(.backgroundColor, range: range)
+        attributedString.removeAttribute(.shadow, range: range)
+        if let backgroundColor = style.backgroundColor {
+            attributedString.addAttribute(.backgroundColor, value: backgroundColor, range: range)
+        }
+        if let shadow = style.shadow {
+            attributedString.addAttribute(.shadow, value: shadow, range: range)
+        }
+    }
+
     private func tokens(for captures: [TreeSitterCapture], localTo localRange: ByteRange) -> [TreeSitterSyntaxHighlightToken] {
         var tokens: [TreeSitterSyntaxHighlightToken] = []
         for capture in captures where capture.byteRange.overlaps(localRange) {
@@ -146,6 +175,10 @@ private extension TreeSitterSyntaxHighlighter {
 private extension TreeSitterSyntaxHighlighter {
     private func token(from capture: TreeSitterCapture, in byteRange: ByteRange) -> TreeSitterSyntaxHighlightToken {
         let range = NSRange(byteRange)
+        if let style = theme.highlightStyle(for: capture.name) {
+            return TreeSitterSyntaxHighlightToken(
+                range: range, textColor: nil, shadow: nil, font: nil, fontTraits: [], completeStyle: style)
+        }
         let textColor = theme.textColor(for: capture.name)
         let shadow = theme.shadow(for: capture.name)
         let font = theme.font(for: capture.name)