fix(refs): preserve non-UTF-8 names at native API boundaries (#2064)

Packed-reference enumeration now delegates to Git/Gix and no longer has the
original UTF-8 file-reader failure. However, tag paths retain undecodable
bytes as surrogate escapes, and passing those strings back into GixPython
raises `UnicodeEncodeError` before validation or reference lookup.

Make `safe_encode()` the byte-preserving inverse of `safe_decode()`. Use it
at the supported Gix reference validation, lookup, revision, reflog and edit
boundaries, and when serializing Git reference transactions. GixPython 0.3.0
accepts bytes in these APIs, so this fixes adapter conversion without adding
a reference-storage implementation or requiring an upstream workaround.

Add a scripted packed-ref fixture with lightweight and annotated non-UTF-8
tag names for both SHA-1 and SHA-256. Git 2.54.0's `for-each-ref` supplies
the reference inventory; `git-for-each-ref.adoc` documents the `refname` and
`objectname` projections used for the baseline. The regression checks byte
round trips, tag/commit resolution, CLI parity and absence of CLI fallback
during native resolution. Packed records are fixture inputs because APFS
rejects those bytes in loose reference filenames.

Validation: 42 reference tests and 24 subtests passed with CLI; 144 reference
and backend tests and 24 subtests passed with Gix. Ruff, ShellCheck and mypy
passed. Windows behavior will be validated by CI.

Author: codex

doc/gix-backend.md

@@ -389,6 +389,15 @@ repository-format validation gap.
 | `Repo.untracked_files` | Native directory walk | Extra status options and non-root command directories |
 | `Repo.ignored` | Native excludes with tracked-file suppression | Symlink/submodule traversal, unsupported path normalization, and unavailable path metadata |
 
+Reference names and revisions are encoded with `surrogateescape` before calling
+GixPython's byte-capable `Target.Symbolic`, `Repository.try_find_reference`,
+`Repository.rev_parse_single`, `Repository.rev_parse` and `RefEdit` APIs.
+This preserves non-UTF-8 packed tag names without interpreting reference storage
+in Python. `test_non_utf8_tag_names_preserve_bytes_and_resolve` covers lightweight
+and annotated tags in SHA-1 and SHA-256 repositories and forbids CLI fallback
+during native resolution. This corrects adapter input conversion; GixPython 0.3.0
+already accepts the original bytes through those APIs.
+
 Repository-backed native operations fall back for reftable (GIX-1) and repositories
 with `extensions.compatObjectFormat` (GIX-19). Process-returning calls, timeouts,
 custom storage/environment, global Git options, and unhandled command options

doc/source/changes.rst

@@ -51,6 +51,9 @@ Git executable and safety
 API changes
 -----------
 
+* Packed tag names containing non-UTF-8 bytes remain losslessly represented as
+  surrogate-escaped strings. ``safe_encode()`` recovers their original bytes,
+  and Gix reference validation and resolution receive those bytes directly.
 * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``,
   and raw rename fields. Staged-file diffs no longer fail while decoding legacy
   filenames; blob contents remain byte streams regardless of their encoding.

git/_backend.py

@@ -16,7 +16,7 @@
 from threading import Lock
 from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, cast
 
-from git.compat import defenc, safe_decode
+from git.compat import defenc, safe_decode, safe_encode
 from git.exc import GitCommandError
 
 _selection = os.environ.get("GIT_PYTHON_BACKEND")
@@ -218,7 +218,7 @@ def _oid(repo: Any, ref: Any, env: Optional[Dict[str, Any]] = None) -> Any:
         _index(repo, {"env": env or {}})
     if not hasattr(repo, "rev_parse_single"):
         raise _Unsupported("revision feature disabled")
-    return repo.rev_parse_single(ref)
+    return repo.rev_parse_single(safe_encode(ref))
 
 
 def object_data(command: Any, ref: bytes, *, stream: bool = False) -> Any:
@@ -296,7 +296,7 @@ def revision_info(command: Any, ref: str) -> Any:
         repo = _repository(command, {})
         if ref.startswith(":"):
             _index(repo, {"env": command.environment()})
-        spec = repo.rev_parse(ref)
+        spec = repo.rev_parse(safe_encode(ref))
         oid = spec.single()
         if oid is None:
             raise _Unsupported("multiple revisions")
@@ -362,7 +362,7 @@ def check_ref_name(name: str) -> Any:
     if gix is None:
         return NotImplemented
     try:
-        gix.Target.Symbolic(name)
+        gix.Target.Symbolic(safe_encode(name))
     except gix.Error as exc:
         if "/" not in name:
             return _fallback("Reference.validate", "standalone reference names (GIX-20)")
@@ -380,10 +380,11 @@ def reference_info(command: Any, path: str) -> Any:
         if path != "HEAD" and not path.startswith("refs/"):
             raise _Unsupported("partial reference names")
         repo = _repository(command, {})
-        reference = repo.try_find_reference(path)
+        encoded_path = safe_encode(path)
+        reference = repo.try_find_reference(encoded_path)
         result = None
         if reference is not None:
-            if reference.name() != os.fsencode(path):
+            if reference.name() != encoded_path:
                 raise _Unsupported("partial reference lookup")
             target = reference.target()
             name = target.try_name()
@@ -400,7 +401,7 @@ def reference_info(command: Any, path: str) -> Any:
 def _symbolic_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     if len(args) != 4 or args[:3] != ["--quiet", "--no-recurse", "--"]:
         raise _Unsupported("reference mutation or options")
-    reference = repo.try_find_reference(args[-1])
+    reference = repo.try_find_reference(safe_encode(args[-1]))
     if reference is None:
         raise _Unsupported("missing reference diagnostics")
     target = reference.target().try_name()
@@ -415,7 +416,7 @@ def _for_each_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     split = args.index("--") if "--" in args else len(args)
     if args[1:split] or len(args[split + 1 :]) > 1:
         raise _Unsupported("root refs or reference options")
-    prefix = os.fsencode(args[-1]) if len(args) > split + 1 else b""
+    prefix = safe_encode(args[-1]) if len(args) > split + 1 else b""
     if any(char in prefix for char in b"*?["):
         raise _Unsupported("reference glob patterns")
     with repo.references().all() as refs:
@@ -739,7 +740,7 @@ def _date(signature: Any) -> bytes:
 
 def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     if len(args) == 3 and args[:2] == ["exists", "--"]:
-        reference = repo.try_find_reference(args[-1])
+        reference = repo.try_find_reference(safe_encode(args[-1]))
         if reference is None:
             raise _Unsupported("orphan reflog lookup")
         if not reference.log_exists():
@@ -761,7 +762,7 @@ def _reflog(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
         or args[-1] != "--"
     ):
         raise _Unsupported("reflog writing or format")
-    reference = repo.try_find_reference(args[-2])
+    reference = repo.try_find_reference(safe_encode(args[-2]))
     if reference is None:
         raise _Unsupported("orphan reflog lookup")
     cursor = reference.log_iter().rev()
@@ -1174,7 +1175,7 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     if delete:
         if len(operands) != 1 or deref:
             raise _Unsupported("reference deletion options")
-        edit = gix.RefEdit.delete(operands[0], gix.PreviousValue.Any).with_deref(False)
+        edit = gix.RefEdit.delete(safe_encode(operands[0]), gix.PreviousValue.Any).with_deref(False)
         _write("update_ref", lambda: repo.edit_references_as([edit]))
         return b""
     if len(operands) != 2:
@@ -1184,7 +1185,8 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     kind = repo.find_header(oid).kind()
     if (name == "HEAD" or name.startswith("refs/heads/")) and kind != "commit":
         raise _Unsupported("branch target validation")
-    reference = repo.try_find_reference(name)
+    encoded_name = safe_encode(name)
+    reference = repo.try_find_reference(encoded_name)
     if reference is not None:
         if deref and name != "HEAD" and reference.target().try_name() is not None:
             raise _Unsupported("symbolic reference update (GIX-10)")
@@ -1196,13 +1198,15 @@ def _update_ref(repo: Any, args: List[str], kwargs: Dict[str, Any]) -> bytes:
     for _ in range(5):
         if head is None or head.target().try_name() is None:
             break
-        if head.target().try_name() == os.fsencode(name):
+        if head.target().try_name() == encoded_name:
             raise _Unsupported("active branch HEAD reflog (GIX-10)")
         head = head.follow()
     log = gix.LogChange()
     log.force_create_reflog = force_log
     log.message = message
-    edit = gix.RefEdit.update_with_log(name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref(deref)
+    edit = gix.RefEdit.update_with_log(encoded_name, gix.Target.Object(oid), gix.PreviousValue.Any, log).with_deref(
+        deref
+    )
     signature = _committer(repo, kwargs.get("env", {}))
     _write("update_ref", lambda: repo.edit_references_as([edit], signature))
     return b""

git/compat.py

@@ -65,9 +65,9 @@ def safe_encode(s: AnyStr) -> bytes: ...
 
 
 def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]:
-    """Safely encode a binary string to Unicode."""
+    """Encode text to bytes, preserving bytes recovered by :func:`safe_decode`."""
     if isinstance(s, str):
-        return s.encode(defenc)
+        return s.encode(defenc, "surrogateescape")
     elif isinstance(s, bytes):
         return s
     elif s is None:

git/refs/symbolic.py

@@ -11,6 +11,7 @@
 
 from git import _backend
 from git.cmd import Git
+from git.compat import safe_encode
 from git.exc import GitCommandError, UnsafeOptionError
 from git.objects.base import Object
 from git.objects.commit import Commit
@@ -462,7 +463,7 @@ def _transaction(repo: "Repo", commands: str, logmsg: Union[str, None] = None) -
             if first_line:
                 options.extend(("-m", first_line))
         with tempfile.TemporaryFile() as stream:
-            stream.write(commands.encode("utf-8"))
+            stream.write(safe_encode(commands))
             stream.seek(0)
             repo.git._call_process_safe("update_ref", "--no-deref", "--stdin", "-z", *options, istream=stream)
 

test/test_refs.py

@@ -10,27 +10,53 @@
 from pathlib import Path
 from unittest.mock import patch
 
+import pytest
+
 from gitdb.exc import BadName
 
 import git.refs as refs
 from git import (
     Commit,
     GitCommandError,
     GitConfigParser,
+    Git,
     Head,
     Reference,
     RefLog,
     RemoteReference,
     SymbolicReference,
     TagReference,
+    _backend,
 )
+from git.compat import safe_encode
 from git.exc import UnsafeOptionError
 from git.objects.tag import TagObject
 from git.util import Actor
 from test.lib import PathLikeMock, TestBase, requires_symlinks, with_rw_repo
 from test.lib.fixtures import fixture_repo, read_only_fixture, writable_fixture
 
 
+@pytest.mark.parametrize("object_format", ["sha1", "sha256"])
+def test_non_utf8_tag_names_preserve_bytes_and_resolve(object_format):
+    with fixture_repo("legacy-refs.sh", object_format) as repo:
+        tags = repo.tags
+        assert len(tags) == 2
+        assert {safe_encode(tag.path) for tag in tags} == {b"refs/tags/release-\xf5", b"refs/tags/annotated-\xe3"}
+        native_guard = (
+            patch.object(Git, "execute", side_effect=AssertionError("unexpected reference CLI call"))
+            if _backend.name == "gix"
+            else contextlib.nullcontext()
+        )
+        with native_guard:
+            for tag in tags:
+                assert tag.commit == repo.head.commit
+                assert repo.rev_parse(tag.path).hexsha == tag.object.hexsha
+                assert repo.commit(tag.path) == repo.head.commit
+        if _backend.name == "gix":
+            with patch.object(_backend, "gix", None):
+                assert {tag.path for tag in repo.tags} == {tag.path for tag in tags}
+
+
 class TestRefs(TestBase):
     def test_from_path(self):
         # Should be able to create any reference directly.

tests/fixtures/legacy-refs.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+git init --initial-branch=main --object-format="$1" repo
+cd repo
+tree=$(git mktree </dev/null)
+commit=$(printf initial | git commit-tree "$tree")
+git update-ref refs/heads/main "$commit"
+tag=$(printf 'object %s\ntype commit\ntag annotated\ntagger Example <example@example.invalid> 1100000000 +0000\n\nrelease\n' "$commit" | git hash-object -t tag -w --stdin)
+# Packed refs can carry bytes that the fixture filesystem rejects in loose names.
+printf '# pack-refs with: peeled fully-peeled sorted\n%s refs/tags/annotated-\343\n^%s\n%s refs/tags/release-\365\n' \
+    "$tag" "$commit" "$commit" >.git/packed-refs