fix(refs): preserve Unicode whitespace in command records

Git separates reference and remote URL records with LF, but Unicode
`splitlines()` also split valid names at U+0085, U+2028 and U+2029. Enumeration
could return a truncated, nonexistent reference. Fetch parsing and progress
handling also stripped valid Unicode whitespace from the ends of names.

Split these outputs only at LF, trim fetch column padding as ASCII spaces,
and remove only CR/LF framing from progress records. Preserve empty remote-URL
output behavior. Reference storage and native lookup remain owned by Git/Gix.

Scripted SHA-1/SHA-256 repositories contain leading, middle and trailing
separators in branch, tag and remote-ref names. Tests compare Git's inventory,
resolve the names, feed bytes/text through the fetch progress handler, check
local stale refs and retain Unicode separators in URLs. All 26 focused cases
pass on each backend after reproducing the failures. Existing progress tests,
Ruff, mypy and whitespace checks pass.

Author: codex

doc/source/changes.rst

@@ -54,6 +54,8 @@ API changes
 * Packed tag names containing non-UTF-8 bytes are represented losslessly as
   surrogate-escaped strings. ``safe_encode()`` recovers their original bytes,
   and Gix reference validation and resolution receive those bytes directly.
+  Reference enumeration, fetch results, stale references and remote URLs preserve
+  Unicode line separators within names instead of treating them as record breaks.
 * Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``,
   and raw rename fields. Staged-file diffs no longer fail while decoding legacy
   filenames; blob contents remain byte streams regardless of their encoding.

git/refs/symbolic.py

@@ -698,7 +698,7 @@ def _iter_items(
         output = repo.git._call_process_safe(
             "for_each_ref", "--format=%(refname)", *options, "--", *([prefix] if prefix else [])
         )
-        for path in sorted(output.splitlines()):
+        for path in sorted(output.split("\n")):
             try:
                 yield cls.from_path(repo, path)
             except ValueError:

git/remote.py

@@ -380,7 +380,7 @@ def _from_line(cls, repo: "Repo", line: str, refspecs: Sequence[str] = (), refs:
         if match is None:
             raise ValueError("Failed to parse fetch output: %r" % line)
         control, operation, source, destination, note = match.groups()
-        source = source.strip()
+        source = source.strip(" ")
         if control not in cls._flag_map:
             raise ValueError("Unknown fetch status: %r" % control)
         flags = cls._flag_map[cast(flagKeyLiteral, control)]
@@ -393,7 +393,7 @@ def _from_line(cls, repo: "Repo", line: str, refspecs: Sequence[str] = (), refs:
         old_commit = None
         if control in (" ", "+"):
             old_commit = repo.rev_parse(operation.split("..", 1)[0])
-        destination = destination.strip()
+        destination = destination.strip(" ")
         if destination == "FETCH_HEAD":
             ref = SymbolicReference(repo, destination)
         else:
@@ -618,7 +618,9 @@ def delete_url(self, url: str, **kwargs: Any) -> "Remote":
     @property
     def urls(self) -> Iterator[str]:
         """:return: Iterator yielding all configured URL targets on a remote as strings"""
-        yield from self.repo.git._call_process_safe("remote", "get-url", "--all", "--", self.name).splitlines()
+        output = self.repo.git._call_process_safe("remote", "get-url", "--all", "--", self.name)
+        if output:
+            yield from output.split("\n")
 
     @property
     def refs(self) -> IterableList[RemoteReference]:
@@ -655,7 +657,7 @@ def stale_refs(self) -> IterableList[Reference]:
         out_refs: IterableList[Reference] = IterableList(RemoteReference._id_attribute_, "%s/" % self.name)
         for line in self.repo.git._call_process_safe(
             "remote", "prune", "--dry-run", "--", self, _allow_network=True
-        ).splitlines()[2:]:
+        ).split("\n")[2:]:
             # expecting
             # * [would prune] origin/new_branch
             token = " * [would prune] "
@@ -817,7 +819,7 @@ def _get_fetch_info_from_stderr(
                     fetch_info_lines.append(line)
                     continue
 
-        refs = self.repo.git._call_process_safe("for_each_ref", "--format=%(refname)").splitlines()
+        refs = self.repo.git._call_process_safe("for_each_ref", "--format=%(refname)").split("\n")
         try:
             configured = self.config_reader.config.get_values(self._config_section_name(), "fetch")
         except (cp.NoSectionError, cp.NoOptionError):

git/util.py

@@ -812,7 +812,7 @@ def new_message_handler(self) -> Callable[[str], None]:
         """
 
         def handler(line: AnyStr) -> None:
-            self._parse_progress_line(line.rstrip())
+            self._parse_progress_line(line.rstrip(b"\r\n") if isinstance(line, bytes) else line.rstrip("\r\n"))
 
         # END handler
 

test/test_refs.py

@@ -9,7 +9,7 @@
 import unittest
 from itertools import chain
 from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 
@@ -24,6 +24,8 @@
     Head,
     Reference,
     RefLog,
+    Remote,
+    RemoteProgress,
     RemoteReference,
     SymbolicReference,
     TagReference,
@@ -37,6 +39,58 @@
 from test.lib.fixtures import fixture_repo, read_only_fixture, writable_fixture
 
 
+@pytest.mark.parametrize("object_format", ["sha1", "sha256"])
+def test_reference_names_preserve_unicode_line_separators(object_format):
+    names = [
+        before + separator + after
+        for separator in ("\x85", "\u2028", "\u2029")
+        for before, after in (("", "after"), ("before", "after"), ("before", ""))
+    ]
+    with fixture_repo("ref-line-separators.sh", object_format) as repo:
+        expected = {
+            f"{prefix}/{name}" for prefix in ("refs/heads", "refs/tags", "refs/remotes/origin") for name in names
+        } | {"refs/heads/main"}
+        assert set(repo.git.for_each_ref("--format=%(refname)").split("\n")) == expected
+        assert {ref.path for ref in repo.references} == expected
+        assert {head.name for head in repo.heads} == set(names) | {"main"}
+        assert {tag.name for tag in repo.tags} == set(names)
+        assert {ref.remote_head for ref in RemoteReference.iter_items(repo, remote="origin")} == set(names)
+        for ref in repo.references:
+            assert ref.commit == repo.head.commit
+
+
+@pytest.mark.parametrize("separator", ["\x85", "\u2028", "\u2029"])
+@pytest.mark.parametrize("before,after", [("", "after"), ("before", "after"), ("before", "")])
+@pytest.mark.parametrize("as_bytes", [False, True])
+def test_fetch_resolves_unicode_line_separator_names_from_inventory(separator, before, after, as_bytes):
+    name = before + separator + after
+    with fixture_repo("ref-line-separators.sh", "sha1") as repo:
+        progress = RemoteProgress()
+        line = f" * [new branch] {name}     -> origin/{name}"
+        progress.new_message_handler()((line + "\n").encode() if as_bytes else line + "\n")
+        assert progress.other_lines == [line]
+        with patch("git.remote.handle_process_output"):
+            result = Remote(repo, "inventory")._get_fetch_info_from_stderr(Mock(), progress)
+        assert len(result) == 1
+        assert result[0].ref.path == f"refs/remotes/origin/{name}"
+        assert result[0].remote_ref_path == name
+
+
+def test_remote_urls_preserve_unicode_line_separators():
+    with fixture_repo("ref-line-separators.sh", "sha1") as repo:
+        assert list(repo.remote("unicode").urls) == [
+            f"https://example.invalid/before{separator}after" for separator in ("\x85", "\u2028", "\u2029")
+        ]
+        with patch.object(Git, "_call_process_safe", return_value=""):
+            assert list(Remote(repo, "empty").urls) == []
+
+
+def test_stale_refs_preserve_unicode_line_separators():
+    with fixture_repo("ref-line-separators.sh", "sha1") as repo:
+        origin = repo.remote("origin")
+        assert {ref.path for ref in origin.stale_refs} == {ref.path for ref in origin.refs}
+
+
 @pytest.mark.parametrize("object_format", ["sha1", "sha256"])
 def test_non_utf8_tag_names_preserve_bytes(object_format):
     with fixture_repo("legacy-refs.sh", object_format) as repo:

tests/fixtures/ref-line-separators.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eu
+git init --initial-branch=main --object-format="$1" repo
+git init --bare --initial-branch=main --object-format="$1" upstream
+cd repo
+git remote add origin ../upstream
+tree=$(git mktree </dev/null)
+commit=$(git commit-tree "$tree" -m initial)
+git update-ref refs/heads/main "$commit"
+for separator in $'\302\205' $'\342\200\250' $'\342\200\251'; do
+    git config --add remote.unicode.url "https://example.invalid/before${separator}after"
+    for prefix in refs/heads refs/tags refs/remotes/origin; do
+        for name in "${separator}after" "before${separator}after" "before${separator}"; do
+            git update-ref "$prefix/$name" "$commit"
+        done
+    done
+done