Skip to content

Commit b3d736a

Browse files
codexByron
authored andcommitted
fix(diff): preserve legacy filename bytes in staged diffs (#2030)
Aider-style staged-file reads fail with `UnicodeDecodeError` when a raw diff contains a non-UTF-8 filename. Index entries and blob streams already preserve the legacy bytes, but `Diff._handle_diff_line()` decoded the entire NUL-delimited output before separating metadata from paths. Keep raw diff fields as bytes, decode only the ASCII mode/OID/status metadata, and pass original path bytes into `Diff`. This also preserves rename and copy destinations without requiring callers to reencode them. The existing text path properties retain their display behavior. Git 2.54.0's `Documentation/diff-options.adoc` specifies that `--raw -z` returns unmodified pathnames separated by NULs. All six new regressions failed before the fix: five cover raw add/delete/ modify/rename/copy records, and one stages Windows-1252 filename and content bytes in a real index and reads its diff and blobs. All diff and index tests now pass with CLI and Gix backends: 150 passed, 2 skipped, and 9 subtests on each. Ruff and ShellCheck pass. Windows execution remains for CI validation; no conversion of the legacy files is needed.
1 parent b2189d9 commit b3d736a

5 files changed

Lines changed: 50 additions & 12 deletions

File tree

‎doc/source/changes.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ Git executable and safety
5151
API changes
5252
-----------
5353

54+
* Raw diffs preserve non-UTF-8 filename bytes in ``a_rawpath``, ``b_rawpath``,
55+
and raw rename fields. Staged-file diffs no longer fail while decoding legacy
56+
filenames; blob contents remain byte streams regardless of their encoding.
5457
* ``Tree.cache`` and ``TreeModifier`` remain supported. They edit one tree's
5558
in-memory entries; object IDs and parent trees do not change automatically.
5659
Tree serialization now delegates construction to Git or the supported Gix

‎git/diff.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -761,34 +761,29 @@ def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> N
761761

762762
@staticmethod
763763
def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None:
764-
lines = lines_bytes.decode(defenc)
765-
766764
# Discard everything before the first colon, and the colon itself.
767-
_, _, lines = lines.partition(":")
765+
_, _, lines = lines_bytes.partition(b":")
768766

769767
# Each record is a metadata field starting with a colon, followed by one path,
770768
# or two for copies and renames, each terminated by NUL. A path may itself
771769
# start with a colon or contain a line feed, so read the fields in order
772770
# instead of splitting at colons.
773-
fields = iter(lines.rstrip("\n").split("\x00"))
771+
fields = iter(lines.rstrip(b"\n").split(b"\x00"))
774772
for meta in fields:
775773
if not meta:
776774
# The line data is empty, skip.
777775
continue
778-
if meta.startswith(":"):
779-
meta = meta[1:]
776+
metadata = (meta[1:] if meta.startswith(b":") else meta).decode("ascii")
780777
a_blob_id: Optional[str]
781778
b_blob_id: Optional[str]
782-
old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4)
779+
old_mode, new_mode, a_blob_id, b_blob_id, _change_type = metadata.split(None, 4)
783780
# Change type can be R100
784781
# R: status letter
785782
# 100: score (in case of copy and rename)
786783
change_type: Lit_change_type = cast(Lit_change_type, _change_type[0])
787784
score_str = "".join(_change_type[1:])
788785
score = int(score_str) if score_str.isdigit() else None
789-
path = next(fields, "")
790-
a_path = path.encode(defenc)
791-
b_path = path.encode(defenc)
786+
a_path = b_path = next(fields, b"")
792787
deleted_file = False
793788
new_file = False
794789
copied_file = False
@@ -805,9 +800,9 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]
805800
new_file = True
806801
elif change_type == "C":
807802
copied_file = True
808-
b_path = next(fields, "").encode(defenc)
803+
b_path = next(fields, b"")
809804
elif change_type == "R":
810-
b_path = next(fields, "").encode(defenc)
805+
b_path = next(fields, b"")
811806
rename_from, rename_to = a_path, b_path
812807
elif change_type == "T":
813808
# Nothing to do.

‎test/test_diff.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import os.path as osp
77

88
import ddt
9+
import pytest
910

1011
from git import NULL_TREE, Diff, DiffIndex, Diffable, Repo
1112
from git.diff import decode_path
@@ -19,6 +20,22 @@ def to_raw(input):
1920
return input.replace(b"\t", b"\x00")
2021

2122

23+
@pytest.mark.parametrize("change", [b"A", b"D", b"M", b"R100", b"C100"])
24+
def test_raw_diff_preserves_non_utf8_paths(change):
25+
before = b"legacy-\xf5\t:\n.pas"
26+
after = b"renamed-\xe3.pas"
27+
output = b":100644 100644 " + b"1" * 40 + b" " + b"2" * 40 + b" " + change + b"\0" + before + b"\0"
28+
if change.startswith((b"R", b"C")):
29+
output += after + b"\0"
30+
diffs = DiffIndex()
31+
Diff._handle_diff_line(output, None, diffs)
32+
assert len(diffs) == 1
33+
assert diffs[0].a_rawpath == before
34+
assert diffs[0].b_rawpath == (after if change.startswith((b"R", b"C")) else before)
35+
if change.startswith(b"R"):
36+
assert (diffs[0].raw_rename_from, diffs[0].raw_rename_to) == (before, after)
37+
38+
2239
@ddt.ddt
2340
class TestDiff(TestBase):
2441
def setUp(self):

‎test/test_index.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
from git import NULL_TREE, BlobFilter, Diff, Git, IndexFile, Object, Repo, Tree
2020
from git.diff import NULL_TREE_SHA
21+
from git.compat import defenc
2122
from git.exc import (
2223
CheckoutError,
2324
GitCommandError,
@@ -38,6 +39,19 @@
3839
_logger = logging.getLogger(__name__)
3940

4041

42+
def test_staged_legacy_filename_and_blob_bytes():
43+
with fixture_repo("legacy-index.sh") as repo:
44+
(entry,) = repo.index.iter_entries()
45+
assert entry.path.encode(defenc, "surrogateescape") == b"fonte-\xf5.pas"
46+
assert repo.odb.stream(entry.binsha).read() == b"// vers\xe3o nova\n"
47+
(diff,) = repo.index.diff("HEAD")
48+
assert (diff.a_rawpath, diff.b_rawpath) == (b"fonte-\xf5.pas", b"fonte-\xf5.pas")
49+
assert {blob.data_stream.read() for blob in (diff.a_blob, diff.b_blob)} == {
50+
b"// vers\xe3o original\n",
51+
b"// vers\xe3o nova\n",
52+
}
53+
54+
4155
@ddt.ddt
4256
class TestIndex(TestBase):
4357
@with_rw_repo("HEAD")

‎tests/fixtures/legacy-index.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
set -eu
3+
git init --initial-branch=main repo
4+
cd repo
5+
original=$(printf '// vers\343o original\n' | git hash-object -w --stdin)
6+
printf '100644 %s\tfonte-\365.pas\0' "$original" | git update-index -z --index-info
7+
git commit -m initial
8+
staged=$(printf '// vers\343o nova\n' | git hash-object -w --stdin)
9+
printf '100644 %s\tfonte-\365.pas\0' "$staged" | git update-index -z --index-info

0 commit comments

Comments
 (0)