fix: bound tar.gz metadata decompression

docs/reference/catalogs.md

@@ -109,6 +109,16 @@ configuration does not mean its sources or components were approved. Run the
 relevant `catalog list` command and inspect every install-allowed source before
 installing or running components from an unfamiliar project.
 
+Shared archive validation limits the decompressed tar.gz stream to 54 MiB,
+including tar headers, padding, and GNU/PAX metadata as well as file contents.
+Each GNU/PAX metadata payload is also capped at 64 KiB before it is read or
+parsed. These limits apply during format detection and extraction; ordinary
+member-size and total-file-size limits still apply. Packages that exceed these
+limits must be reduced before installation.
+Old GNU sparse archives and GNU sparse 1.0 archives are rejected before their
+separately stored maps are parsed; sparse maps contained in bounded PAX 0.0/0.1
+metadata remain supported.
+
 ## Host a private catalog
 
 1. Review and approve the entries and release artifacts before registering an

src/specify_cli/_download_security.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import gzip
 import io
 import re
 import socket
@@ -32,6 +33,9 @@
 # ``ZipFile`` reads this whole structure into memory. Four MiB leaves roughly
 # 8 KiB of filename/extra/comment metadata for each of the 512 allowed entries.
 MAX_ZIP_CENTRAL_DIRECTORY_BYTES = 4 * 1024 * 1024
+# Include tar headers, padding, and GNU/PAX metadata, not just file payloads.
+MAX_TAR_STREAM_BYTES = MAX_ZIP_TOTAL_BYTES + 4 * 1024 * 1024
+MAX_TAR_METADATA_BYTES = 64 * 1024
 READ_CHUNK_SIZE = 64 * 1024
 
 # Tighter ceilings for responses that are read fully into memory and parsed as
@@ -92,6 +96,74 @@
 }
 
 
+class _TarLimitError(tarfile.ReadError):
+    pass
+
+
+def is_archive_safety_error(exc: Exception) -> bool:
+    """Identify a tar safety signal retained by shared error translation."""
+    return isinstance(exc, _TarLimitError) or isinstance(exc.__cause__, _TarLimitError)
+
+
+class _BoundedTarInfo(tarfile.TarInfo):
+    """Check every metadata header, including headers parsed recursively."""
+
+    def _proc_member(self, archive: tarfile.TarFile) -> tarfile.TarInfo:
+        if self.type == tarfile.GNUTYPE_SPARSE:
+            raise _TarLimitError("Unsupported GNU sparse metadata in tar.gz archive")
+        if self.type in (
+            tarfile.GNUTYPE_LONGNAME,
+            tarfile.GNUTYPE_LONGLINK,
+            tarfile.XHDTYPE,
+            tarfile.XGLTYPE,
+            tarfile.SOLARIS_XHDTYPE,
+        ) and self.size > MAX_TAR_METADATA_BYTES:
+            raise _TarLimitError(
+                "tar.gz metadata payload exceeds maximum size "
+                f"of {MAX_TAR_METADATA_BYTES} bytes"
+            )
+        return super()._proc_member(archive)
+
+    def _proc_gnusparse_10(
+        self,
+        next_member: tarfile.TarInfo,
+        pax_headers: dict[str, str],
+        archive: tarfile.TarFile,
+    ) -> NoReturn:
+        # This map lives in file data, outside the bounded PAX header payload.
+        raise _TarLimitError("Unsupported GNU sparse metadata in tar.gz archive")
+
+
+class _BoundedTarGzipFile(gzip.GzipFile):
+    """Bound decompressed offsets before tarfile reads metadata or skips data."""
+
+    def _check_position(self, position: int) -> None:
+        if position > MAX_TAR_STREAM_BYTES:
+            raise _TarLimitError(
+                "tar.gz archive exceeds maximum decompressed stream size "
+                f"of {MAX_TAR_STREAM_BYTES} bytes"
+            )
+
+    def read(self, size: int = -1) -> bytes:
+        position = super().seek(0, io.SEEK_CUR)
+        if size >= 0:
+            self._check_position(position + size)
+        else:
+            size = MAX_TAR_STREAM_BYTES - position + 1
+        data = super().read(size)
+        self._check_position(position + len(data))
+        return data
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        if whence == io.SEEK_CUR:
+            offset += super().seek(0, io.SEEK_CUR)
+        elif whence != io.SEEK_SET:
+            # SEEK_END would decompress the whole stream before checking it.
+            raise io.UnsupportedOperation("Unbounded tar.gz seek")
+        self._check_position(offset)
+        return super().seek(offset)
+
+
 def archive_format_from_name(name: str) -> ArchiveFormat | None:
     """Return the supported archive format declared by a path or URL."""
     try:
@@ -178,15 +250,24 @@ def detect_archive_format(
             is_tar_gz = False
             if is_gzip:
                 try:
-                    with tarfile.open(fileobj=archive_file, mode="r:gz"):
+                    with (
+                        _BoundedTarGzipFile(fileobj=archive_file, mode="rb") as source,
+                        tarfile.open(
+                            fileobj=source, mode="r:", tarinfo=_BoundedTarInfo
+                        ),
+                    ):
                         is_tar_gz = True
-                except _TAR_DECOMPRESSION_ERRORS:
+                except _TarLimitError:
+                    raise
+                except (*_TAR_DECOMPRESSION_ERRORS, gzip.BadGzipFile):
                     # A truncated gzip stream raises a bare EOFError here rather
                     # than a TarError, so catching only TarError let it escape
                     # this probe as a raw exception instead of leaving
                     # ``is_tar_gz`` False and reporting the format mismatch.
                     pass
             archive_file.seek(0)
+        except _TarLimitError as exc:
+            _raise_from(error_type, str(exc), exc)
         except OSError as exc:
             _raise_from(error_type, f"Invalid archive: {archive_path}", exc)
 
@@ -1087,18 +1168,23 @@ def safe_extract_tar(
     except OSError as exc:
         _raise_from(error_type, f"Invalid tar extraction target: {target_dir}", exc)
 
-    try:
-        if archive_file is not None:
-            archive_file.seek(0)
-        archive = tarfile.open(
-            archive_path if archive_file is None else None,
-            mode="r:gz",
-            fileobj=archive_file,
-        )
-    except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
-        _raise_from(error_type, f"Invalid tar.gz archive: {archive_path}", exc)
+    with ExitStack() as stack:
+        try:
+            if archive_file is not None:
+                archive_file.seek(0)
+            source = stack.enter_context(
+                _BoundedTarGzipFile(
+                    archive_path if archive_file is None else None,
+                    mode="rb",
+                    fileobj=archive_file,
+                )
+            )
+            archive = stack.enter_context(
+                tarfile.open(fileobj=source, mode="r:", tarinfo=_BoundedTarInfo)
+            )
+        except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
+            _raise_from(error_type, f"Invalid tar.gz archive: {archive_path}: {exc}", exc)
 
-    with archive:
         validated: list[tuple[tarfile.TarInfo, str, bool]] = []
         validated_paths: dict[tuple[str, ...], tuple[str, bool]] = {}
         total_size = 0
@@ -1170,7 +1256,7 @@ def safe_extract_tar(
         except (*_TAR_DECOMPRESSION_ERRORS, OSError) as exc:
             _raise_from(
                 error_type,
-                f"Invalid tar.gz archive: {archive_path}",
+                f"Invalid tar.gz archive: {archive_path}: {exc}",
                 exc,
             )
 

src/specify_cli/extensions/_commands.py

@@ -24,6 +24,7 @@
 from .._download_security import (
     archive_format_from_name,
     detect_archive_format,
+    is_archive_safety_error,
     is_https_or_localhost_http,
     read_response_limited,
 )
@@ -254,6 +255,8 @@ def install_extension_from_url(
                 error_type=ExtensionError,
             )
         except ExtensionError as exc:
+            if is_archive_safety_error(exc):
+                raise
             raise ExtensionError(
                 f"{url} did not return a ZIP archive or tar.gz/tgz archive "
                 f"(got {len(archive_data)} bytes). This usually means the request "

src/specify_cli/workflows/_commands.py

@@ -23,6 +23,7 @@
     archive_format_from_name,
     archive_suffix as archive_suffix,
     detect_archive_format,
+    is_archive_safety_error,
     is_https_or_localhost_http,
     is_safe_download_redirect,
     read_response_limited,
@@ -296,7 +297,9 @@ def _sniff_workflow_archive_format(data: bytes):
             Path("workflow-download"),
             archive_file=BytesIO(data),
         )
-    except ValueError:
+    except ValueError as exc:
+        if is_archive_safety_error(exc):
+            raise
         return None
 
 

tests/archive_helpers.py

@@ -0,0 +1,53 @@
+"""Small compressed archives exercising shared tar safety rejections."""
+
+import gzip
+import io
+import tarfile
+
+
+def tar_safety_archive(kind: str) -> bytes:
+    if kind in ("stream", "metadata"):
+        size = 4096 if kind == "stream" else 65537
+        prefix = f"{size} comment=".encode()
+        payload = prefix + b"x" * (size - len(prefix) - 1) + b"\n"
+        info = tarfile.TarInfo("metadata")
+        info.type = tarfile.XHDTYPE
+        info.size = size
+        plain = info.tobuf() + payload + b"\0" * (-size % tarfile.BLOCKSIZE)
+        plain += tarfile.TarInfo("file.txt").tobuf()
+    elif kind == "gnu-sparse":
+        info = tarfile.TarInfo("file.txt")
+        info.type = tarfile.GNUTYPE_SPARSE
+        header = bytearray(info.tobuf(format=tarfile.GNU_FORMAT))
+        header[482] = 1  # The old GNU format stores map extensions after this header.
+        header[483:495] = tarfile.itn(2, 12)
+        header[148:156] = b" " * 8
+        header[148:156] = f"{sum(header):06o}\0 ".encode()
+        blocks = []
+        for index in range(160):
+            block = bytearray(tarfile.BLOCKSIZE)
+            for entry in range(21):
+                offset = entry * 24
+                block[offset:offset + 24] = tarfile.itn(1, 12) * 2
+            block[504] = int(index < 159)
+            blocks.append(bytes(block))
+        plain = bytes(header) + b"".join(blocks)
+    elif kind == "pax-sparse-1.0":
+        sparse_map = (b"20000\n" + b"1\n1\n" * 20000)
+        payload = sparse_map.ljust(
+            (len(sparse_map) + 511) // 512 * 512, b"\0"
+        )
+        info = tarfile.TarInfo("file.txt")
+        info.size = len(payload)
+        info.pax_headers = {
+            "GNU.sparse.major": "1",
+            "GNU.sparse.minor": "0",
+            "GNU.sparse.realsize": "2",
+        }
+        buffer = io.BytesIO()
+        with tarfile.open(fileobj=buffer, mode="w", format=tarfile.PAX_FORMAT) as archive:
+            archive.addfile(info, io.BytesIO(payload))
+        plain = buffer.getvalue()
+    else:
+        raise ValueError(f"Unknown test archive kind: {kind}")
+    return gzip.compress(plain + b"\0" * tarfile.BLOCKSIZE)

tests/specify_cli/extensions/test_command_add.py

@@ -27,6 +27,7 @@
     ValidationError,
 )
 from tests.conftest import strip_ansi
+from tests.archive_helpers import tar_safety_archive
 from tests.specify_cli.extensions._helpers import (
     LISTED_COMMAND_NAME_CASES,
     MINIMAL_ZIP_BYTES as _MINIMAL_ZIP_BYTES,
@@ -40,6 +41,54 @@
 class TestExtensionAddCLI:
     """CLI tests for ``specify extension add``."""
 
+    @pytest.mark.parametrize(
+        "kind, diagnostic",
+        [
+            ("stream", "maximum decompressed"),
+            ("metadata", "metadata payload exceeds"),
+            ("gnu-sparse", "Unsupported GNU sparse"),
+            ("pax-sparse-1.0", "Unsupported GNU sparse"),
+        ],
+    )
+    @pytest.mark.parametrize("suffix", ["ext.tar.gz", "assets/42"])
+    def test_add_from_url_preserves_archive_safety_errors(
+        self, project_dir, monkeypatch, kind, diagnostic, suffix
+    ):
+        from specify_cli import _download_security
+
+        if kind == "stream":
+            monkeypatch.setattr(_download_security, "MAX_TAR_STREAM_BYTES", 2048)
+        with (
+            patch.object(Path, "cwd", return_value=project_dir),
+            patch("typer.confirm", return_value=True),
+            patch(
+                "specify_cli.extensions._commands._validate_safe_cache_dir",
+                side_effect=_validate_safe_cache_dir_test_stand_in,
+            ),
+            patch(
+                "specify_cli.extensions._commands._safe_open_download_zip",
+                side_effect=_open_test_download_zip,
+            ),
+            patch(
+                "specify_cli.authentication.http.open_url",
+                return_value=io.BytesIO(tar_safety_archive(kind)),
+            ),
+            patch.object(ExtensionManager, "install_from_zip") as install,
+        ):
+            result = CliRunner().invoke(
+                app,
+                ["extension", "add", "unsafe", "--from", f"https://example.com/{suffix}"],
+            )
+
+        output = " ".join(strip_ansi(result.output).split())
+        assert result.exit_code == 1, output
+        assert isinstance(result.exception, SystemExit), repr(result.exception)
+        assert diagnostic in output
+        assert "not authenticated" not in output
+        assert "Traceback" not in output
+        install.assert_not_called()
+        assert not (project_dir / ".specify" / "extensions" / "unsafe").exists()
+
     @pytest.mark.parametrize("command", [
         pytest.param(["speckit.test-ext.hello"], id="list"),
         pytest.param({"name": "speckit.test-ext.hello"}, id="mapping"),

tests/specify_cli/workflows/test_command_add.py

@@ -18,6 +18,8 @@
 from typer.testing import CliRunner
 
 from specify_cli import app
+from tests.archive_helpers import tar_safety_archive
+from tests.conftest import strip_ansi
 
 runner = CliRunner()
 
@@ -1382,6 +1384,55 @@ def test_add_from_suffixless_url_sniffs_archive(
         installed = project_dir / ".specify" / "workflows" / "align-wf"
         assert (installed / "assets" / "sniffed.txt").read_text() == "sniffed\n"
 
+    @pytest.mark.parametrize("source", ["url", "catalog"])
+    @pytest.mark.parametrize(
+        "kind, diagnostic",
+        [
+            ("stream", "maximum decompressed"),
+            ("metadata", "metadata payload exceeds"),
+            ("gnu-sparse", "Unsupported GNU sparse"),
+            ("pax-sparse-1.0", "Unsupported GNU sparse"),
+        ],
+    )
+    def test_suffixless_download_preserves_archive_safety_errors(
+        self, project_dir, monkeypatch, source, kind, diagnostic
+    ):
+        from unittest.mock import patch
+        from specify_cli import _download_security
+        from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry
+
+        monkeypatch.chdir(project_dir)
+        if kind == "stream":
+            monkeypatch.setattr(_download_security, "MAX_TAR_STREAM_BYTES", 2048)
+        url = "https://example.com/assets/12345"
+        data = tar_safety_archive(kind)
+        info = {
+            "id": "align-wf", "name": "Align Workflow", "version": "1.0.0",
+            "url": url, "_install_allowed": True, "_catalog_name": "test",
+        }
+        args = ["workflow", "add", "align-wf"]
+        if source == "url":
+            args += ["--from", url]
+        with (
+            patch.object(WorkflowCatalog, "get_workflow_info", return_value=info),
+            patch(
+                "specify_cli.authentication.http.open_url",
+                side_effect=lambda *_args, **_kwargs: self._FakeResponse(
+                    data, url, {"Content-Type": "application/octet-stream"},
+                ),
+            ),
+        ):
+            result = CliRunner().invoke(app, args, input="y\n")
+
+        output = " ".join(strip_ansi(result.output).split())
+        assert result.exit_code == 1, output
+        assert isinstance(result.exception, SystemExit), repr(result.exception)
+        assert diagnostic in output
+        assert "Invalid workflow YAML" not in output
+        assert "Traceback" not in output
+        assert not WorkflowRegistry(project_dir).is_installed("align-wf")
+        assert not (project_dir / ".specify" / "workflows" / "align-wf").exists()
+
     @pytest.mark.parametrize("suffix", [".zip", ".tar.gz", ".tgz"])
     def test_add_catalog_installs_complete_archive_package_and_sha(
         self, project_dir, monkeypatch, suffix