diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6862480bae..1103aabded 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,6 +10,8 @@ on: - '.github/workflows/deploy.yml' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + schedule: + - cron: "17 6 * * *" # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: @@ -47,6 +49,16 @@ jobs: run: | pip install -r docs/requirements-docs.txt + - name: Restore download statistics + uses: actions/cache@v4 + with: + path: docs/source/_static/pypi-downloads.json + key: pypi-downloads-v1-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: pypi-downloads-v1- + + - name: Refresh download statistics + run: python docs/scripts/update_pypi_downloads.py + - name: Build documentation run: | cd docs diff --git a/.github/workflows/pre-release.yaml b/.github/workflows/pre-release.yaml index 4af4c74e49..1e32700d79 100644 --- a/.github/workflows/pre-release.yaml +++ b/.github/workflows/pre-release.yaml @@ -176,7 +176,10 @@ jobs: export CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}" source /usr/local/Ascend/cann/set_env.sh cd build - cmake --build . -j"$(nproc)" + # Cap parallelism: the mooncake-store TUs (~3-4GB each with Ascend + # headers) at -j$(nproc)=4 peak at 12-16GB and get the 16GB runner + # killed mid-compile. -j2 peaks at ~7GB and builds in ~2x the time. + cmake --build . -j2 sudo cmake --install . - name: Generate Python version tag diff --git a/.github/workflows/release-npu.yaml b/.github/workflows/release-npu.yaml index 9ef99593d3..2c3deab775 100644 --- a/.github/workflows/release-npu.yaml +++ b/.github/workflows/release-npu.yaml @@ -128,7 +128,10 @@ jobs: source /usr/local/Ascend/cann/set_env.sh export PATH="/usr/local/go/bin:${PATH}" cd build - cmake --build . -j"$(nproc)" + # Cap parallelism: the mooncake-store TUs (~3-4GB each with Ascend + # headers) at -j$(nproc)=4 peak at 12-16GB and get the 16GB runner + # killed mid-compile. -j2 peaks at ~7GB and builds in ~2x the time. + cmake --build . -j2 sudo cmake --install . - name: Generate Python version tag diff --git a/docs/README.md b/docs/README.md index b3f6798dbe..f7d65c1615 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,3 +43,23 @@ If port 8000 is already in use, you can specify a different port, for example: ```bash python -m http.server 3000 -d build/html/ ``` + +## PyPI downloads page + +`source/community/pypi-downloads.md` uses a same-origin snapshot in +`source/_static/pypi-downloads.json`; it does not depend on an external app. +The Pages workflow refreshes this snapshot during deployment and runs daily. +Successful endpoint responses are reused for 24 hours, and the workflow cache +keeps the last available responses when PyPIStats is unavailable. Collection +age and partial data are visible on the page. The committed snapshot also +supports offline documentation builds. + +To refresh manually and run the offline checks, from `docs/`: + +```bash +python scripts/update_pypi_downloads.py +python -m unittest discover -s tests -p 'test_pypi_downloads.py' +node --test tests/pypi-downloads.test.cjs +``` + +Avoid repeatedly fetching the same PyPIStats endpoints within a day. diff --git a/docs/scripts/update_pypi_downloads.py b/docs/scripts/update_pypi_downloads.py new file mode 100644 index 0000000000..cc1f402ed1 --- /dev/null +++ b/docs/scripts/update_pypi_downloads.py @@ -0,0 +1,135 @@ +"""Refresh the static PyPIStats snapshot, keeping usable data on upstream failure.""" + +import argparse +import copy +import datetime as dt +import json +import re +import time +import urllib.request +from pathlib import Path + +PREFIX = "mooncake-transfer-engine" +ENDPOINTS = ("recent", "overall", "python_major", "python_minor", "system") +DEFAULT_PATH = ( + Path(__file__).resolve().parents[1] / "source/_static/pypi-downloads.json" +) + + +def now(): + return dt.datetime.now(dt.timezone.utc) + + +def fresh(result): + try: + age = now() - dt.datetime.fromisoformat(result["fetchedAt"]) + return result["status"] == "ok" and dt.timedelta(0) <= age < dt.timedelta( + days=1 + ) + except (KeyError, ValueError, TypeError): + return False + + +def open_url(url): + request = urllib.request.Request( + url, headers={"User-Agent": "Mooncake-Docs-PyPIStats/1.0"} + ) + return urllib.request.urlopen(request, timeout=30) + + +def discover(): + with open_url("https://pypi.org/simple/") as response: + names = set() + for line in response: + names.update( + re.findall(rb">(mooncake-transfer-engine(?:-[a-z0-9-]+)?)<", line) + ) + if PREFIX.encode() not in names: + raise ValueError("Incomplete PyPI package index") + return sorted(name.decode() for name in names) + + +def validate(payload, name, endpoint): + if payload.get("package") != name: + raise ValueError("Package name mismatch") + data = payload["data"] + if endpoint == "recent": + values = [data[key] for key in ("last_day", "last_week", "last_month")] + else: + values = [row["downloads"] for row in data] + for row in data: + dt.date.fromisoformat(row["date"]) + if row["category"] is not None and not isinstance(row["category"], str): + raise ValueError("Invalid category") + keys = [(row["date"], row["category"]) for row in data] + if len(set(keys)) != len(keys): + raise ValueError("Duplicate date/category") + if not all(type(value) is int and value >= 0 for value in values): + raise ValueError("Invalid download count") + return payload + + +def update(previous): + result = copy.deepcopy(previous) + catalog = result["catalog"] + if not fresh(catalog): + try: + result["catalog"] = dict( + names=discover(), + source="https://pypi.org/simple/", + fetchedAt=now().isoformat(), + status="ok", + ) + except Exception as error: + catalog.update(status="stale", error=str(error)) + by_name = {p["name"]: p for p in result["packages"]} + result["packages"] = [] + rate_limited = False + for name in result["catalog"]["names"]: + package = by_name.get(name, {"name": name, "endpoints": {}}) + for endpoint in ENDPOINTS: + old = package["endpoints"].get(endpoint, {}) + if fresh(old) and "payload" in old: + continue + url = f"https://pypistats.org/api/packages/{name}/{endpoint}" + try: + if rate_limited: + raise RuntimeError("Skipped after upstream rate limit") + try: + with open_url(url) as response: + payload = validate(json.load(response), name, endpoint) + finally: + time.sleep(1.1) + package["endpoints"][endpoint] = dict( + payload=payload, + source=url, + fetchedAt=now().isoformat(), + status="ok", + ) + except Exception as error: + rate_limited |= getattr(error, "code", None) == 429 + package["endpoints"][endpoint] = { + **old, + "source": url, + "error": str(error), + "status": "stale" if "payload" in old else "error", + } + print(f"Warning: {name}/{endpoint}: {error}") + result["packages"].append(package) + result["collectedAt"] = now().isoformat() + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, default=DEFAULT_PATH) + args = parser.parse_args() + result = update(json.loads(args.output.read_text())) + temporary = args.output.with_suffix(".tmp") + temporary.write_text(json.dumps(result, separators=(",", ":")) + "\n") + temporary.replace(args.output) + print(f"Saved {len(result['packages'])} packages to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/_static/pypi-downloads.css b/docs/source/_static/pypi-downloads.css new file mode 100644 index 0000000000..f64169336b --- /dev/null +++ b/docs/source/_static/pypi-downloads.css @@ -0,0 +1,54 @@ +/* Scoped to the downloads widget; navigation and theme stay Sphinx-native. */ +.pypi-downloads { margin: 1.2rem 0 1.6rem; } +.pypi-downloads button, .pypi-downloads select { + font: inherit; font-size: .8rem; color: var(--pst-color-text-base); + border: 1px solid var(--pst-color-border); border-radius: .35rem; + padding: .3rem .55rem; background: var(--pst-color-background); +} +.pypi-downloads button { cursor: pointer; } +.pypi-downloads :focus-visible { outline: 2px solid var(--pst-color-primary); outline-offset: 2px; } +.pypi-downloads .pypi-status { color: var(--pst-color-text-muted); font-size: .76rem; line-height: 1.6; margin: .6rem 0; } +.pypi-controls { display: flex; align-items: center; gap: .8rem; flex-wrap: wrap; font-size: .8rem; } +.pypi-controls label { display: flex; align-items: center; gap: .35rem; margin: 0; } +.pypi-controls button { margin-left: auto; } +.pypi-packages { display: flex; gap: .35rem; flex-wrap: wrap; margin: .8rem 0 1.3rem; } +.pypi-packages button[aria-pressed="true"] { color: var(--pst-color-primary); background: var(--pst-color-surface); border-color: var(--pst-color-primary); } +.pypi-packages button[aria-pressed="false"] { opacity: .6; } +.pypi-metrics { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: .7rem; } +.pypi-metrics > div { padding: .8rem 1rem; border: 1px solid var(--pst-color-border); border-radius: .45rem; } +.pypi-metrics span, .pypi-metrics small { display: block; color: var(--pst-color-text-muted); font-size: .75rem; } +.pypi-metrics strong { display: block; font-size: 1.6rem; font-weight: 600; margin: .3rem 0; font-variant-numeric: tabular-nums; } +.pypi-metrics small { font-size: .65rem; } +.pypi-chart-heading { display: flex; justify-content: space-between; gap: .8rem; flex-wrap: wrap; align-items: baseline; margin-top: 1.5rem; } +.pypi-chart-heading strong { font-size: .95rem; } +.pypi-chart-heading span { font-size: .7rem; color: var(--pst-color-text-muted); } +.pypi-chart svg { width: 100%; height: auto; display: block; } +.pypi-chart svg text { font: 11px sans-serif; fill: var(--pst-color-text-muted); } +.pypi-grid { stroke: var(--pst-color-border); stroke-width: 1; stroke-dasharray: 3 4; } +.pypi-point { opacity: .2; } +.pypi-point:hover, .pypi-point:focus { opacity: 1; r: 5; outline: none; } +.pypi-legend { display: flex; flex-wrap: wrap; gap: .8rem; font-size: .7rem; color: var(--pst-color-text-muted); } +.pypi-legend span { display: flex; align-items: center; gap: .3rem; } +.pypi-legend i { width: .4rem; height: .4rem; border-radius: 50%; } +.pypi-table { overflow-x: auto; margin-top: 1rem; } +.pypi-table table { width: 100%; border-collapse: collapse; font-size: .78rem; } +.pypi-table th, .pypi-table td { padding: .55rem .65rem; border-bottom: 1px solid var(--pst-color-border); text-align: right; white-space: nowrap; font-variant-numeric: tabular-nums; } +.pypi-table th:first-child { text-align: left; } +.pypi-table thead th { color: var(--pst-color-text-muted); font-size: .7rem; font-weight: 500; } +.pypi-table tbody th { font-weight: 500; } +.pypi-environment { border-top: 1px solid var(--pst-color-border); padding-top: .8rem; margin-top: 1.3rem; } +.pypi-environment summary { cursor: pointer; font-size: .85rem; font-weight: 600; } +.pypi-columns { display: grid; grid-template-columns: 1fr 1fr; gap: 2rem; margin-top: 1rem; } +.pypi-columns h2 { font-size: .9rem; margin: 0 0 .6rem; } +.pypi-bar { margin: .8rem 0; } +.pypi-bar > span { display: flex; justify-content: space-between; gap: .5rem; font-size: .7rem; } +.pypi-bar meter { width: 100%; height: .5rem; display: block; margin-top: .25rem; accent-color: var(--pst-color-primary); } +.pypi-downloads .pypi-warning { font-size: .8rem; border-left: 3px solid var(--pst-color-warning); padding: .5rem .8rem; } +@media (max-width: 600px) { + .pypi-metrics { gap: .4rem; } + .pypi-metrics > div { padding: .65rem .5rem; } + .pypi-metrics strong { font-size: 1.25rem; } + .pypi-metrics span { font-size: .65rem; } + .pypi-columns { grid-template-columns: 1fr; gap: 1rem; } + .pypi-controls { gap: .5rem; } +} diff --git a/docs/source/_static/pypi-downloads.js b/docs/source/_static/pypi-downloads.js new file mode 100644 index 0000000000..86b1dabbcd --- /dev/null +++ b/docs/source/_static/pypi-downloads.js @@ -0,0 +1,162 @@ +// Static, same-origin PyPIStats snapshot. No browser requests to PyPIStats. +(() => { + "use strict"; + const PREFIX = "mooncake-transfer-engine"; + const COLORS = ["#2674d9", "#149b89", "#9371ce", "#ba8029", "#cf647b", "#4c94b1", "#9b7559", "#697d99", "#a45fa8"]; + const shortName = name => name === PREFIX ? "Original" : name.slice(PREFIX.length + 1); + const format = value => value == null ? "—" : value.toLocaleString("en-US"); + const rows = (pkg, endpoint) => { + const data = pkg.endpoints[endpoint]?.payload?.data; + return Array.isArray(data) ? data : []; + }; + const recent = (pkg, key) => pkg.endpoints.recent?.payload?.data?.[key] ?? null; + const sum = values => values.some(v => v != null) ? values.reduce((a, b) => a + (b ?? 0), 0) : null; + const offset = (date, days) => new Date(Date.parse(date + "T00:00:00Z") + days * 86400000).toISOString().slice(0, 10); + const inRange = (row, start, end) => row.date >= start && row.date <= end; + const rangeTotal = (pkg, start, end, category) => sum(rows(pkg, "overall") + .filter(r => inRange(r, start, end) && r.category === category).map(r => r.downloads)); + function series(packages, start, end, category) { + if (!start || !end || start > end) return []; + const maps = packages.map(p => new Map(rows(p, "overall").filter(r => r.category === category).map(r => [r.date, r.downloads]))); + const result = []; + for (let date = start; date <= end; date = offset(date, 1)) { + const values = maps.map(m => m.get(date) ?? null); + result.push({ date, values, total: sum(values), reporting: values.filter(v => v != null).length }); + } + return result; + } + function breakdown(packages, endpoint, start, end) { + const values = new Map(); + packages.flatMap(p => rows(p, endpoint)).filter(r => inRange(r, start, end)).forEach(r => { + const key = r.category == null || r.category === "null" ? "Unknown" : r.category; + values.set(key, (values.get(key) ?? 0) + r.downloads); + }); + return [...values].sort((a, b) => b[1] - a[1]); + } + // Pure aggregation functions are also exercised by the Node test runner. + if (typeof module !== "undefined") module.exports = { sum, recent, rangeTotal, series, breakdown }; + if (typeof document === "undefined") return; + const el = (tag, props = {}, ...children) => { + const node = document.createElement(tag); + for (const [key, value] of Object.entries(props)) { + if (key === "text") node.textContent = value; + else if (key.startsWith("on")) node.addEventListener(key.slice(2), value); + else node.setAttribute(key, value); + } + for (const child of children.flat()) if (child != null) node.append(child instanceof Node ? child : String(child)); + return node; + }; + const svg = (tag, props, text) => { + const node = document.createElementNS("http://www.w3.org/2000/svg", tag); + Object.entries(props).forEach(([key, value]) => node.setAttribute(key, value)); + if (text != null) node.textContent = text; + return node; + }; + function plot(days, packages, compare) { + const chart = svg("svg", { viewBox: "0 0 720 225", role: "img", "aria-label": "Daily downloads for selected packages" }); + const max = Math.max(1, ...days.flatMap(d => compare ? d.values : [d.total])); + const x = i => 52 + i / Math.max(1, days.length - 1) * 648; + const y = value => 190 - value / max * 165; + for (let i = 0; i <= 4; i++) { + const value = max * i / 4; + chart.append(svg("line", { x1: 52, x2: 700, y1: y(value), y2: y(value), class: "pypi-grid" })); + chart.append(svg("text", { x: 42, y: y(value) + 4, "text-anchor": "end" }, Intl.NumberFormat("en", { notation: "compact", maximumFractionDigits: 1 }).format(value))); + } + [0, Math.floor((days.length - 1) / 2), days.length - 1].filter((v, i, a) => a.indexOf(v) === i).forEach(i => { + chart.append(svg("text", { x: x(i), y: 217, "text-anchor": "middle" }, days[i].date.slice(5))); + }); + const lines = compare ? packages.map((p, i) => ({ name: shortName(p.name), values: days.map(d => d.values[i]), color: COLORS[i % COLORS.length] })) + : [{ name: "Total", values: days.map(d => d.total), color: COLORS[0] }]; + lines.forEach(line => { + let path = "", pen = false; + line.values.forEach((v, i) => { + if (v == null) { pen = false; return; } + path += `${pen ? "L" : "M"}${x(i)},${y(v)} `; + pen = true; + const point = svg("circle", { cx: x(i), cy: y(v), r: 3, fill: line.color, class: "pypi-point", tabindex: "0" }); + const label = `${days[i].date} · ${line.name}: ${format(v)} · ${days[i].reporting}/${packages.length} packages reporting`; + point.setAttribute("aria-label", label); + point.append(svg("title", {}, label)); + chart.append(point); + }); + chart.append(svg("path", { d: path, fill: "none", stroke: line.color, "stroke-width": 2, "pointer-events": "none" })); + }); + const legend = el("div", { class: "pypi-legend" }, lines.map(l => el("span", {}, el("i", { style: `background:${l.color}` }), l.name))); + return el("div", { class: "pypi-chart" }, chart, legend); + } + async function mount(root) { + try { + const response = await fetch(root.dataset.source); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const data = await response.json(); + const selected = new Set(data.catalog.names); + const dates = data.packages.flatMap(p => rows(p, "overall").map(r => r.date)).sort(); + const first = dates[0], last = dates.at(-1); + if (!first || !last) throw new Error("No daily records in this snapshot"); + const controls = el("div", { class: "pypi-controls" }); + const output = el("div", { "aria-live": "polite" }); + const period = el("select", { "aria-label": "Date range", onchange: render }, + [[7, "7 days"], [30, "30 days"], [90, "90 days"], [180, "180 days"], [0, "All dates"]].map(([v, label]) => el("option", { value: v }, label))); + period.value = "30"; + const mode = el("select", { "aria-label": "Chart mode", onchange: render }, el("option", { value: "total" }, "Total"), el("option", { value: "packages" }, "By package")); + const mirrors = el("input", { type: "checkbox", onchange: render }); + const chips = el("div", { class: "pypi-packages", "aria-label": "Packages" }); + data.catalog.names.forEach(name => chips.append(el("button", { type: "button", "aria-pressed": "true", title: name, onclick: event => { + selected.has(name) ? selected.delete(name) : selected.add(name); + event.currentTarget.setAttribute("aria-pressed", String(selected.has(name))); render(); + } }, shortName(name)))); + const all = el("button", { type: "button", onclick: () => { + data.catalog.names.forEach(name => selected.add(name)); + chips.querySelectorAll("button").forEach(b => b.setAttribute("aria-pressed", "true")); render(); + } }, "Select all"); + controls.append(el("label", {}, "Range ", period), mode, el("label", {}, mirrors, " Include mirrors"), all); + const oldest = data.packages.flatMap(p => Object.values(p.endpoints)).filter(r => r.payload && r.fetchedAt).map(r => r.fetchedAt).sort()[0]; + const issues = data.packages.flatMap(p => Object.values(p.endpoints)).filter(r => r.status !== "ok").length; + const age = oldest ? Math.floor((Date.now() - Date.parse(oldest)) / 86400000) : 0; + const status = `Data through ${last} · ${data.catalog.names.length} packages · Collected ${oldest ? oldest.slice(0, 16).replace("T", " ") + " UTC" : "unknown"}`; + root.replaceChildren(el("p", { class: "pypi-status" }, status), controls, chips, output); + if (issues || age > 2 || data.catalog.status !== "ok") root.insertBefore(el("p", { class: "pypi-warning" }, `Some data may be outdated or incomplete (${issues} endpoint warnings). Available records are retained.`), controls); + function render() { + const packages = data.packages.filter(p => selected.has(p.name)); + const start = Number(period.value) ? [first, offset(last, 1 - Number(period.value))].sort().at(-1) : first; + const category = mirrors.checked ? "with_mirrors" : "without_mirrors"; + output.replaceChildren(); + if (!packages.length) { output.append(el("p", {}, "Select a package to view statistics.")); return; } + const cards = el("div", { class: "pypi-metrics" }); + [["Last month · API", "last_month"], ["Last week · API", "last_week"], ["Last day · API", "last_day"]].forEach(([label, key]) => { + const values = packages.map(p => recent(p, key)); + cards.append(el("div", {}, el("span", {}, label), el("strong", {}, format(sum(values))), el("small", {}, `${values.filter(v => v != null).length}/${packages.length} packages · Excludes mirrors`))); + }); + const total = sum(packages.map(p => rangeTotal(p, start, last, category))); + const days = series(packages, start, last, category); + output.append(cards, el("div", { class: "pypi-chart-heading" }, el("strong", {}, `Selected range: ${format(total)}`), el("span", {}, `${start} — ${last} · ${mirrors.checked ? "Includes" : "Excludes"} mirrors`)), plot(days, packages, mode.value === "packages")); + const partial = days.filter(d => d.reporting < packages.length).length; + if (partial) output.append(el("p", { class: "pypi-status" }, `${partial} days have incomplete coverage. The chart sums returned records; missing values are not zero.`)); + const table = el("table", {}, el("thead", {}, el("tr", {}, ["Package", "Day · API", "Week · API", "Month · API", "Selected range"].map(t => el("th", { scope: "col" }, t))))); + const body = el("tbody"); + [...packages].sort((a, b) => (rangeTotal(b, start, last, category) ?? -1) - (rangeTotal(a, start, last, category) ?? -1)).forEach(p => { + body.append(el("tr", {}, el("th", { scope: "row" }, el("a", { href: `https://pypistats.org/packages/${p.name}`, title: p.name }, shortName(p.name))), ...["last_day", "last_week", "last_month"].map(k => el("td", {}, format(recent(p, k)))), el("td", {}, format(rangeTotal(p, start, last, category))))); + }); + table.append(body); + output.append(el("div", { class: "pypi-table" }, table), el("p", { class: "pypi-status" }, "Recent API totals can differ from daily sums. Selected range follows the date and mirror filters.")); + const python = el("select", { "aria-label": "Python version level" }, el("option", { value: "python_minor" }, "Minor versions"), el("option", { value: "python_major" }, "Major versions")); + const py = el("div"), os = el("div"); + function bars(target, endpoint) { + const values = breakdown(packages, endpoint, start, last), total = sum(values.map(v => v[1])); + const reporting = packages.filter(p => rows(p, endpoint).some(r => inRange(r, start, last))).length; + target.replaceChildren(el("p", { class: "pypi-status" }, `${reporting}/${packages.length} packages with data · Excludes mirrors`)); + values.forEach(([label, value]) => target.append(el("div", { class: "pypi-bar" }, el("span", {}, label, el("span", {}, `${format(value)} · ${total ? (value / total * 100).toFixed(1) : "0.0"}%`)), el("meter", { min: "0", max: total || 1, value, "aria-label": label })))); + if (!values.length) target.append(el("p", {}, "No data for this range.")); + } + python.addEventListener("change", () => bars(py, python.value)); + bars(py, python.value); bars(os, "system"); + output.append(el("details", { class: "pypi-environment" }, el("summary", {}, "Environment breakdown"), el("div", { class: "pypi-columns" }, el("section", {}, el("h2", {}, "Python versions"), python, py), el("section", {}, el("h2", {}, "Operating systems"), os)))); + } + render(); + } catch (error) { + root.replaceChildren(el("p", { role: "alert" }, "Download statistics are unavailable. ", el("a", { href: "https://pypistats.org/packages/mooncake-transfer-engine" }, "View PyPIStats"))); + console.error("PyPI downloads:", error); + } + } + document.querySelectorAll("[data-pypi-downloads]").forEach(mount); +})(); diff --git a/docs/source/_static/pypi-downloads.json b/docs/source/_static/pypi-downloads.json new file mode 100644 index 0000000000..f50d09ab9c --- /dev/null +++ b/docs/source/_static/pypi-downloads.json @@ -0,0 +1 @@ +{"collectedAt":"2026-10-29T07:13:19.721202+00:00","catalog":{"names":["mooncake-transfer-engine","mooncake-transfer-engine-cuda13","mooncake-transfer-engine-efa","mooncake-transfer-engine-efa-cuda13","mooncake-transfer-engine-efa-non-cuda","mooncake-transfer-engine-musa","mooncake-transfer-engine-non-cuda","mooncake-transfer-engine-npu","mooncake-transfer-engine-rocm"],"fetchedAt":"2026-10-29T07:11:17.777818+00:00","source":"https://pypi.org/simple/","status":"ok"},"packages":[{"name":"mooncake-transfer-engine","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:11:20.226928+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine/recent","payload":{"data":{"last_day":1388,"last_month":67148,"last_week":17677},"package":"mooncake-transfer-engine","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:11:22.941365+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-04-08","downloads":12383},{"category":"with_mirrors","date":"2026-04-09","downloads":13307},{"category":"with_mirrors","date":"2026-04-10","downloads":12507},{"category":"with_mirrors","date":"2026-04-11","downloads":6497},{"category":"with_mirrors","date":"2026-04-12","downloads":4989},{"category":"with_mirrors","date":"2026-04-13","downloads":10493},{"category":"with_mirrors","date":"2026-04-14","downloads":12795},{"category":"with_mirrors","date":"2026-04-15","downloads":16027},{"category":"with_mirrors","date":"2026-04-16","downloads":12862},{"category":"with_mirrors","date":"2026-04-17","downloads":12378},{"category":"with_mirrors","date":"2026-04-18","downloads":7348},{"category":"with_mirrors","date":"2026-04-19","downloads":4260},{"category":"with_mirrors","date":"2026-04-20","downloads":10557},{"category":"with_mirrors","date":"2026-04-21","downloads":14940},{"category":"with_mirrors","date":"2026-04-22","downloads":14647},{"category":"with_mirrors","date":"2026-04-23","downloads":13424},{"category":"with_mirrors","date":"2026-04-24","downloads":16798},{"category":"with_mirrors","date":"2026-04-25","downloads":7384},{"category":"with_mirrors","date":"2026-04-26","downloads":7231},{"category":"with_mirrors","date":"2026-04-27","downloads":11620},{"category":"with_mirrors","date":"2026-04-28","downloads":14441},{"category":"with_mirrors","date":"2026-04-29","downloads":15077},{"category":"with_mirrors","date":"2026-04-30","downloads":13326},{"category":"with_mirrors","date":"2026-05-01","downloads":14463},{"category":"with_mirrors","date":"2026-05-02","downloads":5111},{"category":"with_mirrors","date":"2026-05-03","downloads":6295},{"category":"with_mirrors","date":"2026-05-04","downloads":10009},{"category":"with_mirrors","date":"2026-05-05","downloads":14958},{"category":"with_mirrors","date":"2026-05-06","downloads":15701},{"category":"with_mirrors","date":"2026-05-07","downloads":15737},{"category":"with_mirrors","date":"2026-05-08","downloads":12643},{"category":"with_mirrors","date":"2026-05-09","downloads":5965},{"category":"with_mirrors","date":"2026-05-10","downloads":3475},{"category":"with_mirrors","date":"2026-05-11","downloads":10587},{"category":"with_mirrors","date":"2026-05-12","downloads":14202},{"category":"with_mirrors","date":"2026-05-13","downloads":14103},{"category":"with_mirrors","date":"2026-05-14","downloads":11997},{"category":"with_mirrors","date":"2026-05-15","downloads":12321},{"category":"with_mirrors","date":"2026-05-16","downloads":5346},{"category":"with_mirrors","date":"2026-05-17","downloads":6190},{"category":"with_mirrors","date":"2026-05-18","downloads":11413},{"category":"with_mirrors","date":"2026-05-19","downloads":10267},{"category":"with_mirrors","date":"2026-05-20","downloads":12623},{"category":"with_mirrors","date":"2026-05-21","downloads":16024},{"category":"with_mirrors","date":"2026-05-22","downloads":12853},{"category":"with_mirrors","date":"2026-05-23","downloads":6237},{"category":"with_mirrors","date":"2026-05-24","downloads":5533},{"category":"with_mirrors","date":"2026-05-25","downloads":5636},{"category":"with_mirrors","date":"2026-05-26","downloads":11354},{"category":"with_mirrors","date":"2026-05-27","downloads":14141},{"category":"with_mirrors","date":"2026-05-28","downloads":13384},{"category":"with_mirrors","date":"2026-05-29","downloads":16171},{"category":"with_mirrors","date":"2026-05-30","downloads":5980},{"category":"with_mirrors","date":"2026-05-31","downloads":6335},{"category":"with_mirrors","date":"2026-06-01","downloads":9687},{"category":"with_mirrors","date":"2026-06-02","downloads":9208},{"category":"with_mirrors","date":"2026-06-03","downloads":8072},{"category":"with_mirrors","date":"2026-06-04","downloads":7641},{"category":"with_mirrors","date":"2026-06-05","downloads":6916},{"category":"with_mirrors","date":"2026-06-06","downloads":4308},{"category":"with_mirrors","date":"2026-06-07","downloads":3044},{"category":"with_mirrors","date":"2026-06-08","downloads":6632},{"category":"with_mirrors","date":"2026-06-09","downloads":9017},{"category":"with_mirrors","date":"2026-06-10","downloads":9894},{"category":"with_mirrors","date":"2026-06-11","downloads":9823},{"category":"with_mirrors","date":"2026-06-12","downloads":7833},{"category":"with_mirrors","date":"2026-06-13","downloads":3682},{"category":"with_mirrors","date":"2026-06-14","downloads":2916},{"category":"with_mirrors","date":"2026-06-15","downloads":5271},{"category":"with_mirrors","date":"2026-06-16","downloads":6155},{"category":"with_mirrors","date":"2026-06-17","downloads":5458},{"category":"with_mirrors","date":"2026-06-18","downloads":5192},{"category":"with_mirrors","date":"2026-06-19","downloads":5166},{"category":"with_mirrors","date":"2026-06-20","downloads":2744},{"category":"with_mirrors","date":"2026-06-21","downloads":2151},{"category":"with_mirrors","date":"2026-06-22","downloads":3989},{"category":"with_mirrors","date":"2026-06-23","downloads":5032},{"category":"with_mirrors","date":"2026-06-24","downloads":5818},{"category":"with_mirrors","date":"2026-06-25","downloads":6434},{"category":"with_mirrors","date":"2026-06-26","downloads":7092},{"category":"with_mirrors","date":"2026-06-27","downloads":3692},{"category":"with_mirrors","date":"2026-06-28","downloads":3002},{"category":"with_mirrors","date":"2026-06-29","downloads":7464},{"category":"with_mirrors","date":"2026-06-30","downloads":8106},{"category":"with_mirrors","date":"2026-07-01","downloads":7817},{"category":"with_mirrors","date":"2026-07-02","downloads":7806},{"category":"with_mirrors","date":"2026-07-03","downloads":5930},{"category":"with_mirrors","date":"2026-07-04","downloads":2805},{"category":"with_mirrors","date":"2026-07-05","downloads":2073},{"category":"with_mirrors","date":"2026-07-06","downloads":7282},{"category":"with_mirrors","date":"2026-07-07","downloads":8016},{"category":"with_mirrors","date":"2026-07-08","downloads":8646},{"category":"with_mirrors","date":"2026-07-09","downloads":8184},{"category":"with_mirrors","date":"2026-07-10","downloads":8333},{"category":"with_mirrors","date":"2026-07-11","downloads":3534},{"category":"with_mirrors","date":"2026-07-12","downloads":2326},{"category":"with_mirrors","date":"2026-07-13","downloads":8348},{"category":"with_mirrors","date":"2026-07-14","downloads":10613},{"category":"with_mirrors","date":"2026-07-15","downloads":11767},{"category":"with_mirrors","date":"2026-07-16","downloads":10389},{"category":"with_mirrors","date":"2026-07-17","downloads":10139},{"category":"with_mirrors","date":"2026-07-18","downloads":5444},{"category":"with_mirrors","date":"2026-07-19","downloads":2464},{"category":"with_mirrors","date":"2026-07-20","downloads":9694},{"category":"with_mirrors","date":"2026-07-21","downloads":12897},{"category":"with_mirrors","date":"2026-07-22","downloads":12346},{"category":"with_mirrors","date":"2026-07-23","downloads":12065},{"category":"with_mirrors","date":"2026-07-24","downloads":12812},{"category":"with_mirrors","date":"2026-07-25","downloads":5912},{"category":"with_mirrors","date":"2026-07-26","downloads":4352},{"category":"with_mirrors","date":"2026-07-27","downloads":11818},{"category":"with_mirrors","date":"2026-07-28","downloads":15125},{"category":"with_mirrors","date":"2026-07-29","downloads":14343},{"category":"with_mirrors","date":"2026-07-30","downloads":15562},{"category":"with_mirrors","date":"2026-07-31","downloads":14399},{"category":"with_mirrors","date":"2026-08-01","downloads":10342},{"category":"with_mirrors","date":"2026-08-02","downloads":6398},{"category":"with_mirrors","date":"2026-08-03","downloads":13240},{"category":"with_mirrors","date":"2026-08-04","downloads":15924},{"category":"with_mirrors","date":"2026-08-05","downloads":15977},{"category":"with_mirrors","date":"2026-08-06","downloads":10060},{"category":"with_mirrors","date":"2026-08-07","downloads":17677},{"category":"with_mirrors","date":"2026-08-08","downloads":7331},{"category":"with_mirrors","date":"2026-08-09","downloads":3186},{"category":"with_mirrors","date":"2026-08-10","downloads":4174},{"category":"with_mirrors","date":"2026-08-11","downloads":4361},{"category":"with_mirrors","date":"2026-08-12","downloads":5513},{"category":"with_mirrors","date":"2026-08-13","downloads":3662},{"category":"with_mirrors","date":"2026-08-14","downloads":4758},{"category":"with_mirrors","date":"2026-08-15","downloads":1755},{"category":"with_mirrors","date":"2026-08-16","downloads":1933},{"category":"with_mirrors","date":"2026-08-17","downloads":2752},{"category":"with_mirrors","date":"2026-08-18","downloads":3708},{"category":"with_mirrors","date":"2026-08-19","downloads":4042},{"category":"with_mirrors","date":"2026-08-20","downloads":3763},{"category":"with_mirrors","date":"2026-08-21","downloads":4109},{"category":"with_mirrors","date":"2026-08-22","downloads":3161},{"category":"with_mirrors","date":"2026-08-23","downloads":2170},{"category":"with_mirrors","date":"2026-08-24","downloads":2411},{"category":"with_mirrors","date":"2026-08-25","downloads":2632},{"category":"with_mirrors","date":"2026-08-26","downloads":2967},{"category":"with_mirrors","date":"2026-08-27","downloads":2456},{"category":"with_mirrors","date":"2026-08-28","downloads":2199},{"category":"with_mirrors","date":"2026-08-29","downloads":1158},{"category":"with_mirrors","date":"2026-08-30","downloads":1178},{"category":"with_mirrors","date":"2026-08-31","downloads":2692},{"category":"with_mirrors","date":"2026-09-01","downloads":2656},{"category":"with_mirrors","date":"2026-09-02","downloads":2677},{"category":"with_mirrors","date":"2026-09-03","downloads":2632},{"category":"with_mirrors","date":"2026-09-04","downloads":2356},{"category":"with_mirrors","date":"2026-09-05","downloads":1436},{"category":"with_mirrors","date":"2026-09-06","downloads":1561},{"category":"with_mirrors","date":"2026-09-07","downloads":1877},{"category":"with_mirrors","date":"2026-09-08","downloads":2678},{"category":"with_mirrors","date":"2026-09-09","downloads":2395},{"category":"with_mirrors","date":"2026-09-10","downloads":2656},{"category":"with_mirrors","date":"2026-09-11","downloads":2805},{"category":"with_mirrors","date":"2026-09-12","downloads":2017},{"category":"with_mirrors","date":"2026-09-13","downloads":1803},{"category":"with_mirrors","date":"2026-09-14","downloads":2087},{"category":"with_mirrors","date":"2026-09-15","downloads":2906},{"category":"with_mirrors","date":"2026-09-16","downloads":3002},{"category":"with_mirrors","date":"2026-09-17","downloads":3099},{"category":"with_mirrors","date":"2026-09-18","downloads":2943},{"category":"with_mirrors","date":"2026-09-19","downloads":1943},{"category":"with_mirrors","date":"2026-09-20","downloads":2494},{"category":"with_mirrors","date":"2026-09-21","downloads":2600},{"category":"with_mirrors","date":"2026-09-22","downloads":3227},{"category":"with_mirrors","date":"2026-09-23","downloads":2716},{"category":"with_mirrors","date":"2026-09-24","downloads":2963},{"category":"with_mirrors","date":"2026-09-25","downloads":2388},{"category":"with_mirrors","date":"2026-09-26","downloads":1521},{"category":"with_mirrors","date":"2026-09-27","downloads":1374},{"category":"with_mirrors","date":"2026-09-28","downloads":1896},{"category":"with_mirrors","date":"2026-09-29","downloads":2214},{"category":"with_mirrors","date":"2026-09-30","downloads":2271},{"category":"with_mirrors","date":"2026-10-01","downloads":2542},{"category":"with_mirrors","date":"2026-10-02","downloads":2603},{"category":"with_mirrors","date":"2026-10-03","downloads":1660},{"category":"with_mirrors","date":"2026-10-04","downloads":1759},{"category":"with_mirrors","date":"2026-10-05","downloads":2286},{"category":"with_mirrors","date":"2026-10-06","downloads":3061},{"category":"with_mirrors","date":"2026-10-07","downloads":3594},{"category":"with_mirrors","date":"2026-10-08","downloads":3894},{"category":"with_mirrors","date":"2026-10-09","downloads":3225},{"category":"with_mirrors","date":"2026-10-10","downloads":2129},{"category":"with_mirrors","date":"2026-10-11","downloads":1731},{"category":"with_mirrors","date":"2026-10-12","downloads":2735},{"category":"with_mirrors","date":"2026-10-13","downloads":3646},{"category":"with_mirrors","date":"2026-10-14","downloads":4014},{"category":"with_mirrors","date":"2026-10-15","downloads":4311},{"category":"with_mirrors","date":"2026-10-16","downloads":2842},{"category":"with_mirrors","date":"2026-10-17","downloads":2572},{"category":"with_mirrors","date":"2026-10-18","downloads":2187},{"category":"with_mirrors","date":"2026-10-19","downloads":3585},{"category":"with_mirrors","date":"2026-10-20","downloads":3016},{"category":"with_mirrors","date":"2026-10-21","downloads":2932},{"category":"with_mirrors","date":"2026-10-22","downloads":2782},{"category":"with_mirrors","date":"2026-10-23","downloads":3446},{"category":"with_mirrors","date":"2026-10-24","downloads":1989},{"category":"with_mirrors","date":"2026-10-25","downloads":2462},{"category":"with_mirrors","date":"2026-10-26","downloads":1955},{"category":"with_mirrors","date":"2026-10-27","downloads":1493},{"category":"with_mirrors","date":"2026-10-28","downloads":1418},{"category":"without_mirrors","date":"2026-04-08","downloads":12353},{"category":"without_mirrors","date":"2026-04-09","downloads":13263},{"category":"without_mirrors","date":"2026-04-10","downloads":12327},{"category":"without_mirrors","date":"2026-04-11","downloads":6248},{"category":"without_mirrors","date":"2026-04-12","downloads":4954},{"category":"without_mirrors","date":"2026-04-13","downloads":10443},{"category":"without_mirrors","date":"2026-04-14","downloads":12780},{"category":"without_mirrors","date":"2026-04-15","downloads":15781},{"category":"without_mirrors","date":"2026-04-16","downloads":12839},{"category":"without_mirrors","date":"2026-04-17","downloads":11937},{"category":"without_mirrors","date":"2026-04-18","downloads":6683},{"category":"without_mirrors","date":"2026-04-19","downloads":4207},{"category":"without_mirrors","date":"2026-04-20","downloads":10450},{"category":"without_mirrors","date":"2026-04-21","downloads":14562},{"category":"without_mirrors","date":"2026-04-22","downloads":14123},{"category":"without_mirrors","date":"2026-04-23","downloads":13268},{"category":"without_mirrors","date":"2026-04-24","downloads":16784},{"category":"without_mirrors","date":"2026-04-25","downloads":7336},{"category":"without_mirrors","date":"2026-04-26","downloads":6904},{"category":"without_mirrors","date":"2026-04-27","downloads":11394},{"category":"without_mirrors","date":"2026-04-28","downloads":14371},{"category":"without_mirrors","date":"2026-04-29","downloads":15060},{"category":"without_mirrors","date":"2026-04-30","downloads":13289},{"category":"without_mirrors","date":"2026-05-01","downloads":14429},{"category":"without_mirrors","date":"2026-05-02","downloads":5095},{"category":"without_mirrors","date":"2026-05-03","downloads":6277},{"category":"without_mirrors","date":"2026-05-04","downloads":9967},{"category":"without_mirrors","date":"2026-05-05","downloads":14877},{"category":"without_mirrors","date":"2026-05-06","downloads":15668},{"category":"without_mirrors","date":"2026-05-07","downloads":15723},{"category":"without_mirrors","date":"2026-05-08","downloads":12635},{"category":"without_mirrors","date":"2026-05-09","downloads":5935},{"category":"without_mirrors","date":"2026-05-10","downloads":3259},{"category":"without_mirrors","date":"2026-05-11","downloads":10494},{"category":"without_mirrors","date":"2026-05-12","downloads":14094},{"category":"without_mirrors","date":"2026-05-13","downloads":13826},{"category":"without_mirrors","date":"2026-05-14","downloads":11874},{"category":"without_mirrors","date":"2026-05-15","downloads":12278},{"category":"without_mirrors","date":"2026-05-16","downloads":5144},{"category":"without_mirrors","date":"2026-05-17","downloads":5962},{"category":"without_mirrors","date":"2026-05-18","downloads":11175},{"category":"without_mirrors","date":"2026-05-19","downloads":10225},{"category":"without_mirrors","date":"2026-05-20","downloads":12528},{"category":"without_mirrors","date":"2026-05-21","downloads":15688},{"category":"without_mirrors","date":"2026-05-22","downloads":12685},{"category":"without_mirrors","date":"2026-05-23","downloads":6126},{"category":"without_mirrors","date":"2026-05-24","downloads":5137},{"category":"without_mirrors","date":"2026-05-25","downloads":5478},{"category":"without_mirrors","date":"2026-05-26","downloads":11220},{"category":"without_mirrors","date":"2026-05-27","downloads":14011},{"category":"without_mirrors","date":"2026-05-28","downloads":13315},{"category":"without_mirrors","date":"2026-05-29","downloads":15945},{"category":"without_mirrors","date":"2026-05-30","downloads":5935},{"category":"without_mirrors","date":"2026-05-31","downloads":6304},{"category":"without_mirrors","date":"2026-06-01","downloads":9663},{"category":"without_mirrors","date":"2026-06-02","downloads":9005},{"category":"without_mirrors","date":"2026-06-03","downloads":8049},{"category":"without_mirrors","date":"2026-06-04","downloads":7517},{"category":"without_mirrors","date":"2026-06-05","downloads":6870},{"category":"without_mirrors","date":"2026-06-06","downloads":3901},{"category":"without_mirrors","date":"2026-06-07","downloads":2626},{"category":"without_mirrors","date":"2026-06-08","downloads":6608},{"category":"without_mirrors","date":"2026-06-09","downloads":8956},{"category":"without_mirrors","date":"2026-06-10","downloads":9859},{"category":"without_mirrors","date":"2026-06-11","downloads":9610},{"category":"without_mirrors","date":"2026-06-12","downloads":7810},{"category":"without_mirrors","date":"2026-06-13","downloads":3665},{"category":"without_mirrors","date":"2026-06-14","downloads":2904},{"category":"without_mirrors","date":"2026-06-15","downloads":5021},{"category":"without_mirrors","date":"2026-06-16","downloads":6135},{"category":"without_mirrors","date":"2026-06-17","downloads":5415},{"category":"without_mirrors","date":"2026-06-18","downloads":5155},{"category":"without_mirrors","date":"2026-06-19","downloads":4461},{"category":"without_mirrors","date":"2026-06-20","downloads":2702},{"category":"without_mirrors","date":"2026-06-21","downloads":2142},{"category":"without_mirrors","date":"2026-06-22","downloads":3953},{"category":"without_mirrors","date":"2026-06-23","downloads":4950},{"category":"without_mirrors","date":"2026-06-24","downloads":5777},{"category":"without_mirrors","date":"2026-06-25","downloads":6294},{"category":"without_mirrors","date":"2026-06-26","downloads":6949},{"category":"without_mirrors","date":"2026-06-27","downloads":3457},{"category":"without_mirrors","date":"2026-06-28","downloads":2769},{"category":"without_mirrors","date":"2026-06-29","downloads":6902},{"category":"without_mirrors","date":"2026-06-30","downloads":7833},{"category":"without_mirrors","date":"2026-07-01","downloads":7739},{"category":"without_mirrors","date":"2026-07-02","downloads":7477},{"category":"without_mirrors","date":"2026-07-03","downloads":5668},{"category":"without_mirrors","date":"2026-07-04","downloads":2742},{"category":"without_mirrors","date":"2026-07-05","downloads":1990},{"category":"without_mirrors","date":"2026-07-06","downloads":7142},{"category":"without_mirrors","date":"2026-07-07","downloads":7997},{"category":"without_mirrors","date":"2026-07-08","downloads":8566},{"category":"without_mirrors","date":"2026-07-09","downloads":8158},{"category":"without_mirrors","date":"2026-07-10","downloads":8301},{"category":"without_mirrors","date":"2026-07-11","downloads":3513},{"category":"without_mirrors","date":"2026-07-12","downloads":2252},{"category":"without_mirrors","date":"2026-07-13","downloads":8274},{"category":"without_mirrors","date":"2026-07-14","downloads":10569},{"category":"without_mirrors","date":"2026-07-15","downloads":11587},{"category":"without_mirrors","date":"2026-07-16","downloads":9947},{"category":"without_mirrors","date":"2026-07-17","downloads":9918},{"category":"without_mirrors","date":"2026-07-18","downloads":5316},{"category":"without_mirrors","date":"2026-07-19","downloads":2457},{"category":"without_mirrors","date":"2026-07-20","downloads":9663},{"category":"without_mirrors","date":"2026-07-21","downloads":12853},{"category":"without_mirrors","date":"2026-07-22","downloads":12336},{"category":"without_mirrors","date":"2026-07-23","downloads":11661},{"category":"without_mirrors","date":"2026-07-24","downloads":12655},{"category":"without_mirrors","date":"2026-07-25","downloads":5386},{"category":"without_mirrors","date":"2026-07-26","downloads":4212},{"category":"without_mirrors","date":"2026-07-27","downloads":11720},{"category":"without_mirrors","date":"2026-07-28","downloads":15020},{"category":"without_mirrors","date":"2026-07-29","downloads":14270},{"category":"without_mirrors","date":"2026-07-30","downloads":15511},{"category":"without_mirrors","date":"2026-07-31","downloads":14365},{"category":"without_mirrors","date":"2026-08-01","downloads":10039},{"category":"without_mirrors","date":"2026-08-02","downloads":6189},{"category":"without_mirrors","date":"2026-08-03","downloads":12884},{"category":"without_mirrors","date":"2026-08-04","downloads":15641},{"category":"without_mirrors","date":"2026-08-05","downloads":15948},{"category":"without_mirrors","date":"2026-08-06","downloads":10042},{"category":"without_mirrors","date":"2026-08-07","downloads":17274},{"category":"without_mirrors","date":"2026-08-08","downloads":7301},{"category":"without_mirrors","date":"2026-08-09","downloads":2947},{"category":"without_mirrors","date":"2026-08-10","downloads":4125},{"category":"without_mirrors","date":"2026-08-11","downloads":4327},{"category":"without_mirrors","date":"2026-08-12","downloads":5422},{"category":"without_mirrors","date":"2026-08-13","downloads":3631},{"category":"without_mirrors","date":"2026-08-14","downloads":4200},{"category":"without_mirrors","date":"2026-08-15","downloads":1715},{"category":"without_mirrors","date":"2026-08-16","downloads":1873},{"category":"without_mirrors","date":"2026-08-17","downloads":2741},{"category":"without_mirrors","date":"2026-08-18","downloads":3675},{"category":"without_mirrors","date":"2026-08-19","downloads":4022},{"category":"without_mirrors","date":"2026-08-20","downloads":3728},{"category":"without_mirrors","date":"2026-08-21","downloads":3801},{"category":"without_mirrors","date":"2026-08-22","downloads":2899},{"category":"without_mirrors","date":"2026-08-23","downloads":1976},{"category":"without_mirrors","date":"2026-08-24","downloads":2331},{"category":"without_mirrors","date":"2026-08-25","downloads":2336},{"category":"without_mirrors","date":"2026-08-26","downloads":2219},{"category":"without_mirrors","date":"2026-08-27","downloads":2278},{"category":"without_mirrors","date":"2026-08-28","downloads":2085},{"category":"without_mirrors","date":"2026-08-29","downloads":1123},{"category":"without_mirrors","date":"2026-08-30","downloads":1150},{"category":"without_mirrors","date":"2026-08-31","downloads":2066},{"category":"without_mirrors","date":"2026-09-01","downloads":2105},{"category":"without_mirrors","date":"2026-09-02","downloads":2642},{"category":"without_mirrors","date":"2026-09-03","downloads":2347},{"category":"without_mirrors","date":"2026-09-04","downloads":2224},{"category":"without_mirrors","date":"2026-09-05","downloads":1411},{"category":"without_mirrors","date":"2026-09-06","downloads":1326},{"category":"without_mirrors","date":"2026-09-07","downloads":1784},{"category":"without_mirrors","date":"2026-09-08","downloads":2190},{"category":"without_mirrors","date":"2026-09-09","downloads":2348},{"category":"without_mirrors","date":"2026-09-10","downloads":2550},{"category":"without_mirrors","date":"2026-09-11","downloads":2561},{"category":"without_mirrors","date":"2026-09-12","downloads":1990},{"category":"without_mirrors","date":"2026-09-13","downloads":1752},{"category":"without_mirrors","date":"2026-09-14","downloads":2075},{"category":"without_mirrors","date":"2026-09-15","downloads":2883},{"category":"without_mirrors","date":"2026-09-16","downloads":2957},{"category":"without_mirrors","date":"2026-09-17","downloads":3061},{"category":"without_mirrors","date":"2026-09-18","downloads":2932},{"category":"without_mirrors","date":"2026-09-19","downloads":1937},{"category":"without_mirrors","date":"2026-09-20","downloads":1819},{"category":"without_mirrors","date":"2026-09-21","downloads":2587},{"category":"without_mirrors","date":"2026-09-22","downloads":3174},{"category":"without_mirrors","date":"2026-09-23","downloads":2710},{"category":"without_mirrors","date":"2026-09-24","downloads":2935},{"category":"without_mirrors","date":"2026-09-25","downloads":2354},{"category":"without_mirrors","date":"2026-09-26","downloads":1477},{"category":"without_mirrors","date":"2026-09-27","downloads":1352},{"category":"without_mirrors","date":"2026-09-28","downloads":1848},{"category":"without_mirrors","date":"2026-09-29","downloads":2142},{"category":"without_mirrors","date":"2026-09-30","downloads":2240},{"category":"without_mirrors","date":"2026-10-01","downloads":2492},{"category":"without_mirrors","date":"2026-10-02","downloads":2359},{"category":"without_mirrors","date":"2026-10-03","downloads":1648},{"category":"without_mirrors","date":"2026-10-04","downloads":1455},{"category":"without_mirrors","date":"2026-10-05","downloads":2210},{"category":"without_mirrors","date":"2026-10-06","downloads":3030},{"category":"without_mirrors","date":"2026-10-07","downloads":3319},{"category":"without_mirrors","date":"2026-10-08","downloads":3833},{"category":"without_mirrors","date":"2026-10-09","downloads":3171},{"category":"without_mirrors","date":"2026-10-10","downloads":2118},{"category":"without_mirrors","date":"2026-10-11","downloads":1701},{"category":"without_mirrors","date":"2026-10-12","downloads":2721},{"category":"without_mirrors","date":"2026-10-13","downloads":3385},{"category":"without_mirrors","date":"2026-10-14","downloads":3746},{"category":"without_mirrors","date":"2026-10-15","downloads":3488},{"category":"without_mirrors","date":"2026-10-16","downloads":2743},{"category":"without_mirrors","date":"2026-10-17","downloads":2407},{"category":"without_mirrors","date":"2026-10-18","downloads":1926},{"category":"without_mirrors","date":"2026-10-19","downloads":3310},{"category":"without_mirrors","date":"2026-10-20","downloads":3003},{"category":"without_mirrors","date":"2026-10-21","downloads":2637},{"category":"without_mirrors","date":"2026-10-22","downloads":2539},{"category":"without_mirrors","date":"2026-10-23","downloads":3161},{"category":"without_mirrors","date":"2026-10-24","downloads":1952},{"category":"without_mirrors","date":"2026-10-25","downloads":2449},{"category":"without_mirrors","date":"2026-10-26","downloads":1936},{"category":"without_mirrors","date":"2026-10-27","downloads":1467},{"category":"without_mirrors","date":"2026-10-28","downloads":1388}],"package":"mooncake-transfer-engine","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:11:25.626931+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine/python_major","payload":{"data":[{"category":"3","date":"2026-04-08","downloads":12292},{"category":"3","date":"2026-04-09","downloads":13187},{"category":"3","date":"2026-04-10","downloads":12316},{"category":"3","date":"2026-04-11","downloads":6234},{"category":"3","date":"2026-04-12","downloads":4931},{"category":"3","date":"2026-04-13","downloads":10434},{"category":"3","date":"2026-04-14","downloads":12715},{"category":"3","date":"2026-04-15","downloads":15538},{"category":"3","date":"2026-04-16","downloads":12831},{"category":"3","date":"2026-04-17","downloads":11934},{"category":"3","date":"2026-04-18","downloads":6677},{"category":"3","date":"2026-04-19","downloads":4192},{"category":"3","date":"2026-04-20","downloads":10433},{"category":"3","date":"2026-04-21","downloads":14559},{"category":"3","date":"2026-04-22","downloads":13824},{"category":"3","date":"2026-04-23","downloads":13238},{"category":"3","date":"2026-04-24","downloads":16762},{"category":"3","date":"2026-04-25","downloads":7290},{"category":"3","date":"2026-04-26","downloads":6887},{"category":"3","date":"2026-04-27","downloads":11376},{"category":"3","date":"2026-04-28","downloads":14222},{"category":"3","date":"2026-04-29","downloads":15045},{"category":"3","date":"2026-04-30","downloads":13285},{"category":"3","date":"2026-05-01","downloads":14418},{"category":"3","date":"2026-05-02","downloads":5095},{"category":"3","date":"2026-05-03","downloads":6276},{"category":"3","date":"2026-05-04","downloads":9960},{"category":"3","date":"2026-05-05","downloads":14780},{"category":"3","date":"2026-05-06","downloads":15661},{"category":"3","date":"2026-05-07","downloads":15713},{"category":"3","date":"2026-05-08","downloads":12479},{"category":"3","date":"2026-05-09","downloads":5756},{"category":"3","date":"2026-05-10","downloads":3254},{"category":"3","date":"2026-05-11","downloads":10486},{"category":"3","date":"2026-05-12","downloads":14088},{"category":"3","date":"2026-05-13","downloads":13822},{"category":"3","date":"2026-05-14","downloads":11871},{"category":"3","date":"2026-05-15","downloads":12157},{"category":"3","date":"2026-05-16","downloads":5077},{"category":"3","date":"2026-05-17","downloads":5956},{"category":"3","date":"2026-05-18","downloads":11167},{"category":"3","date":"2026-05-19","downloads":10222},{"category":"3","date":"2026-05-20","downloads":12510},{"category":"3","date":"2026-05-21","downloads":15497},{"category":"3","date":"2026-05-22","downloads":12651},{"category":"3","date":"2026-05-23","downloads":6084},{"category":"3","date":"2026-05-24","downloads":4925},{"category":"3","date":"2026-05-25","downloads":5440},{"category":"3","date":"2026-05-26","downloads":11203},{"category":"3","date":"2026-05-27","downloads":13975},{"category":"3","date":"2026-05-28","downloads":13267},{"category":"3","date":"2026-05-29","downloads":15925},{"category":"3","date":"2026-05-30","downloads":5923},{"category":"3","date":"2026-05-31","downloads":6281},{"category":"3","date":"2026-06-01","downloads":9642},{"category":"3","date":"2026-06-02","downloads":8978},{"category":"3","date":"2026-06-03","downloads":8016},{"category":"3","date":"2026-06-04","downloads":7501},{"category":"3","date":"2026-06-05","downloads":6837},{"category":"3","date":"2026-06-06","downloads":3739},{"category":"3","date":"2026-06-07","downloads":2622},{"category":"3","date":"2026-06-08","downloads":6594},{"category":"3","date":"2026-06-09","downloads":8892},{"category":"3","date":"2026-06-10","downloads":9840},{"category":"3","date":"2026-06-11","downloads":9598},{"category":"3","date":"2026-06-12","downloads":7798},{"category":"3","date":"2026-06-13","downloads":3656},{"category":"3","date":"2026-06-14","downloads":2888},{"category":"3","date":"2026-06-15","downloads":4969},{"category":"3","date":"2026-06-16","downloads":6116},{"category":"3","date":"2026-06-17","downloads":5400},{"category":"3","date":"2026-06-18","downloads":5126},{"category":"3","date":"2026-06-19","downloads":4407},{"category":"3","date":"2026-06-20","downloads":2646},{"category":"3","date":"2026-06-21","downloads":2137},{"category":"3","date":"2026-06-22","downloads":3924},{"category":"3","date":"2026-06-23","downloads":4763},{"category":"3","date":"2026-06-24","downloads":5766},{"category":"3","date":"2026-06-25","downloads":6287},{"category":"3","date":"2026-06-26","downloads":6944},{"category":"3","date":"2026-06-27","downloads":3315},{"category":"3","date":"2026-06-28","downloads":2765},{"category":"3","date":"2026-06-29","downloads":6893},{"category":"3","date":"2026-06-30","downloads":7814},{"category":"3","date":"2026-07-01","downloads":7720},{"category":"3","date":"2026-07-02","downloads":7458},{"category":"3","date":"2026-07-03","downloads":5659},{"category":"3","date":"2026-07-04","downloads":2729},{"category":"3","date":"2026-07-05","downloads":1983},{"category":"3","date":"2026-07-06","downloads":7134},{"category":"3","date":"2026-07-07","downloads":7990},{"category":"3","date":"2026-07-08","downloads":8538},{"category":"3","date":"2026-07-09","downloads":8138},{"category":"3","date":"2026-07-10","downloads":8295},{"category":"3","date":"2026-07-11","downloads":3492},{"category":"3","date":"2026-07-12","downloads":2251},{"category":"3","date":"2026-07-13","downloads":8263},{"category":"3","date":"2026-07-14","downloads":10557},{"category":"3","date":"2026-07-15","downloads":11577},{"category":"3","date":"2026-07-16","downloads":9936},{"category":"3","date":"2026-07-17","downloads":9912},{"category":"3","date":"2026-07-18","downloads":5307},{"category":"3","date":"2026-07-19","downloads":2448},{"category":"3","date":"2026-07-20","downloads":9648},{"category":"3","date":"2026-07-21","downloads":12843},{"category":"3","date":"2026-07-22","downloads":12232},{"category":"3","date":"2026-07-23","downloads":11497},{"category":"3","date":"2026-07-24","downloads":12611},{"category":"3","date":"2026-07-25","downloads":5165},{"category":"3","date":"2026-07-26","downloads":4192},{"category":"3","date":"2026-07-27","downloads":11679},{"category":"3","date":"2026-07-28","downloads":14918},{"category":"3","date":"2026-07-29","downloads":14231},{"category":"3","date":"2026-07-30","downloads":15438},{"category":"3","date":"2026-07-31","downloads":14302},{"category":"3","date":"2026-08-01","downloads":10030},{"category":"3","date":"2026-08-02","downloads":6177},{"category":"3","date":"2026-08-03","downloads":12860},{"category":"3","date":"2026-08-04","downloads":15613},{"category":"3","date":"2026-08-05","downloads":15918},{"category":"3","date":"2026-08-06","downloads":10027},{"category":"3","date":"2026-08-07","downloads":17257},{"category":"3","date":"2026-08-08","downloads":7296},{"category":"3","date":"2026-08-09","downloads":2945},{"category":"3","date":"2026-08-10","downloads":4021},{"category":"3","date":"2026-08-11","downloads":4305},{"category":"3","date":"2026-08-12","downloads":5412},{"category":"3","date":"2026-08-13","downloads":3616},{"category":"3","date":"2026-08-14","downloads":4173},{"category":"3","date":"2026-08-15","downloads":1704},{"category":"3","date":"2026-08-16","downloads":1865},{"category":"3","date":"2026-08-17","downloads":2719},{"category":"3","date":"2026-08-18","downloads":3577},{"category":"3","date":"2026-08-19","downloads":4007},{"category":"3","date":"2026-08-20","downloads":3713},{"category":"3","date":"2026-08-21","downloads":3695},{"category":"3","date":"2026-08-22","downloads":2877},{"category":"3","date":"2026-08-23","downloads":1964},{"category":"3","date":"2026-08-24","downloads":2294},{"category":"3","date":"2026-08-25","downloads":2307},{"category":"3","date":"2026-08-26","downloads":1924},{"category":"3","date":"2026-08-27","downloads":2024},{"category":"3","date":"2026-08-28","downloads":1817},{"category":"3","date":"2026-08-29","downloads":1080},{"category":"3","date":"2026-08-30","downloads":1146},{"category":"3","date":"2026-08-31","downloads":1758},{"category":"3","date":"2026-09-01","downloads":1984},{"category":"3","date":"2026-09-02","downloads":2577},{"category":"3","date":"2026-09-03","downloads":2319},{"category":"3","date":"2026-09-04","downloads":2194},{"category":"3","date":"2026-09-05","downloads":1384},{"category":"3","date":"2026-09-06","downloads":1304},{"category":"3","date":"2026-09-07","downloads":1733},{"category":"3","date":"2026-09-08","downloads":2162},{"category":"3","date":"2026-09-09","downloads":2330},{"category":"3","date":"2026-09-10","downloads":2521},{"category":"3","date":"2026-09-11","downloads":2543},{"category":"3","date":"2026-09-12","downloads":1961},{"category":"3","date":"2026-09-13","downloads":1681},{"category":"3","date":"2026-09-14","downloads":2048},{"category":"3","date":"2026-09-15","downloads":2840},{"category":"3","date":"2026-09-16","downloads":2935},{"category":"3","date":"2026-09-17","downloads":3027},{"category":"3","date":"2026-09-18","downloads":2915},{"category":"3","date":"2026-09-19","downloads":1676},{"category":"3","date":"2026-09-20","downloads":1799},{"category":"3","date":"2026-09-21","downloads":2552},{"category":"3","date":"2026-09-22","downloads":3136},{"category":"3","date":"2026-09-23","downloads":2685},{"category":"3","date":"2026-09-24","downloads":2923},{"category":"3","date":"2026-09-25","downloads":2326},{"category":"3","date":"2026-09-26","downloads":1470},{"category":"3","date":"2026-09-27","downloads":1316},{"category":"3","date":"2026-09-28","downloads":1821},{"category":"3","date":"2026-09-29","downloads":2126},{"category":"3","date":"2026-09-30","downloads":2219},{"category":"3","date":"2026-10-01","downloads":2484},{"category":"3","date":"2026-10-02","downloads":2331},{"category":"3","date":"2026-10-03","downloads":1583},{"category":"3","date":"2026-10-04","downloads":1402},{"category":"3","date":"2026-10-05","downloads":2177},{"category":"3","date":"2026-10-06","downloads":3020},{"category":"3","date":"2026-10-07","downloads":3303},{"category":"3","date":"2026-10-08","downloads":3817},{"category":"3","date":"2026-10-09","downloads":2960},{"category":"3","date":"2026-10-10","downloads":2086},{"category":"3","date":"2026-10-11","downloads":1677},{"category":"3","date":"2026-10-12","downloads":2701},{"category":"3","date":"2026-10-13","downloads":3359},{"category":"3","date":"2026-10-14","downloads":3713},{"category":"3","date":"2026-10-15","downloads":3441},{"category":"3","date":"2026-10-16","downloads":2709},{"category":"3","date":"2026-10-17","downloads":2398},{"category":"3","date":"2026-10-18","downloads":1916},{"category":"3","date":"2026-10-19","downloads":3287},{"category":"3","date":"2026-10-20","downloads":2955},{"category":"3","date":"2026-10-21","downloads":2594},{"category":"3","date":"2026-10-22","downloads":2522},{"category":"3","date":"2026-10-23","downloads":3075},{"category":"3","date":"2026-10-24","downloads":1836},{"category":"3","date":"2026-10-25","downloads":2434},{"category":"3","date":"2026-10-26","downloads":1910},{"category":"3","date":"2026-10-27","downloads":1443},{"category":"3","date":"2026-10-28","downloads":1367},{"category":"null","date":"2026-04-08","downloads":61},{"category":"null","date":"2026-04-09","downloads":76},{"category":"null","date":"2026-04-10","downloads":11},{"category":"null","date":"2026-04-11","downloads":14},{"category":"null","date":"2026-04-12","downloads":23},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":65},{"category":"null","date":"2026-04-15","downloads":243},{"category":"null","date":"2026-04-16","downloads":8},{"category":"null","date":"2026-04-17","downloads":3},{"category":"null","date":"2026-04-18","downloads":6},{"category":"null","date":"2026-04-19","downloads":15},{"category":"null","date":"2026-04-20","downloads":17},{"category":"null","date":"2026-04-21","downloads":3},{"category":"null","date":"2026-04-22","downloads":299},{"category":"null","date":"2026-04-23","downloads":30},{"category":"null","date":"2026-04-24","downloads":22},{"category":"null","date":"2026-04-25","downloads":46},{"category":"null","date":"2026-04-26","downloads":17},{"category":"null","date":"2026-04-27","downloads":18},{"category":"null","date":"2026-04-28","downloads":149},{"category":"null","date":"2026-04-29","downloads":15},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-01","downloads":11},{"category":"null","date":"2026-05-03","downloads":1},{"category":"null","date":"2026-05-04","downloads":7},{"category":"null","date":"2026-05-05","downloads":97},{"category":"null","date":"2026-05-06","downloads":7},{"category":"null","date":"2026-05-07","downloads":10},{"category":"null","date":"2026-05-08","downloads":156},{"category":"null","date":"2026-05-09","downloads":179},{"category":"null","date":"2026-05-10","downloads":5},{"category":"null","date":"2026-05-11","downloads":8},{"category":"null","date":"2026-05-12","downloads":6},{"category":"null","date":"2026-05-13","downloads":4},{"category":"null","date":"2026-05-14","downloads":3},{"category":"null","date":"2026-05-15","downloads":121},{"category":"null","date":"2026-05-16","downloads":67},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":8},{"category":"null","date":"2026-05-19","downloads":3},{"category":"null","date":"2026-05-20","downloads":18},{"category":"null","date":"2026-05-21","downloads":191},{"category":"null","date":"2026-05-22","downloads":34},{"category":"null","date":"2026-05-23","downloads":42},{"category":"null","date":"2026-05-24","downloads":212},{"category":"null","date":"2026-05-25","downloads":38},{"category":"null","date":"2026-05-26","downloads":17},{"category":"null","date":"2026-05-27","downloads":36},{"category":"null","date":"2026-05-28","downloads":48},{"category":"null","date":"2026-05-29","downloads":20},{"category":"null","date":"2026-05-30","downloads":12},{"category":"null","date":"2026-05-31","downloads":23},{"category":"null","date":"2026-06-01","downloads":21},{"category":"null","date":"2026-06-02","downloads":27},{"category":"null","date":"2026-06-03","downloads":33},{"category":"null","date":"2026-06-04","downloads":16},{"category":"null","date":"2026-06-05","downloads":33},{"category":"null","date":"2026-06-06","downloads":162},{"category":"null","date":"2026-06-07","downloads":4},{"category":"null","date":"2026-06-08","downloads":14},{"category":"null","date":"2026-06-09","downloads":64},{"category":"null","date":"2026-06-10","downloads":19},{"category":"null","date":"2026-06-11","downloads":12},{"category":"null","date":"2026-06-12","downloads":12},{"category":"null","date":"2026-06-13","downloads":9},{"category":"null","date":"2026-06-14","downloads":16},{"category":"null","date":"2026-06-15","downloads":52},{"category":"null","date":"2026-06-16","downloads":19},{"category":"null","date":"2026-06-17","downloads":15},{"category":"null","date":"2026-06-18","downloads":29},{"category":"null","date":"2026-06-19","downloads":54},{"category":"null","date":"2026-06-20","downloads":56},{"category":"null","date":"2026-06-21","downloads":5},{"category":"null","date":"2026-06-22","downloads":29},{"category":"null","date":"2026-06-23","downloads":187},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":7},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":142},{"category":"null","date":"2026-06-28","downloads":4},{"category":"null","date":"2026-06-29","downloads":9},{"category":"null","date":"2026-06-30","downloads":19},{"category":"null","date":"2026-07-01","downloads":19},{"category":"null","date":"2026-07-02","downloads":19},{"category":"null","date":"2026-07-03","downloads":9},{"category":"null","date":"2026-07-04","downloads":13},{"category":"null","date":"2026-07-05","downloads":7},{"category":"null","date":"2026-07-06","downloads":8},{"category":"null","date":"2026-07-07","downloads":7},{"category":"null","date":"2026-07-08","downloads":28},{"category":"null","date":"2026-07-09","downloads":20},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":21},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":11},{"category":"null","date":"2026-07-14","downloads":12},{"category":"null","date":"2026-07-15","downloads":10},{"category":"null","date":"2026-07-16","downloads":11},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":9},{"category":"null","date":"2026-07-19","downloads":9},{"category":"null","date":"2026-07-20","downloads":15},{"category":"null","date":"2026-07-21","downloads":10},{"category":"null","date":"2026-07-22","downloads":104},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":44},{"category":"null","date":"2026-07-25","downloads":221},{"category":"null","date":"2026-07-26","downloads":20},{"category":"null","date":"2026-07-27","downloads":41},{"category":"null","date":"2026-07-28","downloads":102},{"category":"null","date":"2026-07-29","downloads":39},{"category":"null","date":"2026-07-30","downloads":73},{"category":"null","date":"2026-07-31","downloads":63},{"category":"null","date":"2026-08-01","downloads":9},{"category":"null","date":"2026-08-02","downloads":12},{"category":"null","date":"2026-08-03","downloads":24},{"category":"null","date":"2026-08-04","downloads":28},{"category":"null","date":"2026-08-05","downloads":30},{"category":"null","date":"2026-08-06","downloads":15},{"category":"null","date":"2026-08-07","downloads":17},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":104},{"category":"null","date":"2026-08-11","downloads":22},{"category":"null","date":"2026-08-12","downloads":10},{"category":"null","date":"2026-08-13","downloads":15},{"category":"null","date":"2026-08-14","downloads":27},{"category":"null","date":"2026-08-15","downloads":11},{"category":"null","date":"2026-08-16","downloads":8},{"category":"null","date":"2026-08-17","downloads":22},{"category":"null","date":"2026-08-18","downloads":98},{"category":"null","date":"2026-08-19","downloads":15},{"category":"null","date":"2026-08-20","downloads":15},{"category":"null","date":"2026-08-21","downloads":106},{"category":"null","date":"2026-08-22","downloads":22},{"category":"null","date":"2026-08-23","downloads":12},{"category":"null","date":"2026-08-24","downloads":37},{"category":"null","date":"2026-08-25","downloads":29},{"category":"null","date":"2026-08-26","downloads":295},{"category":"null","date":"2026-08-27","downloads":254},{"category":"null","date":"2026-08-28","downloads":268},{"category":"null","date":"2026-08-29","downloads":43},{"category":"null","date":"2026-08-30","downloads":4},{"category":"null","date":"2026-08-31","downloads":308},{"category":"null","date":"2026-09-01","downloads":121},{"category":"null","date":"2026-09-02","downloads":65},{"category":"null","date":"2026-09-03","downloads":28},{"category":"null","date":"2026-09-04","downloads":30},{"category":"null","date":"2026-09-05","downloads":27},{"category":"null","date":"2026-09-06","downloads":22},{"category":"null","date":"2026-09-07","downloads":51},{"category":"null","date":"2026-09-08","downloads":28},{"category":"null","date":"2026-09-09","downloads":18},{"category":"null","date":"2026-09-10","downloads":29},{"category":"null","date":"2026-09-11","downloads":18},{"category":"null","date":"2026-09-12","downloads":29},{"category":"null","date":"2026-09-13","downloads":71},{"category":"null","date":"2026-09-14","downloads":27},{"category":"null","date":"2026-09-15","downloads":43},{"category":"null","date":"2026-09-16","downloads":22},{"category":"null","date":"2026-09-17","downloads":34},{"category":"null","date":"2026-09-18","downloads":17},{"category":"null","date":"2026-09-19","downloads":261},{"category":"null","date":"2026-09-20","downloads":20},{"category":"null","date":"2026-09-21","downloads":35},{"category":"null","date":"2026-09-22","downloads":38},{"category":"null","date":"2026-09-23","downloads":25},{"category":"null","date":"2026-09-24","downloads":12},{"category":"null","date":"2026-09-25","downloads":28},{"category":"null","date":"2026-09-26","downloads":7},{"category":"null","date":"2026-09-27","downloads":36},{"category":"null","date":"2026-09-28","downloads":27},{"category":"null","date":"2026-09-29","downloads":16},{"category":"null","date":"2026-09-30","downloads":21},{"category":"null","date":"2026-10-01","downloads":8},{"category":"null","date":"2026-10-02","downloads":28},{"category":"null","date":"2026-10-03","downloads":65},{"category":"null","date":"2026-10-04","downloads":53},{"category":"null","date":"2026-10-05","downloads":33},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-07","downloads":16},{"category":"null","date":"2026-10-08","downloads":16},{"category":"null","date":"2026-10-09","downloads":211},{"category":"null","date":"2026-10-10","downloads":32},{"category":"null","date":"2026-10-11","downloads":24},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-13","downloads":26},{"category":"null","date":"2026-10-14","downloads":33},{"category":"null","date":"2026-10-15","downloads":47},{"category":"null","date":"2026-10-16","downloads":34},{"category":"null","date":"2026-10-17","downloads":9},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":23},{"category":"null","date":"2026-10-20","downloads":48},{"category":"null","date":"2026-10-21","downloads":43},{"category":"null","date":"2026-10-22","downloads":17},{"category":"null","date":"2026-10-23","downloads":86},{"category":"null","date":"2026-10-24","downloads":116},{"category":"null","date":"2026-10-25","downloads":15},{"category":"null","date":"2026-10-26","downloads":26},{"category":"null","date":"2026-10-27","downloads":24},{"category":"null","date":"2026-10-28","downloads":21}],"package":"mooncake-transfer-engine","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:11:28.649550+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine/python_minor","payload":{"data":[{"category":"3.10","date":"2026-04-08","downloads":21},{"category":"3.10","date":"2026-04-09","downloads":110},{"category":"3.10","date":"2026-04-10","downloads":97},{"category":"3.10","date":"2026-04-11","downloads":44},{"category":"3.10","date":"2026-04-12","downloads":152},{"category":"3.10","date":"2026-04-13","downloads":109},{"category":"3.10","date":"2026-04-14","downloads":117},{"category":"3.10","date":"2026-04-15","downloads":131},{"category":"3.10","date":"2026-04-16","downloads":56},{"category":"3.10","date":"2026-04-17","downloads":45},{"category":"3.10","date":"2026-04-18","downloads":201},{"category":"3.10","date":"2026-04-19","downloads":67},{"category":"3.10","date":"2026-04-20","downloads":52},{"category":"3.10","date":"2026-04-21","downloads":38},{"category":"3.10","date":"2026-04-22","downloads":215},{"category":"3.10","date":"2026-04-23","downloads":31},{"category":"3.10","date":"2026-04-24","downloads":18},{"category":"3.10","date":"2026-04-25","downloads":9},{"category":"3.10","date":"2026-04-26","downloads":37},{"category":"3.10","date":"2026-04-27","downloads":51},{"category":"3.10","date":"2026-04-28","downloads":93},{"category":"3.10","date":"2026-04-29","downloads":26},{"category":"3.10","date":"2026-04-30","downloads":54},{"category":"3.10","date":"2026-05-01","downloads":47},{"category":"3.10","date":"2026-05-02","downloads":75},{"category":"3.10","date":"2026-05-03","downloads":8},{"category":"3.10","date":"2026-05-04","downloads":14},{"category":"3.10","date":"2026-05-05","downloads":42},{"category":"3.10","date":"2026-05-06","downloads":168},{"category":"3.10","date":"2026-05-07","downloads":112},{"category":"3.10","date":"2026-05-08","downloads":51},{"category":"3.10","date":"2026-05-09","downloads":48},{"category":"3.10","date":"2026-05-10","downloads":34},{"category":"3.10","date":"2026-05-11","downloads":108},{"category":"3.10","date":"2026-05-12","downloads":107},{"category":"3.10","date":"2026-05-13","downloads":66},{"category":"3.10","date":"2026-05-14","downloads":86},{"category":"3.10","date":"2026-05-15","downloads":43},{"category":"3.10","date":"2026-05-16","downloads":72},{"category":"3.10","date":"2026-05-17","downloads":1215},{"category":"3.10","date":"2026-05-18","downloads":91},{"category":"3.10","date":"2026-05-19","downloads":68},{"category":"3.10","date":"2026-05-20","downloads":41},{"category":"3.10","date":"2026-05-21","downloads":50},{"category":"3.10","date":"2026-05-22","downloads":86},{"category":"3.10","date":"2026-05-23","downloads":53},{"category":"3.10","date":"2026-05-24","downloads":56},{"category":"3.10","date":"2026-05-25","downloads":25},{"category":"3.10","date":"2026-05-26","downloads":53},{"category":"3.10","date":"2026-05-27","downloads":52},{"category":"3.10","date":"2026-05-28","downloads":35},{"category":"3.10","date":"2026-05-29","downloads":28},{"category":"3.10","date":"2026-05-30","downloads":29},{"category":"3.10","date":"2026-05-31","downloads":27},{"category":"3.10","date":"2026-06-01","downloads":55},{"category":"3.10","date":"2026-06-02","downloads":107},{"category":"3.10","date":"2026-06-03","downloads":96},{"category":"3.10","date":"2026-06-04","downloads":109},{"category":"3.10","date":"2026-06-05","downloads":107},{"category":"3.10","date":"2026-06-06","downloads":79},{"category":"3.10","date":"2026-06-07","downloads":73},{"category":"3.10","date":"2026-06-08","downloads":97},{"category":"3.10","date":"2026-06-09","downloads":263},{"category":"3.10","date":"2026-06-10","downloads":565},{"category":"3.10","date":"2026-06-11","downloads":604},{"category":"3.10","date":"2026-06-12","downloads":555},{"category":"3.10","date":"2026-06-13","downloads":532},{"category":"3.10","date":"2026-06-14","downloads":530},{"category":"3.10","date":"2026-06-15","downloads":204},{"category":"3.10","date":"2026-06-16","downloads":80},{"category":"3.10","date":"2026-06-17","downloads":17},{"category":"3.10","date":"2026-06-18","downloads":13},{"category":"3.10","date":"2026-06-19","downloads":17},{"category":"3.10","date":"2026-06-20","downloads":11},{"category":"3.10","date":"2026-06-21","downloads":14},{"category":"3.10","date":"2026-06-22","downloads":18},{"category":"3.10","date":"2026-06-23","downloads":35},{"category":"3.10","date":"2026-06-24","downloads":79},{"category":"3.10","date":"2026-06-25","downloads":40},{"category":"3.10","date":"2026-06-26","downloads":22},{"category":"3.10","date":"2026-06-27","downloads":15},{"category":"3.10","date":"2026-06-28","downloads":11},{"category":"3.10","date":"2026-06-29","downloads":42},{"category":"3.10","date":"2026-06-30","downloads":20},{"category":"3.10","date":"2026-07-01","downloads":19},{"category":"3.10","date":"2026-07-02","downloads":15},{"category":"3.10","date":"2026-07-03","downloads":21},{"category":"3.10","date":"2026-07-04","downloads":18},{"category":"3.10","date":"2026-07-05","downloads":11},{"category":"3.10","date":"2026-07-06","downloads":16},{"category":"3.10","date":"2026-07-07","downloads":57},{"category":"3.10","date":"2026-07-08","downloads":111},{"category":"3.10","date":"2026-07-09","downloads":41},{"category":"3.10","date":"2026-07-10","downloads":40},{"category":"3.10","date":"2026-07-11","downloads":8},{"category":"3.10","date":"2026-07-12","downloads":8},{"category":"3.10","date":"2026-07-13","downloads":17},{"category":"3.10","date":"2026-07-14","downloads":27},{"category":"3.10","date":"2026-07-15","downloads":19},{"category":"3.10","date":"2026-07-16","downloads":25},{"category":"3.10","date":"2026-07-17","downloads":16},{"category":"3.10","date":"2026-07-18","downloads":8},{"category":"3.10","date":"2026-07-19","downloads":2},{"category":"3.10","date":"2026-07-20","downloads":16},{"category":"3.10","date":"2026-07-21","downloads":22},{"category":"3.10","date":"2026-07-22","downloads":17},{"category":"3.10","date":"2026-07-23","downloads":104},{"category":"3.10","date":"2026-07-24","downloads":11},{"category":"3.10","date":"2026-07-25","downloads":33},{"category":"3.10","date":"2026-07-26","downloads":12},{"category":"3.10","date":"2026-07-27","downloads":53},{"category":"3.10","date":"2026-07-28","downloads":45},{"category":"3.10","date":"2026-07-29","downloads":204},{"category":"3.10","date":"2026-07-30","downloads":48},{"category":"3.10","date":"2026-07-31","downloads":69},{"category":"3.10","date":"2026-08-01","downloads":11},{"category":"3.10","date":"2026-08-02","downloads":6},{"category":"3.10","date":"2026-08-03","downloads":32},{"category":"3.10","date":"2026-08-04","downloads":15},{"category":"3.10","date":"2026-08-05","downloads":46},{"category":"3.10","date":"2026-08-06","downloads":211},{"category":"3.10","date":"2026-08-07","downloads":193},{"category":"3.10","date":"2026-08-08","downloads":240},{"category":"3.10","date":"2026-08-09","downloads":182},{"category":"3.10","date":"2026-08-10","downloads":148},{"category":"3.10","date":"2026-08-11","downloads":83},{"category":"3.10","date":"2026-08-12","downloads":57},{"category":"3.10","date":"2026-08-13","downloads":117},{"category":"3.10","date":"2026-08-14","downloads":92},{"category":"3.10","date":"2026-08-15","downloads":61},{"category":"3.10","date":"2026-08-16","downloads":10},{"category":"3.10","date":"2026-08-17","downloads":41},{"category":"3.10","date":"2026-08-18","downloads":113},{"category":"3.10","date":"2026-08-19","downloads":15},{"category":"3.10","date":"2026-08-20","downloads":47},{"category":"3.10","date":"2026-08-21","downloads":59},{"category":"3.10","date":"2026-08-22","downloads":24},{"category":"3.10","date":"2026-08-23","downloads":30},{"category":"3.10","date":"2026-08-24","downloads":11},{"category":"3.10","date":"2026-08-25","downloads":23},{"category":"3.10","date":"2026-08-26","downloads":17},{"category":"3.10","date":"2026-08-27","downloads":15},{"category":"3.10","date":"2026-08-28","downloads":6},{"category":"3.10","date":"2026-08-29","downloads":14},{"category":"3.10","date":"2026-08-30","downloads":9},{"category":"3.10","date":"2026-08-31","downloads":9},{"category":"3.10","date":"2026-09-01","downloads":14},{"category":"3.10","date":"2026-09-02","downloads":33},{"category":"3.10","date":"2026-09-03","downloads":13},{"category":"3.10","date":"2026-09-04","downloads":33},{"category":"3.10","date":"2026-09-05","downloads":1},{"category":"3.10","date":"2026-09-06","downloads":10},{"category":"3.10","date":"2026-09-07","downloads":14},{"category":"3.10","date":"2026-09-08","downloads":18},{"category":"3.10","date":"2026-09-09","downloads":21},{"category":"3.10","date":"2026-09-10","downloads":10},{"category":"3.10","date":"2026-09-11","downloads":8},{"category":"3.10","date":"2026-09-12","downloads":1},{"category":"3.10","date":"2026-09-13","downloads":3},{"category":"3.10","date":"2026-09-14","downloads":23},{"category":"3.10","date":"2026-09-15","downloads":18},{"category":"3.10","date":"2026-09-16","downloads":53},{"category":"3.10","date":"2026-09-17","downloads":19},{"category":"3.10","date":"2026-09-18","downloads":7},{"category":"3.10","date":"2026-09-19","downloads":15},{"category":"3.10","date":"2026-09-20","downloads":24},{"category":"3.10","date":"2026-09-21","downloads":26},{"category":"3.10","date":"2026-09-22","downloads":18},{"category":"3.10","date":"2026-09-23","downloads":73},{"category":"3.10","date":"2026-09-24","downloads":78},{"category":"3.10","date":"2026-09-25","downloads":15},{"category":"3.10","date":"2026-09-26","downloads":12},{"category":"3.10","date":"2026-09-27","downloads":4},{"category":"3.10","date":"2026-09-28","downloads":7},{"category":"3.10","date":"2026-09-29","downloads":12},{"category":"3.10","date":"2026-09-30","downloads":8},{"category":"3.10","date":"2026-10-01","downloads":13},{"category":"3.10","date":"2026-10-02","downloads":42},{"category":"3.10","date":"2026-10-03","downloads":3},{"category":"3.10","date":"2026-10-04","downloads":11},{"category":"3.10","date":"2026-10-05","downloads":20},{"category":"3.10","date":"2026-10-06","downloads":9},{"category":"3.10","date":"2026-10-07","downloads":12},{"category":"3.10","date":"2026-10-08","downloads":7},{"category":"3.10","date":"2026-10-09","downloads":14},{"category":"3.10","date":"2026-10-10","downloads":14},{"category":"3.10","date":"2026-10-11","downloads":18},{"category":"3.10","date":"2026-10-12","downloads":62},{"category":"3.10","date":"2026-10-13","downloads":51},{"category":"3.10","date":"2026-10-14","downloads":19},{"category":"3.10","date":"2026-10-15","downloads":7},{"category":"3.10","date":"2026-10-16","downloads":76},{"category":"3.10","date":"2026-10-17","downloads":92},{"category":"3.10","date":"2026-10-18","downloads":24},{"category":"3.10","date":"2026-10-19","downloads":22},{"category":"3.10","date":"2026-10-20","downloads":27},{"category":"3.10","date":"2026-10-21","downloads":45},{"category":"3.10","date":"2026-10-22","downloads":89},{"category":"3.10","date":"2026-10-23","downloads":20},{"category":"3.10","date":"2026-10-24","downloads":4},{"category":"3.10","date":"2026-10-25","downloads":10},{"category":"3.10","date":"2026-10-26","downloads":8},{"category":"3.10","date":"2026-10-27","downloads":3},{"category":"3.10","date":"2026-10-28","downloads":9},{"category":"3.11","date":"2026-04-08","downloads":22},{"category":"3.11","date":"2026-04-09","downloads":76},{"category":"3.11","date":"2026-04-10","downloads":69},{"category":"3.11","date":"2026-04-11","downloads":24},{"category":"3.11","date":"2026-04-12","downloads":38},{"category":"3.11","date":"2026-04-13","downloads":93},{"category":"3.11","date":"2026-04-14","downloads":77},{"category":"3.11","date":"2026-04-15","downloads":73},{"category":"3.11","date":"2026-04-16","downloads":48},{"category":"3.11","date":"2026-04-17","downloads":29},{"category":"3.11","date":"2026-04-18","downloads":49},{"category":"3.11","date":"2026-04-19","downloads":18},{"category":"3.11","date":"2026-04-20","downloads":79},{"category":"3.11","date":"2026-04-21","downloads":74},{"category":"3.11","date":"2026-04-22","downloads":173},{"category":"3.11","date":"2026-04-23","downloads":28},{"category":"3.11","date":"2026-04-24","downloads":13},{"category":"3.11","date":"2026-04-25","downloads":282},{"category":"3.11","date":"2026-04-26","downloads":84},{"category":"3.11","date":"2026-04-27","downloads":64},{"category":"3.11","date":"2026-04-28","downloads":23},{"category":"3.11","date":"2026-04-29","downloads":57},{"category":"3.11","date":"2026-04-30","downloads":62},{"category":"3.11","date":"2026-05-01","downloads":71},{"category":"3.11","date":"2026-05-02","downloads":22},{"category":"3.11","date":"2026-05-03","downloads":28},{"category":"3.11","date":"2026-05-04","downloads":53},{"category":"3.11","date":"2026-05-05","downloads":125},{"category":"3.11","date":"2026-05-06","downloads":19},{"category":"3.11","date":"2026-05-07","downloads":63},{"category":"3.11","date":"2026-05-08","downloads":76},{"category":"3.11","date":"2026-05-09","downloads":11},{"category":"3.11","date":"2026-05-10","downloads":6},{"category":"3.11","date":"2026-05-11","downloads":204},{"category":"3.11","date":"2026-05-12","downloads":236},{"category":"3.11","date":"2026-05-13","downloads":35},{"category":"3.11","date":"2026-05-14","downloads":100},{"category":"3.11","date":"2026-05-15","downloads":55},{"category":"3.11","date":"2026-05-16","downloads":228},{"category":"3.11","date":"2026-05-17","downloads":38},{"category":"3.11","date":"2026-05-18","downloads":14},{"category":"3.11","date":"2026-05-19","downloads":39},{"category":"3.11","date":"2026-05-20","downloads":18},{"category":"3.11","date":"2026-05-21","downloads":92},{"category":"3.11","date":"2026-05-22","downloads":30},{"category":"3.11","date":"2026-05-23","downloads":30},{"category":"3.11","date":"2026-05-24","downloads":60},{"category":"3.11","date":"2026-05-25","downloads":45},{"category":"3.11","date":"2026-05-26","downloads":48},{"category":"3.11","date":"2026-05-27","downloads":67},{"category":"3.11","date":"2026-05-28","downloads":21},{"category":"3.11","date":"2026-05-29","downloads":33},{"category":"3.11","date":"2026-05-30","downloads":17},{"category":"3.11","date":"2026-05-31","downloads":28},{"category":"3.11","date":"2026-06-01","downloads":31},{"category":"3.11","date":"2026-06-02","downloads":47},{"category":"3.11","date":"2026-06-03","downloads":16},{"category":"3.11","date":"2026-06-04","downloads":21},{"category":"3.11","date":"2026-06-05","downloads":21},{"category":"3.11","date":"2026-06-06","downloads":18},{"category":"3.11","date":"2026-06-07","downloads":41},{"category":"3.11","date":"2026-06-08","downloads":41},{"category":"3.11","date":"2026-06-09","downloads":110},{"category":"3.11","date":"2026-06-10","downloads":176},{"category":"3.11","date":"2026-06-11","downloads":63},{"category":"3.11","date":"2026-06-12","downloads":21},{"category":"3.11","date":"2026-06-13","downloads":14},{"category":"3.11","date":"2026-06-14","downloads":10},{"category":"3.11","date":"2026-06-15","downloads":20},{"category":"3.11","date":"2026-06-16","downloads":38},{"category":"3.11","date":"2026-06-17","downloads":17},{"category":"3.11","date":"2026-06-18","downloads":12},{"category":"3.11","date":"2026-06-19","downloads":141},{"category":"3.11","date":"2026-06-20","downloads":154},{"category":"3.11","date":"2026-06-21","downloads":100},{"category":"3.11","date":"2026-06-22","downloads":13},{"category":"3.11","date":"2026-06-23","downloads":30},{"category":"3.11","date":"2026-06-24","downloads":257},{"category":"3.11","date":"2026-06-25","downloads":2},{"category":"3.11","date":"2026-06-26","downloads":24},{"category":"3.11","date":"2026-06-27","downloads":19},{"category":"3.11","date":"2026-06-28","downloads":33},{"category":"3.11","date":"2026-06-29","downloads":26},{"category":"3.11","date":"2026-06-30","downloads":66},{"category":"3.11","date":"2026-07-01","downloads":22},{"category":"3.11","date":"2026-07-02","downloads":16},{"category":"3.11","date":"2026-07-03","downloads":22},{"category":"3.11","date":"2026-07-04","downloads":25},{"category":"3.11","date":"2026-07-05","downloads":18},{"category":"3.11","date":"2026-07-06","downloads":74},{"category":"3.11","date":"2026-07-07","downloads":43},{"category":"3.11","date":"2026-07-08","downloads":24},{"category":"3.11","date":"2026-07-09","downloads":10},{"category":"3.11","date":"2026-07-10","downloads":29},{"category":"3.11","date":"2026-07-11","downloads":24},{"category":"3.11","date":"2026-07-12","downloads":7},{"category":"3.11","date":"2026-07-13","downloads":12},{"category":"3.11","date":"2026-07-14","downloads":52},{"category":"3.11","date":"2026-07-15","downloads":21},{"category":"3.11","date":"2026-07-16","downloads":16},{"category":"3.11","date":"2026-07-17","downloads":36},{"category":"3.11","date":"2026-07-18","downloads":12},{"category":"3.11","date":"2026-07-19","downloads":14},{"category":"3.11","date":"2026-07-20","downloads":32},{"category":"3.11","date":"2026-07-21","downloads":34},{"category":"3.11","date":"2026-07-22","downloads":30},{"category":"3.11","date":"2026-07-23","downloads":150},{"category":"3.11","date":"2026-07-24","downloads":32},{"category":"3.11","date":"2026-07-25","downloads":22},{"category":"3.11","date":"2026-07-26","downloads":5},{"category":"3.11","date":"2026-07-27","downloads":20},{"category":"3.11","date":"2026-07-28","downloads":35},{"category":"3.11","date":"2026-07-29","downloads":36},{"category":"3.11","date":"2026-07-30","downloads":71},{"category":"3.11","date":"2026-07-31","downloads":71},{"category":"3.11","date":"2026-08-01","downloads":11},{"category":"3.11","date":"2026-08-02","downloads":14},{"category":"3.11","date":"2026-08-03","downloads":77},{"category":"3.11","date":"2026-08-04","downloads":39},{"category":"3.11","date":"2026-08-05","downloads":170},{"category":"3.11","date":"2026-08-06","downloads":164},{"category":"3.11","date":"2026-08-07","downloads":141},{"category":"3.11","date":"2026-08-08","downloads":58},{"category":"3.11","date":"2026-08-09","downloads":32},{"category":"3.11","date":"2026-08-10","downloads":139},{"category":"3.11","date":"2026-08-11","downloads":187},{"category":"3.11","date":"2026-08-12","downloads":124},{"category":"3.11","date":"2026-08-13","downloads":215},{"category":"3.11","date":"2026-08-14","downloads":140},{"category":"3.11","date":"2026-08-15","downloads":39},{"category":"3.11","date":"2026-08-16","downloads":29},{"category":"3.11","date":"2026-08-17","downloads":46},{"category":"3.11","date":"2026-08-18","downloads":48},{"category":"3.11","date":"2026-08-19","downloads":118},{"category":"3.11","date":"2026-08-20","downloads":192},{"category":"3.11","date":"2026-08-21","downloads":214},{"category":"3.11","date":"2026-08-22","downloads":35},{"category":"3.11","date":"2026-08-23","downloads":28},{"category":"3.11","date":"2026-08-24","downloads":45},{"category":"3.11","date":"2026-08-25","downloads":228},{"category":"3.11","date":"2026-08-26","downloads":297},{"category":"3.11","date":"2026-08-27","downloads":269},{"category":"3.11","date":"2026-08-28","downloads":263},{"category":"3.11","date":"2026-08-29","downloads":301},{"category":"3.11","date":"2026-08-30","downloads":325},{"category":"3.11","date":"2026-08-31","downloads":350},{"category":"3.11","date":"2026-09-01","downloads":270},{"category":"3.11","date":"2026-09-02","downloads":327},{"category":"3.11","date":"2026-09-03","downloads":292},{"category":"3.11","date":"2026-09-04","downloads":308},{"category":"3.11","date":"2026-09-05","downloads":267},{"category":"3.11","date":"2026-09-06","downloads":252},{"category":"3.11","date":"2026-09-07","downloads":260},{"category":"3.11","date":"2026-09-08","downloads":284},{"category":"3.11","date":"2026-09-09","downloads":246},{"category":"3.11","date":"2026-09-10","downloads":248},{"category":"3.11","date":"2026-09-11","downloads":256},{"category":"3.11","date":"2026-09-12","downloads":247},{"category":"3.11","date":"2026-09-13","downloads":250},{"category":"3.11","date":"2026-09-14","downloads":244},{"category":"3.11","date":"2026-09-15","downloads":262},{"category":"3.11","date":"2026-09-16","downloads":262},{"category":"3.11","date":"2026-09-17","downloads":283},{"category":"3.11","date":"2026-09-18","downloads":372},{"category":"3.11","date":"2026-09-19","downloads":260},{"category":"3.11","date":"2026-09-20","downloads":274},{"category":"3.11","date":"2026-09-21","downloads":279},{"category":"3.11","date":"2026-09-22","downloads":268},{"category":"3.11","date":"2026-09-23","downloads":329},{"category":"3.11","date":"2026-09-24","downloads":401},{"category":"3.11","date":"2026-09-25","downloads":319},{"category":"3.11","date":"2026-09-26","downloads":279},{"category":"3.11","date":"2026-09-27","downloads":254},{"category":"3.11","date":"2026-09-28","downloads":241},{"category":"3.11","date":"2026-09-29","downloads":260},{"category":"3.11","date":"2026-09-30","downloads":330},{"category":"3.11","date":"2026-10-01","downloads":316},{"category":"3.11","date":"2026-10-02","downloads":295},{"category":"3.11","date":"2026-10-03","downloads":281},{"category":"3.11","date":"2026-10-04","downloads":291},{"category":"3.11","date":"2026-10-05","downloads":421},{"category":"3.11","date":"2026-10-06","downloads":420},{"category":"3.11","date":"2026-10-07","downloads":245},{"category":"3.11","date":"2026-10-08","downloads":305},{"category":"3.11","date":"2026-10-09","downloads":310},{"category":"3.11","date":"2026-10-10","downloads":288},{"category":"3.11","date":"2026-10-11","downloads":308},{"category":"3.11","date":"2026-10-12","downloads":401},{"category":"3.11","date":"2026-10-13","downloads":422},{"category":"3.11","date":"2026-10-14","downloads":447},{"category":"3.11","date":"2026-10-15","downloads":409},{"category":"3.11","date":"2026-10-16","downloads":473},{"category":"3.11","date":"2026-10-17","downloads":583},{"category":"3.11","date":"2026-10-18","downloads":452},{"category":"3.11","date":"2026-10-19","downloads":331},{"category":"3.11","date":"2026-10-20","downloads":378},{"category":"3.11","date":"2026-10-21","downloads":380},{"category":"3.11","date":"2026-10-22","downloads":415},{"category":"3.11","date":"2026-10-23","downloads":488},{"category":"3.11","date":"2026-10-24","downloads":332},{"category":"3.11","date":"2026-10-25","downloads":507},{"category":"3.11","date":"2026-10-26","downloads":696},{"category":"3.11","date":"2026-10-27","downloads":395},{"category":"3.11","date":"2026-10-28","downloads":166},{"category":"3.12","date":"2026-04-08","downloads":12226},{"category":"3.12","date":"2026-04-09","downloads":12981},{"category":"3.12","date":"2026-04-10","downloads":12125},{"category":"3.12","date":"2026-04-11","downloads":6164},{"category":"3.12","date":"2026-04-12","downloads":4737},{"category":"3.12","date":"2026-04-13","downloads":10208},{"category":"3.12","date":"2026-04-14","downloads":12506},{"category":"3.12","date":"2026-04-15","downloads":15323},{"category":"3.12","date":"2026-04-16","downloads":12709},{"category":"3.12","date":"2026-04-17","downloads":11844},{"category":"3.12","date":"2026-04-18","downloads":6405},{"category":"3.12","date":"2026-04-19","downloads":4093},{"category":"3.12","date":"2026-04-20","downloads":10289},{"category":"3.12","date":"2026-04-21","downloads":14434},{"category":"3.12","date":"2026-04-22","downloads":13422},{"category":"3.12","date":"2026-04-23","downloads":13161},{"category":"3.12","date":"2026-04-24","downloads":16607},{"category":"3.12","date":"2026-04-25","downloads":6991},{"category":"3.12","date":"2026-04-26","downloads":6760},{"category":"3.12","date":"2026-04-27","downloads":11245},{"category":"3.12","date":"2026-04-28","downloads":14101},{"category":"3.12","date":"2026-04-29","downloads":14954},{"category":"3.12","date":"2026-04-30","downloads":13128},{"category":"3.12","date":"2026-05-01","downloads":14271},{"category":"3.12","date":"2026-05-02","downloads":4957},{"category":"3.12","date":"2026-05-03","downloads":6218},{"category":"3.12","date":"2026-05-04","downloads":9825},{"category":"3.12","date":"2026-05-05","downloads":14412},{"category":"3.12","date":"2026-05-06","downloads":15458},{"category":"3.12","date":"2026-05-07","downloads":15491},{"category":"3.12","date":"2026-05-08","downloads":12227},{"category":"3.12","date":"2026-05-09","downloads":5596},{"category":"3.12","date":"2026-05-10","downloads":3205},{"category":"3.12","date":"2026-05-11","downloads":10113},{"category":"3.12","date":"2026-05-12","downloads":13696},{"category":"3.12","date":"2026-05-13","downloads":13638},{"category":"3.12","date":"2026-05-14","downloads":11643},{"category":"3.12","date":"2026-05-15","downloads":12044},{"category":"3.12","date":"2026-05-16","downloads":4764},{"category":"3.12","date":"2026-05-17","downloads":4683},{"category":"3.12","date":"2026-05-18","downloads":11053},{"category":"3.12","date":"2026-05-19","downloads":10071},{"category":"3.12","date":"2026-05-20","downloads":12443},{"category":"3.12","date":"2026-05-21","downloads":15291},{"category":"3.12","date":"2026-05-22","downloads":12520},{"category":"3.12","date":"2026-05-23","downloads":5982},{"category":"3.12","date":"2026-05-24","downloads":4771},{"category":"3.12","date":"2026-05-25","downloads":5346},{"category":"3.12","date":"2026-05-26","downloads":11059},{"category":"3.12","date":"2026-05-27","downloads":13816},{"category":"3.12","date":"2026-05-28","downloads":13179},{"category":"3.12","date":"2026-05-29","downloads":15823},{"category":"3.12","date":"2026-05-30","downloads":5789},{"category":"3.12","date":"2026-05-31","downloads":6194},{"category":"3.12","date":"2026-06-01","downloads":9527},{"category":"3.12","date":"2026-06-02","downloads":8768},{"category":"3.12","date":"2026-06-03","downloads":7879},{"category":"3.12","date":"2026-06-04","downloads":7330},{"category":"3.12","date":"2026-06-05","downloads":6678},{"category":"3.12","date":"2026-06-06","downloads":3615},{"category":"3.12","date":"2026-06-07","downloads":2467},{"category":"3.12","date":"2026-06-08","downloads":6360},{"category":"3.12","date":"2026-06-09","downloads":8440},{"category":"3.12","date":"2026-06-10","downloads":9075},{"category":"3.12","date":"2026-06-11","downloads":8908},{"category":"3.12","date":"2026-06-12","downloads":7198},{"category":"3.12","date":"2026-06-13","downloads":3047},{"category":"3.12","date":"2026-06-14","downloads":2331},{"category":"3.12","date":"2026-06-15","downloads":4721},{"category":"3.12","date":"2026-06-16","downloads":5976},{"category":"3.12","date":"2026-06-17","downloads":5340},{"category":"3.12","date":"2026-06-18","downloads":5081},{"category":"3.12","date":"2026-06-19","downloads":4184},{"category":"3.12","date":"2026-06-20","downloads":2413},{"category":"3.12","date":"2026-06-21","downloads":1985},{"category":"3.12","date":"2026-06-22","downloads":3821},{"category":"3.12","date":"2026-06-23","downloads":4656},{"category":"3.12","date":"2026-06-24","downloads":5179},{"category":"3.12","date":"2026-06-25","downloads":6220},{"category":"3.12","date":"2026-06-26","downloads":6869},{"category":"3.12","date":"2026-06-27","downloads":3260},{"category":"3.12","date":"2026-06-28","downloads":2701},{"category":"3.12","date":"2026-06-29","downloads":6802},{"category":"3.12","date":"2026-06-30","downloads":7696},{"category":"3.12","date":"2026-07-01","downloads":7645},{"category":"3.12","date":"2026-07-02","downloads":7399},{"category":"3.12","date":"2026-07-03","downloads":5595},{"category":"3.12","date":"2026-07-04","downloads":2657},{"category":"3.12","date":"2026-07-05","downloads":1942},{"category":"3.12","date":"2026-07-06","downloads":7036},{"category":"3.12","date":"2026-07-07","downloads":7886},{"category":"3.12","date":"2026-07-08","downloads":8399},{"category":"3.12","date":"2026-07-09","downloads":8076},{"category":"3.12","date":"2026-07-10","downloads":8218},{"category":"3.12","date":"2026-07-11","downloads":3450},{"category":"3.12","date":"2026-07-12","downloads":2235},{"category":"3.12","date":"2026-07-13","downloads":8224},{"category":"3.12","date":"2026-07-14","downloads":10472},{"category":"3.12","date":"2026-07-15","downloads":11527},{"category":"3.12","date":"2026-07-16","downloads":9885},{"category":"3.12","date":"2026-07-17","downloads":9842},{"category":"3.12","date":"2026-07-18","downloads":5285},{"category":"3.12","date":"2026-07-19","downloads":2415},{"category":"3.12","date":"2026-07-20","downloads":9583},{"category":"3.12","date":"2026-07-21","downloads":12765},{"category":"3.12","date":"2026-07-22","downloads":12174},{"category":"3.12","date":"2026-07-23","downloads":11219},{"category":"3.12","date":"2026-07-24","downloads":12558},{"category":"3.12","date":"2026-07-25","downloads":5082},{"category":"3.12","date":"2026-07-26","downloads":4156},{"category":"3.12","date":"2026-07-27","downloads":11595},{"category":"3.12","date":"2026-07-28","downloads":14829},{"category":"3.12","date":"2026-07-29","downloads":13988},{"category":"3.12","date":"2026-07-30","downloads":15304},{"category":"3.12","date":"2026-07-31","downloads":14144},{"category":"3.12","date":"2026-08-01","downloads":10002},{"category":"3.12","date":"2026-08-02","downloads":6148},{"category":"3.12","date":"2026-08-03","downloads":12727},{"category":"3.12","date":"2026-08-04","downloads":15549},{"category":"3.12","date":"2026-08-05","downloads":15698},{"category":"3.12","date":"2026-08-06","downloads":9642},{"category":"3.12","date":"2026-08-07","downloads":16913},{"category":"3.12","date":"2026-08-08","downloads":6992},{"category":"3.12","date":"2026-08-09","downloads":2710},{"category":"3.12","date":"2026-08-10","downloads":3708},{"category":"3.12","date":"2026-08-11","downloads":4028},{"category":"3.12","date":"2026-08-12","downloads":5222},{"category":"3.12","date":"2026-08-13","downloads":3276},{"category":"3.12","date":"2026-08-14","downloads":3933},{"category":"3.12","date":"2026-08-15","downloads":1604},{"category":"3.12","date":"2026-08-16","downloads":1820},{"category":"3.12","date":"2026-08-17","downloads":2586},{"category":"3.12","date":"2026-08-18","downloads":3399},{"category":"3.12","date":"2026-08-19","downloads":3859},{"category":"3.12","date":"2026-08-20","downloads":3465},{"category":"3.12","date":"2026-08-21","downloads":3409},{"category":"3.12","date":"2026-08-22","downloads":2810},{"category":"3.12","date":"2026-08-23","downloads":1895},{"category":"3.12","date":"2026-08-24","downloads":2233},{"category":"3.12","date":"2026-08-25","downloads":2054},{"category":"3.12","date":"2026-08-26","downloads":1596},{"category":"3.12","date":"2026-08-27","downloads":1730},{"category":"3.12","date":"2026-08-28","downloads":1541},{"category":"3.12","date":"2026-08-29","downloads":679},{"category":"3.12","date":"2026-08-30","downloads":812},{"category":"3.12","date":"2026-08-31","downloads":1381},{"category":"3.12","date":"2026-09-01","downloads":1643},{"category":"3.12","date":"2026-09-02","downloads":2197},{"category":"3.12","date":"2026-09-03","downloads":2004},{"category":"3.12","date":"2026-09-04","downloads":1849},{"category":"3.12","date":"2026-09-05","downloads":1116},{"category":"3.12","date":"2026-09-06","downloads":1038},{"category":"3.12","date":"2026-09-07","downloads":1454},{"category":"3.12","date":"2026-09-08","downloads":1853},{"category":"3.12","date":"2026-09-09","downloads":2058},{"category":"3.12","date":"2026-09-10","downloads":2254},{"category":"3.12","date":"2026-09-11","downloads":2261},{"category":"3.12","date":"2026-09-12","downloads":1711},{"category":"3.12","date":"2026-09-13","downloads":1428},{"category":"3.12","date":"2026-09-14","downloads":1771},{"category":"3.12","date":"2026-09-15","downloads":2557},{"category":"3.12","date":"2026-09-16","downloads":2616},{"category":"3.12","date":"2026-09-17","downloads":2704},{"category":"3.12","date":"2026-09-18","downloads":2535},{"category":"3.12","date":"2026-09-19","downloads":1394},{"category":"3.12","date":"2026-09-20","downloads":1499},{"category":"3.12","date":"2026-09-21","downloads":2244},{"category":"3.12","date":"2026-09-22","downloads":2846},{"category":"3.12","date":"2026-09-23","downloads":2280},{"category":"3.12","date":"2026-09-24","downloads":2440},{"category":"3.12","date":"2026-09-25","downloads":1988},{"category":"3.12","date":"2026-09-26","downloads":1167},{"category":"3.12","date":"2026-09-27","downloads":1057},{"category":"3.12","date":"2026-09-28","downloads":1561},{"category":"3.12","date":"2026-09-29","downloads":1853},{"category":"3.12","date":"2026-09-30","downloads":1879},{"category":"3.12","date":"2026-10-01","downloads":2150},{"category":"3.12","date":"2026-10-02","downloads":1989},{"category":"3.12","date":"2026-10-03","downloads":1293},{"category":"3.12","date":"2026-10-04","downloads":1088},{"category":"3.12","date":"2026-10-05","downloads":1722},{"category":"3.12","date":"2026-10-06","downloads":2566},{"category":"3.12","date":"2026-10-07","downloads":2933},{"category":"3.12","date":"2026-10-08","downloads":3491},{"category":"3.12","date":"2026-10-09","downloads":2630},{"category":"3.12","date":"2026-10-10","downloads":1777},{"category":"3.12","date":"2026-10-11","downloads":1335},{"category":"3.12","date":"2026-10-12","downloads":2220},{"category":"3.12","date":"2026-10-13","downloads":2864},{"category":"3.12","date":"2026-10-14","downloads":3219},{"category":"3.12","date":"2026-10-15","downloads":3019},{"category":"3.12","date":"2026-10-16","downloads":2154},{"category":"3.12","date":"2026-10-17","downloads":1719},{"category":"3.12","date":"2026-10-18","downloads":1432},{"category":"3.12","date":"2026-10-19","downloads":2915},{"category":"3.12","date":"2026-10-20","downloads":2539},{"category":"3.12","date":"2026-10-21","downloads":2155},{"category":"3.12","date":"2026-10-22","downloads":2002},{"category":"3.12","date":"2026-10-23","downloads":2560},{"category":"3.12","date":"2026-10-24","downloads":1491},{"category":"3.12","date":"2026-10-25","downloads":1915},{"category":"3.12","date":"2026-10-26","downloads":1174},{"category":"3.12","date":"2026-10-27","downloads":1035},{"category":"3.12","date":"2026-10-28","downloads":1176},{"category":"3.13","date":"2026-04-08","downloads":22},{"category":"3.13","date":"2026-04-09","downloads":19},{"category":"3.13","date":"2026-04-10","downloads":20},{"category":"3.13","date":"2026-04-11","downloads":2},{"category":"3.13","date":"2026-04-12","downloads":3},{"category":"3.13","date":"2026-04-13","downloads":24},{"category":"3.13","date":"2026-04-14","downloads":15},{"category":"3.13","date":"2026-04-15","downloads":11},{"category":"3.13","date":"2026-04-16","downloads":15},{"category":"3.13","date":"2026-04-17","downloads":16},{"category":"3.13","date":"2026-04-18","downloads":21},{"category":"3.13","date":"2026-04-19","downloads":14},{"category":"3.13","date":"2026-04-20","downloads":13},{"category":"3.13","date":"2026-04-21","downloads":11},{"category":"3.13","date":"2026-04-22","downloads":14},{"category":"3.13","date":"2026-04-23","downloads":14},{"category":"3.13","date":"2026-04-24","downloads":123},{"category":"3.13","date":"2026-04-25","downloads":8},{"category":"3.13","date":"2026-04-26","downloads":6},{"category":"3.13","date":"2026-04-27","downloads":12},{"category":"3.13","date":"2026-04-28","downloads":5},{"category":"3.13","date":"2026-04-29","downloads":8},{"category":"3.13","date":"2026-04-30","downloads":41},{"category":"3.13","date":"2026-05-01","downloads":29},{"category":"3.13","date":"2026-05-02","downloads":37},{"category":"3.13","date":"2026-05-03","downloads":22},{"category":"3.13","date":"2026-05-04","downloads":68},{"category":"3.13","date":"2026-05-05","downloads":26},{"category":"3.13","date":"2026-05-06","downloads":16},{"category":"3.13","date":"2026-05-07","downloads":43},{"category":"3.13","date":"2026-05-08","downloads":119},{"category":"3.13","date":"2026-05-09","downloads":101},{"category":"3.13","date":"2026-05-10","downloads":8},{"category":"3.13","date":"2026-05-11","downloads":53},{"category":"3.13","date":"2026-05-12","downloads":44},{"category":"3.13","date":"2026-05-13","downloads":71},{"category":"3.13","date":"2026-05-14","downloads":26},{"category":"3.13","date":"2026-05-15","downloads":14},{"category":"3.13","date":"2026-05-16","downloads":13},{"category":"3.13","date":"2026-05-17","downloads":19},{"category":"3.13","date":"2026-05-18","downloads":8},{"category":"3.13","date":"2026-05-19","downloads":44},{"category":"3.13","date":"2026-05-20","downloads":6},{"category":"3.13","date":"2026-05-21","downloads":54},{"category":"3.13","date":"2026-05-22","downloads":14},{"category":"3.13","date":"2026-05-23","downloads":15},{"category":"3.13","date":"2026-05-24","downloads":27},{"category":"3.13","date":"2026-05-25","downloads":23},{"category":"3.13","date":"2026-05-26","downloads":20},{"category":"3.13","date":"2026-05-27","downloads":19},{"category":"3.13","date":"2026-05-28","downloads":30},{"category":"3.13","date":"2026-05-29","downloads":35},{"category":"3.13","date":"2026-05-30","downloads":87},{"category":"3.13","date":"2026-05-31","downloads":30},{"category":"3.13","date":"2026-06-01","downloads":21},{"category":"3.13","date":"2026-06-02","downloads":52},{"category":"3.13","date":"2026-06-03","downloads":20},{"category":"3.13","date":"2026-06-04","downloads":28},{"category":"3.13","date":"2026-06-05","downloads":26},{"category":"3.13","date":"2026-06-06","downloads":27},{"category":"3.13","date":"2026-06-07","downloads":35},{"category":"3.13","date":"2026-06-08","downloads":84},{"category":"3.13","date":"2026-06-09","downloads":30},{"category":"3.13","date":"2026-06-10","downloads":12},{"category":"3.13","date":"2026-06-11","downloads":17},{"category":"3.13","date":"2026-06-12","downloads":17},{"category":"3.13","date":"2026-06-13","downloads":60},{"category":"3.13","date":"2026-06-14","downloads":16},{"category":"3.13","date":"2026-06-15","downloads":24},{"category":"3.13","date":"2026-06-16","downloads":22},{"category":"3.13","date":"2026-06-17","downloads":25},{"category":"3.13","date":"2026-06-18","downloads":20},{"category":"3.13","date":"2026-06-19","downloads":65},{"category":"3.13","date":"2026-06-20","downloads":68},{"category":"3.13","date":"2026-06-21","downloads":38},{"category":"3.13","date":"2026-06-22","downloads":54},{"category":"3.13","date":"2026-06-23","downloads":38},{"category":"3.13","date":"2026-06-24","downloads":247},{"category":"3.13","date":"2026-06-25","downloads":25},{"category":"3.13","date":"2026-06-26","downloads":29},{"category":"3.13","date":"2026-06-27","downloads":21},{"category":"3.13","date":"2026-06-28","downloads":18},{"category":"3.13","date":"2026-06-29","downloads":23},{"category":"3.13","date":"2026-06-30","downloads":30},{"category":"3.13","date":"2026-07-01","downloads":34},{"category":"3.13","date":"2026-07-02","downloads":28},{"category":"3.13","date":"2026-07-03","downloads":16},{"category":"3.13","date":"2026-07-04","downloads":29},{"category":"3.13","date":"2026-07-05","downloads":12},{"category":"3.13","date":"2026-07-06","downloads":8},{"category":"3.13","date":"2026-07-07","downloads":4},{"category":"3.13","date":"2026-07-08","downloads":4},{"category":"3.13","date":"2026-07-09","downloads":10},{"category":"3.13","date":"2026-07-10","downloads":6},{"category":"3.13","date":"2026-07-11","downloads":6},{"category":"3.13","date":"2026-07-12","downloads":1},{"category":"3.13","date":"2026-07-13","downloads":5},{"category":"3.13","date":"2026-07-14","downloads":4},{"category":"3.13","date":"2026-07-15","downloads":8},{"category":"3.13","date":"2026-07-16","downloads":4},{"category":"3.13","date":"2026-07-17","downloads":2},{"category":"3.13","date":"2026-07-18","downloads":2},{"category":"3.13","date":"2026-07-19","downloads":10},{"category":"3.13","date":"2026-07-20","downloads":9},{"category":"3.13","date":"2026-07-21","downloads":22},{"category":"3.13","date":"2026-07-22","downloads":7},{"category":"3.13","date":"2026-07-23","downloads":9},{"category":"3.13","date":"2026-07-24","downloads":9},{"category":"3.13","date":"2026-07-25","downloads":20},{"category":"3.13","date":"2026-07-26","downloads":18},{"category":"3.13","date":"2026-07-27","downloads":7},{"category":"3.13","date":"2026-07-28","downloads":9},{"category":"3.13","date":"2026-07-29","downloads":3},{"category":"3.13","date":"2026-07-30","downloads":13},{"category":"3.13","date":"2026-07-31","downloads":12},{"category":"3.13","date":"2026-08-01","downloads":4},{"category":"3.13","date":"2026-08-02","downloads":8},{"category":"3.13","date":"2026-08-03","downloads":12},{"category":"3.13","date":"2026-08-04","downloads":7},{"category":"3.13","date":"2026-08-05","downloads":4},{"category":"3.13","date":"2026-08-06","downloads":3},{"category":"3.13","date":"2026-08-07","downloads":4},{"category":"3.13","date":"2026-08-08","downloads":2},{"category":"3.13","date":"2026-08-09","downloads":16},{"category":"3.13","date":"2026-08-10","downloads":18},{"category":"3.13","date":"2026-08-11","downloads":6},{"category":"3.13","date":"2026-08-12","downloads":6},{"category":"3.13","date":"2026-08-13","downloads":3},{"category":"3.13","date":"2026-08-14","downloads":8},{"category":"3.13","date":"2026-08-16","downloads":6},{"category":"3.13","date":"2026-08-17","downloads":44},{"category":"3.13","date":"2026-08-18","downloads":7},{"category":"3.13","date":"2026-08-19","downloads":4},{"category":"3.13","date":"2026-08-20","downloads":5},{"category":"3.13","date":"2026-08-21","downloads":5},{"category":"3.13","date":"2026-08-22","downloads":6},{"category":"3.13","date":"2026-08-23","downloads":11},{"category":"3.13","date":"2026-08-24","downloads":5},{"category":"3.13","date":"2026-08-25","downloads":2},{"category":"3.13","date":"2026-08-26","downloads":5},{"category":"3.13","date":"2026-08-27","downloads":8},{"category":"3.13","date":"2026-08-28","downloads":7},{"category":"3.13","date":"2026-08-29","downloads":82},{"category":"3.13","date":"2026-08-31","downloads":4},{"category":"3.13","date":"2026-09-01","downloads":31},{"category":"3.13","date":"2026-09-02","downloads":3},{"category":"3.13","date":"2026-09-03","downloads":6},{"category":"3.13","date":"2026-09-04","downloads":2},{"category":"3.13","date":"2026-09-06","downloads":2},{"category":"3.13","date":"2026-09-07","downloads":4},{"category":"3.13","date":"2026-09-08","downloads":5},{"category":"3.13","date":"2026-09-09","downloads":5},{"category":"3.13","date":"2026-09-10","downloads":9},{"category":"3.13","date":"2026-09-11","downloads":13},{"category":"3.13","date":"2026-09-14","downloads":4},{"category":"3.13","date":"2026-09-15","downloads":2},{"category":"3.13","date":"2026-09-16","downloads":3},{"category":"3.13","date":"2026-09-17","downloads":13},{"category":"3.13","date":"2026-09-18","downloads":1},{"category":"3.13","date":"2026-09-19","downloads":3},{"category":"3.13","date":"2026-09-20","downloads":2},{"category":"3.13","date":"2026-09-21","downloads":3},{"category":"3.13","date":"2026-09-22","downloads":4},{"category":"3.13","date":"2026-09-23","downloads":2},{"category":"3.13","date":"2026-09-24","downloads":2},{"category":"3.13","date":"2026-09-25","downloads":4},{"category":"3.13","date":"2026-09-26","downloads":12},{"category":"3.13","date":"2026-09-28","downloads":10},{"category":"3.13","date":"2026-09-29","downloads":1},{"category":"3.13","date":"2026-09-30","downloads":1},{"category":"3.13","date":"2026-10-01","downloads":4},{"category":"3.13","date":"2026-10-02","downloads":4},{"category":"3.13","date":"2026-10-03","downloads":5},{"category":"3.13","date":"2026-10-04","downloads":11},{"category":"3.13","date":"2026-10-05","downloads":12},{"category":"3.13","date":"2026-10-06","downloads":19},{"category":"3.13","date":"2026-10-07","downloads":113},{"category":"3.13","date":"2026-10-08","downloads":13},{"category":"3.13","date":"2026-10-09","downloads":5},{"category":"3.13","date":"2026-10-10","downloads":7},{"category":"3.13","date":"2026-10-11","downloads":15},{"category":"3.13","date":"2026-10-12","downloads":18},{"category":"3.13","date":"2026-10-13","downloads":18},{"category":"3.13","date":"2026-10-14","downloads":25},{"category":"3.13","date":"2026-10-15","downloads":5},{"category":"3.13","date":"2026-10-16","downloads":4},{"category":"3.13","date":"2026-10-17","downloads":4},{"category":"3.13","date":"2026-10-18","downloads":8},{"category":"3.13","date":"2026-10-19","downloads":14},{"category":"3.13","date":"2026-10-20","downloads":11},{"category":"3.13","date":"2026-10-21","downloads":10},{"category":"3.13","date":"2026-10-22","downloads":10},{"category":"3.13","date":"2026-10-23","downloads":6},{"category":"3.13","date":"2026-10-24","downloads":5},{"category":"3.13","date":"2026-10-25","downloads":1},{"category":"3.13","date":"2026-10-26","downloads":26},{"category":"3.13","date":"2026-10-27","downloads":10},{"category":"3.13","date":"2026-10-28","downloads":11},{"category":"3.14","date":"2026-04-09","downloads":1},{"category":"3.14","date":"2026-04-10","downloads":5},{"category":"3.14","date":"2026-04-16","downloads":1},{"category":"3.14","date":"2026-04-23","downloads":4},{"category":"3.14","date":"2026-04-24","downloads":1},{"category":"3.14","date":"2026-04-27","downloads":4},{"category":"3.14","date":"2026-05-02","downloads":4},{"category":"3.14","date":"2026-05-05","downloads":174},{"category":"3.14","date":"2026-05-07","downloads":4},{"category":"3.14","date":"2026-05-08","downloads":6},{"category":"3.14","date":"2026-05-11","downloads":4},{"category":"3.14","date":"2026-05-12","downloads":5},{"category":"3.14","date":"2026-05-13","downloads":12},{"category":"3.14","date":"2026-05-14","downloads":16},{"category":"3.14","date":"2026-05-21","downloads":10},{"category":"3.14","date":"2026-05-23","downloads":4},{"category":"3.14","date":"2026-05-24","downloads":10},{"category":"3.14","date":"2026-05-25","downloads":1},{"category":"3.14","date":"2026-05-26","downloads":23},{"category":"3.14","date":"2026-05-27","downloads":21},{"category":"3.14","date":"2026-05-28","downloads":2},{"category":"3.14","date":"2026-05-29","downloads":5},{"category":"3.14","date":"2026-06-01","downloads":8},{"category":"3.14","date":"2026-06-02","downloads":4},{"category":"3.14","date":"2026-06-03","downloads":2},{"category":"3.14","date":"2026-06-04","downloads":6},{"category":"3.14","date":"2026-06-05","downloads":2},{"category":"3.14","date":"2026-06-07","downloads":1},{"category":"3.14","date":"2026-06-08","downloads":12},{"category":"3.14","date":"2026-06-09","downloads":49},{"category":"3.14","date":"2026-06-10","downloads":12},{"category":"3.14","date":"2026-06-11","downloads":6},{"category":"3.14","date":"2026-06-13","downloads":2},{"category":"3.14","date":"2026-06-22","downloads":18},{"category":"3.14","date":"2026-06-23","downloads":4},{"category":"3.14","date":"2026-06-24","downloads":4},{"category":"3.14","date":"2026-07-03","downloads":4},{"category":"3.14","date":"2026-07-09","downloads":1},{"category":"3.14","date":"2026-07-10","downloads":2},{"category":"3.14","date":"2026-07-11","downloads":4},{"category":"3.14","date":"2026-07-13","downloads":5},{"category":"3.14","date":"2026-07-15","downloads":1},{"category":"3.14","date":"2026-07-16","downloads":6},{"category":"3.14","date":"2026-07-17","downloads":16},{"category":"3.14","date":"2026-07-19","downloads":5},{"category":"3.14","date":"2026-07-20","downloads":8},{"category":"3.14","date":"2026-07-22","downloads":4},{"category":"3.14","date":"2026-07-23","downloads":12},{"category":"3.14","date":"2026-07-24","downloads":1},{"category":"3.14","date":"2026-07-25","downloads":8},{"category":"3.14","date":"2026-07-27","downloads":4},{"category":"3.14","date":"2026-07-30","downloads":2},{"category":"3.14","date":"2026-07-31","downloads":2},{"category":"3.14","date":"2026-08-01","downloads":1},{"category":"3.14","date":"2026-08-03","downloads":11},{"category":"3.14","date":"2026-08-04","downloads":1},{"category":"3.14","date":"2026-08-06","downloads":7},{"category":"3.14","date":"2026-08-07","downloads":5},{"category":"3.14","date":"2026-08-08","downloads":4},{"category":"3.14","date":"2026-08-09","downloads":2},{"category":"3.14","date":"2026-08-10","downloads":8},{"category":"3.14","date":"2026-08-12","downloads":3},{"category":"3.14","date":"2026-08-18","downloads":10},{"category":"3.14","date":"2026-08-19","downloads":8},{"category":"3.14","date":"2026-08-20","downloads":4},{"category":"3.14","date":"2026-08-21","downloads":6},{"category":"3.14","date":"2026-08-22","downloads":2},{"category":"3.14","date":"2026-08-26","downloads":8},{"category":"3.14","date":"2026-08-27","downloads":1},{"category":"3.14","date":"2026-08-29","downloads":4},{"category":"3.14","date":"2026-08-31","downloads":4},{"category":"3.14","date":"2026-09-01","downloads":4},{"category":"3.14","date":"2026-09-02","downloads":4},{"category":"3.14","date":"2026-09-03","downloads":4},{"category":"3.14","date":"2026-09-06","downloads":2},{"category":"3.14","date":"2026-09-07","downloads":1},{"category":"3.14","date":"2026-09-08","downloads":2},{"category":"3.14","date":"2026-09-11","downloads":4},{"category":"3.14","date":"2026-09-12","downloads":2},{"category":"3.14","date":"2026-09-14","downloads":5},{"category":"3.14","date":"2026-09-17","downloads":8},{"category":"3.14","date":"2026-09-19","downloads":4},{"category":"3.14","date":"2026-09-24","downloads":1},{"category":"3.14","date":"2026-09-27","downloads":1},{"category":"3.14","date":"2026-09-28","downloads":1},{"category":"3.14","date":"2026-09-30","downloads":1},{"category":"3.14","date":"2026-10-02","downloads":1},{"category":"3.14","date":"2026-10-05","downloads":1},{"category":"3.14","date":"2026-10-08","downloads":1},{"category":"3.14","date":"2026-10-11","downloads":1},{"category":"3.14","date":"2026-10-13","downloads":3},{"category":"3.14","date":"2026-10-14","downloads":2},{"category":"3.14","date":"2026-10-15","downloads":1},{"category":"3.14","date":"2026-10-16","downloads":1},{"category":"3.14","date":"2026-10-19","downloads":5},{"category":"3.14","date":"2026-10-21","downloads":4},{"category":"3.14","date":"2026-10-22","downloads":5},{"category":"3.14","date":"2026-10-23","downloads":1},{"category":"3.14","date":"2026-10-24","downloads":4},{"category":"3.14","date":"2026-10-26","downloads":4},{"category":"3.14","date":"2026-10-28","downloads":4},{"category":"3.15","date":"2026-10-26","downloads":1},{"category":"3.7","date":"2026-08-13","downloads":4},{"category":"3.8","date":"2026-04-16","downloads":2},{"category":"3.8","date":"2026-05-29","downloads":1},{"category":"3.8","date":"2026-06-03","downloads":1},{"category":"3.8","date":"2026-06-04","downloads":2},{"category":"3.8","date":"2026-07-14","downloads":2},{"category":"3.8","date":"2026-08-27","downloads":1},{"category":"3.8","date":"2026-09-11","downloads":1},{"category":"3.8","date":"2026-09-24","downloads":1},{"category":"3.8","date":"2026-10-04","downloads":1},{"category":"3.9","date":"2026-04-08","downloads":1},{"category":"3.9","date":"2026-04-12","downloads":1},{"category":"3.9","date":"2026-04-18","downloads":1},{"category":"3.9","date":"2026-04-21","downloads":2},{"category":"3.9","date":"2026-05-05","downloads":1},{"category":"3.9","date":"2026-05-10","downloads":1},{"category":"3.9","date":"2026-05-11","downloads":4},{"category":"3.9","date":"2026-05-15","downloads":1},{"category":"3.9","date":"2026-05-17","downloads":1},{"category":"3.9","date":"2026-05-18","downloads":1},{"category":"3.9","date":"2026-05-20","downloads":2},{"category":"3.9","date":"2026-05-22","downloads":1},{"category":"3.9","date":"2026-05-24","downloads":1},{"category":"3.9","date":"2026-05-30","downloads":1},{"category":"3.9","date":"2026-05-31","downloads":2},{"category":"3.9","date":"2026-06-03","downloads":2},{"category":"3.9","date":"2026-06-04","downloads":5},{"category":"3.9","date":"2026-06-05","downloads":3},{"category":"3.9","date":"2026-06-07","downloads":5},{"category":"3.9","date":"2026-06-12","downloads":7},{"category":"3.9","date":"2026-06-13","downloads":1},{"category":"3.9","date":"2026-06-14","downloads":1},{"category":"3.9","date":"2026-06-17","downloads":1},{"category":"3.9","date":"2026-06-28","downloads":2},{"category":"3.9","date":"2026-06-30","downloads":2},{"category":"3.9","date":"2026-07-03","downloads":1},{"category":"3.9","date":"2026-07-15","downloads":1},{"category":"3.9","date":"2026-07-19","downloads":2},{"category":"3.9","date":"2026-07-23","downloads":3},{"category":"3.9","date":"2026-07-26","downloads":1},{"category":"3.9","date":"2026-07-31","downloads":4},{"category":"3.9","date":"2026-08-01","downloads":1},{"category":"3.9","date":"2026-08-02","downloads":1},{"category":"3.9","date":"2026-08-03","downloads":1},{"category":"3.9","date":"2026-08-04","downloads":2},{"category":"3.9","date":"2026-08-07","downloads":1},{"category":"3.9","date":"2026-08-09","downloads":3},{"category":"3.9","date":"2026-08-11","downloads":1},{"category":"3.9","date":"2026-08-13","downloads":1},{"category":"3.9","date":"2026-08-17","downloads":2},{"category":"3.9","date":"2026-08-19","downloads":3},{"category":"3.9","date":"2026-08-21","downloads":2},{"category":"3.9","date":"2026-08-26","downloads":1},{"category":"3.9","date":"2026-08-31","downloads":10},{"category":"3.9","date":"2026-09-01","downloads":22},{"category":"3.9","date":"2026-09-02","downloads":13},{"category":"3.9","date":"2026-09-04","downloads":2},{"category":"3.9","date":"2026-09-14","downloads":1},{"category":"3.9","date":"2026-09-15","downloads":1},{"category":"3.9","date":"2026-09-16","downloads":1},{"category":"3.9","date":"2026-09-23","downloads":1},{"category":"3.9","date":"2026-09-28","downloads":1},{"category":"3.9","date":"2026-10-01","downloads":1},{"category":"3.9","date":"2026-10-03","downloads":1},{"category":"3.9","date":"2026-10-05","downloads":1},{"category":"3.9","date":"2026-10-06","downloads":6},{"category":"3.9","date":"2026-10-09","downloads":1},{"category":"3.9","date":"2026-10-13","downloads":1},{"category":"3.9","date":"2026-10-14","downloads":1},{"category":"3.9","date":"2026-10-16","downloads":1},{"category":"3.9","date":"2026-10-22","downloads":1},{"category":"3.9","date":"2026-10-25","downloads":1},{"category":"3.9","date":"2026-10-26","downloads":1},{"category":"3.9","date":"2026-10-28","downloads":1},{"category":"null","date":"2026-04-08","downloads":61},{"category":"null","date":"2026-04-09","downloads":76},{"category":"null","date":"2026-04-10","downloads":11},{"category":"null","date":"2026-04-11","downloads":14},{"category":"null","date":"2026-04-12","downloads":23},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":65},{"category":"null","date":"2026-04-15","downloads":243},{"category":"null","date":"2026-04-16","downloads":8},{"category":"null","date":"2026-04-17","downloads":3},{"category":"null","date":"2026-04-18","downloads":6},{"category":"null","date":"2026-04-19","downloads":15},{"category":"null","date":"2026-04-20","downloads":17},{"category":"null","date":"2026-04-21","downloads":3},{"category":"null","date":"2026-04-22","downloads":299},{"category":"null","date":"2026-04-23","downloads":30},{"category":"null","date":"2026-04-24","downloads":22},{"category":"null","date":"2026-04-25","downloads":46},{"category":"null","date":"2026-04-26","downloads":17},{"category":"null","date":"2026-04-27","downloads":18},{"category":"null","date":"2026-04-28","downloads":149},{"category":"null","date":"2026-04-29","downloads":15},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-01","downloads":11},{"category":"null","date":"2026-05-03","downloads":1},{"category":"null","date":"2026-05-04","downloads":7},{"category":"null","date":"2026-05-05","downloads":97},{"category":"null","date":"2026-05-06","downloads":7},{"category":"null","date":"2026-05-07","downloads":10},{"category":"null","date":"2026-05-08","downloads":156},{"category":"null","date":"2026-05-09","downloads":179},{"category":"null","date":"2026-05-10","downloads":5},{"category":"null","date":"2026-05-11","downloads":8},{"category":"null","date":"2026-05-12","downloads":6},{"category":"null","date":"2026-05-13","downloads":4},{"category":"null","date":"2026-05-14","downloads":3},{"category":"null","date":"2026-05-15","downloads":121},{"category":"null","date":"2026-05-16","downloads":67},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":8},{"category":"null","date":"2026-05-19","downloads":3},{"category":"null","date":"2026-05-20","downloads":18},{"category":"null","date":"2026-05-21","downloads":191},{"category":"null","date":"2026-05-22","downloads":34},{"category":"null","date":"2026-05-23","downloads":42},{"category":"null","date":"2026-05-24","downloads":212},{"category":"null","date":"2026-05-25","downloads":38},{"category":"null","date":"2026-05-26","downloads":17},{"category":"null","date":"2026-05-27","downloads":36},{"category":"null","date":"2026-05-28","downloads":48},{"category":"null","date":"2026-05-29","downloads":20},{"category":"null","date":"2026-05-30","downloads":12},{"category":"null","date":"2026-05-31","downloads":23},{"category":"null","date":"2026-06-01","downloads":21},{"category":"null","date":"2026-06-02","downloads":27},{"category":"null","date":"2026-06-03","downloads":33},{"category":"null","date":"2026-06-04","downloads":16},{"category":"null","date":"2026-06-05","downloads":33},{"category":"null","date":"2026-06-06","downloads":162},{"category":"null","date":"2026-06-07","downloads":4},{"category":"null","date":"2026-06-08","downloads":14},{"category":"null","date":"2026-06-09","downloads":64},{"category":"null","date":"2026-06-10","downloads":19},{"category":"null","date":"2026-06-11","downloads":12},{"category":"null","date":"2026-06-12","downloads":12},{"category":"null","date":"2026-06-13","downloads":9},{"category":"null","date":"2026-06-14","downloads":16},{"category":"null","date":"2026-06-15","downloads":52},{"category":"null","date":"2026-06-16","downloads":19},{"category":"null","date":"2026-06-17","downloads":15},{"category":"null","date":"2026-06-18","downloads":29},{"category":"null","date":"2026-06-19","downloads":54},{"category":"null","date":"2026-06-20","downloads":56},{"category":"null","date":"2026-06-21","downloads":5},{"category":"null","date":"2026-06-22","downloads":29},{"category":"null","date":"2026-06-23","downloads":187},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":7},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":142},{"category":"null","date":"2026-06-28","downloads":4},{"category":"null","date":"2026-06-29","downloads":9},{"category":"null","date":"2026-06-30","downloads":19},{"category":"null","date":"2026-07-01","downloads":19},{"category":"null","date":"2026-07-02","downloads":19},{"category":"null","date":"2026-07-03","downloads":9},{"category":"null","date":"2026-07-04","downloads":13},{"category":"null","date":"2026-07-05","downloads":7},{"category":"null","date":"2026-07-06","downloads":8},{"category":"null","date":"2026-07-07","downloads":7},{"category":"null","date":"2026-07-08","downloads":28},{"category":"null","date":"2026-07-09","downloads":20},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":21},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":11},{"category":"null","date":"2026-07-14","downloads":12},{"category":"null","date":"2026-07-15","downloads":10},{"category":"null","date":"2026-07-16","downloads":11},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":9},{"category":"null","date":"2026-07-19","downloads":9},{"category":"null","date":"2026-07-20","downloads":15},{"category":"null","date":"2026-07-21","downloads":10},{"category":"null","date":"2026-07-22","downloads":104},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":44},{"category":"null","date":"2026-07-25","downloads":221},{"category":"null","date":"2026-07-26","downloads":20},{"category":"null","date":"2026-07-27","downloads":41},{"category":"null","date":"2026-07-28","downloads":102},{"category":"null","date":"2026-07-29","downloads":39},{"category":"null","date":"2026-07-30","downloads":73},{"category":"null","date":"2026-07-31","downloads":63},{"category":"null","date":"2026-08-01","downloads":9},{"category":"null","date":"2026-08-02","downloads":12},{"category":"null","date":"2026-08-03","downloads":24},{"category":"null","date":"2026-08-04","downloads":28},{"category":"null","date":"2026-08-05","downloads":30},{"category":"null","date":"2026-08-06","downloads":15},{"category":"null","date":"2026-08-07","downloads":17},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":104},{"category":"null","date":"2026-08-11","downloads":22},{"category":"null","date":"2026-08-12","downloads":10},{"category":"null","date":"2026-08-13","downloads":15},{"category":"null","date":"2026-08-14","downloads":27},{"category":"null","date":"2026-08-15","downloads":11},{"category":"null","date":"2026-08-16","downloads":8},{"category":"null","date":"2026-08-17","downloads":22},{"category":"null","date":"2026-08-18","downloads":98},{"category":"null","date":"2026-08-19","downloads":15},{"category":"null","date":"2026-08-20","downloads":15},{"category":"null","date":"2026-08-21","downloads":106},{"category":"null","date":"2026-08-22","downloads":22},{"category":"null","date":"2026-08-23","downloads":12},{"category":"null","date":"2026-08-24","downloads":37},{"category":"null","date":"2026-08-25","downloads":29},{"category":"null","date":"2026-08-26","downloads":295},{"category":"null","date":"2026-08-27","downloads":254},{"category":"null","date":"2026-08-28","downloads":268},{"category":"null","date":"2026-08-29","downloads":43},{"category":"null","date":"2026-08-30","downloads":4},{"category":"null","date":"2026-08-31","downloads":308},{"category":"null","date":"2026-09-01","downloads":121},{"category":"null","date":"2026-09-02","downloads":65},{"category":"null","date":"2026-09-03","downloads":28},{"category":"null","date":"2026-09-04","downloads":30},{"category":"null","date":"2026-09-05","downloads":27},{"category":"null","date":"2026-09-06","downloads":22},{"category":"null","date":"2026-09-07","downloads":51},{"category":"null","date":"2026-09-08","downloads":28},{"category":"null","date":"2026-09-09","downloads":18},{"category":"null","date":"2026-09-10","downloads":29},{"category":"null","date":"2026-09-11","downloads":18},{"category":"null","date":"2026-09-12","downloads":29},{"category":"null","date":"2026-09-13","downloads":71},{"category":"null","date":"2026-09-14","downloads":27},{"category":"null","date":"2026-09-15","downloads":43},{"category":"null","date":"2026-09-16","downloads":22},{"category":"null","date":"2026-09-17","downloads":34},{"category":"null","date":"2026-09-18","downloads":17},{"category":"null","date":"2026-09-19","downloads":261},{"category":"null","date":"2026-09-20","downloads":20},{"category":"null","date":"2026-09-21","downloads":35},{"category":"null","date":"2026-09-22","downloads":38},{"category":"null","date":"2026-09-23","downloads":25},{"category":"null","date":"2026-09-24","downloads":12},{"category":"null","date":"2026-09-25","downloads":28},{"category":"null","date":"2026-09-26","downloads":7},{"category":"null","date":"2026-09-27","downloads":36},{"category":"null","date":"2026-09-28","downloads":27},{"category":"null","date":"2026-09-29","downloads":16},{"category":"null","date":"2026-09-30","downloads":21},{"category":"null","date":"2026-10-01","downloads":8},{"category":"null","date":"2026-10-02","downloads":28},{"category":"null","date":"2026-10-03","downloads":65},{"category":"null","date":"2026-10-04","downloads":53},{"category":"null","date":"2026-10-05","downloads":33},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-07","downloads":16},{"category":"null","date":"2026-10-08","downloads":16},{"category":"null","date":"2026-10-09","downloads":211},{"category":"null","date":"2026-10-10","downloads":32},{"category":"null","date":"2026-10-11","downloads":24},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-13","downloads":26},{"category":"null","date":"2026-10-14","downloads":33},{"category":"null","date":"2026-10-15","downloads":47},{"category":"null","date":"2026-10-16","downloads":34},{"category":"null","date":"2026-10-17","downloads":9},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":23},{"category":"null","date":"2026-10-20","downloads":48},{"category":"null","date":"2026-10-21","downloads":43},{"category":"null","date":"2026-10-22","downloads":17},{"category":"null","date":"2026-10-23","downloads":86},{"category":"null","date":"2026-10-24","downloads":116},{"category":"null","date":"2026-10-25","downloads":15},{"category":"null","date":"2026-10-26","downloads":26},{"category":"null","date":"2026-10-27","downloads":24},{"category":"null","date":"2026-10-28","downloads":21}],"package":"mooncake-transfer-engine","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:11:31.414079+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine/system","payload":{"data":[{"category":"Darwin","date":"2026-04-08","downloads":9},{"category":"Darwin","date":"2026-04-09","downloads":8},{"category":"Darwin","date":"2026-04-10","downloads":8},{"category":"Darwin","date":"2026-04-11","downloads":8},{"category":"Darwin","date":"2026-04-13","downloads":9},{"category":"Darwin","date":"2026-04-14","downloads":9},{"category":"Darwin","date":"2026-04-15","downloads":10},{"category":"Darwin","date":"2026-04-16","downloads":6},{"category":"Darwin","date":"2026-04-17","downloads":7},{"category":"Darwin","date":"2026-04-18","downloads":1},{"category":"Darwin","date":"2026-04-19","downloads":7},{"category":"Darwin","date":"2026-04-20","downloads":13},{"category":"Darwin","date":"2026-04-21","downloads":8},{"category":"Darwin","date":"2026-04-22","downloads":11},{"category":"Darwin","date":"2026-04-23","downloads":4},{"category":"Darwin","date":"2026-04-24","downloads":1},{"category":"Darwin","date":"2026-04-26","downloads":1},{"category":"Darwin","date":"2026-04-27","downloads":7},{"category":"Darwin","date":"2026-04-28","downloads":11},{"category":"Darwin","date":"2026-04-29","downloads":3},{"category":"Darwin","date":"2026-04-30","downloads":6},{"category":"Darwin","date":"2026-05-01","downloads":5},{"category":"Darwin","date":"2026-05-02","downloads":2},{"category":"Darwin","date":"2026-05-04","downloads":31},{"category":"Darwin","date":"2026-05-05","downloads":6},{"category":"Darwin","date":"2026-05-06","downloads":6},{"category":"Darwin","date":"2026-05-07","downloads":6},{"category":"Darwin","date":"2026-05-08","downloads":9},{"category":"Darwin","date":"2026-05-09","downloads":2},{"category":"Darwin","date":"2026-05-10","downloads":1},{"category":"Darwin","date":"2026-05-11","downloads":6},{"category":"Darwin","date":"2026-05-12","downloads":22},{"category":"Darwin","date":"2026-05-13","downloads":5},{"category":"Darwin","date":"2026-05-14","downloads":2},{"category":"Darwin","date":"2026-05-15","downloads":1},{"category":"Darwin","date":"2026-05-16","downloads":176},{"category":"Darwin","date":"2026-05-17","downloads":2},{"category":"Darwin","date":"2026-05-18","downloads":7},{"category":"Darwin","date":"2026-05-19","downloads":6},{"category":"Darwin","date":"2026-05-20","downloads":6},{"category":"Darwin","date":"2026-05-21","downloads":12},{"category":"Darwin","date":"2026-05-22","downloads":15},{"category":"Darwin","date":"2026-05-23","downloads":3},{"category":"Darwin","date":"2026-05-24","downloads":24},{"category":"Darwin","date":"2026-05-25","downloads":5},{"category":"Darwin","date":"2026-05-26","downloads":2},{"category":"Darwin","date":"2026-05-27","downloads":9},{"category":"Darwin","date":"2026-05-28","downloads":5},{"category":"Darwin","date":"2026-05-29","downloads":6},{"category":"Darwin","date":"2026-05-30","downloads":2},{"category":"Darwin","date":"2026-05-31","downloads":2},{"category":"Darwin","date":"2026-06-01","downloads":5},{"category":"Darwin","date":"2026-06-02","downloads":10},{"category":"Darwin","date":"2026-06-03","downloads":6},{"category":"Darwin","date":"2026-06-04","downloads":9},{"category":"Darwin","date":"2026-06-05","downloads":7},{"category":"Darwin","date":"2026-06-06","downloads":2},{"category":"Darwin","date":"2026-06-07","downloads":5},{"category":"Darwin","date":"2026-06-08","downloads":11},{"category":"Darwin","date":"2026-06-09","downloads":10},{"category":"Darwin","date":"2026-06-10","downloads":12},{"category":"Darwin","date":"2026-06-11","downloads":13},{"category":"Darwin","date":"2026-06-12","downloads":15},{"category":"Darwin","date":"2026-06-13","downloads":1},{"category":"Darwin","date":"2026-06-14","downloads":1},{"category":"Darwin","date":"2026-06-16","downloads":5},{"category":"Darwin","date":"2026-06-17","downloads":10},{"category":"Darwin","date":"2026-06-18","downloads":9},{"category":"Darwin","date":"2026-06-19","downloads":3},{"category":"Darwin","date":"2026-06-20","downloads":1},{"category":"Darwin","date":"2026-06-21","downloads":3},{"category":"Darwin","date":"2026-06-22","downloads":5},{"category":"Darwin","date":"2026-06-23","downloads":8},{"category":"Darwin","date":"2026-06-24","downloads":7},{"category":"Darwin","date":"2026-06-25","downloads":8},{"category":"Darwin","date":"2026-06-26","downloads":5},{"category":"Darwin","date":"2026-06-27","downloads":5},{"category":"Darwin","date":"2026-06-28","downloads":2},{"category":"Darwin","date":"2026-06-29","downloads":8},{"category":"Darwin","date":"2026-06-30","downloads":4},{"category":"Darwin","date":"2026-07-01","downloads":3},{"category":"Darwin","date":"2026-07-02","downloads":4},{"category":"Darwin","date":"2026-07-03","downloads":4},{"category":"Darwin","date":"2026-07-04","downloads":4},{"category":"Darwin","date":"2026-07-06","downloads":11},{"category":"Darwin","date":"2026-07-07","downloads":5},{"category":"Darwin","date":"2026-07-08","downloads":12},{"category":"Darwin","date":"2026-07-09","downloads":14},{"category":"Darwin","date":"2026-07-10","downloads":8},{"category":"Darwin","date":"2026-07-12","downloads":2},{"category":"Darwin","date":"2026-07-13","downloads":8},{"category":"Darwin","date":"2026-07-14","downloads":4},{"category":"Darwin","date":"2026-07-15","downloads":13},{"category":"Darwin","date":"2026-07-16","downloads":16},{"category":"Darwin","date":"2026-07-17","downloads":9},{"category":"Darwin","date":"2026-07-18","downloads":4},{"category":"Darwin","date":"2026-07-19","downloads":4},{"category":"Darwin","date":"2026-07-20","downloads":9},{"category":"Darwin","date":"2026-07-21","downloads":13},{"category":"Darwin","date":"2026-07-22","downloads":13},{"category":"Darwin","date":"2026-07-23","downloads":33},{"category":"Darwin","date":"2026-07-24","downloads":14},{"category":"Darwin","date":"2026-07-25","downloads":21},{"category":"Darwin","date":"2026-07-26","downloads":4},{"category":"Darwin","date":"2026-07-27","downloads":11},{"category":"Darwin","date":"2026-07-28","downloads":10},{"category":"Darwin","date":"2026-07-29","downloads":19},{"category":"Darwin","date":"2026-07-30","downloads":6},{"category":"Darwin","date":"2026-07-31","downloads":13},{"category":"Darwin","date":"2026-08-01","downloads":1},{"category":"Darwin","date":"2026-08-02","downloads":8},{"category":"Darwin","date":"2026-08-03","downloads":10},{"category":"Darwin","date":"2026-08-04","downloads":4},{"category":"Darwin","date":"2026-08-05","downloads":13},{"category":"Darwin","date":"2026-08-06","downloads":7},{"category":"Darwin","date":"2026-08-07","downloads":11},{"category":"Darwin","date":"2026-08-08","downloads":12},{"category":"Darwin","date":"2026-08-09","downloads":5},{"category":"Darwin","date":"2026-08-10","downloads":10},{"category":"Darwin","date":"2026-08-11","downloads":6},{"category":"Darwin","date":"2026-08-12","downloads":20},{"category":"Darwin","date":"2026-08-13","downloads":5},{"category":"Darwin","date":"2026-08-14","downloads":8},{"category":"Darwin","date":"2026-08-15","downloads":4},{"category":"Darwin","date":"2026-08-16","downloads":2},{"category":"Darwin","date":"2026-08-17","downloads":8},{"category":"Darwin","date":"2026-08-18","downloads":6},{"category":"Darwin","date":"2026-08-19","downloads":14},{"category":"Darwin","date":"2026-08-20","downloads":5},{"category":"Darwin","date":"2026-08-21","downloads":9},{"category":"Darwin","date":"2026-08-22","downloads":5},{"category":"Darwin","date":"2026-08-23","downloads":2},{"category":"Darwin","date":"2026-08-24","downloads":12},{"category":"Darwin","date":"2026-08-26","downloads":27},{"category":"Darwin","date":"2026-08-27","downloads":3},{"category":"Darwin","date":"2026-08-28","downloads":1},{"category":"Darwin","date":"2026-08-30","downloads":1},{"category":"Darwin","date":"2026-08-31","downloads":40},{"category":"Darwin","date":"2026-09-01","downloads":1},{"category":"Darwin","date":"2026-09-02","downloads":4},{"category":"Darwin","date":"2026-09-04","downloads":1},{"category":"Darwin","date":"2026-09-07","downloads":1},{"category":"Darwin","date":"2026-09-09","downloads":2},{"category":"Darwin","date":"2026-09-12","downloads":2},{"category":"Darwin","date":"2026-09-14","downloads":3},{"category":"Darwin","date":"2026-09-15","downloads":1},{"category":"Darwin","date":"2026-09-16","downloads":2},{"category":"Darwin","date":"2026-09-18","downloads":1},{"category":"Darwin","date":"2026-09-19","downloads":1},{"category":"Darwin","date":"2026-09-22","downloads":4},{"category":"Darwin","date":"2026-09-24","downloads":5},{"category":"Darwin","date":"2026-09-25","downloads":1},{"category":"Darwin","date":"2026-09-26","downloads":7},{"category":"Darwin","date":"2026-09-27","downloads":1},{"category":"Darwin","date":"2026-09-28","downloads":3},{"category":"Darwin","date":"2026-09-29","downloads":2},{"category":"Darwin","date":"2026-09-30","downloads":1},{"category":"Darwin","date":"2026-10-01","downloads":1},{"category":"Darwin","date":"2026-10-02","downloads":1},{"category":"Darwin","date":"2026-10-03","downloads":1},{"category":"Darwin","date":"2026-10-05","downloads":5},{"category":"Darwin","date":"2026-10-06","downloads":3},{"category":"Darwin","date":"2026-10-08","downloads":1},{"category":"Darwin","date":"2026-10-09","downloads":3},{"category":"Darwin","date":"2026-10-11","downloads":1},{"category":"Darwin","date":"2026-10-13","downloads":7},{"category":"Darwin","date":"2026-10-14","downloads":2},{"category":"Darwin","date":"2026-10-15","downloads":2},{"category":"Darwin","date":"2026-10-16","downloads":1},{"category":"Darwin","date":"2026-10-19","downloads":7},{"category":"Darwin","date":"2026-10-22","downloads":3},{"category":"Darwin","date":"2026-10-23","downloads":1},{"category":"Darwin","date":"2026-10-26","downloads":2},{"category":"Darwin","date":"2026-10-27","downloads":6},{"category":"Linux","date":"2026-04-08","downloads":12283},{"category":"Linux","date":"2026-04-09","downloads":13177},{"category":"Linux","date":"2026-04-10","downloads":12299},{"category":"Linux","date":"2026-04-11","downloads":6226},{"category":"Linux","date":"2026-04-12","downloads":4931},{"category":"Linux","date":"2026-04-13","downloads":10424},{"category":"Linux","date":"2026-04-14","downloads":12706},{"category":"Linux","date":"2026-04-15","downloads":15527},{"category":"Linux","date":"2026-04-16","downloads":12823},{"category":"Linux","date":"2026-04-17","downloads":11923},{"category":"Linux","date":"2026-04-18","downloads":6668},{"category":"Linux","date":"2026-04-19","downloads":4177},{"category":"Linux","date":"2026-04-20","downloads":10419},{"category":"Linux","date":"2026-04-21","downloads":14547},{"category":"Linux","date":"2026-04-22","downloads":13813},{"category":"Linux","date":"2026-04-23","downloads":13234},{"category":"Linux","date":"2026-04-24","downloads":16761},{"category":"Linux","date":"2026-04-25","downloads":7290},{"category":"Linux","date":"2026-04-26","downloads":6886},{"category":"Linux","date":"2026-04-27","downloads":11369},{"category":"Linux","date":"2026-04-28","downloads":14210},{"category":"Linux","date":"2026-04-29","downloads":15041},{"category":"Linux","date":"2026-04-30","downloads":13279},{"category":"Linux","date":"2026-05-01","downloads":14413},{"category":"Linux","date":"2026-05-02","downloads":5093},{"category":"Linux","date":"2026-05-03","downloads":6276},{"category":"Linux","date":"2026-05-04","downloads":9925},{"category":"Linux","date":"2026-05-05","downloads":14771},{"category":"Linux","date":"2026-05-06","downloads":15655},{"category":"Linux","date":"2026-05-07","downloads":15707},{"category":"Linux","date":"2026-05-08","downloads":12470},{"category":"Linux","date":"2026-05-09","downloads":5750},{"category":"Linux","date":"2026-05-10","downloads":3253},{"category":"Linux","date":"2026-05-11","downloads":10476},{"category":"Linux","date":"2026-05-12","downloads":14066},{"category":"Linux","date":"2026-05-13","downloads":13817},{"category":"Linux","date":"2026-05-14","downloads":11868},{"category":"Linux","date":"2026-05-15","downloads":12156},{"category":"Linux","date":"2026-05-16","downloads":4901},{"category":"Linux","date":"2026-05-17","downloads":5954},{"category":"Linux","date":"2026-05-18","downloads":11160},{"category":"Linux","date":"2026-05-19","downloads":10212},{"category":"Linux","date":"2026-05-20","downloads":12503},{"category":"Linux","date":"2026-05-21","downloads":15485},{"category":"Linux","date":"2026-05-22","downloads":12634},{"category":"Linux","date":"2026-05-23","downloads":6081},{"category":"Linux","date":"2026-05-24","downloads":4901},{"category":"Linux","date":"2026-05-25","downloads":5435},{"category":"Linux","date":"2026-05-26","downloads":11197},{"category":"Linux","date":"2026-05-27","downloads":13962},{"category":"Linux","date":"2026-05-28","downloads":13262},{"category":"Linux","date":"2026-05-29","downloads":15919},{"category":"Linux","date":"2026-05-30","downloads":5920},{"category":"Linux","date":"2026-05-31","downloads":6279},{"category":"Linux","date":"2026-06-01","downloads":9636},{"category":"Linux","date":"2026-06-02","downloads":8968},{"category":"Linux","date":"2026-06-03","downloads":8010},{"category":"Linux","date":"2026-06-04","downloads":7487},{"category":"Linux","date":"2026-06-05","downloads":6825},{"category":"Linux","date":"2026-06-06","downloads":3737},{"category":"Linux","date":"2026-06-07","downloads":2609},{"category":"Linux","date":"2026-06-08","downloads":6570},{"category":"Linux","date":"2026-06-09","downloads":8880},{"category":"Linux","date":"2026-06-10","downloads":9824},{"category":"Linux","date":"2026-06-11","downloads":9577},{"category":"Linux","date":"2026-06-12","downloads":7783},{"category":"Linux","date":"2026-06-13","downloads":3655},{"category":"Linux","date":"2026-06-14","downloads":2886},{"category":"Linux","date":"2026-06-15","downloads":4969},{"category":"Linux","date":"2026-06-16","downloads":6111},{"category":"Linux","date":"2026-06-17","downloads":5386},{"category":"Linux","date":"2026-06-18","downloads":5116},{"category":"Linux","date":"2026-06-19","downloads":4404},{"category":"Linux","date":"2026-06-20","downloads":2645},{"category":"Linux","date":"2026-06-21","downloads":2134},{"category":"Linux","date":"2026-06-22","downloads":3915},{"category":"Linux","date":"2026-06-23","downloads":4752},{"category":"Linux","date":"2026-06-24","downloads":5759},{"category":"Linux","date":"2026-06-25","downloads":6279},{"category":"Linux","date":"2026-06-26","downloads":6934},{"category":"Linux","date":"2026-06-27","downloads":3310},{"category":"Linux","date":"2026-06-28","downloads":2763},{"category":"Linux","date":"2026-06-29","downloads":6885},{"category":"Linux","date":"2026-06-30","downloads":7810},{"category":"Linux","date":"2026-07-01","downloads":7713},{"category":"Linux","date":"2026-07-02","downloads":7453},{"category":"Linux","date":"2026-07-03","downloads":5655},{"category":"Linux","date":"2026-07-04","downloads":2725},{"category":"Linux","date":"2026-07-05","downloads":1983},{"category":"Linux","date":"2026-07-06","downloads":7123},{"category":"Linux","date":"2026-07-07","downloads":7985},{"category":"Linux","date":"2026-07-08","downloads":8526},{"category":"Linux","date":"2026-07-09","downloads":8124},{"category":"Linux","date":"2026-07-10","downloads":8283},{"category":"Linux","date":"2026-07-11","downloads":3492},{"category":"Linux","date":"2026-07-12","downloads":2249},{"category":"Linux","date":"2026-07-13","downloads":8251},{"category":"Linux","date":"2026-07-14","downloads":10549},{"category":"Linux","date":"2026-07-15","downloads":11560},{"category":"Linux","date":"2026-07-16","downloads":9920},{"category":"Linux","date":"2026-07-17","downloads":9899},{"category":"Linux","date":"2026-07-18","downloads":5303},{"category":"Linux","date":"2026-07-19","downloads":2444},{"category":"Linux","date":"2026-07-20","downloads":9639},{"category":"Linux","date":"2026-07-21","downloads":12830},{"category":"Linux","date":"2026-07-22","downloads":12219},{"category":"Linux","date":"2026-07-23","downloads":11464},{"category":"Linux","date":"2026-07-24","downloads":12597},{"category":"Linux","date":"2026-07-25","downloads":5144},{"category":"Linux","date":"2026-07-26","downloads":4188},{"category":"Linux","date":"2026-07-27","downloads":11668},{"category":"Linux","date":"2026-07-28","downloads":14906},{"category":"Linux","date":"2026-07-29","downloads":14212},{"category":"Linux","date":"2026-07-30","downloads":15430},{"category":"Linux","date":"2026-07-31","downloads":14289},{"category":"Linux","date":"2026-08-01","downloads":10025},{"category":"Linux","date":"2026-08-02","downloads":6169},{"category":"Linux","date":"2026-08-03","downloads":12847},{"category":"Linux","date":"2026-08-04","downloads":15609},{"category":"Linux","date":"2026-08-05","downloads":15905},{"category":"Linux","date":"2026-08-06","downloads":10018},{"category":"Linux","date":"2026-08-07","downloads":17246},{"category":"Linux","date":"2026-08-08","downloads":7284},{"category":"Linux","date":"2026-08-09","downloads":2936},{"category":"Linux","date":"2026-08-10","downloads":4007},{"category":"Linux","date":"2026-08-11","downloads":4298},{"category":"Linux","date":"2026-08-12","downloads":5387},{"category":"Linux","date":"2026-08-13","downloads":3607},{"category":"Linux","date":"2026-08-14","downloads":4165},{"category":"Linux","date":"2026-08-15","downloads":1700},{"category":"Linux","date":"2026-08-16","downloads":1863},{"category":"Linux","date":"2026-08-17","downloads":2711},{"category":"Linux","date":"2026-08-18","downloads":3571},{"category":"Linux","date":"2026-08-19","downloads":3992},{"category":"Linux","date":"2026-08-20","downloads":3706},{"category":"Linux","date":"2026-08-21","downloads":3682},{"category":"Linux","date":"2026-08-22","downloads":2872},{"category":"Linux","date":"2026-08-23","downloads":1954},{"category":"Linux","date":"2026-08-24","downloads":2280},{"category":"Linux","date":"2026-08-25","downloads":2307},{"category":"Linux","date":"2026-08-26","downloads":1897},{"category":"Linux","date":"2026-08-27","downloads":2019},{"category":"Linux","date":"2026-08-28","downloads":1816},{"category":"Linux","date":"2026-08-29","downloads":1076},{"category":"Linux","date":"2026-08-30","downloads":1145},{"category":"Linux","date":"2026-08-31","downloads":1718},{"category":"Linux","date":"2026-09-01","downloads":1979},{"category":"Linux","date":"2026-09-02","downloads":2573},{"category":"Linux","date":"2026-09-03","downloads":2315},{"category":"Linux","date":"2026-09-04","downloads":2193},{"category":"Linux","date":"2026-09-05","downloads":1384},{"category":"Linux","date":"2026-09-06","downloads":1304},{"category":"Linux","date":"2026-09-07","downloads":1732},{"category":"Linux","date":"2026-09-08","downloads":2162},{"category":"Linux","date":"2026-09-09","downloads":2328},{"category":"Linux","date":"2026-09-10","downloads":2521},{"category":"Linux","date":"2026-09-11","downloads":2542},{"category":"Linux","date":"2026-09-12","downloads":1959},{"category":"Linux","date":"2026-09-13","downloads":1681},{"category":"Linux","date":"2026-09-14","downloads":2045},{"category":"Linux","date":"2026-09-15","downloads":2839},{"category":"Linux","date":"2026-09-16","downloads":2933},{"category":"Linux","date":"2026-09-17","downloads":3027},{"category":"Linux","date":"2026-09-18","downloads":2914},{"category":"Linux","date":"2026-09-19","downloads":1675},{"category":"Linux","date":"2026-09-20","downloads":1798},{"category":"Linux","date":"2026-09-21","downloads":2552},{"category":"Linux","date":"2026-09-22","downloads":3128},{"category":"Linux","date":"2026-09-23","downloads":2685},{"category":"Linux","date":"2026-09-24","downloads":2918},{"category":"Linux","date":"2026-09-25","downloads":2325},{"category":"Linux","date":"2026-09-26","downloads":1463},{"category":"Linux","date":"2026-09-27","downloads":1314},{"category":"Linux","date":"2026-09-28","downloads":1818},{"category":"Linux","date":"2026-09-29","downloads":2124},{"category":"Linux","date":"2026-09-30","downloads":2217},{"category":"Linux","date":"2026-10-01","downloads":2483},{"category":"Linux","date":"2026-10-02","downloads":2326},{"category":"Linux","date":"2026-10-03","downloads":1582},{"category":"Linux","date":"2026-10-04","downloads":1402},{"category":"Linux","date":"2026-10-05","downloads":2172},{"category":"Linux","date":"2026-10-06","downloads":3017},{"category":"Linux","date":"2026-10-07","downloads":3303},{"category":"Linux","date":"2026-10-08","downloads":3817},{"category":"Linux","date":"2026-10-09","downloads":2953},{"category":"Linux","date":"2026-10-10","downloads":2086},{"category":"Linux","date":"2026-10-11","downloads":1676},{"category":"Linux","date":"2026-10-12","downloads":2701},{"category":"Linux","date":"2026-10-13","downloads":3352},{"category":"Linux","date":"2026-10-14","downloads":3707},{"category":"Linux","date":"2026-10-15","downloads":3439},{"category":"Linux","date":"2026-10-16","downloads":2701},{"category":"Linux","date":"2026-10-17","downloads":2398},{"category":"Linux","date":"2026-10-18","downloads":1915},{"category":"Linux","date":"2026-10-19","downloads":3279},{"category":"Linux","date":"2026-10-20","downloads":2955},{"category":"Linux","date":"2026-10-21","downloads":2594},{"category":"Linux","date":"2026-10-22","downloads":2515},{"category":"Linux","date":"2026-10-23","downloads":3074},{"category":"Linux","date":"2026-10-24","downloads":1836},{"category":"Linux","date":"2026-10-25","downloads":2434},{"category":"Linux","date":"2026-10-26","downloads":1908},{"category":"Linux","date":"2026-10-27","downloads":1437},{"category":"Linux","date":"2026-10-28","downloads":1360},{"category":"Windows","date":"2026-04-09","downloads":2},{"category":"Windows","date":"2026-04-10","downloads":9},{"category":"Windows","date":"2026-04-13","downloads":1},{"category":"Windows","date":"2026-04-15","downloads":1},{"category":"Windows","date":"2026-04-16","downloads":2},{"category":"Windows","date":"2026-04-17","downloads":4},{"category":"Windows","date":"2026-04-18","downloads":8},{"category":"Windows","date":"2026-04-19","downloads":8},{"category":"Windows","date":"2026-04-20","downloads":1},{"category":"Windows","date":"2026-04-21","downloads":4},{"category":"Windows","date":"2026-04-28","downloads":1},{"category":"Windows","date":"2026-04-29","downloads":1},{"category":"Windows","date":"2026-05-04","downloads":4},{"category":"Windows","date":"2026-05-05","downloads":3},{"category":"Windows","date":"2026-05-09","downloads":4},{"category":"Windows","date":"2026-05-11","downloads":4},{"category":"Windows","date":"2026-05-14","downloads":1},{"category":"Windows","date":"2026-05-19","downloads":4},{"category":"Windows","date":"2026-05-20","downloads":1},{"category":"Windows","date":"2026-05-22","downloads":2},{"category":"Windows","date":"2026-05-26","downloads":4},{"category":"Windows","date":"2026-05-27","downloads":4},{"category":"Windows","date":"2026-05-30","downloads":1},{"category":"Windows","date":"2026-06-01","downloads":1},{"category":"Windows","date":"2026-06-04","downloads":5},{"category":"Windows","date":"2026-06-05","downloads":5},{"category":"Windows","date":"2026-06-07","downloads":8},{"category":"Windows","date":"2026-06-08","downloads":13},{"category":"Windows","date":"2026-06-09","downloads":2},{"category":"Windows","date":"2026-06-10","downloads":4},{"category":"Windows","date":"2026-06-11","downloads":8},{"category":"Windows","date":"2026-06-14","downloads":1},{"category":"Windows","date":"2026-06-17","downloads":4},{"category":"Windows","date":"2026-06-18","downloads":1},{"category":"Windows","date":"2026-06-22","downloads":4},{"category":"Windows","date":"2026-06-23","downloads":3},{"category":"Windows","date":"2026-06-26","downloads":5},{"category":"Windows","date":"2026-07-01","downloads":4},{"category":"Windows","date":"2026-07-02","downloads":1},{"category":"Windows","date":"2026-07-10","downloads":4},{"category":"Windows","date":"2026-07-13","downloads":4},{"category":"Windows","date":"2026-07-14","downloads":4},{"category":"Windows","date":"2026-07-15","downloads":4},{"category":"Windows","date":"2026-07-17","downloads":4},{"category":"Windows","date":"2026-07-28","downloads":2},{"category":"Windows","date":"2026-07-30","downloads":2},{"category":"Windows","date":"2026-08-03","downloads":3},{"category":"Windows","date":"2026-08-06","downloads":2},{"category":"Windows","date":"2026-08-09","downloads":4},{"category":"Windows","date":"2026-08-10","downloads":4},{"category":"Windows","date":"2026-08-11","downloads":1},{"category":"Windows","date":"2026-08-12","downloads":5},{"category":"Windows","date":"2026-08-13","downloads":4},{"category":"Windows","date":"2026-08-19","downloads":1},{"category":"Windows","date":"2026-08-20","downloads":2},{"category":"Windows","date":"2026-08-21","downloads":4},{"category":"Windows","date":"2026-08-23","downloads":8},{"category":"Windows","date":"2026-08-24","downloads":2},{"category":"Windows","date":"2026-08-27","downloads":2},{"category":"Windows","date":"2026-08-29","downloads":4},{"category":"Windows","date":"2026-09-01","downloads":4},{"category":"Windows","date":"2026-09-03","downloads":4},{"category":"Windows","date":"2026-09-11","downloads":1},{"category":"Windows","date":"2026-09-20","downloads":1},{"category":"Windows","date":"2026-09-22","downloads":4},{"category":"Windows","date":"2026-09-27","downloads":1},{"category":"Windows","date":"2026-09-30","downloads":1},{"category":"Windows","date":"2026-10-02","downloads":4},{"category":"Windows","date":"2026-10-09","downloads":4},{"category":"Windows","date":"2026-10-14","downloads":4},{"category":"Windows","date":"2026-10-16","downloads":7},{"category":"Windows","date":"2026-10-18","downloads":1},{"category":"Windows","date":"2026-10-19","downloads":1},{"category":"Windows","date":"2026-10-22","downloads":4},{"category":"Windows","date":"2026-10-28","downloads":7},{"category":"null","date":"2026-04-08","downloads":61},{"category":"null","date":"2026-04-09","downloads":76},{"category":"null","date":"2026-04-10","downloads":11},{"category":"null","date":"2026-04-11","downloads":14},{"category":"null","date":"2026-04-12","downloads":23},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":65},{"category":"null","date":"2026-04-15","downloads":243},{"category":"null","date":"2026-04-16","downloads":8},{"category":"null","date":"2026-04-17","downloads":3},{"category":"null","date":"2026-04-18","downloads":6},{"category":"null","date":"2026-04-19","downloads":15},{"category":"null","date":"2026-04-20","downloads":17},{"category":"null","date":"2026-04-21","downloads":3},{"category":"null","date":"2026-04-22","downloads":299},{"category":"null","date":"2026-04-23","downloads":30},{"category":"null","date":"2026-04-24","downloads":22},{"category":"null","date":"2026-04-25","downloads":46},{"category":"null","date":"2026-04-26","downloads":17},{"category":"null","date":"2026-04-27","downloads":18},{"category":"null","date":"2026-04-28","downloads":149},{"category":"null","date":"2026-04-29","downloads":15},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-01","downloads":11},{"category":"null","date":"2026-05-03","downloads":1},{"category":"null","date":"2026-05-04","downloads":7},{"category":"null","date":"2026-05-05","downloads":97},{"category":"null","date":"2026-05-06","downloads":7},{"category":"null","date":"2026-05-07","downloads":10},{"category":"null","date":"2026-05-08","downloads":156},{"category":"null","date":"2026-05-09","downloads":179},{"category":"null","date":"2026-05-10","downloads":5},{"category":"null","date":"2026-05-11","downloads":8},{"category":"null","date":"2026-05-12","downloads":6},{"category":"null","date":"2026-05-13","downloads":4},{"category":"null","date":"2026-05-14","downloads":3},{"category":"null","date":"2026-05-15","downloads":121},{"category":"null","date":"2026-05-16","downloads":67},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":8},{"category":"null","date":"2026-05-19","downloads":3},{"category":"null","date":"2026-05-20","downloads":18},{"category":"null","date":"2026-05-21","downloads":191},{"category":"null","date":"2026-05-22","downloads":34},{"category":"null","date":"2026-05-23","downloads":42},{"category":"null","date":"2026-05-24","downloads":212},{"category":"null","date":"2026-05-25","downloads":38},{"category":"null","date":"2026-05-26","downloads":17},{"category":"null","date":"2026-05-27","downloads":36},{"category":"null","date":"2026-05-28","downloads":48},{"category":"null","date":"2026-05-29","downloads":20},{"category":"null","date":"2026-05-30","downloads":12},{"category":"null","date":"2026-05-31","downloads":23},{"category":"null","date":"2026-06-01","downloads":21},{"category":"null","date":"2026-06-02","downloads":27},{"category":"null","date":"2026-06-03","downloads":33},{"category":"null","date":"2026-06-04","downloads":16},{"category":"null","date":"2026-06-05","downloads":33},{"category":"null","date":"2026-06-06","downloads":162},{"category":"null","date":"2026-06-07","downloads":4},{"category":"null","date":"2026-06-08","downloads":14},{"category":"null","date":"2026-06-09","downloads":64},{"category":"null","date":"2026-06-10","downloads":19},{"category":"null","date":"2026-06-11","downloads":12},{"category":"null","date":"2026-06-12","downloads":12},{"category":"null","date":"2026-06-13","downloads":9},{"category":"null","date":"2026-06-14","downloads":16},{"category":"null","date":"2026-06-15","downloads":52},{"category":"null","date":"2026-06-16","downloads":19},{"category":"null","date":"2026-06-17","downloads":15},{"category":"null","date":"2026-06-18","downloads":29},{"category":"null","date":"2026-06-19","downloads":54},{"category":"null","date":"2026-06-20","downloads":56},{"category":"null","date":"2026-06-21","downloads":5},{"category":"null","date":"2026-06-22","downloads":29},{"category":"null","date":"2026-06-23","downloads":187},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":7},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":142},{"category":"null","date":"2026-06-28","downloads":4},{"category":"null","date":"2026-06-29","downloads":9},{"category":"null","date":"2026-06-30","downloads":19},{"category":"null","date":"2026-07-01","downloads":19},{"category":"null","date":"2026-07-02","downloads":19},{"category":"null","date":"2026-07-03","downloads":9},{"category":"null","date":"2026-07-04","downloads":13},{"category":"null","date":"2026-07-05","downloads":7},{"category":"null","date":"2026-07-06","downloads":8},{"category":"null","date":"2026-07-07","downloads":7},{"category":"null","date":"2026-07-08","downloads":28},{"category":"null","date":"2026-07-09","downloads":20},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":21},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":11},{"category":"null","date":"2026-07-14","downloads":12},{"category":"null","date":"2026-07-15","downloads":10},{"category":"null","date":"2026-07-16","downloads":11},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":9},{"category":"null","date":"2026-07-19","downloads":9},{"category":"null","date":"2026-07-20","downloads":15},{"category":"null","date":"2026-07-21","downloads":10},{"category":"null","date":"2026-07-22","downloads":104},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":44},{"category":"null","date":"2026-07-25","downloads":221},{"category":"null","date":"2026-07-26","downloads":20},{"category":"null","date":"2026-07-27","downloads":41},{"category":"null","date":"2026-07-28","downloads":102},{"category":"null","date":"2026-07-29","downloads":39},{"category":"null","date":"2026-07-30","downloads":73},{"category":"null","date":"2026-07-31","downloads":63},{"category":"null","date":"2026-08-01","downloads":9},{"category":"null","date":"2026-08-02","downloads":12},{"category":"null","date":"2026-08-03","downloads":24},{"category":"null","date":"2026-08-04","downloads":28},{"category":"null","date":"2026-08-05","downloads":30},{"category":"null","date":"2026-08-06","downloads":15},{"category":"null","date":"2026-08-07","downloads":17},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":104},{"category":"null","date":"2026-08-11","downloads":22},{"category":"null","date":"2026-08-12","downloads":10},{"category":"null","date":"2026-08-13","downloads":15},{"category":"null","date":"2026-08-14","downloads":27},{"category":"null","date":"2026-08-15","downloads":11},{"category":"null","date":"2026-08-16","downloads":8},{"category":"null","date":"2026-08-17","downloads":22},{"category":"null","date":"2026-08-18","downloads":98},{"category":"null","date":"2026-08-19","downloads":15},{"category":"null","date":"2026-08-20","downloads":15},{"category":"null","date":"2026-08-21","downloads":106},{"category":"null","date":"2026-08-22","downloads":22},{"category":"null","date":"2026-08-23","downloads":12},{"category":"null","date":"2026-08-24","downloads":37},{"category":"null","date":"2026-08-25","downloads":29},{"category":"null","date":"2026-08-26","downloads":295},{"category":"null","date":"2026-08-27","downloads":254},{"category":"null","date":"2026-08-28","downloads":268},{"category":"null","date":"2026-08-29","downloads":43},{"category":"null","date":"2026-08-30","downloads":4},{"category":"null","date":"2026-08-31","downloads":308},{"category":"null","date":"2026-09-01","downloads":121},{"category":"null","date":"2026-09-02","downloads":65},{"category":"null","date":"2026-09-03","downloads":28},{"category":"null","date":"2026-09-04","downloads":30},{"category":"null","date":"2026-09-05","downloads":27},{"category":"null","date":"2026-09-06","downloads":22},{"category":"null","date":"2026-09-07","downloads":51},{"category":"null","date":"2026-09-08","downloads":28},{"category":"null","date":"2026-09-09","downloads":18},{"category":"null","date":"2026-09-10","downloads":29},{"category":"null","date":"2026-09-11","downloads":18},{"category":"null","date":"2026-09-12","downloads":29},{"category":"null","date":"2026-09-13","downloads":71},{"category":"null","date":"2026-09-14","downloads":27},{"category":"null","date":"2026-09-15","downloads":43},{"category":"null","date":"2026-09-16","downloads":22},{"category":"null","date":"2026-09-17","downloads":34},{"category":"null","date":"2026-09-18","downloads":17},{"category":"null","date":"2026-09-19","downloads":261},{"category":"null","date":"2026-09-20","downloads":20},{"category":"null","date":"2026-09-21","downloads":35},{"category":"null","date":"2026-09-22","downloads":38},{"category":"null","date":"2026-09-23","downloads":25},{"category":"null","date":"2026-09-24","downloads":12},{"category":"null","date":"2026-09-25","downloads":28},{"category":"null","date":"2026-09-26","downloads":7},{"category":"null","date":"2026-09-27","downloads":36},{"category":"null","date":"2026-09-28","downloads":27},{"category":"null","date":"2026-09-29","downloads":16},{"category":"null","date":"2026-09-30","downloads":21},{"category":"null","date":"2026-10-01","downloads":8},{"category":"null","date":"2026-10-02","downloads":28},{"category":"null","date":"2026-10-03","downloads":65},{"category":"null","date":"2026-10-04","downloads":53},{"category":"null","date":"2026-10-05","downloads":33},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-07","downloads":16},{"category":"null","date":"2026-10-08","downloads":15},{"category":"null","date":"2026-10-09","downloads":211},{"category":"null","date":"2026-10-10","downloads":32},{"category":"null","date":"2026-10-11","downloads":24},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-13","downloads":26},{"category":"null","date":"2026-10-14","downloads":33},{"category":"null","date":"2026-10-15","downloads":47},{"category":"null","date":"2026-10-16","downloads":34},{"category":"null","date":"2026-10-17","downloads":9},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":23},{"category":"null","date":"2026-10-20","downloads":48},{"category":"null","date":"2026-10-21","downloads":43},{"category":"null","date":"2026-10-22","downloads":17},{"category":"null","date":"2026-10-23","downloads":86},{"category":"null","date":"2026-10-24","downloads":116},{"category":"null","date":"2026-10-25","downloads":15},{"category":"null","date":"2026-10-26","downloads":26},{"category":"null","date":"2026-10-27","downloads":24},{"category":"null","date":"2026-10-28","downloads":21},{"category":"other","date":"2026-08-01","downloads":4}],"package":"mooncake-transfer-engine","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training."},{"name":"mooncake-transfer-engine-cuda13","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:11:34.592331+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-cuda13/recent","payload":{"data":{"last_day":22693,"last_month":152966,"last_week":104604},"package":"mooncake-transfer-engine-cuda13","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:11:37.254921+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-cuda13/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-04-08","downloads":27},{"category":"with_mirrors","date":"2026-04-09","downloads":34},{"category":"with_mirrors","date":"2026-04-10","downloads":103},{"category":"with_mirrors","date":"2026-04-11","downloads":128},{"category":"with_mirrors","date":"2026-04-12","downloads":20},{"category":"with_mirrors","date":"2026-04-13","downloads":129},{"category":"with_mirrors","date":"2026-04-14","downloads":84},{"category":"with_mirrors","date":"2026-04-15","downloads":286},{"category":"with_mirrors","date":"2026-04-16","downloads":135},{"category":"with_mirrors","date":"2026-04-17","downloads":69},{"category":"with_mirrors","date":"2026-04-18","downloads":281},{"category":"with_mirrors","date":"2026-04-19","downloads":105},{"category":"with_mirrors","date":"2026-04-20","downloads":110},{"category":"with_mirrors","date":"2026-04-21","downloads":384},{"category":"with_mirrors","date":"2026-04-22","downloads":815},{"category":"with_mirrors","date":"2026-04-23","downloads":244},{"category":"with_mirrors","date":"2026-04-24","downloads":195},{"category":"with_mirrors","date":"2026-04-25","downloads":105},{"category":"with_mirrors","date":"2026-04-26","downloads":95},{"category":"with_mirrors","date":"2026-04-27","downloads":183},{"category":"with_mirrors","date":"2026-04-28","downloads":172},{"category":"with_mirrors","date":"2026-04-29","downloads":257},{"category":"with_mirrors","date":"2026-04-30","downloads":695},{"category":"with_mirrors","date":"2026-05-01","downloads":369},{"category":"with_mirrors","date":"2026-05-02","downloads":339},{"category":"with_mirrors","date":"2026-05-03","downloads":330},{"category":"with_mirrors","date":"2026-05-04","downloads":390},{"category":"with_mirrors","date":"2026-05-05","downloads":704},{"category":"with_mirrors","date":"2026-05-06","downloads":549},{"category":"with_mirrors","date":"2026-05-07","downloads":398},{"category":"with_mirrors","date":"2026-05-08","downloads":635},{"category":"with_mirrors","date":"2026-05-09","downloads":332},{"category":"with_mirrors","date":"2026-05-10","downloads":349},{"category":"with_mirrors","date":"2026-05-11","downloads":552},{"category":"with_mirrors","date":"2026-05-12","downloads":649},{"category":"with_mirrors","date":"2026-05-13","downloads":798},{"category":"with_mirrors","date":"2026-05-14","downloads":477},{"category":"with_mirrors","date":"2026-05-15","downloads":701},{"category":"with_mirrors","date":"2026-05-16","downloads":403},{"category":"with_mirrors","date":"2026-05-17","downloads":320},{"category":"with_mirrors","date":"2026-05-18","downloads":684},{"category":"with_mirrors","date":"2026-05-19","downloads":730},{"category":"with_mirrors","date":"2026-05-20","downloads":572},{"category":"with_mirrors","date":"2026-05-21","downloads":1746},{"category":"with_mirrors","date":"2026-05-22","downloads":1261},{"category":"with_mirrors","date":"2026-05-23","downloads":311},{"category":"with_mirrors","date":"2026-05-24","downloads":987},{"category":"with_mirrors","date":"2026-05-25","downloads":1030},{"category":"with_mirrors","date":"2026-05-26","downloads":765},{"category":"with_mirrors","date":"2026-05-27","downloads":2005},{"category":"with_mirrors","date":"2026-05-28","downloads":1743},{"category":"with_mirrors","date":"2026-05-29","downloads":1247},{"category":"with_mirrors","date":"2026-05-30","downloads":1254},{"category":"with_mirrors","date":"2026-05-31","downloads":715},{"category":"with_mirrors","date":"2026-06-01","downloads":1873},{"category":"with_mirrors","date":"2026-06-02","downloads":3041},{"category":"with_mirrors","date":"2026-06-03","downloads":3162},{"category":"with_mirrors","date":"2026-06-04","downloads":3479},{"category":"with_mirrors","date":"2026-06-05","downloads":3270},{"category":"with_mirrors","date":"2026-06-06","downloads":1832},{"category":"with_mirrors","date":"2026-06-07","downloads":1350},{"category":"with_mirrors","date":"2026-06-08","downloads":2330},{"category":"with_mirrors","date":"2026-06-09","downloads":3632},{"category":"with_mirrors","date":"2026-06-10","downloads":2938},{"category":"with_mirrors","date":"2026-06-11","downloads":3815},{"category":"with_mirrors","date":"2026-06-12","downloads":3327},{"category":"with_mirrors","date":"2026-06-13","downloads":1962},{"category":"with_mirrors","date":"2026-06-14","downloads":1896},{"category":"with_mirrors","date":"2026-06-15","downloads":2353},{"category":"with_mirrors","date":"2026-06-16","downloads":2880},{"category":"with_mirrors","date":"2026-06-17","downloads":2261},{"category":"with_mirrors","date":"2026-06-18","downloads":1850},{"category":"with_mirrors","date":"2026-06-19","downloads":1534},{"category":"with_mirrors","date":"2026-06-20","downloads":1372},{"category":"with_mirrors","date":"2026-06-21","downloads":1840},{"category":"with_mirrors","date":"2026-06-22","downloads":2697},{"category":"with_mirrors","date":"2026-06-23","downloads":3140},{"category":"with_mirrors","date":"2026-06-24","downloads":2735},{"category":"with_mirrors","date":"2026-06-25","downloads":3155},{"category":"with_mirrors","date":"2026-06-26","downloads":4008},{"category":"with_mirrors","date":"2026-06-27","downloads":1700},{"category":"with_mirrors","date":"2026-06-28","downloads":1541},{"category":"with_mirrors","date":"2026-06-29","downloads":3596},{"category":"with_mirrors","date":"2026-06-30","downloads":3683},{"category":"with_mirrors","date":"2026-07-01","downloads":2774},{"category":"with_mirrors","date":"2026-07-02","downloads":3162},{"category":"with_mirrors","date":"2026-07-03","downloads":3261},{"category":"with_mirrors","date":"2026-07-04","downloads":1873},{"category":"with_mirrors","date":"2026-07-05","downloads":2091},{"category":"with_mirrors","date":"2026-07-06","downloads":3235},{"category":"with_mirrors","date":"2026-07-07","downloads":3985},{"category":"with_mirrors","date":"2026-07-08","downloads":4022},{"category":"with_mirrors","date":"2026-07-09","downloads":3815},{"category":"with_mirrors","date":"2026-07-10","downloads":3425},{"category":"with_mirrors","date":"2026-07-11","downloads":2414},{"category":"with_mirrors","date":"2026-07-12","downloads":2511},{"category":"with_mirrors","date":"2026-07-13","downloads":5633},{"category":"with_mirrors","date":"2026-07-14","downloads":8460},{"category":"with_mirrors","date":"2026-07-15","downloads":5806},{"category":"with_mirrors","date":"2026-07-16","downloads":4629},{"category":"with_mirrors","date":"2026-07-17","downloads":6077},{"category":"with_mirrors","date":"2026-07-18","downloads":3117},{"category":"with_mirrors","date":"2026-07-19","downloads":2325},{"category":"with_mirrors","date":"2026-07-20","downloads":5689},{"category":"with_mirrors","date":"2026-07-21","downloads":10868},{"category":"with_mirrors","date":"2026-07-22","downloads":5418},{"category":"with_mirrors","date":"2026-07-23","downloads":6564},{"category":"with_mirrors","date":"2026-07-24","downloads":8528},{"category":"with_mirrors","date":"2026-07-25","downloads":4570},{"category":"with_mirrors","date":"2026-07-26","downloads":3507},{"category":"with_mirrors","date":"2026-07-27","downloads":5872},{"category":"with_mirrors","date":"2026-07-28","downloads":7508},{"category":"with_mirrors","date":"2026-07-29","downloads":9310},{"category":"with_mirrors","date":"2026-07-30","downloads":9006},{"category":"with_mirrors","date":"2026-07-31","downloads":9373},{"category":"with_mirrors","date":"2026-08-01","downloads":10026},{"category":"with_mirrors","date":"2026-08-02","downloads":5199},{"category":"with_mirrors","date":"2026-08-03","downloads":8889},{"category":"with_mirrors","date":"2026-08-04","downloads":12661},{"category":"with_mirrors","date":"2026-08-05","downloads":7384},{"category":"with_mirrors","date":"2026-08-06","downloads":6845},{"category":"with_mirrors","date":"2026-08-07","downloads":8449},{"category":"with_mirrors","date":"2026-08-08","downloads":8342},{"category":"with_mirrors","date":"2026-08-09","downloads":5577},{"category":"with_mirrors","date":"2026-08-10","downloads":7314},{"category":"with_mirrors","date":"2026-08-11","downloads":6412},{"category":"with_mirrors","date":"2026-08-12","downloads":8587},{"category":"with_mirrors","date":"2026-08-13","downloads":6985},{"category":"with_mirrors","date":"2026-08-14","downloads":9107},{"category":"with_mirrors","date":"2026-08-15","downloads":4229},{"category":"with_mirrors","date":"2026-08-16","downloads":4899},{"category":"with_mirrors","date":"2026-08-17","downloads":7412},{"category":"with_mirrors","date":"2026-08-18","downloads":7694},{"category":"with_mirrors","date":"2026-08-19","downloads":7619},{"category":"with_mirrors","date":"2026-08-20","downloads":9190},{"category":"with_mirrors","date":"2026-08-21","downloads":9581},{"category":"with_mirrors","date":"2026-08-22","downloads":5017},{"category":"with_mirrors","date":"2026-08-23","downloads":3506},{"category":"with_mirrors","date":"2026-08-24","downloads":3930},{"category":"with_mirrors","date":"2026-08-25","downloads":4243},{"category":"with_mirrors","date":"2026-08-26","downloads":4637},{"category":"with_mirrors","date":"2026-08-27","downloads":3630},{"category":"with_mirrors","date":"2026-08-28","downloads":3902},{"category":"with_mirrors","date":"2026-08-29","downloads":2050},{"category":"with_mirrors","date":"2026-08-30","downloads":2300},{"category":"with_mirrors","date":"2026-08-31","downloads":4103},{"category":"with_mirrors","date":"2026-09-01","downloads":4199},{"category":"with_mirrors","date":"2026-09-02","downloads":4974},{"category":"with_mirrors","date":"2026-09-03","downloads":4672},{"category":"with_mirrors","date":"2026-09-04","downloads":3654},{"category":"with_mirrors","date":"2026-09-05","downloads":3443},{"category":"with_mirrors","date":"2026-09-06","downloads":2725},{"category":"with_mirrors","date":"2026-09-07","downloads":2503},{"category":"with_mirrors","date":"2026-09-08","downloads":3992},{"category":"with_mirrors","date":"2026-09-09","downloads":4482},{"category":"with_mirrors","date":"2026-09-10","downloads":3396},{"category":"with_mirrors","date":"2026-09-11","downloads":4001},{"category":"with_mirrors","date":"2026-09-12","downloads":2190},{"category":"with_mirrors","date":"2026-09-13","downloads":2233},{"category":"with_mirrors","date":"2026-09-14","downloads":3467},{"category":"with_mirrors","date":"2026-09-15","downloads":4256},{"category":"with_mirrors","date":"2026-09-16","downloads":3983},{"category":"with_mirrors","date":"2026-09-17","downloads":4878},{"category":"with_mirrors","date":"2026-09-18","downloads":6192},{"category":"with_mirrors","date":"2026-09-19","downloads":3157},{"category":"with_mirrors","date":"2026-09-20","downloads":3243},{"category":"with_mirrors","date":"2026-09-21","downloads":5340},{"category":"with_mirrors","date":"2026-09-22","downloads":5675},{"category":"with_mirrors","date":"2026-09-23","downloads":5502},{"category":"with_mirrors","date":"2026-09-24","downloads":8085},{"category":"with_mirrors","date":"2026-09-25","downloads":4750},{"category":"with_mirrors","date":"2026-09-26","downloads":3065},{"category":"with_mirrors","date":"2026-09-27","downloads":2987},{"category":"with_mirrors","date":"2026-09-28","downloads":4370},{"category":"with_mirrors","date":"2026-09-29","downloads":5618},{"category":"with_mirrors","date":"2026-09-30","downloads":7222},{"category":"with_mirrors","date":"2026-10-01","downloads":8579},{"category":"with_mirrors","date":"2026-10-02","downloads":14467},{"category":"with_mirrors","date":"2026-10-03","downloads":8528},{"category":"with_mirrors","date":"2026-10-04","downloads":5672},{"category":"with_mirrors","date":"2026-10-05","downloads":10860},{"category":"with_mirrors","date":"2026-10-06","downloads":11839},{"category":"with_mirrors","date":"2026-10-07","downloads":18142},{"category":"with_mirrors","date":"2026-10-08","downloads":20701},{"category":"with_mirrors","date":"2026-10-09","downloads":21506},{"category":"with_mirrors","date":"2026-10-10","downloads":11173},{"category":"with_mirrors","date":"2026-10-11","downloads":7866},{"category":"with_mirrors","date":"2026-10-12","downloads":16609},{"category":"with_mirrors","date":"2026-10-13","downloads":19436},{"category":"with_mirrors","date":"2026-10-14","downloads":21061},{"category":"with_mirrors","date":"2026-10-15","downloads":21122},{"category":"with_mirrors","date":"2026-10-16","downloads":17549},{"category":"with_mirrors","date":"2026-10-17","downloads":10162},{"category":"with_mirrors","date":"2026-10-18","downloads":8578},{"category":"with_mirrors","date":"2026-10-19","downloads":15023},{"category":"with_mirrors","date":"2026-10-20","downloads":16961},{"category":"with_mirrors","date":"2026-10-21","downloads":20492},{"category":"with_mirrors","date":"2026-10-22","downloads":14413},{"category":"with_mirrors","date":"2026-10-23","downloads":13927},{"category":"with_mirrors","date":"2026-10-24","downloads":13386},{"category":"with_mirrors","date":"2026-10-25","downloads":9530},{"category":"with_mirrors","date":"2026-10-26","downloads":16985},{"category":"with_mirrors","date":"2026-10-27","downloads":26890},{"category":"with_mirrors","date":"2026-10-28","downloads":22753},{"category":"without_mirrors","date":"2026-04-08","downloads":8},{"category":"without_mirrors","date":"2026-04-09","downloads":18},{"category":"without_mirrors","date":"2026-04-10","downloads":103},{"category":"without_mirrors","date":"2026-04-11","downloads":53},{"category":"without_mirrors","date":"2026-04-12","downloads":12},{"category":"without_mirrors","date":"2026-04-13","downloads":97},{"category":"without_mirrors","date":"2026-04-14","downloads":76},{"category":"without_mirrors","date":"2026-04-15","downloads":226},{"category":"without_mirrors","date":"2026-04-16","downloads":135},{"category":"without_mirrors","date":"2026-04-17","downloads":18},{"category":"without_mirrors","date":"2026-04-18","downloads":176},{"category":"without_mirrors","date":"2026-04-19","downloads":79},{"category":"without_mirrors","date":"2026-04-20","downloads":65},{"category":"without_mirrors","date":"2026-04-21","downloads":93},{"category":"without_mirrors","date":"2026-04-22","downloads":390},{"category":"without_mirrors","date":"2026-04-23","downloads":148},{"category":"without_mirrors","date":"2026-04-24","downloads":144},{"category":"without_mirrors","date":"2026-04-25","downloads":47},{"category":"without_mirrors","date":"2026-04-26","downloads":42},{"category":"without_mirrors","date":"2026-04-27","downloads":121},{"category":"without_mirrors","date":"2026-04-28","downloads":111},{"category":"without_mirrors","date":"2026-04-29","downloads":234},{"category":"without_mirrors","date":"2026-04-30","downloads":690},{"category":"without_mirrors","date":"2026-05-01","downloads":337},{"category":"without_mirrors","date":"2026-05-02","downloads":332},{"category":"without_mirrors","date":"2026-05-03","downloads":319},{"category":"without_mirrors","date":"2026-05-04","downloads":358},{"category":"without_mirrors","date":"2026-05-05","downloads":631},{"category":"without_mirrors","date":"2026-05-06","downloads":527},{"category":"without_mirrors","date":"2026-05-07","downloads":398},{"category":"without_mirrors","date":"2026-05-08","downloads":633},{"category":"without_mirrors","date":"2026-05-09","downloads":290},{"category":"without_mirrors","date":"2026-05-10","downloads":273},{"category":"without_mirrors","date":"2026-05-11","downloads":545},{"category":"without_mirrors","date":"2026-05-12","downloads":623},{"category":"without_mirrors","date":"2026-05-13","downloads":729},{"category":"without_mirrors","date":"2026-05-14","downloads":442},{"category":"without_mirrors","date":"2026-05-15","downloads":647},{"category":"without_mirrors","date":"2026-05-16","downloads":403},{"category":"without_mirrors","date":"2026-05-17","downloads":264},{"category":"without_mirrors","date":"2026-05-18","downloads":634},{"category":"without_mirrors","date":"2026-05-19","downloads":698},{"category":"without_mirrors","date":"2026-05-20","downloads":564},{"category":"without_mirrors","date":"2026-05-21","downloads":1437},{"category":"without_mirrors","date":"2026-05-22","downloads":1000},{"category":"without_mirrors","date":"2026-05-23","downloads":194},{"category":"without_mirrors","date":"2026-05-24","downloads":632},{"category":"without_mirrors","date":"2026-05-25","downloads":817},{"category":"without_mirrors","date":"2026-05-26","downloads":677},{"category":"without_mirrors","date":"2026-05-27","downloads":1941},{"category":"without_mirrors","date":"2026-05-28","downloads":1671},{"category":"without_mirrors","date":"2026-05-29","downloads":1157},{"category":"without_mirrors","date":"2026-05-30","downloads":1199},{"category":"without_mirrors","date":"2026-05-31","downloads":694},{"category":"without_mirrors","date":"2026-06-01","downloads":1844},{"category":"without_mirrors","date":"2026-06-02","downloads":3007},{"category":"without_mirrors","date":"2026-06-03","downloads":3103},{"category":"without_mirrors","date":"2026-06-04","downloads":3382},{"category":"without_mirrors","date":"2026-06-05","downloads":3229},{"category":"without_mirrors","date":"2026-06-06","downloads":1714},{"category":"without_mirrors","date":"2026-06-07","downloads":1187},{"category":"without_mirrors","date":"2026-06-08","downloads":2318},{"category":"without_mirrors","date":"2026-06-09","downloads":3602},{"category":"without_mirrors","date":"2026-06-10","downloads":2924},{"category":"without_mirrors","date":"2026-06-11","downloads":3772},{"category":"without_mirrors","date":"2026-06-12","downloads":3322},{"category":"without_mirrors","date":"2026-06-13","downloads":1914},{"category":"without_mirrors","date":"2026-06-14","downloads":1896},{"category":"without_mirrors","date":"2026-06-15","downloads":2305},{"category":"without_mirrors","date":"2026-06-16","downloads":2772},{"category":"without_mirrors","date":"2026-06-17","downloads":2252},{"category":"without_mirrors","date":"2026-06-18","downloads":1818},{"category":"without_mirrors","date":"2026-06-19","downloads":1532},{"category":"without_mirrors","date":"2026-06-20","downloads":1319},{"category":"without_mirrors","date":"2026-06-21","downloads":1838},{"category":"without_mirrors","date":"2026-06-22","downloads":2673},{"category":"without_mirrors","date":"2026-06-23","downloads":3071},{"category":"without_mirrors","date":"2026-06-24","downloads":2709},{"category":"without_mirrors","date":"2026-06-25","downloads":3154},{"category":"without_mirrors","date":"2026-06-26","downloads":3975},{"category":"without_mirrors","date":"2026-06-27","downloads":1654},{"category":"without_mirrors","date":"2026-06-28","downloads":1490},{"category":"without_mirrors","date":"2026-06-29","downloads":3592},{"category":"without_mirrors","date":"2026-06-30","downloads":3669},{"category":"without_mirrors","date":"2026-07-01","downloads":2722},{"category":"without_mirrors","date":"2026-07-02","downloads":3099},{"category":"without_mirrors","date":"2026-07-03","downloads":3216},{"category":"without_mirrors","date":"2026-07-04","downloads":1852},{"category":"without_mirrors","date":"2026-07-05","downloads":2065},{"category":"without_mirrors","date":"2026-07-06","downloads":3182},{"category":"without_mirrors","date":"2026-07-07","downloads":3939},{"category":"without_mirrors","date":"2026-07-08","downloads":3974},{"category":"without_mirrors","date":"2026-07-09","downloads":3776},{"category":"without_mirrors","date":"2026-07-10","downloads":3360},{"category":"without_mirrors","date":"2026-07-11","downloads":2393},{"category":"without_mirrors","date":"2026-07-12","downloads":2449},{"category":"without_mirrors","date":"2026-07-13","downloads":5600},{"category":"without_mirrors","date":"2026-07-14","downloads":8458},{"category":"without_mirrors","date":"2026-07-15","downloads":5706},{"category":"without_mirrors","date":"2026-07-16","downloads":4477},{"category":"without_mirrors","date":"2026-07-17","downloads":6030},{"category":"without_mirrors","date":"2026-07-18","downloads":3090},{"category":"without_mirrors","date":"2026-07-19","downloads":2323},{"category":"without_mirrors","date":"2026-07-20","downloads":5675},{"category":"without_mirrors","date":"2026-07-21","downloads":10862},{"category":"without_mirrors","date":"2026-07-22","downloads":5409},{"category":"without_mirrors","date":"2026-07-23","downloads":6490},{"category":"without_mirrors","date":"2026-07-24","downloads":8111},{"category":"without_mirrors","date":"2026-07-25","downloads":4075},{"category":"without_mirrors","date":"2026-07-26","downloads":3405},{"category":"without_mirrors","date":"2026-07-27","downloads":5769},{"category":"without_mirrors","date":"2026-07-28","downloads":7410},{"category":"without_mirrors","date":"2026-07-29","downloads":9277},{"category":"without_mirrors","date":"2026-07-30","downloads":8971},{"category":"without_mirrors","date":"2026-07-31","downloads":9303},{"category":"without_mirrors","date":"2026-08-01","downloads":9856},{"category":"without_mirrors","date":"2026-08-02","downloads":5112},{"category":"without_mirrors","date":"2026-08-03","downloads":8683},{"category":"without_mirrors","date":"2026-08-04","downloads":12512},{"category":"without_mirrors","date":"2026-08-05","downloads":7356},{"category":"without_mirrors","date":"2026-08-06","downloads":6831},{"category":"without_mirrors","date":"2026-08-07","downloads":8266},{"category":"without_mirrors","date":"2026-08-08","downloads":8321},{"category":"without_mirrors","date":"2026-08-09","downloads":5499},{"category":"without_mirrors","date":"2026-08-10","downloads":7293},{"category":"without_mirrors","date":"2026-08-11","downloads":6375},{"category":"without_mirrors","date":"2026-08-12","downloads":8584},{"category":"without_mirrors","date":"2026-08-13","downloads":6964},{"category":"without_mirrors","date":"2026-08-14","downloads":8975},{"category":"without_mirrors","date":"2026-08-15","downloads":4080},{"category":"without_mirrors","date":"2026-08-16","downloads":4848},{"category":"without_mirrors","date":"2026-08-17","downloads":7411},{"category":"without_mirrors","date":"2026-08-18","downloads":7685},{"category":"without_mirrors","date":"2026-08-19","downloads":7614},{"category":"without_mirrors","date":"2026-08-20","downloads":9167},{"category":"without_mirrors","date":"2026-08-21","downloads":9470},{"category":"without_mirrors","date":"2026-08-22","downloads":4937},{"category":"without_mirrors","date":"2026-08-23","downloads":3434},{"category":"without_mirrors","date":"2026-08-24","downloads":3917},{"category":"without_mirrors","date":"2026-08-25","downloads":4192},{"category":"without_mirrors","date":"2026-08-26","downloads":4093},{"category":"without_mirrors","date":"2026-08-27","downloads":3488},{"category":"without_mirrors","date":"2026-08-28","downloads":3840},{"category":"without_mirrors","date":"2026-08-29","downloads":2010},{"category":"without_mirrors","date":"2026-08-30","downloads":2244},{"category":"without_mirrors","date":"2026-08-31","downloads":3533},{"category":"without_mirrors","date":"2026-09-01","downloads":3911},{"category":"without_mirrors","date":"2026-09-02","downloads":4949},{"category":"without_mirrors","date":"2026-09-03","downloads":4545},{"category":"without_mirrors","date":"2026-09-04","downloads":3532},{"category":"without_mirrors","date":"2026-09-05","downloads":3419},{"category":"without_mirrors","date":"2026-09-06","downloads":2645},{"category":"without_mirrors","date":"2026-09-07","downloads":2444},{"category":"without_mirrors","date":"2026-09-08","downloads":3630},{"category":"without_mirrors","date":"2026-09-09","downloads":4439},{"category":"without_mirrors","date":"2026-09-10","downloads":3339},{"category":"without_mirrors","date":"2026-09-11","downloads":3935},{"category":"without_mirrors","date":"2026-09-12","downloads":2167},{"category":"without_mirrors","date":"2026-09-13","downloads":2103},{"category":"without_mirrors","date":"2026-09-14","downloads":3457},{"category":"without_mirrors","date":"2026-09-15","downloads":4218},{"category":"without_mirrors","date":"2026-09-16","downloads":3963},{"category":"without_mirrors","date":"2026-09-17","downloads":4852},{"category":"without_mirrors","date":"2026-09-18","downloads":5964},{"category":"without_mirrors","date":"2026-09-19","downloads":3151},{"category":"without_mirrors","date":"2026-09-20","downloads":2992},{"category":"without_mirrors","date":"2026-09-21","downloads":5297},{"category":"without_mirrors","date":"2026-09-22","downloads":5620},{"category":"without_mirrors","date":"2026-09-23","downloads":5338},{"category":"without_mirrors","date":"2026-09-24","downloads":7985},{"category":"without_mirrors","date":"2026-09-25","downloads":4707},{"category":"without_mirrors","date":"2026-09-26","downloads":2973},{"category":"without_mirrors","date":"2026-09-27","downloads":2929},{"category":"without_mirrors","date":"2026-09-28","downloads":4332},{"category":"without_mirrors","date":"2026-09-29","downloads":5455},{"category":"without_mirrors","date":"2026-09-30","downloads":7178},{"category":"without_mirrors","date":"2026-10-01","downloads":8547},{"category":"without_mirrors","date":"2026-10-02","downloads":14370},{"category":"without_mirrors","date":"2026-10-03","downloads":8524},{"category":"without_mirrors","date":"2026-10-04","downloads":5580},{"category":"without_mirrors","date":"2026-10-05","downloads":10832},{"category":"without_mirrors","date":"2026-10-06","downloads":11801},{"category":"without_mirrors","date":"2026-10-07","downloads":17991},{"category":"without_mirrors","date":"2026-10-08","downloads":20690},{"category":"without_mirrors","date":"2026-10-09","downloads":21213},{"category":"without_mirrors","date":"2026-10-10","downloads":11071},{"category":"without_mirrors","date":"2026-10-11","downloads":7777},{"category":"without_mirrors","date":"2026-10-12","downloads":16480},{"category":"without_mirrors","date":"2026-10-13","downloads":19327},{"category":"without_mirrors","date":"2026-10-14","downloads":21010},{"category":"without_mirrors","date":"2026-10-15","downloads":20716},{"category":"without_mirrors","date":"2026-10-16","downloads":17453},{"category":"without_mirrors","date":"2026-10-17","downloads":10097},{"category":"without_mirrors","date":"2026-10-18","downloads":8348},{"category":"without_mirrors","date":"2026-10-19","downloads":14657},{"category":"without_mirrors","date":"2026-10-20","downloads":16862},{"category":"without_mirrors","date":"2026-10-21","downloads":20135},{"category":"without_mirrors","date":"2026-10-22","downloads":14169},{"category":"without_mirrors","date":"2026-10-23","downloads":13715},{"category":"without_mirrors","date":"2026-10-24","downloads":13354},{"category":"without_mirrors","date":"2026-10-25","downloads":9514},{"category":"without_mirrors","date":"2026-10-26","downloads":16855},{"category":"without_mirrors","date":"2026-10-27","downloads":26794},{"category":"without_mirrors","date":"2026-10-28","downloads":22693}],"package":"mooncake-transfer-engine-cuda13","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:11:39.913257+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-cuda13/python_major","payload":{"data":[{"category":"3","date":"2026-04-08","downloads":8},{"category":"3","date":"2026-04-09","downloads":12},{"category":"3","date":"2026-04-10","downloads":102},{"category":"3","date":"2026-04-11","downloads":42},{"category":"3","date":"2026-04-12","downloads":12},{"category":"3","date":"2026-04-13","downloads":88},{"category":"3","date":"2026-04-14","downloads":36},{"category":"3","date":"2026-04-15","downloads":222},{"category":"3","date":"2026-04-16","downloads":134},{"category":"3","date":"2026-04-17","downloads":18},{"category":"3","date":"2026-04-18","downloads":176},{"category":"3","date":"2026-04-19","downloads":63},{"category":"3","date":"2026-04-20","downloads":64},{"category":"3","date":"2026-04-21","downloads":89},{"category":"3","date":"2026-04-22","downloads":198},{"category":"3","date":"2026-04-23","downloads":134},{"category":"3","date":"2026-04-24","downloads":130},{"category":"3","date":"2026-04-25","downloads":24},{"category":"3","date":"2026-04-26","downloads":40},{"category":"3","date":"2026-04-27","downloads":112},{"category":"3","date":"2026-04-28","downloads":110},{"category":"3","date":"2026-04-29","downloads":230},{"category":"3","date":"2026-04-30","downloads":681},{"category":"3","date":"2026-05-01","downloads":337},{"category":"3","date":"2026-05-02","downloads":330},{"category":"3","date":"2026-05-03","downloads":319},{"category":"3","date":"2026-05-04","downloads":358},{"category":"3","date":"2026-05-05","downloads":583},{"category":"3","date":"2026-05-06","downloads":519},{"category":"3","date":"2026-05-07","downloads":390},{"category":"3","date":"2026-05-08","downloads":632},{"category":"3","date":"2026-05-09","downloads":290},{"category":"3","date":"2026-05-10","downloads":273},{"category":"3","date":"2026-05-11","downloads":544},{"category":"3","date":"2026-05-12","downloads":620},{"category":"3","date":"2026-05-13","downloads":729},{"category":"3","date":"2026-05-14","downloads":432},{"category":"3","date":"2026-05-15","downloads":646},{"category":"3","date":"2026-05-16","downloads":399},{"category":"3","date":"2026-05-17","downloads":258},{"category":"3","date":"2026-05-18","downloads":625},{"category":"3","date":"2026-05-19","downloads":696},{"category":"3","date":"2026-05-20","downloads":564},{"category":"3","date":"2026-05-21","downloads":1258},{"category":"3","date":"2026-05-22","downloads":980},{"category":"3","date":"2026-05-23","downloads":166},{"category":"3","date":"2026-05-24","downloads":454},{"category":"3","date":"2026-05-25","downloads":784},{"category":"3","date":"2026-05-26","downloads":663},{"category":"3","date":"2026-05-27","downloads":1916},{"category":"3","date":"2026-05-28","downloads":1667},{"category":"3","date":"2026-05-29","downloads":1153},{"category":"3","date":"2026-05-30","downloads":1199},{"category":"3","date":"2026-05-31","downloads":677},{"category":"3","date":"2026-06-01","downloads":1842},{"category":"3","date":"2026-06-02","downloads":2990},{"category":"3","date":"2026-06-03","downloads":3068},{"category":"3","date":"2026-06-04","downloads":3372},{"category":"3","date":"2026-06-05","downloads":3192},{"category":"3","date":"2026-06-06","downloads":1669},{"category":"3","date":"2026-06-07","downloads":1187},{"category":"3","date":"2026-06-08","downloads":2307},{"category":"3","date":"2026-06-09","downloads":3567},{"category":"3","date":"2026-06-10","downloads":2900},{"category":"3","date":"2026-06-11","downloads":3743},{"category":"3","date":"2026-06-12","downloads":3315},{"category":"3","date":"2026-06-13","downloads":1896},{"category":"3","date":"2026-06-14","downloads":1893},{"category":"3","date":"2026-06-15","downloads":2304},{"category":"3","date":"2026-06-16","downloads":2751},{"category":"3","date":"2026-06-17","downloads":2236},{"category":"3","date":"2026-06-18","downloads":1816},{"category":"3","date":"2026-06-19","downloads":1477},{"category":"3","date":"2026-06-20","downloads":1299},{"category":"3","date":"2026-06-21","downloads":1826},{"category":"3","date":"2026-06-22","downloads":2659},{"category":"3","date":"2026-06-23","downloads":3026},{"category":"3","date":"2026-06-24","downloads":2698},{"category":"3","date":"2026-06-25","downloads":3153},{"category":"3","date":"2026-06-26","downloads":3974},{"category":"3","date":"2026-06-27","downloads":1645},{"category":"3","date":"2026-06-28","downloads":1488},{"category":"3","date":"2026-06-29","downloads":3585},{"category":"3","date":"2026-06-30","downloads":3661},{"category":"3","date":"2026-07-01","downloads":2712},{"category":"3","date":"2026-07-02","downloads":3096},{"category":"3","date":"2026-07-03","downloads":3213},{"category":"3","date":"2026-07-04","downloads":1850},{"category":"3","date":"2026-07-05","downloads":2059},{"category":"3","date":"2026-07-06","downloads":3178},{"category":"3","date":"2026-07-07","downloads":3938},{"category":"3","date":"2026-07-08","downloads":3972},{"category":"3","date":"2026-07-09","downloads":3773},{"category":"3","date":"2026-07-10","downloads":3354},{"category":"3","date":"2026-07-11","downloads":2388},{"category":"3","date":"2026-07-12","downloads":2448},{"category":"3","date":"2026-07-13","downloads":5599},{"category":"3","date":"2026-07-14","downloads":8451},{"category":"3","date":"2026-07-15","downloads":5688},{"category":"3","date":"2026-07-16","downloads":4471},{"category":"3","date":"2026-07-17","downloads":6024},{"category":"3","date":"2026-07-18","downloads":3088},{"category":"3","date":"2026-07-19","downloads":2321},{"category":"3","date":"2026-07-20","downloads":5665},{"category":"3","date":"2026-07-21","downloads":10858},{"category":"3","date":"2026-07-22","downloads":5357},{"category":"3","date":"2026-07-23","downloads":6488},{"category":"3","date":"2026-07-24","downloads":7900},{"category":"3","date":"2026-07-25","downloads":3870},{"category":"3","date":"2026-07-26","downloads":3376},{"category":"3","date":"2026-07-27","downloads":5745},{"category":"3","date":"2026-07-28","downloads":7400},{"category":"3","date":"2026-07-29","downloads":9267},{"category":"3","date":"2026-07-30","downloads":8952},{"category":"3","date":"2026-07-31","downloads":9242},{"category":"3","date":"2026-08-01","downloads":9836},{"category":"3","date":"2026-08-02","downloads":5112},{"category":"3","date":"2026-08-03","downloads":8657},{"category":"3","date":"2026-08-04","downloads":12469},{"category":"3","date":"2026-08-05","downloads":7340},{"category":"3","date":"2026-08-06","downloads":6807},{"category":"3","date":"2026-08-07","downloads":8257},{"category":"3","date":"2026-08-08","downloads":8314},{"category":"3","date":"2026-08-09","downloads":5498},{"category":"3","date":"2026-08-10","downloads":7283},{"category":"3","date":"2026-08-11","downloads":6360},{"category":"3","date":"2026-08-12","downloads":8551},{"category":"3","date":"2026-08-13","downloads":6952},{"category":"3","date":"2026-08-14","downloads":8951},{"category":"3","date":"2026-08-15","downloads":4076},{"category":"3","date":"2026-08-16","downloads":4846},{"category":"3","date":"2026-08-17","downloads":7407},{"category":"3","date":"2026-08-18","downloads":7674},{"category":"3","date":"2026-08-19","downloads":7605},{"category":"3","date":"2026-08-20","downloads":9148},{"category":"3","date":"2026-08-21","downloads":9463},{"category":"3","date":"2026-08-22","downloads":4933},{"category":"3","date":"2026-08-23","downloads":3429},{"category":"3","date":"2026-08-24","downloads":3901},{"category":"3","date":"2026-08-25","downloads":4178},{"category":"3","date":"2026-08-26","downloads":3900},{"category":"3","date":"2026-08-27","downloads":3319},{"category":"3","date":"2026-08-28","downloads":3745},{"category":"3","date":"2026-08-29","downloads":1984},{"category":"3","date":"2026-08-30","downloads":2232},{"category":"3","date":"2026-08-31","downloads":3267},{"category":"3","date":"2026-09-01","downloads":3835},{"category":"3","date":"2026-09-02","downloads":4828},{"category":"3","date":"2026-09-03","downloads":4498},{"category":"3","date":"2026-09-04","downloads":3494},{"category":"3","date":"2026-09-05","downloads":3385},{"category":"3","date":"2026-09-06","downloads":2630},{"category":"3","date":"2026-09-07","downloads":2428},{"category":"3","date":"2026-09-08","downloads":3608},{"category":"3","date":"2026-09-09","downloads":4408},{"category":"3","date":"2026-09-10","downloads":3326},{"category":"3","date":"2026-09-11","downloads":3872},{"category":"3","date":"2026-09-12","downloads":2156},{"category":"3","date":"2026-09-13","downloads":2089},{"category":"3","date":"2026-09-14","downloads":3414},{"category":"3","date":"2026-09-15","downloads":4204},{"category":"3","date":"2026-09-16","downloads":3946},{"category":"3","date":"2026-09-17","downloads":4832},{"category":"3","date":"2026-09-18","downloads":5952},{"category":"3","date":"2026-09-19","downloads":3052},{"category":"3","date":"2026-09-20","downloads":2980},{"category":"3","date":"2026-09-21","downloads":5263},{"category":"3","date":"2026-09-22","downloads":5607},{"category":"3","date":"2026-09-23","downloads":5297},{"category":"3","date":"2026-09-24","downloads":7969},{"category":"3","date":"2026-09-25","downloads":4699},{"category":"3","date":"2026-09-26","downloads":2940},{"category":"3","date":"2026-09-27","downloads":2828},{"category":"3","date":"2026-09-28","downloads":4299},{"category":"3","date":"2026-09-29","downloads":5392},{"category":"3","date":"2026-09-30","downloads":7144},{"category":"3","date":"2026-10-01","downloads":7292},{"category":"3","date":"2026-10-02","downloads":7432},{"category":"3","date":"2026-10-03","downloads":5573},{"category":"3","date":"2026-10-04","downloads":4502},{"category":"3","date":"2026-10-05","downloads":8494},{"category":"3","date":"2026-10-06","downloads":9517},{"category":"3","date":"2026-10-07","downloads":9206},{"category":"3","date":"2026-10-08","downloads":11622},{"category":"3","date":"2026-10-09","downloads":13514},{"category":"3","date":"2026-10-10","downloads":8926},{"category":"3","date":"2026-10-11","downloads":6140},{"category":"3","date":"2026-10-12","downloads":12197},{"category":"3","date":"2026-10-13","downloads":12278},{"category":"3","date":"2026-10-14","downloads":12834},{"category":"3","date":"2026-10-15","downloads":11585},{"category":"3","date":"2026-10-16","downloads":9544},{"category":"3","date":"2026-10-17","downloads":6336},{"category":"3","date":"2026-10-18","downloads":6039},{"category":"3","date":"2026-10-19","downloads":8454},{"category":"3","date":"2026-10-20","downloads":9037},{"category":"3","date":"2026-10-21","downloads":11568},{"category":"3","date":"2026-10-22","downloads":8154},{"category":"3","date":"2026-10-23","downloads":8980},{"category":"3","date":"2026-10-24","downloads":10872},{"category":"3","date":"2026-10-25","downloads":8395},{"category":"3","date":"2026-10-26","downloads":11574},{"category":"3","date":"2026-10-27","downloads":17648},{"category":"3","date":"2026-10-28","downloads":14796},{"category":"null","date":"2026-04-09","downloads":6},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":11},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":40},{"category":"null","date":"2026-04-15","downloads":4},{"category":"null","date":"2026-04-16","downloads":1},{"category":"null","date":"2026-04-19","downloads":16},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-21","downloads":4},{"category":"null","date":"2026-04-22","downloads":192},{"category":"null","date":"2026-04-23","downloads":14},{"category":"null","date":"2026-04-24","downloads":14},{"category":"null","date":"2026-04-25","downloads":23},{"category":"null","date":"2026-04-26","downloads":2},{"category":"null","date":"2026-04-27","downloads":9},{"category":"null","date":"2026-04-28","downloads":1},{"category":"null","date":"2026-04-29","downloads":4},{"category":"null","date":"2026-04-30","downloads":9},{"category":"null","date":"2026-05-02","downloads":2},{"category":"null","date":"2026-05-05","downloads":48},{"category":"null","date":"2026-05-06","downloads":8},{"category":"null","date":"2026-05-07","downloads":8},{"category":"null","date":"2026-05-08","downloads":1},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":10},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":4},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":9},{"category":"null","date":"2026-05-19","downloads":2},{"category":"null","date":"2026-05-21","downloads":179},{"category":"null","date":"2026-05-22","downloads":20},{"category":"null","date":"2026-05-23","downloads":28},{"category":"null","date":"2026-05-24","downloads":178},{"category":"null","date":"2026-05-25","downloads":33},{"category":"null","date":"2026-05-26","downloads":14},{"category":"null","date":"2026-05-27","downloads":25},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":4},{"category":"null","date":"2026-05-31","downloads":17},{"category":"null","date":"2026-06-01","downloads":2},{"category":"null","date":"2026-06-02","downloads":17},{"category":"null","date":"2026-06-03","downloads":35},{"category":"null","date":"2026-06-04","downloads":10},{"category":"null","date":"2026-06-05","downloads":37},{"category":"null","date":"2026-06-06","downloads":45},{"category":"null","date":"2026-06-08","downloads":11},{"category":"null","date":"2026-06-09","downloads":35},{"category":"null","date":"2026-06-10","downloads":24},{"category":"null","date":"2026-06-11","downloads":29},{"category":"null","date":"2026-06-12","downloads":7},{"category":"null","date":"2026-06-13","downloads":18},{"category":"null","date":"2026-06-14","downloads":3},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":21},{"category":"null","date":"2026-06-17","downloads":16},{"category":"null","date":"2026-06-18","downloads":2},{"category":"null","date":"2026-06-19","downloads":55},{"category":"null","date":"2026-06-20","downloads":20},{"category":"null","date":"2026-06-21","downloads":12},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":45},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":1},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-28","downloads":2},{"category":"null","date":"2026-06-29","downloads":7},{"category":"null","date":"2026-06-30","downloads":8},{"category":"null","date":"2026-07-01","downloads":10},{"category":"null","date":"2026-07-02","downloads":3},{"category":"null","date":"2026-07-03","downloads":3},{"category":"null","date":"2026-07-04","downloads":2},{"category":"null","date":"2026-07-05","downloads":6},{"category":"null","date":"2026-07-06","downloads":4},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":2},{"category":"null","date":"2026-07-09","downloads":3},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":5},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":7},{"category":"null","date":"2026-07-15","downloads":18},{"category":"null","date":"2026-07-16","downloads":6},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":2},{"category":"null","date":"2026-07-19","downloads":2},{"category":"null","date":"2026-07-20","downloads":10},{"category":"null","date":"2026-07-21","downloads":4},{"category":"null","date":"2026-07-22","downloads":52},{"category":"null","date":"2026-07-23","downloads":2},{"category":"null","date":"2026-07-24","downloads":211},{"category":"null","date":"2026-07-25","downloads":205},{"category":"null","date":"2026-07-26","downloads":29},{"category":"null","date":"2026-07-27","downloads":24},{"category":"null","date":"2026-07-28","downloads":10},{"category":"null","date":"2026-07-29","downloads":10},{"category":"null","date":"2026-07-30","downloads":19},{"category":"null","date":"2026-07-31","downloads":61},{"category":"null","date":"2026-08-01","downloads":20},{"category":"null","date":"2026-08-03","downloads":26},{"category":"null","date":"2026-08-04","downloads":43},{"category":"null","date":"2026-08-05","downloads":16},{"category":"null","date":"2026-08-06","downloads":24},{"category":"null","date":"2026-08-07","downloads":9},{"category":"null","date":"2026-08-08","downloads":7},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-10","downloads":10},{"category":"null","date":"2026-08-11","downloads":15},{"category":"null","date":"2026-08-12","downloads":33},{"category":"null","date":"2026-08-13","downloads":12},{"category":"null","date":"2026-08-14","downloads":24},{"category":"null","date":"2026-08-15","downloads":4},{"category":"null","date":"2026-08-16","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":11},{"category":"null","date":"2026-08-19","downloads":9},{"category":"null","date":"2026-08-20","downloads":19},{"category":"null","date":"2026-08-21","downloads":7},{"category":"null","date":"2026-08-22","downloads":4},{"category":"null","date":"2026-08-23","downloads":5},{"category":"null","date":"2026-08-24","downloads":16},{"category":"null","date":"2026-08-25","downloads":14},{"category":"null","date":"2026-08-26","downloads":193},{"category":"null","date":"2026-08-27","downloads":169},{"category":"null","date":"2026-08-28","downloads":95},{"category":"null","date":"2026-08-29","downloads":26},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":266},{"category":"null","date":"2026-09-01","downloads":76},{"category":"null","date":"2026-09-02","downloads":121},{"category":"null","date":"2026-09-03","downloads":47},{"category":"null","date":"2026-09-04","downloads":38},{"category":"null","date":"2026-09-05","downloads":34},{"category":"null","date":"2026-09-06","downloads":15},{"category":"null","date":"2026-09-07","downloads":16},{"category":"null","date":"2026-09-08","downloads":22},{"category":"null","date":"2026-09-09","downloads":31},{"category":"null","date":"2026-09-10","downloads":13},{"category":"null","date":"2026-09-11","downloads":63},{"category":"null","date":"2026-09-12","downloads":11},{"category":"null","date":"2026-09-13","downloads":14},{"category":"null","date":"2026-09-14","downloads":43},{"category":"null","date":"2026-09-15","downloads":14},{"category":"null","date":"2026-09-16","downloads":17},{"category":"null","date":"2026-09-17","downloads":20},{"category":"null","date":"2026-09-18","downloads":12},{"category":"null","date":"2026-09-19","downloads":99},{"category":"null","date":"2026-09-20","downloads":12},{"category":"null","date":"2026-09-21","downloads":34},{"category":"null","date":"2026-09-22","downloads":13},{"category":"null","date":"2026-09-23","downloads":41},{"category":"null","date":"2026-09-24","downloads":16},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":33},{"category":"null","date":"2026-09-27","downloads":101},{"category":"null","date":"2026-09-28","downloads":33},{"category":"null","date":"2026-09-29","downloads":63},{"category":"null","date":"2026-09-30","downloads":34},{"category":"null","date":"2026-10-01","downloads":1255},{"category":"null","date":"2026-10-02","downloads":6938},{"category":"null","date":"2026-10-03","downloads":2951},{"category":"null","date":"2026-10-04","downloads":1078},{"category":"null","date":"2026-10-05","downloads":2338},{"category":"null","date":"2026-10-06","downloads":2284},{"category":"null","date":"2026-10-07","downloads":8785},{"category":"null","date":"2026-10-08","downloads":9068},{"category":"null","date":"2026-10-09","downloads":7699},{"category":"null","date":"2026-10-10","downloads":2145},{"category":"null","date":"2026-10-11","downloads":1637},{"category":"null","date":"2026-10-12","downloads":4283},{"category":"null","date":"2026-10-13","downloads":7049},{"category":"null","date":"2026-10-14","downloads":8176},{"category":"null","date":"2026-10-15","downloads":9131},{"category":"null","date":"2026-10-16","downloads":7909},{"category":"null","date":"2026-10-17","downloads":3761},{"category":"null","date":"2026-10-18","downloads":2309},{"category":"null","date":"2026-10-19","downloads":6203},{"category":"null","date":"2026-10-20","downloads":7825},{"category":"null","date":"2026-10-21","downloads":8567},{"category":"null","date":"2026-10-22","downloads":6015},{"category":"null","date":"2026-10-23","downloads":4735},{"category":"null","date":"2026-10-24","downloads":2482},{"category":"null","date":"2026-10-25","downloads":1119},{"category":"null","date":"2026-10-26","downloads":5281},{"category":"null","date":"2026-10-27","downloads":9146},{"category":"null","date":"2026-10-28","downloads":7897}],"package":"mooncake-transfer-engine-cuda13","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:11:43.071570+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-cuda13/python_minor","payload":{"data":[{"category":"3.10","date":"2026-04-08","downloads":2},{"category":"3.10","date":"2026-04-10","downloads":4},{"category":"3.10","date":"2026-04-11","downloads":3},{"category":"3.10","date":"2026-04-12","downloads":5},{"category":"3.10","date":"2026-04-13","downloads":3},{"category":"3.10","date":"2026-04-15","downloads":2},{"category":"3.10","date":"2026-04-18","downloads":134},{"category":"3.10","date":"2026-04-19","downloads":60},{"category":"3.10","date":"2026-04-20","downloads":2},{"category":"3.10","date":"2026-04-21","downloads":16},{"category":"3.10","date":"2026-04-22","downloads":91},{"category":"3.10","date":"2026-04-23","downloads":2},{"category":"3.10","date":"2026-04-24","downloads":18},{"category":"3.10","date":"2026-04-25","downloads":3},{"category":"3.10","date":"2026-04-27","downloads":4},{"category":"3.10","date":"2026-04-28","downloads":24},{"category":"3.10","date":"2026-04-29","downloads":7},{"category":"3.10","date":"2026-04-30","downloads":49},{"category":"3.10","date":"2026-05-01","downloads":2},{"category":"3.10","date":"2026-05-02","downloads":86},{"category":"3.10","date":"2026-05-03","downloads":9},{"category":"3.10","date":"2026-05-04","downloads":4},{"category":"3.10","date":"2026-05-05","downloads":60},{"category":"3.10","date":"2026-05-06","downloads":6},{"category":"3.10","date":"2026-05-07","downloads":6},{"category":"3.10","date":"2026-05-08","downloads":3},{"category":"3.10","date":"2026-05-09","downloads":1},{"category":"3.10","date":"2026-05-10","downloads":1},{"category":"3.10","date":"2026-05-11","downloads":1},{"category":"3.10","date":"2026-05-12","downloads":24},{"category":"3.10","date":"2026-05-13","downloads":15},{"category":"3.10","date":"2026-05-14","downloads":2},{"category":"3.10","date":"2026-05-16","downloads":3},{"category":"3.10","date":"2026-05-18","downloads":3},{"category":"3.10","date":"2026-05-19","downloads":27},{"category":"3.10","date":"2026-05-20","downloads":4},{"category":"3.10","date":"2026-05-21","downloads":64},{"category":"3.10","date":"2026-05-22","downloads":64},{"category":"3.10","date":"2026-05-23","downloads":3},{"category":"3.10","date":"2026-05-24","downloads":68},{"category":"3.10","date":"2026-05-25","downloads":51},{"category":"3.10","date":"2026-05-26","downloads":7},{"category":"3.10","date":"2026-05-27","downloads":25},{"category":"3.10","date":"2026-05-28","downloads":4},{"category":"3.10","date":"2026-05-29","downloads":5},{"category":"3.10","date":"2026-05-30","downloads":2},{"category":"3.10","date":"2026-06-01","downloads":2},{"category":"3.10","date":"2026-06-03","downloads":8},{"category":"3.10","date":"2026-06-05","downloads":2},{"category":"3.10","date":"2026-06-08","downloads":2},{"category":"3.10","date":"2026-06-09","downloads":16},{"category":"3.10","date":"2026-06-10","downloads":69},{"category":"3.10","date":"2026-06-11","downloads":15},{"category":"3.10","date":"2026-06-12","downloads":14},{"category":"3.10","date":"2026-06-13","downloads":8},{"category":"3.10","date":"2026-06-14","downloads":10},{"category":"3.10","date":"2026-06-15","downloads":11},{"category":"3.10","date":"2026-06-16","downloads":7},{"category":"3.10","date":"2026-06-17","downloads":4},{"category":"3.10","date":"2026-06-18","downloads":13},{"category":"3.10","date":"2026-06-19","downloads":8},{"category":"3.10","date":"2026-06-20","downloads":1},{"category":"3.10","date":"2026-06-21","downloads":43},{"category":"3.10","date":"2026-06-22","downloads":17},{"category":"3.10","date":"2026-06-23","downloads":19},{"category":"3.10","date":"2026-06-24","downloads":21},{"category":"3.10","date":"2026-06-25","downloads":7},{"category":"3.10","date":"2026-06-26","downloads":4},{"category":"3.10","date":"2026-06-27","downloads":4},{"category":"3.10","date":"2026-06-28","downloads":5},{"category":"3.10","date":"2026-06-29","downloads":2},{"category":"3.10","date":"2026-07-01","downloads":3},{"category":"3.10","date":"2026-07-02","downloads":22},{"category":"3.10","date":"2026-07-03","downloads":10},{"category":"3.10","date":"2026-07-04","downloads":8},{"category":"3.10","date":"2026-07-05","downloads":8},{"category":"3.10","date":"2026-07-06","downloads":2},{"category":"3.10","date":"2026-07-07","downloads":6},{"category":"3.10","date":"2026-07-08","downloads":10},{"category":"3.10","date":"2026-07-10","downloads":19},{"category":"3.10","date":"2026-07-11","downloads":4},{"category":"3.10","date":"2026-07-12","downloads":6},{"category":"3.10","date":"2026-07-13","downloads":20},{"category":"3.10","date":"2026-07-14","downloads":25},{"category":"3.10","date":"2026-07-15","downloads":24},{"category":"3.10","date":"2026-07-16","downloads":23},{"category":"3.10","date":"2026-07-17","downloads":26},{"category":"3.10","date":"2026-07-18","downloads":20},{"category":"3.10","date":"2026-07-19","downloads":21},{"category":"3.10","date":"2026-07-20","downloads":20},{"category":"3.10","date":"2026-07-21","downloads":26},{"category":"3.10","date":"2026-07-22","downloads":30},{"category":"3.10","date":"2026-07-23","downloads":73},{"category":"3.10","date":"2026-07-24","downloads":83},{"category":"3.10","date":"2026-07-25","downloads":162},{"category":"3.10","date":"2026-07-26","downloads":34},{"category":"3.10","date":"2026-07-27","downloads":73},{"category":"3.10","date":"2026-07-28","downloads":36},{"category":"3.10","date":"2026-07-29","downloads":21},{"category":"3.10","date":"2026-07-30","downloads":11},{"category":"3.10","date":"2026-07-31","downloads":20},{"category":"3.10","date":"2026-08-01","downloads":20},{"category":"3.10","date":"2026-08-02","downloads":27},{"category":"3.10","date":"2026-08-03","downloads":23},{"category":"3.10","date":"2026-08-04","downloads":17},{"category":"3.10","date":"2026-08-05","downloads":23},{"category":"3.10","date":"2026-08-06","downloads":35},{"category":"3.10","date":"2026-08-07","downloads":29},{"category":"3.10","date":"2026-08-08","downloads":27},{"category":"3.10","date":"2026-08-09","downloads":26},{"category":"3.10","date":"2026-08-10","downloads":14},{"category":"3.10","date":"2026-08-11","downloads":13},{"category":"3.10","date":"2026-08-12","downloads":24},{"category":"3.10","date":"2026-08-13","downloads":3},{"category":"3.10","date":"2026-08-14","downloads":47},{"category":"3.10","date":"2026-08-15","downloads":64},{"category":"3.10","date":"2026-08-16","downloads":49},{"category":"3.10","date":"2026-08-17","downloads":56},{"category":"3.10","date":"2026-08-18","downloads":21},{"category":"3.10","date":"2026-08-19","downloads":12},{"category":"3.10","date":"2026-08-20","downloads":18},{"category":"3.10","date":"2026-08-21","downloads":28},{"category":"3.10","date":"2026-08-22","downloads":40},{"category":"3.10","date":"2026-08-23","downloads":38},{"category":"3.10","date":"2026-08-24","downloads":38},{"category":"3.10","date":"2026-08-25","downloads":20},{"category":"3.10","date":"2026-08-26","downloads":86},{"category":"3.10","date":"2026-08-27","downloads":55},{"category":"3.10","date":"2026-08-28","downloads":20},{"category":"3.10","date":"2026-08-29","downloads":35},{"category":"3.10","date":"2026-08-30","downloads":18},{"category":"3.10","date":"2026-08-31","downloads":21},{"category":"3.10","date":"2026-09-01","downloads":24},{"category":"3.10","date":"2026-09-02","downloads":32},{"category":"3.10","date":"2026-09-03","downloads":16},{"category":"3.10","date":"2026-09-04","downloads":16},{"category":"3.10","date":"2026-09-05","downloads":21},{"category":"3.10","date":"2026-09-06","downloads":12},{"category":"3.10","date":"2026-09-07","downloads":21},{"category":"3.10","date":"2026-09-08","downloads":6},{"category":"3.10","date":"2026-09-09","downloads":24},{"category":"3.10","date":"2026-09-10","downloads":21},{"category":"3.10","date":"2026-09-11","downloads":24},{"category":"3.10","date":"2026-09-12","downloads":10},{"category":"3.10","date":"2026-09-13","downloads":13},{"category":"3.10","date":"2026-09-14","downloads":16},{"category":"3.10","date":"2026-09-15","downloads":12},{"category":"3.10","date":"2026-09-16","downloads":11},{"category":"3.10","date":"2026-09-17","downloads":32},{"category":"3.10","date":"2026-09-18","downloads":22},{"category":"3.10","date":"2026-09-20","downloads":16},{"category":"3.10","date":"2026-09-21","downloads":16},{"category":"3.10","date":"2026-09-22","downloads":20},{"category":"3.10","date":"2026-09-23","downloads":31},{"category":"3.10","date":"2026-09-24","downloads":19},{"category":"3.10","date":"2026-09-25","downloads":18},{"category":"3.10","date":"2026-09-26","downloads":12},{"category":"3.10","date":"2026-09-27","downloads":7},{"category":"3.10","date":"2026-09-28","downloads":23},{"category":"3.10","date":"2026-09-29","downloads":29},{"category":"3.10","date":"2026-09-30","downloads":66},{"category":"3.10","date":"2026-10-01","downloads":30},{"category":"3.10","date":"2026-10-02","downloads":21},{"category":"3.10","date":"2026-10-03","downloads":18},{"category":"3.10","date":"2026-10-04","downloads":10},{"category":"3.10","date":"2026-10-05","downloads":15},{"category":"3.10","date":"2026-10-06","downloads":11},{"category":"3.10","date":"2026-10-07","downloads":15},{"category":"3.10","date":"2026-10-08","downloads":33},{"category":"3.10","date":"2026-10-09","downloads":22},{"category":"3.10","date":"2026-10-10","downloads":10},{"category":"3.10","date":"2026-10-11","downloads":14},{"category":"3.10","date":"2026-10-12","downloads":27},{"category":"3.10","date":"2026-10-13","downloads":15},{"category":"3.10","date":"2026-10-14","downloads":16},{"category":"3.10","date":"2026-10-15","downloads":7},{"category":"3.10","date":"2026-10-16","downloads":29},{"category":"3.10","date":"2026-10-17","downloads":45},{"category":"3.10","date":"2026-10-18","downloads":30},{"category":"3.10","date":"2026-10-19","downloads":27},{"category":"3.10","date":"2026-10-20","downloads":68},{"category":"3.10","date":"2026-10-21","downloads":64},{"category":"3.10","date":"2026-10-22","downloads":16},{"category":"3.10","date":"2026-10-23","downloads":12},{"category":"3.10","date":"2026-10-24","downloads":9},{"category":"3.10","date":"2026-10-25","downloads":9},{"category":"3.10","date":"2026-10-26","downloads":91},{"category":"3.10","date":"2026-10-27","downloads":15},{"category":"3.10","date":"2026-10-28","downloads":12},{"category":"3.11","date":"2026-04-10","downloads":4},{"category":"3.11","date":"2026-04-11","downloads":8},{"category":"3.11","date":"2026-04-16","downloads":6},{"category":"3.11","date":"2026-04-20","downloads":14},{"category":"3.11","date":"2026-04-21","downloads":6},{"category":"3.11","date":"2026-04-22","downloads":12},{"category":"3.11","date":"2026-04-23","downloads":2},{"category":"3.11","date":"2026-04-29","downloads":1},{"category":"3.11","date":"2026-05-01","downloads":1},{"category":"3.11","date":"2026-05-04","downloads":2},{"category":"3.11","date":"2026-05-05","downloads":1},{"category":"3.11","date":"2026-05-06","downloads":1},{"category":"3.11","date":"2026-05-07","downloads":1},{"category":"3.11","date":"2026-05-08","downloads":2},{"category":"3.11","date":"2026-05-09","downloads":7},{"category":"3.11","date":"2026-05-11","downloads":4},{"category":"3.11","date":"2026-05-12","downloads":2},{"category":"3.11","date":"2026-05-13","downloads":5},{"category":"3.11","date":"2026-05-16","downloads":27},{"category":"3.11","date":"2026-05-18","downloads":2},{"category":"3.11","date":"2026-05-19","downloads":4},{"category":"3.11","date":"2026-05-20","downloads":1},{"category":"3.11","date":"2026-05-21","downloads":8},{"category":"3.11","date":"2026-05-22","downloads":10},{"category":"3.11","date":"2026-05-23","downloads":2},{"category":"3.11","date":"2026-05-24","downloads":16},{"category":"3.11","date":"2026-05-25","downloads":6},{"category":"3.11","date":"2026-05-26","downloads":11},{"category":"3.11","date":"2026-05-27","downloads":9},{"category":"3.11","date":"2026-05-28","downloads":2},{"category":"3.11","date":"2026-05-31","downloads":2},{"category":"3.11","date":"2026-06-01","downloads":8},{"category":"3.11","date":"2026-06-02","downloads":2},{"category":"3.11","date":"2026-06-03","downloads":3},{"category":"3.11","date":"2026-06-04","downloads":1},{"category":"3.11","date":"2026-06-05","downloads":4},{"category":"3.11","date":"2026-06-06","downloads":1},{"category":"3.11","date":"2026-06-08","downloads":4},{"category":"3.11","date":"2026-06-09","downloads":47},{"category":"3.11","date":"2026-06-10","downloads":46},{"category":"3.11","date":"2026-06-11","downloads":62},{"category":"3.11","date":"2026-06-12","downloads":41},{"category":"3.11","date":"2026-06-13","downloads":31},{"category":"3.11","date":"2026-06-14","downloads":12},{"category":"3.11","date":"2026-06-15","downloads":13},{"category":"3.11","date":"2026-06-16","downloads":27},{"category":"3.11","date":"2026-06-17","downloads":22},{"category":"3.11","date":"2026-06-18","downloads":14},{"category":"3.11","date":"2026-06-19","downloads":10},{"category":"3.11","date":"2026-06-20","downloads":16},{"category":"3.11","date":"2026-06-21","downloads":12},{"category":"3.11","date":"2026-06-22","downloads":23},{"category":"3.11","date":"2026-06-23","downloads":7},{"category":"3.11","date":"2026-06-24","downloads":37},{"category":"3.11","date":"2026-06-25","downloads":13},{"category":"3.11","date":"2026-06-26","downloads":6},{"category":"3.11","date":"2026-06-27","downloads":13},{"category":"3.11","date":"2026-06-28","downloads":5},{"category":"3.11","date":"2026-06-29","downloads":19},{"category":"3.11","date":"2026-06-30","downloads":4},{"category":"3.11","date":"2026-07-01","downloads":26},{"category":"3.11","date":"2026-07-02","downloads":43},{"category":"3.11","date":"2026-07-03","downloads":5},{"category":"3.11","date":"2026-07-04","downloads":3},{"category":"3.11","date":"2026-07-05","downloads":6},{"category":"3.11","date":"2026-07-06","downloads":39},{"category":"3.11","date":"2026-07-07","downloads":17},{"category":"3.11","date":"2026-07-08","downloads":14},{"category":"3.11","date":"2026-07-09","downloads":79},{"category":"3.11","date":"2026-07-10","downloads":103},{"category":"3.11","date":"2026-07-11","downloads":57},{"category":"3.11","date":"2026-07-12","downloads":9},{"category":"3.11","date":"2026-07-13","downloads":28},{"category":"3.11","date":"2026-07-14","downloads":58},{"category":"3.11","date":"2026-07-15","downloads":89},{"category":"3.11","date":"2026-07-16","downloads":64},{"category":"3.11","date":"2026-07-17","downloads":38},{"category":"3.11","date":"2026-07-18","downloads":11},{"category":"3.11","date":"2026-07-19","downloads":55},{"category":"3.11","date":"2026-07-20","downloads":23},{"category":"3.11","date":"2026-07-21","downloads":61},{"category":"3.11","date":"2026-07-22","downloads":242},{"category":"3.11","date":"2026-07-23","downloads":187},{"category":"3.11","date":"2026-07-24","downloads":217},{"category":"3.11","date":"2026-07-25","downloads":108},{"category":"3.11","date":"2026-07-26","downloads":44},{"category":"3.11","date":"2026-07-27","downloads":53},{"category":"3.11","date":"2026-07-28","downloads":43},{"category":"3.11","date":"2026-07-29","downloads":30},{"category":"3.11","date":"2026-07-30","downloads":38},{"category":"3.11","date":"2026-07-31","downloads":37},{"category":"3.11","date":"2026-08-01","downloads":19},{"category":"3.11","date":"2026-08-02","downloads":24},{"category":"3.11","date":"2026-08-03","downloads":52},{"category":"3.11","date":"2026-08-04","downloads":109},{"category":"3.11","date":"2026-08-05","downloads":82},{"category":"3.11","date":"2026-08-06","downloads":48},{"category":"3.11","date":"2026-08-07","downloads":57},{"category":"3.11","date":"2026-08-08","downloads":107},{"category":"3.11","date":"2026-08-09","downloads":64},{"category":"3.11","date":"2026-08-10","downloads":127},{"category":"3.11","date":"2026-08-11","downloads":94},{"category":"3.11","date":"2026-08-12","downloads":59},{"category":"3.11","date":"2026-08-13","downloads":58},{"category":"3.11","date":"2026-08-14","downloads":94},{"category":"3.11","date":"2026-08-15","downloads":40},{"category":"3.11","date":"2026-08-16","downloads":21},{"category":"3.11","date":"2026-08-17","downloads":41},{"category":"3.11","date":"2026-08-18","downloads":40},{"category":"3.11","date":"2026-08-19","downloads":39},{"category":"3.11","date":"2026-08-20","downloads":74},{"category":"3.11","date":"2026-08-21","downloads":90},{"category":"3.11","date":"2026-08-22","downloads":40},{"category":"3.11","date":"2026-08-23","downloads":28},{"category":"3.11","date":"2026-08-24","downloads":105},{"category":"3.11","date":"2026-08-25","downloads":63},{"category":"3.11","date":"2026-08-26","downloads":66},{"category":"3.11","date":"2026-08-27","downloads":57},{"category":"3.11","date":"2026-08-28","downloads":35},{"category":"3.11","date":"2026-08-29","downloads":9},{"category":"3.11","date":"2026-08-30","downloads":2},{"category":"3.11","date":"2026-08-31","downloads":66},{"category":"3.11","date":"2026-09-01","downloads":76},{"category":"3.11","date":"2026-09-02","downloads":251},{"category":"3.11","date":"2026-09-03","downloads":99},{"category":"3.11","date":"2026-09-04","downloads":30},{"category":"3.11","date":"2026-09-05","downloads":13},{"category":"3.11","date":"2026-09-06","downloads":24},{"category":"3.11","date":"2026-09-07","downloads":38},{"category":"3.11","date":"2026-09-08","downloads":31},{"category":"3.11","date":"2026-09-09","downloads":13},{"category":"3.11","date":"2026-09-10","downloads":16},{"category":"3.11","date":"2026-09-11","downloads":13},{"category":"3.11","date":"2026-09-12","downloads":7},{"category":"3.11","date":"2026-09-13","downloads":10},{"category":"3.11","date":"2026-09-14","downloads":17},{"category":"3.11","date":"2026-09-15","downloads":25},{"category":"3.11","date":"2026-09-16","downloads":16},{"category":"3.11","date":"2026-09-17","downloads":15},{"category":"3.11","date":"2026-09-18","downloads":21},{"category":"3.11","date":"2026-09-19","downloads":29},{"category":"3.11","date":"2026-09-20","downloads":30},{"category":"3.11","date":"2026-09-21","downloads":34},{"category":"3.11","date":"2026-09-22","downloads":46},{"category":"3.11","date":"2026-09-23","downloads":31},{"category":"3.11","date":"2026-09-24","downloads":56},{"category":"3.11","date":"2026-09-25","downloads":46},{"category":"3.11","date":"2026-09-26","downloads":18},{"category":"3.11","date":"2026-09-27","downloads":6},{"category":"3.11","date":"2026-09-28","downloads":43},{"category":"3.11","date":"2026-09-29","downloads":30},{"category":"3.11","date":"2026-09-30","downloads":33},{"category":"3.11","date":"2026-10-01","downloads":32},{"category":"3.11","date":"2026-10-02","downloads":65},{"category":"3.11","date":"2026-10-03","downloads":33},{"category":"3.11","date":"2026-10-04","downloads":11},{"category":"3.11","date":"2026-10-05","downloads":17},{"category":"3.11","date":"2026-10-06","downloads":31},{"category":"3.11","date":"2026-10-07","downloads":42},{"category":"3.11","date":"2026-10-08","downloads":32},{"category":"3.11","date":"2026-10-09","downloads":69},{"category":"3.11","date":"2026-10-10","downloads":18},{"category":"3.11","date":"2026-10-11","downloads":10},{"category":"3.11","date":"2026-10-12","downloads":27},{"category":"3.11","date":"2026-10-13","downloads":24},{"category":"3.11","date":"2026-10-14","downloads":44},{"category":"3.11","date":"2026-10-15","downloads":47},{"category":"3.11","date":"2026-10-16","downloads":20},{"category":"3.11","date":"2026-10-17","downloads":17},{"category":"3.11","date":"2026-10-18","downloads":23},{"category":"3.11","date":"2026-10-19","downloads":36},{"category":"3.11","date":"2026-10-20","downloads":92},{"category":"3.11","date":"2026-10-21","downloads":74},{"category":"3.11","date":"2026-10-22","downloads":64},{"category":"3.11","date":"2026-10-23","downloads":64},{"category":"3.11","date":"2026-10-24","downloads":47},{"category":"3.11","date":"2026-10-25","downloads":53},{"category":"3.11","date":"2026-10-26","downloads":86},{"category":"3.11","date":"2026-10-27","downloads":120},{"category":"3.11","date":"2026-10-28","downloads":146},{"category":"3.12","date":"2026-04-08","downloads":6},{"category":"3.12","date":"2026-04-09","downloads":10},{"category":"3.12","date":"2026-04-10","downloads":94},{"category":"3.12","date":"2026-04-11","downloads":31},{"category":"3.12","date":"2026-04-12","downloads":7},{"category":"3.12","date":"2026-04-13","downloads":85},{"category":"3.12","date":"2026-04-14","downloads":36},{"category":"3.12","date":"2026-04-15","downloads":220},{"category":"3.12","date":"2026-04-16","downloads":128},{"category":"3.12","date":"2026-04-17","downloads":18},{"category":"3.12","date":"2026-04-18","downloads":42},{"category":"3.12","date":"2026-04-19","downloads":3},{"category":"3.12","date":"2026-04-20","downloads":48},{"category":"3.12","date":"2026-04-21","downloads":67},{"category":"3.12","date":"2026-04-22","downloads":95},{"category":"3.12","date":"2026-04-23","downloads":130},{"category":"3.12","date":"2026-04-24","downloads":112},{"category":"3.12","date":"2026-04-25","downloads":21},{"category":"3.12","date":"2026-04-26","downloads":40},{"category":"3.12","date":"2026-04-27","downloads":106},{"category":"3.12","date":"2026-04-28","downloads":86},{"category":"3.12","date":"2026-04-29","downloads":222},{"category":"3.12","date":"2026-04-30","downloads":631},{"category":"3.12","date":"2026-05-01","downloads":333},{"category":"3.12","date":"2026-05-02","downloads":240},{"category":"3.12","date":"2026-05-03","downloads":309},{"category":"3.12","date":"2026-05-04","downloads":351},{"category":"3.12","date":"2026-05-05","downloads":522},{"category":"3.12","date":"2026-05-06","downloads":511},{"category":"3.12","date":"2026-05-07","downloads":383},{"category":"3.12","date":"2026-05-08","downloads":627},{"category":"3.12","date":"2026-05-09","downloads":281},{"category":"3.12","date":"2026-05-10","downloads":270},{"category":"3.12","date":"2026-05-11","downloads":538},{"category":"3.12","date":"2026-05-12","downloads":592},{"category":"3.12","date":"2026-05-13","downloads":709},{"category":"3.12","date":"2026-05-14","downloads":428},{"category":"3.12","date":"2026-05-15","downloads":638},{"category":"3.12","date":"2026-05-16","downloads":365},{"category":"3.12","date":"2026-05-17","downloads":255},{"category":"3.12","date":"2026-05-18","downloads":599},{"category":"3.12","date":"2026-05-19","downloads":660},{"category":"3.12","date":"2026-05-20","downloads":559},{"category":"3.12","date":"2026-05-21","downloads":913},{"category":"3.12","date":"2026-05-22","downloads":807},{"category":"3.12","date":"2026-05-23","downloads":145},{"category":"3.12","date":"2026-05-24","downloads":224},{"category":"3.12","date":"2026-05-25","downloads":669},{"category":"3.12","date":"2026-05-26","downloads":485},{"category":"3.12","date":"2026-05-27","downloads":1081},{"category":"3.12","date":"2026-05-28","downloads":1329},{"category":"3.12","date":"2026-05-29","downloads":925},{"category":"3.12","date":"2026-05-30","downloads":1089},{"category":"3.12","date":"2026-05-31","downloads":553},{"category":"3.12","date":"2026-06-01","downloads":1451},{"category":"3.12","date":"2026-06-02","downloads":2720},{"category":"3.12","date":"2026-06-03","downloads":2011},{"category":"3.12","date":"2026-06-04","downloads":2578},{"category":"3.12","date":"2026-06-05","downloads":2068},{"category":"3.12","date":"2026-06-06","downloads":1382},{"category":"3.12","date":"2026-06-07","downloads":1073},{"category":"3.12","date":"2026-06-08","downloads":1946},{"category":"3.12","date":"2026-06-09","downloads":3153},{"category":"3.12","date":"2026-06-10","downloads":2522},{"category":"3.12","date":"2026-06-11","downloads":3308},{"category":"3.12","date":"2026-06-12","downloads":2837},{"category":"3.12","date":"2026-06-13","downloads":1733},{"category":"3.12","date":"2026-06-14","downloads":1743},{"category":"3.12","date":"2026-06-15","downloads":2056},{"category":"3.12","date":"2026-06-16","downloads":2460},{"category":"3.12","date":"2026-06-17","downloads":2067},{"category":"3.12","date":"2026-06-18","downloads":1671},{"category":"3.12","date":"2026-06-19","downloads":1389},{"category":"3.12","date":"2026-06-20","downloads":1193},{"category":"3.12","date":"2026-06-21","downloads":1685},{"category":"3.12","date":"2026-06-22","downloads":2306},{"category":"3.12","date":"2026-06-23","downloads":2131},{"category":"3.12","date":"2026-06-24","downloads":2055},{"category":"3.12","date":"2026-06-25","downloads":2597},{"category":"3.12","date":"2026-06-26","downloads":3674},{"category":"3.12","date":"2026-06-27","downloads":1444},{"category":"3.12","date":"2026-06-28","downloads":1229},{"category":"3.12","date":"2026-06-29","downloads":3113},{"category":"3.12","date":"2026-06-30","downloads":3288},{"category":"3.12","date":"2026-07-01","downloads":2153},{"category":"3.12","date":"2026-07-02","downloads":2661},{"category":"3.12","date":"2026-07-03","downloads":2895},{"category":"3.12","date":"2026-07-04","downloads":1501},{"category":"3.12","date":"2026-07-05","downloads":1031},{"category":"3.12","date":"2026-07-06","downloads":2831},{"category":"3.12","date":"2026-07-07","downloads":3486},{"category":"3.12","date":"2026-07-08","downloads":3053},{"category":"3.12","date":"2026-07-09","downloads":3235},{"category":"3.12","date":"2026-07-10","downloads":2785},{"category":"3.12","date":"2026-07-11","downloads":1668},{"category":"3.12","date":"2026-07-12","downloads":1404},{"category":"3.12","date":"2026-07-13","downloads":4442},{"category":"3.12","date":"2026-07-14","downloads":7136},{"category":"3.12","date":"2026-07-15","downloads":3990},{"category":"3.12","date":"2026-07-16","downloads":3087},{"category":"3.12","date":"2026-07-17","downloads":4876},{"category":"3.12","date":"2026-07-18","downloads":2342},{"category":"3.12","date":"2026-07-19","downloads":1584},{"category":"3.12","date":"2026-07-20","downloads":3430},{"category":"3.12","date":"2026-07-21","downloads":9332},{"category":"3.12","date":"2026-07-22","downloads":3509},{"category":"3.12","date":"2026-07-23","downloads":4153},{"category":"3.12","date":"2026-07-24","downloads":4450},{"category":"3.12","date":"2026-07-25","downloads":1816},{"category":"3.12","date":"2026-07-26","downloads":2013},{"category":"3.12","date":"2026-07-27","downloads":3601},{"category":"3.12","date":"2026-07-28","downloads":5151},{"category":"3.12","date":"2026-07-29","downloads":7555},{"category":"3.12","date":"2026-07-30","downloads":6804},{"category":"3.12","date":"2026-07-31","downloads":7548},{"category":"3.12","date":"2026-08-01","downloads":8715},{"category":"3.12","date":"2026-08-02","downloads":4373},{"category":"3.12","date":"2026-08-03","downloads":7289},{"category":"3.12","date":"2026-08-04","downloads":10318},{"category":"3.12","date":"2026-08-05","downloads":5969},{"category":"3.12","date":"2026-08-06","downloads":5117},{"category":"3.12","date":"2026-08-07","downloads":6692},{"category":"3.12","date":"2026-08-08","downloads":7036},{"category":"3.12","date":"2026-08-09","downloads":4690},{"category":"3.12","date":"2026-08-10","downloads":6138},{"category":"3.12","date":"2026-08-11","downloads":4790},{"category":"3.12","date":"2026-08-12","downloads":6807},{"category":"3.12","date":"2026-08-13","downloads":4929},{"category":"3.12","date":"2026-08-14","downloads":6252},{"category":"3.12","date":"2026-08-15","downloads":3426},{"category":"3.12","date":"2026-08-16","downloads":4048},{"category":"3.12","date":"2026-08-17","downloads":5564},{"category":"3.12","date":"2026-08-18","downloads":5594},{"category":"3.12","date":"2026-08-19","downloads":5845},{"category":"3.12","date":"2026-08-20","downloads":7512},{"category":"3.12","date":"2026-08-21","downloads":7573},{"category":"3.12","date":"2026-08-22","downloads":3659},{"category":"3.12","date":"2026-08-23","downloads":2285},{"category":"3.12","date":"2026-08-24","downloads":2688},{"category":"3.12","date":"2026-08-25","downloads":3117},{"category":"3.12","date":"2026-08-26","downloads":2426},{"category":"3.12","date":"2026-08-27","downloads":2300},{"category":"3.12","date":"2026-08-28","downloads":2500},{"category":"3.12","date":"2026-08-29","downloads":1450},{"category":"3.12","date":"2026-08-30","downloads":1779},{"category":"3.12","date":"2026-08-31","downloads":2538},{"category":"3.12","date":"2026-09-01","downloads":2866},{"category":"3.12","date":"2026-09-02","downloads":3653},{"category":"3.12","date":"2026-09-03","downloads":3793},{"category":"3.12","date":"2026-09-04","downloads":2469},{"category":"3.12","date":"2026-09-05","downloads":1775},{"category":"3.12","date":"2026-09-06","downloads":1383},{"category":"3.12","date":"2026-09-07","downloads":1744},{"category":"3.12","date":"2026-09-08","downloads":2505},{"category":"3.12","date":"2026-09-09","downloads":2815},{"category":"3.12","date":"2026-09-10","downloads":2406},{"category":"3.12","date":"2026-09-11","downloads":3052},{"category":"3.12","date":"2026-09-12","downloads":1514},{"category":"3.12","date":"2026-09-13","downloads":1627},{"category":"3.12","date":"2026-09-14","downloads":2626},{"category":"3.12","date":"2026-09-15","downloads":3344},{"category":"3.12","date":"2026-09-16","downloads":3238},{"category":"3.12","date":"2026-09-17","downloads":3741},{"category":"3.12","date":"2026-09-18","downloads":4466},{"category":"3.12","date":"2026-09-19","downloads":2196},{"category":"3.12","date":"2026-09-20","downloads":2407},{"category":"3.12","date":"2026-09-21","downloads":4003},{"category":"3.12","date":"2026-09-22","downloads":4149},{"category":"3.12","date":"2026-09-23","downloads":4063},{"category":"3.12","date":"2026-09-24","downloads":4721},{"category":"3.12","date":"2026-09-25","downloads":3412},{"category":"3.12","date":"2026-09-26","downloads":1721},{"category":"3.12","date":"2026-09-27","downloads":1668},{"category":"3.12","date":"2026-09-28","downloads":2557},{"category":"3.12","date":"2026-09-29","downloads":3825},{"category":"3.12","date":"2026-09-30","downloads":4888},{"category":"3.12","date":"2026-10-01","downloads":4789},{"category":"3.12","date":"2026-10-02","downloads":5045},{"category":"3.12","date":"2026-10-03","downloads":3886},{"category":"3.12","date":"2026-10-04","downloads":3066},{"category":"3.12","date":"2026-10-05","downloads":5334},{"category":"3.12","date":"2026-10-06","downloads":6118},{"category":"3.12","date":"2026-10-07","downloads":6734},{"category":"3.12","date":"2026-10-08","downloads":9068},{"category":"3.12","date":"2026-10-09","downloads":9492},{"category":"3.12","date":"2026-10-10","downloads":5679},{"category":"3.12","date":"2026-10-11","downloads":3381},{"category":"3.12","date":"2026-10-12","downloads":6855},{"category":"3.12","date":"2026-10-13","downloads":7551},{"category":"3.12","date":"2026-10-14","downloads":8731},{"category":"3.12","date":"2026-10-15","downloads":8912},{"category":"3.12","date":"2026-10-16","downloads":6491},{"category":"3.12","date":"2026-10-17","downloads":4694},{"category":"3.12","date":"2026-10-18","downloads":4977},{"category":"3.12","date":"2026-10-19","downloads":6835},{"category":"3.12","date":"2026-10-20","downloads":7805},{"category":"3.12","date":"2026-10-21","downloads":7863},{"category":"3.12","date":"2026-10-22","downloads":6531},{"category":"3.12","date":"2026-10-23","downloads":7037},{"category":"3.12","date":"2026-10-24","downloads":4495},{"category":"3.12","date":"2026-10-25","downloads":3673},{"category":"3.12","date":"2026-10-26","downloads":7200},{"category":"3.12","date":"2026-10-27","downloads":12771},{"category":"3.12","date":"2026-10-28","downloads":8071},{"category":"3.13","date":"2026-04-09","downloads":2},{"category":"3.13","date":"2026-04-27","downloads":2},{"category":"3.13","date":"2026-04-30","downloads":1},{"category":"3.13","date":"2026-05-01","downloads":1},{"category":"3.13","date":"2026-05-02","downloads":4},{"category":"3.13","date":"2026-05-03","downloads":1},{"category":"3.13","date":"2026-05-04","downloads":1},{"category":"3.13","date":"2026-05-06","downloads":1},{"category":"3.13","date":"2026-05-09","downloads":1},{"category":"3.13","date":"2026-05-10","downloads":2},{"category":"3.13","date":"2026-05-11","downloads":1},{"category":"3.13","date":"2026-05-12","downloads":2},{"category":"3.13","date":"2026-05-14","downloads":2},{"category":"3.13","date":"2026-05-15","downloads":8},{"category":"3.13","date":"2026-05-16","downloads":4},{"category":"3.13","date":"2026-05-17","downloads":3},{"category":"3.13","date":"2026-05-18","downloads":21},{"category":"3.13","date":"2026-05-19","downloads":5},{"category":"3.13","date":"2026-05-21","downloads":273},{"category":"3.13","date":"2026-05-22","downloads":99},{"category":"3.13","date":"2026-05-23","downloads":16},{"category":"3.13","date":"2026-05-24","downloads":146},{"category":"3.13","date":"2026-05-25","downloads":58},{"category":"3.13","date":"2026-05-26","downloads":160},{"category":"3.13","date":"2026-05-27","downloads":801},{"category":"3.13","date":"2026-05-28","downloads":332},{"category":"3.13","date":"2026-05-29","downloads":223},{"category":"3.13","date":"2026-05-30","downloads":108},{"category":"3.13","date":"2026-05-31","downloads":122},{"category":"3.13","date":"2026-06-01","downloads":381},{"category":"3.13","date":"2026-06-02","downloads":256},{"category":"3.13","date":"2026-06-03","downloads":1046},{"category":"3.13","date":"2026-06-04","downloads":793},{"category":"3.13","date":"2026-06-05","downloads":1116},{"category":"3.13","date":"2026-06-06","downloads":286},{"category":"3.13","date":"2026-06-07","downloads":114},{"category":"3.13","date":"2026-06-08","downloads":339},{"category":"3.13","date":"2026-06-09","downloads":287},{"category":"3.13","date":"2026-06-10","downloads":215},{"category":"3.13","date":"2026-06-11","downloads":294},{"category":"3.13","date":"2026-06-12","downloads":422},{"category":"3.13","date":"2026-06-13","downloads":124},{"category":"3.13","date":"2026-06-14","downloads":127},{"category":"3.13","date":"2026-06-15","downloads":224},{"category":"3.13","date":"2026-06-16","downloads":253},{"category":"3.13","date":"2026-06-17","downloads":143},{"category":"3.13","date":"2026-06-18","downloads":115},{"category":"3.13","date":"2026-06-19","downloads":70},{"category":"3.13","date":"2026-06-20","downloads":89},{"category":"3.13","date":"2026-06-21","downloads":86},{"category":"3.13","date":"2026-06-22","downloads":288},{"category":"3.13","date":"2026-06-23","downloads":869},{"category":"3.13","date":"2026-06-24","downloads":585},{"category":"3.13","date":"2026-06-25","downloads":536},{"category":"3.13","date":"2026-06-26","downloads":290},{"category":"3.13","date":"2026-06-27","downloads":184},{"category":"3.13","date":"2026-06-28","downloads":249},{"category":"3.13","date":"2026-06-29","downloads":451},{"category":"3.13","date":"2026-06-30","downloads":368},{"category":"3.13","date":"2026-07-01","downloads":530},{"category":"3.13","date":"2026-07-02","downloads":370},{"category":"3.13","date":"2026-07-03","downloads":302},{"category":"3.13","date":"2026-07-04","downloads":338},{"category":"3.13","date":"2026-07-05","downloads":1014},{"category":"3.13","date":"2026-07-06","downloads":306},{"category":"3.13","date":"2026-07-07","downloads":429},{"category":"3.13","date":"2026-07-08","downloads":895},{"category":"3.13","date":"2026-07-09","downloads":459},{"category":"3.13","date":"2026-07-10","downloads":447},{"category":"3.13","date":"2026-07-11","downloads":659},{"category":"3.13","date":"2026-07-12","downloads":1029},{"category":"3.13","date":"2026-07-13","downloads":1109},{"category":"3.13","date":"2026-07-14","downloads":1232},{"category":"3.13","date":"2026-07-15","downloads":1582},{"category":"3.13","date":"2026-07-16","downloads":1297},{"category":"3.13","date":"2026-07-17","downloads":1084},{"category":"3.13","date":"2026-07-18","downloads":713},{"category":"3.13","date":"2026-07-19","downloads":661},{"category":"3.13","date":"2026-07-20","downloads":2190},{"category":"3.13","date":"2026-07-21","downloads":1439},{"category":"3.13","date":"2026-07-22","downloads":1574},{"category":"3.13","date":"2026-07-23","downloads":2075},{"category":"3.13","date":"2026-07-24","downloads":3150},{"category":"3.13","date":"2026-07-25","downloads":1783},{"category":"3.13","date":"2026-07-26","downloads":1285},{"category":"3.13","date":"2026-07-27","downloads":2017},{"category":"3.13","date":"2026-07-28","downloads":2169},{"category":"3.13","date":"2026-07-29","downloads":1659},{"category":"3.13","date":"2026-07-30","downloads":2096},{"category":"3.13","date":"2026-07-31","downloads":1636},{"category":"3.13","date":"2026-08-01","downloads":1082},{"category":"3.13","date":"2026-08-02","downloads":688},{"category":"3.13","date":"2026-08-03","downloads":1293},{"category":"3.13","date":"2026-08-04","downloads":2021},{"category":"3.13","date":"2026-08-05","downloads":1266},{"category":"3.13","date":"2026-08-06","downloads":1607},{"category":"3.13","date":"2026-08-07","downloads":1478},{"category":"3.13","date":"2026-08-08","downloads":1144},{"category":"3.13","date":"2026-08-09","downloads":718},{"category":"3.13","date":"2026-08-10","downloads":1004},{"category":"3.13","date":"2026-08-11","downloads":1462},{"category":"3.13","date":"2026-08-12","downloads":1660},{"category":"3.13","date":"2026-08-13","downloads":1958},{"category":"3.13","date":"2026-08-14","downloads":2556},{"category":"3.13","date":"2026-08-15","downloads":544},{"category":"3.13","date":"2026-08-16","downloads":728},{"category":"3.13","date":"2026-08-17","downloads":1745},{"category":"3.13","date":"2026-08-18","downloads":2019},{"category":"3.13","date":"2026-08-19","downloads":1709},{"category":"3.13","date":"2026-08-20","downloads":1544},{"category":"3.13","date":"2026-08-21","downloads":1772},{"category":"3.13","date":"2026-08-22","downloads":1194},{"category":"3.13","date":"2026-08-23","downloads":1076},{"category":"3.13","date":"2026-08-24","downloads":1069},{"category":"3.13","date":"2026-08-25","downloads":978},{"category":"3.13","date":"2026-08-26","downloads":1321},{"category":"3.13","date":"2026-08-27","downloads":904},{"category":"3.13","date":"2026-08-28","downloads":1187},{"category":"3.13","date":"2026-08-29","downloads":489},{"category":"3.13","date":"2026-08-30","downloads":433},{"category":"3.13","date":"2026-08-31","downloads":641},{"category":"3.13","date":"2026-09-01","downloads":859},{"category":"3.13","date":"2026-09-02","downloads":881},{"category":"3.13","date":"2026-09-03","downloads":590},{"category":"3.13","date":"2026-09-04","downloads":979},{"category":"3.13","date":"2026-09-05","downloads":1576},{"category":"3.13","date":"2026-09-06","downloads":1211},{"category":"3.13","date":"2026-09-07","downloads":625},{"category":"3.13","date":"2026-09-08","downloads":1066},{"category":"3.13","date":"2026-09-09","downloads":1556},{"category":"3.13","date":"2026-09-10","downloads":881},{"category":"3.13","date":"2026-09-11","downloads":783},{"category":"3.13","date":"2026-09-12","downloads":625},{"category":"3.13","date":"2026-09-13","downloads":439},{"category":"3.13","date":"2026-09-14","downloads":754},{"category":"3.13","date":"2026-09-15","downloads":822},{"category":"3.13","date":"2026-09-16","downloads":681},{"category":"3.13","date":"2026-09-17","downloads":1044},{"category":"3.13","date":"2026-09-18","downloads":1443},{"category":"3.13","date":"2026-09-19","downloads":825},{"category":"3.13","date":"2026-09-20","downloads":527},{"category":"3.13","date":"2026-09-21","downloads":1210},{"category":"3.13","date":"2026-09-22","downloads":1392},{"category":"3.13","date":"2026-09-23","downloads":1171},{"category":"3.13","date":"2026-09-24","downloads":3173},{"category":"3.13","date":"2026-09-25","downloads":1223},{"category":"3.13","date":"2026-09-26","downloads":1188},{"category":"3.13","date":"2026-09-27","downloads":1147},{"category":"3.13","date":"2026-09-28","downloads":1674},{"category":"3.13","date":"2026-09-29","downloads":1508},{"category":"3.13","date":"2026-09-30","downloads":2154},{"category":"3.13","date":"2026-10-01","downloads":2441},{"category":"3.13","date":"2026-10-02","downloads":2301},{"category":"3.13","date":"2026-10-03","downloads":1635},{"category":"3.13","date":"2026-10-04","downloads":1415},{"category":"3.13","date":"2026-10-05","downloads":3127},{"category":"3.13","date":"2026-10-06","downloads":3354},{"category":"3.13","date":"2026-10-07","downloads":2414},{"category":"3.13","date":"2026-10-08","downloads":2487},{"category":"3.13","date":"2026-10-09","downloads":3930},{"category":"3.13","date":"2026-10-10","downloads":3219},{"category":"3.13","date":"2026-10-11","downloads":2733},{"category":"3.13","date":"2026-10-12","downloads":5286},{"category":"3.13","date":"2026-10-13","downloads":4685},{"category":"3.13","date":"2026-10-14","downloads":4042},{"category":"3.13","date":"2026-10-15","downloads":2619},{"category":"3.13","date":"2026-10-16","downloads":3004},{"category":"3.13","date":"2026-10-17","downloads":1580},{"category":"3.13","date":"2026-10-18","downloads":1009},{"category":"3.13","date":"2026-10-19","downloads":1555},{"category":"3.13","date":"2026-10-20","downloads":1071},{"category":"3.13","date":"2026-10-21","downloads":3567},{"category":"3.13","date":"2026-10-22","downloads":1541},{"category":"3.13","date":"2026-10-23","downloads":1866},{"category":"3.13","date":"2026-10-24","downloads":6320},{"category":"3.13","date":"2026-10-25","downloads":4659},{"category":"3.13","date":"2026-10-26","downloads":4194},{"category":"3.13","date":"2026-10-27","downloads":4742},{"category":"3.13","date":"2026-10-28","downloads":6567},{"category":"3.14","date":"2026-06-02","downloads":12},{"category":"3.14","date":"2026-06-05","downloads":2},{"category":"3.14","date":"2026-06-08","downloads":16},{"category":"3.14","date":"2026-06-09","downloads":64},{"category":"3.14","date":"2026-06-10","downloads":48},{"category":"3.14","date":"2026-06-11","downloads":64},{"category":"3.14","date":"2026-06-12","downloads":1},{"category":"3.14","date":"2026-06-14","downloads":1},{"category":"3.14","date":"2026-06-16","downloads":4},{"category":"3.14","date":"2026-06-18","downloads":3},{"category":"3.14","date":"2026-06-22","downloads":25},{"category":"3.14","date":"2026-07-03","downloads":1},{"category":"3.14","date":"2026-07-15","downloads":2},{"category":"3.14","date":"2026-07-18","downloads":1},{"category":"3.14","date":"2026-07-20","downloads":2},{"category":"3.14","date":"2026-07-22","downloads":2},{"category":"3.14","date":"2026-07-25","downloads":1},{"category":"3.14","date":"2026-07-27","downloads":1},{"category":"3.14","date":"2026-07-28","downloads":1},{"category":"3.14","date":"2026-07-29","downloads":2},{"category":"3.14","date":"2026-07-30","downloads":3},{"category":"3.14","date":"2026-07-31","downloads":1},{"category":"3.14","date":"2026-08-04","downloads":4},{"category":"3.14","date":"2026-08-14","downloads":2},{"category":"3.14","date":"2026-08-15","downloads":2},{"category":"3.14","date":"2026-08-17","downloads":1},{"category":"3.14","date":"2026-08-23","downloads":2},{"category":"3.14","date":"2026-08-24","downloads":1},{"category":"3.14","date":"2026-08-28","downloads":2},{"category":"3.14","date":"2026-09-19","downloads":1},{"category":"3.14","date":"2026-09-23","downloads":1},{"category":"3.14","date":"2026-09-26","downloads":1},{"category":"3.14","date":"2026-09-28","downloads":2},{"category":"3.14","date":"2026-09-30","downloads":3},{"category":"3.14","date":"2026-10-03","downloads":1},{"category":"3.14","date":"2026-10-05","downloads":1},{"category":"3.14","date":"2026-10-06","downloads":2},{"category":"3.14","date":"2026-10-07","downloads":1},{"category":"3.14","date":"2026-10-09","downloads":1},{"category":"3.14","date":"2026-10-11","downloads":2},{"category":"3.14","date":"2026-10-12","downloads":2},{"category":"3.14","date":"2026-10-13","downloads":2},{"category":"3.14","date":"2026-10-14","downloads":1},{"category":"3.14","date":"2026-10-22","downloads":2},{"category":"3.14","date":"2026-10-24","downloads":1},{"category":"3.14","date":"2026-10-25","downloads":1},{"category":"3.14","date":"2026-10-26","downloads":2},{"category":"3.7","date":"2026-08-13","downloads":4},{"category":"3.8","date":"2026-07-18","downloads":1},{"category":"3.8","date":"2026-08-27","downloads":2},{"category":"3.8","date":"2026-08-31","downloads":1},{"category":"3.8","date":"2026-10-08","downloads":2},{"category":"3.8","date":"2026-10-26","downloads":1},{"category":"3.9","date":"2026-06-30","downloads":1},{"category":"3.9","date":"2026-07-15","downloads":1},{"category":"3.9","date":"2026-08-07","downloads":1},{"category":"3.9","date":"2026-08-11","downloads":1},{"category":"3.9","date":"2026-08-12","downloads":1},{"category":"3.9","date":"2026-08-26","downloads":1},{"category":"3.9","date":"2026-08-27","downloads":1},{"category":"3.9","date":"2026-08-28","downloads":1},{"category":"3.9","date":"2026-08-29","downloads":1},{"category":"3.9","date":"2026-09-01","downloads":10},{"category":"3.9","date":"2026-09-02","downloads":11},{"category":"3.9","date":"2026-09-10","downloads":2},{"category":"3.9","date":"2026-09-14","downloads":1},{"category":"3.9","date":"2026-09-15","downloads":1},{"category":"3.9","date":"2026-09-19","downloads":1},{"category":"3.9","date":"2026-10-06","downloads":1},{"category":"3.9","date":"2026-10-13","downloads":1},{"category":"3.9","date":"2026-10-19","downloads":1},{"category":"3.9","date":"2026-10-20","downloads":1},{"category":"3.9","date":"2026-10-23","downloads":1},{"category":"null","date":"2026-04-09","downloads":6},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":11},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":40},{"category":"null","date":"2026-04-15","downloads":4},{"category":"null","date":"2026-04-16","downloads":1},{"category":"null","date":"2026-04-19","downloads":16},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-21","downloads":4},{"category":"null","date":"2026-04-22","downloads":192},{"category":"null","date":"2026-04-23","downloads":14},{"category":"null","date":"2026-04-24","downloads":14},{"category":"null","date":"2026-04-25","downloads":23},{"category":"null","date":"2026-04-26","downloads":2},{"category":"null","date":"2026-04-27","downloads":9},{"category":"null","date":"2026-04-28","downloads":1},{"category":"null","date":"2026-04-29","downloads":4},{"category":"null","date":"2026-04-30","downloads":9},{"category":"null","date":"2026-05-02","downloads":2},{"category":"null","date":"2026-05-05","downloads":48},{"category":"null","date":"2026-05-06","downloads":8},{"category":"null","date":"2026-05-07","downloads":8},{"category":"null","date":"2026-05-08","downloads":1},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":10},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":4},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":9},{"category":"null","date":"2026-05-19","downloads":2},{"category":"null","date":"2026-05-21","downloads":179},{"category":"null","date":"2026-05-22","downloads":20},{"category":"null","date":"2026-05-23","downloads":28},{"category":"null","date":"2026-05-24","downloads":178},{"category":"null","date":"2026-05-25","downloads":33},{"category":"null","date":"2026-05-26","downloads":14},{"category":"null","date":"2026-05-27","downloads":25},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":4},{"category":"null","date":"2026-05-31","downloads":17},{"category":"null","date":"2026-06-01","downloads":2},{"category":"null","date":"2026-06-02","downloads":17},{"category":"null","date":"2026-06-03","downloads":35},{"category":"null","date":"2026-06-04","downloads":10},{"category":"null","date":"2026-06-05","downloads":37},{"category":"null","date":"2026-06-06","downloads":45},{"category":"null","date":"2026-06-08","downloads":11},{"category":"null","date":"2026-06-09","downloads":35},{"category":"null","date":"2026-06-10","downloads":24},{"category":"null","date":"2026-06-11","downloads":29},{"category":"null","date":"2026-06-12","downloads":7},{"category":"null","date":"2026-06-13","downloads":18},{"category":"null","date":"2026-06-14","downloads":3},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":21},{"category":"null","date":"2026-06-17","downloads":16},{"category":"null","date":"2026-06-18","downloads":2},{"category":"null","date":"2026-06-19","downloads":55},{"category":"null","date":"2026-06-20","downloads":20},{"category":"null","date":"2026-06-21","downloads":12},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":45},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":1},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-28","downloads":2},{"category":"null","date":"2026-06-29","downloads":7},{"category":"null","date":"2026-06-30","downloads":8},{"category":"null","date":"2026-07-01","downloads":10},{"category":"null","date":"2026-07-02","downloads":3},{"category":"null","date":"2026-07-03","downloads":3},{"category":"null","date":"2026-07-04","downloads":2},{"category":"null","date":"2026-07-05","downloads":6},{"category":"null","date":"2026-07-06","downloads":4},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":2},{"category":"null","date":"2026-07-09","downloads":3},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":5},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":7},{"category":"null","date":"2026-07-15","downloads":18},{"category":"null","date":"2026-07-16","downloads":6},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":2},{"category":"null","date":"2026-07-19","downloads":2},{"category":"null","date":"2026-07-20","downloads":10},{"category":"null","date":"2026-07-21","downloads":4},{"category":"null","date":"2026-07-22","downloads":52},{"category":"null","date":"2026-07-23","downloads":2},{"category":"null","date":"2026-07-24","downloads":211},{"category":"null","date":"2026-07-25","downloads":205},{"category":"null","date":"2026-07-26","downloads":29},{"category":"null","date":"2026-07-27","downloads":24},{"category":"null","date":"2026-07-28","downloads":10},{"category":"null","date":"2026-07-29","downloads":10},{"category":"null","date":"2026-07-30","downloads":19},{"category":"null","date":"2026-07-31","downloads":61},{"category":"null","date":"2026-08-01","downloads":20},{"category":"null","date":"2026-08-03","downloads":26},{"category":"null","date":"2026-08-04","downloads":43},{"category":"null","date":"2026-08-05","downloads":16},{"category":"null","date":"2026-08-06","downloads":24},{"category":"null","date":"2026-08-07","downloads":9},{"category":"null","date":"2026-08-08","downloads":7},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-10","downloads":10},{"category":"null","date":"2026-08-11","downloads":15},{"category":"null","date":"2026-08-12","downloads":33},{"category":"null","date":"2026-08-13","downloads":12},{"category":"null","date":"2026-08-14","downloads":24},{"category":"null","date":"2026-08-15","downloads":4},{"category":"null","date":"2026-08-16","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":11},{"category":"null","date":"2026-08-19","downloads":9},{"category":"null","date":"2026-08-20","downloads":19},{"category":"null","date":"2026-08-21","downloads":7},{"category":"null","date":"2026-08-22","downloads":4},{"category":"null","date":"2026-08-23","downloads":5},{"category":"null","date":"2026-08-24","downloads":16},{"category":"null","date":"2026-08-25","downloads":14},{"category":"null","date":"2026-08-26","downloads":193},{"category":"null","date":"2026-08-27","downloads":169},{"category":"null","date":"2026-08-28","downloads":95},{"category":"null","date":"2026-08-29","downloads":26},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":266},{"category":"null","date":"2026-09-01","downloads":76},{"category":"null","date":"2026-09-02","downloads":121},{"category":"null","date":"2026-09-03","downloads":47},{"category":"null","date":"2026-09-04","downloads":38},{"category":"null","date":"2026-09-05","downloads":34},{"category":"null","date":"2026-09-06","downloads":15},{"category":"null","date":"2026-09-07","downloads":16},{"category":"null","date":"2026-09-08","downloads":22},{"category":"null","date":"2026-09-09","downloads":31},{"category":"null","date":"2026-09-10","downloads":13},{"category":"null","date":"2026-09-11","downloads":63},{"category":"null","date":"2026-09-12","downloads":11},{"category":"null","date":"2026-09-13","downloads":14},{"category":"null","date":"2026-09-14","downloads":43},{"category":"null","date":"2026-09-15","downloads":14},{"category":"null","date":"2026-09-16","downloads":17},{"category":"null","date":"2026-09-17","downloads":20},{"category":"null","date":"2026-09-18","downloads":12},{"category":"null","date":"2026-09-19","downloads":99},{"category":"null","date":"2026-09-20","downloads":12},{"category":"null","date":"2026-09-21","downloads":34},{"category":"null","date":"2026-09-22","downloads":13},{"category":"null","date":"2026-09-23","downloads":41},{"category":"null","date":"2026-09-24","downloads":16},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":33},{"category":"null","date":"2026-09-27","downloads":101},{"category":"null","date":"2026-09-28","downloads":33},{"category":"null","date":"2026-09-29","downloads":63},{"category":"null","date":"2026-09-30","downloads":34},{"category":"null","date":"2026-10-01","downloads":1255},{"category":"null","date":"2026-10-02","downloads":6938},{"category":"null","date":"2026-10-03","downloads":2951},{"category":"null","date":"2026-10-04","downloads":1078},{"category":"null","date":"2026-10-05","downloads":2338},{"category":"null","date":"2026-10-06","downloads":2284},{"category":"null","date":"2026-10-07","downloads":8785},{"category":"null","date":"2026-10-08","downloads":9068},{"category":"null","date":"2026-10-09","downloads":7699},{"category":"null","date":"2026-10-10","downloads":2145},{"category":"null","date":"2026-10-11","downloads":1637},{"category":"null","date":"2026-10-12","downloads":4283},{"category":"null","date":"2026-10-13","downloads":7049},{"category":"null","date":"2026-10-14","downloads":8176},{"category":"null","date":"2026-10-15","downloads":9131},{"category":"null","date":"2026-10-16","downloads":7909},{"category":"null","date":"2026-10-17","downloads":3761},{"category":"null","date":"2026-10-18","downloads":2309},{"category":"null","date":"2026-10-19","downloads":6203},{"category":"null","date":"2026-10-20","downloads":7825},{"category":"null","date":"2026-10-21","downloads":8567},{"category":"null","date":"2026-10-22","downloads":6015},{"category":"null","date":"2026-10-23","downloads":4735},{"category":"null","date":"2026-10-24","downloads":2482},{"category":"null","date":"2026-10-25","downloads":1119},{"category":"null","date":"2026-10-26","downloads":5281},{"category":"null","date":"2026-10-27","downloads":9146},{"category":"null","date":"2026-10-28","downloads":7897}],"package":"mooncake-transfer-engine-cuda13","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:11:45.762220+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-cuda13/system","payload":{"data":[{"category":"Darwin","date":"2026-04-08","downloads":2},{"category":"Darwin","date":"2026-04-20","downloads":8},{"category":"Darwin","date":"2026-04-22","downloads":8},{"category":"Darwin","date":"2026-05-01","downloads":1},{"category":"Darwin","date":"2026-05-04","downloads":1},{"category":"Darwin","date":"2026-05-05","downloads":3},{"category":"Darwin","date":"2026-05-06","downloads":1},{"category":"Darwin","date":"2026-05-08","downloads":1},{"category":"Darwin","date":"2026-05-09","downloads":1},{"category":"Darwin","date":"2026-05-10","downloads":2},{"category":"Darwin","date":"2026-05-12","downloads":1},{"category":"Darwin","date":"2026-05-13","downloads":1},{"category":"Darwin","date":"2026-05-16","downloads":28},{"category":"Darwin","date":"2026-05-18","downloads":2},{"category":"Darwin","date":"2026-05-19","downloads":1},{"category":"Darwin","date":"2026-05-21","downloads":11},{"category":"Darwin","date":"2026-05-22","downloads":10},{"category":"Darwin","date":"2026-05-24","downloads":16},{"category":"Darwin","date":"2026-05-26","downloads":2},{"category":"Darwin","date":"2026-05-27","downloads":1},{"category":"Darwin","date":"2026-05-28","downloads":1},{"category":"Darwin","date":"2026-05-29","downloads":2},{"category":"Darwin","date":"2026-05-31","downloads":2},{"category":"Darwin","date":"2026-06-01","downloads":3},{"category":"Darwin","date":"2026-06-02","downloads":12},{"category":"Darwin","date":"2026-06-03","downloads":8},{"category":"Darwin","date":"2026-06-04","downloads":12},{"category":"Darwin","date":"2026-06-05","downloads":9},{"category":"Darwin","date":"2026-06-06","downloads":2},{"category":"Darwin","date":"2026-06-07","downloads":4},{"category":"Darwin","date":"2026-06-08","downloads":11},{"category":"Darwin","date":"2026-06-09","downloads":10},{"category":"Darwin","date":"2026-06-10","downloads":10},{"category":"Darwin","date":"2026-06-11","downloads":11},{"category":"Darwin","date":"2026-06-12","downloads":11},{"category":"Darwin","date":"2026-06-13","downloads":3},{"category":"Darwin","date":"2026-06-14","downloads":3},{"category":"Darwin","date":"2026-06-15","downloads":3},{"category":"Darwin","date":"2026-06-16","downloads":8},{"category":"Darwin","date":"2026-06-17","downloads":13},{"category":"Darwin","date":"2026-06-18","downloads":5},{"category":"Darwin","date":"2026-06-19","downloads":5},{"category":"Darwin","date":"2026-06-20","downloads":6},{"category":"Darwin","date":"2026-06-21","downloads":3},{"category":"Darwin","date":"2026-06-22","downloads":8},{"category":"Darwin","date":"2026-06-23","downloads":4},{"category":"Darwin","date":"2026-06-24","downloads":5},{"category":"Darwin","date":"2026-06-25","downloads":14},{"category":"Darwin","date":"2026-06-26","downloads":11},{"category":"Darwin","date":"2026-06-27","downloads":2},{"category":"Darwin","date":"2026-06-28","downloads":4},{"category":"Darwin","date":"2026-06-29","downloads":8},{"category":"Darwin","date":"2026-06-30","downloads":10},{"category":"Darwin","date":"2026-07-01","downloads":9},{"category":"Darwin","date":"2026-07-02","downloads":8},{"category":"Darwin","date":"2026-07-03","downloads":5},{"category":"Darwin","date":"2026-07-04","downloads":2},{"category":"Darwin","date":"2026-07-05","downloads":1},{"category":"Darwin","date":"2026-07-06","downloads":16},{"category":"Darwin","date":"2026-07-07","downloads":7},{"category":"Darwin","date":"2026-07-08","downloads":16},{"category":"Darwin","date":"2026-07-09","downloads":15},{"category":"Darwin","date":"2026-07-10","downloads":10},{"category":"Darwin","date":"2026-07-11","downloads":11},{"category":"Darwin","date":"2026-07-12","downloads":5},{"category":"Darwin","date":"2026-07-13","downloads":11},{"category":"Darwin","date":"2026-07-14","downloads":16},{"category":"Darwin","date":"2026-07-15","downloads":16},{"category":"Darwin","date":"2026-07-16","downloads":19},{"category":"Darwin","date":"2026-07-17","downloads":9},{"category":"Darwin","date":"2026-07-18","downloads":6},{"category":"Darwin","date":"2026-07-19","downloads":5},{"category":"Darwin","date":"2026-07-20","downloads":18},{"category":"Darwin","date":"2026-07-21","downloads":16},{"category":"Darwin","date":"2026-07-22","downloads":16},{"category":"Darwin","date":"2026-07-23","downloads":10},{"category":"Darwin","date":"2026-07-24","downloads":42},{"category":"Darwin","date":"2026-07-25","downloads":28},{"category":"Darwin","date":"2026-07-26","downloads":11},{"category":"Darwin","date":"2026-07-27","downloads":15},{"category":"Darwin","date":"2026-07-28","downloads":19},{"category":"Darwin","date":"2026-07-29","downloads":20},{"category":"Darwin","date":"2026-07-30","downloads":16},{"category":"Darwin","date":"2026-07-31","downloads":19},{"category":"Darwin","date":"2026-08-01","downloads":3},{"category":"Darwin","date":"2026-08-02","downloads":4},{"category":"Darwin","date":"2026-08-03","downloads":16},{"category":"Darwin","date":"2026-08-04","downloads":11},{"category":"Darwin","date":"2026-08-05","downloads":12},{"category":"Darwin","date":"2026-08-06","downloads":9},{"category":"Darwin","date":"2026-08-07","downloads":17},{"category":"Darwin","date":"2026-08-08","downloads":12},{"category":"Darwin","date":"2026-08-09","downloads":7},{"category":"Darwin","date":"2026-08-10","downloads":8},{"category":"Darwin","date":"2026-08-11","downloads":13},{"category":"Darwin","date":"2026-08-12","downloads":17},{"category":"Darwin","date":"2026-08-13","downloads":15},{"category":"Darwin","date":"2026-08-14","downloads":11},{"category":"Darwin","date":"2026-08-15","downloads":9},{"category":"Darwin","date":"2026-08-16","downloads":4},{"category":"Darwin","date":"2026-08-17","downloads":4},{"category":"Darwin","date":"2026-08-18","downloads":10},{"category":"Darwin","date":"2026-08-19","downloads":15},{"category":"Darwin","date":"2026-08-20","downloads":17},{"category":"Darwin","date":"2026-08-21","downloads":8},{"category":"Darwin","date":"2026-08-22","downloads":4},{"category":"Darwin","date":"2026-08-23","downloads":5},{"category":"Darwin","date":"2026-08-24","downloads":10},{"category":"Darwin","date":"2026-08-26","downloads":24},{"category":"Darwin","date":"2026-08-27","downloads":3},{"category":"Darwin","date":"2026-08-28","downloads":3},{"category":"Darwin","date":"2026-08-29","downloads":1},{"category":"Darwin","date":"2026-08-31","downloads":41},{"category":"Darwin","date":"2026-09-02","downloads":1},{"category":"Darwin","date":"2026-09-06","downloads":2},{"category":"Darwin","date":"2026-09-09","downloads":1},{"category":"Darwin","date":"2026-09-10","downloads":2},{"category":"Darwin","date":"2026-09-14","downloads":2},{"category":"Darwin","date":"2026-09-15","downloads":1},{"category":"Darwin","date":"2026-09-19","downloads":1},{"category":"Darwin","date":"2026-09-23","downloads":1},{"category":"Darwin","date":"2026-09-24","downloads":1},{"category":"Darwin","date":"2026-09-25","downloads":2},{"category":"Darwin","date":"2026-09-26","downloads":1},{"category":"Darwin","date":"2026-09-28","downloads":1},{"category":"Darwin","date":"2026-09-29","downloads":2},{"category":"Darwin","date":"2026-10-01","downloads":1},{"category":"Darwin","date":"2026-10-06","downloads":3},{"category":"Darwin","date":"2026-10-07","downloads":1},{"category":"Darwin","date":"2026-10-08","downloads":2},{"category":"Darwin","date":"2026-10-09","downloads":1},{"category":"Darwin","date":"2026-10-10","downloads":1},{"category":"Darwin","date":"2026-10-11","downloads":2},{"category":"Darwin","date":"2026-10-12","downloads":1},{"category":"Darwin","date":"2026-10-13","downloads":1},{"category":"Darwin","date":"2026-10-14","downloads":3},{"category":"Darwin","date":"2026-10-17","downloads":1},{"category":"Darwin","date":"2026-10-19","downloads":2},{"category":"Darwin","date":"2026-10-20","downloads":2},{"category":"Darwin","date":"2026-10-21","downloads":1},{"category":"Darwin","date":"2026-10-22","downloads":2},{"category":"Darwin","date":"2026-10-23","downloads":3},{"category":"Darwin","date":"2026-10-25","downloads":1},{"category":"Darwin","date":"2026-10-26","downloads":2},{"category":"Darwin","date":"2026-10-27","downloads":1},{"category":"Darwin","date":"2026-10-28","downloads":1},{"category":"Linux","date":"2026-04-08","downloads":6},{"category":"Linux","date":"2026-04-09","downloads":12},{"category":"Linux","date":"2026-04-10","downloads":102},{"category":"Linux","date":"2026-04-11","downloads":42},{"category":"Linux","date":"2026-04-12","downloads":12},{"category":"Linux","date":"2026-04-13","downloads":88},{"category":"Linux","date":"2026-04-14","downloads":36},{"category":"Linux","date":"2026-04-15","downloads":222},{"category":"Linux","date":"2026-04-16","downloads":134},{"category":"Linux","date":"2026-04-17","downloads":18},{"category":"Linux","date":"2026-04-18","downloads":176},{"category":"Linux","date":"2026-04-19","downloads":63},{"category":"Linux","date":"2026-04-20","downloads":56},{"category":"Linux","date":"2026-04-21","downloads":89},{"category":"Linux","date":"2026-04-22","downloads":190},{"category":"Linux","date":"2026-04-23","downloads":134},{"category":"Linux","date":"2026-04-24","downloads":130},{"category":"Linux","date":"2026-04-25","downloads":24},{"category":"Linux","date":"2026-04-26","downloads":40},{"category":"Linux","date":"2026-04-27","downloads":112},{"category":"Linux","date":"2026-04-28","downloads":110},{"category":"Linux","date":"2026-04-29","downloads":230},{"category":"Linux","date":"2026-04-30","downloads":681},{"category":"Linux","date":"2026-05-01","downloads":336},{"category":"Linux","date":"2026-05-02","downloads":330},{"category":"Linux","date":"2026-05-03","downloads":318},{"category":"Linux","date":"2026-05-04","downloads":356},{"category":"Linux","date":"2026-05-05","downloads":580},{"category":"Linux","date":"2026-05-06","downloads":517},{"category":"Linux","date":"2026-05-07","downloads":390},{"category":"Linux","date":"2026-05-08","downloads":631},{"category":"Linux","date":"2026-05-09","downloads":288},{"category":"Linux","date":"2026-05-10","downloads":271},{"category":"Linux","date":"2026-05-11","downloads":544},{"category":"Linux","date":"2026-05-12","downloads":619},{"category":"Linux","date":"2026-05-13","downloads":728},{"category":"Linux","date":"2026-05-14","downloads":432},{"category":"Linux","date":"2026-05-15","downloads":646},{"category":"Linux","date":"2026-05-16","downloads":371},{"category":"Linux","date":"2026-05-17","downloads":258},{"category":"Linux","date":"2026-05-18","downloads":622},{"category":"Linux","date":"2026-05-19","downloads":695},{"category":"Linux","date":"2026-05-20","downloads":563},{"category":"Linux","date":"2026-05-21","downloads":1246},{"category":"Linux","date":"2026-05-22","downloads":970},{"category":"Linux","date":"2026-05-23","downloads":166},{"category":"Linux","date":"2026-05-24","downloads":438},{"category":"Linux","date":"2026-05-25","downloads":778},{"category":"Linux","date":"2026-05-26","downloads":660},{"category":"Linux","date":"2026-05-27","downloads":1915},{"category":"Linux","date":"2026-05-28","downloads":1666},{"category":"Linux","date":"2026-05-29","downloads":1151},{"category":"Linux","date":"2026-05-30","downloads":1199},{"category":"Linux","date":"2026-05-31","downloads":675},{"category":"Linux","date":"2026-06-01","downloads":1839},{"category":"Linux","date":"2026-06-02","downloads":2977},{"category":"Linux","date":"2026-06-03","downloads":3060},{"category":"Linux","date":"2026-06-04","downloads":3359},{"category":"Linux","date":"2026-06-05","downloads":3182},{"category":"Linux","date":"2026-06-06","downloads":1667},{"category":"Linux","date":"2026-06-07","downloads":1183},{"category":"Linux","date":"2026-06-08","downloads":2296},{"category":"Linux","date":"2026-06-09","downloads":3557},{"category":"Linux","date":"2026-06-10","downloads":2889},{"category":"Linux","date":"2026-06-11","downloads":3731},{"category":"Linux","date":"2026-06-12","downloads":3304},{"category":"Linux","date":"2026-06-13","downloads":1893},{"category":"Linux","date":"2026-06-14","downloads":1890},{"category":"Linux","date":"2026-06-15","downloads":2301},{"category":"Linux","date":"2026-06-16","downloads":2741},{"category":"Linux","date":"2026-06-17","downloads":2222},{"category":"Linux","date":"2026-06-18","downloads":1811},{"category":"Linux","date":"2026-06-19","downloads":1472},{"category":"Linux","date":"2026-06-20","downloads":1293},{"category":"Linux","date":"2026-06-21","downloads":1823},{"category":"Linux","date":"2026-06-22","downloads":2650},{"category":"Linux","date":"2026-06-23","downloads":3022},{"category":"Linux","date":"2026-06-24","downloads":2693},{"category":"Linux","date":"2026-06-25","downloads":3138},{"category":"Linux","date":"2026-06-26","downloads":3962},{"category":"Linux","date":"2026-06-27","downloads":1643},{"category":"Linux","date":"2026-06-28","downloads":1483},{"category":"Linux","date":"2026-06-29","downloads":3576},{"category":"Linux","date":"2026-06-30","downloads":3651},{"category":"Linux","date":"2026-07-01","downloads":2701},{"category":"Linux","date":"2026-07-02","downloads":3088},{"category":"Linux","date":"2026-07-03","downloads":3207},{"category":"Linux","date":"2026-07-04","downloads":1847},{"category":"Linux","date":"2026-07-05","downloads":2058},{"category":"Linux","date":"2026-07-06","downloads":3161},{"category":"Linux","date":"2026-07-07","downloads":3931},{"category":"Linux","date":"2026-07-08","downloads":3956},{"category":"Linux","date":"2026-07-09","downloads":3758},{"category":"Linux","date":"2026-07-10","downloads":3344},{"category":"Linux","date":"2026-07-11","downloads":2376},{"category":"Linux","date":"2026-07-12","downloads":2443},{"category":"Linux","date":"2026-07-13","downloads":5588},{"category":"Linux","date":"2026-07-14","downloads":8434},{"category":"Linux","date":"2026-07-15","downloads":5672},{"category":"Linux","date":"2026-07-16","downloads":4452},{"category":"Linux","date":"2026-07-17","downloads":6014},{"category":"Linux","date":"2026-07-18","downloads":3081},{"category":"Linux","date":"2026-07-19","downloads":2314},{"category":"Linux","date":"2026-07-20","downloads":5647},{"category":"Linux","date":"2026-07-21","downloads":10842},{"category":"Linux","date":"2026-07-22","downloads":5341},{"category":"Linux","date":"2026-07-23","downloads":6478},{"category":"Linux","date":"2026-07-24","downloads":7858},{"category":"Linux","date":"2026-07-25","downloads":3842},{"category":"Linux","date":"2026-07-26","downloads":3364},{"category":"Linux","date":"2026-07-27","downloads":5729},{"category":"Linux","date":"2026-07-28","downloads":7380},{"category":"Linux","date":"2026-07-29","downloads":9246},{"category":"Linux","date":"2026-07-30","downloads":8934},{"category":"Linux","date":"2026-07-31","downloads":9222},{"category":"Linux","date":"2026-08-01","downloads":9832},{"category":"Linux","date":"2026-08-02","downloads":5108},{"category":"Linux","date":"2026-08-03","downloads":8641},{"category":"Linux","date":"2026-08-04","downloads":12458},{"category":"Linux","date":"2026-08-05","downloads":7328},{"category":"Linux","date":"2026-08-06","downloads":6798},{"category":"Linux","date":"2026-08-07","downloads":8240},{"category":"Linux","date":"2026-08-08","downloads":8302},{"category":"Linux","date":"2026-08-09","downloads":5491},{"category":"Linux","date":"2026-08-10","downloads":7274},{"category":"Linux","date":"2026-08-11","downloads":6339},{"category":"Linux","date":"2026-08-12","downloads":8534},{"category":"Linux","date":"2026-08-13","downloads":6932},{"category":"Linux","date":"2026-08-14","downloads":8940},{"category":"Linux","date":"2026-08-15","downloads":4062},{"category":"Linux","date":"2026-08-16","downloads":4842},{"category":"Linux","date":"2026-08-17","downloads":7402},{"category":"Linux","date":"2026-08-18","downloads":7664},{"category":"Linux","date":"2026-08-19","downloads":7582},{"category":"Linux","date":"2026-08-20","downloads":9130},{"category":"Linux","date":"2026-08-21","downloads":9454},{"category":"Linux","date":"2026-08-22","downloads":4927},{"category":"Linux","date":"2026-08-23","downloads":3421},{"category":"Linux","date":"2026-08-24","downloads":3891},{"category":"Linux","date":"2026-08-25","downloads":4178},{"category":"Linux","date":"2026-08-26","downloads":3876},{"category":"Linux","date":"2026-08-27","downloads":3316},{"category":"Linux","date":"2026-08-28","downloads":3742},{"category":"Linux","date":"2026-08-29","downloads":1983},{"category":"Linux","date":"2026-08-30","downloads":2232},{"category":"Linux","date":"2026-08-31","downloads":3225},{"category":"Linux","date":"2026-09-01","downloads":3835},{"category":"Linux","date":"2026-09-02","downloads":4827},{"category":"Linux","date":"2026-09-03","downloads":4498},{"category":"Linux","date":"2026-09-04","downloads":3494},{"category":"Linux","date":"2026-09-05","downloads":3385},{"category":"Linux","date":"2026-09-06","downloads":2628},{"category":"Linux","date":"2026-09-07","downloads":2428},{"category":"Linux","date":"2026-09-08","downloads":3608},{"category":"Linux","date":"2026-09-09","downloads":4407},{"category":"Linux","date":"2026-09-10","downloads":3324},{"category":"Linux","date":"2026-09-11","downloads":3872},{"category":"Linux","date":"2026-09-12","downloads":2156},{"category":"Linux","date":"2026-09-13","downloads":2089},{"category":"Linux","date":"2026-09-14","downloads":3412},{"category":"Linux","date":"2026-09-15","downloads":4203},{"category":"Linux","date":"2026-09-16","downloads":3946},{"category":"Linux","date":"2026-09-17","downloads":4832},{"category":"Linux","date":"2026-09-18","downloads":5952},{"category":"Linux","date":"2026-09-19","downloads":3051},{"category":"Linux","date":"2026-09-20","downloads":2980},{"category":"Linux","date":"2026-09-21","downloads":5263},{"category":"Linux","date":"2026-09-22","downloads":5607},{"category":"Linux","date":"2026-09-23","downloads":5296},{"category":"Linux","date":"2026-09-24","downloads":7968},{"category":"Linux","date":"2026-09-25","downloads":4697},{"category":"Linux","date":"2026-09-26","downloads":2939},{"category":"Linux","date":"2026-09-27","downloads":2828},{"category":"Linux","date":"2026-09-28","downloads":4297},{"category":"Linux","date":"2026-09-29","downloads":5390},{"category":"Linux","date":"2026-09-30","downloads":7143},{"category":"Linux","date":"2026-10-01","downloads":7296},{"category":"Linux","date":"2026-10-02","downloads":7443},{"category":"Linux","date":"2026-10-03","downloads":5584},{"category":"Linux","date":"2026-10-04","downloads":4506},{"category":"Linux","date":"2026-10-05","downloads":8505},{"category":"Linux","date":"2026-10-06","downloads":9535},{"category":"Linux","date":"2026-10-07","downloads":9228},{"category":"Linux","date":"2026-10-08","downloads":11654},{"category":"Linux","date":"2026-10-09","downloads":13554},{"category":"Linux","date":"2026-10-10","downloads":8945},{"category":"Linux","date":"2026-10-11","downloads":6153},{"category":"Linux","date":"2026-10-12","downloads":12224},{"category":"Linux","date":"2026-10-13","downloads":12324},{"category":"Linux","date":"2026-10-14","downloads":12880},{"category":"Linux","date":"2026-10-15","downloads":11629},{"category":"Linux","date":"2026-10-16","downloads":9569},{"category":"Linux","date":"2026-10-17","downloads":6350},{"category":"Linux","date":"2026-10-18","downloads":6045},{"category":"Linux","date":"2026-10-19","downloads":8490},{"category":"Linux","date":"2026-10-20","downloads":9071},{"category":"Linux","date":"2026-10-21","downloads":11594},{"category":"Linux","date":"2026-10-22","downloads":8190},{"category":"Linux","date":"2026-10-23","downloads":9021},{"category":"Linux","date":"2026-10-24","downloads":10883},{"category":"Linux","date":"2026-10-25","downloads":8402},{"category":"Linux","date":"2026-10-26","downloads":11591},{"category":"Linux","date":"2026-10-27","downloads":17671},{"category":"Linux","date":"2026-10-28","downloads":14828},{"category":"Windows","date":"2026-05-03","downloads":1},{"category":"Windows","date":"2026-05-04","downloads":1},{"category":"Windows","date":"2026-05-06","downloads":1},{"category":"Windows","date":"2026-05-09","downloads":1},{"category":"Windows","date":"2026-05-18","downloads":1},{"category":"Windows","date":"2026-05-20","downloads":1},{"category":"Windows","date":"2026-05-21","downloads":1},{"category":"Windows","date":"2026-05-25","downloads":6},{"category":"Windows","date":"2026-05-26","downloads":1},{"category":"Windows","date":"2026-06-02","downloads":1},{"category":"Windows","date":"2026-06-04","downloads":1},{"category":"Windows","date":"2026-06-05","downloads":1},{"category":"Windows","date":"2026-06-10","downloads":1},{"category":"Windows","date":"2026-06-11","downloads":1},{"category":"Windows","date":"2026-06-16","downloads":2},{"category":"Windows","date":"2026-06-17","downloads":1},{"category":"Windows","date":"2026-06-22","downloads":1},{"category":"Windows","date":"2026-06-25","downloads":1},{"category":"Windows","date":"2026-06-26","downloads":1},{"category":"Windows","date":"2026-06-28","downloads":1},{"category":"Windows","date":"2026-06-29","downloads":1},{"category":"Windows","date":"2026-07-01","downloads":2},{"category":"Windows","date":"2026-07-03","downloads":1},{"category":"Windows","date":"2026-07-04","downloads":1},{"category":"Windows","date":"2026-07-06","downloads":1},{"category":"Windows","date":"2026-07-11","downloads":1},{"category":"Windows","date":"2026-07-14","downloads":1},{"category":"Windows","date":"2026-07-17","downloads":1},{"category":"Windows","date":"2026-07-18","downloads":1},{"category":"Windows","date":"2026-07-19","downloads":2},{"category":"Windows","date":"2026-07-26","downloads":1},{"category":"Windows","date":"2026-07-27","downloads":1},{"category":"Windows","date":"2026-07-28","downloads":1},{"category":"Windows","date":"2026-07-29","downloads":1},{"category":"Windows","date":"2026-07-30","downloads":2},{"category":"Windows","date":"2026-07-31","downloads":1},{"category":"Windows","date":"2026-08-01","downloads":1},{"category":"Windows","date":"2026-08-10","downloads":1},{"category":"Windows","date":"2026-08-11","downloads":8},{"category":"Windows","date":"2026-08-13","downloads":5},{"category":"Windows","date":"2026-08-15","downloads":5},{"category":"Windows","date":"2026-08-17","downloads":1},{"category":"Windows","date":"2026-08-19","downloads":8},{"category":"Windows","date":"2026-08-20","downloads":1},{"category":"Windows","date":"2026-08-21","downloads":1},{"category":"Windows","date":"2026-08-22","downloads":2},{"category":"Windows","date":"2026-08-23","downloads":3},{"category":"Windows","date":"2026-08-31","downloads":1},{"category":"Windows","date":"2026-09-28","downloads":1},{"category":"Windows","date":"2026-09-30","downloads":1},{"category":"Windows","date":"2026-10-09","downloads":4},{"category":"Windows","date":"2026-10-12","downloads":1},{"category":"Windows","date":"2026-10-15","downloads":1},{"category":"Windows","date":"2026-10-17","downloads":1},{"category":"Windows","date":"2026-10-18","downloads":1},{"category":"Windows","date":"2026-10-26","downloads":1},{"category":"null","date":"2026-04-09","downloads":6},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":11},{"category":"null","date":"2026-04-13","downloads":9},{"category":"null","date":"2026-04-14","downloads":40},{"category":"null","date":"2026-04-15","downloads":4},{"category":"null","date":"2026-04-16","downloads":1},{"category":"null","date":"2026-04-19","downloads":16},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-21","downloads":4},{"category":"null","date":"2026-04-22","downloads":192},{"category":"null","date":"2026-04-23","downloads":14},{"category":"null","date":"2026-04-24","downloads":14},{"category":"null","date":"2026-04-25","downloads":23},{"category":"null","date":"2026-04-26","downloads":2},{"category":"null","date":"2026-04-27","downloads":9},{"category":"null","date":"2026-04-28","downloads":1},{"category":"null","date":"2026-04-29","downloads":4},{"category":"null","date":"2026-04-30","downloads":9},{"category":"null","date":"2026-05-02","downloads":2},{"category":"null","date":"2026-05-05","downloads":48},{"category":"null","date":"2026-05-06","downloads":8},{"category":"null","date":"2026-05-07","downloads":8},{"category":"null","date":"2026-05-08","downloads":1},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":10},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":4},{"category":"null","date":"2026-05-17","downloads":6},{"category":"null","date":"2026-05-18","downloads":9},{"category":"null","date":"2026-05-19","downloads":2},{"category":"null","date":"2026-05-21","downloads":179},{"category":"null","date":"2026-05-22","downloads":20},{"category":"null","date":"2026-05-23","downloads":28},{"category":"null","date":"2026-05-24","downloads":178},{"category":"null","date":"2026-05-25","downloads":33},{"category":"null","date":"2026-05-26","downloads":14},{"category":"null","date":"2026-05-27","downloads":25},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":4},{"category":"null","date":"2026-05-31","downloads":17},{"category":"null","date":"2026-06-01","downloads":2},{"category":"null","date":"2026-06-02","downloads":17},{"category":"null","date":"2026-06-03","downloads":35},{"category":"null","date":"2026-06-04","downloads":10},{"category":"null","date":"2026-06-05","downloads":37},{"category":"null","date":"2026-06-06","downloads":45},{"category":"null","date":"2026-06-08","downloads":11},{"category":"null","date":"2026-06-09","downloads":35},{"category":"null","date":"2026-06-10","downloads":24},{"category":"null","date":"2026-06-11","downloads":29},{"category":"null","date":"2026-06-12","downloads":7},{"category":"null","date":"2026-06-13","downloads":18},{"category":"null","date":"2026-06-14","downloads":3},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":21},{"category":"null","date":"2026-06-17","downloads":16},{"category":"null","date":"2026-06-18","downloads":2},{"category":"null","date":"2026-06-19","downloads":55},{"category":"null","date":"2026-06-20","downloads":20},{"category":"null","date":"2026-06-21","downloads":12},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":45},{"category":"null","date":"2026-06-24","downloads":11},{"category":"null","date":"2026-06-25","downloads":1},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-28","downloads":2},{"category":"null","date":"2026-06-29","downloads":7},{"category":"null","date":"2026-06-30","downloads":8},{"category":"null","date":"2026-07-01","downloads":10},{"category":"null","date":"2026-07-02","downloads":3},{"category":"null","date":"2026-07-03","downloads":3},{"category":"null","date":"2026-07-04","downloads":2},{"category":"null","date":"2026-07-05","downloads":6},{"category":"null","date":"2026-07-06","downloads":4},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":2},{"category":"null","date":"2026-07-09","downloads":3},{"category":"null","date":"2026-07-10","downloads":6},{"category":"null","date":"2026-07-11","downloads":5},{"category":"null","date":"2026-07-12","downloads":1},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":7},{"category":"null","date":"2026-07-15","downloads":18},{"category":"null","date":"2026-07-16","downloads":6},{"category":"null","date":"2026-07-17","downloads":6},{"category":"null","date":"2026-07-18","downloads":2},{"category":"null","date":"2026-07-19","downloads":2},{"category":"null","date":"2026-07-20","downloads":10},{"category":"null","date":"2026-07-21","downloads":4},{"category":"null","date":"2026-07-22","downloads":52},{"category":"null","date":"2026-07-23","downloads":2},{"category":"null","date":"2026-07-24","downloads":211},{"category":"null","date":"2026-07-25","downloads":205},{"category":"null","date":"2026-07-26","downloads":29},{"category":"null","date":"2026-07-27","downloads":24},{"category":"null","date":"2026-07-28","downloads":10},{"category":"null","date":"2026-07-29","downloads":10},{"category":"null","date":"2026-07-30","downloads":19},{"category":"null","date":"2026-07-31","downloads":61},{"category":"null","date":"2026-08-01","downloads":20},{"category":"null","date":"2026-08-03","downloads":26},{"category":"null","date":"2026-08-04","downloads":43},{"category":"null","date":"2026-08-05","downloads":16},{"category":"null","date":"2026-08-06","downloads":24},{"category":"null","date":"2026-08-07","downloads":9},{"category":"null","date":"2026-08-08","downloads":7},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-10","downloads":10},{"category":"null","date":"2026-08-11","downloads":15},{"category":"null","date":"2026-08-12","downloads":33},{"category":"null","date":"2026-08-13","downloads":12},{"category":"null","date":"2026-08-14","downloads":24},{"category":"null","date":"2026-08-15","downloads":4},{"category":"null","date":"2026-08-16","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":11},{"category":"null","date":"2026-08-19","downloads":9},{"category":"null","date":"2026-08-20","downloads":19},{"category":"null","date":"2026-08-21","downloads":7},{"category":"null","date":"2026-08-22","downloads":4},{"category":"null","date":"2026-08-23","downloads":5},{"category":"null","date":"2026-08-24","downloads":16},{"category":"null","date":"2026-08-25","downloads":14},{"category":"null","date":"2026-08-26","downloads":193},{"category":"null","date":"2026-08-27","downloads":169},{"category":"null","date":"2026-08-28","downloads":95},{"category":"null","date":"2026-08-29","downloads":26},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":266},{"category":"null","date":"2026-09-01","downloads":76},{"category":"null","date":"2026-09-02","downloads":121},{"category":"null","date":"2026-09-03","downloads":47},{"category":"null","date":"2026-09-04","downloads":38},{"category":"null","date":"2026-09-05","downloads":34},{"category":"null","date":"2026-09-06","downloads":15},{"category":"null","date":"2026-09-07","downloads":16},{"category":"null","date":"2026-09-08","downloads":22},{"category":"null","date":"2026-09-09","downloads":31},{"category":"null","date":"2026-09-10","downloads":13},{"category":"null","date":"2026-09-11","downloads":63},{"category":"null","date":"2026-09-12","downloads":11},{"category":"null","date":"2026-09-13","downloads":14},{"category":"null","date":"2026-09-14","downloads":43},{"category":"null","date":"2026-09-15","downloads":14},{"category":"null","date":"2026-09-16","downloads":17},{"category":"null","date":"2026-09-17","downloads":20},{"category":"null","date":"2026-09-18","downloads":12},{"category":"null","date":"2026-09-19","downloads":99},{"category":"null","date":"2026-09-20","downloads":12},{"category":"null","date":"2026-09-21","downloads":34},{"category":"null","date":"2026-09-22","downloads":13},{"category":"null","date":"2026-09-23","downloads":41},{"category":"null","date":"2026-09-24","downloads":16},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":33},{"category":"null","date":"2026-09-27","downloads":101},{"category":"null","date":"2026-09-28","downloads":33},{"category":"null","date":"2026-09-29","downloads":63},{"category":"null","date":"2026-09-30","downloads":34},{"category":"null","date":"2026-10-01","downloads":1250},{"category":"null","date":"2026-10-02","downloads":6927},{"category":"null","date":"2026-10-03","downloads":2940},{"category":"null","date":"2026-10-04","downloads":1074},{"category":"null","date":"2026-10-05","downloads":2327},{"category":"null","date":"2026-10-06","downloads":2263},{"category":"null","date":"2026-10-07","downloads":8762},{"category":"null","date":"2026-10-08","downloads":9034},{"category":"null","date":"2026-10-09","downloads":7654},{"category":"null","date":"2026-10-10","downloads":2125},{"category":"null","date":"2026-10-11","downloads":1622},{"category":"null","date":"2026-10-12","downloads":4254},{"category":"null","date":"2026-10-13","downloads":7002},{"category":"null","date":"2026-10-14","downloads":8127},{"category":"null","date":"2026-10-15","downloads":9086},{"category":"null","date":"2026-10-16","downloads":7884},{"category":"null","date":"2026-10-17","downloads":3745},{"category":"null","date":"2026-10-18","downloads":2302},{"category":"null","date":"2026-10-19","downloads":6165},{"category":"null","date":"2026-10-20","downloads":7789},{"category":"null","date":"2026-10-21","downloads":8540},{"category":"null","date":"2026-10-22","downloads":5977},{"category":"null","date":"2026-10-23","downloads":4691},{"category":"null","date":"2026-10-24","downloads":2471},{"category":"null","date":"2026-10-25","downloads":1111},{"category":"null","date":"2026-10-26","downloads":5261},{"category":"null","date":"2026-10-27","downloads":9122},{"category":"null","date":"2026-10-28","downloads":7864}],"package":"mooncake-transfer-engine-cuda13","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (CUDA 13 version)"},{"name":"mooncake-transfer-engine-efa","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:11:48.907362+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa/recent","payload":{"data":{"last_day":1,"last_month":166,"last_week":49},"package":"mooncake-transfer-engine-efa","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:11:51.277506+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-07-23","downloads":365},{"category":"with_mirrors","date":"2026-07-24","downloads":353},{"category":"with_mirrors","date":"2026-07-25","downloads":275},{"category":"with_mirrors","date":"2026-07-26","downloads":17},{"category":"with_mirrors","date":"2026-07-27","downloads":63},{"category":"with_mirrors","date":"2026-07-28","downloads":49},{"category":"with_mirrors","date":"2026-07-30","downloads":28},{"category":"with_mirrors","date":"2026-07-31","downloads":20},{"category":"with_mirrors","date":"2026-08-01","downloads":26},{"category":"with_mirrors","date":"2026-08-02","downloads":32},{"category":"with_mirrors","date":"2026-08-03","downloads":66},{"category":"with_mirrors","date":"2026-08-04","downloads":55},{"category":"with_mirrors","date":"2026-08-05","downloads":1},{"category":"with_mirrors","date":"2026-08-06","downloads":12},{"category":"with_mirrors","date":"2026-08-07","downloads":25},{"category":"with_mirrors","date":"2026-08-08","downloads":59},{"category":"with_mirrors","date":"2026-08-09","downloads":12},{"category":"with_mirrors","date":"2026-08-10","downloads":9},{"category":"with_mirrors","date":"2026-08-11","downloads":14},{"category":"with_mirrors","date":"2026-08-12","downloads":3},{"category":"with_mirrors","date":"2026-08-13","downloads":1},{"category":"with_mirrors","date":"2026-08-14","downloads":21},{"category":"with_mirrors","date":"2026-08-15","downloads":16},{"category":"with_mirrors","date":"2026-08-16","downloads":24},{"category":"with_mirrors","date":"2026-08-17","downloads":8},{"category":"with_mirrors","date":"2026-08-18","downloads":11},{"category":"with_mirrors","date":"2026-08-19","downloads":7},{"category":"with_mirrors","date":"2026-08-20","downloads":23},{"category":"with_mirrors","date":"2026-08-21","downloads":49},{"category":"with_mirrors","date":"2026-08-22","downloads":19},{"category":"with_mirrors","date":"2026-08-23","downloads":2},{"category":"with_mirrors","date":"2026-08-24","downloads":2},{"category":"with_mirrors","date":"2026-08-25","downloads":17},{"category":"with_mirrors","date":"2026-08-26","downloads":276},{"category":"with_mirrors","date":"2026-08-27","downloads":102},{"category":"with_mirrors","date":"2026-08-28","downloads":70},{"category":"with_mirrors","date":"2026-08-29","downloads":18},{"category":"with_mirrors","date":"2026-08-30","downloads":26},{"category":"with_mirrors","date":"2026-08-31","downloads":304},{"category":"with_mirrors","date":"2026-09-01","downloads":164},{"category":"with_mirrors","date":"2026-09-02","downloads":36},{"category":"with_mirrors","date":"2026-09-03","downloads":39},{"category":"with_mirrors","date":"2026-09-04","downloads":24},{"category":"with_mirrors","date":"2026-09-05","downloads":27},{"category":"with_mirrors","date":"2026-09-06","downloads":19},{"category":"with_mirrors","date":"2026-09-07","downloads":53},{"category":"with_mirrors","date":"2026-09-08","downloads":39},{"category":"with_mirrors","date":"2026-09-09","downloads":26},{"category":"with_mirrors","date":"2026-09-10","downloads":28},{"category":"with_mirrors","date":"2026-09-11","downloads":10},{"category":"with_mirrors","date":"2026-09-12","downloads":12},{"category":"with_mirrors","date":"2026-09-13","downloads":19},{"category":"with_mirrors","date":"2026-09-14","downloads":5},{"category":"with_mirrors","date":"2026-09-15","downloads":8},{"category":"with_mirrors","date":"2026-09-16","downloads":6},{"category":"with_mirrors","date":"2026-09-17","downloads":8},{"category":"with_mirrors","date":"2026-09-18","downloads":1},{"category":"with_mirrors","date":"2026-09-19","downloads":24},{"category":"with_mirrors","date":"2026-09-20","downloads":64},{"category":"with_mirrors","date":"2026-09-21","downloads":4},{"category":"with_mirrors","date":"2026-09-22","downloads":25},{"category":"with_mirrors","date":"2026-09-23","downloads":2},{"category":"with_mirrors","date":"2026-09-24","downloads":2},{"category":"with_mirrors","date":"2026-09-25","downloads":25},{"category":"with_mirrors","date":"2026-09-26","downloads":16},{"category":"with_mirrors","date":"2026-09-27","downloads":16},{"category":"with_mirrors","date":"2026-09-28","downloads":11},{"category":"with_mirrors","date":"2026-09-29","downloads":29},{"category":"with_mirrors","date":"2026-09-30","downloads":1},{"category":"with_mirrors","date":"2026-10-01","downloads":1},{"category":"with_mirrors","date":"2026-10-02","downloads":41},{"category":"with_mirrors","date":"2026-10-03","downloads":3},{"category":"with_mirrors","date":"2026-10-04","downloads":18},{"category":"with_mirrors","date":"2026-10-05","downloads":4},{"category":"with_mirrors","date":"2026-10-06","downloads":8},{"category":"with_mirrors","date":"2026-10-07","downloads":41},{"category":"with_mirrors","date":"2026-10-08","downloads":4},{"category":"with_mirrors","date":"2026-10-09","downloads":4},{"category":"with_mirrors","date":"2026-10-10","downloads":1},{"category":"with_mirrors","date":"2026-10-11","downloads":16},{"category":"with_mirrors","date":"2026-10-12","downloads":6},{"category":"with_mirrors","date":"2026-10-13","downloads":22},{"category":"with_mirrors","date":"2026-10-14","downloads":31},{"category":"with_mirrors","date":"2026-10-15","downloads":56},{"category":"with_mirrors","date":"2026-10-16","downloads":23},{"category":"with_mirrors","date":"2026-10-17","downloads":17},{"category":"with_mirrors","date":"2026-10-18","downloads":35},{"category":"with_mirrors","date":"2026-10-19","downloads":20},{"category":"with_mirrors","date":"2026-10-20","downloads":39},{"category":"with_mirrors","date":"2026-10-21","downloads":23},{"category":"with_mirrors","date":"2026-10-22","downloads":30},{"category":"with_mirrors","date":"2026-10-23","downloads":17},{"category":"with_mirrors","date":"2026-10-24","downloads":1},{"category":"with_mirrors","date":"2026-10-26","downloads":2},{"category":"with_mirrors","date":"2026-10-27","downloads":3},{"category":"with_mirrors","date":"2026-10-28","downloads":2},{"category":"without_mirrors","date":"2026-07-23","downloads":192},{"category":"without_mirrors","date":"2026-07-24","downloads":142},{"category":"without_mirrors","date":"2026-07-25","downloads":187},{"category":"without_mirrors","date":"2026-07-27","downloads":49},{"category":"without_mirrors","date":"2026-07-28","downloads":2},{"category":"without_mirrors","date":"2026-07-30","downloads":20},{"category":"without_mirrors","date":"2026-07-31","downloads":20},{"category":"without_mirrors","date":"2026-08-01","downloads":2},{"category":"without_mirrors","date":"2026-08-03","downloads":8},{"category":"without_mirrors","date":"2026-08-04","downloads":9},{"category":"without_mirrors","date":"2026-08-05","downloads":1},{"category":"without_mirrors","date":"2026-08-06","downloads":10},{"category":"without_mirrors","date":"2026-08-07","downloads":3},{"category":"without_mirrors","date":"2026-08-08","downloads":56},{"category":"without_mirrors","date":"2026-08-11","downloads":6},{"category":"without_mirrors","date":"2026-08-12","downloads":3},{"category":"without_mirrors","date":"2026-08-13","downloads":1},{"category":"without_mirrors","date":"2026-08-14","downloads":5},{"category":"without_mirrors","date":"2026-08-17","downloads":4},{"category":"without_mirrors","date":"2026-08-18","downloads":11},{"category":"without_mirrors","date":"2026-08-19","downloads":7},{"category":"without_mirrors","date":"2026-08-20","downloads":11},{"category":"without_mirrors","date":"2026-08-21","downloads":25},{"category":"without_mirrors","date":"2026-08-22","downloads":5},{"category":"without_mirrors","date":"2026-08-23","downloads":2},{"category":"without_mirrors","date":"2026-08-26","downloads":125},{"category":"without_mirrors","date":"2026-08-27","downloads":28},{"category":"without_mirrors","date":"2026-08-28","downloads":37},{"category":"without_mirrors","date":"2026-08-29","downloads":8},{"category":"without_mirrors","date":"2026-08-30","downloads":9},{"category":"without_mirrors","date":"2026-08-31","downloads":137},{"category":"without_mirrors","date":"2026-09-01","downloads":13},{"category":"without_mirrors","date":"2026-09-02","downloads":13},{"category":"without_mirrors","date":"2026-09-04","downloads":11},{"category":"without_mirrors","date":"2026-09-05","downloads":9},{"category":"without_mirrors","date":"2026-09-06","downloads":2},{"category":"without_mirrors","date":"2026-09-07","downloads":27},{"category":"without_mirrors","date":"2026-09-08","downloads":4},{"category":"without_mirrors","date":"2026-09-09","downloads":6},{"category":"without_mirrors","date":"2026-09-10","downloads":13},{"category":"without_mirrors","date":"2026-09-11","downloads":3},{"category":"without_mirrors","date":"2026-09-12","downloads":4},{"category":"without_mirrors","date":"2026-09-13","downloads":3},{"category":"without_mirrors","date":"2026-09-14","downloads":4},{"category":"without_mirrors","date":"2026-09-16","downloads":3},{"category":"without_mirrors","date":"2026-09-17","downloads":2},{"category":"without_mirrors","date":"2026-09-19","downloads":24},{"category":"without_mirrors","date":"2026-09-20","downloads":4},{"category":"without_mirrors","date":"2026-09-21","downloads":4},{"category":"without_mirrors","date":"2026-09-23","downloads":1},{"category":"without_mirrors","date":"2026-09-24","downloads":1},{"category":"without_mirrors","date":"2026-09-25","downloads":25},{"category":"without_mirrors","date":"2026-09-27","downloads":16},{"category":"without_mirrors","date":"2026-09-28","downloads":2},{"category":"without_mirrors","date":"2026-09-30","downloads":1},{"category":"without_mirrors","date":"2026-10-02","downloads":13},{"category":"without_mirrors","date":"2026-10-03","downloads":3},{"category":"without_mirrors","date":"2026-10-05","downloads":1},{"category":"without_mirrors","date":"2026-10-06","downloads":4},{"category":"without_mirrors","date":"2026-10-07","downloads":1},{"category":"without_mirrors","date":"2026-10-09","downloads":1},{"category":"without_mirrors","date":"2026-10-10","downloads":1},{"category":"without_mirrors","date":"2026-10-12","downloads":4},{"category":"without_mirrors","date":"2026-10-14","downloads":28},{"category":"without_mirrors","date":"2026-10-15","downloads":8},{"category":"without_mirrors","date":"2026-10-16","downloads":7},{"category":"without_mirrors","date":"2026-10-18","downloads":1},{"category":"without_mirrors","date":"2026-10-19","downloads":3},{"category":"without_mirrors","date":"2026-10-20","downloads":26},{"category":"without_mirrors","date":"2026-10-22","downloads":3},{"category":"without_mirrors","date":"2026-10-23","downloads":17},{"category":"without_mirrors","date":"2026-10-24","downloads":1},{"category":"without_mirrors","date":"2026-10-26","downloads":2},{"category":"without_mirrors","date":"2026-10-27","downloads":3},{"category":"without_mirrors","date":"2026-10-28","downloads":1}],"package":"mooncake-transfer-engine-efa","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:11:53.690367+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa/python_major","payload":{"data":[{"category":"3","date":"2026-07-23","downloads":28},{"category":"3","date":"2026-07-24","downloads":24},{"category":"3","date":"2026-07-25","downloads":182},{"category":"3","date":"2026-07-27","downloads":44},{"category":"3","date":"2026-07-28","downloads":1},{"category":"3","date":"2026-08-03","downloads":6},{"category":"3","date":"2026-08-06","downloads":2},{"category":"3","date":"2026-08-08","downloads":56},{"category":"3","date":"2026-08-11","downloads":5},{"category":"3","date":"2026-08-14","downloads":3},{"category":"3","date":"2026-08-17","downloads":4},{"category":"3","date":"2026-08-18","downloads":5},{"category":"3","date":"2026-08-19","downloads":4},{"category":"3","date":"2026-08-20","downloads":8},{"category":"3","date":"2026-08-23","downloads":2},{"category":"3","date":"2026-08-26","downloads":14},{"category":"3","date":"2026-08-27","downloads":3},{"category":"3","date":"2026-08-28","downloads":4},{"category":"3","date":"2026-08-29","downloads":2},{"category":"3","date":"2026-08-30","downloads":3},{"category":"3","date":"2026-08-31","downloads":21},{"category":"3","date":"2026-09-01","downloads":1},{"category":"3","date":"2026-09-02","downloads":1},{"category":"3","date":"2026-09-04","downloads":4},{"category":"3","date":"2026-09-05","downloads":1},{"category":"3","date":"2026-09-07","downloads":7},{"category":"3","date":"2026-09-10","downloads":3},{"category":"3","date":"2026-09-23","downloads":1},{"category":"3","date":"2026-09-25","downloads":7},{"category":"3","date":"2026-09-30","downloads":1},{"category":"3","date":"2026-10-03","downloads":3},{"category":"3","date":"2026-10-07","downloads":1},{"category":"3","date":"2026-10-09","downloads":1},{"category":"3","date":"2026-10-14","downloads":25},{"category":"3","date":"2026-10-20","downloads":25},{"category":"3","date":"2026-10-22","downloads":2},{"category":"3","date":"2026-10-23","downloads":16},{"category":"3","date":"2026-10-26","downloads":2},{"category":"3","date":"2026-10-27","downloads":3},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":118},{"category":"null","date":"2026-07-25","downloads":5},{"category":"null","date":"2026-07-27","downloads":5},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-30","downloads":20},{"category":"null","date":"2026-07-31","downloads":20},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-03","downloads":2},{"category":"null","date":"2026-08-04","downloads":9},{"category":"null","date":"2026-08-05","downloads":1},{"category":"null","date":"2026-08-06","downloads":8},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":3},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-18","downloads":6},{"category":"null","date":"2026-08-19","downloads":3},{"category":"null","date":"2026-08-20","downloads":3},{"category":"null","date":"2026-08-21","downloads":25},{"category":"null","date":"2026-08-22","downloads":5},{"category":"null","date":"2026-08-26","downloads":111},{"category":"null","date":"2026-08-27","downloads":25},{"category":"null","date":"2026-08-28","downloads":33},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-30","downloads":6},{"category":"null","date":"2026-08-31","downloads":116},{"category":"null","date":"2026-09-01","downloads":12},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":2},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":4},{"category":"null","date":"2026-09-09","downloads":6},{"category":"null","date":"2026-09-10","downloads":10},{"category":"null","date":"2026-09-11","downloads":3},{"category":"null","date":"2026-09-12","downloads":4},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":2},{"category":"null","date":"2026-09-19","downloads":24},{"category":"null","date":"2026-09-20","downloads":4},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":18},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-02","downloads":13},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-10","downloads":1},{"category":"null","date":"2026-10-12","downloads":4},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":8},{"category":"null","date":"2026-10-16","downloads":7},{"category":"null","date":"2026-10-18","downloads":1},{"category":"null","date":"2026-10-19","downloads":3},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1},{"category":"null","date":"2026-10-24","downloads":1},{"category":"null","date":"2026-10-28","downloads":1}],"package":"mooncake-transfer-engine-efa","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:11:56.051740+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa/python_minor","payload":{"data":[{"category":"3.10","date":"2026-08-03","downloads":2},{"category":"3.10","date":"2026-09-07","downloads":1},{"category":"3.11","date":"2026-07-24","downloads":16},{"category":"3.11","date":"2026-08-19","downloads":4},{"category":"3.11","date":"2026-08-20","downloads":2},{"category":"3.11","date":"2026-08-26","downloads":8},{"category":"3.11","date":"2026-08-29","downloads":1},{"category":"3.11","date":"2026-08-31","downloads":16},{"category":"3.11","date":"2026-10-14","downloads":25},{"category":"3.11","date":"2026-10-20","downloads":25},{"category":"3.11","date":"2026-10-23","downloads":16},{"category":"3.12","date":"2026-07-23","downloads":28},{"category":"3.12","date":"2026-07-24","downloads":8},{"category":"3.12","date":"2026-07-25","downloads":178},{"category":"3.12","date":"2026-07-27","downloads":44},{"category":"3.12","date":"2026-07-28","downloads":1},{"category":"3.12","date":"2026-08-03","downloads":4},{"category":"3.12","date":"2026-08-08","downloads":56},{"category":"3.12","date":"2026-08-14","downloads":3},{"category":"3.12","date":"2026-08-17","downloads":4},{"category":"3.12","date":"2026-08-18","downloads":5},{"category":"3.12","date":"2026-08-20","downloads":6},{"category":"3.12","date":"2026-08-23","downloads":2},{"category":"3.12","date":"2026-08-26","downloads":6},{"category":"3.12","date":"2026-08-27","downloads":3},{"category":"3.12","date":"2026-08-28","downloads":4},{"category":"3.12","date":"2026-08-29","downloads":1},{"category":"3.12","date":"2026-08-30","downloads":3},{"category":"3.12","date":"2026-08-31","downloads":5},{"category":"3.12","date":"2026-09-01","downloads":1},{"category":"3.12","date":"2026-09-02","downloads":1},{"category":"3.12","date":"2026-09-04","downloads":4},{"category":"3.12","date":"2026-09-05","downloads":1},{"category":"3.12","date":"2026-09-07","downloads":6},{"category":"3.12","date":"2026-09-10","downloads":3},{"category":"3.12","date":"2026-09-23","downloads":1},{"category":"3.12","date":"2026-09-25","downloads":6},{"category":"3.12","date":"2026-09-30","downloads":1},{"category":"3.12","date":"2026-10-03","downloads":3},{"category":"3.12","date":"2026-10-07","downloads":1},{"category":"3.12","date":"2026-10-09","downloads":1},{"category":"3.12","date":"2026-10-22","downloads":2},{"category":"3.12","date":"2026-10-26","downloads":2},{"category":"3.12","date":"2026-10-27","downloads":1},{"category":"3.13","date":"2026-07-25","downloads":4},{"category":"3.13","date":"2026-08-06","downloads":2},{"category":"3.13","date":"2026-08-11","downloads":5},{"category":"3.13","date":"2026-09-25","downloads":1},{"category":"3.13","date":"2026-10-27","downloads":2},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":118},{"category":"null","date":"2026-07-25","downloads":5},{"category":"null","date":"2026-07-27","downloads":5},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-30","downloads":20},{"category":"null","date":"2026-07-31","downloads":20},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-03","downloads":2},{"category":"null","date":"2026-08-04","downloads":9},{"category":"null","date":"2026-08-05","downloads":1},{"category":"null","date":"2026-08-06","downloads":8},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":3},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-18","downloads":6},{"category":"null","date":"2026-08-19","downloads":3},{"category":"null","date":"2026-08-20","downloads":3},{"category":"null","date":"2026-08-21","downloads":25},{"category":"null","date":"2026-08-22","downloads":5},{"category":"null","date":"2026-08-26","downloads":111},{"category":"null","date":"2026-08-27","downloads":25},{"category":"null","date":"2026-08-28","downloads":33},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-30","downloads":6},{"category":"null","date":"2026-08-31","downloads":116},{"category":"null","date":"2026-09-01","downloads":12},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":2},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":4},{"category":"null","date":"2026-09-09","downloads":6},{"category":"null","date":"2026-09-10","downloads":10},{"category":"null","date":"2026-09-11","downloads":3},{"category":"null","date":"2026-09-12","downloads":4},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":2},{"category":"null","date":"2026-09-19","downloads":24},{"category":"null","date":"2026-09-20","downloads":4},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":18},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-02","downloads":13},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-10","downloads":1},{"category":"null","date":"2026-10-12","downloads":4},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":8},{"category":"null","date":"2026-10-16","downloads":7},{"category":"null","date":"2026-10-18","downloads":1},{"category":"null","date":"2026-10-19","downloads":3},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1},{"category":"null","date":"2026-10-24","downloads":1},{"category":"null","date":"2026-10-28","downloads":1}],"package":"mooncake-transfer-engine-efa","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:11:58.456601+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa/system","payload":{"data":[{"category":"Darwin","date":"2026-07-24","downloads":16},{"category":"Darwin","date":"2026-08-26","downloads":8},{"category":"Darwin","date":"2026-08-31","downloads":16},{"category":"Darwin","date":"2026-09-25","downloads":1},{"category":"Linux","date":"2026-07-23","downloads":28},{"category":"Linux","date":"2026-07-24","downloads":8},{"category":"Linux","date":"2026-07-25","downloads":182},{"category":"Linux","date":"2026-07-27","downloads":44},{"category":"Linux","date":"2026-07-28","downloads":1},{"category":"Linux","date":"2026-08-03","downloads":6},{"category":"Linux","date":"2026-08-06","downloads":2},{"category":"Linux","date":"2026-08-08","downloads":56},{"category":"Linux","date":"2026-08-11","downloads":5},{"category":"Linux","date":"2026-08-14","downloads":3},{"category":"Linux","date":"2026-08-17","downloads":4},{"category":"Linux","date":"2026-08-18","downloads":5},{"category":"Linux","date":"2026-08-19","downloads":4},{"category":"Linux","date":"2026-08-20","downloads":8},{"category":"Linux","date":"2026-08-23","downloads":2},{"category":"Linux","date":"2026-08-26","downloads":6},{"category":"Linux","date":"2026-08-27","downloads":3},{"category":"Linux","date":"2026-08-28","downloads":4},{"category":"Linux","date":"2026-08-29","downloads":2},{"category":"Linux","date":"2026-08-30","downloads":3},{"category":"Linux","date":"2026-08-31","downloads":5},{"category":"Linux","date":"2026-09-01","downloads":1},{"category":"Linux","date":"2026-09-02","downloads":1},{"category":"Linux","date":"2026-09-04","downloads":4},{"category":"Linux","date":"2026-09-05","downloads":1},{"category":"Linux","date":"2026-09-07","downloads":7},{"category":"Linux","date":"2026-09-10","downloads":3},{"category":"Linux","date":"2026-09-23","downloads":1},{"category":"Linux","date":"2026-09-25","downloads":6},{"category":"Linux","date":"2026-09-30","downloads":1},{"category":"Linux","date":"2026-10-03","downloads":3},{"category":"Linux","date":"2026-10-07","downloads":1},{"category":"Linux","date":"2026-10-09","downloads":1},{"category":"Linux","date":"2026-10-14","downloads":25},{"category":"Linux","date":"2026-10-20","downloads":25},{"category":"Linux","date":"2026-10-22","downloads":2},{"category":"Linux","date":"2026-10-23","downloads":16},{"category":"Linux","date":"2026-10-26","downloads":2},{"category":"Linux","date":"2026-10-27","downloads":3},{"category":"null","date":"2026-07-23","downloads":164},{"category":"null","date":"2026-07-24","downloads":118},{"category":"null","date":"2026-07-25","downloads":5},{"category":"null","date":"2026-07-27","downloads":5},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-30","downloads":20},{"category":"null","date":"2026-07-31","downloads":20},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-03","downloads":2},{"category":"null","date":"2026-08-04","downloads":9},{"category":"null","date":"2026-08-05","downloads":1},{"category":"null","date":"2026-08-06","downloads":8},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":3},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-18","downloads":6},{"category":"null","date":"2026-08-19","downloads":3},{"category":"null","date":"2026-08-20","downloads":3},{"category":"null","date":"2026-08-21","downloads":25},{"category":"null","date":"2026-08-22","downloads":5},{"category":"null","date":"2026-08-26","downloads":111},{"category":"null","date":"2026-08-27","downloads":25},{"category":"null","date":"2026-08-28","downloads":33},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-30","downloads":6},{"category":"null","date":"2026-08-31","downloads":116},{"category":"null","date":"2026-09-01","downloads":12},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":2},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":4},{"category":"null","date":"2026-09-09","downloads":6},{"category":"null","date":"2026-09-10","downloads":10},{"category":"null","date":"2026-09-11","downloads":3},{"category":"null","date":"2026-09-12","downloads":4},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":2},{"category":"null","date":"2026-09-19","downloads":24},{"category":"null","date":"2026-09-20","downloads":4},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":18},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-02","downloads":13},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-10","downloads":1},{"category":"null","date":"2026-10-12","downloads":4},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":8},{"category":"null","date":"2026-10-16","downloads":7},{"category":"null","date":"2026-10-18","downloads":1},{"category":"null","date":"2026-10-19","downloads":3},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1},{"category":"null","date":"2026-10-24","downloads":1},{"category":"null","date":"2026-10-28","downloads":1}],"package":"mooncake-transfer-engine-efa","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (AWS EFA, CUDA version)"},{"name":"mooncake-transfer-engine-efa-cuda13","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:12:01.630680+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-cuda13/recent","payload":{"data":{"last_day":19,"last_month":3734,"last_week":404},"package":"mooncake-transfer-engine-efa-cuda13","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:12:04.017385+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-cuda13/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-08-26","downloads":325},{"category":"with_mirrors","date":"2026-08-27","downloads":56},{"category":"with_mirrors","date":"2026-08-28","downloads":52},{"category":"with_mirrors","date":"2026-08-29","downloads":24},{"category":"with_mirrors","date":"2026-08-30","downloads":30},{"category":"with_mirrors","date":"2026-08-31","downloads":333},{"category":"with_mirrors","date":"2026-09-01","downloads":151},{"category":"with_mirrors","date":"2026-09-02","downloads":40},{"category":"with_mirrors","date":"2026-09-03","downloads":28},{"category":"with_mirrors","date":"2026-09-04","downloads":26},{"category":"with_mirrors","date":"2026-09-05","downloads":15},{"category":"with_mirrors","date":"2026-09-06","downloads":12},{"category":"with_mirrors","date":"2026-09-07","downloads":20},{"category":"with_mirrors","date":"2026-09-08","downloads":37},{"category":"with_mirrors","date":"2026-09-09","downloads":49},{"category":"with_mirrors","date":"2026-09-10","downloads":47},{"category":"with_mirrors","date":"2026-09-11","downloads":77},{"category":"with_mirrors","date":"2026-09-12","downloads":41},{"category":"with_mirrors","date":"2026-09-13","downloads":86},{"category":"with_mirrors","date":"2026-09-14","downloads":277},{"category":"with_mirrors","date":"2026-09-15","downloads":275},{"category":"with_mirrors","date":"2026-09-16","downloads":171},{"category":"with_mirrors","date":"2026-09-17","downloads":166},{"category":"with_mirrors","date":"2026-09-18","downloads":255},{"category":"with_mirrors","date":"2026-09-19","downloads":121},{"category":"with_mirrors","date":"2026-09-20","downloads":89},{"category":"with_mirrors","date":"2026-09-21","downloads":271},{"category":"with_mirrors","date":"2026-09-22","downloads":349},{"category":"with_mirrors","date":"2026-09-23","downloads":350},{"category":"with_mirrors","date":"2026-09-24","downloads":184},{"category":"with_mirrors","date":"2026-09-25","downloads":171},{"category":"with_mirrors","date":"2026-09-26","downloads":341},{"category":"with_mirrors","date":"2026-09-27","downloads":22},{"category":"with_mirrors","date":"2026-09-28","downloads":199},{"category":"with_mirrors","date":"2026-09-29","downloads":91},{"category":"with_mirrors","date":"2026-09-30","downloads":103},{"category":"with_mirrors","date":"2026-10-01","downloads":44},{"category":"with_mirrors","date":"2026-10-02","downloads":61},{"category":"with_mirrors","date":"2026-10-03","downloads":14},{"category":"with_mirrors","date":"2026-10-04","downloads":21},{"category":"with_mirrors","date":"2026-10-05","downloads":26},{"category":"with_mirrors","date":"2026-10-06","downloads":57},{"category":"with_mirrors","date":"2026-10-07","downloads":61},{"category":"with_mirrors","date":"2026-10-08","downloads":26},{"category":"with_mirrors","date":"2026-10-09","downloads":13},{"category":"with_mirrors","date":"2026-10-10","downloads":27},{"category":"with_mirrors","date":"2026-10-11","downloads":23},{"category":"with_mirrors","date":"2026-10-12","downloads":53},{"category":"with_mirrors","date":"2026-10-13","downloads":33},{"category":"with_mirrors","date":"2026-10-14","downloads":84},{"category":"with_mirrors","date":"2026-10-15","downloads":84},{"category":"with_mirrors","date":"2026-10-16","downloads":87},{"category":"with_mirrors","date":"2026-10-17","downloads":56},{"category":"with_mirrors","date":"2026-10-18","downloads":30},{"category":"with_mirrors","date":"2026-10-19","downloads":27},{"category":"with_mirrors","date":"2026-10-20","downloads":39},{"category":"with_mirrors","date":"2026-10-21","downloads":114},{"category":"with_mirrors","date":"2026-10-22","downloads":115},{"category":"with_mirrors","date":"2026-10-23","downloads":113},{"category":"with_mirrors","date":"2026-10-24","downloads":37},{"category":"with_mirrors","date":"2026-10-25","downloads":32},{"category":"with_mirrors","date":"2026-10-26","downloads":30},{"category":"with_mirrors","date":"2026-10-27","downloads":20},{"category":"with_mirrors","date":"2026-10-28","downloads":20},{"category":"without_mirrors","date":"2026-08-26","downloads":192},{"category":"without_mirrors","date":"2026-08-27","downloads":2},{"category":"without_mirrors","date":"2026-08-28","downloads":19},{"category":"without_mirrors","date":"2026-08-29","downloads":14},{"category":"without_mirrors","date":"2026-08-30","downloads":16},{"category":"without_mirrors","date":"2026-08-31","downloads":143},{"category":"without_mirrors","date":"2026-09-01","downloads":60},{"category":"without_mirrors","date":"2026-09-02","downloads":32},{"category":"without_mirrors","date":"2026-09-03","downloads":1},{"category":"without_mirrors","date":"2026-09-04","downloads":13},{"category":"without_mirrors","date":"2026-09-05","downloads":5},{"category":"without_mirrors","date":"2026-09-06","downloads":4},{"category":"without_mirrors","date":"2026-09-07","downloads":12},{"category":"without_mirrors","date":"2026-09-08","downloads":9},{"category":"without_mirrors","date":"2026-09-09","downloads":30},{"category":"without_mirrors","date":"2026-09-10","downloads":32},{"category":"without_mirrors","date":"2026-09-11","downloads":69},{"category":"without_mirrors","date":"2026-09-12","downloads":35},{"category":"without_mirrors","date":"2026-09-13","downloads":78},{"category":"without_mirrors","date":"2026-09-14","downloads":277},{"category":"without_mirrors","date":"2026-09-15","downloads":267},{"category":"without_mirrors","date":"2026-09-16","downloads":169},{"category":"without_mirrors","date":"2026-09-17","downloads":156},{"category":"without_mirrors","date":"2026-09-18","downloads":253},{"category":"without_mirrors","date":"2026-09-19","downloads":120},{"category":"without_mirrors","date":"2026-09-20","downloads":65},{"category":"without_mirrors","date":"2026-09-21","downloads":271},{"category":"without_mirrors","date":"2026-09-22","downloads":331},{"category":"without_mirrors","date":"2026-09-23","downloads":349},{"category":"without_mirrors","date":"2026-09-24","downloads":182},{"category":"without_mirrors","date":"2026-09-25","downloads":171},{"category":"without_mirrors","date":"2026-09-26","downloads":331},{"category":"without_mirrors","date":"2026-09-27","downloads":18},{"category":"without_mirrors","date":"2026-09-28","downloads":194},{"category":"without_mirrors","date":"2026-09-29","downloads":74},{"category":"without_mirrors","date":"2026-09-30","downloads":103},{"category":"without_mirrors","date":"2026-10-01","downloads":43},{"category":"without_mirrors","date":"2026-10-02","downloads":44},{"category":"without_mirrors","date":"2026-10-03","downloads":13},{"category":"without_mirrors","date":"2026-10-04","downloads":11},{"category":"without_mirrors","date":"2026-10-05","downloads":23},{"category":"without_mirrors","date":"2026-10-06","downloads":53},{"category":"without_mirrors","date":"2026-10-07","downloads":41},{"category":"without_mirrors","date":"2026-10-08","downloads":22},{"category":"without_mirrors","date":"2026-10-09","downloads":10},{"category":"without_mirrors","date":"2026-10-10","downloads":24},{"category":"without_mirrors","date":"2026-10-11","downloads":15},{"category":"without_mirrors","date":"2026-10-12","downloads":48},{"category":"without_mirrors","date":"2026-10-13","downloads":20},{"category":"without_mirrors","date":"2026-10-14","downloads":81},{"category":"without_mirrors","date":"2026-10-15","downloads":68},{"category":"without_mirrors","date":"2026-10-16","downloads":79},{"category":"without_mirrors","date":"2026-10-17","downloads":47},{"category":"without_mirrors","date":"2026-10-18","downloads":2},{"category":"without_mirrors","date":"2026-10-19","downloads":26},{"category":"without_mirrors","date":"2026-10-20","downloads":29},{"category":"without_mirrors","date":"2026-10-21","downloads":102},{"category":"without_mirrors","date":"2026-10-22","downloads":91},{"category":"without_mirrors","date":"2026-10-23","downloads":108},{"category":"without_mirrors","date":"2026-10-24","downloads":36},{"category":"without_mirrors","date":"2026-10-25","downloads":22},{"category":"without_mirrors","date":"2026-10-26","downloads":16},{"category":"without_mirrors","date":"2026-10-27","downloads":20},{"category":"without_mirrors","date":"2026-10-28","downloads":19}],"package":"mooncake-transfer-engine-efa-cuda13","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:12:06.405803+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-cuda13/python_major","payload":{"data":[{"category":"3","date":"2026-08-26","downloads":14},{"category":"3","date":"2026-08-27","downloads":1},{"category":"3","date":"2026-08-28","downloads":1},{"category":"3","date":"2026-08-30","downloads":4},{"category":"3","date":"2026-08-31","downloads":29},{"category":"3","date":"2026-09-01","downloads":56},{"category":"3","date":"2026-09-02","downloads":20},{"category":"3","date":"2026-09-04","downloads":5},{"category":"3","date":"2026-09-05","downloads":2},{"category":"3","date":"2026-09-08","downloads":6},{"category":"3","date":"2026-09-09","downloads":5},{"category":"3","date":"2026-09-10","downloads":18},{"category":"3","date":"2026-09-11","downloads":58},{"category":"3","date":"2026-09-12","downloads":16},{"category":"3","date":"2026-09-13","downloads":31},{"category":"3","date":"2026-09-14","downloads":231},{"category":"3","date":"2026-09-15","downloads":211},{"category":"3","date":"2026-09-16","downloads":112},{"category":"3","date":"2026-09-17","downloads":97},{"category":"3","date":"2026-09-18","downloads":195},{"category":"3","date":"2026-09-19","downloads":53},{"category":"3","date":"2026-09-20","downloads":60},{"category":"3","date":"2026-09-21","downloads":253},{"category":"3","date":"2026-09-22","downloads":328},{"category":"3","date":"2026-09-23","downloads":349},{"category":"3","date":"2026-09-24","downloads":178},{"category":"3","date":"2026-09-25","downloads":168},{"category":"3","date":"2026-09-26","downloads":329},{"category":"3","date":"2026-09-27","downloads":9},{"category":"3","date":"2026-09-28","downloads":187},{"category":"3","date":"2026-09-29","downloads":74},{"category":"3","date":"2026-09-30","downloads":102},{"category":"3","date":"2026-10-01","downloads":42},{"category":"3","date":"2026-10-02","downloads":41},{"category":"3","date":"2026-10-03","downloads":13},{"category":"3","date":"2026-10-04","downloads":9},{"category":"3","date":"2026-10-05","downloads":23},{"category":"3","date":"2026-10-06","downloads":52},{"category":"3","date":"2026-10-07","downloads":41},{"category":"3","date":"2026-10-08","downloads":21},{"category":"3","date":"2026-10-09","downloads":10},{"category":"3","date":"2026-10-10","downloads":24},{"category":"3","date":"2026-10-11","downloads":11},{"category":"3","date":"2026-10-12","downloads":36},{"category":"3","date":"2026-10-13","downloads":19},{"category":"3","date":"2026-10-14","downloads":78},{"category":"3","date":"2026-10-15","downloads":53},{"category":"3","date":"2026-10-16","downloads":78},{"category":"3","date":"2026-10-17","downloads":47},{"category":"3","date":"2026-10-18","downloads":2},{"category":"3","date":"2026-10-19","downloads":26},{"category":"3","date":"2026-10-20","downloads":26},{"category":"3","date":"2026-10-21","downloads":99},{"category":"3","date":"2026-10-22","downloads":90},{"category":"3","date":"2026-10-23","downloads":108},{"category":"3","date":"2026-10-24","downloads":36},{"category":"3","date":"2026-10-25","downloads":22},{"category":"3","date":"2026-10-26","downloads":16},{"category":"3","date":"2026-10-27","downloads":19},{"category":"3","date":"2026-10-28","downloads":17},{"category":"null","date":"2026-08-26","downloads":178},{"category":"null","date":"2026-08-27","downloads":1},{"category":"null","date":"2026-08-28","downloads":18},{"category":"null","date":"2026-08-29","downloads":14},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":114},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":3},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":12},{"category":"null","date":"2026-09-08","downloads":3},{"category":"null","date":"2026-09-09","downloads":25},{"category":"null","date":"2026-09-10","downloads":14},{"category":"null","date":"2026-09-11","downloads":11},{"category":"null","date":"2026-09-12","downloads":19},{"category":"null","date":"2026-09-13","downloads":47},{"category":"null","date":"2026-09-14","downloads":46},{"category":"null","date":"2026-09-15","downloads":56},{"category":"null","date":"2026-09-16","downloads":57},{"category":"null","date":"2026-09-17","downloads":59},{"category":"null","date":"2026-09-18","downloads":58},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":5},{"category":"null","date":"2026-09-21","downloads":18},{"category":"null","date":"2026-09-22","downloads":3},{"category":"null","date":"2026-09-24","downloads":4},{"category":"null","date":"2026-09-25","downloads":3},{"category":"null","date":"2026-09-26","downloads":2},{"category":"null","date":"2026-09-27","downloads":9},{"category":"null","date":"2026-09-28","downloads":7},{"category":"null","date":"2026-09-30","downloads":1},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":3},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-08","downloads":1},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":12},{"category":"null","date":"2026-10-13","downloads":1},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":15},{"category":"null","date":"2026-10-16","downloads":1},{"category":"null","date":"2026-10-20","downloads":3},{"category":"null","date":"2026-10-21","downloads":3},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-27","downloads":1},{"category":"null","date":"2026-10-28","downloads":2}],"package":"mooncake-transfer-engine-efa-cuda13","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:12:08.823265+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-cuda13/python_minor","payload":{"data":[{"category":"3.10","date":"2026-08-30","downloads":1},{"category":"3.11","date":"2026-08-26","downloads":8},{"category":"3.11","date":"2026-08-31","downloads":20},{"category":"3.11","date":"2026-09-24","downloads":1},{"category":"3.11","date":"2026-10-08","downloads":1},{"category":"3.11","date":"2026-10-09","downloads":1},{"category":"3.12","date":"2026-08-26","downloads":6},{"category":"3.12","date":"2026-08-27","downloads":1},{"category":"3.12","date":"2026-08-28","downloads":1},{"category":"3.12","date":"2026-08-30","downloads":3},{"category":"3.12","date":"2026-08-31","downloads":5},{"category":"3.12","date":"2026-09-01","downloads":56},{"category":"3.12","date":"2026-09-02","downloads":20},{"category":"3.12","date":"2026-09-04","downloads":5},{"category":"3.12","date":"2026-09-05","downloads":2},{"category":"3.12","date":"2026-09-08","downloads":6},{"category":"3.12","date":"2026-09-09","downloads":5},{"category":"3.12","date":"2026-09-10","downloads":18},{"category":"3.12","date":"2026-09-11","downloads":58},{"category":"3.12","date":"2026-09-12","downloads":16},{"category":"3.12","date":"2026-09-13","downloads":31},{"category":"3.12","date":"2026-09-14","downloads":231},{"category":"3.12","date":"2026-09-15","downloads":211},{"category":"3.12","date":"2026-09-16","downloads":112},{"category":"3.12","date":"2026-09-17","downloads":97},{"category":"3.12","date":"2026-09-18","downloads":195},{"category":"3.12","date":"2026-09-19","downloads":53},{"category":"3.12","date":"2026-09-20","downloads":60},{"category":"3.12","date":"2026-09-21","downloads":253},{"category":"3.12","date":"2026-09-22","downloads":328},{"category":"3.12","date":"2026-09-23","downloads":349},{"category":"3.12","date":"2026-09-24","downloads":177},{"category":"3.12","date":"2026-09-25","downloads":168},{"category":"3.12","date":"2026-09-26","downloads":329},{"category":"3.12","date":"2026-09-27","downloads":9},{"category":"3.12","date":"2026-09-28","downloads":186},{"category":"3.12","date":"2026-09-29","downloads":73},{"category":"3.12","date":"2026-09-30","downloads":102},{"category":"3.12","date":"2026-10-01","downloads":42},{"category":"3.12","date":"2026-10-02","downloads":41},{"category":"3.12","date":"2026-10-03","downloads":13},{"category":"3.12","date":"2026-10-04","downloads":9},{"category":"3.12","date":"2026-10-05","downloads":23},{"category":"3.12","date":"2026-10-06","downloads":52},{"category":"3.12","date":"2026-10-07","downloads":41},{"category":"3.12","date":"2026-10-08","downloads":20},{"category":"3.12","date":"2026-10-09","downloads":9},{"category":"3.12","date":"2026-10-10","downloads":20},{"category":"3.12","date":"2026-10-11","downloads":10},{"category":"3.12","date":"2026-10-12","downloads":36},{"category":"3.12","date":"2026-10-13","downloads":19},{"category":"3.12","date":"2026-10-14","downloads":76},{"category":"3.12","date":"2026-10-15","downloads":53},{"category":"3.12","date":"2026-10-16","downloads":78},{"category":"3.12","date":"2026-10-17","downloads":47},{"category":"3.12","date":"2026-10-18","downloads":2},{"category":"3.12","date":"2026-10-19","downloads":26},{"category":"3.12","date":"2026-10-20","downloads":25},{"category":"3.12","date":"2026-10-21","downloads":98},{"category":"3.12","date":"2026-10-22","downloads":90},{"category":"3.12","date":"2026-10-23","downloads":108},{"category":"3.12","date":"2026-10-24","downloads":36},{"category":"3.12","date":"2026-10-25","downloads":22},{"category":"3.12","date":"2026-10-26","downloads":16},{"category":"3.12","date":"2026-10-27","downloads":19},{"category":"3.12","date":"2026-10-28","downloads":17},{"category":"3.13","date":"2026-08-31","downloads":4},{"category":"3.13","date":"2026-09-28","downloads":1},{"category":"3.13","date":"2026-10-10","downloads":4},{"category":"3.13","date":"2026-10-14","downloads":2},{"category":"3.13","date":"2026-10-20","downloads":1},{"category":"3.14","date":"2026-09-29","downloads":1},{"category":"3.14","date":"2026-10-11","downloads":1},{"category":"3.14","date":"2026-10-21","downloads":1},{"category":"null","date":"2026-08-26","downloads":178},{"category":"null","date":"2026-08-27","downloads":1},{"category":"null","date":"2026-08-28","downloads":18},{"category":"null","date":"2026-08-29","downloads":14},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":114},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":3},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":12},{"category":"null","date":"2026-09-08","downloads":3},{"category":"null","date":"2026-09-09","downloads":25},{"category":"null","date":"2026-09-10","downloads":14},{"category":"null","date":"2026-09-11","downloads":11},{"category":"null","date":"2026-09-12","downloads":19},{"category":"null","date":"2026-09-13","downloads":47},{"category":"null","date":"2026-09-14","downloads":46},{"category":"null","date":"2026-09-15","downloads":56},{"category":"null","date":"2026-09-16","downloads":57},{"category":"null","date":"2026-09-17","downloads":59},{"category":"null","date":"2026-09-18","downloads":58},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":5},{"category":"null","date":"2026-09-21","downloads":18},{"category":"null","date":"2026-09-22","downloads":3},{"category":"null","date":"2026-09-24","downloads":4},{"category":"null","date":"2026-09-25","downloads":3},{"category":"null","date":"2026-09-26","downloads":2},{"category":"null","date":"2026-09-27","downloads":9},{"category":"null","date":"2026-09-28","downloads":7},{"category":"null","date":"2026-09-30","downloads":1},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":3},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-08","downloads":1},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":12},{"category":"null","date":"2026-10-13","downloads":1},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":15},{"category":"null","date":"2026-10-16","downloads":1},{"category":"null","date":"2026-10-20","downloads":3},{"category":"null","date":"2026-10-21","downloads":3},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-27","downloads":1},{"category":"null","date":"2026-10-28","downloads":2}],"package":"mooncake-transfer-engine-efa-cuda13","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:12:11.243314+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-cuda13/system","payload":{"data":[{"category":"Darwin","date":"2026-08-26","downloads":8},{"category":"Darwin","date":"2026-08-31","downloads":20},{"category":"Darwin","date":"2026-09-24","downloads":1},{"category":"Darwin","date":"2026-09-28","downloads":1},{"category":"Darwin","date":"2026-09-29","downloads":1},{"category":"Darwin","date":"2026-10-11","downloads":1},{"category":"Darwin","date":"2026-10-21","downloads":1},{"category":"Linux","date":"2026-08-26","downloads":6},{"category":"Linux","date":"2026-08-27","downloads":1},{"category":"Linux","date":"2026-08-28","downloads":1},{"category":"Linux","date":"2026-08-30","downloads":4},{"category":"Linux","date":"2026-08-31","downloads":9},{"category":"Linux","date":"2026-09-01","downloads":56},{"category":"Linux","date":"2026-09-02","downloads":20},{"category":"Linux","date":"2026-09-04","downloads":5},{"category":"Linux","date":"2026-09-05","downloads":2},{"category":"Linux","date":"2026-09-08","downloads":6},{"category":"Linux","date":"2026-09-09","downloads":5},{"category":"Linux","date":"2026-09-10","downloads":18},{"category":"Linux","date":"2026-09-11","downloads":58},{"category":"Linux","date":"2026-09-12","downloads":16},{"category":"Linux","date":"2026-09-13","downloads":31},{"category":"Linux","date":"2026-09-14","downloads":231},{"category":"Linux","date":"2026-09-15","downloads":211},{"category":"Linux","date":"2026-09-16","downloads":112},{"category":"Linux","date":"2026-09-17","downloads":97},{"category":"Linux","date":"2026-09-18","downloads":195},{"category":"Linux","date":"2026-09-19","downloads":53},{"category":"Linux","date":"2026-09-20","downloads":60},{"category":"Linux","date":"2026-09-21","downloads":253},{"category":"Linux","date":"2026-09-22","downloads":328},{"category":"Linux","date":"2026-09-23","downloads":349},{"category":"Linux","date":"2026-09-24","downloads":177},{"category":"Linux","date":"2026-09-25","downloads":168},{"category":"Linux","date":"2026-09-26","downloads":329},{"category":"Linux","date":"2026-09-27","downloads":9},{"category":"Linux","date":"2026-09-28","downloads":186},{"category":"Linux","date":"2026-09-29","downloads":73},{"category":"Linux","date":"2026-09-30","downloads":102},{"category":"Linux","date":"2026-10-01","downloads":42},{"category":"Linux","date":"2026-10-02","downloads":41},{"category":"Linux","date":"2026-10-03","downloads":13},{"category":"Linux","date":"2026-10-04","downloads":9},{"category":"Linux","date":"2026-10-05","downloads":23},{"category":"Linux","date":"2026-10-06","downloads":52},{"category":"Linux","date":"2026-10-07","downloads":41},{"category":"Linux","date":"2026-10-08","downloads":21},{"category":"Linux","date":"2026-10-09","downloads":10},{"category":"Linux","date":"2026-10-10","downloads":24},{"category":"Linux","date":"2026-10-11","downloads":10},{"category":"Linux","date":"2026-10-12","downloads":36},{"category":"Linux","date":"2026-10-13","downloads":19},{"category":"Linux","date":"2026-10-14","downloads":78},{"category":"Linux","date":"2026-10-15","downloads":53},{"category":"Linux","date":"2026-10-16","downloads":78},{"category":"Linux","date":"2026-10-17","downloads":47},{"category":"Linux","date":"2026-10-18","downloads":2},{"category":"Linux","date":"2026-10-19","downloads":26},{"category":"Linux","date":"2026-10-20","downloads":26},{"category":"Linux","date":"2026-10-21","downloads":98},{"category":"Linux","date":"2026-10-22","downloads":90},{"category":"Linux","date":"2026-10-23","downloads":108},{"category":"Linux","date":"2026-10-24","downloads":36},{"category":"Linux","date":"2026-10-25","downloads":22},{"category":"Linux","date":"2026-10-26","downloads":16},{"category":"Linux","date":"2026-10-27","downloads":19},{"category":"Linux","date":"2026-10-28","downloads":17},{"category":"null","date":"2026-08-26","downloads":178},{"category":"null","date":"2026-08-27","downloads":1},{"category":"null","date":"2026-08-28","downloads":18},{"category":"null","date":"2026-08-29","downloads":14},{"category":"null","date":"2026-08-30","downloads":12},{"category":"null","date":"2026-08-31","downloads":114},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":12},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":3},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":12},{"category":"null","date":"2026-09-08","downloads":3},{"category":"null","date":"2026-09-09","downloads":25},{"category":"null","date":"2026-09-10","downloads":14},{"category":"null","date":"2026-09-11","downloads":11},{"category":"null","date":"2026-09-12","downloads":19},{"category":"null","date":"2026-09-13","downloads":47},{"category":"null","date":"2026-09-14","downloads":46},{"category":"null","date":"2026-09-15","downloads":56},{"category":"null","date":"2026-09-16","downloads":57},{"category":"null","date":"2026-09-17","downloads":59},{"category":"null","date":"2026-09-18","downloads":58},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":5},{"category":"null","date":"2026-09-21","downloads":18},{"category":"null","date":"2026-09-22","downloads":3},{"category":"null","date":"2026-09-24","downloads":4},{"category":"null","date":"2026-09-25","downloads":3},{"category":"null","date":"2026-09-26","downloads":2},{"category":"null","date":"2026-09-27","downloads":9},{"category":"null","date":"2026-09-28","downloads":7},{"category":"null","date":"2026-09-30","downloads":1},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":3},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-08","downloads":1},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":12},{"category":"null","date":"2026-10-13","downloads":1},{"category":"null","date":"2026-10-14","downloads":3},{"category":"null","date":"2026-10-15","downloads":15},{"category":"null","date":"2026-10-16","downloads":1},{"category":"null","date":"2026-10-20","downloads":3},{"category":"null","date":"2026-10-21","downloads":3},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-27","downloads":1},{"category":"null","date":"2026-10-28","downloads":2}],"package":"mooncake-transfer-engine-efa-cuda13","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (AWS EFA, CUDA 13 version)"},{"name":"mooncake-transfer-engine-efa-non-cuda","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:12:14.419359+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-non-cuda/recent","payload":{"data":{"last_day":2,"last_month":164,"last_week":2},"package":"mooncake-transfer-engine-efa-non-cuda","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:12:16.797551+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-non-cuda/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-07-23","downloads":336},{"category":"with_mirrors","date":"2026-07-24","downloads":336},{"category":"with_mirrors","date":"2026-07-25","downloads":100},{"category":"with_mirrors","date":"2026-07-26","downloads":13},{"category":"with_mirrors","date":"2026-07-27","downloads":28},{"category":"with_mirrors","date":"2026-07-28","downloads":42},{"category":"with_mirrors","date":"2026-07-29","downloads":4},{"category":"with_mirrors","date":"2026-07-30","downloads":18},{"category":"with_mirrors","date":"2026-07-31","downloads":18},{"category":"with_mirrors","date":"2026-08-01","downloads":26},{"category":"with_mirrors","date":"2026-08-02","downloads":32},{"category":"with_mirrors","date":"2026-08-03","downloads":58},{"category":"with_mirrors","date":"2026-08-04","downloads":54},{"category":"with_mirrors","date":"2026-08-06","downloads":15},{"category":"with_mirrors","date":"2026-08-07","downloads":42},{"category":"with_mirrors","date":"2026-08-08","downloads":11},{"category":"with_mirrors","date":"2026-08-09","downloads":11},{"category":"with_mirrors","date":"2026-08-10","downloads":9},{"category":"with_mirrors","date":"2026-08-11","downloads":9},{"category":"with_mirrors","date":"2026-08-14","downloads":19},{"category":"with_mirrors","date":"2026-08-15","downloads":16},{"category":"with_mirrors","date":"2026-08-16","downloads":24},{"category":"with_mirrors","date":"2026-08-17","downloads":4},{"category":"with_mirrors","date":"2026-08-18","downloads":2},{"category":"with_mirrors","date":"2026-08-19","downloads":4},{"category":"with_mirrors","date":"2026-08-20","downloads":52},{"category":"with_mirrors","date":"2026-08-21","downloads":44},{"category":"with_mirrors","date":"2026-08-22","downloads":25},{"category":"with_mirrors","date":"2026-08-23","downloads":2},{"category":"with_mirrors","date":"2026-08-24","downloads":2},{"category":"with_mirrors","date":"2026-08-25","downloads":15},{"category":"with_mirrors","date":"2026-08-26","downloads":234},{"category":"with_mirrors","date":"2026-08-27","downloads":94},{"category":"with_mirrors","date":"2026-08-28","downloads":64},{"category":"with_mirrors","date":"2026-08-29","downloads":23},{"category":"with_mirrors","date":"2026-08-30","downloads":25},{"category":"with_mirrors","date":"2026-08-31","downloads":303},{"category":"with_mirrors","date":"2026-09-01","downloads":128},{"category":"with_mirrors","date":"2026-09-02","downloads":18},{"category":"with_mirrors","date":"2026-09-03","downloads":43},{"category":"with_mirrors","date":"2026-09-04","downloads":22},{"category":"with_mirrors","date":"2026-09-05","downloads":27},{"category":"with_mirrors","date":"2026-09-06","downloads":24},{"category":"with_mirrors","date":"2026-09-07","downloads":56},{"category":"with_mirrors","date":"2026-09-08","downloads":42},{"category":"with_mirrors","date":"2026-09-09","downloads":31},{"category":"with_mirrors","date":"2026-09-10","downloads":24},{"category":"with_mirrors","date":"2026-09-11","downloads":13},{"category":"with_mirrors","date":"2026-09-12","downloads":8},{"category":"with_mirrors","date":"2026-09-13","downloads":18},{"category":"with_mirrors","date":"2026-09-14","downloads":5},{"category":"with_mirrors","date":"2026-09-15","downloads":8},{"category":"with_mirrors","date":"2026-09-16","downloads":10},{"category":"with_mirrors","date":"2026-09-17","downloads":8},{"category":"with_mirrors","date":"2026-09-18","downloads":3},{"category":"with_mirrors","date":"2026-09-19","downloads":16},{"category":"with_mirrors","date":"2026-09-20","downloads":63},{"category":"with_mirrors","date":"2026-09-21","downloads":8},{"category":"with_mirrors","date":"2026-09-22","downloads":27},{"category":"with_mirrors","date":"2026-09-23","downloads":1},{"category":"with_mirrors","date":"2026-09-24","downloads":1},{"category":"with_mirrors","date":"2026-09-25","downloads":8},{"category":"with_mirrors","date":"2026-09-26","downloads":21},{"category":"with_mirrors","date":"2026-09-27","downloads":17},{"category":"with_mirrors","date":"2026-09-28","downloads":7},{"category":"with_mirrors","date":"2026-09-29","downloads":29},{"category":"with_mirrors","date":"2026-10-01","downloads":8},{"category":"with_mirrors","date":"2026-10-02","downloads":30},{"category":"with_mirrors","date":"2026-10-03","downloads":1},{"category":"with_mirrors","date":"2026-10-04","downloads":23},{"category":"with_mirrors","date":"2026-10-05","downloads":7},{"category":"with_mirrors","date":"2026-10-06","downloads":6},{"category":"with_mirrors","date":"2026-10-07","downloads":45},{"category":"with_mirrors","date":"2026-10-08","downloads":8},{"category":"with_mirrors","date":"2026-10-09","downloads":4},{"category":"with_mirrors","date":"2026-10-10","downloads":4},{"category":"with_mirrors","date":"2026-10-11","downloads":18},{"category":"with_mirrors","date":"2026-10-13","downloads":22},{"category":"with_mirrors","date":"2026-10-14","downloads":4},{"category":"with_mirrors","date":"2026-10-15","downloads":54},{"category":"with_mirrors","date":"2026-10-16","downloads":25},{"category":"with_mirrors","date":"2026-10-17","downloads":19},{"category":"with_mirrors","date":"2026-10-18","downloads":37},{"category":"with_mirrors","date":"2026-10-19","downloads":28},{"category":"with_mirrors","date":"2026-10-20","downloads":13},{"category":"with_mirrors","date":"2026-10-21","downloads":23},{"category":"with_mirrors","date":"2026-10-22","downloads":27},{"category":"with_mirrors","date":"2026-10-23","downloads":1},{"category":"with_mirrors","date":"2026-10-26","downloads":4},{"category":"with_mirrors","date":"2026-10-27","downloads":2},{"category":"with_mirrors","date":"2026-10-28","downloads":3},{"category":"without_mirrors","date":"2026-07-23","downloads":161},{"category":"without_mirrors","date":"2026-07-24","downloads":148},{"category":"without_mirrors","date":"2026-07-25","downloads":6},{"category":"without_mirrors","date":"2026-07-26","downloads":1},{"category":"without_mirrors","date":"2026-07-27","downloads":10},{"category":"without_mirrors","date":"2026-07-29","downloads":4},{"category":"without_mirrors","date":"2026-07-30","downloads":10},{"category":"without_mirrors","date":"2026-07-31","downloads":18},{"category":"without_mirrors","date":"2026-08-01","downloads":2},{"category":"without_mirrors","date":"2026-08-04","downloads":8},{"category":"without_mirrors","date":"2026-08-06","downloads":13},{"category":"without_mirrors","date":"2026-08-07","downloads":21},{"category":"without_mirrors","date":"2026-08-08","downloads":6},{"category":"without_mirrors","date":"2026-08-11","downloads":1},{"category":"without_mirrors","date":"2026-08-14","downloads":3},{"category":"without_mirrors","date":"2026-08-18","downloads":2},{"category":"without_mirrors","date":"2026-08-19","downloads":4},{"category":"without_mirrors","date":"2026-08-20","downloads":36},{"category":"without_mirrors","date":"2026-08-21","downloads":20},{"category":"without_mirrors","date":"2026-08-22","downloads":11},{"category":"without_mirrors","date":"2026-08-23","downloads":2},{"category":"without_mirrors","date":"2026-08-26","downloads":82},{"category":"without_mirrors","date":"2026-08-27","downloads":19},{"category":"without_mirrors","date":"2026-08-28","downloads":31},{"category":"without_mirrors","date":"2026-08-29","downloads":13},{"category":"without_mirrors","date":"2026-08-30","downloads":8},{"category":"without_mirrors","date":"2026-08-31","downloads":130},{"category":"without_mirrors","date":"2026-09-01","downloads":13},{"category":"without_mirrors","date":"2026-09-02","downloads":10},{"category":"without_mirrors","date":"2026-09-03","downloads":4},{"category":"without_mirrors","date":"2026-09-04","downloads":9},{"category":"without_mirrors","date":"2026-09-05","downloads":9},{"category":"without_mirrors","date":"2026-09-06","downloads":8},{"category":"without_mirrors","date":"2026-09-07","downloads":32},{"category":"without_mirrors","date":"2026-09-08","downloads":7},{"category":"without_mirrors","date":"2026-09-09","downloads":11},{"category":"without_mirrors","date":"2026-09-10","downloads":9},{"category":"without_mirrors","date":"2026-09-11","downloads":5},{"category":"without_mirrors","date":"2026-09-12","downloads":2},{"category":"without_mirrors","date":"2026-09-13","downloads":2},{"category":"without_mirrors","date":"2026-09-14","downloads":4},{"category":"without_mirrors","date":"2026-09-16","downloads":8},{"category":"without_mirrors","date":"2026-09-17","downloads":3},{"category":"without_mirrors","date":"2026-09-18","downloads":3},{"category":"without_mirrors","date":"2026-09-19","downloads":16},{"category":"without_mirrors","date":"2026-09-20","downloads":3},{"category":"without_mirrors","date":"2026-09-21","downloads":4},{"category":"without_mirrors","date":"2026-09-25","downloads":8},{"category":"without_mirrors","date":"2026-09-26","downloads":1},{"category":"without_mirrors","date":"2026-09-27","downloads":17},{"category":"without_mirrors","date":"2026-09-28","downloads":2},{"category":"without_mirrors","date":"2026-10-01","downloads":7},{"category":"without_mirrors","date":"2026-10-02","downloads":2},{"category":"without_mirrors","date":"2026-10-03","downloads":1},{"category":"without_mirrors","date":"2026-10-04","downloads":5},{"category":"without_mirrors","date":"2026-10-05","downloads":4},{"category":"without_mirrors","date":"2026-10-06","downloads":2},{"category":"without_mirrors","date":"2026-10-07","downloads":5},{"category":"without_mirrors","date":"2026-10-08","downloads":3},{"category":"without_mirrors","date":"2026-10-09","downloads":1},{"category":"without_mirrors","date":"2026-10-10","downloads":4},{"category":"without_mirrors","date":"2026-10-11","downloads":1},{"category":"without_mirrors","date":"2026-10-14","downloads":1},{"category":"without_mirrors","date":"2026-10-15","downloads":6},{"category":"without_mirrors","date":"2026-10-16","downloads":9},{"category":"without_mirrors","date":"2026-10-17","downloads":2},{"category":"without_mirrors","date":"2026-10-18","downloads":3},{"category":"without_mirrors","date":"2026-10-19","downloads":10},{"category":"without_mirrors","date":"2026-10-20","downloads":1},{"category":"without_mirrors","date":"2026-10-23","downloads":1},{"category":"without_mirrors","date":"2026-10-27","downloads":2},{"category":"without_mirrors","date":"2026-10-28","downloads":2}],"package":"mooncake-transfer-engine-efa-non-cuda","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:12:19.220660+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-non-cuda/python_major","payload":{"data":[{"category":"3","date":"2026-07-23","downloads":30},{"category":"3","date":"2026-07-24","downloads":20},{"category":"3","date":"2026-07-25","downloads":4},{"category":"3","date":"2026-07-30","downloads":2},{"category":"3","date":"2026-08-06","downloads":2},{"category":"3","date":"2026-08-07","downloads":18},{"category":"3","date":"2026-08-08","downloads":4},{"category":"3","date":"2026-08-20","downloads":24},{"category":"3","date":"2026-08-23","downloads":2},{"category":"3","date":"2026-08-26","downloads":15},{"category":"3","date":"2026-08-29","downloads":1},{"category":"3","date":"2026-08-31","downloads":26},{"category":"3","date":"2026-09-04","downloads":1},{"category":"3","date":"2026-09-05","downloads":2},{"category":"3","date":"2026-09-09","downloads":1},{"category":"3","date":"2026-09-10","downloads":3},{"category":"3","date":"2026-09-18","downloads":3},{"category":"3","date":"2026-09-27","downloads":1},{"category":"3","date":"2026-10-01","downloads":4},{"category":"3","date":"2026-10-03","downloads":1},{"category":"3","date":"2026-10-04","downloads":1},{"category":"3","date":"2026-10-06","downloads":1},{"category":"3","date":"2026-10-07","downloads":1},{"category":"3","date":"2026-10-08","downloads":3},{"category":"3","date":"2026-10-10","downloads":4},{"category":"3","date":"2026-10-17","downloads":1},{"category":"3","date":"2026-10-19","downloads":4},{"category":"3","date":"2026-10-27","downloads":2},{"category":"3","date":"2026-10-28","downloads":2},{"category":"null","date":"2026-07-23","downloads":131},{"category":"null","date":"2026-07-24","downloads":128},{"category":"null","date":"2026-07-25","downloads":2},{"category":"null","date":"2026-07-26","downloads":1},{"category":"null","date":"2026-07-27","downloads":10},{"category":"null","date":"2026-07-29","downloads":4},{"category":"null","date":"2026-07-30","downloads":8},{"category":"null","date":"2026-07-31","downloads":18},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-04","downloads":8},{"category":"null","date":"2026-08-06","downloads":11},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-08","downloads":2},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-14","downloads":3},{"category":"null","date":"2026-08-18","downloads":2},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":12},{"category":"null","date":"2026-08-21","downloads":20},{"category":"null","date":"2026-08-22","downloads":11},{"category":"null","date":"2026-08-26","downloads":67},{"category":"null","date":"2026-08-27","downloads":19},{"category":"null","date":"2026-08-28","downloads":31},{"category":"null","date":"2026-08-29","downloads":12},{"category":"null","date":"2026-08-30","downloads":8},{"category":"null","date":"2026-08-31","downloads":104},{"category":"null","date":"2026-09-01","downloads":13},{"category":"null","date":"2026-09-02","downloads":10},{"category":"null","date":"2026-09-03","downloads":4},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":7},{"category":"null","date":"2026-09-06","downloads":8},{"category":"null","date":"2026-09-07","downloads":32},{"category":"null","date":"2026-09-08","downloads":7},{"category":"null","date":"2026-09-09","downloads":10},{"category":"null","date":"2026-09-10","downloads":6},{"category":"null","date":"2026-09-11","downloads":5},{"category":"null","date":"2026-09-12","downloads":2},{"category":"null","date":"2026-09-13","downloads":2},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":8},{"category":"null","date":"2026-09-17","downloads":3},{"category":"null","date":"2026-09-19","downloads":16},{"category":"null","date":"2026-09-20","downloads":3},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":1},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-04","downloads":4},{"category":"null","date":"2026-10-05","downloads":4},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":4},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-11","downloads":1},{"category":"null","date":"2026-10-14","downloads":1},{"category":"null","date":"2026-10-15","downloads":6},{"category":"null","date":"2026-10-16","downloads":9},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":3},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-efa-non-cuda","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:12:21.609219+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-non-cuda/python_minor","payload":{"data":[{"category":"3.10","date":"2026-07-30","downloads":2},{"category":"3.10","date":"2026-09-27","downloads":1},{"category":"3.10","date":"2026-10-06","downloads":1},{"category":"3.11","date":"2026-07-24","downloads":16},{"category":"3.11","date":"2026-08-26","downloads":10},{"category":"3.11","date":"2026-08-29","downloads":1},{"category":"3.11","date":"2026-08-31","downloads":20},{"category":"3.12","date":"2026-07-23","downloads":30},{"category":"3.12","date":"2026-07-24","downloads":4},{"category":"3.12","date":"2026-08-06","downloads":2},{"category":"3.12","date":"2026-08-07","downloads":18},{"category":"3.12","date":"2026-08-08","downloads":4},{"category":"3.12","date":"2026-08-20","downloads":24},{"category":"3.12","date":"2026-08-23","downloads":2},{"category":"3.12","date":"2026-08-26","downloads":5},{"category":"3.12","date":"2026-08-31","downloads":4},{"category":"3.12","date":"2026-09-04","downloads":1},{"category":"3.12","date":"2026-09-05","downloads":2},{"category":"3.12","date":"2026-09-09","downloads":1},{"category":"3.12","date":"2026-09-10","downloads":3},{"category":"3.12","date":"2026-09-18","downloads":3},{"category":"3.12","date":"2026-10-01","downloads":3},{"category":"3.12","date":"2026-10-03","downloads":1},{"category":"3.12","date":"2026-10-04","downloads":1},{"category":"3.12","date":"2026-10-07","downloads":1},{"category":"3.12","date":"2026-10-08","downloads":3},{"category":"3.12","date":"2026-10-17","downloads":1},{"category":"3.12","date":"2026-10-19","downloads":4},{"category":"3.12","date":"2026-10-28","downloads":2},{"category":"3.13","date":"2026-07-25","downloads":4},{"category":"3.13","date":"2026-08-31","downloads":2},{"category":"3.13","date":"2026-10-10","downloads":4},{"category":"3.13","date":"2026-10-27","downloads":2},{"category":"3.9","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-07-23","downloads":131},{"category":"null","date":"2026-07-24","downloads":128},{"category":"null","date":"2026-07-25","downloads":2},{"category":"null","date":"2026-07-26","downloads":1},{"category":"null","date":"2026-07-27","downloads":10},{"category":"null","date":"2026-07-29","downloads":4},{"category":"null","date":"2026-07-30","downloads":8},{"category":"null","date":"2026-07-31","downloads":18},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-04","downloads":8},{"category":"null","date":"2026-08-06","downloads":11},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-08","downloads":2},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-14","downloads":3},{"category":"null","date":"2026-08-18","downloads":2},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":12},{"category":"null","date":"2026-08-21","downloads":20},{"category":"null","date":"2026-08-22","downloads":11},{"category":"null","date":"2026-08-26","downloads":67},{"category":"null","date":"2026-08-27","downloads":19},{"category":"null","date":"2026-08-28","downloads":31},{"category":"null","date":"2026-08-29","downloads":12},{"category":"null","date":"2026-08-30","downloads":8},{"category":"null","date":"2026-08-31","downloads":104},{"category":"null","date":"2026-09-01","downloads":13},{"category":"null","date":"2026-09-02","downloads":10},{"category":"null","date":"2026-09-03","downloads":4},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":7},{"category":"null","date":"2026-09-06","downloads":8},{"category":"null","date":"2026-09-07","downloads":32},{"category":"null","date":"2026-09-08","downloads":7},{"category":"null","date":"2026-09-09","downloads":10},{"category":"null","date":"2026-09-10","downloads":6},{"category":"null","date":"2026-09-11","downloads":5},{"category":"null","date":"2026-09-12","downloads":2},{"category":"null","date":"2026-09-13","downloads":2},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":8},{"category":"null","date":"2026-09-17","downloads":3},{"category":"null","date":"2026-09-19","downloads":16},{"category":"null","date":"2026-09-20","downloads":3},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":1},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-04","downloads":4},{"category":"null","date":"2026-10-05","downloads":4},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":4},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-11","downloads":1},{"category":"null","date":"2026-10-14","downloads":1},{"category":"null","date":"2026-10-15","downloads":6},{"category":"null","date":"2026-10-16","downloads":9},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":3},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-efa-non-cuda","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:12:24.096298+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-efa-non-cuda/system","payload":{"data":[{"category":"Darwin","date":"2026-07-24","downloads":16},{"category":"Darwin","date":"2026-08-26","downloads":10},{"category":"Darwin","date":"2026-08-31","downloads":20},{"category":"Linux","date":"2026-07-23","downloads":30},{"category":"Linux","date":"2026-07-24","downloads":4},{"category":"Linux","date":"2026-07-25","downloads":4},{"category":"Linux","date":"2026-07-30","downloads":2},{"category":"Linux","date":"2026-08-06","downloads":2},{"category":"Linux","date":"2026-08-07","downloads":18},{"category":"Linux","date":"2026-08-08","downloads":4},{"category":"Linux","date":"2026-08-20","downloads":24},{"category":"Linux","date":"2026-08-23","downloads":2},{"category":"Linux","date":"2026-08-26","downloads":5},{"category":"Linux","date":"2026-08-29","downloads":1},{"category":"Linux","date":"2026-08-31","downloads":6},{"category":"Linux","date":"2026-09-04","downloads":1},{"category":"Linux","date":"2026-09-05","downloads":2},{"category":"Linux","date":"2026-09-09","downloads":1},{"category":"Linux","date":"2026-09-10","downloads":3},{"category":"Linux","date":"2026-09-18","downloads":3},{"category":"Linux","date":"2026-09-27","downloads":1},{"category":"Linux","date":"2026-10-01","downloads":4},{"category":"Linux","date":"2026-10-03","downloads":1},{"category":"Linux","date":"2026-10-04","downloads":1},{"category":"Linux","date":"2026-10-06","downloads":1},{"category":"Linux","date":"2026-10-07","downloads":1},{"category":"Linux","date":"2026-10-08","downloads":3},{"category":"Linux","date":"2026-10-10","downloads":4},{"category":"Linux","date":"2026-10-17","downloads":1},{"category":"Linux","date":"2026-10-19","downloads":4},{"category":"Linux","date":"2026-10-27","downloads":2},{"category":"Linux","date":"2026-10-28","downloads":2},{"category":"null","date":"2026-07-23","downloads":131},{"category":"null","date":"2026-07-24","downloads":128},{"category":"null","date":"2026-07-25","downloads":2},{"category":"null","date":"2026-07-26","downloads":1},{"category":"null","date":"2026-07-27","downloads":10},{"category":"null","date":"2026-07-29","downloads":4},{"category":"null","date":"2026-07-30","downloads":8},{"category":"null","date":"2026-07-31","downloads":18},{"category":"null","date":"2026-08-01","downloads":2},{"category":"null","date":"2026-08-04","downloads":8},{"category":"null","date":"2026-08-06","downloads":11},{"category":"null","date":"2026-08-07","downloads":3},{"category":"null","date":"2026-08-08","downloads":2},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-14","downloads":3},{"category":"null","date":"2026-08-18","downloads":2},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":12},{"category":"null","date":"2026-08-21","downloads":20},{"category":"null","date":"2026-08-22","downloads":11},{"category":"null","date":"2026-08-26","downloads":67},{"category":"null","date":"2026-08-27","downloads":19},{"category":"null","date":"2026-08-28","downloads":31},{"category":"null","date":"2026-08-29","downloads":12},{"category":"null","date":"2026-08-30","downloads":8},{"category":"null","date":"2026-08-31","downloads":104},{"category":"null","date":"2026-09-01","downloads":13},{"category":"null","date":"2026-09-02","downloads":10},{"category":"null","date":"2026-09-03","downloads":4},{"category":"null","date":"2026-09-04","downloads":8},{"category":"null","date":"2026-09-05","downloads":7},{"category":"null","date":"2026-09-06","downloads":8},{"category":"null","date":"2026-09-07","downloads":32},{"category":"null","date":"2026-09-08","downloads":7},{"category":"null","date":"2026-09-09","downloads":10},{"category":"null","date":"2026-09-10","downloads":6},{"category":"null","date":"2026-09-11","downloads":5},{"category":"null","date":"2026-09-12","downloads":2},{"category":"null","date":"2026-09-13","downloads":2},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-16","downloads":8},{"category":"null","date":"2026-09-17","downloads":3},{"category":"null","date":"2026-09-19","downloads":16},{"category":"null","date":"2026-09-20","downloads":3},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-25","downloads":8},{"category":"null","date":"2026-09-26","downloads":1},{"category":"null","date":"2026-09-27","downloads":16},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-04","downloads":4},{"category":"null","date":"2026-10-05","downloads":4},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":4},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-11","downloads":1},{"category":"null","date":"2026-10-14","downloads":1},{"category":"null","date":"2026-10-15","downloads":6},{"category":"null","date":"2026-10-16","downloads":9},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":3},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-efa-non-cuda","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (AWS EFA, Non-CUDA version)"},{"name":"mooncake-transfer-engine-musa","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:12:27.357129+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-musa/recent","payload":{"data":{"last_day":0,"last_month":59,"last_week":6},"package":"mooncake-transfer-engine-musa","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:12:30.069564+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-musa/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-04-08","downloads":1},{"category":"with_mirrors","date":"2026-04-10","downloads":3},{"category":"with_mirrors","date":"2026-04-11","downloads":5},{"category":"with_mirrors","date":"2026-04-13","downloads":2},{"category":"with_mirrors","date":"2026-04-15","downloads":3},{"category":"with_mirrors","date":"2026-04-17","downloads":6},{"category":"with_mirrors","date":"2026-04-18","downloads":8},{"category":"with_mirrors","date":"2026-04-19","downloads":2},{"category":"with_mirrors","date":"2026-04-20","downloads":1},{"category":"with_mirrors","date":"2026-04-21","downloads":4},{"category":"with_mirrors","date":"2026-04-22","downloads":5},{"category":"with_mirrors","date":"2026-04-23","downloads":1},{"category":"with_mirrors","date":"2026-04-24","downloads":2},{"category":"with_mirrors","date":"2026-04-26","downloads":4},{"category":"with_mirrors","date":"2026-04-27","downloads":10},{"category":"with_mirrors","date":"2026-04-28","downloads":3},{"category":"with_mirrors","date":"2026-04-30","downloads":1},{"category":"with_mirrors","date":"2026-05-02","downloads":1},{"category":"with_mirrors","date":"2026-05-05","downloads":12},{"category":"with_mirrors","date":"2026-05-06","downloads":6},{"category":"with_mirrors","date":"2026-05-09","downloads":4},{"category":"with_mirrors","date":"2026-05-10","downloads":3},{"category":"with_mirrors","date":"2026-05-11","downloads":1},{"category":"with_mirrors","date":"2026-05-13","downloads":13},{"category":"with_mirrors","date":"2026-05-14","downloads":2},{"category":"with_mirrors","date":"2026-05-15","downloads":2},{"category":"with_mirrors","date":"2026-05-16","downloads":5},{"category":"with_mirrors","date":"2026-05-17","downloads":6},{"category":"with_mirrors","date":"2026-05-18","downloads":4},{"category":"with_mirrors","date":"2026-05-19","downloads":2},{"category":"with_mirrors","date":"2026-05-21","downloads":2},{"category":"with_mirrors","date":"2026-05-22","downloads":2},{"category":"with_mirrors","date":"2026-05-23","downloads":3},{"category":"with_mirrors","date":"2026-05-24","downloads":3},{"category":"with_mirrors","date":"2026-05-25","downloads":3},{"category":"with_mirrors","date":"2026-05-27","downloads":2},{"category":"with_mirrors","date":"2026-05-28","downloads":1},{"category":"with_mirrors","date":"2026-05-29","downloads":11},{"category":"with_mirrors","date":"2026-05-30","downloads":1},{"category":"with_mirrors","date":"2026-05-31","downloads":4},{"category":"with_mirrors","date":"2026-06-01","downloads":4},{"category":"with_mirrors","date":"2026-06-02","downloads":1},{"category":"with_mirrors","date":"2026-06-03","downloads":4},{"category":"with_mirrors","date":"2026-06-04","downloads":1},{"category":"with_mirrors","date":"2026-06-05","downloads":11},{"category":"with_mirrors","date":"2026-06-06","downloads":23},{"category":"with_mirrors","date":"2026-06-07","downloads":10},{"category":"with_mirrors","date":"2026-06-08","downloads":2},{"category":"with_mirrors","date":"2026-06-09","downloads":3},{"category":"with_mirrors","date":"2026-06-10","downloads":3},{"category":"with_mirrors","date":"2026-06-11","downloads":2},{"category":"with_mirrors","date":"2026-06-12","downloads":3},{"category":"with_mirrors","date":"2026-06-13","downloads":4},{"category":"with_mirrors","date":"2026-06-15","downloads":1},{"category":"with_mirrors","date":"2026-06-16","downloads":6},{"category":"with_mirrors","date":"2026-06-17","downloads":2},{"category":"with_mirrors","date":"2026-06-18","downloads":3},{"category":"with_mirrors","date":"2026-06-19","downloads":1},{"category":"with_mirrors","date":"2026-06-21","downloads":3},{"category":"with_mirrors","date":"2026-06-23","downloads":6},{"category":"with_mirrors","date":"2026-06-24","downloads":6},{"category":"with_mirrors","date":"2026-06-25","downloads":1},{"category":"with_mirrors","date":"2026-06-26","downloads":4},{"category":"with_mirrors","date":"2026-06-27","downloads":12},{"category":"with_mirrors","date":"2026-06-28","downloads":7},{"category":"with_mirrors","date":"2026-06-29","downloads":2},{"category":"with_mirrors","date":"2026-06-30","downloads":3},{"category":"with_mirrors","date":"2026-07-01","downloads":2},{"category":"with_mirrors","date":"2026-07-03","downloads":6},{"category":"with_mirrors","date":"2026-07-04","downloads":4},{"category":"with_mirrors","date":"2026-07-05","downloads":1},{"category":"with_mirrors","date":"2026-07-06","downloads":4},{"category":"with_mirrors","date":"2026-07-07","downloads":1},{"category":"with_mirrors","date":"2026-07-08","downloads":2},{"category":"with_mirrors","date":"2026-07-09","downloads":4},{"category":"with_mirrors","date":"2026-07-10","downloads":1},{"category":"with_mirrors","date":"2026-07-12","downloads":2},{"category":"with_mirrors","date":"2026-07-14","downloads":2},{"category":"with_mirrors","date":"2026-07-15","downloads":3},{"category":"with_mirrors","date":"2026-07-16","downloads":14},{"category":"with_mirrors","date":"2026-07-17","downloads":2},{"category":"with_mirrors","date":"2026-07-18","downloads":4},{"category":"with_mirrors","date":"2026-07-19","downloads":3},{"category":"with_mirrors","date":"2026-07-20","downloads":1},{"category":"with_mirrors","date":"2026-07-21","downloads":1},{"category":"with_mirrors","date":"2026-07-22","downloads":2},{"category":"with_mirrors","date":"2026-07-23","downloads":6},{"category":"with_mirrors","date":"2026-07-24","downloads":229},{"category":"with_mirrors","date":"2026-07-25","downloads":49},{"category":"with_mirrors","date":"2026-07-26","downloads":10},{"category":"with_mirrors","date":"2026-07-27","downloads":12},{"category":"with_mirrors","date":"2026-07-28","downloads":13},{"category":"with_mirrors","date":"2026-07-29","downloads":2},{"category":"with_mirrors","date":"2026-07-30","downloads":4},{"category":"with_mirrors","date":"2026-07-31","downloads":6},{"category":"with_mirrors","date":"2026-08-01","downloads":6},{"category":"with_mirrors","date":"2026-08-02","downloads":11},{"category":"with_mirrors","date":"2026-08-03","downloads":17},{"category":"with_mirrors","date":"2026-08-04","downloads":19},{"category":"with_mirrors","date":"2026-08-05","downloads":2},{"category":"with_mirrors","date":"2026-08-06","downloads":5},{"category":"with_mirrors","date":"2026-08-07","downloads":11},{"category":"with_mirrors","date":"2026-08-08","downloads":10},{"category":"with_mirrors","date":"2026-08-09","downloads":6},{"category":"with_mirrors","date":"2026-08-10","downloads":2},{"category":"with_mirrors","date":"2026-08-11","downloads":2},{"category":"with_mirrors","date":"2026-08-13","downloads":7},{"category":"with_mirrors","date":"2026-08-14","downloads":9},{"category":"with_mirrors","date":"2026-08-15","downloads":2},{"category":"with_mirrors","date":"2026-08-16","downloads":10},{"category":"with_mirrors","date":"2026-08-17","downloads":2},{"category":"with_mirrors","date":"2026-08-19","downloads":2},{"category":"with_mirrors","date":"2026-08-20","downloads":3},{"category":"with_mirrors","date":"2026-08-21","downloads":12},{"category":"with_mirrors","date":"2026-08-22","downloads":13},{"category":"with_mirrors","date":"2026-08-23","downloads":3},{"category":"with_mirrors","date":"2026-08-24","downloads":3},{"category":"with_mirrors","date":"2026-08-25","downloads":3},{"category":"with_mirrors","date":"2026-08-26","downloads":121},{"category":"with_mirrors","date":"2026-08-27","downloads":34},{"category":"with_mirrors","date":"2026-08-28","downloads":24},{"category":"with_mirrors","date":"2026-08-29","downloads":14},{"category":"with_mirrors","date":"2026-08-30","downloads":7},{"category":"with_mirrors","date":"2026-08-31","downloads":148},{"category":"with_mirrors","date":"2026-09-01","downloads":44},{"category":"with_mirrors","date":"2026-09-02","downloads":8},{"category":"with_mirrors","date":"2026-09-03","downloads":15},{"category":"with_mirrors","date":"2026-09-04","downloads":8},{"category":"with_mirrors","date":"2026-09-05","downloads":4},{"category":"with_mirrors","date":"2026-09-06","downloads":9},{"category":"with_mirrors","date":"2026-09-07","downloads":9},{"category":"with_mirrors","date":"2026-09-08","downloads":18},{"category":"with_mirrors","date":"2026-09-09","downloads":7},{"category":"with_mirrors","date":"2026-09-10","downloads":4},{"category":"with_mirrors","date":"2026-09-11","downloads":6},{"category":"with_mirrors","date":"2026-09-12","downloads":1},{"category":"with_mirrors","date":"2026-09-13","downloads":7},{"category":"with_mirrors","date":"2026-09-14","downloads":4},{"category":"with_mirrors","date":"2026-09-15","downloads":7},{"category":"with_mirrors","date":"2026-09-16","downloads":1},{"category":"with_mirrors","date":"2026-09-17","downloads":3},{"category":"with_mirrors","date":"2026-09-19","downloads":10},{"category":"with_mirrors","date":"2026-09-20","downloads":21},{"category":"with_mirrors","date":"2026-09-21","downloads":6},{"category":"with_mirrors","date":"2026-09-22","downloads":8},{"category":"with_mirrors","date":"2026-09-23","downloads":2},{"category":"with_mirrors","date":"2026-09-24","downloads":3},{"category":"with_mirrors","date":"2026-09-25","downloads":3},{"category":"with_mirrors","date":"2026-09-26","downloads":6},{"category":"with_mirrors","date":"2026-09-27","downloads":5},{"category":"with_mirrors","date":"2026-09-28","downloads":7},{"category":"with_mirrors","date":"2026-09-29","downloads":10},{"category":"with_mirrors","date":"2026-09-30","downloads":4},{"category":"with_mirrors","date":"2026-10-01","downloads":2},{"category":"with_mirrors","date":"2026-10-02","downloads":11},{"category":"with_mirrors","date":"2026-10-04","downloads":8},{"category":"with_mirrors","date":"2026-10-05","downloads":4},{"category":"with_mirrors","date":"2026-10-06","downloads":4},{"category":"with_mirrors","date":"2026-10-07","downloads":12},{"category":"with_mirrors","date":"2026-10-08","downloads":2},{"category":"with_mirrors","date":"2026-10-09","downloads":3},{"category":"with_mirrors","date":"2026-10-11","downloads":4},{"category":"with_mirrors","date":"2026-10-12","downloads":4},{"category":"with_mirrors","date":"2026-10-13","downloads":9},{"category":"with_mirrors","date":"2026-10-14","downloads":5},{"category":"with_mirrors","date":"2026-10-15","downloads":20},{"category":"with_mirrors","date":"2026-10-16","downloads":9},{"category":"with_mirrors","date":"2026-10-17","downloads":6},{"category":"with_mirrors","date":"2026-10-18","downloads":15},{"category":"with_mirrors","date":"2026-10-19","downloads":8},{"category":"with_mirrors","date":"2026-10-20","downloads":2},{"category":"with_mirrors","date":"2026-10-21","downloads":10},{"category":"with_mirrors","date":"2026-10-22","downloads":20},{"category":"with_mirrors","date":"2026-10-23","downloads":1},{"category":"with_mirrors","date":"2026-10-26","downloads":10},{"category":"with_mirrors","date":"2026-10-27","downloads":1},{"category":"with_mirrors","date":"2026-10-28","downloads":2},{"category":"without_mirrors","date":"2026-04-10","downloads":1},{"category":"without_mirrors","date":"2026-04-11","downloads":1},{"category":"without_mirrors","date":"2026-04-15","downloads":1},{"category":"without_mirrors","date":"2026-04-18","downloads":1},{"category":"without_mirrors","date":"2026-04-19","downloads":1},{"category":"without_mirrors","date":"2026-04-20","downloads":1},{"category":"without_mirrors","date":"2026-04-22","downloads":2},{"category":"without_mirrors","date":"2026-04-23","downloads":1},{"category":"without_mirrors","date":"2026-04-26","downloads":1},{"category":"without_mirrors","date":"2026-04-27","downloads":7},{"category":"without_mirrors","date":"2026-05-05","downloads":11},{"category":"without_mirrors","date":"2026-05-06","downloads":2},{"category":"without_mirrors","date":"2026-05-11","downloads":1},{"category":"without_mirrors","date":"2026-05-16","downloads":5},{"category":"without_mirrors","date":"2026-05-17","downloads":1},{"category":"without_mirrors","date":"2026-05-18","downloads":2},{"category":"without_mirrors","date":"2026-05-23","downloads":2},{"category":"without_mirrors","date":"2026-05-24","downloads":1},{"category":"without_mirrors","date":"2026-05-25","downloads":1},{"category":"without_mirrors","date":"2026-05-29","downloads":1},{"category":"without_mirrors","date":"2026-06-01","downloads":1},{"category":"without_mirrors","date":"2026-06-05","downloads":10},{"category":"without_mirrors","date":"2026-06-06","downloads":15},{"category":"without_mirrors","date":"2026-06-10","downloads":2},{"category":"without_mirrors","date":"2026-06-13","downloads":1},{"category":"without_mirrors","date":"2026-06-18","downloads":1},{"category":"without_mirrors","date":"2026-06-21","downloads":3},{"category":"without_mirrors","date":"2026-06-23","downloads":5},{"category":"without_mirrors","date":"2026-06-27","downloads":7},{"category":"without_mirrors","date":"2026-06-28","downloads":1},{"category":"without_mirrors","date":"2026-06-29","downloads":1},{"category":"without_mirrors","date":"2026-06-30","downloads":3},{"category":"without_mirrors","date":"2026-07-04","downloads":1},{"category":"without_mirrors","date":"2026-07-07","downloads":1},{"category":"without_mirrors","date":"2026-07-14","downloads":1},{"category":"without_mirrors","date":"2026-07-15","downloads":1},{"category":"without_mirrors","date":"2026-07-16","downloads":2},{"category":"without_mirrors","date":"2026-07-22","downloads":2},{"category":"without_mirrors","date":"2026-07-23","downloads":3},{"category":"without_mirrors","date":"2026-07-24","downloads":83},{"category":"without_mirrors","date":"2026-07-25","downloads":10},{"category":"without_mirrors","date":"2026-07-27","downloads":1},{"category":"without_mirrors","date":"2026-07-28","downloads":2},{"category":"without_mirrors","date":"2026-07-29","downloads":2},{"category":"without_mirrors","date":"2026-07-30","downloads":2},{"category":"without_mirrors","date":"2026-07-31","downloads":6},{"category":"without_mirrors","date":"2026-08-01","downloads":1},{"category":"without_mirrors","date":"2026-08-03","downloads":1},{"category":"without_mirrors","date":"2026-08-04","downloads":3},{"category":"without_mirrors","date":"2026-08-05","downloads":2},{"category":"without_mirrors","date":"2026-08-06","downloads":5},{"category":"without_mirrors","date":"2026-08-08","downloads":7},{"category":"without_mirrors","date":"2026-08-09","downloads":1},{"category":"without_mirrors","date":"2026-08-14","downloads":2},{"category":"without_mirrors","date":"2026-08-16","downloads":1},{"category":"without_mirrors","date":"2026-08-17","downloads":1},{"category":"without_mirrors","date":"2026-08-19","downloads":1},{"category":"without_mirrors","date":"2026-08-21","downloads":2},{"category":"without_mirrors","date":"2026-08-22","downloads":2},{"category":"without_mirrors","date":"2026-08-23","downloads":3},{"category":"without_mirrors","date":"2026-08-26","downloads":44},{"category":"without_mirrors","date":"2026-08-27","downloads":10},{"category":"without_mirrors","date":"2026-08-28","downloads":10},{"category":"without_mirrors","date":"2026-08-29","downloads":7},{"category":"without_mirrors","date":"2026-08-31","downloads":61},{"category":"without_mirrors","date":"2026-09-01","downloads":4},{"category":"without_mirrors","date":"2026-09-02","downloads":4},{"category":"without_mirrors","date":"2026-09-03","downloads":1},{"category":"without_mirrors","date":"2026-09-04","downloads":4},{"category":"without_mirrors","date":"2026-09-05","downloads":1},{"category":"without_mirrors","date":"2026-09-06","downloads":3},{"category":"without_mirrors","date":"2026-09-07","downloads":3},{"category":"without_mirrors","date":"2026-09-08","downloads":5},{"category":"without_mirrors","date":"2026-09-09","downloads":3},{"category":"without_mirrors","date":"2026-09-10","downloads":2},{"category":"without_mirrors","date":"2026-09-13","downloads":1},{"category":"without_mirrors","date":"2026-09-14","downloads":4},{"category":"without_mirrors","date":"2026-09-17","downloads":1},{"category":"without_mirrors","date":"2026-09-19","downloads":6},{"category":"without_mirrors","date":"2026-09-21","downloads":6},{"category":"without_mirrors","date":"2026-09-22","downloads":5},{"category":"without_mirrors","date":"2026-09-23","downloads":1},{"category":"without_mirrors","date":"2026-09-24","downloads":3},{"category":"without_mirrors","date":"2026-09-25","downloads":3},{"category":"without_mirrors","date":"2026-09-27","downloads":5},{"category":"without_mirrors","date":"2026-09-28","downloads":3},{"category":"without_mirrors","date":"2026-09-29","downloads":2},{"category":"without_mirrors","date":"2026-10-02","downloads":2},{"category":"without_mirrors","date":"2026-10-05","downloads":1},{"category":"without_mirrors","date":"2026-10-06","downloads":3},{"category":"without_mirrors","date":"2026-10-12","downloads":3},{"category":"without_mirrors","date":"2026-10-15","downloads":2},{"category":"without_mirrors","date":"2026-10-16","downloads":3},{"category":"without_mirrors","date":"2026-10-17","downloads":1},{"category":"without_mirrors","date":"2026-10-18","downloads":4},{"category":"without_mirrors","date":"2026-10-20","downloads":1},{"category":"without_mirrors","date":"2026-10-21","downloads":2},{"category":"without_mirrors","date":"2026-10-22","downloads":1},{"category":"without_mirrors","date":"2026-10-23","downloads":1},{"category":"without_mirrors","date":"2026-10-26","downloads":1},{"category":"without_mirrors","date":"2026-10-27","downloads":1}],"package":"mooncake-transfer-engine-musa","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:12:32.448510+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-musa/python_major","payload":{"data":[{"category":"3","date":"2026-04-15","downloads":1},{"category":"3","date":"2026-04-18","downloads":1},{"category":"3","date":"2026-04-22","downloads":2},{"category":"3","date":"2026-04-23","downloads":1},{"category":"3","date":"2026-04-26","downloads":1},{"category":"3","date":"2026-04-27","downloads":2},{"category":"3","date":"2026-05-16","downloads":2},{"category":"3","date":"2026-06-13","downloads":1},{"category":"3","date":"2026-07-24","downloads":10},{"category":"3","date":"2026-07-25","downloads":4},{"category":"3","date":"2026-07-29","downloads":2},{"category":"3","date":"2026-08-05","downloads":2},{"category":"3","date":"2026-08-08","downloads":7},{"category":"3","date":"2026-08-16","downloads":1},{"category":"3","date":"2026-08-17","downloads":1},{"category":"3","date":"2026-08-21","downloads":2},{"category":"3","date":"2026-08-22","downloads":2},{"category":"3","date":"2026-08-23","downloads":2},{"category":"3","date":"2026-08-26","downloads":3},{"category":"3","date":"2026-08-27","downloads":1},{"category":"3","date":"2026-08-28","downloads":1},{"category":"3","date":"2026-08-29","downloads":1},{"category":"3","date":"2026-08-31","downloads":5},{"category":"3","date":"2026-09-23","downloads":1},{"category":"3","date":"2026-09-24","downloads":3},{"category":"3","date":"2026-09-25","downloads":3},{"category":"3","date":"2026-09-29","downloads":2},{"category":"3","date":"2026-10-12","downloads":3},{"category":"3","date":"2026-10-26","downloads":1},{"category":"3","date":"2026-10-27","downloads":1},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":1},{"category":"null","date":"2026-04-19","downloads":1},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-27","downloads":5},{"category":"null","date":"2026-05-05","downloads":11},{"category":"null","date":"2026-05-06","downloads":2},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-16","downloads":3},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":2},{"category":"null","date":"2026-05-23","downloads":2},{"category":"null","date":"2026-05-24","downloads":1},{"category":"null","date":"2026-05-25","downloads":1},{"category":"null","date":"2026-05-29","downloads":1},{"category":"null","date":"2026-06-01","downloads":1},{"category":"null","date":"2026-06-05","downloads":10},{"category":"null","date":"2026-06-06","downloads":15},{"category":"null","date":"2026-06-10","downloads":2},{"category":"null","date":"2026-06-18","downloads":1},{"category":"null","date":"2026-06-21","downloads":3},{"category":"null","date":"2026-06-23","downloads":5},{"category":"null","date":"2026-06-27","downloads":7},{"category":"null","date":"2026-06-28","downloads":1},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-15","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-22","downloads":2},{"category":"null","date":"2026-07-23","downloads":3},{"category":"null","date":"2026-07-24","downloads":73},{"category":"null","date":"2026-07-25","downloads":6},{"category":"null","date":"2026-07-27","downloads":1},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-30","downloads":2},{"category":"null","date":"2026-07-31","downloads":6},{"category":"null","date":"2026-08-01","downloads":1},{"category":"null","date":"2026-08-03","downloads":1},{"category":"null","date":"2026-08-04","downloads":3},{"category":"null","date":"2026-08-06","downloads":5},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-19","downloads":1},{"category":"null","date":"2026-08-23","downloads":1},{"category":"null","date":"2026-08-26","downloads":41},{"category":"null","date":"2026-08-27","downloads":9},{"category":"null","date":"2026-08-28","downloads":9},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-31","downloads":56},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":4},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":4},{"category":"null","date":"2026-09-05","downloads":1},{"category":"null","date":"2026-09-06","downloads":3},{"category":"null","date":"2026-09-07","downloads":3},{"category":"null","date":"2026-09-08","downloads":5},{"category":"null","date":"2026-09-09","downloads":3},{"category":"null","date":"2026-09-10","downloads":2},{"category":"null","date":"2026-09-13","downloads":1},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-19","downloads":6},{"category":"null","date":"2026-09-21","downloads":6},{"category":"null","date":"2026-09-22","downloads":5},{"category":"null","date":"2026-09-27","downloads":5},{"category":"null","date":"2026-09-28","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":3},{"category":"null","date":"2026-10-15","downloads":2},{"category":"null","date":"2026-10-16","downloads":3},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":4},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-musa","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:12:34.820366+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-musa/python_minor","payload":{"data":[{"category":"3.10","date":"2026-07-25","downloads":4},{"category":"3.10","date":"2026-08-05","downloads":2},{"category":"3.10","date":"2026-08-08","downloads":7},{"category":"3.10","date":"2026-08-16","downloads":1},{"category":"3.10","date":"2026-08-17","downloads":1},{"category":"3.10","date":"2026-08-21","downloads":2},{"category":"3.10","date":"2026-08-22","downloads":2},{"category":"3.10","date":"2026-08-26","downloads":1},{"category":"3.10","date":"2026-08-27","downloads":1},{"category":"3.10","date":"2026-08-28","downloads":1},{"category":"3.10","date":"2026-08-31","downloads":1},{"category":"3.10","date":"2026-09-23","downloads":1},{"category":"3.10","date":"2026-09-24","downloads":3},{"category":"3.10","date":"2026-09-25","downloads":3},{"category":"3.10","date":"2026-09-29","downloads":2},{"category":"3.10","date":"2026-10-12","downloads":3},{"category":"3.10","date":"2026-10-26","downloads":1},{"category":"3.11","date":"2026-05-16","downloads":2},{"category":"3.11","date":"2026-06-13","downloads":1},{"category":"3.11","date":"2026-07-24","downloads":4},{"category":"3.11","date":"2026-08-26","downloads":2},{"category":"3.11","date":"2026-08-29","downloads":1},{"category":"3.11","date":"2026-08-31","downloads":4},{"category":"3.12","date":"2026-07-24","downloads":6},{"category":"3.12","date":"2026-07-29","downloads":2},{"category":"3.12","date":"2026-08-23","downloads":2},{"category":"3.12","date":"2026-10-27","downloads":1},{"category":"3.13","date":"2026-04-22","downloads":2},{"category":"3.13","date":"2026-04-27","downloads":2},{"category":"3.7","date":"2026-04-15","downloads":1},{"category":"3.7","date":"2026-04-18","downloads":1},{"category":"3.7","date":"2026-04-23","downloads":1},{"category":"3.7","date":"2026-04-26","downloads":1},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":1},{"category":"null","date":"2026-04-19","downloads":1},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-27","downloads":5},{"category":"null","date":"2026-05-05","downloads":11},{"category":"null","date":"2026-05-06","downloads":2},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-16","downloads":3},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":2},{"category":"null","date":"2026-05-23","downloads":2},{"category":"null","date":"2026-05-24","downloads":1},{"category":"null","date":"2026-05-25","downloads":1},{"category":"null","date":"2026-05-29","downloads":1},{"category":"null","date":"2026-06-01","downloads":1},{"category":"null","date":"2026-06-05","downloads":10},{"category":"null","date":"2026-06-06","downloads":15},{"category":"null","date":"2026-06-10","downloads":2},{"category":"null","date":"2026-06-18","downloads":1},{"category":"null","date":"2026-06-21","downloads":3},{"category":"null","date":"2026-06-23","downloads":5},{"category":"null","date":"2026-06-27","downloads":7},{"category":"null","date":"2026-06-28","downloads":1},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-15","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-22","downloads":2},{"category":"null","date":"2026-07-23","downloads":3},{"category":"null","date":"2026-07-24","downloads":73},{"category":"null","date":"2026-07-25","downloads":6},{"category":"null","date":"2026-07-27","downloads":1},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-30","downloads":2},{"category":"null","date":"2026-07-31","downloads":6},{"category":"null","date":"2026-08-01","downloads":1},{"category":"null","date":"2026-08-03","downloads":1},{"category":"null","date":"2026-08-04","downloads":3},{"category":"null","date":"2026-08-06","downloads":5},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-19","downloads":1},{"category":"null","date":"2026-08-23","downloads":1},{"category":"null","date":"2026-08-26","downloads":41},{"category":"null","date":"2026-08-27","downloads":9},{"category":"null","date":"2026-08-28","downloads":9},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-31","downloads":56},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":4},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":4},{"category":"null","date":"2026-09-05","downloads":1},{"category":"null","date":"2026-09-06","downloads":3},{"category":"null","date":"2026-09-07","downloads":3},{"category":"null","date":"2026-09-08","downloads":5},{"category":"null","date":"2026-09-09","downloads":3},{"category":"null","date":"2026-09-10","downloads":2},{"category":"null","date":"2026-09-13","downloads":1},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-19","downloads":6},{"category":"null","date":"2026-09-21","downloads":6},{"category":"null","date":"2026-09-22","downloads":5},{"category":"null","date":"2026-09-27","downloads":5},{"category":"null","date":"2026-09-28","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":3},{"category":"null","date":"2026-10-15","downloads":2},{"category":"null","date":"2026-10-16","downloads":3},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":4},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-musa","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:12:37.219158+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-musa/system","payload":{"data":[{"category":"Darwin","date":"2026-05-16","downloads":2},{"category":"Darwin","date":"2026-07-24","downloads":4},{"category":"Darwin","date":"2026-08-26","downloads":2},{"category":"Darwin","date":"2026-08-31","downloads":4},{"category":"Linux","date":"2026-04-15","downloads":1},{"category":"Linux","date":"2026-04-18","downloads":1},{"category":"Linux","date":"2026-04-23","downloads":1},{"category":"Linux","date":"2026-04-26","downloads":1},{"category":"Linux","date":"2026-04-27","downloads":2},{"category":"Linux","date":"2026-06-13","downloads":1},{"category":"Linux","date":"2026-07-24","downloads":6},{"category":"Linux","date":"2026-07-25","downloads":4},{"category":"Linux","date":"2026-08-05","downloads":2},{"category":"Linux","date":"2026-08-08","downloads":7},{"category":"Linux","date":"2026-08-16","downloads":1},{"category":"Linux","date":"2026-08-17","downloads":1},{"category":"Linux","date":"2026-08-21","downloads":2},{"category":"Linux","date":"2026-08-22","downloads":2},{"category":"Linux","date":"2026-08-23","downloads":2},{"category":"Linux","date":"2026-08-26","downloads":1},{"category":"Linux","date":"2026-08-27","downloads":1},{"category":"Linux","date":"2026-08-28","downloads":1},{"category":"Linux","date":"2026-08-29","downloads":1},{"category":"Linux","date":"2026-08-31","downloads":1},{"category":"Linux","date":"2026-09-23","downloads":1},{"category":"Linux","date":"2026-09-24","downloads":3},{"category":"Linux","date":"2026-09-25","downloads":3},{"category":"Linux","date":"2026-09-29","downloads":2},{"category":"Linux","date":"2026-10-12","downloads":3},{"category":"Linux","date":"2026-10-26","downloads":1},{"category":"Linux","date":"2026-10-27","downloads":1},{"category":"Windows","date":"2026-04-22","downloads":2},{"category":"Windows","date":"2026-07-29","downloads":2},{"category":"null","date":"2026-04-10","downloads":1},{"category":"null","date":"2026-04-11","downloads":1},{"category":"null","date":"2026-04-19","downloads":1},{"category":"null","date":"2026-04-20","downloads":1},{"category":"null","date":"2026-04-27","downloads":5},{"category":"null","date":"2026-05-05","downloads":11},{"category":"null","date":"2026-05-06","downloads":2},{"category":"null","date":"2026-05-11","downloads":1},{"category":"null","date":"2026-05-16","downloads":3},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":2},{"category":"null","date":"2026-05-23","downloads":2},{"category":"null","date":"2026-05-24","downloads":1},{"category":"null","date":"2026-05-25","downloads":1},{"category":"null","date":"2026-05-29","downloads":1},{"category":"null","date":"2026-06-01","downloads":1},{"category":"null","date":"2026-06-05","downloads":10},{"category":"null","date":"2026-06-06","downloads":15},{"category":"null","date":"2026-06-10","downloads":2},{"category":"null","date":"2026-06-18","downloads":1},{"category":"null","date":"2026-06-21","downloads":3},{"category":"null","date":"2026-06-23","downloads":5},{"category":"null","date":"2026-06-27","downloads":7},{"category":"null","date":"2026-06-28","downloads":1},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-15","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-22","downloads":2},{"category":"null","date":"2026-07-23","downloads":3},{"category":"null","date":"2026-07-24","downloads":73},{"category":"null","date":"2026-07-25","downloads":6},{"category":"null","date":"2026-07-27","downloads":1},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-30","downloads":2},{"category":"null","date":"2026-07-31","downloads":6},{"category":"null","date":"2026-08-01","downloads":1},{"category":"null","date":"2026-08-03","downloads":1},{"category":"null","date":"2026-08-04","downloads":3},{"category":"null","date":"2026-08-06","downloads":5},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-14","downloads":2},{"category":"null","date":"2026-08-19","downloads":1},{"category":"null","date":"2026-08-23","downloads":1},{"category":"null","date":"2026-08-26","downloads":41},{"category":"null","date":"2026-08-27","downloads":9},{"category":"null","date":"2026-08-28","downloads":9},{"category":"null","date":"2026-08-29","downloads":6},{"category":"null","date":"2026-08-31","downloads":56},{"category":"null","date":"2026-09-01","downloads":4},{"category":"null","date":"2026-09-02","downloads":4},{"category":"null","date":"2026-09-03","downloads":1},{"category":"null","date":"2026-09-04","downloads":4},{"category":"null","date":"2026-09-05","downloads":1},{"category":"null","date":"2026-09-06","downloads":3},{"category":"null","date":"2026-09-07","downloads":3},{"category":"null","date":"2026-09-08","downloads":5},{"category":"null","date":"2026-09-09","downloads":3},{"category":"null","date":"2026-09-10","downloads":2},{"category":"null","date":"2026-09-13","downloads":1},{"category":"null","date":"2026-09-14","downloads":4},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-19","downloads":6},{"category":"null","date":"2026-09-21","downloads":6},{"category":"null","date":"2026-09-22","downloads":5},{"category":"null","date":"2026-09-27","downloads":5},{"category":"null","date":"2026-09-28","downloads":3},{"category":"null","date":"2026-10-02","downloads":2},{"category":"null","date":"2026-10-05","downloads":1},{"category":"null","date":"2026-10-06","downloads":3},{"category":"null","date":"2026-10-15","downloads":2},{"category":"null","date":"2026-10-16","downloads":3},{"category":"null","date":"2026-10-17","downloads":1},{"category":"null","date":"2026-10-18","downloads":4},{"category":"null","date":"2026-10-20","downloads":1},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-22","downloads":1},{"category":"null","date":"2026-10-23","downloads":1}],"package":"mooncake-transfer-engine-musa","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (MUSA version)"},{"name":"mooncake-transfer-engine-non-cuda","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:12:40.433800+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-non-cuda/recent","payload":{"data":{"last_day":168,"last_month":17925,"last_week":1542},"package":"mooncake-transfer-engine-non-cuda","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:12:43.148537+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-non-cuda/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-04-08","downloads":38},{"category":"with_mirrors","date":"2026-04-09","downloads":45},{"category":"with_mirrors","date":"2026-04-10","downloads":73},{"category":"with_mirrors","date":"2026-04-11","downloads":123},{"category":"with_mirrors","date":"2026-04-12","downloads":67},{"category":"with_mirrors","date":"2026-04-13","downloads":78},{"category":"with_mirrors","date":"2026-04-14","downloads":39},{"category":"with_mirrors","date":"2026-04-15","downloads":136},{"category":"with_mirrors","date":"2026-04-16","downloads":23},{"category":"with_mirrors","date":"2026-04-17","downloads":140},{"category":"with_mirrors","date":"2026-04-18","downloads":273},{"category":"with_mirrors","date":"2026-04-19","downloads":20},{"category":"with_mirrors","date":"2026-04-20","downloads":47},{"category":"with_mirrors","date":"2026-04-21","downloads":195},{"category":"with_mirrors","date":"2026-04-22","downloads":409},{"category":"with_mirrors","date":"2026-04-23","downloads":73},{"category":"with_mirrors","date":"2026-04-24","downloads":18},{"category":"with_mirrors","date":"2026-04-25","downloads":52},{"category":"with_mirrors","date":"2026-04-26","downloads":80},{"category":"with_mirrors","date":"2026-04-27","downloads":158},{"category":"with_mirrors","date":"2026-04-28","downloads":45},{"category":"with_mirrors","date":"2026-04-29","downloads":42},{"category":"with_mirrors","date":"2026-04-30","downloads":29},{"category":"with_mirrors","date":"2026-05-01","downloads":110},{"category":"with_mirrors","date":"2026-05-02","downloads":78},{"category":"with_mirrors","date":"2026-05-03","downloads":13},{"category":"with_mirrors","date":"2026-05-04","downloads":27},{"category":"with_mirrors","date":"2026-05-05","downloads":103},{"category":"with_mirrors","date":"2026-05-06","downloads":33},{"category":"with_mirrors","date":"2026-05-07","downloads":32},{"category":"with_mirrors","date":"2026-05-08","downloads":20},{"category":"with_mirrors","date":"2026-05-09","downloads":79},{"category":"with_mirrors","date":"2026-05-10","downloads":51},{"category":"with_mirrors","date":"2026-05-11","downloads":10},{"category":"with_mirrors","date":"2026-05-12","downloads":33},{"category":"with_mirrors","date":"2026-05-13","downloads":117},{"category":"with_mirrors","date":"2026-05-14","downloads":61},{"category":"with_mirrors","date":"2026-05-15","downloads":56},{"category":"with_mirrors","date":"2026-05-16","downloads":53},{"category":"with_mirrors","date":"2026-05-17","downloads":76},{"category":"with_mirrors","date":"2026-05-18","downloads":105},{"category":"with_mirrors","date":"2026-05-19","downloads":45},{"category":"with_mirrors","date":"2026-05-20","downloads":30},{"category":"with_mirrors","date":"2026-05-21","downloads":359},{"category":"with_mirrors","date":"2026-05-22","downloads":204},{"category":"with_mirrors","date":"2026-05-23","downloads":454},{"category":"with_mirrors","date":"2026-05-24","downloads":125},{"category":"with_mirrors","date":"2026-05-25","downloads":93},{"category":"with_mirrors","date":"2026-05-26","downloads":62},{"category":"with_mirrors","date":"2026-05-27","downloads":67},{"category":"with_mirrors","date":"2026-05-28","downloads":76},{"category":"with_mirrors","date":"2026-05-29","downloads":117},{"category":"with_mirrors","date":"2026-05-30","downloads":93},{"category":"with_mirrors","date":"2026-05-31","downloads":47},{"category":"with_mirrors","date":"2026-06-01","downloads":35},{"category":"with_mirrors","date":"2026-06-02","downloads":164},{"category":"with_mirrors","date":"2026-06-03","downloads":61},{"category":"with_mirrors","date":"2026-06-04","downloads":132},{"category":"with_mirrors","date":"2026-06-05","downloads":195},{"category":"with_mirrors","date":"2026-06-06","downloads":329},{"category":"with_mirrors","date":"2026-06-07","downloads":269},{"category":"with_mirrors","date":"2026-06-08","downloads":253},{"category":"with_mirrors","date":"2026-06-09","downloads":152},{"category":"with_mirrors","date":"2026-06-10","downloads":83},{"category":"with_mirrors","date":"2026-06-11","downloads":113},{"category":"with_mirrors","date":"2026-06-12","downloads":25},{"category":"with_mirrors","date":"2026-06-13","downloads":78},{"category":"with_mirrors","date":"2026-06-14","downloads":13},{"category":"with_mirrors","date":"2026-06-15","downloads":45},{"category":"with_mirrors","date":"2026-06-16","downloads":115},{"category":"with_mirrors","date":"2026-06-17","downloads":34},{"category":"with_mirrors","date":"2026-06-18","downloads":246},{"category":"with_mirrors","date":"2026-06-19","downloads":13},{"category":"with_mirrors","date":"2026-06-20","downloads":41},{"category":"with_mirrors","date":"2026-06-21","downloads":4},{"category":"with_mirrors","date":"2026-06-22","downloads":112},{"category":"with_mirrors","date":"2026-06-23","downloads":297},{"category":"with_mirrors","date":"2026-06-24","downloads":233},{"category":"with_mirrors","date":"2026-06-25","downloads":85},{"category":"with_mirrors","date":"2026-06-26","downloads":55},{"category":"with_mirrors","date":"2026-06-27","downloads":88},{"category":"with_mirrors","date":"2026-06-28","downloads":78},{"category":"with_mirrors","date":"2026-06-29","downloads":136},{"category":"with_mirrors","date":"2026-06-30","downloads":345},{"category":"with_mirrors","date":"2026-07-01","downloads":112},{"category":"with_mirrors","date":"2026-07-02","downloads":80},{"category":"with_mirrors","date":"2026-07-03","downloads":201},{"category":"with_mirrors","date":"2026-07-04","downloads":95},{"category":"with_mirrors","date":"2026-07-05","downloads":49},{"category":"with_mirrors","date":"2026-07-06","downloads":128},{"category":"with_mirrors","date":"2026-07-07","downloads":63},{"category":"with_mirrors","date":"2026-07-08","downloads":77},{"category":"with_mirrors","date":"2026-07-09","downloads":163},{"category":"with_mirrors","date":"2026-07-10","downloads":125},{"category":"with_mirrors","date":"2026-07-11","downloads":56},{"category":"with_mirrors","date":"2026-07-12","downloads":256},{"category":"with_mirrors","date":"2026-07-13","downloads":81},{"category":"with_mirrors","date":"2026-07-14","downloads":307},{"category":"with_mirrors","date":"2026-07-15","downloads":59},{"category":"with_mirrors","date":"2026-07-16","downloads":138},{"category":"with_mirrors","date":"2026-07-17","downloads":89},{"category":"with_mirrors","date":"2026-07-18","downloads":48},{"category":"with_mirrors","date":"2026-07-19","downloads":8},{"category":"with_mirrors","date":"2026-07-20","downloads":14},{"category":"with_mirrors","date":"2026-07-21","downloads":296},{"category":"with_mirrors","date":"2026-07-22","downloads":98},{"category":"with_mirrors","date":"2026-07-23","downloads":438},{"category":"with_mirrors","date":"2026-07-24","downloads":575},{"category":"with_mirrors","date":"2026-07-25","downloads":170},{"category":"with_mirrors","date":"2026-07-26","downloads":189},{"category":"with_mirrors","date":"2026-07-27","downloads":711},{"category":"with_mirrors","date":"2026-07-28","downloads":1319},{"category":"with_mirrors","date":"2026-07-29","downloads":1306},{"category":"with_mirrors","date":"2026-07-30","downloads":1243},{"category":"with_mirrors","date":"2026-07-31","downloads":1407},{"category":"with_mirrors","date":"2026-08-01","downloads":641},{"category":"with_mirrors","date":"2026-08-02","downloads":672},{"category":"with_mirrors","date":"2026-08-03","downloads":632},{"category":"with_mirrors","date":"2026-08-04","downloads":711},{"category":"with_mirrors","date":"2026-08-05","downloads":643},{"category":"with_mirrors","date":"2026-08-06","downloads":587},{"category":"with_mirrors","date":"2026-08-07","downloads":781},{"category":"with_mirrors","date":"2026-08-08","downloads":384},{"category":"with_mirrors","date":"2026-08-09","downloads":406},{"category":"with_mirrors","date":"2026-08-10","downloads":625},{"category":"with_mirrors","date":"2026-08-11","downloads":754},{"category":"with_mirrors","date":"2026-08-12","downloads":743},{"category":"with_mirrors","date":"2026-08-13","downloads":840},{"category":"with_mirrors","date":"2026-08-14","downloads":814},{"category":"with_mirrors","date":"2026-08-15","downloads":509},{"category":"with_mirrors","date":"2026-08-16","downloads":342},{"category":"with_mirrors","date":"2026-08-17","downloads":790},{"category":"with_mirrors","date":"2026-08-18","downloads":847},{"category":"with_mirrors","date":"2026-08-19","downloads":1469},{"category":"with_mirrors","date":"2026-08-20","downloads":837},{"category":"with_mirrors","date":"2026-08-21","downloads":766},{"category":"with_mirrors","date":"2026-08-22","downloads":136},{"category":"with_mirrors","date":"2026-08-23","downloads":238},{"category":"with_mirrors","date":"2026-08-24","downloads":942},{"category":"with_mirrors","date":"2026-08-25","downloads":439},{"category":"with_mirrors","date":"2026-08-26","downloads":1580},{"category":"with_mirrors","date":"2026-08-27","downloads":999},{"category":"with_mirrors","date":"2026-08-28","downloads":1009},{"category":"with_mirrors","date":"2026-08-29","downloads":96},{"category":"with_mirrors","date":"2026-08-30","downloads":81},{"category":"with_mirrors","date":"2026-08-31","downloads":1164},{"category":"with_mirrors","date":"2026-09-01","downloads":2889},{"category":"with_mirrors","date":"2026-09-02","downloads":1824},{"category":"with_mirrors","date":"2026-09-03","downloads":1750},{"category":"with_mirrors","date":"2026-09-04","downloads":746},{"category":"with_mirrors","date":"2026-09-05","downloads":823},{"category":"with_mirrors","date":"2026-09-06","downloads":1833},{"category":"with_mirrors","date":"2026-09-07","downloads":365},{"category":"with_mirrors","date":"2026-09-08","downloads":1093},{"category":"with_mirrors","date":"2026-09-09","downloads":1176},{"category":"with_mirrors","date":"2026-09-10","downloads":853},{"category":"with_mirrors","date":"2026-09-11","downloads":980},{"category":"with_mirrors","date":"2026-09-12","downloads":373},{"category":"with_mirrors","date":"2026-09-13","downloads":274},{"category":"with_mirrors","date":"2026-09-14","downloads":1111},{"category":"with_mirrors","date":"2026-09-15","downloads":1346},{"category":"with_mirrors","date":"2026-09-16","downloads":1584},{"category":"with_mirrors","date":"2026-09-17","downloads":1139},{"category":"with_mirrors","date":"2026-09-18","downloads":751},{"category":"with_mirrors","date":"2026-09-19","downloads":261},{"category":"with_mirrors","date":"2026-09-20","downloads":352},{"category":"with_mirrors","date":"2026-09-21","downloads":348},{"category":"with_mirrors","date":"2026-09-22","downloads":503},{"category":"with_mirrors","date":"2026-09-23","downloads":455},{"category":"with_mirrors","date":"2026-09-24","downloads":459},{"category":"with_mirrors","date":"2026-09-25","downloads":436},{"category":"with_mirrors","date":"2026-09-26","downloads":215},{"category":"with_mirrors","date":"2026-09-27","downloads":131},{"category":"with_mirrors","date":"2026-09-28","downloads":534},{"category":"with_mirrors","date":"2026-09-29","downloads":650},{"category":"with_mirrors","date":"2026-09-30","downloads":799},{"category":"with_mirrors","date":"2026-10-01","downloads":185},{"category":"with_mirrors","date":"2026-10-02","downloads":157},{"category":"with_mirrors","date":"2026-10-03","downloads":165},{"category":"with_mirrors","date":"2026-10-04","downloads":157},{"category":"with_mirrors","date":"2026-10-05","downloads":151},{"category":"with_mirrors","date":"2026-10-06","downloads":284},{"category":"with_mirrors","date":"2026-10-07","downloads":326},{"category":"with_mirrors","date":"2026-10-08","downloads":233},{"category":"with_mirrors","date":"2026-10-09","downloads":174},{"category":"with_mirrors","date":"2026-10-10","downloads":129},{"category":"with_mirrors","date":"2026-10-11","downloads":73},{"category":"with_mirrors","date":"2026-10-12","downloads":104},{"category":"with_mirrors","date":"2026-10-13","downloads":252},{"category":"with_mirrors","date":"2026-10-14","downloads":197},{"category":"with_mirrors","date":"2026-10-15","downloads":381},{"category":"with_mirrors","date":"2026-10-16","downloads":217},{"category":"with_mirrors","date":"2026-10-17","downloads":100},{"category":"with_mirrors","date":"2026-10-18","downloads":319},{"category":"with_mirrors","date":"2026-10-19","downloads":607},{"category":"with_mirrors","date":"2026-10-20","downloads":864},{"category":"with_mirrors","date":"2026-10-21","downloads":274},{"category":"with_mirrors","date":"2026-10-22","downloads":401},{"category":"with_mirrors","date":"2026-10-23","downloads":162},{"category":"with_mirrors","date":"2026-10-24","downloads":152},{"category":"with_mirrors","date":"2026-10-25","downloads":15},{"category":"with_mirrors","date":"2026-10-26","downloads":92},{"category":"with_mirrors","date":"2026-10-27","downloads":136},{"category":"with_mirrors","date":"2026-10-28","downloads":173},{"category":"without_mirrors","date":"2026-04-08","downloads":37},{"category":"without_mirrors","date":"2026-04-09","downloads":37},{"category":"without_mirrors","date":"2026-04-10","downloads":38},{"category":"without_mirrors","date":"2026-04-11","downloads":49},{"category":"without_mirrors","date":"2026-04-12","downloads":63},{"category":"without_mirrors","date":"2026-04-13","downloads":58},{"category":"without_mirrors","date":"2026-04-14","downloads":37},{"category":"without_mirrors","date":"2026-04-15","downloads":72},{"category":"without_mirrors","date":"2026-04-16","downloads":14},{"category":"without_mirrors","date":"2026-04-17","downloads":48},{"category":"without_mirrors","date":"2026-04-18","downloads":6},{"category":"without_mirrors","date":"2026-04-19","downloads":20},{"category":"without_mirrors","date":"2026-04-20","downloads":22},{"category":"without_mirrors","date":"2026-04-21","downloads":34},{"category":"without_mirrors","date":"2026-04-22","downloads":189},{"category":"without_mirrors","date":"2026-04-23","downloads":22},{"category":"without_mirrors","date":"2026-04-24","downloads":16},{"category":"without_mirrors","date":"2026-04-25","downloads":36},{"category":"without_mirrors","date":"2026-04-26","downloads":12},{"category":"without_mirrors","date":"2026-04-27","downloads":101},{"category":"without_mirrors","date":"2026-04-28","downloads":18},{"category":"without_mirrors","date":"2026-04-29","downloads":38},{"category":"without_mirrors","date":"2026-04-30","downloads":18},{"category":"without_mirrors","date":"2026-05-01","downloads":98},{"category":"without_mirrors","date":"2026-05-02","downloads":72},{"category":"without_mirrors","date":"2026-05-03","downloads":8},{"category":"without_mirrors","date":"2026-05-04","downloads":10},{"category":"without_mirrors","date":"2026-05-05","downloads":66},{"category":"without_mirrors","date":"2026-05-06","downloads":20},{"category":"without_mirrors","date":"2026-05-07","downloads":32},{"category":"without_mirrors","date":"2026-05-08","downloads":20},{"category":"without_mirrors","date":"2026-05-09","downloads":64},{"category":"without_mirrors","date":"2026-05-10","downloads":4},{"category":"without_mirrors","date":"2026-05-11","downloads":6},{"category":"without_mirrors","date":"2026-05-12","downloads":33},{"category":"without_mirrors","date":"2026-05-13","downloads":27},{"category":"without_mirrors","date":"2026-05-14","downloads":60},{"category":"without_mirrors","date":"2026-05-15","downloads":44},{"category":"without_mirrors","date":"2026-05-16","downloads":53},{"category":"without_mirrors","date":"2026-05-17","downloads":19},{"category":"without_mirrors","date":"2026-05-18","downloads":57},{"category":"without_mirrors","date":"2026-05-19","downloads":23},{"category":"without_mirrors","date":"2026-05-20","downloads":26},{"category":"without_mirrors","date":"2026-05-21","downloads":182},{"category":"without_mirrors","date":"2026-05-22","downloads":89},{"category":"without_mirrors","date":"2026-05-23","downloads":209},{"category":"without_mirrors","date":"2026-05-24","downloads":48},{"category":"without_mirrors","date":"2026-05-25","downloads":56},{"category":"without_mirrors","date":"2026-05-26","downloads":16},{"category":"without_mirrors","date":"2026-05-27","downloads":36},{"category":"without_mirrors","date":"2026-05-28","downloads":72},{"category":"without_mirrors","date":"2026-05-29","downloads":55},{"category":"without_mirrors","date":"2026-05-30","downloads":76},{"category":"without_mirrors","date":"2026-05-31","downloads":34},{"category":"without_mirrors","date":"2026-06-01","downloads":24},{"category":"without_mirrors","date":"2026-06-02","downloads":150},{"category":"without_mirrors","date":"2026-06-03","downloads":53},{"category":"without_mirrors","date":"2026-06-04","downloads":83},{"category":"without_mirrors","date":"2026-06-05","downloads":177},{"category":"without_mirrors","date":"2026-06-06","downloads":176},{"category":"without_mirrors","date":"2026-06-07","downloads":163},{"category":"without_mirrors","date":"2026-06-08","downloads":247},{"category":"without_mirrors","date":"2026-06-09","downloads":134},{"category":"without_mirrors","date":"2026-06-10","downloads":65},{"category":"without_mirrors","date":"2026-06-11","downloads":69},{"category":"without_mirrors","date":"2026-06-12","downloads":23},{"category":"without_mirrors","date":"2026-06-13","downloads":25},{"category":"without_mirrors","date":"2026-06-14","downloads":13},{"category":"without_mirrors","date":"2026-06-15","downloads":25},{"category":"without_mirrors","date":"2026-06-16","downloads":16},{"category":"without_mirrors","date":"2026-06-17","downloads":29},{"category":"without_mirrors","date":"2026-06-18","downloads":228},{"category":"without_mirrors","date":"2026-06-19","downloads":12},{"category":"without_mirrors","date":"2026-06-20","downloads":14},{"category":"without_mirrors","date":"2026-06-21","downloads":3},{"category":"without_mirrors","date":"2026-06-22","downloads":57},{"category":"without_mirrors","date":"2026-06-23","downloads":277},{"category":"without_mirrors","date":"2026-06-24","downloads":223},{"category":"without_mirrors","date":"2026-06-25","downloads":85},{"category":"without_mirrors","date":"2026-06-26","downloads":25},{"category":"without_mirrors","date":"2026-06-27","downloads":34},{"category":"without_mirrors","date":"2026-06-28","downloads":24},{"category":"without_mirrors","date":"2026-06-29","downloads":135},{"category":"without_mirrors","date":"2026-06-30","downloads":340},{"category":"without_mirrors","date":"2026-07-01","downloads":57},{"category":"without_mirrors","date":"2026-07-02","downloads":55},{"category":"without_mirrors","date":"2026-07-03","downloads":135},{"category":"without_mirrors","date":"2026-07-04","downloads":79},{"category":"without_mirrors","date":"2026-07-05","downloads":48},{"category":"without_mirrors","date":"2026-07-06","downloads":82},{"category":"without_mirrors","date":"2026-07-07","downloads":59},{"category":"without_mirrors","date":"2026-07-08","downloads":59},{"category":"without_mirrors","date":"2026-07-09","downloads":149},{"category":"without_mirrors","date":"2026-07-10","downloads":108},{"category":"without_mirrors","date":"2026-07-11","downloads":48},{"category":"without_mirrors","date":"2026-07-12","downloads":220},{"category":"without_mirrors","date":"2026-07-13","downloads":64},{"category":"without_mirrors","date":"2026-07-14","downloads":306},{"category":"without_mirrors","date":"2026-07-15","downloads":21},{"category":"without_mirrors","date":"2026-07-16","downloads":36},{"category":"without_mirrors","date":"2026-07-17","downloads":38},{"category":"without_mirrors","date":"2026-07-18","downloads":32},{"category":"without_mirrors","date":"2026-07-19","downloads":7},{"category":"without_mirrors","date":"2026-07-20","downloads":12},{"category":"without_mirrors","date":"2026-07-21","downloads":280},{"category":"without_mirrors","date":"2026-07-22","downloads":97},{"category":"without_mirrors","date":"2026-07-23","downloads":223},{"category":"without_mirrors","date":"2026-07-24","downloads":355},{"category":"without_mirrors","date":"2026-07-25","downloads":78},{"category":"without_mirrors","date":"2026-07-26","downloads":156},{"category":"without_mirrors","date":"2026-07-27","downloads":685},{"category":"without_mirrors","date":"2026-07-28","downloads":1271},{"category":"without_mirrors","date":"2026-07-29","downloads":1285},{"category":"without_mirrors","date":"2026-07-30","downloads":1232},{"category":"without_mirrors","date":"2026-07-31","downloads":1403},{"category":"without_mirrors","date":"2026-08-01","downloads":556},{"category":"without_mirrors","date":"2026-08-02","downloads":558},{"category":"without_mirrors","date":"2026-08-03","downloads":508},{"category":"without_mirrors","date":"2026-08-04","downloads":617},{"category":"without_mirrors","date":"2026-08-05","downloads":631},{"category":"without_mirrors","date":"2026-08-06","downloads":578},{"category":"without_mirrors","date":"2026-08-07","downloads":651},{"category":"without_mirrors","date":"2026-08-08","downloads":373},{"category":"without_mirrors","date":"2026-08-09","downloads":351},{"category":"without_mirrors","date":"2026-08-10","downloads":602},{"category":"without_mirrors","date":"2026-08-11","downloads":753},{"category":"without_mirrors","date":"2026-08-12","downloads":732},{"category":"without_mirrors","date":"2026-08-13","downloads":831},{"category":"without_mirrors","date":"2026-08-14","downloads":705},{"category":"without_mirrors","date":"2026-08-15","downloads":395},{"category":"without_mirrors","date":"2026-08-16","downloads":318},{"category":"without_mirrors","date":"2026-08-17","downloads":782},{"category":"without_mirrors","date":"2026-08-18","downloads":845},{"category":"without_mirrors","date":"2026-08-19","downloads":1464},{"category":"without_mirrors","date":"2026-08-20","downloads":821},{"category":"without_mirrors","date":"2026-08-21","downloads":688},{"category":"without_mirrors","date":"2026-08-22","downloads":56},{"category":"without_mirrors","date":"2026-08-23","downloads":208},{"category":"without_mirrors","date":"2026-08-24","downloads":939},{"category":"without_mirrors","date":"2026-08-25","downloads":423},{"category":"without_mirrors","date":"2026-08-26","downloads":1244},{"category":"without_mirrors","date":"2026-08-27","downloads":869},{"category":"without_mirrors","date":"2026-08-28","downloads":934},{"category":"without_mirrors","date":"2026-08-29","downloads":77},{"category":"without_mirrors","date":"2026-08-30","downloads":64},{"category":"without_mirrors","date":"2026-08-31","downloads":842},{"category":"without_mirrors","date":"2026-09-01","downloads":2634},{"category":"without_mirrors","date":"2026-09-02","downloads":1808},{"category":"without_mirrors","date":"2026-09-03","downloads":1643},{"category":"without_mirrors","date":"2026-09-04","downloads":675},{"category":"without_mirrors","date":"2026-09-05","downloads":805},{"category":"without_mirrors","date":"2026-09-06","downloads":1766},{"category":"without_mirrors","date":"2026-09-07","downloads":323},{"category":"without_mirrors","date":"2026-09-08","downloads":989},{"category":"without_mirrors","date":"2026-09-09","downloads":1135},{"category":"without_mirrors","date":"2026-09-10","downloads":807},{"category":"without_mirrors","date":"2026-09-11","downloads":942},{"category":"without_mirrors","date":"2026-09-12","downloads":360},{"category":"without_mirrors","date":"2026-09-13","downloads":245},{"category":"without_mirrors","date":"2026-09-14","downloads":1109},{"category":"without_mirrors","date":"2026-09-15","downloads":1338},{"category":"without_mirrors","date":"2026-09-16","downloads":1577},{"category":"without_mirrors","date":"2026-09-17","downloads":1131},{"category":"without_mirrors","date":"2026-09-18","downloads":744},{"category":"without_mirrors","date":"2026-09-19","downloads":261},{"category":"without_mirrors","date":"2026-09-20","downloads":134},{"category":"without_mirrors","date":"2026-09-21","downloads":345},{"category":"without_mirrors","date":"2026-09-22","downloads":466},{"category":"without_mirrors","date":"2026-09-23","downloads":453},{"category":"without_mirrors","date":"2026-09-24","downloads":458},{"category":"without_mirrors","date":"2026-09-25","downloads":418},{"category":"without_mirrors","date":"2026-09-26","downloads":214},{"category":"without_mirrors","date":"2026-09-27","downloads":121},{"category":"without_mirrors","date":"2026-09-28","downloads":525},{"category":"without_mirrors","date":"2026-09-29","downloads":604},{"category":"without_mirrors","date":"2026-09-30","downloads":798},{"category":"without_mirrors","date":"2026-10-01","downloads":174},{"category":"without_mirrors","date":"2026-10-02","downloads":89},{"category":"without_mirrors","date":"2026-10-03","downloads":164},{"category":"without_mirrors","date":"2026-10-04","downloads":88},{"category":"without_mirrors","date":"2026-10-05","downloads":147},{"category":"without_mirrors","date":"2026-10-06","downloads":275},{"category":"without_mirrors","date":"2026-10-07","downloads":223},{"category":"without_mirrors","date":"2026-10-08","downloads":232},{"category":"without_mirrors","date":"2026-10-09","downloads":153},{"category":"without_mirrors","date":"2026-10-10","downloads":122},{"category":"without_mirrors","date":"2026-10-11","downloads":62},{"category":"without_mirrors","date":"2026-10-12","downloads":100},{"category":"without_mirrors","date":"2026-10-13","downloads":164},{"category":"without_mirrors","date":"2026-10-14","downloads":190},{"category":"without_mirrors","date":"2026-10-15","downloads":159},{"category":"without_mirrors","date":"2026-10-16","downloads":168},{"category":"without_mirrors","date":"2026-10-17","downloads":67},{"category":"without_mirrors","date":"2026-10-18","downloads":213},{"category":"without_mirrors","date":"2026-10-19","downloads":535},{"category":"without_mirrors","date":"2026-10-20","downloads":862},{"category":"without_mirrors","date":"2026-10-21","downloads":164},{"category":"without_mirrors","date":"2026-10-22","downloads":149},{"category":"without_mirrors","date":"2026-10-23","downloads":115},{"category":"without_mirrors","date":"2026-10-24","downloads":151},{"category":"without_mirrors","date":"2026-10-25","downloads":14},{"category":"without_mirrors","date":"2026-10-26","downloads":87},{"category":"without_mirrors","date":"2026-10-27","downloads":133},{"category":"without_mirrors","date":"2026-10-28","downloads":168}],"package":"mooncake-transfer-engine-non-cuda","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:12:45.942595+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-non-cuda/python_major","payload":{"data":[{"category":"3","date":"2026-04-08","downloads":36},{"category":"3","date":"2026-04-09","downloads":37},{"category":"3","date":"2026-04-10","downloads":33},{"category":"3","date":"2026-04-11","downloads":32},{"category":"3","date":"2026-04-12","downloads":62},{"category":"3","date":"2026-04-13","downloads":54},{"category":"3","date":"2026-04-14","downloads":8},{"category":"3","date":"2026-04-15","downloads":30},{"category":"3","date":"2026-04-16","downloads":12},{"category":"3","date":"2026-04-17","downloads":47},{"category":"3","date":"2026-04-18","downloads":6},{"category":"3","date":"2026-04-19","downloads":6},{"category":"3","date":"2026-04-20","downloads":20},{"category":"3","date":"2026-04-21","downloads":34},{"category":"3","date":"2026-04-22","downloads":38},{"category":"3","date":"2026-04-23","downloads":14},{"category":"3","date":"2026-04-24","downloads":11},{"category":"3","date":"2026-04-25","downloads":26},{"category":"3","date":"2026-04-26","downloads":8},{"category":"3","date":"2026-04-27","downloads":91},{"category":"3","date":"2026-04-28","downloads":14},{"category":"3","date":"2026-04-29","downloads":36},{"category":"3","date":"2026-04-30","downloads":14},{"category":"3","date":"2026-05-01","downloads":98},{"category":"3","date":"2026-05-02","downloads":72},{"category":"3","date":"2026-05-03","downloads":6},{"category":"3","date":"2026-05-04","downloads":10},{"category":"3","date":"2026-05-05","downloads":10},{"category":"3","date":"2026-05-06","downloads":14},{"category":"3","date":"2026-05-07","downloads":28},{"category":"3","date":"2026-05-08","downloads":20},{"category":"3","date":"2026-05-09","downloads":64},{"category":"3","date":"2026-05-10","downloads":4},{"category":"3","date":"2026-05-11","downloads":6},{"category":"3","date":"2026-05-12","downloads":30},{"category":"3","date":"2026-05-13","downloads":27},{"category":"3","date":"2026-05-14","downloads":59},{"category":"3","date":"2026-05-15","downloads":43},{"category":"3","date":"2026-05-16","downloads":47},{"category":"3","date":"2026-05-17","downloads":18},{"category":"3","date":"2026-05-18","downloads":54},{"category":"3","date":"2026-05-19","downloads":22},{"category":"3","date":"2026-05-20","downloads":26},{"category":"3","date":"2026-05-21","downloads":37},{"category":"3","date":"2026-05-22","downloads":81},{"category":"3","date":"2026-05-23","downloads":47},{"category":"3","date":"2026-05-24","downloads":39},{"category":"3","date":"2026-05-25","downloads":35},{"category":"3","date":"2026-05-26","downloads":13},{"category":"3","date":"2026-05-27","downloads":20},{"category":"3","date":"2026-05-28","downloads":68},{"category":"3","date":"2026-05-29","downloads":48},{"category":"3","date":"2026-05-30","downloads":74},{"category":"3","date":"2026-05-31","downloads":26},{"category":"3","date":"2026-06-01","downloads":24},{"category":"3","date":"2026-06-02","downloads":140},{"category":"3","date":"2026-06-03","downloads":36},{"category":"3","date":"2026-06-04","downloads":77},{"category":"3","date":"2026-06-05","downloads":143},{"category":"3","date":"2026-06-06","downloads":140},{"category":"3","date":"2026-06-07","downloads":154},{"category":"3","date":"2026-06-08","downloads":227},{"category":"3","date":"2026-06-09","downloads":124},{"category":"3","date":"2026-06-10","downloads":56},{"category":"3","date":"2026-06-11","downloads":69},{"category":"3","date":"2026-06-12","downloads":20},{"category":"3","date":"2026-06-13","downloads":24},{"category":"3","date":"2026-06-14","downloads":12},{"category":"3","date":"2026-06-15","downloads":24},{"category":"3","date":"2026-06-16","downloads":14},{"category":"3","date":"2026-06-17","downloads":22},{"category":"3","date":"2026-06-18","downloads":51},{"category":"3","date":"2026-06-19","downloads":10},{"category":"3","date":"2026-06-20","downloads":10},{"category":"3","date":"2026-06-21","downloads":2},{"category":"3","date":"2026-06-22","downloads":51},{"category":"3","date":"2026-06-23","downloads":224},{"category":"3","date":"2026-06-24","downloads":219},{"category":"3","date":"2026-06-25","downloads":85},{"category":"3","date":"2026-06-26","downloads":20},{"category":"3","date":"2026-06-27","downloads":11},{"category":"3","date":"2026-06-28","downloads":24},{"category":"3","date":"2026-06-29","downloads":134},{"category":"3","date":"2026-06-30","downloads":337},{"category":"3","date":"2026-07-01","downloads":53},{"category":"3","date":"2026-07-02","downloads":54},{"category":"3","date":"2026-07-03","downloads":135},{"category":"3","date":"2026-07-04","downloads":78},{"category":"3","date":"2026-07-05","downloads":48},{"category":"3","date":"2026-07-06","downloads":82},{"category":"3","date":"2026-07-07","downloads":58},{"category":"3","date":"2026-07-08","downloads":58},{"category":"3","date":"2026-07-09","downloads":147},{"category":"3","date":"2026-07-10","downloads":108},{"category":"3","date":"2026-07-11","downloads":46},{"category":"3","date":"2026-07-12","downloads":220},{"category":"3","date":"2026-07-13","downloads":63},{"category":"3","date":"2026-07-14","downloads":305},{"category":"3","date":"2026-07-15","downloads":21},{"category":"3","date":"2026-07-16","downloads":34},{"category":"3","date":"2026-07-17","downloads":38},{"category":"3","date":"2026-07-18","downloads":31},{"category":"3","date":"2026-07-19","downloads":7},{"category":"3","date":"2026-07-20","downloads":12},{"category":"3","date":"2026-07-21","downloads":279},{"category":"3","date":"2026-07-22","downloads":50},{"category":"3","date":"2026-07-23","downloads":152},{"category":"3","date":"2026-07-24","downloads":235},{"category":"3","date":"2026-07-25","downloads":74},{"category":"3","date":"2026-07-26","downloads":148},{"category":"3","date":"2026-07-27","downloads":674},{"category":"3","date":"2026-07-28","downloads":1269},{"category":"3","date":"2026-07-29","downloads":1278},{"category":"3","date":"2026-07-30","downloads":1208},{"category":"3","date":"2026-07-31","downloads":1387},{"category":"3","date":"2026-08-01","downloads":552},{"category":"3","date":"2026-08-02","downloads":555},{"category":"3","date":"2026-08-03","downloads":504},{"category":"3","date":"2026-08-04","downloads":607},{"category":"3","date":"2026-08-05","downloads":622},{"category":"3","date":"2026-08-06","downloads":569},{"category":"3","date":"2026-08-07","downloads":638},{"category":"3","date":"2026-08-08","downloads":372},{"category":"3","date":"2026-08-09","downloads":350},{"category":"3","date":"2026-08-10","downloads":602},{"category":"3","date":"2026-08-11","downloads":752},{"category":"3","date":"2026-08-12","downloads":724},{"category":"3","date":"2026-08-13","downloads":827},{"category":"3","date":"2026-08-14","downloads":673},{"category":"3","date":"2026-08-15","downloads":394},{"category":"3","date":"2026-08-16","downloads":318},{"category":"3","date":"2026-08-17","downloads":779},{"category":"3","date":"2026-08-18","downloads":836},{"category":"3","date":"2026-08-19","downloads":1459},{"category":"3","date":"2026-08-20","downloads":805},{"category":"3","date":"2026-08-21","downloads":665},{"category":"3","date":"2026-08-22","downloads":53},{"category":"3","date":"2026-08-23","downloads":208},{"category":"3","date":"2026-08-24","downloads":938},{"category":"3","date":"2026-08-25","downloads":421},{"category":"3","date":"2026-08-26","downloads":1027},{"category":"3","date":"2026-08-27","downloads":783},{"category":"3","date":"2026-08-28","downloads":832},{"category":"3","date":"2026-08-29","downloads":72},{"category":"3","date":"2026-08-30","downloads":63},{"category":"3","date":"2026-08-31","downloads":623},{"category":"3","date":"2026-09-01","downloads":2586},{"category":"3","date":"2026-09-02","downloads":1723},{"category":"3","date":"2026-09-03","downloads":1537},{"category":"3","date":"2026-09-04","downloads":657},{"category":"3","date":"2026-09-05","downloads":797},{"category":"3","date":"2026-09-06","downloads":1756},{"category":"3","date":"2026-09-07","downloads":303},{"category":"3","date":"2026-09-08","downloads":955},{"category":"3","date":"2026-09-09","downloads":1134},{"category":"3","date":"2026-09-10","downloads":796},{"category":"3","date":"2026-09-11","downloads":941},{"category":"3","date":"2026-09-12","downloads":353},{"category":"3","date":"2026-09-13","downloads":241},{"category":"3","date":"2026-09-14","downloads":1096},{"category":"3","date":"2026-09-15","downloads":1325},{"category":"3","date":"2026-09-16","downloads":1574},{"category":"3","date":"2026-09-17","downloads":1107},{"category":"3","date":"2026-09-18","downloads":740},{"category":"3","date":"2026-09-19","downloads":194},{"category":"3","date":"2026-09-20","downloads":126},{"category":"3","date":"2026-09-21","downloads":329},{"category":"3","date":"2026-09-22","downloads":464},{"category":"3","date":"2026-09-23","downloads":452},{"category":"3","date":"2026-09-24","downloads":445},{"category":"3","date":"2026-09-25","downloads":402},{"category":"3","date":"2026-09-26","downloads":145},{"category":"3","date":"2026-09-27","downloads":94},{"category":"3","date":"2026-09-28","downloads":392},{"category":"3","date":"2026-09-29","downloads":596},{"category":"3","date":"2026-09-30","downloads":790},{"category":"3","date":"2026-10-01","downloads":168},{"category":"3","date":"2026-10-02","downloads":85},{"category":"3","date":"2026-10-03","downloads":160},{"category":"3","date":"2026-10-04","downloads":79},{"category":"3","date":"2026-10-05","downloads":144},{"category":"3","date":"2026-10-06","downloads":265},{"category":"3","date":"2026-10-07","downloads":223},{"category":"3","date":"2026-10-08","downloads":228},{"category":"3","date":"2026-10-09","downloads":145},{"category":"3","date":"2026-10-10","downloads":120},{"category":"3","date":"2026-10-11","downloads":59},{"category":"3","date":"2026-10-12","downloads":97},{"category":"3","date":"2026-10-13","downloads":156},{"category":"3","date":"2026-10-14","downloads":182},{"category":"3","date":"2026-10-15","downloads":146},{"category":"3","date":"2026-10-16","downloads":153},{"category":"3","date":"2026-10-17","downloads":65},{"category":"3","date":"2026-10-18","downloads":204},{"category":"3","date":"2026-10-19","downloads":525},{"category":"3","date":"2026-10-20","downloads":856},{"category":"3","date":"2026-10-21","downloads":159},{"category":"3","date":"2026-10-22","downloads":140},{"category":"3","date":"2026-10-23","downloads":107},{"category":"3","date":"2026-10-24","downloads":146},{"category":"3","date":"2026-10-25","downloads":13},{"category":"3","date":"2026-10-26","downloads":85},{"category":"3","date":"2026-10-27","downloads":130},{"category":"3","date":"2026-10-28","downloads":162},{"category":"null","date":"2026-04-08","downloads":1},{"category":"null","date":"2026-04-10","downloads":5},{"category":"null","date":"2026-04-11","downloads":17},{"category":"null","date":"2026-04-12","downloads":1},{"category":"null","date":"2026-04-13","downloads":4},{"category":"null","date":"2026-04-14","downloads":29},{"category":"null","date":"2026-04-15","downloads":42},{"category":"null","date":"2026-04-16","downloads":2},{"category":"null","date":"2026-04-17","downloads":1},{"category":"null","date":"2026-04-19","downloads":14},{"category":"null","date":"2026-04-20","downloads":2},{"category":"null","date":"2026-04-22","downloads":151},{"category":"null","date":"2026-04-23","downloads":8},{"category":"null","date":"2026-04-24","downloads":5},{"category":"null","date":"2026-04-25","downloads":10},{"category":"null","date":"2026-04-26","downloads":4},{"category":"null","date":"2026-04-27","downloads":10},{"category":"null","date":"2026-04-28","downloads":4},{"category":"null","date":"2026-04-29","downloads":2},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-03","downloads":2},{"category":"null","date":"2026-05-05","downloads":56},{"category":"null","date":"2026-05-06","downloads":6},{"category":"null","date":"2026-05-07","downloads":4},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":1},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":6},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":3},{"category":"null","date":"2026-05-19","downloads":1},{"category":"null","date":"2026-05-21","downloads":145},{"category":"null","date":"2026-05-22","downloads":8},{"category":"null","date":"2026-05-23","downloads":162},{"category":"null","date":"2026-05-24","downloads":9},{"category":"null","date":"2026-05-25","downloads":21},{"category":"null","date":"2026-05-26","downloads":3},{"category":"null","date":"2026-05-27","downloads":16},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":7},{"category":"null","date":"2026-05-30","downloads":2},{"category":"null","date":"2026-05-31","downloads":8},{"category":"null","date":"2026-06-02","downloads":10},{"category":"null","date":"2026-06-03","downloads":17},{"category":"null","date":"2026-06-04","downloads":6},{"category":"null","date":"2026-06-05","downloads":34},{"category":"null","date":"2026-06-06","downloads":36},{"category":"null","date":"2026-06-07","downloads":9},{"category":"null","date":"2026-06-08","downloads":20},{"category":"null","date":"2026-06-09","downloads":10},{"category":"null","date":"2026-06-10","downloads":9},{"category":"null","date":"2026-06-12","downloads":3},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":1},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":2},{"category":"null","date":"2026-06-17","downloads":7},{"category":"null","date":"2026-06-18","downloads":177},{"category":"null","date":"2026-06-19","downloads":2},{"category":"null","date":"2026-06-20","downloads":4},{"category":"null","date":"2026-06-21","downloads":1},{"category":"null","date":"2026-06-22","downloads":6},{"category":"null","date":"2026-06-23","downloads":53},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":23},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-01","downloads":4},{"category":"null","date":"2026-07-02","downloads":1},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-21","downloads":1},{"category":"null","date":"2026-07-22","downloads":47},{"category":"null","date":"2026-07-23","downloads":71},{"category":"null","date":"2026-07-24","downloads":120},{"category":"null","date":"2026-07-25","downloads":4},{"category":"null","date":"2026-07-26","downloads":8},{"category":"null","date":"2026-07-27","downloads":11},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-29","downloads":7},{"category":"null","date":"2026-07-30","downloads":24},{"category":"null","date":"2026-07-31","downloads":16},{"category":"null","date":"2026-08-01","downloads":4},{"category":"null","date":"2026-08-02","downloads":3},{"category":"null","date":"2026-08-03","downloads":4},{"category":"null","date":"2026-08-04","downloads":10},{"category":"null","date":"2026-08-05","downloads":9},{"category":"null","date":"2026-08-06","downloads":9},{"category":"null","date":"2026-08-07","downloads":13},{"category":"null","date":"2026-08-08","downloads":1},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":8},{"category":"null","date":"2026-08-13","downloads":4},{"category":"null","date":"2026-08-14","downloads":32},{"category":"null","date":"2026-08-15","downloads":1},{"category":"null","date":"2026-08-17","downloads":3},{"category":"null","date":"2026-08-18","downloads":9},{"category":"null","date":"2026-08-19","downloads":5},{"category":"null","date":"2026-08-20","downloads":16},{"category":"null","date":"2026-08-21","downloads":23},{"category":"null","date":"2026-08-22","downloads":3},{"category":"null","date":"2026-08-24","downloads":1},{"category":"null","date":"2026-08-25","downloads":2},{"category":"null","date":"2026-08-26","downloads":217},{"category":"null","date":"2026-08-27","downloads":86},{"category":"null","date":"2026-08-28","downloads":102},{"category":"null","date":"2026-08-29","downloads":5},{"category":"null","date":"2026-08-30","downloads":1},{"category":"null","date":"2026-08-31","downloads":219},{"category":"null","date":"2026-09-01","downloads":48},{"category":"null","date":"2026-09-02","downloads":85},{"category":"null","date":"2026-09-03","downloads":106},{"category":"null","date":"2026-09-04","downloads":18},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":10},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":34},{"category":"null","date":"2026-09-09","downloads":1},{"category":"null","date":"2026-09-10","downloads":11},{"category":"null","date":"2026-09-11","downloads":1},{"category":"null","date":"2026-09-12","downloads":7},{"category":"null","date":"2026-09-13","downloads":4},{"category":"null","date":"2026-09-14","downloads":13},{"category":"null","date":"2026-09-15","downloads":13},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":24},{"category":"null","date":"2026-09-18","downloads":4},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":8},{"category":"null","date":"2026-09-21","downloads":16},{"category":"null","date":"2026-09-22","downloads":2},{"category":"null","date":"2026-09-23","downloads":1},{"category":"null","date":"2026-09-24","downloads":13},{"category":"null","date":"2026-09-25","downloads":16},{"category":"null","date":"2026-09-26","downloads":69},{"category":"null","date":"2026-09-27","downloads":27},{"category":"null","date":"2026-09-28","downloads":133},{"category":"null","date":"2026-09-29","downloads":8},{"category":"null","date":"2026-09-30","downloads":8},{"category":"null","date":"2026-10-01","downloads":6},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":4},{"category":"null","date":"2026-10-04","downloads":9},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-08","downloads":4},{"category":"null","date":"2026-10-09","downloads":8},{"category":"null","date":"2026-10-10","downloads":2},{"category":"null","date":"2026-10-11","downloads":3},{"category":"null","date":"2026-10-12","downloads":3},{"category":"null","date":"2026-10-13","downloads":8},{"category":"null","date":"2026-10-14","downloads":8},{"category":"null","date":"2026-10-15","downloads":13},{"category":"null","date":"2026-10-16","downloads":15},{"category":"null","date":"2026-10-17","downloads":2},{"category":"null","date":"2026-10-18","downloads":9},{"category":"null","date":"2026-10-19","downloads":10},{"category":"null","date":"2026-10-20","downloads":6},{"category":"null","date":"2026-10-21","downloads":5},{"category":"null","date":"2026-10-22","downloads":9},{"category":"null","date":"2026-10-23","downloads":8},{"category":"null","date":"2026-10-24","downloads":5},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-26","downloads":2},{"category":"null","date":"2026-10-27","downloads":3},{"category":"null","date":"2026-10-28","downloads":6}],"package":"mooncake-transfer-engine-non-cuda","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:12:49.029341+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-non-cuda/python_minor","payload":{"data":[{"category":"3.10","date":"2026-04-10","downloads":17},{"category":"3.10","date":"2026-04-11","downloads":4},{"category":"3.10","date":"2026-04-12","downloads":14},{"category":"3.10","date":"2026-04-13","downloads":42},{"category":"3.10","date":"2026-04-14","downloads":2},{"category":"3.10","date":"2026-04-17","downloads":1},{"category":"3.10","date":"2026-04-21","downloads":3},{"category":"3.10","date":"2026-04-22","downloads":5},{"category":"3.10","date":"2026-04-23","downloads":3},{"category":"3.10","date":"2026-04-24","downloads":2},{"category":"3.10","date":"2026-04-26","downloads":4},{"category":"3.10","date":"2026-04-27","downloads":3},{"category":"3.10","date":"2026-04-28","downloads":2},{"category":"3.10","date":"2026-04-29","downloads":10},{"category":"3.10","date":"2026-04-30","downloads":4},{"category":"3.10","date":"2026-05-03","downloads":2},{"category":"3.10","date":"2026-05-04","downloads":2},{"category":"3.10","date":"2026-05-05","downloads":2},{"category":"3.10","date":"2026-05-06","downloads":2},{"category":"3.10","date":"2026-05-13","downloads":3},{"category":"3.10","date":"2026-05-14","downloads":7},{"category":"3.10","date":"2026-05-15","downloads":17},{"category":"3.10","date":"2026-05-16","downloads":2},{"category":"3.10","date":"2026-05-17","downloads":3},{"category":"3.10","date":"2026-05-18","downloads":5},{"category":"3.10","date":"2026-05-19","downloads":4},{"category":"3.10","date":"2026-05-21","downloads":5},{"category":"3.10","date":"2026-05-22","downloads":6},{"category":"3.10","date":"2026-05-23","downloads":9},{"category":"3.10","date":"2026-05-24","downloads":2},{"category":"3.10","date":"2026-05-25","downloads":2},{"category":"3.10","date":"2026-05-26","downloads":3},{"category":"3.10","date":"2026-05-27","downloads":1},{"category":"3.10","date":"2026-05-28","downloads":5},{"category":"3.10","date":"2026-05-29","downloads":2},{"category":"3.10","date":"2026-05-30","downloads":6},{"category":"3.10","date":"2026-05-31","downloads":16},{"category":"3.10","date":"2026-06-01","downloads":2},{"category":"3.10","date":"2026-06-02","downloads":60},{"category":"3.10","date":"2026-06-03","downloads":10},{"category":"3.10","date":"2026-06-04","downloads":8},{"category":"3.10","date":"2026-06-05","downloads":91},{"category":"3.10","date":"2026-06-06","downloads":140},{"category":"3.10","date":"2026-06-07","downloads":143},{"category":"3.10","date":"2026-06-08","downloads":147},{"category":"3.10","date":"2026-06-09","downloads":66},{"category":"3.10","date":"2026-06-10","downloads":10},{"category":"3.10","date":"2026-06-11","downloads":43},{"category":"3.10","date":"2026-06-12","downloads":8},{"category":"3.10","date":"2026-06-13","downloads":18},{"category":"3.10","date":"2026-06-14","downloads":3},{"category":"3.10","date":"2026-06-15","downloads":4},{"category":"3.10","date":"2026-06-16","downloads":4},{"category":"3.10","date":"2026-06-17","downloads":2},{"category":"3.10","date":"2026-06-18","downloads":11},{"category":"3.10","date":"2026-06-19","downloads":6},{"category":"3.10","date":"2026-06-22","downloads":13},{"category":"3.10","date":"2026-06-23","downloads":216},{"category":"3.10","date":"2026-06-24","downloads":204},{"category":"3.10","date":"2026-06-25","downloads":65},{"category":"3.10","date":"2026-06-26","downloads":8},{"category":"3.10","date":"2026-06-27","downloads":1},{"category":"3.10","date":"2026-06-28","downloads":2},{"category":"3.10","date":"2026-06-29","downloads":90},{"category":"3.10","date":"2026-06-30","downloads":300},{"category":"3.10","date":"2026-07-01","downloads":13},{"category":"3.10","date":"2026-07-02","downloads":14},{"category":"3.10","date":"2026-07-03","downloads":99},{"category":"3.10","date":"2026-07-06","downloads":8},{"category":"3.10","date":"2026-07-07","downloads":2},{"category":"3.10","date":"2026-07-09","downloads":39},{"category":"3.10","date":"2026-07-10","downloads":12},{"category":"3.10","date":"2026-07-11","downloads":15},{"category":"3.10","date":"2026-07-12","downloads":12},{"category":"3.10","date":"2026-07-13","downloads":3},{"category":"3.10","date":"2026-07-14","downloads":195},{"category":"3.10","date":"2026-07-15","downloads":14},{"category":"3.10","date":"2026-07-16","downloads":10},{"category":"3.10","date":"2026-07-18","downloads":1},{"category":"3.10","date":"2026-07-21","downloads":122},{"category":"3.10","date":"2026-07-22","downloads":6},{"category":"3.10","date":"2026-07-23","downloads":21},{"category":"3.10","date":"2026-07-24","downloads":137},{"category":"3.10","date":"2026-07-25","downloads":1},{"category":"3.10","date":"2026-07-26","downloads":5},{"category":"3.10","date":"2026-07-27","downloads":559},{"category":"3.10","date":"2026-07-28","downloads":1219},{"category":"3.10","date":"2026-07-29","downloads":1193},{"category":"3.10","date":"2026-07-30","downloads":1188},{"category":"3.10","date":"2026-07-31","downloads":1121},{"category":"3.10","date":"2026-08-01","downloads":544},{"category":"3.10","date":"2026-08-02","downloads":543},{"category":"3.10","date":"2026-08-03","downloads":444},{"category":"3.10","date":"2026-08-04","downloads":286},{"category":"3.10","date":"2026-08-05","downloads":281},{"category":"3.10","date":"2026-08-06","downloads":295},{"category":"3.10","date":"2026-08-07","downloads":287},{"category":"3.10","date":"2026-08-08","downloads":291},{"category":"3.10","date":"2026-08-09","downloads":288},{"category":"3.10","date":"2026-08-10","downloads":275},{"category":"3.10","date":"2026-08-11","downloads":287},{"category":"3.10","date":"2026-08-12","downloads":284},{"category":"3.10","date":"2026-08-13","downloads":359},{"category":"3.10","date":"2026-08-14","downloads":296},{"category":"3.10","date":"2026-08-15","downloads":288},{"category":"3.10","date":"2026-08-16","downloads":284},{"category":"3.10","date":"2026-08-17","downloads":402},{"category":"3.10","date":"2026-08-18","downloads":564},{"category":"3.10","date":"2026-08-19","downloads":797},{"category":"3.10","date":"2026-08-20","downloads":75},{"category":"3.10","date":"2026-08-21","downloads":7},{"category":"3.10","date":"2026-08-22","downloads":7},{"category":"3.10","date":"2026-08-23","downloads":30},{"category":"3.10","date":"2026-08-24","downloads":142},{"category":"3.10","date":"2026-08-25","downloads":41},{"category":"3.10","date":"2026-08-26","downloads":81},{"category":"3.10","date":"2026-08-27","downloads":50},{"category":"3.10","date":"2026-08-28","downloads":45},{"category":"3.10","date":"2026-08-29","downloads":5},{"category":"3.10","date":"2026-08-30","downloads":4},{"category":"3.10","date":"2026-08-31","downloads":14},{"category":"3.10","date":"2026-09-01","downloads":1672},{"category":"3.10","date":"2026-09-02","downloads":843},{"category":"3.10","date":"2026-09-03","downloads":73},{"category":"3.10","date":"2026-09-04","downloads":9},{"category":"3.10","date":"2026-09-05","downloads":49},{"category":"3.10","date":"2026-09-06","downloads":45},{"category":"3.10","date":"2026-09-07","downloads":10},{"category":"3.10","date":"2026-09-08","downloads":10},{"category":"3.10","date":"2026-09-09","downloads":18},{"category":"3.10","date":"2026-09-10","downloads":3},{"category":"3.10","date":"2026-09-11","downloads":2},{"category":"3.10","date":"2026-09-12","downloads":1},{"category":"3.10","date":"2026-09-13","downloads":3},{"category":"3.10","date":"2026-09-14","downloads":7},{"category":"3.10","date":"2026-09-15","downloads":2},{"category":"3.10","date":"2026-09-16","downloads":63},{"category":"3.10","date":"2026-09-17","downloads":6},{"category":"3.10","date":"2026-09-18","downloads":1},{"category":"3.10","date":"2026-09-19","downloads":3},{"category":"3.10","date":"2026-09-20","downloads":3},{"category":"3.10","date":"2026-09-21","downloads":3},{"category":"3.10","date":"2026-09-22","downloads":9},{"category":"3.10","date":"2026-09-23","downloads":6},{"category":"3.10","date":"2026-09-24","downloads":4},{"category":"3.10","date":"2026-09-25","downloads":1},{"category":"3.10","date":"2026-09-26","downloads":2},{"category":"3.10","date":"2026-09-27","downloads":1},{"category":"3.10","date":"2026-09-28","downloads":5},{"category":"3.10","date":"2026-09-30","downloads":364},{"category":"3.10","date":"2026-10-01","downloads":3},{"category":"3.10","date":"2026-10-02","downloads":32},{"category":"3.10","date":"2026-10-03","downloads":67},{"category":"3.10","date":"2026-10-04","downloads":31},{"category":"3.10","date":"2026-10-05","downloads":20},{"category":"3.10","date":"2026-10-06","downloads":10},{"category":"3.10","date":"2026-10-07","downloads":3},{"category":"3.10","date":"2026-10-08","downloads":15},{"category":"3.10","date":"2026-10-09","downloads":2},{"category":"3.10","date":"2026-10-10","downloads":20},{"category":"3.10","date":"2026-10-11","downloads":6},{"category":"3.10","date":"2026-10-12","downloads":2},{"category":"3.10","date":"2026-10-13","downloads":13},{"category":"3.10","date":"2026-10-14","downloads":2},{"category":"3.10","date":"2026-10-15","downloads":2},{"category":"3.10","date":"2026-10-16","downloads":13},{"category":"3.10","date":"2026-10-17","downloads":12},{"category":"3.10","date":"2026-10-18","downloads":13},{"category":"3.10","date":"2026-10-19","downloads":19},{"category":"3.10","date":"2026-10-20","downloads":10},{"category":"3.10","date":"2026-10-21","downloads":10},{"category":"3.10","date":"2026-10-22","downloads":6},{"category":"3.10","date":"2026-10-23","downloads":9},{"category":"3.10","date":"2026-10-24","downloads":7},{"category":"3.10","date":"2026-10-25","downloads":2},{"category":"3.10","date":"2026-10-26","downloads":11},{"category":"3.10","date":"2026-10-27","downloads":12},{"category":"3.10","date":"2026-10-28","downloads":10},{"category":"3.11","date":"2026-04-08","downloads":26},{"category":"3.11","date":"2026-04-09","downloads":28},{"category":"3.11","date":"2026-04-10","downloads":12},{"category":"3.11","date":"2026-04-11","downloads":24},{"category":"3.11","date":"2026-04-12","downloads":44},{"category":"3.11","date":"2026-04-13","downloads":4},{"category":"3.11","date":"2026-04-15","downloads":24},{"category":"3.11","date":"2026-04-17","downloads":24},{"category":"3.11","date":"2026-04-18","downloads":2},{"category":"3.11","date":"2026-04-20","downloads":4},{"category":"3.11","date":"2026-04-21","downloads":26},{"category":"3.11","date":"2026-04-22","downloads":20},{"category":"3.11","date":"2026-04-23","downloads":4},{"category":"3.11","date":"2026-04-24","downloads":2},{"category":"3.11","date":"2026-04-25","downloads":20},{"category":"3.11","date":"2026-04-27","downloads":80},{"category":"3.11","date":"2026-04-28","downloads":8},{"category":"3.11","date":"2026-04-29","downloads":20},{"category":"3.11","date":"2026-05-01","downloads":68},{"category":"3.11","date":"2026-05-02","downloads":68},{"category":"3.11","date":"2026-05-06","downloads":4},{"category":"3.11","date":"2026-05-07","downloads":12},{"category":"3.11","date":"2026-05-08","downloads":16},{"category":"3.11","date":"2026-05-09","downloads":56},{"category":"3.11","date":"2026-05-11","downloads":4},{"category":"3.11","date":"2026-05-12","downloads":16},{"category":"3.11","date":"2026-05-13","downloads":16},{"category":"3.11","date":"2026-05-14","downloads":44},{"category":"3.11","date":"2026-05-15","downloads":16},{"category":"3.11","date":"2026-05-16","downloads":35},{"category":"3.11","date":"2026-05-17","downloads":6},{"category":"3.11","date":"2026-05-18","downloads":20},{"category":"3.11","date":"2026-05-19","downloads":8},{"category":"3.11","date":"2026-05-20","downloads":16},{"category":"3.11","date":"2026-05-21","downloads":4},{"category":"3.11","date":"2026-05-22","downloads":60},{"category":"3.11","date":"2026-05-23","downloads":8},{"category":"3.11","date":"2026-05-24","downloads":2},{"category":"3.11","date":"2026-05-25","downloads":12},{"category":"3.11","date":"2026-05-26","downloads":2},{"category":"3.11","date":"2026-05-27","downloads":4},{"category":"3.11","date":"2026-05-28","downloads":18},{"category":"3.11","date":"2026-05-29","downloads":26},{"category":"3.11","date":"2026-05-30","downloads":66},{"category":"3.11","date":"2026-06-01","downloads":20},{"category":"3.11","date":"2026-06-02","downloads":2},{"category":"3.11","date":"2026-06-03","downloads":4},{"category":"3.11","date":"2026-06-04","downloads":49},{"category":"3.11","date":"2026-06-05","downloads":52},{"category":"3.11","date":"2026-06-07","downloads":2},{"category":"3.11","date":"2026-06-08","downloads":48},{"category":"3.11","date":"2026-06-09","downloads":18},{"category":"3.11","date":"2026-06-10","downloads":26},{"category":"3.11","date":"2026-06-11","downloads":20},{"category":"3.11","date":"2026-06-12","downloads":4},{"category":"3.11","date":"2026-06-13","downloads":2},{"category":"3.11","date":"2026-06-14","downloads":2},{"category":"3.11","date":"2026-06-15","downloads":12},{"category":"3.11","date":"2026-06-16","downloads":10},{"category":"3.11","date":"2026-06-17","downloads":14},{"category":"3.11","date":"2026-06-18","downloads":24},{"category":"3.11","date":"2026-06-22","downloads":10},{"category":"3.11","date":"2026-06-24","downloads":10},{"category":"3.11","date":"2026-06-25","downloads":12},{"category":"3.11","date":"2026-06-26","downloads":4},{"category":"3.11","date":"2026-06-27","downloads":4},{"category":"3.11","date":"2026-06-29","downloads":16},{"category":"3.11","date":"2026-06-30","downloads":24},{"category":"3.11","date":"2026-07-01","downloads":36},{"category":"3.11","date":"2026-07-02","downloads":32},{"category":"3.11","date":"2026-07-03","downloads":32},{"category":"3.11","date":"2026-07-04","downloads":30},{"category":"3.11","date":"2026-07-06","downloads":16},{"category":"3.11","date":"2026-07-07","downloads":8},{"category":"3.11","date":"2026-07-10","downloads":28},{"category":"3.11","date":"2026-07-11","downloads":6},{"category":"3.11","date":"2026-07-12","downloads":36},{"category":"3.11","date":"2026-07-13","downloads":8},{"category":"3.11","date":"2026-07-14","downloads":4},{"category":"3.11","date":"2026-07-18","downloads":4},{"category":"3.11","date":"2026-07-20","downloads":8},{"category":"3.11","date":"2026-07-23","downloads":8},{"category":"3.11","date":"2026-07-24","downloads":9},{"category":"3.11","date":"2026-07-26","downloads":2},{"category":"3.11","date":"2026-07-27","downloads":21},{"category":"3.11","date":"2026-07-28","downloads":15},{"category":"3.11","date":"2026-07-29","downloads":20},{"category":"3.11","date":"2026-07-30","downloads":12},{"category":"3.11","date":"2026-07-31","downloads":22},{"category":"3.11","date":"2026-08-01","downloads":4},{"category":"3.11","date":"2026-08-02","downloads":12},{"category":"3.11","date":"2026-08-03","downloads":44},{"category":"3.11","date":"2026-08-04","downloads":259},{"category":"3.11","date":"2026-08-05","downloads":325},{"category":"3.11","date":"2026-08-06","downloads":159},{"category":"3.11","date":"2026-08-07","downloads":340},{"category":"3.11","date":"2026-08-08","downloads":61},{"category":"3.11","date":"2026-08-09","downloads":58},{"category":"3.11","date":"2026-08-10","downloads":325},{"category":"3.11","date":"2026-08-11","downloads":413},{"category":"3.11","date":"2026-08-12","downloads":423},{"category":"3.11","date":"2026-08-13","downloads":434},{"category":"3.11","date":"2026-08-14","downloads":314},{"category":"3.11","date":"2026-08-15","downloads":90},{"category":"3.11","date":"2026-08-16","downloads":30},{"category":"3.11","date":"2026-08-17","downloads":354},{"category":"3.11","date":"2026-08-18","downloads":252},{"category":"3.11","date":"2026-08-19","downloads":282},{"category":"3.11","date":"2026-08-20","downloads":298},{"category":"3.11","date":"2026-08-21","downloads":288},{"category":"3.11","date":"2026-08-22","downloads":44},{"category":"3.11","date":"2026-08-23","downloads":48},{"category":"3.11","date":"2026-08-24","downloads":157},{"category":"3.11","date":"2026-08-25","downloads":223},{"category":"3.11","date":"2026-08-26","downloads":264},{"category":"3.11","date":"2026-08-27","downloads":208},{"category":"3.11","date":"2026-08-28","downloads":153},{"category":"3.11","date":"2026-08-29","downloads":10},{"category":"3.11","date":"2026-08-30","downloads":6},{"category":"3.11","date":"2026-08-31","downloads":136},{"category":"3.11","date":"2026-09-01","downloads":110},{"category":"3.11","date":"2026-09-02","downloads":186},{"category":"3.11","date":"2026-09-03","downloads":202},{"category":"3.11","date":"2026-09-04","downloads":174},{"category":"3.11","date":"2026-09-05","downloads":60},{"category":"3.11","date":"2026-09-06","downloads":87},{"category":"3.11","date":"2026-09-07","downloads":76},{"category":"3.11","date":"2026-09-08","downloads":247},{"category":"3.11","date":"2026-09-09","downloads":257},{"category":"3.11","date":"2026-09-10","downloads":203},{"category":"3.11","date":"2026-09-11","downloads":214},{"category":"3.11","date":"2026-09-12","downloads":81},{"category":"3.11","date":"2026-09-13","downloads":34},{"category":"3.11","date":"2026-09-14","downloads":170},{"category":"3.11","date":"2026-09-15","downloads":309},{"category":"3.11","date":"2026-09-16","downloads":271},{"category":"3.11","date":"2026-09-17","downloads":190},{"category":"3.11","date":"2026-09-18","downloads":177},{"category":"3.11","date":"2026-09-19","downloads":49},{"category":"3.11","date":"2026-09-20","downloads":15},{"category":"3.11","date":"2026-09-21","downloads":25},{"category":"3.11","date":"2026-09-22","downloads":21},{"category":"3.11","date":"2026-09-23","downloads":46},{"category":"3.11","date":"2026-09-24","downloads":31},{"category":"3.11","date":"2026-09-25","downloads":19},{"category":"3.11","date":"2026-09-26","downloads":8},{"category":"3.11","date":"2026-09-27","downloads":9},{"category":"3.11","date":"2026-09-28","downloads":34},{"category":"3.11","date":"2026-09-29","downloads":52},{"category":"3.11","date":"2026-09-30","downloads":42},{"category":"3.11","date":"2026-10-01","downloads":34},{"category":"3.11","date":"2026-10-02","downloads":9},{"category":"3.11","date":"2026-10-03","downloads":24},{"category":"3.11","date":"2026-10-04","downloads":3},{"category":"3.11","date":"2026-10-05","downloads":23},{"category":"3.11","date":"2026-10-06","downloads":20},{"category":"3.11","date":"2026-10-07","downloads":26},{"category":"3.11","date":"2026-10-08","downloads":19},{"category":"3.11","date":"2026-10-09","downloads":29},{"category":"3.11","date":"2026-10-10","downloads":4},{"category":"3.11","date":"2026-10-11","downloads":4},{"category":"3.11","date":"2026-10-12","downloads":30},{"category":"3.11","date":"2026-10-13","downloads":32},{"category":"3.11","date":"2026-10-14","downloads":36},{"category":"3.11","date":"2026-10-15","downloads":28},{"category":"3.11","date":"2026-10-16","downloads":14},{"category":"3.11","date":"2026-10-17","downloads":3},{"category":"3.11","date":"2026-10-18","downloads":1},{"category":"3.11","date":"2026-10-19","downloads":25},{"category":"3.11","date":"2026-10-20","downloads":9},{"category":"3.11","date":"2026-10-21","downloads":15},{"category":"3.11","date":"2026-10-22","downloads":8},{"category":"3.11","date":"2026-10-23","downloads":46},{"category":"3.11","date":"2026-10-24","downloads":5},{"category":"3.11","date":"2026-10-25","downloads":4},{"category":"3.11","date":"2026-10-26","downloads":16},{"category":"3.11","date":"2026-10-27","downloads":8},{"category":"3.12","date":"2026-04-08","downloads":10},{"category":"3.12","date":"2026-04-09","downloads":9},{"category":"3.12","date":"2026-04-10","downloads":4},{"category":"3.12","date":"2026-04-11","downloads":4},{"category":"3.12","date":"2026-04-12","downloads":4},{"category":"3.12","date":"2026-04-13","downloads":8},{"category":"3.12","date":"2026-04-14","downloads":4},{"category":"3.12","date":"2026-04-15","downloads":6},{"category":"3.12","date":"2026-04-16","downloads":12},{"category":"3.12","date":"2026-04-17","downloads":22},{"category":"3.12","date":"2026-04-18","downloads":4},{"category":"3.12","date":"2026-04-19","downloads":6},{"category":"3.12","date":"2026-04-20","downloads":16},{"category":"3.12","date":"2026-04-21","downloads":5},{"category":"3.12","date":"2026-04-22","downloads":13},{"category":"3.12","date":"2026-04-23","downloads":6},{"category":"3.12","date":"2026-04-24","downloads":4},{"category":"3.12","date":"2026-04-25","downloads":5},{"category":"3.12","date":"2026-04-26","downloads":4},{"category":"3.12","date":"2026-04-27","downloads":6},{"category":"3.12","date":"2026-04-28","downloads":4},{"category":"3.12","date":"2026-04-29","downloads":6},{"category":"3.12","date":"2026-04-30","downloads":10},{"category":"3.12","date":"2026-05-01","downloads":30},{"category":"3.12","date":"2026-05-02","downloads":4},{"category":"3.12","date":"2026-05-03","downloads":4},{"category":"3.12","date":"2026-05-04","downloads":8},{"category":"3.12","date":"2026-05-05","downloads":8},{"category":"3.12","date":"2026-05-06","downloads":8},{"category":"3.12","date":"2026-05-07","downloads":16},{"category":"3.12","date":"2026-05-08","downloads":4},{"category":"3.12","date":"2026-05-09","downloads":8},{"category":"3.12","date":"2026-05-10","downloads":4},{"category":"3.12","date":"2026-05-11","downloads":2},{"category":"3.12","date":"2026-05-12","downloads":14},{"category":"3.12","date":"2026-05-13","downloads":8},{"category":"3.12","date":"2026-05-14","downloads":8},{"category":"3.12","date":"2026-05-15","downloads":10},{"category":"3.12","date":"2026-05-16","downloads":10},{"category":"3.12","date":"2026-05-17","downloads":6},{"category":"3.12","date":"2026-05-18","downloads":14},{"category":"3.12","date":"2026-05-19","downloads":10},{"category":"3.12","date":"2026-05-20","downloads":4},{"category":"3.12","date":"2026-05-21","downloads":18},{"category":"3.12","date":"2026-05-22","downloads":15},{"category":"3.12","date":"2026-05-23","downloads":22},{"category":"3.12","date":"2026-05-24","downloads":19},{"category":"3.12","date":"2026-05-25","downloads":10},{"category":"3.12","date":"2026-05-26","downloads":8},{"category":"3.12","date":"2026-05-27","downloads":8},{"category":"3.12","date":"2026-05-28","downloads":16},{"category":"3.12","date":"2026-05-29","downloads":10},{"category":"3.12","date":"2026-05-30","downloads":2},{"category":"3.12","date":"2026-05-31","downloads":10},{"category":"3.12","date":"2026-06-01","downloads":2},{"category":"3.12","date":"2026-06-02","downloads":68},{"category":"3.12","date":"2026-06-03","downloads":18},{"category":"3.12","date":"2026-06-04","downloads":8},{"category":"3.12","date":"2026-06-07","downloads":4},{"category":"3.12","date":"2026-06-08","downloads":8},{"category":"3.12","date":"2026-06-09","downloads":4},{"category":"3.12","date":"2026-06-10","downloads":4},{"category":"3.12","date":"2026-06-11","downloads":6},{"category":"3.12","date":"2026-06-12","downloads":6},{"category":"3.12","date":"2026-06-13","downloads":2},{"category":"3.12","date":"2026-06-14","downloads":4},{"category":"3.12","date":"2026-06-15","downloads":8},{"category":"3.12","date":"2026-06-17","downloads":6},{"category":"3.12","date":"2026-06-18","downloads":16},{"category":"3.12","date":"2026-06-19","downloads":2},{"category":"3.12","date":"2026-06-20","downloads":8},{"category":"3.12","date":"2026-06-22","downloads":4},{"category":"3.12","date":"2026-06-23","downloads":6},{"category":"3.12","date":"2026-06-24","downloads":4},{"category":"3.12","date":"2026-06-25","downloads":8},{"category":"3.12","date":"2026-06-26","downloads":8},{"category":"3.12","date":"2026-06-27","downloads":2},{"category":"3.12","date":"2026-06-28","downloads":18},{"category":"3.12","date":"2026-06-29","downloads":28},{"category":"3.12","date":"2026-06-30","downloads":11},{"category":"3.12","date":"2026-07-01","downloads":4},{"category":"3.12","date":"2026-07-02","downloads":6},{"category":"3.12","date":"2026-07-03","downloads":4},{"category":"3.12","date":"2026-07-04","downloads":40},{"category":"3.12","date":"2026-07-05","downloads":48},{"category":"3.12","date":"2026-07-06","downloads":58},{"category":"3.12","date":"2026-07-07","downloads":48},{"category":"3.12","date":"2026-07-08","downloads":58},{"category":"3.12","date":"2026-07-09","downloads":108},{"category":"3.12","date":"2026-07-10","downloads":68},{"category":"3.12","date":"2026-07-11","downloads":25},{"category":"3.12","date":"2026-07-12","downloads":170},{"category":"3.12","date":"2026-07-13","downloads":52},{"category":"3.12","date":"2026-07-14","downloads":106},{"category":"3.12","date":"2026-07-15","downloads":1},{"category":"3.12","date":"2026-07-16","downloads":22},{"category":"3.12","date":"2026-07-17","downloads":36},{"category":"3.12","date":"2026-07-18","downloads":26},{"category":"3.12","date":"2026-07-19","downloads":7},{"category":"3.12","date":"2026-07-20","downloads":4},{"category":"3.12","date":"2026-07-21","downloads":154},{"category":"3.12","date":"2026-07-22","downloads":42},{"category":"3.12","date":"2026-07-23","downloads":111},{"category":"3.12","date":"2026-07-24","downloads":78},{"category":"3.12","date":"2026-07-25","downloads":73},{"category":"3.12","date":"2026-07-26","downloads":98},{"category":"3.12","date":"2026-07-27","downloads":94},{"category":"3.12","date":"2026-07-28","downloads":35},{"category":"3.12","date":"2026-07-29","downloads":65},{"category":"3.12","date":"2026-07-30","downloads":6},{"category":"3.12","date":"2026-07-31","downloads":244},{"category":"3.12","date":"2026-08-01","downloads":4},{"category":"3.12","date":"2026-08-03","downloads":16},{"category":"3.12","date":"2026-08-04","downloads":62},{"category":"3.12","date":"2026-08-05","downloads":12},{"category":"3.12","date":"2026-08-06","downloads":115},{"category":"3.12","date":"2026-08-07","downloads":6},{"category":"3.12","date":"2026-08-08","downloads":20},{"category":"3.12","date":"2026-08-09","downloads":4},{"category":"3.12","date":"2026-08-10","downloads":2},{"category":"3.12","date":"2026-08-11","downloads":52},{"category":"3.12","date":"2026-08-12","downloads":17},{"category":"3.12","date":"2026-08-13","downloads":32},{"category":"3.12","date":"2026-08-14","downloads":63},{"category":"3.12","date":"2026-08-15","downloads":16},{"category":"3.12","date":"2026-08-16","downloads":4},{"category":"3.12","date":"2026-08-17","downloads":22},{"category":"3.12","date":"2026-08-18","downloads":20},{"category":"3.12","date":"2026-08-19","downloads":376},{"category":"3.12","date":"2026-08-20","downloads":432},{"category":"3.12","date":"2026-08-21","downloads":370},{"category":"3.12","date":"2026-08-22","downloads":2},{"category":"3.12","date":"2026-08-23","downloads":130},{"category":"3.12","date":"2026-08-24","downloads":639},{"category":"3.12","date":"2026-08-25","downloads":157},{"category":"3.12","date":"2026-08-26","downloads":681},{"category":"3.12","date":"2026-08-27","downloads":524},{"category":"3.12","date":"2026-08-28","downloads":634},{"category":"3.12","date":"2026-08-29","downloads":57},{"category":"3.12","date":"2026-08-30","downloads":53},{"category":"3.12","date":"2026-08-31","downloads":432},{"category":"3.12","date":"2026-09-01","downloads":799},{"category":"3.12","date":"2026-09-02","downloads":694},{"category":"3.12","date":"2026-09-03","downloads":1260},{"category":"3.12","date":"2026-09-04","downloads":474},{"category":"3.12","date":"2026-09-05","downloads":688},{"category":"3.12","date":"2026-09-06","downloads":1624},{"category":"3.12","date":"2026-09-07","downloads":216},{"category":"3.12","date":"2026-09-08","downloads":698},{"category":"3.12","date":"2026-09-09","downloads":859},{"category":"3.12","date":"2026-09-10","downloads":589},{"category":"3.12","date":"2026-09-11","downloads":725},{"category":"3.12","date":"2026-09-12","downloads":271},{"category":"3.12","date":"2026-09-13","downloads":204},{"category":"3.12","date":"2026-09-14","downloads":917},{"category":"3.12","date":"2026-09-15","downloads":1014},{"category":"3.12","date":"2026-09-16","downloads":1239},{"category":"3.12","date":"2026-09-17","downloads":910},{"category":"3.12","date":"2026-09-18","downloads":562},{"category":"3.12","date":"2026-09-19","downloads":142},{"category":"3.12","date":"2026-09-20","downloads":108},{"category":"3.12","date":"2026-09-21","downloads":301},{"category":"3.12","date":"2026-09-22","downloads":434},{"category":"3.12","date":"2026-09-23","downloads":400},{"category":"3.12","date":"2026-09-24","downloads":406},{"category":"3.12","date":"2026-09-25","downloads":378},{"category":"3.12","date":"2026-09-26","downloads":134},{"category":"3.12","date":"2026-09-27","downloads":81},{"category":"3.12","date":"2026-09-28","downloads":353},{"category":"3.12","date":"2026-09-29","downloads":544},{"category":"3.12","date":"2026-09-30","downloads":383},{"category":"3.12","date":"2026-10-01","downloads":131},{"category":"3.12","date":"2026-10-02","downloads":43},{"category":"3.12","date":"2026-10-03","downloads":67},{"category":"3.12","date":"2026-10-04","downloads":44},{"category":"3.12","date":"2026-10-05","downloads":100},{"category":"3.12","date":"2026-10-06","downloads":231},{"category":"3.12","date":"2026-10-07","downloads":193},{"category":"3.12","date":"2026-10-08","downloads":193},{"category":"3.12","date":"2026-10-09","downloads":113},{"category":"3.12","date":"2026-10-10","downloads":95},{"category":"3.12","date":"2026-10-11","downloads":48},{"category":"3.12","date":"2026-10-12","downloads":64},{"category":"3.12","date":"2026-10-13","downloads":107},{"category":"3.12","date":"2026-10-14","downloads":143},{"category":"3.12","date":"2026-10-15","downloads":115},{"category":"3.12","date":"2026-10-16","downloads":120},{"category":"3.12","date":"2026-10-17","downloads":47},{"category":"3.12","date":"2026-10-18","downloads":162},{"category":"3.12","date":"2026-10-19","downloads":477},{"category":"3.12","date":"2026-10-20","downloads":832},{"category":"3.12","date":"2026-10-21","downloads":131},{"category":"3.12","date":"2026-10-22","downloads":125},{"category":"3.12","date":"2026-10-23","downloads":41},{"category":"3.12","date":"2026-10-24","downloads":34},{"category":"3.12","date":"2026-10-25","downloads":5},{"category":"3.12","date":"2026-10-26","downloads":56},{"category":"3.12","date":"2026-10-27","downloads":108},{"category":"3.12","date":"2026-10-28","downloads":151},{"category":"3.13","date":"2026-04-14","downloads":2},{"category":"3.13","date":"2026-04-25","downloads":1},{"category":"3.13","date":"2026-04-27","downloads":2},{"category":"3.13","date":"2026-05-17","downloads":2},{"category":"3.13","date":"2026-05-18","downloads":11},{"category":"3.13","date":"2026-05-20","downloads":6},{"category":"3.13","date":"2026-05-21","downloads":6},{"category":"3.13","date":"2026-05-23","downloads":8},{"category":"3.13","date":"2026-05-24","downloads":15},{"category":"3.13","date":"2026-05-25","downloads":11},{"category":"3.13","date":"2026-05-27","downloads":5},{"category":"3.13","date":"2026-05-28","downloads":21},{"category":"3.13","date":"2026-05-29","downloads":8},{"category":"3.13","date":"2026-06-02","downloads":8},{"category":"3.13","date":"2026-06-03","downloads":4},{"category":"3.13","date":"2026-06-04","downloads":10},{"category":"3.13","date":"2026-06-07","downloads":4},{"category":"3.13","date":"2026-06-08","downloads":4},{"category":"3.13","date":"2026-06-09","downloads":6},{"category":"3.13","date":"2026-06-10","downloads":6},{"category":"3.13","date":"2026-06-12","downloads":2},{"category":"3.13","date":"2026-06-13","downloads":2},{"category":"3.13","date":"2026-06-14","downloads":2},{"category":"3.13","date":"2026-06-19","downloads":2},{"category":"3.13","date":"2026-06-20","downloads":2},{"category":"3.13","date":"2026-06-21","downloads":2},{"category":"3.13","date":"2026-06-22","downloads":4},{"category":"3.13","date":"2026-06-23","downloads":2},{"category":"3.13","date":"2026-06-27","downloads":4},{"category":"3.13","date":"2026-06-28","downloads":4},{"category":"3.13","date":"2026-07-02","downloads":2},{"category":"3.13","date":"2026-07-04","downloads":8},{"category":"3.13","date":"2026-07-12","downloads":2},{"category":"3.13","date":"2026-07-15","downloads":6},{"category":"3.13","date":"2026-07-16","downloads":2},{"category":"3.13","date":"2026-07-17","downloads":2},{"category":"3.13","date":"2026-07-21","downloads":3},{"category":"3.13","date":"2026-07-22","downloads":2},{"category":"3.13","date":"2026-07-23","downloads":2},{"category":"3.13","date":"2026-07-24","downloads":1},{"category":"3.13","date":"2026-07-26","downloads":41},{"category":"3.13","date":"2026-07-30","downloads":2},{"category":"3.13","date":"2026-08-05","downloads":4},{"category":"3.13","date":"2026-08-07","downloads":4},{"category":"3.13","date":"2026-08-13","downloads":2},{"category":"3.13","date":"2026-08-27","downloads":1},{"category":"3.13","date":"2026-08-31","downloads":18},{"category":"3.13","date":"2026-09-01","downloads":3},{"category":"3.13","date":"2026-09-03","downloads":2},{"category":"3.13","date":"2026-09-07","downloads":1},{"category":"3.13","date":"2026-09-10","downloads":1},{"category":"3.13","date":"2026-09-14","downloads":2},{"category":"3.13","date":"2026-09-16","downloads":1},{"category":"3.13","date":"2026-09-17","downloads":1},{"category":"3.13","date":"2026-09-25","downloads":4},{"category":"3.13","date":"2026-09-26","downloads":1},{"category":"3.13","date":"2026-09-27","downloads":3},{"category":"3.13","date":"2026-09-30","downloads":1},{"category":"3.13","date":"2026-10-02","downloads":1},{"category":"3.13","date":"2026-10-03","downloads":1},{"category":"3.13","date":"2026-10-04","downloads":1},{"category":"3.13","date":"2026-10-05","downloads":1},{"category":"3.13","date":"2026-10-06","downloads":3},{"category":"3.13","date":"2026-10-07","downloads":1},{"category":"3.13","date":"2026-10-08","downloads":1},{"category":"3.13","date":"2026-10-09","downloads":1},{"category":"3.13","date":"2026-10-10","downloads":1},{"category":"3.13","date":"2026-10-11","downloads":1},{"category":"3.13","date":"2026-10-12","downloads":1},{"category":"3.13","date":"2026-10-13","downloads":3},{"category":"3.13","date":"2026-10-14","downloads":1},{"category":"3.13","date":"2026-10-15","downloads":1},{"category":"3.13","date":"2026-10-16","downloads":6},{"category":"3.13","date":"2026-10-17","downloads":3},{"category":"3.13","date":"2026-10-18","downloads":28},{"category":"3.13","date":"2026-10-19","downloads":4},{"category":"3.13","date":"2026-10-20","downloads":5},{"category":"3.13","date":"2026-10-21","downloads":3},{"category":"3.13","date":"2026-10-22","downloads":1},{"category":"3.13","date":"2026-10-23","downloads":11},{"category":"3.13","date":"2026-10-24","downloads":100},{"category":"3.13","date":"2026-10-25","downloads":2},{"category":"3.13","date":"2026-10-26","downloads":1},{"category":"3.13","date":"2026-10-27","downloads":2},{"category":"3.13","date":"2026-10-28","downloads":1},{"category":"3.14","date":"2026-04-24","downloads":1},{"category":"3.14","date":"2026-05-27","downloads":2},{"category":"3.14","date":"2026-05-28","downloads":8},{"category":"3.14","date":"2026-05-29","downloads":2},{"category":"3.14","date":"2026-06-02","downloads":2},{"category":"3.14","date":"2026-06-04","downloads":2},{"category":"3.14","date":"2026-06-08","downloads":20},{"category":"3.14","date":"2026-06-09","downloads":30},{"category":"3.14","date":"2026-06-10","downloads":10},{"category":"3.14","date":"2026-06-22","downloads":20},{"category":"3.14","date":"2026-07-23","downloads":10},{"category":"3.14","date":"2026-07-24","downloads":10},{"category":"3.14","date":"2026-07-26","downloads":2},{"category":"3.14","date":"2026-08-31","downloads":10},{"category":"3.8","date":"2026-10-26","downloads":1},{"category":"3.9","date":"2026-04-23","downloads":1},{"category":"3.9","date":"2026-04-24","downloads":2},{"category":"3.9","date":"2026-05-17","downloads":1},{"category":"3.9","date":"2026-05-18","downloads":4},{"category":"3.9","date":"2026-05-21","downloads":4},{"category":"3.9","date":"2026-05-24","downloads":1},{"category":"3.9","date":"2026-06-07","downloads":1},{"category":"3.9","date":"2026-06-14","downloads":1},{"category":"3.9","date":"2026-06-24","downloads":1},{"category":"3.9","date":"2026-06-30","downloads":2},{"category":"3.9","date":"2026-08-07","downloads":1},{"category":"3.9","date":"2026-08-17","downloads":1},{"category":"3.9","date":"2026-08-19","downloads":4},{"category":"3.9","date":"2026-08-26","downloads":1},{"category":"3.9","date":"2026-08-31","downloads":13},{"category":"3.9","date":"2026-09-01","downloads":2},{"category":"3.9","date":"2026-09-24","downloads":4},{"category":"3.9","date":"2026-10-03","downloads":1},{"category":"3.9","date":"2026-10-06","downloads":1},{"category":"3.9","date":"2026-10-13","downloads":1},{"category":"null","date":"2026-04-08","downloads":1},{"category":"null","date":"2026-04-10","downloads":5},{"category":"null","date":"2026-04-11","downloads":17},{"category":"null","date":"2026-04-12","downloads":1},{"category":"null","date":"2026-04-13","downloads":4},{"category":"null","date":"2026-04-14","downloads":29},{"category":"null","date":"2026-04-15","downloads":42},{"category":"null","date":"2026-04-16","downloads":2},{"category":"null","date":"2026-04-17","downloads":1},{"category":"null","date":"2026-04-19","downloads":14},{"category":"null","date":"2026-04-20","downloads":2},{"category":"null","date":"2026-04-22","downloads":151},{"category":"null","date":"2026-04-23","downloads":8},{"category":"null","date":"2026-04-24","downloads":5},{"category":"null","date":"2026-04-25","downloads":10},{"category":"null","date":"2026-04-26","downloads":4},{"category":"null","date":"2026-04-27","downloads":10},{"category":"null","date":"2026-04-28","downloads":4},{"category":"null","date":"2026-04-29","downloads":2},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-03","downloads":2},{"category":"null","date":"2026-05-05","downloads":56},{"category":"null","date":"2026-05-06","downloads":6},{"category":"null","date":"2026-05-07","downloads":4},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":1},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":6},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":3},{"category":"null","date":"2026-05-19","downloads":1},{"category":"null","date":"2026-05-21","downloads":145},{"category":"null","date":"2026-05-22","downloads":8},{"category":"null","date":"2026-05-23","downloads":162},{"category":"null","date":"2026-05-24","downloads":9},{"category":"null","date":"2026-05-25","downloads":21},{"category":"null","date":"2026-05-26","downloads":3},{"category":"null","date":"2026-05-27","downloads":16},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":7},{"category":"null","date":"2026-05-30","downloads":2},{"category":"null","date":"2026-05-31","downloads":8},{"category":"null","date":"2026-06-02","downloads":10},{"category":"null","date":"2026-06-03","downloads":17},{"category":"null","date":"2026-06-04","downloads":6},{"category":"null","date":"2026-06-05","downloads":34},{"category":"null","date":"2026-06-06","downloads":36},{"category":"null","date":"2026-06-07","downloads":9},{"category":"null","date":"2026-06-08","downloads":20},{"category":"null","date":"2026-06-09","downloads":10},{"category":"null","date":"2026-06-10","downloads":9},{"category":"null","date":"2026-06-12","downloads":3},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":1},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":2},{"category":"null","date":"2026-06-17","downloads":7},{"category":"null","date":"2026-06-18","downloads":177},{"category":"null","date":"2026-06-19","downloads":2},{"category":"null","date":"2026-06-20","downloads":4},{"category":"null","date":"2026-06-21","downloads":1},{"category":"null","date":"2026-06-22","downloads":6},{"category":"null","date":"2026-06-23","downloads":53},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":23},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-01","downloads":4},{"category":"null","date":"2026-07-02","downloads":1},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-21","downloads":1},{"category":"null","date":"2026-07-22","downloads":47},{"category":"null","date":"2026-07-23","downloads":71},{"category":"null","date":"2026-07-24","downloads":120},{"category":"null","date":"2026-07-25","downloads":4},{"category":"null","date":"2026-07-26","downloads":8},{"category":"null","date":"2026-07-27","downloads":11},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-29","downloads":7},{"category":"null","date":"2026-07-30","downloads":24},{"category":"null","date":"2026-07-31","downloads":16},{"category":"null","date":"2026-08-01","downloads":4},{"category":"null","date":"2026-08-02","downloads":3},{"category":"null","date":"2026-08-03","downloads":4},{"category":"null","date":"2026-08-04","downloads":10},{"category":"null","date":"2026-08-05","downloads":9},{"category":"null","date":"2026-08-06","downloads":9},{"category":"null","date":"2026-08-07","downloads":13},{"category":"null","date":"2026-08-08","downloads":1},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":8},{"category":"null","date":"2026-08-13","downloads":4},{"category":"null","date":"2026-08-14","downloads":32},{"category":"null","date":"2026-08-15","downloads":1},{"category":"null","date":"2026-08-17","downloads":3},{"category":"null","date":"2026-08-18","downloads":9},{"category":"null","date":"2026-08-19","downloads":5},{"category":"null","date":"2026-08-20","downloads":16},{"category":"null","date":"2026-08-21","downloads":23},{"category":"null","date":"2026-08-22","downloads":3},{"category":"null","date":"2026-08-24","downloads":1},{"category":"null","date":"2026-08-25","downloads":2},{"category":"null","date":"2026-08-26","downloads":217},{"category":"null","date":"2026-08-27","downloads":86},{"category":"null","date":"2026-08-28","downloads":102},{"category":"null","date":"2026-08-29","downloads":5},{"category":"null","date":"2026-08-30","downloads":1},{"category":"null","date":"2026-08-31","downloads":219},{"category":"null","date":"2026-09-01","downloads":48},{"category":"null","date":"2026-09-02","downloads":85},{"category":"null","date":"2026-09-03","downloads":106},{"category":"null","date":"2026-09-04","downloads":18},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":10},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":34},{"category":"null","date":"2026-09-09","downloads":1},{"category":"null","date":"2026-09-10","downloads":11},{"category":"null","date":"2026-09-11","downloads":1},{"category":"null","date":"2026-09-12","downloads":7},{"category":"null","date":"2026-09-13","downloads":4},{"category":"null","date":"2026-09-14","downloads":13},{"category":"null","date":"2026-09-15","downloads":13},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":24},{"category":"null","date":"2026-09-18","downloads":4},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":8},{"category":"null","date":"2026-09-21","downloads":16},{"category":"null","date":"2026-09-22","downloads":2},{"category":"null","date":"2026-09-23","downloads":1},{"category":"null","date":"2026-09-24","downloads":13},{"category":"null","date":"2026-09-25","downloads":16},{"category":"null","date":"2026-09-26","downloads":69},{"category":"null","date":"2026-09-27","downloads":27},{"category":"null","date":"2026-09-28","downloads":133},{"category":"null","date":"2026-09-29","downloads":8},{"category":"null","date":"2026-09-30","downloads":8},{"category":"null","date":"2026-10-01","downloads":6},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":4},{"category":"null","date":"2026-10-04","downloads":9},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-08","downloads":4},{"category":"null","date":"2026-10-09","downloads":8},{"category":"null","date":"2026-10-10","downloads":2},{"category":"null","date":"2026-10-11","downloads":3},{"category":"null","date":"2026-10-12","downloads":3},{"category":"null","date":"2026-10-13","downloads":8},{"category":"null","date":"2026-10-14","downloads":8},{"category":"null","date":"2026-10-15","downloads":13},{"category":"null","date":"2026-10-16","downloads":15},{"category":"null","date":"2026-10-17","downloads":2},{"category":"null","date":"2026-10-18","downloads":9},{"category":"null","date":"2026-10-19","downloads":10},{"category":"null","date":"2026-10-20","downloads":6},{"category":"null","date":"2026-10-21","downloads":5},{"category":"null","date":"2026-10-22","downloads":9},{"category":"null","date":"2026-10-23","downloads":8},{"category":"null","date":"2026-10-24","downloads":5},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-26","downloads":2},{"category":"null","date":"2026-10-27","downloads":3},{"category":"null","date":"2026-10-28","downloads":6}],"package":"mooncake-transfer-engine-non-cuda","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:12:51.811754+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-non-cuda/system","payload":{"data":[{"category":"Darwin","date":"2026-04-20","downloads":4},{"category":"Darwin","date":"2026-04-22","downloads":4},{"category":"Darwin","date":"2026-05-16","downloads":35},{"category":"Darwin","date":"2026-05-21","downloads":4},{"category":"Darwin","date":"2026-05-22","downloads":4},{"category":"Darwin","date":"2026-05-23","downloads":8},{"category":"Darwin","date":"2026-06-04","downloads":1},{"category":"Darwin","date":"2026-06-23","downloads":2},{"category":"Darwin","date":"2026-07-23","downloads":8},{"category":"Darwin","date":"2026-07-24","downloads":9},{"category":"Darwin","date":"2026-08-04","downloads":2},{"category":"Darwin","date":"2026-08-12","downloads":2},{"category":"Darwin","date":"2026-08-13","downloads":6},{"category":"Darwin","date":"2026-08-18","downloads":2},{"category":"Darwin","date":"2026-08-25","downloads":2},{"category":"Darwin","date":"2026-08-26","downloads":18},{"category":"Darwin","date":"2026-08-27","downloads":1},{"category":"Darwin","date":"2026-08-28","downloads":1},{"category":"Darwin","date":"2026-08-31","downloads":40},{"category":"Darwin","date":"2026-09-01","downloads":1},{"category":"Darwin","date":"2026-09-03","downloads":2},{"category":"Darwin","date":"2026-09-10","downloads":1},{"category":"Darwin","date":"2026-09-17","downloads":1},{"category":"Darwin","date":"2026-09-21","downloads":1},{"category":"Darwin","date":"2026-09-22","downloads":2},{"category":"Darwin","date":"2026-09-24","downloads":3},{"category":"Darwin","date":"2026-09-29","downloads":1},{"category":"Darwin","date":"2026-09-30","downloads":3},{"category":"Darwin","date":"2026-10-01","downloads":1},{"category":"Darwin","date":"2026-10-03","downloads":1},{"category":"Darwin","date":"2026-10-07","downloads":1},{"category":"Darwin","date":"2026-10-08","downloads":1},{"category":"Linux","date":"2026-04-08","downloads":36},{"category":"Linux","date":"2026-04-09","downloads":37},{"category":"Linux","date":"2026-04-10","downloads":33},{"category":"Linux","date":"2026-04-11","downloads":32},{"category":"Linux","date":"2026-04-12","downloads":62},{"category":"Linux","date":"2026-04-13","downloads":54},{"category":"Linux","date":"2026-04-14","downloads":8},{"category":"Linux","date":"2026-04-15","downloads":30},{"category":"Linux","date":"2026-04-16","downloads":12},{"category":"Linux","date":"2026-04-17","downloads":47},{"category":"Linux","date":"2026-04-18","downloads":6},{"category":"Linux","date":"2026-04-19","downloads":6},{"category":"Linux","date":"2026-04-20","downloads":16},{"category":"Linux","date":"2026-04-21","downloads":34},{"category":"Linux","date":"2026-04-22","downloads":34},{"category":"Linux","date":"2026-04-23","downloads":14},{"category":"Linux","date":"2026-04-24","downloads":11},{"category":"Linux","date":"2026-04-25","downloads":26},{"category":"Linux","date":"2026-04-26","downloads":8},{"category":"Linux","date":"2026-04-27","downloads":91},{"category":"Linux","date":"2026-04-28","downloads":14},{"category":"Linux","date":"2026-04-29","downloads":36},{"category":"Linux","date":"2026-04-30","downloads":14},{"category":"Linux","date":"2026-05-01","downloads":98},{"category":"Linux","date":"2026-05-02","downloads":72},{"category":"Linux","date":"2026-05-03","downloads":6},{"category":"Linux","date":"2026-05-04","downloads":10},{"category":"Linux","date":"2026-05-05","downloads":10},{"category":"Linux","date":"2026-05-06","downloads":14},{"category":"Linux","date":"2026-05-07","downloads":28},{"category":"Linux","date":"2026-05-08","downloads":20},{"category":"Linux","date":"2026-05-09","downloads":64},{"category":"Linux","date":"2026-05-10","downloads":4},{"category":"Linux","date":"2026-05-11","downloads":6},{"category":"Linux","date":"2026-05-12","downloads":30},{"category":"Linux","date":"2026-05-13","downloads":27},{"category":"Linux","date":"2026-05-14","downloads":59},{"category":"Linux","date":"2026-05-15","downloads":43},{"category":"Linux","date":"2026-05-16","downloads":12},{"category":"Linux","date":"2026-05-17","downloads":18},{"category":"Linux","date":"2026-05-18","downloads":54},{"category":"Linux","date":"2026-05-19","downloads":22},{"category":"Linux","date":"2026-05-20","downloads":26},{"category":"Linux","date":"2026-05-21","downloads":33},{"category":"Linux","date":"2026-05-22","downloads":77},{"category":"Linux","date":"2026-05-23","downloads":39},{"category":"Linux","date":"2026-05-24","downloads":39},{"category":"Linux","date":"2026-05-25","downloads":35},{"category":"Linux","date":"2026-05-26","downloads":13},{"category":"Linux","date":"2026-05-27","downloads":20},{"category":"Linux","date":"2026-05-28","downloads":68},{"category":"Linux","date":"2026-05-29","downloads":48},{"category":"Linux","date":"2026-05-30","downloads":74},{"category":"Linux","date":"2026-05-31","downloads":26},{"category":"Linux","date":"2026-06-01","downloads":24},{"category":"Linux","date":"2026-06-02","downloads":140},{"category":"Linux","date":"2026-06-03","downloads":36},{"category":"Linux","date":"2026-06-04","downloads":76},{"category":"Linux","date":"2026-06-05","downloads":143},{"category":"Linux","date":"2026-06-06","downloads":140},{"category":"Linux","date":"2026-06-07","downloads":154},{"category":"Linux","date":"2026-06-08","downloads":227},{"category":"Linux","date":"2026-06-09","downloads":124},{"category":"Linux","date":"2026-06-10","downloads":56},{"category":"Linux","date":"2026-06-11","downloads":69},{"category":"Linux","date":"2026-06-12","downloads":20},{"category":"Linux","date":"2026-06-13","downloads":24},{"category":"Linux","date":"2026-06-14","downloads":12},{"category":"Linux","date":"2026-06-15","downloads":24},{"category":"Linux","date":"2026-06-16","downloads":14},{"category":"Linux","date":"2026-06-17","downloads":22},{"category":"Linux","date":"2026-06-18","downloads":51},{"category":"Linux","date":"2026-06-19","downloads":10},{"category":"Linux","date":"2026-06-20","downloads":10},{"category":"Linux","date":"2026-06-21","downloads":2},{"category":"Linux","date":"2026-06-22","downloads":51},{"category":"Linux","date":"2026-06-23","downloads":222},{"category":"Linux","date":"2026-06-24","downloads":219},{"category":"Linux","date":"2026-06-25","downloads":85},{"category":"Linux","date":"2026-06-26","downloads":20},{"category":"Linux","date":"2026-06-27","downloads":11},{"category":"Linux","date":"2026-06-28","downloads":24},{"category":"Linux","date":"2026-06-29","downloads":134},{"category":"Linux","date":"2026-06-30","downloads":337},{"category":"Linux","date":"2026-07-01","downloads":53},{"category":"Linux","date":"2026-07-02","downloads":54},{"category":"Linux","date":"2026-07-03","downloads":135},{"category":"Linux","date":"2026-07-04","downloads":78},{"category":"Linux","date":"2026-07-05","downloads":48},{"category":"Linux","date":"2026-07-06","downloads":82},{"category":"Linux","date":"2026-07-07","downloads":58},{"category":"Linux","date":"2026-07-08","downloads":58},{"category":"Linux","date":"2026-07-09","downloads":147},{"category":"Linux","date":"2026-07-10","downloads":108},{"category":"Linux","date":"2026-07-11","downloads":46},{"category":"Linux","date":"2026-07-12","downloads":220},{"category":"Linux","date":"2026-07-13","downloads":63},{"category":"Linux","date":"2026-07-14","downloads":305},{"category":"Linux","date":"2026-07-15","downloads":21},{"category":"Linux","date":"2026-07-16","downloads":34},{"category":"Linux","date":"2026-07-17","downloads":38},{"category":"Linux","date":"2026-07-18","downloads":31},{"category":"Linux","date":"2026-07-19","downloads":7},{"category":"Linux","date":"2026-07-20","downloads":12},{"category":"Linux","date":"2026-07-21","downloads":279},{"category":"Linux","date":"2026-07-22","downloads":50},{"category":"Linux","date":"2026-07-23","downloads":144},{"category":"Linux","date":"2026-07-24","downloads":226},{"category":"Linux","date":"2026-07-25","downloads":74},{"category":"Linux","date":"2026-07-26","downloads":148},{"category":"Linux","date":"2026-07-27","downloads":674},{"category":"Linux","date":"2026-07-28","downloads":1269},{"category":"Linux","date":"2026-07-29","downloads":1278},{"category":"Linux","date":"2026-07-30","downloads":1208},{"category":"Linux","date":"2026-07-31","downloads":1387},{"category":"Linux","date":"2026-08-01","downloads":552},{"category":"Linux","date":"2026-08-02","downloads":555},{"category":"Linux","date":"2026-08-03","downloads":504},{"category":"Linux","date":"2026-08-04","downloads":605},{"category":"Linux","date":"2026-08-05","downloads":622},{"category":"Linux","date":"2026-08-06","downloads":569},{"category":"Linux","date":"2026-08-07","downloads":638},{"category":"Linux","date":"2026-08-08","downloads":372},{"category":"Linux","date":"2026-08-09","downloads":350},{"category":"Linux","date":"2026-08-10","downloads":602},{"category":"Linux","date":"2026-08-11","downloads":752},{"category":"Linux","date":"2026-08-12","downloads":722},{"category":"Linux","date":"2026-08-13","downloads":821},{"category":"Linux","date":"2026-08-14","downloads":673},{"category":"Linux","date":"2026-08-15","downloads":394},{"category":"Linux","date":"2026-08-16","downloads":318},{"category":"Linux","date":"2026-08-17","downloads":779},{"category":"Linux","date":"2026-08-18","downloads":834},{"category":"Linux","date":"2026-08-19","downloads":1459},{"category":"Linux","date":"2026-08-20","downloads":805},{"category":"Linux","date":"2026-08-21","downloads":665},{"category":"Linux","date":"2026-08-22","downloads":53},{"category":"Linux","date":"2026-08-23","downloads":208},{"category":"Linux","date":"2026-08-24","downloads":938},{"category":"Linux","date":"2026-08-25","downloads":419},{"category":"Linux","date":"2026-08-26","downloads":1009},{"category":"Linux","date":"2026-08-27","downloads":782},{"category":"Linux","date":"2026-08-28","downloads":831},{"category":"Linux","date":"2026-08-29","downloads":72},{"category":"Linux","date":"2026-08-30","downloads":63},{"category":"Linux","date":"2026-08-31","downloads":582},{"category":"Linux","date":"2026-09-01","downloads":2585},{"category":"Linux","date":"2026-09-02","downloads":1723},{"category":"Linux","date":"2026-09-03","downloads":1535},{"category":"Linux","date":"2026-09-04","downloads":657},{"category":"Linux","date":"2026-09-05","downloads":797},{"category":"Linux","date":"2026-09-06","downloads":1756},{"category":"Linux","date":"2026-09-07","downloads":303},{"category":"Linux","date":"2026-09-08","downloads":955},{"category":"Linux","date":"2026-09-09","downloads":1134},{"category":"Linux","date":"2026-09-10","downloads":795},{"category":"Linux","date":"2026-09-11","downloads":941},{"category":"Linux","date":"2026-09-12","downloads":353},{"category":"Linux","date":"2026-09-13","downloads":241},{"category":"Linux","date":"2026-09-14","downloads":1096},{"category":"Linux","date":"2026-09-15","downloads":1325},{"category":"Linux","date":"2026-09-16","downloads":1573},{"category":"Linux","date":"2026-09-17","downloads":1106},{"category":"Linux","date":"2026-09-18","downloads":740},{"category":"Linux","date":"2026-09-19","downloads":194},{"category":"Linux","date":"2026-09-20","downloads":126},{"category":"Linux","date":"2026-09-21","downloads":328},{"category":"Linux","date":"2026-09-22","downloads":462},{"category":"Linux","date":"2026-09-23","downloads":452},{"category":"Linux","date":"2026-09-24","downloads":442},{"category":"Linux","date":"2026-09-25","downloads":402},{"category":"Linux","date":"2026-09-26","downloads":145},{"category":"Linux","date":"2026-09-27","downloads":94},{"category":"Linux","date":"2026-09-28","downloads":392},{"category":"Linux","date":"2026-09-29","downloads":595},{"category":"Linux","date":"2026-09-30","downloads":787},{"category":"Linux","date":"2026-10-01","downloads":167},{"category":"Linux","date":"2026-10-02","downloads":85},{"category":"Linux","date":"2026-10-03","downloads":159},{"category":"Linux","date":"2026-10-04","downloads":79},{"category":"Linux","date":"2026-10-05","downloads":144},{"category":"Linux","date":"2026-10-06","downloads":265},{"category":"Linux","date":"2026-10-07","downloads":222},{"category":"Linux","date":"2026-10-08","downloads":227},{"category":"Linux","date":"2026-10-09","downloads":145},{"category":"Linux","date":"2026-10-10","downloads":114},{"category":"Linux","date":"2026-10-11","downloads":59},{"category":"Linux","date":"2026-10-12","downloads":97},{"category":"Linux","date":"2026-10-13","downloads":156},{"category":"Linux","date":"2026-10-14","downloads":182},{"category":"Linux","date":"2026-10-15","downloads":146},{"category":"Linux","date":"2026-10-16","downloads":153},{"category":"Linux","date":"2026-10-17","downloads":65},{"category":"Linux","date":"2026-10-18","downloads":204},{"category":"Linux","date":"2026-10-19","downloads":525},{"category":"Linux","date":"2026-10-20","downloads":856},{"category":"Linux","date":"2026-10-21","downloads":159},{"category":"Linux","date":"2026-10-22","downloads":140},{"category":"Linux","date":"2026-10-23","downloads":107},{"category":"Linux","date":"2026-10-24","downloads":146},{"category":"Linux","date":"2026-10-25","downloads":13},{"category":"Linux","date":"2026-10-26","downloads":84},{"category":"Linux","date":"2026-10-27","downloads":130},{"category":"Linux","date":"2026-10-28","downloads":162},{"category":"Windows","date":"2026-08-31","downloads":1},{"category":"Windows","date":"2026-09-16","downloads":1},{"category":"Windows","date":"2026-10-10","downloads":6},{"category":"Windows","date":"2026-10-26","downloads":1},{"category":"null","date":"2026-04-08","downloads":1},{"category":"null","date":"2026-04-10","downloads":5},{"category":"null","date":"2026-04-11","downloads":17},{"category":"null","date":"2026-04-12","downloads":1},{"category":"null","date":"2026-04-13","downloads":4},{"category":"null","date":"2026-04-14","downloads":29},{"category":"null","date":"2026-04-15","downloads":42},{"category":"null","date":"2026-04-16","downloads":2},{"category":"null","date":"2026-04-17","downloads":1},{"category":"null","date":"2026-04-19","downloads":14},{"category":"null","date":"2026-04-20","downloads":2},{"category":"null","date":"2026-04-22","downloads":151},{"category":"null","date":"2026-04-23","downloads":8},{"category":"null","date":"2026-04-24","downloads":5},{"category":"null","date":"2026-04-25","downloads":10},{"category":"null","date":"2026-04-26","downloads":4},{"category":"null","date":"2026-04-27","downloads":10},{"category":"null","date":"2026-04-28","downloads":4},{"category":"null","date":"2026-04-29","downloads":2},{"category":"null","date":"2026-04-30","downloads":4},{"category":"null","date":"2026-05-03","downloads":2},{"category":"null","date":"2026-05-05","downloads":56},{"category":"null","date":"2026-05-06","downloads":6},{"category":"null","date":"2026-05-07","downloads":4},{"category":"null","date":"2026-05-12","downloads":3},{"category":"null","date":"2026-05-14","downloads":1},{"category":"null","date":"2026-05-15","downloads":1},{"category":"null","date":"2026-05-16","downloads":6},{"category":"null","date":"2026-05-17","downloads":1},{"category":"null","date":"2026-05-18","downloads":3},{"category":"null","date":"2026-05-19","downloads":1},{"category":"null","date":"2026-05-21","downloads":145},{"category":"null","date":"2026-05-22","downloads":8},{"category":"null","date":"2026-05-23","downloads":162},{"category":"null","date":"2026-05-24","downloads":9},{"category":"null","date":"2026-05-25","downloads":21},{"category":"null","date":"2026-05-26","downloads":3},{"category":"null","date":"2026-05-27","downloads":16},{"category":"null","date":"2026-05-28","downloads":4},{"category":"null","date":"2026-05-29","downloads":7},{"category":"null","date":"2026-05-30","downloads":2},{"category":"null","date":"2026-05-31","downloads":8},{"category":"null","date":"2026-06-02","downloads":10},{"category":"null","date":"2026-06-03","downloads":17},{"category":"null","date":"2026-06-04","downloads":6},{"category":"null","date":"2026-06-05","downloads":34},{"category":"null","date":"2026-06-06","downloads":36},{"category":"null","date":"2026-06-07","downloads":9},{"category":"null","date":"2026-06-08","downloads":20},{"category":"null","date":"2026-06-09","downloads":10},{"category":"null","date":"2026-06-10","downloads":9},{"category":"null","date":"2026-06-12","downloads":3},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":1},{"category":"null","date":"2026-06-15","downloads":1},{"category":"null","date":"2026-06-16","downloads":2},{"category":"null","date":"2026-06-17","downloads":7},{"category":"null","date":"2026-06-18","downloads":177},{"category":"null","date":"2026-06-19","downloads":2},{"category":"null","date":"2026-06-20","downloads":4},{"category":"null","date":"2026-06-21","downloads":1},{"category":"null","date":"2026-06-22","downloads":6},{"category":"null","date":"2026-06-23","downloads":53},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-26","downloads":5},{"category":"null","date":"2026-06-27","downloads":23},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":3},{"category":"null","date":"2026-07-01","downloads":4},{"category":"null","date":"2026-07-02","downloads":1},{"category":"null","date":"2026-07-04","downloads":1},{"category":"null","date":"2026-07-07","downloads":1},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-13","downloads":1},{"category":"null","date":"2026-07-14","downloads":1},{"category":"null","date":"2026-07-16","downloads":2},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-21","downloads":1},{"category":"null","date":"2026-07-22","downloads":47},{"category":"null","date":"2026-07-23","downloads":71},{"category":"null","date":"2026-07-24","downloads":120},{"category":"null","date":"2026-07-25","downloads":4},{"category":"null","date":"2026-07-26","downloads":8},{"category":"null","date":"2026-07-27","downloads":11},{"category":"null","date":"2026-07-28","downloads":2},{"category":"null","date":"2026-07-29","downloads":7},{"category":"null","date":"2026-07-30","downloads":24},{"category":"null","date":"2026-07-31","downloads":16},{"category":"null","date":"2026-08-01","downloads":4},{"category":"null","date":"2026-08-02","downloads":3},{"category":"null","date":"2026-08-03","downloads":4},{"category":"null","date":"2026-08-04","downloads":10},{"category":"null","date":"2026-08-05","downloads":9},{"category":"null","date":"2026-08-06","downloads":9},{"category":"null","date":"2026-08-07","downloads":13},{"category":"null","date":"2026-08-08","downloads":1},{"category":"null","date":"2026-08-09","downloads":1},{"category":"null","date":"2026-08-11","downloads":1},{"category":"null","date":"2026-08-12","downloads":8},{"category":"null","date":"2026-08-13","downloads":4},{"category":"null","date":"2026-08-14","downloads":32},{"category":"null","date":"2026-08-15","downloads":1},{"category":"null","date":"2026-08-17","downloads":3},{"category":"null","date":"2026-08-18","downloads":9},{"category":"null","date":"2026-08-19","downloads":5},{"category":"null","date":"2026-08-20","downloads":16},{"category":"null","date":"2026-08-21","downloads":23},{"category":"null","date":"2026-08-22","downloads":3},{"category":"null","date":"2026-08-24","downloads":1},{"category":"null","date":"2026-08-25","downloads":2},{"category":"null","date":"2026-08-26","downloads":217},{"category":"null","date":"2026-08-27","downloads":86},{"category":"null","date":"2026-08-28","downloads":102},{"category":"null","date":"2026-08-29","downloads":5},{"category":"null","date":"2026-08-30","downloads":1},{"category":"null","date":"2026-08-31","downloads":219},{"category":"null","date":"2026-09-01","downloads":48},{"category":"null","date":"2026-09-02","downloads":85},{"category":"null","date":"2026-09-03","downloads":106},{"category":"null","date":"2026-09-04","downloads":18},{"category":"null","date":"2026-09-05","downloads":8},{"category":"null","date":"2026-09-06","downloads":10},{"category":"null","date":"2026-09-07","downloads":20},{"category":"null","date":"2026-09-08","downloads":34},{"category":"null","date":"2026-09-09","downloads":1},{"category":"null","date":"2026-09-10","downloads":11},{"category":"null","date":"2026-09-11","downloads":1},{"category":"null","date":"2026-09-12","downloads":7},{"category":"null","date":"2026-09-13","downloads":4},{"category":"null","date":"2026-09-14","downloads":13},{"category":"null","date":"2026-09-15","downloads":13},{"category":"null","date":"2026-09-16","downloads":3},{"category":"null","date":"2026-09-17","downloads":24},{"category":"null","date":"2026-09-18","downloads":4},{"category":"null","date":"2026-09-19","downloads":67},{"category":"null","date":"2026-09-20","downloads":8},{"category":"null","date":"2026-09-21","downloads":16},{"category":"null","date":"2026-09-22","downloads":2},{"category":"null","date":"2026-09-23","downloads":1},{"category":"null","date":"2026-09-24","downloads":13},{"category":"null","date":"2026-09-25","downloads":16},{"category":"null","date":"2026-09-26","downloads":69},{"category":"null","date":"2026-09-27","downloads":27},{"category":"null","date":"2026-09-28","downloads":133},{"category":"null","date":"2026-09-29","downloads":8},{"category":"null","date":"2026-09-30","downloads":8},{"category":"null","date":"2026-10-01","downloads":6},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":4},{"category":"null","date":"2026-10-04","downloads":9},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":10},{"category":"null","date":"2026-10-08","downloads":4},{"category":"null","date":"2026-10-09","downloads":8},{"category":"null","date":"2026-10-10","downloads":2},{"category":"null","date":"2026-10-11","downloads":3},{"category":"null","date":"2026-10-12","downloads":3},{"category":"null","date":"2026-10-13","downloads":8},{"category":"null","date":"2026-10-14","downloads":8},{"category":"null","date":"2026-10-15","downloads":13},{"category":"null","date":"2026-10-16","downloads":15},{"category":"null","date":"2026-10-17","downloads":2},{"category":"null","date":"2026-10-18","downloads":9},{"category":"null","date":"2026-10-19","downloads":10},{"category":"null","date":"2026-10-20","downloads":6},{"category":"null","date":"2026-10-21","downloads":5},{"category":"null","date":"2026-10-22","downloads":9},{"category":"null","date":"2026-10-23","downloads":8},{"category":"null","date":"2026-10-24","downloads":5},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-26","downloads":2},{"category":"null","date":"2026-10-27","downloads":3},{"category":"null","date":"2026-10-28","downloads":6}],"package":"mooncake-transfer-engine-non-cuda","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (Non-CUDA version)"},{"name":"mooncake-transfer-engine-npu","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:12:55.261911+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-npu/recent","payload":{"data":{"last_day":26,"last_month":756,"last_week":218},"package":"mooncake-transfer-engine-npu","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:12:57.952900+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-npu/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-06-07","downloads":185},{"category":"with_mirrors","date":"2026-06-08","downloads":135},{"category":"with_mirrors","date":"2026-06-09","downloads":37},{"category":"with_mirrors","date":"2026-06-10","downloads":302},{"category":"with_mirrors","date":"2026-06-11","downloads":99},{"category":"with_mirrors","date":"2026-06-12","downloads":38},{"category":"with_mirrors","date":"2026-06-13","downloads":28},{"category":"with_mirrors","date":"2026-06-14","downloads":16},{"category":"with_mirrors","date":"2026-06-15","downloads":56},{"category":"with_mirrors","date":"2026-06-16","downloads":145},{"category":"with_mirrors","date":"2026-06-17","downloads":80},{"category":"with_mirrors","date":"2026-06-18","downloads":46},{"category":"with_mirrors","date":"2026-06-19","downloads":29},{"category":"with_mirrors","date":"2026-06-20","downloads":43},{"category":"with_mirrors","date":"2026-06-21","downloads":4},{"category":"with_mirrors","date":"2026-06-22","downloads":30},{"category":"with_mirrors","date":"2026-06-23","downloads":52},{"category":"with_mirrors","date":"2026-06-24","downloads":44},{"category":"with_mirrors","date":"2026-06-25","downloads":12},{"category":"with_mirrors","date":"2026-06-26","downloads":9},{"category":"with_mirrors","date":"2026-06-27","downloads":24},{"category":"with_mirrors","date":"2026-06-28","downloads":17},{"category":"with_mirrors","date":"2026-06-29","downloads":8},{"category":"with_mirrors","date":"2026-06-30","downloads":4},{"category":"with_mirrors","date":"2026-07-01","downloads":20},{"category":"with_mirrors","date":"2026-07-02","downloads":19},{"category":"with_mirrors","date":"2026-07-03","downloads":17},{"category":"with_mirrors","date":"2026-07-04","downloads":35},{"category":"with_mirrors","date":"2026-07-05","downloads":21},{"category":"with_mirrors","date":"2026-07-06","downloads":35},{"category":"with_mirrors","date":"2026-07-07","downloads":23},{"category":"with_mirrors","date":"2026-07-08","downloads":23},{"category":"with_mirrors","date":"2026-07-09","downloads":6},{"category":"with_mirrors","date":"2026-07-10","downloads":15},{"category":"with_mirrors","date":"2026-07-11","downloads":4},{"category":"with_mirrors","date":"2026-07-12","downloads":32},{"category":"with_mirrors","date":"2026-07-13","downloads":15},{"category":"with_mirrors","date":"2026-07-14","downloads":20},{"category":"with_mirrors","date":"2026-07-15","downloads":18},{"category":"with_mirrors","date":"2026-07-16","downloads":47},{"category":"with_mirrors","date":"2026-07-17","downloads":15},{"category":"with_mirrors","date":"2026-07-18","downloads":13},{"category":"with_mirrors","date":"2026-07-19","downloads":2},{"category":"with_mirrors","date":"2026-07-20","downloads":10},{"category":"with_mirrors","date":"2026-07-21","downloads":25},{"category":"with_mirrors","date":"2026-07-22","downloads":50},{"category":"with_mirrors","date":"2026-07-23","downloads":532},{"category":"with_mirrors","date":"2026-07-24","downloads":673},{"category":"with_mirrors","date":"2026-07-25","downloads":298},{"category":"with_mirrors","date":"2026-07-26","downloads":80},{"category":"with_mirrors","date":"2026-07-27","downloads":78},{"category":"with_mirrors","date":"2026-07-28","downloads":78},{"category":"with_mirrors","date":"2026-07-29","downloads":24},{"category":"with_mirrors","date":"2026-07-30","downloads":64},{"category":"with_mirrors","date":"2026-07-31","downloads":124},{"category":"with_mirrors","date":"2026-08-01","downloads":57},{"category":"with_mirrors","date":"2026-08-02","downloads":62},{"category":"with_mirrors","date":"2026-08-03","downloads":192},{"category":"with_mirrors","date":"2026-08-04","downloads":170},{"category":"with_mirrors","date":"2026-08-05","downloads":11},{"category":"with_mirrors","date":"2026-08-06","downloads":37},{"category":"with_mirrors","date":"2026-08-07","downloads":100},{"category":"with_mirrors","date":"2026-08-08","downloads":10},{"category":"with_mirrors","date":"2026-08-09","downloads":64},{"category":"with_mirrors","date":"2026-08-10","downloads":40},{"category":"with_mirrors","date":"2026-08-11","downloads":46},{"category":"with_mirrors","date":"2026-08-12","downloads":31},{"category":"with_mirrors","date":"2026-08-13","downloads":65},{"category":"with_mirrors","date":"2026-08-14","downloads":70},{"category":"with_mirrors","date":"2026-08-15","downloads":48},{"category":"with_mirrors","date":"2026-08-16","downloads":60},{"category":"with_mirrors","date":"2026-08-17","downloads":28},{"category":"with_mirrors","date":"2026-08-18","downloads":5},{"category":"with_mirrors","date":"2026-08-19","downloads":7},{"category":"with_mirrors","date":"2026-08-20","downloads":45},{"category":"with_mirrors","date":"2026-08-21","downloads":105},{"category":"with_mirrors","date":"2026-08-22","downloads":48},{"category":"with_mirrors","date":"2026-08-23","downloads":8},{"category":"with_mirrors","date":"2026-08-24","downloads":16},{"category":"with_mirrors","date":"2026-08-25","downloads":69},{"category":"with_mirrors","date":"2026-08-26","downloads":480},{"category":"with_mirrors","date":"2026-08-27","downloads":241},{"category":"with_mirrors","date":"2026-08-28","downloads":144},{"category":"with_mirrors","date":"2026-08-29","downloads":85},{"category":"with_mirrors","date":"2026-08-30","downloads":96},{"category":"with_mirrors","date":"2026-08-31","downloads":550},{"category":"with_mirrors","date":"2026-09-01","downloads":423},{"category":"with_mirrors","date":"2026-09-02","downloads":102},{"category":"with_mirrors","date":"2026-09-03","downloads":173},{"category":"with_mirrors","date":"2026-09-04","downloads":124},{"category":"with_mirrors","date":"2026-09-05","downloads":35},{"category":"with_mirrors","date":"2026-09-06","downloads":55},{"category":"with_mirrors","date":"2026-09-07","downloads":64},{"category":"with_mirrors","date":"2026-09-08","downloads":128},{"category":"with_mirrors","date":"2026-09-09","downloads":77},{"category":"with_mirrors","date":"2026-09-10","downloads":97},{"category":"with_mirrors","date":"2026-09-11","downloads":172},{"category":"with_mirrors","date":"2026-09-12","downloads":33},{"category":"with_mirrors","date":"2026-09-13","downloads":58},{"category":"with_mirrors","date":"2026-09-14","downloads":16},{"category":"with_mirrors","date":"2026-09-15","downloads":35},{"category":"with_mirrors","date":"2026-09-16","downloads":54},{"category":"with_mirrors","date":"2026-09-17","downloads":40},{"category":"with_mirrors","date":"2026-09-18","downloads":52},{"category":"with_mirrors","date":"2026-09-19","downloads":50},{"category":"with_mirrors","date":"2026-09-20","downloads":190},{"category":"with_mirrors","date":"2026-09-21","downloads":91},{"category":"with_mirrors","date":"2026-09-22","downloads":62},{"category":"with_mirrors","date":"2026-09-23","downloads":52},{"category":"with_mirrors","date":"2026-09-24","downloads":67},{"category":"with_mirrors","date":"2026-09-25","downloads":2},{"category":"with_mirrors","date":"2026-09-26","downloads":64},{"category":"with_mirrors","date":"2026-09-27","downloads":45},{"category":"with_mirrors","date":"2026-09-28","downloads":61},{"category":"with_mirrors","date":"2026-09-29","downloads":99},{"category":"with_mirrors","date":"2026-09-30","downloads":7},{"category":"with_mirrors","date":"2026-10-01","downloads":10},{"category":"with_mirrors","date":"2026-10-02","downloads":94},{"category":"with_mirrors","date":"2026-10-03","downloads":4},{"category":"with_mirrors","date":"2026-10-04","downloads":54},{"category":"with_mirrors","date":"2026-10-05","downloads":7},{"category":"with_mirrors","date":"2026-10-06","downloads":27},{"category":"with_mirrors","date":"2026-10-07","downloads":141},{"category":"with_mirrors","date":"2026-10-08","downloads":31},{"category":"with_mirrors","date":"2026-10-09","downloads":34},{"category":"with_mirrors","date":"2026-10-10","downloads":124},{"category":"with_mirrors","date":"2026-10-11","downloads":97},{"category":"with_mirrors","date":"2026-10-12","downloads":21},{"category":"with_mirrors","date":"2026-10-13","downloads":74},{"category":"with_mirrors","date":"2026-10-14","downloads":45},{"category":"with_mirrors","date":"2026-10-15","downloads":211},{"category":"with_mirrors","date":"2026-10-16","downloads":100},{"category":"with_mirrors","date":"2026-10-17","downloads":81},{"category":"with_mirrors","date":"2026-10-18","downloads":116},{"category":"with_mirrors","date":"2026-10-19","downloads":139},{"category":"with_mirrors","date":"2026-10-20","downloads":59},{"category":"with_mirrors","date":"2026-10-21","downloads":109},{"category":"with_mirrors","date":"2026-10-22","downloads":90},{"category":"with_mirrors","date":"2026-10-23","downloads":85},{"category":"with_mirrors","date":"2026-10-24","downloads":21},{"category":"with_mirrors","date":"2026-10-25","downloads":6},{"category":"with_mirrors","date":"2026-10-26","downloads":79},{"category":"with_mirrors","date":"2026-10-27","downloads":27},{"category":"with_mirrors","date":"2026-10-28","downloads":139},{"category":"without_mirrors","date":"2026-06-07","downloads":71},{"category":"without_mirrors","date":"2026-06-08","downloads":27},{"category":"without_mirrors","date":"2026-06-09","downloads":2},{"category":"without_mirrors","date":"2026-06-10","downloads":87},{"category":"without_mirrors","date":"2026-06-11","downloads":18},{"category":"without_mirrors","date":"2026-06-12","downloads":25},{"category":"without_mirrors","date":"2026-06-13","downloads":3},{"category":"without_mirrors","date":"2026-06-14","downloads":8},{"category":"without_mirrors","date":"2026-06-15","downloads":24},{"category":"without_mirrors","date":"2026-06-16","downloads":40},{"category":"without_mirrors","date":"2026-06-17","downloads":29},{"category":"without_mirrors","date":"2026-06-18","downloads":14},{"category":"without_mirrors","date":"2026-06-19","downloads":1},{"category":"without_mirrors","date":"2026-06-20","downloads":3},{"category":"without_mirrors","date":"2026-06-21","downloads":4},{"category":"without_mirrors","date":"2026-06-22","downloads":16},{"category":"without_mirrors","date":"2026-06-23","downloads":30},{"category":"without_mirrors","date":"2026-06-24","downloads":11},{"category":"without_mirrors","date":"2026-06-25","downloads":2},{"category":"without_mirrors","date":"2026-06-26","downloads":3},{"category":"without_mirrors","date":"2026-06-27","downloads":11},{"category":"without_mirrors","date":"2026-06-29","downloads":7},{"category":"without_mirrors","date":"2026-06-30","downloads":4},{"category":"without_mirrors","date":"2026-07-01","downloads":5},{"category":"without_mirrors","date":"2026-07-02","downloads":2},{"category":"without_mirrors","date":"2026-07-03","downloads":5},{"category":"without_mirrors","date":"2026-07-04","downloads":5},{"category":"without_mirrors","date":"2026-07-06","downloads":5},{"category":"without_mirrors","date":"2026-07-07","downloads":4},{"category":"without_mirrors","date":"2026-07-08","downloads":9},{"category":"without_mirrors","date":"2026-07-09","downloads":2},{"category":"without_mirrors","date":"2026-07-10","downloads":4},{"category":"without_mirrors","date":"2026-07-11","downloads":4},{"category":"without_mirrors","date":"2026-07-12","downloads":2},{"category":"without_mirrors","date":"2026-07-13","downloads":5},{"category":"without_mirrors","date":"2026-07-14","downloads":10},{"category":"without_mirrors","date":"2026-07-15","downloads":7},{"category":"without_mirrors","date":"2026-07-16","downloads":6},{"category":"without_mirrors","date":"2026-07-17","downloads":5},{"category":"without_mirrors","date":"2026-07-18","downloads":1},{"category":"without_mirrors","date":"2026-07-19","downloads":1},{"category":"without_mirrors","date":"2026-07-20","downloads":9},{"category":"without_mirrors","date":"2026-07-21","downloads":14},{"category":"without_mirrors","date":"2026-07-22","downloads":50},{"category":"without_mirrors","date":"2026-07-23","downloads":213},{"category":"without_mirrors","date":"2026-07-24","downloads":242},{"category":"without_mirrors","date":"2026-07-25","downloads":61},{"category":"without_mirrors","date":"2026-07-26","downloads":10},{"category":"without_mirrors","date":"2026-07-27","downloads":8},{"category":"without_mirrors","date":"2026-07-28","downloads":10},{"category":"without_mirrors","date":"2026-07-29","downloads":24},{"category":"without_mirrors","date":"2026-07-30","downloads":44},{"category":"without_mirrors","date":"2026-07-31","downloads":64},{"category":"without_mirrors","date":"2026-08-01","downloads":8},{"category":"without_mirrors","date":"2026-08-02","downloads":2},{"category":"without_mirrors","date":"2026-08-03","downloads":8},{"category":"without_mirrors","date":"2026-08-04","downloads":55},{"category":"without_mirrors","date":"2026-08-05","downloads":11},{"category":"without_mirrors","date":"2026-08-06","downloads":36},{"category":"without_mirrors","date":"2026-08-07","downloads":9},{"category":"without_mirrors","date":"2026-08-08","downloads":5},{"category":"without_mirrors","date":"2026-08-09","downloads":4},{"category":"without_mirrors","date":"2026-08-10","downloads":20},{"category":"without_mirrors","date":"2026-08-11","downloads":5},{"category":"without_mirrors","date":"2026-08-12","downloads":25},{"category":"without_mirrors","date":"2026-08-13","downloads":5},{"category":"without_mirrors","date":"2026-08-14","downloads":10},{"category":"without_mirrors","date":"2026-08-15","downloads":8},{"category":"without_mirrors","date":"2026-08-17","downloads":8},{"category":"without_mirrors","date":"2026-08-18","downloads":5},{"category":"without_mirrors","date":"2026-08-19","downloads":6},{"category":"without_mirrors","date":"2026-08-20","downloads":25},{"category":"without_mirrors","date":"2026-08-21","downloads":25},{"category":"without_mirrors","date":"2026-08-22","downloads":7},{"category":"without_mirrors","date":"2026-08-23","downloads":8},{"category":"without_mirrors","date":"2026-08-24","downloads":8},{"category":"without_mirrors","date":"2026-08-25","downloads":20},{"category":"without_mirrors","date":"2026-08-26","downloads":167},{"category":"without_mirrors","date":"2026-08-27","downloads":93},{"category":"without_mirrors","date":"2026-08-28","downloads":79},{"category":"without_mirrors","date":"2026-08-29","downloads":32},{"category":"without_mirrors","date":"2026-08-30","downloads":15},{"category":"without_mirrors","date":"2026-08-31","downloads":265},{"category":"without_mirrors","date":"2026-09-01","downloads":101},{"category":"without_mirrors","date":"2026-09-02","downloads":28},{"category":"without_mirrors","date":"2026-09-03","downloads":57},{"category":"without_mirrors","date":"2026-09-04","downloads":36},{"category":"without_mirrors","date":"2026-09-05","downloads":13},{"category":"without_mirrors","date":"2026-09-06","downloads":5},{"category":"without_mirrors","date":"2026-09-07","downloads":23},{"category":"without_mirrors","date":"2026-09-08","downloads":75},{"category":"without_mirrors","date":"2026-09-09","downloads":25},{"category":"without_mirrors","date":"2026-09-10","downloads":55},{"category":"without_mirrors","date":"2026-09-11","downloads":20},{"category":"without_mirrors","date":"2026-09-12","downloads":19},{"category":"without_mirrors","date":"2026-09-13","downloads":8},{"category":"without_mirrors","date":"2026-09-14","downloads":16},{"category":"without_mirrors","date":"2026-09-15","downloads":13},{"category":"without_mirrors","date":"2026-09-16","downloads":44},{"category":"without_mirrors","date":"2026-09-17","downloads":24},{"category":"without_mirrors","date":"2026-09-18","downloads":49},{"category":"without_mirrors","date":"2026-09-19","downloads":49},{"category":"without_mirrors","date":"2026-09-20","downloads":19},{"category":"without_mirrors","date":"2026-09-21","downloads":41},{"category":"without_mirrors","date":"2026-09-22","downloads":21},{"category":"without_mirrors","date":"2026-09-23","downloads":50},{"category":"without_mirrors","date":"2026-09-24","downloads":36},{"category":"without_mirrors","date":"2026-09-25","downloads":1},{"category":"without_mirrors","date":"2026-09-26","downloads":14},{"category":"without_mirrors","date":"2026-09-27","downloads":45},{"category":"without_mirrors","date":"2026-09-28","downloads":42},{"category":"without_mirrors","date":"2026-09-29","downloads":26},{"category":"without_mirrors","date":"2026-09-30","downloads":6},{"category":"without_mirrors","date":"2026-10-01","downloads":7},{"category":"without_mirrors","date":"2026-10-02","downloads":14},{"category":"without_mirrors","date":"2026-10-03","downloads":4},{"category":"without_mirrors","date":"2026-10-04","downloads":1},{"category":"without_mirrors","date":"2026-10-05","downloads":4},{"category":"without_mirrors","date":"2026-10-06","downloads":17},{"category":"without_mirrors","date":"2026-10-07","downloads":30},{"category":"without_mirrors","date":"2026-10-08","downloads":21},{"category":"without_mirrors","date":"2026-10-09","downloads":24},{"category":"without_mirrors","date":"2026-10-10","downloads":37},{"category":"without_mirrors","date":"2026-10-11","downloads":10},{"category":"without_mirrors","date":"2026-10-12","downloads":21},{"category":"without_mirrors","date":"2026-10-13","downloads":12},{"category":"without_mirrors","date":"2026-10-14","downloads":38},{"category":"without_mirrors","date":"2026-10-15","downloads":53},{"category":"without_mirrors","date":"2026-10-16","downloads":45},{"category":"without_mirrors","date":"2026-10-17","downloads":30},{"category":"without_mirrors","date":"2026-10-18","downloads":17},{"category":"without_mirrors","date":"2026-10-19","downloads":86},{"category":"without_mirrors","date":"2026-10-20","downloads":52},{"category":"without_mirrors","date":"2026-10-21","downloads":49},{"category":"without_mirrors","date":"2026-10-22","downloads":29},{"category":"without_mirrors","date":"2026-10-23","downloads":22},{"category":"without_mirrors","date":"2026-10-24","downloads":20},{"category":"without_mirrors","date":"2026-10-25","downloads":3},{"category":"without_mirrors","date":"2026-10-26","downloads":43},{"category":"without_mirrors","date":"2026-10-27","downloads":27},{"category":"without_mirrors","date":"2026-10-28","downloads":26}],"package":"mooncake-transfer-engine-npu","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:13:00.391932+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-npu/python_major","payload":{"data":[{"category":"3","date":"2026-06-07","downloads":4},{"category":"3","date":"2026-06-08","downloads":4},{"category":"3","date":"2026-06-10","downloads":16},{"category":"3","date":"2026-06-11","downloads":8},{"category":"3","date":"2026-06-13","downloads":2},{"category":"3","date":"2026-06-15","downloads":10},{"category":"3","date":"2026-06-16","downloads":6},{"category":"3","date":"2026-06-17","downloads":18},{"category":"3","date":"2026-06-18","downloads":11},{"category":"3","date":"2026-06-20","downloads":2},{"category":"3","date":"2026-06-22","downloads":2},{"category":"3","date":"2026-06-23","downloads":16},{"category":"3","date":"2026-06-24","downloads":7},{"category":"3","date":"2026-06-26","downloads":2},{"category":"3","date":"2026-06-27","downloads":2},{"category":"3","date":"2026-06-29","downloads":6},{"category":"3","date":"2026-06-30","downloads":2},{"category":"3","date":"2026-07-01","downloads":5},{"category":"3","date":"2026-07-02","downloads":2},{"category":"3","date":"2026-07-03","downloads":3},{"category":"3","date":"2026-07-06","downloads":5},{"category":"3","date":"2026-07-08","downloads":8},{"category":"3","date":"2026-07-10","downloads":3},{"category":"3","date":"2026-07-11","downloads":2},{"category":"3","date":"2026-07-13","downloads":2},{"category":"3","date":"2026-07-14","downloads":5},{"category":"3","date":"2026-07-15","downloads":5},{"category":"3","date":"2026-07-16","downloads":2},{"category":"3","date":"2026-07-17","downloads":4},{"category":"3","date":"2026-07-20","downloads":8},{"category":"3","date":"2026-07-21","downloads":12},{"category":"3","date":"2026-07-22","downloads":29},{"category":"3","date":"2026-07-23","downloads":27},{"category":"3","date":"2026-07-24","downloads":36},{"category":"3","date":"2026-07-25","downloads":7},{"category":"3","date":"2026-07-27","downloads":8},{"category":"3","date":"2026-07-28","downloads":9},{"category":"3","date":"2026-07-29","downloads":3},{"category":"3","date":"2026-07-30","downloads":11},{"category":"3","date":"2026-07-31","downloads":6},{"category":"3","date":"2026-08-03","downloads":2},{"category":"3","date":"2026-08-04","downloads":15},{"category":"3","date":"2026-08-05","downloads":9},{"category":"3","date":"2026-08-06","downloads":15},{"category":"3","date":"2026-08-07","downloads":5},{"category":"3","date":"2026-08-09","downloads":2},{"category":"3","date":"2026-08-10","downloads":19},{"category":"3","date":"2026-08-11","downloads":2},{"category":"3","date":"2026-08-12","downloads":23},{"category":"3","date":"2026-08-13","downloads":4},{"category":"3","date":"2026-08-14","downloads":3},{"category":"3","date":"2026-08-15","downloads":6},{"category":"3","date":"2026-08-17","downloads":4},{"category":"3","date":"2026-08-19","downloads":2},{"category":"3","date":"2026-08-20","downloads":18},{"category":"3","date":"2026-08-21","downloads":12},{"category":"3","date":"2026-08-22","downloads":6},{"category":"3","date":"2026-08-23","downloads":2},{"category":"3","date":"2026-08-24","downloads":6},{"category":"3","date":"2026-08-25","downloads":9},{"category":"3","date":"2026-08-26","downloads":40},{"category":"3","date":"2026-08-27","downloads":19},{"category":"3","date":"2026-08-28","downloads":12},{"category":"3","date":"2026-08-29","downloads":14},{"category":"3","date":"2026-08-30","downloads":5},{"category":"3","date":"2026-08-31","downloads":43},{"category":"3","date":"2026-09-01","downloads":12},{"category":"3","date":"2026-09-02","downloads":6},{"category":"3","date":"2026-09-03","downloads":23},{"category":"3","date":"2026-09-04","downloads":16},{"category":"3","date":"2026-09-05","downloads":1},{"category":"3","date":"2026-09-06","downloads":1},{"category":"3","date":"2026-09-07","downloads":18},{"category":"3","date":"2026-09-08","downloads":31},{"category":"3","date":"2026-09-09","downloads":12},{"category":"3","date":"2026-09-10","downloads":34},{"category":"3","date":"2026-09-11","downloads":8},{"category":"3","date":"2026-09-12","downloads":4},{"category":"3","date":"2026-09-13","downloads":5},{"category":"3","date":"2026-09-14","downloads":10},{"category":"3","date":"2026-09-15","downloads":9},{"category":"3","date":"2026-09-16","downloads":13},{"category":"3","date":"2026-09-17","downloads":10},{"category":"3","date":"2026-09-18","downloads":12},{"category":"3","date":"2026-09-19","downloads":1},{"category":"3","date":"2026-09-20","downloads":6},{"category":"3","date":"2026-09-21","downloads":22},{"category":"3","date":"2026-09-22","downloads":17},{"category":"3","date":"2026-09-23","downloads":44},{"category":"3","date":"2026-09-24","downloads":34},{"category":"3","date":"2026-09-25","downloads":1},{"category":"3","date":"2026-09-26","downloads":9},{"category":"3","date":"2026-09-28","downloads":19},{"category":"3","date":"2026-09-29","downloads":22},{"category":"3","date":"2026-09-30","downloads":6},{"category":"3","date":"2026-10-01","downloads":2},{"category":"3","date":"2026-10-02","downloads":2},{"category":"3","date":"2026-10-03","downloads":4},{"category":"3","date":"2026-10-04","downloads":1},{"category":"3","date":"2026-10-05","downloads":4},{"category":"3","date":"2026-10-06","downloads":13},{"category":"3","date":"2026-10-07","downloads":20},{"category":"3","date":"2026-10-08","downloads":16},{"category":"3","date":"2026-10-09","downloads":23},{"category":"3","date":"2026-10-10","downloads":23},{"category":"3","date":"2026-10-11","downloads":6},{"category":"3","date":"2026-10-12","downloads":7},{"category":"3","date":"2026-10-13","downloads":6},{"category":"3","date":"2026-10-14","downloads":21},{"category":"3","date":"2026-10-15","downloads":24},{"category":"3","date":"2026-10-16","downloads":24},{"category":"3","date":"2026-10-17","downloads":24},{"category":"3","date":"2026-10-18","downloads":11},{"category":"3","date":"2026-10-19","downloads":79},{"category":"3","date":"2026-10-20","downloads":44},{"category":"3","date":"2026-10-21","downloads":19},{"category":"3","date":"2026-10-22","downloads":25},{"category":"3","date":"2026-10-23","downloads":2},{"category":"3","date":"2026-10-24","downloads":6},{"category":"3","date":"2026-10-26","downloads":7},{"category":"3","date":"2026-10-27","downloads":22},{"category":"3","date":"2026-10-28","downloads":16},{"category":"null","date":"2026-06-07","downloads":67},{"category":"null","date":"2026-06-08","downloads":23},{"category":"null","date":"2026-06-09","downloads":2},{"category":"null","date":"2026-06-10","downloads":71},{"category":"null","date":"2026-06-11","downloads":10},{"category":"null","date":"2026-06-12","downloads":25},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":8},{"category":"null","date":"2026-06-15","downloads":14},{"category":"null","date":"2026-06-16","downloads":34},{"category":"null","date":"2026-06-17","downloads":11},{"category":"null","date":"2026-06-18","downloads":3},{"category":"null","date":"2026-06-19","downloads":1},{"category":"null","date":"2026-06-20","downloads":1},{"category":"null","date":"2026-06-21","downloads":4},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":14},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-25","downloads":2},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":2},{"category":"null","date":"2026-07-03","downloads":2},{"category":"null","date":"2026-07-04","downloads":5},{"category":"null","date":"2026-07-07","downloads":4},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-10","downloads":1},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-12","downloads":2},{"category":"null","date":"2026-07-13","downloads":3},{"category":"null","date":"2026-07-14","downloads":5},{"category":"null","date":"2026-07-15","downloads":2},{"category":"null","date":"2026-07-16","downloads":4},{"category":"null","date":"2026-07-17","downloads":1},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-19","downloads":1},{"category":"null","date":"2026-07-20","downloads":1},{"category":"null","date":"2026-07-21","downloads":2},{"category":"null","date":"2026-07-22","downloads":21},{"category":"null","date":"2026-07-23","downloads":186},{"category":"null","date":"2026-07-24","downloads":206},{"category":"null","date":"2026-07-25","downloads":54},{"category":"null","date":"2026-07-26","downloads":10},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-29","downloads":21},{"category":"null","date":"2026-07-30","downloads":33},{"category":"null","date":"2026-07-31","downloads":58},{"category":"null","date":"2026-08-01","downloads":8},{"category":"null","date":"2026-08-02","downloads":2},{"category":"null","date":"2026-08-03","downloads":6},{"category":"null","date":"2026-08-04","downloads":40},{"category":"null","date":"2026-08-05","downloads":2},{"category":"null","date":"2026-08-06","downloads":21},{"category":"null","date":"2026-08-07","downloads":4},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":1},{"category":"null","date":"2026-08-11","downloads":3},{"category":"null","date":"2026-08-12","downloads":2},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":7},{"category":"null","date":"2026-08-15","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":5},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":7},{"category":"null","date":"2026-08-21","downloads":13},{"category":"null","date":"2026-08-22","downloads":1},{"category":"null","date":"2026-08-23","downloads":6},{"category":"null","date":"2026-08-24","downloads":2},{"category":"null","date":"2026-08-25","downloads":11},{"category":"null","date":"2026-08-26","downloads":127},{"category":"null","date":"2026-08-27","downloads":74},{"category":"null","date":"2026-08-28","downloads":67},{"category":"null","date":"2026-08-29","downloads":18},{"category":"null","date":"2026-08-30","downloads":10},{"category":"null","date":"2026-08-31","downloads":222},{"category":"null","date":"2026-09-01","downloads":89},{"category":"null","date":"2026-09-02","downloads":22},{"category":"null","date":"2026-09-03","downloads":34},{"category":"null","date":"2026-09-04","downloads":20},{"category":"null","date":"2026-09-05","downloads":12},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":5},{"category":"null","date":"2026-09-08","downloads":44},{"category":"null","date":"2026-09-09","downloads":13},{"category":"null","date":"2026-09-10","downloads":21},{"category":"null","date":"2026-09-11","downloads":12},{"category":"null","date":"2026-09-12","downloads":15},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":6},{"category":"null","date":"2026-09-15","downloads":4},{"category":"null","date":"2026-09-16","downloads":31},{"category":"null","date":"2026-09-17","downloads":14},{"category":"null","date":"2026-09-18","downloads":37},{"category":"null","date":"2026-09-19","downloads":48},{"category":"null","date":"2026-09-20","downloads":13},{"category":"null","date":"2026-09-21","downloads":19},{"category":"null","date":"2026-09-22","downloads":4},{"category":"null","date":"2026-09-23","downloads":6},{"category":"null","date":"2026-09-24","downloads":2},{"category":"null","date":"2026-09-26","downloads":5},{"category":"null","date":"2026-09-27","downloads":45},{"category":"null","date":"2026-09-28","downloads":23},{"category":"null","date":"2026-09-29","downloads":4},{"category":"null","date":"2026-10-01","downloads":5},{"category":"null","date":"2026-10-02","downloads":12},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-07","downloads":10},{"category":"null","date":"2026-10-08","downloads":5},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-10","downloads":14},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":14},{"category":"null","date":"2026-10-13","downloads":6},{"category":"null","date":"2026-10-14","downloads":17},{"category":"null","date":"2026-10-15","downloads":29},{"category":"null","date":"2026-10-16","downloads":21},{"category":"null","date":"2026-10-17","downloads":6},{"category":"null","date":"2026-10-18","downloads":6},{"category":"null","date":"2026-10-19","downloads":7},{"category":"null","date":"2026-10-20","downloads":8},{"category":"null","date":"2026-10-21","downloads":30},{"category":"null","date":"2026-10-22","downloads":4},{"category":"null","date":"2026-10-23","downloads":20},{"category":"null","date":"2026-10-24","downloads":14},{"category":"null","date":"2026-10-25","downloads":3},{"category":"null","date":"2026-10-26","downloads":36},{"category":"null","date":"2026-10-27","downloads":5},{"category":"null","date":"2026-10-28","downloads":10}],"package":"mooncake-transfer-engine-npu","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:13:03.108647+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-npu/python_minor","payload":{"data":[{"category":"3.10","date":"2026-06-11","downloads":2},{"category":"3.10","date":"2026-07-03","downloads":1},{"category":"3.10","date":"2026-07-22","downloads":2},{"category":"3.10","date":"2026-08-07","downloads":2},{"category":"3.10","date":"2026-08-21","downloads":1},{"category":"3.10","date":"2026-08-24","downloads":2},{"category":"3.10","date":"2026-08-29","downloads":1},{"category":"3.10","date":"2026-09-03","downloads":1},{"category":"3.10","date":"2026-09-18","downloads":3},{"category":"3.10","date":"2026-09-28","downloads":1},{"category":"3.10","date":"2026-10-12","downloads":1},{"category":"3.11","date":"2026-06-07","downloads":4},{"category":"3.11","date":"2026-06-08","downloads":4},{"category":"3.11","date":"2026-06-10","downloads":16},{"category":"3.11","date":"2026-06-11","downloads":6},{"category":"3.11","date":"2026-06-13","downloads":2},{"category":"3.11","date":"2026-06-15","downloads":6},{"category":"3.11","date":"2026-06-16","downloads":6},{"category":"3.11","date":"2026-06-17","downloads":17},{"category":"3.11","date":"2026-06-18","downloads":11},{"category":"3.11","date":"2026-06-20","downloads":2},{"category":"3.11","date":"2026-06-23","downloads":4},{"category":"3.11","date":"2026-06-24","downloads":4},{"category":"3.11","date":"2026-06-26","downloads":2},{"category":"3.11","date":"2026-06-27","downloads":2},{"category":"3.11","date":"2026-06-29","downloads":4},{"category":"3.11","date":"2026-07-01","downloads":3},{"category":"3.11","date":"2026-07-02","downloads":2},{"category":"3.11","date":"2026-07-06","downloads":5},{"category":"3.11","date":"2026-07-08","downloads":8},{"category":"3.11","date":"2026-07-10","downloads":3},{"category":"3.11","date":"2026-07-13","downloads":2},{"category":"3.11","date":"2026-07-14","downloads":3},{"category":"3.11","date":"2026-07-15","downloads":5},{"category":"3.11","date":"2026-07-16","downloads":2},{"category":"3.11","date":"2026-07-17","downloads":2},{"category":"3.11","date":"2026-07-20","downloads":4},{"category":"3.11","date":"2026-07-21","downloads":2},{"category":"3.11","date":"2026-07-22","downloads":14},{"category":"3.11","date":"2026-07-23","downloads":23},{"category":"3.11","date":"2026-07-24","downloads":24},{"category":"3.11","date":"2026-07-25","downloads":3},{"category":"3.11","date":"2026-07-28","downloads":5},{"category":"3.11","date":"2026-07-30","downloads":7},{"category":"3.11","date":"2026-07-31","downloads":4},{"category":"3.11","date":"2026-08-04","downloads":6},{"category":"3.11","date":"2026-08-05","downloads":9},{"category":"3.11","date":"2026-08-06","downloads":9},{"category":"3.11","date":"2026-08-09","downloads":2},{"category":"3.11","date":"2026-08-10","downloads":10},{"category":"3.11","date":"2026-08-12","downloads":13},{"category":"3.11","date":"2026-08-13","downloads":3},{"category":"3.11","date":"2026-08-15","downloads":2},{"category":"3.11","date":"2026-08-17","downloads":2},{"category":"3.11","date":"2026-08-19","downloads":2},{"category":"3.11","date":"2026-08-20","downloads":10},{"category":"3.11","date":"2026-08-21","downloads":3},{"category":"3.11","date":"2026-08-22","downloads":2},{"category":"3.11","date":"2026-08-25","downloads":7},{"category":"3.11","date":"2026-08-26","downloads":23},{"category":"3.11","date":"2026-08-27","downloads":1},{"category":"3.11","date":"2026-08-28","downloads":1},{"category":"3.11","date":"2026-08-29","downloads":1},{"category":"3.11","date":"2026-08-31","downloads":36},{"category":"3.11","date":"2026-09-01","downloads":7},{"category":"3.11","date":"2026-09-02","downloads":2},{"category":"3.11","date":"2026-09-03","downloads":11},{"category":"3.11","date":"2026-09-04","downloads":14},{"category":"3.11","date":"2026-09-07","downloads":9},{"category":"3.11","date":"2026-09-08","downloads":17},{"category":"3.11","date":"2026-09-09","downloads":4},{"category":"3.11","date":"2026-09-10","downloads":7},{"category":"3.11","date":"2026-09-12","downloads":1},{"category":"3.11","date":"2026-09-13","downloads":3},{"category":"3.11","date":"2026-09-14","downloads":7},{"category":"3.11","date":"2026-09-15","downloads":8},{"category":"3.11","date":"2026-09-16","downloads":7},{"category":"3.11","date":"2026-09-17","downloads":9},{"category":"3.11","date":"2026-09-18","downloads":7},{"category":"3.11","date":"2026-09-20","downloads":6},{"category":"3.11","date":"2026-09-21","downloads":18},{"category":"3.11","date":"2026-09-22","downloads":3},{"category":"3.11","date":"2026-09-23","downloads":41},{"category":"3.11","date":"2026-09-24","downloads":29},{"category":"3.11","date":"2026-09-25","downloads":1},{"category":"3.11","date":"2026-09-26","downloads":3},{"category":"3.11","date":"2026-09-28","downloads":8},{"category":"3.11","date":"2026-09-29","downloads":8},{"category":"3.11","date":"2026-10-02","downloads":2},{"category":"3.11","date":"2026-10-07","downloads":2},{"category":"3.11","date":"2026-10-08","downloads":2},{"category":"3.11","date":"2026-10-09","downloads":5},{"category":"3.11","date":"2026-10-10","downloads":1},{"category":"3.11","date":"2026-10-12","downloads":2},{"category":"3.11","date":"2026-10-14","downloads":14},{"category":"3.11","date":"2026-10-15","downloads":16},{"category":"3.11","date":"2026-10-16","downloads":21},{"category":"3.11","date":"2026-10-17","downloads":18},{"category":"3.11","date":"2026-10-18","downloads":9},{"category":"3.11","date":"2026-10-19","downloads":7},{"category":"3.11","date":"2026-10-20","downloads":5},{"category":"3.11","date":"2026-10-21","downloads":2},{"category":"3.11","date":"2026-10-22","downloads":1},{"category":"3.11","date":"2026-10-24","downloads":2},{"category":"3.11","date":"2026-10-26","downloads":2},{"category":"3.12","date":"2026-06-22","downloads":2},{"category":"3.12","date":"2026-06-23","downloads":12},{"category":"3.12","date":"2026-06-24","downloads":3},{"category":"3.12","date":"2026-06-29","downloads":2},{"category":"3.12","date":"2026-06-30","downloads":2},{"category":"3.12","date":"2026-07-01","downloads":2},{"category":"3.12","date":"2026-07-03","downloads":2},{"category":"3.12","date":"2026-07-11","downloads":2},{"category":"3.12","date":"2026-07-14","downloads":2},{"category":"3.12","date":"2026-07-17","downloads":2},{"category":"3.12","date":"2026-07-20","downloads":4},{"category":"3.12","date":"2026-07-21","downloads":10},{"category":"3.12","date":"2026-07-22","downloads":11},{"category":"3.12","date":"2026-07-23","downloads":4},{"category":"3.12","date":"2026-07-24","downloads":12},{"category":"3.12","date":"2026-07-25","downloads":4},{"category":"3.12","date":"2026-07-27","downloads":8},{"category":"3.12","date":"2026-07-28","downloads":4},{"category":"3.12","date":"2026-07-29","downloads":3},{"category":"3.12","date":"2026-07-30","downloads":4},{"category":"3.12","date":"2026-07-31","downloads":2},{"category":"3.12","date":"2026-08-03","downloads":2},{"category":"3.12","date":"2026-08-04","downloads":8},{"category":"3.12","date":"2026-08-06","downloads":6},{"category":"3.12","date":"2026-08-07","downloads":3},{"category":"3.12","date":"2026-08-10","downloads":9},{"category":"3.12","date":"2026-08-11","downloads":2},{"category":"3.12","date":"2026-08-12","downloads":8},{"category":"3.12","date":"2026-08-13","downloads":1},{"category":"3.12","date":"2026-08-14","downloads":3},{"category":"3.12","date":"2026-08-15","downloads":4},{"category":"3.12","date":"2026-08-17","downloads":2},{"category":"3.12","date":"2026-08-20","downloads":8},{"category":"3.12","date":"2026-08-21","downloads":8},{"category":"3.12","date":"2026-08-22","downloads":4},{"category":"3.12","date":"2026-08-23","downloads":2},{"category":"3.12","date":"2026-08-24","downloads":2},{"category":"3.12","date":"2026-08-25","downloads":2},{"category":"3.12","date":"2026-08-26","downloads":15},{"category":"3.12","date":"2026-08-27","downloads":17},{"category":"3.12","date":"2026-08-28","downloads":9},{"category":"3.12","date":"2026-08-29","downloads":12},{"category":"3.12","date":"2026-08-30","downloads":5},{"category":"3.12","date":"2026-08-31","downloads":5},{"category":"3.12","date":"2026-09-01","downloads":4},{"category":"3.12","date":"2026-09-02","downloads":4},{"category":"3.12","date":"2026-09-03","downloads":11},{"category":"3.12","date":"2026-09-04","downloads":2},{"category":"3.12","date":"2026-09-05","downloads":1},{"category":"3.12","date":"2026-09-06","downloads":1},{"category":"3.12","date":"2026-09-07","downloads":9},{"category":"3.12","date":"2026-09-08","downloads":14},{"category":"3.12","date":"2026-09-09","downloads":8},{"category":"3.12","date":"2026-09-10","downloads":25},{"category":"3.12","date":"2026-09-11","downloads":8},{"category":"3.12","date":"2026-09-12","downloads":3},{"category":"3.12","date":"2026-09-13","downloads":2},{"category":"3.12","date":"2026-09-14","downloads":1},{"category":"3.12","date":"2026-09-15","downloads":1},{"category":"3.12","date":"2026-09-16","downloads":6},{"category":"3.12","date":"2026-09-17","downloads":1},{"category":"3.12","date":"2026-09-18","downloads":2},{"category":"3.12","date":"2026-09-19","downloads":1},{"category":"3.12","date":"2026-09-21","downloads":4},{"category":"3.12","date":"2026-09-22","downloads":14},{"category":"3.12","date":"2026-09-23","downloads":3},{"category":"3.12","date":"2026-09-24","downloads":1},{"category":"3.12","date":"2026-09-26","downloads":6},{"category":"3.12","date":"2026-09-28","downloads":10},{"category":"3.12","date":"2026-09-29","downloads":11},{"category":"3.12","date":"2026-09-30","downloads":6},{"category":"3.12","date":"2026-10-01","downloads":1},{"category":"3.12","date":"2026-10-03","downloads":4},{"category":"3.12","date":"2026-10-04","downloads":1},{"category":"3.12","date":"2026-10-05","downloads":4},{"category":"3.12","date":"2026-10-06","downloads":13},{"category":"3.12","date":"2026-10-07","downloads":16},{"category":"3.12","date":"2026-10-08","downloads":10},{"category":"3.12","date":"2026-10-09","downloads":18},{"category":"3.12","date":"2026-10-10","downloads":22},{"category":"3.12","date":"2026-10-11","downloads":6},{"category":"3.12","date":"2026-10-12","downloads":4},{"category":"3.12","date":"2026-10-13","downloads":6},{"category":"3.12","date":"2026-10-14","downloads":7},{"category":"3.12","date":"2026-10-15","downloads":8},{"category":"3.12","date":"2026-10-16","downloads":3},{"category":"3.12","date":"2026-10-17","downloads":6},{"category":"3.12","date":"2026-10-18","downloads":2},{"category":"3.12","date":"2026-10-19","downloads":72},{"category":"3.12","date":"2026-10-20","downloads":39},{"category":"3.12","date":"2026-10-21","downloads":17},{"category":"3.12","date":"2026-10-22","downloads":24},{"category":"3.12","date":"2026-10-23","downloads":2},{"category":"3.12","date":"2026-10-24","downloads":4},{"category":"3.12","date":"2026-10-26","downloads":5},{"category":"3.12","date":"2026-10-27","downloads":22},{"category":"3.12","date":"2026-10-28","downloads":16},{"category":"3.13","date":"2026-06-15","downloads":4},{"category":"3.13","date":"2026-07-22","downloads":2},{"category":"3.13","date":"2026-08-27","downloads":1},{"category":"3.13","date":"2026-08-28","downloads":1},{"category":"3.13","date":"2026-09-14","downloads":2},{"category":"3.13","date":"2026-10-07","downloads":2},{"category":"3.14","date":"2026-08-12","downloads":2},{"category":"3.14","date":"2026-08-24","downloads":2},{"category":"3.14","date":"2026-08-28","downloads":1},{"category":"3.14","date":"2026-09-24","downloads":4},{"category":"3.14","date":"2026-09-29","downloads":1},{"category":"3.9","date":"2026-06-17","downloads":1},{"category":"3.9","date":"2026-08-04","downloads":1},{"category":"3.9","date":"2026-08-26","downloads":2},{"category":"3.9","date":"2026-08-31","downloads":2},{"category":"3.9","date":"2026-09-01","downloads":1},{"category":"3.9","date":"2026-09-10","downloads":2},{"category":"3.9","date":"2026-09-29","downloads":2},{"category":"3.9","date":"2026-10-01","downloads":1},{"category":"3.9","date":"2026-10-08","downloads":4},{"category":"null","date":"2026-06-07","downloads":67},{"category":"null","date":"2026-06-08","downloads":23},{"category":"null","date":"2026-06-09","downloads":2},{"category":"null","date":"2026-06-10","downloads":71},{"category":"null","date":"2026-06-11","downloads":10},{"category":"null","date":"2026-06-12","downloads":25},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":8},{"category":"null","date":"2026-06-15","downloads":14},{"category":"null","date":"2026-06-16","downloads":34},{"category":"null","date":"2026-06-17","downloads":11},{"category":"null","date":"2026-06-18","downloads":3},{"category":"null","date":"2026-06-19","downloads":1},{"category":"null","date":"2026-06-20","downloads":1},{"category":"null","date":"2026-06-21","downloads":4},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":14},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-25","downloads":2},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":2},{"category":"null","date":"2026-07-03","downloads":2},{"category":"null","date":"2026-07-04","downloads":5},{"category":"null","date":"2026-07-07","downloads":4},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-10","downloads":1},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-12","downloads":2},{"category":"null","date":"2026-07-13","downloads":3},{"category":"null","date":"2026-07-14","downloads":5},{"category":"null","date":"2026-07-15","downloads":2},{"category":"null","date":"2026-07-16","downloads":4},{"category":"null","date":"2026-07-17","downloads":1},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-19","downloads":1},{"category":"null","date":"2026-07-20","downloads":1},{"category":"null","date":"2026-07-21","downloads":2},{"category":"null","date":"2026-07-22","downloads":21},{"category":"null","date":"2026-07-23","downloads":186},{"category":"null","date":"2026-07-24","downloads":206},{"category":"null","date":"2026-07-25","downloads":54},{"category":"null","date":"2026-07-26","downloads":10},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-29","downloads":21},{"category":"null","date":"2026-07-30","downloads":33},{"category":"null","date":"2026-07-31","downloads":58},{"category":"null","date":"2026-08-01","downloads":8},{"category":"null","date":"2026-08-02","downloads":2},{"category":"null","date":"2026-08-03","downloads":6},{"category":"null","date":"2026-08-04","downloads":40},{"category":"null","date":"2026-08-05","downloads":2},{"category":"null","date":"2026-08-06","downloads":21},{"category":"null","date":"2026-08-07","downloads":4},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":1},{"category":"null","date":"2026-08-11","downloads":3},{"category":"null","date":"2026-08-12","downloads":2},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":7},{"category":"null","date":"2026-08-15","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":5},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":7},{"category":"null","date":"2026-08-21","downloads":13},{"category":"null","date":"2026-08-22","downloads":1},{"category":"null","date":"2026-08-23","downloads":6},{"category":"null","date":"2026-08-24","downloads":2},{"category":"null","date":"2026-08-25","downloads":11},{"category":"null","date":"2026-08-26","downloads":127},{"category":"null","date":"2026-08-27","downloads":74},{"category":"null","date":"2026-08-28","downloads":67},{"category":"null","date":"2026-08-29","downloads":18},{"category":"null","date":"2026-08-30","downloads":10},{"category":"null","date":"2026-08-31","downloads":222},{"category":"null","date":"2026-09-01","downloads":89},{"category":"null","date":"2026-09-02","downloads":22},{"category":"null","date":"2026-09-03","downloads":34},{"category":"null","date":"2026-09-04","downloads":20},{"category":"null","date":"2026-09-05","downloads":12},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":5},{"category":"null","date":"2026-09-08","downloads":44},{"category":"null","date":"2026-09-09","downloads":13},{"category":"null","date":"2026-09-10","downloads":21},{"category":"null","date":"2026-09-11","downloads":12},{"category":"null","date":"2026-09-12","downloads":15},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":6},{"category":"null","date":"2026-09-15","downloads":4},{"category":"null","date":"2026-09-16","downloads":31},{"category":"null","date":"2026-09-17","downloads":14},{"category":"null","date":"2026-09-18","downloads":37},{"category":"null","date":"2026-09-19","downloads":48},{"category":"null","date":"2026-09-20","downloads":13},{"category":"null","date":"2026-09-21","downloads":19},{"category":"null","date":"2026-09-22","downloads":4},{"category":"null","date":"2026-09-23","downloads":6},{"category":"null","date":"2026-09-24","downloads":2},{"category":"null","date":"2026-09-26","downloads":5},{"category":"null","date":"2026-09-27","downloads":45},{"category":"null","date":"2026-09-28","downloads":23},{"category":"null","date":"2026-09-29","downloads":4},{"category":"null","date":"2026-10-01","downloads":5},{"category":"null","date":"2026-10-02","downloads":12},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-07","downloads":10},{"category":"null","date":"2026-10-08","downloads":5},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-10","downloads":14},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":14},{"category":"null","date":"2026-10-13","downloads":6},{"category":"null","date":"2026-10-14","downloads":17},{"category":"null","date":"2026-10-15","downloads":29},{"category":"null","date":"2026-10-16","downloads":21},{"category":"null","date":"2026-10-17","downloads":6},{"category":"null","date":"2026-10-18","downloads":6},{"category":"null","date":"2026-10-19","downloads":7},{"category":"null","date":"2026-10-20","downloads":8},{"category":"null","date":"2026-10-21","downloads":30},{"category":"null","date":"2026-10-22","downloads":4},{"category":"null","date":"2026-10-23","downloads":20},{"category":"null","date":"2026-10-24","downloads":14},{"category":"null","date":"2026-10-25","downloads":3},{"category":"null","date":"2026-10-26","downloads":36},{"category":"null","date":"2026-10-27","downloads":5},{"category":"null","date":"2026-10-28","downloads":10}],"package":"mooncake-transfer-engine-npu","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:13:05.860444+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-npu/system","payload":{"data":[{"category":"Darwin","date":"2026-06-07","downloads":4},{"category":"Darwin","date":"2026-06-08","downloads":4},{"category":"Darwin","date":"2026-06-10","downloads":16},{"category":"Darwin","date":"2026-06-16","downloads":4},{"category":"Darwin","date":"2026-07-22","downloads":2},{"category":"Darwin","date":"2026-07-23","downloads":20},{"category":"Darwin","date":"2026-07-24","downloads":20},{"category":"Darwin","date":"2026-08-12","downloads":2},{"category":"Darwin","date":"2026-08-24","downloads":2},{"category":"Darwin","date":"2026-08-26","downloads":20},{"category":"Darwin","date":"2026-08-28","downloads":1},{"category":"Darwin","date":"2026-08-31","downloads":30},{"category":"Darwin","date":"2026-09-01","downloads":1},{"category":"Darwin","date":"2026-09-24","downloads":5},{"category":"Darwin","date":"2026-10-15","downloads":1},{"category":"Linux","date":"2026-06-11","downloads":8},{"category":"Linux","date":"2026-06-13","downloads":2},{"category":"Linux","date":"2026-06-15","downloads":10},{"category":"Linux","date":"2026-06-16","downloads":2},{"category":"Linux","date":"2026-06-17","downloads":18},{"category":"Linux","date":"2026-06-18","downloads":11},{"category":"Linux","date":"2026-06-20","downloads":2},{"category":"Linux","date":"2026-06-22","downloads":2},{"category":"Linux","date":"2026-06-23","downloads":16},{"category":"Linux","date":"2026-06-24","downloads":7},{"category":"Linux","date":"2026-06-26","downloads":2},{"category":"Linux","date":"2026-06-27","downloads":2},{"category":"Linux","date":"2026-06-29","downloads":6},{"category":"Linux","date":"2026-06-30","downloads":2},{"category":"Linux","date":"2026-07-01","downloads":5},{"category":"Linux","date":"2026-07-02","downloads":2},{"category":"Linux","date":"2026-07-03","downloads":3},{"category":"Linux","date":"2026-07-06","downloads":5},{"category":"Linux","date":"2026-07-08","downloads":8},{"category":"Linux","date":"2026-07-10","downloads":3},{"category":"Linux","date":"2026-07-11","downloads":2},{"category":"Linux","date":"2026-07-13","downloads":2},{"category":"Linux","date":"2026-07-14","downloads":5},{"category":"Linux","date":"2026-07-15","downloads":5},{"category":"Linux","date":"2026-07-16","downloads":2},{"category":"Linux","date":"2026-07-17","downloads":4},{"category":"Linux","date":"2026-07-20","downloads":8},{"category":"Linux","date":"2026-07-21","downloads":12},{"category":"Linux","date":"2026-07-22","downloads":27},{"category":"Linux","date":"2026-07-23","downloads":7},{"category":"Linux","date":"2026-07-24","downloads":16},{"category":"Linux","date":"2026-07-25","downloads":7},{"category":"Linux","date":"2026-07-27","downloads":8},{"category":"Linux","date":"2026-07-28","downloads":9},{"category":"Linux","date":"2026-07-29","downloads":2},{"category":"Linux","date":"2026-07-30","downloads":11},{"category":"Linux","date":"2026-07-31","downloads":4},{"category":"Linux","date":"2026-08-03","downloads":2},{"category":"Linux","date":"2026-08-04","downloads":15},{"category":"Linux","date":"2026-08-05","downloads":9},{"category":"Linux","date":"2026-08-06","downloads":15},{"category":"Linux","date":"2026-08-07","downloads":5},{"category":"Linux","date":"2026-08-09","downloads":2},{"category":"Linux","date":"2026-08-10","downloads":19},{"category":"Linux","date":"2026-08-11","downloads":2},{"category":"Linux","date":"2026-08-12","downloads":21},{"category":"Linux","date":"2026-08-13","downloads":4},{"category":"Linux","date":"2026-08-14","downloads":3},{"category":"Linux","date":"2026-08-15","downloads":6},{"category":"Linux","date":"2026-08-17","downloads":4},{"category":"Linux","date":"2026-08-19","downloads":2},{"category":"Linux","date":"2026-08-20","downloads":18},{"category":"Linux","date":"2026-08-21","downloads":12},{"category":"Linux","date":"2026-08-22","downloads":6},{"category":"Linux","date":"2026-08-23","downloads":2},{"category":"Linux","date":"2026-08-24","downloads":4},{"category":"Linux","date":"2026-08-25","downloads":9},{"category":"Linux","date":"2026-08-26","downloads":20},{"category":"Linux","date":"2026-08-27","downloads":19},{"category":"Linux","date":"2026-08-28","downloads":11},{"category":"Linux","date":"2026-08-29","downloads":14},{"category":"Linux","date":"2026-08-30","downloads":5},{"category":"Linux","date":"2026-08-31","downloads":13},{"category":"Linux","date":"2026-09-01","downloads":11},{"category":"Linux","date":"2026-09-02","downloads":6},{"category":"Linux","date":"2026-09-03","downloads":23},{"category":"Linux","date":"2026-09-04","downloads":16},{"category":"Linux","date":"2026-09-05","downloads":1},{"category":"Linux","date":"2026-09-06","downloads":1},{"category":"Linux","date":"2026-09-07","downloads":18},{"category":"Linux","date":"2026-09-08","downloads":31},{"category":"Linux","date":"2026-09-09","downloads":12},{"category":"Linux","date":"2026-09-10","downloads":34},{"category":"Linux","date":"2026-09-11","downloads":8},{"category":"Linux","date":"2026-09-12","downloads":4},{"category":"Linux","date":"2026-09-13","downloads":5},{"category":"Linux","date":"2026-09-14","downloads":9},{"category":"Linux","date":"2026-09-15","downloads":9},{"category":"Linux","date":"2026-09-16","downloads":13},{"category":"Linux","date":"2026-09-17","downloads":9},{"category":"Linux","date":"2026-09-18","downloads":12},{"category":"Linux","date":"2026-09-19","downloads":1},{"category":"Linux","date":"2026-09-20","downloads":6},{"category":"Linux","date":"2026-09-21","downloads":22},{"category":"Linux","date":"2026-09-22","downloads":17},{"category":"Linux","date":"2026-09-23","downloads":44},{"category":"Linux","date":"2026-09-24","downloads":29},{"category":"Linux","date":"2026-09-25","downloads":1},{"category":"Linux","date":"2026-09-26","downloads":9},{"category":"Linux","date":"2026-09-28","downloads":19},{"category":"Linux","date":"2026-09-29","downloads":21},{"category":"Linux","date":"2026-09-30","downloads":6},{"category":"Linux","date":"2026-10-01","downloads":2},{"category":"Linux","date":"2026-10-02","downloads":2},{"category":"Linux","date":"2026-10-03","downloads":4},{"category":"Linux","date":"2026-10-04","downloads":1},{"category":"Linux","date":"2026-10-05","downloads":4},{"category":"Linux","date":"2026-10-06","downloads":13},{"category":"Linux","date":"2026-10-07","downloads":18},{"category":"Linux","date":"2026-10-08","downloads":16},{"category":"Linux","date":"2026-10-09","downloads":23},{"category":"Linux","date":"2026-10-10","downloads":23},{"category":"Linux","date":"2026-10-11","downloads":6},{"category":"Linux","date":"2026-10-12","downloads":7},{"category":"Linux","date":"2026-10-13","downloads":6},{"category":"Linux","date":"2026-10-14","downloads":21},{"category":"Linux","date":"2026-10-15","downloads":21},{"category":"Linux","date":"2026-10-16","downloads":24},{"category":"Linux","date":"2026-10-17","downloads":24},{"category":"Linux","date":"2026-10-18","downloads":11},{"category":"Linux","date":"2026-10-19","downloads":79},{"category":"Linux","date":"2026-10-20","downloads":44},{"category":"Linux","date":"2026-10-21","downloads":19},{"category":"Linux","date":"2026-10-22","downloads":25},{"category":"Linux","date":"2026-10-23","downloads":2},{"category":"Linux","date":"2026-10-24","downloads":6},{"category":"Linux","date":"2026-10-26","downloads":7},{"category":"Linux","date":"2026-10-27","downloads":22},{"category":"Linux","date":"2026-10-28","downloads":16},{"category":"Windows","date":"2026-07-29","downloads":1},{"category":"Windows","date":"2026-07-31","downloads":2},{"category":"Windows","date":"2026-09-14","downloads":1},{"category":"Windows","date":"2026-09-17","downloads":1},{"category":"Windows","date":"2026-09-29","downloads":1},{"category":"Windows","date":"2026-10-07","downloads":2},{"category":"Windows","date":"2026-10-15","downloads":2},{"category":"null","date":"2026-06-07","downloads":67},{"category":"null","date":"2026-06-08","downloads":23},{"category":"null","date":"2026-06-09","downloads":2},{"category":"null","date":"2026-06-10","downloads":71},{"category":"null","date":"2026-06-11","downloads":10},{"category":"null","date":"2026-06-12","downloads":25},{"category":"null","date":"2026-06-13","downloads":1},{"category":"null","date":"2026-06-14","downloads":8},{"category":"null","date":"2026-06-15","downloads":14},{"category":"null","date":"2026-06-16","downloads":34},{"category":"null","date":"2026-06-17","downloads":11},{"category":"null","date":"2026-06-18","downloads":3},{"category":"null","date":"2026-06-19","downloads":1},{"category":"null","date":"2026-06-20","downloads":1},{"category":"null","date":"2026-06-21","downloads":4},{"category":"null","date":"2026-06-22","downloads":14},{"category":"null","date":"2026-06-23","downloads":14},{"category":"null","date":"2026-06-24","downloads":4},{"category":"null","date":"2026-06-25","downloads":2},{"category":"null","date":"2026-06-26","downloads":1},{"category":"null","date":"2026-06-27","downloads":9},{"category":"null","date":"2026-06-29","downloads":1},{"category":"null","date":"2026-06-30","downloads":2},{"category":"null","date":"2026-07-03","downloads":2},{"category":"null","date":"2026-07-04","downloads":5},{"category":"null","date":"2026-07-07","downloads":4},{"category":"null","date":"2026-07-08","downloads":1},{"category":"null","date":"2026-07-09","downloads":2},{"category":"null","date":"2026-07-10","downloads":1},{"category":"null","date":"2026-07-11","downloads":2},{"category":"null","date":"2026-07-12","downloads":2},{"category":"null","date":"2026-07-13","downloads":3},{"category":"null","date":"2026-07-14","downloads":5},{"category":"null","date":"2026-07-15","downloads":2},{"category":"null","date":"2026-07-16","downloads":4},{"category":"null","date":"2026-07-17","downloads":1},{"category":"null","date":"2026-07-18","downloads":1},{"category":"null","date":"2026-07-19","downloads":1},{"category":"null","date":"2026-07-20","downloads":1},{"category":"null","date":"2026-07-21","downloads":2},{"category":"null","date":"2026-07-22","downloads":21},{"category":"null","date":"2026-07-23","downloads":186},{"category":"null","date":"2026-07-24","downloads":206},{"category":"null","date":"2026-07-25","downloads":54},{"category":"null","date":"2026-07-26","downloads":10},{"category":"null","date":"2026-07-28","downloads":1},{"category":"null","date":"2026-07-29","downloads":21},{"category":"null","date":"2026-07-30","downloads":33},{"category":"null","date":"2026-07-31","downloads":58},{"category":"null","date":"2026-08-01","downloads":8},{"category":"null","date":"2026-08-02","downloads":2},{"category":"null","date":"2026-08-03","downloads":6},{"category":"null","date":"2026-08-04","downloads":40},{"category":"null","date":"2026-08-05","downloads":2},{"category":"null","date":"2026-08-06","downloads":21},{"category":"null","date":"2026-08-07","downloads":4},{"category":"null","date":"2026-08-08","downloads":5},{"category":"null","date":"2026-08-09","downloads":2},{"category":"null","date":"2026-08-10","downloads":1},{"category":"null","date":"2026-08-11","downloads":3},{"category":"null","date":"2026-08-12","downloads":2},{"category":"null","date":"2026-08-13","downloads":1},{"category":"null","date":"2026-08-14","downloads":7},{"category":"null","date":"2026-08-15","downloads":2},{"category":"null","date":"2026-08-17","downloads":4},{"category":"null","date":"2026-08-18","downloads":5},{"category":"null","date":"2026-08-19","downloads":4},{"category":"null","date":"2026-08-20","downloads":7},{"category":"null","date":"2026-08-21","downloads":13},{"category":"null","date":"2026-08-22","downloads":1},{"category":"null","date":"2026-08-23","downloads":6},{"category":"null","date":"2026-08-24","downloads":2},{"category":"null","date":"2026-08-25","downloads":11},{"category":"null","date":"2026-08-26","downloads":127},{"category":"null","date":"2026-08-27","downloads":74},{"category":"null","date":"2026-08-28","downloads":67},{"category":"null","date":"2026-08-29","downloads":18},{"category":"null","date":"2026-08-30","downloads":10},{"category":"null","date":"2026-08-31","downloads":222},{"category":"null","date":"2026-09-01","downloads":89},{"category":"null","date":"2026-09-02","downloads":22},{"category":"null","date":"2026-09-03","downloads":34},{"category":"null","date":"2026-09-04","downloads":20},{"category":"null","date":"2026-09-05","downloads":12},{"category":"null","date":"2026-09-06","downloads":4},{"category":"null","date":"2026-09-07","downloads":5},{"category":"null","date":"2026-09-08","downloads":44},{"category":"null","date":"2026-09-09","downloads":13},{"category":"null","date":"2026-09-10","downloads":21},{"category":"null","date":"2026-09-11","downloads":12},{"category":"null","date":"2026-09-12","downloads":15},{"category":"null","date":"2026-09-13","downloads":3},{"category":"null","date":"2026-09-14","downloads":6},{"category":"null","date":"2026-09-15","downloads":4},{"category":"null","date":"2026-09-16","downloads":31},{"category":"null","date":"2026-09-17","downloads":14},{"category":"null","date":"2026-09-18","downloads":37},{"category":"null","date":"2026-09-19","downloads":48},{"category":"null","date":"2026-09-20","downloads":13},{"category":"null","date":"2026-09-21","downloads":19},{"category":"null","date":"2026-09-22","downloads":4},{"category":"null","date":"2026-09-23","downloads":6},{"category":"null","date":"2026-09-24","downloads":2},{"category":"null","date":"2026-09-26","downloads":5},{"category":"null","date":"2026-09-27","downloads":45},{"category":"null","date":"2026-09-28","downloads":23},{"category":"null","date":"2026-09-29","downloads":4},{"category":"null","date":"2026-10-01","downloads":5},{"category":"null","date":"2026-10-02","downloads":12},{"category":"null","date":"2026-10-06","downloads":4},{"category":"null","date":"2026-10-07","downloads":10},{"category":"null","date":"2026-10-08","downloads":5},{"category":"null","date":"2026-10-09","downloads":1},{"category":"null","date":"2026-10-10","downloads":14},{"category":"null","date":"2026-10-11","downloads":4},{"category":"null","date":"2026-10-12","downloads":14},{"category":"null","date":"2026-10-13","downloads":6},{"category":"null","date":"2026-10-14","downloads":17},{"category":"null","date":"2026-10-15","downloads":29},{"category":"null","date":"2026-10-16","downloads":21},{"category":"null","date":"2026-10-17","downloads":6},{"category":"null","date":"2026-10-18","downloads":6},{"category":"null","date":"2026-10-19","downloads":7},{"category":"null","date":"2026-10-20","downloads":8},{"category":"null","date":"2026-10-21","downloads":30},{"category":"null","date":"2026-10-22","downloads":4},{"category":"null","date":"2026-10-23","downloads":20},{"category":"null","date":"2026-10-24","downloads":14},{"category":"null","date":"2026-10-25","downloads":3},{"category":"null","date":"2026-10-26","downloads":36},{"category":"null","date":"2026-10-27","downloads":5},{"category":"null","date":"2026-10-28","downloads":10}],"package":"mooncake-transfer-engine-npu","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (Ascend NPU version)"},{"name":"mooncake-transfer-engine-rocm","endpoints":{"recent":{"status":"ok","fetchedAt":"2026-10-29T07:13:08.971104+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-rocm/recent","payload":{"data":{"last_day":460,"last_month":5575,"last_week":2490},"package":"mooncake-transfer-engine-rocm","type":"recent_downloads"}},"overall":{"status":"ok","fetchedAt":"2026-10-29T07:13:11.402863+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-rocm/overall","payload":{"data":[{"category":"with_mirrors","date":"2026-08-26","downloads":287},{"category":"with_mirrors","date":"2026-08-27","downloads":67},{"category":"with_mirrors","date":"2026-08-28","downloads":45},{"category":"with_mirrors","date":"2026-08-29","downloads":62},{"category":"with_mirrors","date":"2026-08-30","downloads":5},{"category":"with_mirrors","date":"2026-08-31","downloads":296},{"category":"with_mirrors","date":"2026-09-01","downloads":98},{"category":"with_mirrors","date":"2026-09-02","downloads":41},{"category":"with_mirrors","date":"2026-09-03","downloads":205},{"category":"with_mirrors","date":"2026-09-04","downloads":158},{"category":"with_mirrors","date":"2026-09-05","downloads":109},{"category":"with_mirrors","date":"2026-09-06","downloads":73},{"category":"with_mirrors","date":"2026-09-07","downloads":94},{"category":"with_mirrors","date":"2026-09-08","downloads":172},{"category":"with_mirrors","date":"2026-09-09","downloads":297},{"category":"with_mirrors","date":"2026-09-10","downloads":166},{"category":"with_mirrors","date":"2026-09-11","downloads":216},{"category":"with_mirrors","date":"2026-09-12","downloads":87},{"category":"with_mirrors","date":"2026-09-13","downloads":87},{"category":"with_mirrors","date":"2026-09-14","downloads":170},{"category":"with_mirrors","date":"2026-09-15","downloads":152},{"category":"with_mirrors","date":"2026-09-16","downloads":187},{"category":"with_mirrors","date":"2026-09-17","downloads":227},{"category":"with_mirrors","date":"2026-09-18","downloads":205},{"category":"with_mirrors","date":"2026-09-19","downloads":88},{"category":"with_mirrors","date":"2026-09-20","downloads":109},{"category":"with_mirrors","date":"2026-09-21","downloads":176},{"category":"with_mirrors","date":"2026-09-22","downloads":236},{"category":"with_mirrors","date":"2026-09-23","downloads":225},{"category":"with_mirrors","date":"2026-09-24","downloads":236},{"category":"with_mirrors","date":"2026-09-25","downloads":210},{"category":"with_mirrors","date":"2026-09-26","downloads":174},{"category":"with_mirrors","date":"2026-09-27","downloads":169},{"category":"with_mirrors","date":"2026-09-28","downloads":241},{"category":"with_mirrors","date":"2026-09-29","downloads":335},{"category":"with_mirrors","date":"2026-09-30","downloads":296},{"category":"with_mirrors","date":"2026-10-01","downloads":259},{"category":"with_mirrors","date":"2026-10-02","downloads":454},{"category":"with_mirrors","date":"2026-10-03","downloads":172},{"category":"with_mirrors","date":"2026-10-04","downloads":160},{"category":"with_mirrors","date":"2026-10-05","downloads":243},{"category":"with_mirrors","date":"2026-10-06","downloads":236},{"category":"with_mirrors","date":"2026-10-07","downloads":255},{"category":"with_mirrors","date":"2026-10-08","downloads":280},{"category":"with_mirrors","date":"2026-10-09","downloads":286},{"category":"with_mirrors","date":"2026-10-10","downloads":211},{"category":"with_mirrors","date":"2026-10-11","downloads":163},{"category":"with_mirrors","date":"2026-10-12","downloads":260},{"category":"with_mirrors","date":"2026-10-13","downloads":329},{"category":"with_mirrors","date":"2026-10-14","downloads":365},{"category":"with_mirrors","date":"2026-10-15","downloads":394},{"category":"with_mirrors","date":"2026-10-16","downloads":303},{"category":"with_mirrors","date":"2026-10-17","downloads":219},{"category":"with_mirrors","date":"2026-10-18","downloads":295},{"category":"with_mirrors","date":"2026-10-19","downloads":425},{"category":"with_mirrors","date":"2026-10-20","downloads":414},{"category":"with_mirrors","date":"2026-10-21","downloads":368},{"category":"with_mirrors","date":"2026-10-22","downloads":489},{"category":"with_mirrors","date":"2026-10-23","downloads":427},{"category":"with_mirrors","date":"2026-10-24","downloads":199},{"category":"with_mirrors","date":"2026-10-25","downloads":208},{"category":"with_mirrors","date":"2026-10-26","downloads":436},{"category":"with_mirrors","date":"2026-10-27","downloads":534},{"category":"with_mirrors","date":"2026-10-28","downloads":468},{"category":"without_mirrors","date":"2026-08-26","downloads":161},{"category":"without_mirrors","date":"2026-08-27","downloads":12},{"category":"without_mirrors","date":"2026-08-28","downloads":11},{"category":"without_mirrors","date":"2026-08-29","downloads":47},{"category":"without_mirrors","date":"2026-08-31","downloads":117},{"category":"without_mirrors","date":"2026-09-01","downloads":11},{"category":"without_mirrors","date":"2026-09-02","downloads":34},{"category":"without_mirrors","date":"2026-09-03","downloads":176},{"category":"without_mirrors","date":"2026-09-04","downloads":145},{"category":"without_mirrors","date":"2026-09-05","downloads":99},{"category":"without_mirrors","date":"2026-09-06","downloads":63},{"category":"without_mirrors","date":"2026-09-07","downloads":82},{"category":"without_mirrors","date":"2026-09-08","downloads":142},{"category":"without_mirrors","date":"2026-09-09","downloads":272},{"category":"without_mirrors","date":"2026-09-10","downloads":140},{"category":"without_mirrors","date":"2026-09-11","downloads":206},{"category":"without_mirrors","date":"2026-09-12","downloads":81},{"category":"without_mirrors","date":"2026-09-13","downloads":79},{"category":"without_mirrors","date":"2026-09-14","downloads":164},{"category":"without_mirrors","date":"2026-09-15","downloads":144},{"category":"without_mirrors","date":"2026-09-16","downloads":178},{"category":"without_mirrors","date":"2026-09-17","downloads":211},{"category":"without_mirrors","date":"2026-09-18","downloads":204},{"category":"without_mirrors","date":"2026-09-19","downloads":84},{"category":"without_mirrors","date":"2026-09-20","downloads":84},{"category":"without_mirrors","date":"2026-09-21","downloads":171},{"category":"without_mirrors","date":"2026-09-22","downloads":216},{"category":"without_mirrors","date":"2026-09-23","downloads":219},{"category":"without_mirrors","date":"2026-09-24","downloads":225},{"category":"without_mirrors","date":"2026-09-25","downloads":207},{"category":"without_mirrors","date":"2026-09-26","downloads":166},{"category":"without_mirrors","date":"2026-09-27","downloads":160},{"category":"without_mirrors","date":"2026-09-28","downloads":230},{"category":"without_mirrors","date":"2026-09-29","downloads":316},{"category":"without_mirrors","date":"2026-09-30","downloads":296},{"category":"without_mirrors","date":"2026-10-01","downloads":246},{"category":"without_mirrors","date":"2026-10-02","downloads":430},{"category":"without_mirrors","date":"2026-10-03","downloads":172},{"category":"without_mirrors","date":"2026-10-04","downloads":147},{"category":"without_mirrors","date":"2026-10-05","downloads":240},{"category":"without_mirrors","date":"2026-10-06","downloads":231},{"category":"without_mirrors","date":"2026-10-07","downloads":235},{"category":"without_mirrors","date":"2026-10-08","downloads":269},{"category":"without_mirrors","date":"2026-10-09","downloads":279},{"category":"without_mirrors","date":"2026-10-10","downloads":208},{"category":"without_mirrors","date":"2026-10-11","downloads":154},{"category":"without_mirrors","date":"2026-10-12","downloads":253},{"category":"without_mirrors","date":"2026-10-13","downloads":312},{"category":"without_mirrors","date":"2026-10-14","downloads":362},{"category":"without_mirrors","date":"2026-10-15","downloads":378},{"category":"without_mirrors","date":"2026-10-16","downloads":295},{"category":"without_mirrors","date":"2026-10-17","downloads":207},{"category":"without_mirrors","date":"2026-10-18","downloads":266},{"category":"without_mirrors","date":"2026-10-19","downloads":420},{"category":"without_mirrors","date":"2026-10-20","downloads":406},{"category":"without_mirrors","date":"2026-10-21","downloads":356},{"category":"without_mirrors","date":"2026-10-22","downloads":468},{"category":"without_mirrors","date":"2026-10-23","downloads":426},{"category":"without_mirrors","date":"2026-10-24","downloads":199},{"category":"without_mirrors","date":"2026-10-25","downloads":200},{"category":"without_mirrors","date":"2026-10-26","downloads":435},{"category":"without_mirrors","date":"2026-10-27","downloads":511},{"category":"without_mirrors","date":"2026-10-28","downloads":460}],"package":"mooncake-transfer-engine-rocm","type":"overall_downloads"}},"python_major":{"status":"ok","fetchedAt":"2026-10-29T07:13:13.805715+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-rocm/python_major","payload":{"data":[{"category":"2","date":"2026-10-28","downloads":1},{"category":"3","date":"2026-08-26","downloads":4},{"category":"3","date":"2026-08-27","downloads":8},{"category":"3","date":"2026-08-29","downloads":27},{"category":"3","date":"2026-08-31","downloads":23},{"category":"3","date":"2026-09-01","downloads":5},{"category":"3","date":"2026-09-02","downloads":18},{"category":"3","date":"2026-09-03","downloads":155},{"category":"3","date":"2026-09-04","downloads":138},{"category":"3","date":"2026-09-05","downloads":94},{"category":"3","date":"2026-09-06","downloads":62},{"category":"3","date":"2026-09-07","downloads":82},{"category":"3","date":"2026-09-08","downloads":130},{"category":"3","date":"2026-09-09","downloads":272},{"category":"3","date":"2026-09-10","downloads":131},{"category":"3","date":"2026-09-11","downloads":204},{"category":"3","date":"2026-09-12","downloads":80},{"category":"3","date":"2026-09-13","downloads":79},{"category":"3","date":"2026-09-14","downloads":157},{"category":"3","date":"2026-09-15","downloads":136},{"category":"3","date":"2026-09-16","downloads":171},{"category":"3","date":"2026-09-17","downloads":210},{"category":"3","date":"2026-09-18","downloads":203},{"category":"3","date":"2026-09-19","downloads":76},{"category":"3","date":"2026-09-20","downloads":74},{"category":"3","date":"2026-09-21","downloads":167},{"category":"3","date":"2026-09-22","downloads":216},{"category":"3","date":"2026-09-23","downloads":216},{"category":"3","date":"2026-09-24","downloads":224},{"category":"3","date":"2026-09-25","downloads":205},{"category":"3","date":"2026-09-26","downloads":166},{"category":"3","date":"2026-09-27","downloads":152},{"category":"3","date":"2026-09-28","downloads":228},{"category":"3","date":"2026-09-29","downloads":316},{"category":"3","date":"2026-09-30","downloads":296},{"category":"3","date":"2026-10-01","downloads":245},{"category":"3","date":"2026-10-02","downloads":426},{"category":"3","date":"2026-10-03","downloads":171},{"category":"3","date":"2026-10-04","downloads":145},{"category":"3","date":"2026-10-05","downloads":237},{"category":"3","date":"2026-10-06","downloads":230},{"category":"3","date":"2026-10-07","downloads":233},{"category":"3","date":"2026-10-08","downloads":269},{"category":"3","date":"2026-10-09","downloads":277},{"category":"3","date":"2026-10-10","downloads":200},{"category":"3","date":"2026-10-11","downloads":148},{"category":"3","date":"2026-10-12","downloads":233},{"category":"3","date":"2026-10-13","downloads":312},{"category":"3","date":"2026-10-14","downloads":357},{"category":"3","date":"2026-10-15","downloads":360},{"category":"3","date":"2026-10-16","downloads":295},{"category":"3","date":"2026-10-17","downloads":204},{"category":"3","date":"2026-10-18","downloads":256},{"category":"3","date":"2026-10-19","downloads":414},{"category":"3","date":"2026-10-20","downloads":402},{"category":"3","date":"2026-10-21","downloads":354},{"category":"3","date":"2026-10-22","downloads":468},{"category":"3","date":"2026-10-23","downloads":423},{"category":"3","date":"2026-10-24","downloads":197},{"category":"3","date":"2026-10-25","downloads":199},{"category":"3","date":"2026-10-26","downloads":435},{"category":"3","date":"2026-10-27","downloads":510},{"category":"3","date":"2026-10-28","downloads":459},{"category":"null","date":"2026-08-26","downloads":157},{"category":"null","date":"2026-08-27","downloads":4},{"category":"null","date":"2026-08-28","downloads":11},{"category":"null","date":"2026-08-29","downloads":20},{"category":"null","date":"2026-08-31","downloads":94},{"category":"null","date":"2026-09-01","downloads":6},{"category":"null","date":"2026-09-02","downloads":16},{"category":"null","date":"2026-09-03","downloads":21},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":5},{"category":"null","date":"2026-09-06","downloads":1},{"category":"null","date":"2026-09-08","downloads":12},{"category":"null","date":"2026-09-10","downloads":9},{"category":"null","date":"2026-09-11","downloads":2},{"category":"null","date":"2026-09-12","downloads":1},{"category":"null","date":"2026-09-14","downloads":7},{"category":"null","date":"2026-09-15","downloads":8},{"category":"null","date":"2026-09-16","downloads":7},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-18","downloads":1},{"category":"null","date":"2026-09-19","downloads":8},{"category":"null","date":"2026-09-20","downloads":10},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-23","downloads":3},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":2},{"category":"null","date":"2026-09-27","downloads":8},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":1},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":2},{"category":"null","date":"2026-10-09","downloads":2},{"category":"null","date":"2026-10-10","downloads":8},{"category":"null","date":"2026-10-11","downloads":6},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-14","downloads":5},{"category":"null","date":"2026-10-15","downloads":18},{"category":"null","date":"2026-10-17","downloads":3},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":4},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-23","downloads":3},{"category":"null","date":"2026-10-24","downloads":2},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-27","downloads":1}],"package":"mooncake-transfer-engine-rocm","type":"python_major_downloads"}},"python_minor":{"status":"ok","fetchedAt":"2026-10-29T07:13:16.218642+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-rocm/python_minor","payload":{"data":[{"category":"2.7","date":"2026-10-28","downloads":1},{"category":"3.10","date":"2026-08-26","downloads":1},{"category":"3.10","date":"2026-09-02","downloads":3},{"category":"3.10","date":"2026-09-03","downloads":3},{"category":"3.10","date":"2026-09-10","downloads":1},{"category":"3.10","date":"2026-09-14","downloads":3},{"category":"3.10","date":"2026-09-16","downloads":7},{"category":"3.10","date":"2026-09-17","downloads":9},{"category":"3.10","date":"2026-09-18","downloads":62},{"category":"3.10","date":"2026-09-19","downloads":3},{"category":"3.10","date":"2026-09-21","downloads":6},{"category":"3.10","date":"2026-09-23","downloads":4},{"category":"3.10","date":"2026-09-24","downloads":14},{"category":"3.10","date":"2026-09-25","downloads":6},{"category":"3.10","date":"2026-09-27","downloads":19},{"category":"3.10","date":"2026-09-29","downloads":2},{"category":"3.10","date":"2026-09-30","downloads":1},{"category":"3.10","date":"2026-10-05","downloads":2},{"category":"3.10","date":"2026-10-06","downloads":1},{"category":"3.10","date":"2026-10-09","downloads":1},{"category":"3.10","date":"2026-10-16","downloads":5},{"category":"3.10","date":"2026-10-20","downloads":1},{"category":"3.10","date":"2026-10-23","downloads":2},{"category":"3.10","date":"2026-10-27","downloads":1},{"category":"3.11","date":"2026-08-27","downloads":8},{"category":"3.11","date":"2026-08-31","downloads":20},{"category":"3.11","date":"2026-09-05","downloads":1},{"category":"3.11","date":"2026-09-06","downloads":3},{"category":"3.11","date":"2026-09-11","downloads":1},{"category":"3.11","date":"2026-09-17","downloads":2},{"category":"3.11","date":"2026-09-19","downloads":1},{"category":"3.11","date":"2026-09-22","downloads":1},{"category":"3.11","date":"2026-09-25","downloads":1},{"category":"3.11","date":"2026-09-26","downloads":4},{"category":"3.11","date":"2026-09-28","downloads":1},{"category":"3.11","date":"2026-09-29","downloads":1},{"category":"3.11","date":"2026-09-30","downloads":1},{"category":"3.11","date":"2026-10-01","downloads":1},{"category":"3.11","date":"2026-10-02","downloads":1},{"category":"3.11","date":"2026-10-08","downloads":1},{"category":"3.11","date":"2026-10-11","downloads":1},{"category":"3.11","date":"2026-10-15","downloads":8},{"category":"3.11","date":"2026-10-17","downloads":1},{"category":"3.11","date":"2026-10-20","downloads":7},{"category":"3.11","date":"2026-10-21","downloads":11},{"category":"3.11","date":"2026-10-25","downloads":1},{"category":"3.12","date":"2026-08-26","downloads":3},{"category":"3.12","date":"2026-08-29","downloads":27},{"category":"3.12","date":"2026-08-31","downloads":3},{"category":"3.12","date":"2026-09-01","downloads":5},{"category":"3.12","date":"2026-09-02","downloads":15},{"category":"3.12","date":"2026-09-03","downloads":151},{"category":"3.12","date":"2026-09-04","downloads":126},{"category":"3.12","date":"2026-09-05","downloads":88},{"category":"3.12","date":"2026-09-06","downloads":54},{"category":"3.12","date":"2026-09-07","downloads":78},{"category":"3.12","date":"2026-09-08","downloads":123},{"category":"3.12","date":"2026-09-09","downloads":266},{"category":"3.12","date":"2026-09-10","downloads":127},{"category":"3.12","date":"2026-09-11","downloads":190},{"category":"3.12","date":"2026-09-12","downloads":66},{"category":"3.12","date":"2026-09-13","downloads":69},{"category":"3.12","date":"2026-09-14","downloads":130},{"category":"3.12","date":"2026-09-15","downloads":127},{"category":"3.12","date":"2026-09-16","downloads":157},{"category":"3.12","date":"2026-09-17","downloads":197},{"category":"3.12","date":"2026-09-18","downloads":137},{"category":"3.12","date":"2026-09-19","downloads":70},{"category":"3.12","date":"2026-09-20","downloads":73},{"category":"3.12","date":"2026-09-21","downloads":156},{"category":"3.12","date":"2026-09-22","downloads":210},{"category":"3.12","date":"2026-09-23","downloads":205},{"category":"3.12","date":"2026-09-24","downloads":203},{"category":"3.12","date":"2026-09-25","downloads":195},{"category":"3.12","date":"2026-09-26","downloads":154},{"category":"3.12","date":"2026-09-27","downloads":128},{"category":"3.12","date":"2026-09-28","downloads":215},{"category":"3.12","date":"2026-09-29","downloads":306},{"category":"3.12","date":"2026-09-30","downloads":289},{"category":"3.12","date":"2026-10-01","downloads":239},{"category":"3.12","date":"2026-10-02","downloads":419},{"category":"3.12","date":"2026-10-03","downloads":161},{"category":"3.12","date":"2026-10-04","downloads":135},{"category":"3.12","date":"2026-10-05","downloads":228},{"category":"3.12","date":"2026-10-06","downloads":225},{"category":"3.12","date":"2026-10-07","downloads":229},{"category":"3.12","date":"2026-10-08","downloads":264},{"category":"3.12","date":"2026-10-09","downloads":270},{"category":"3.12","date":"2026-10-10","downloads":190},{"category":"3.12","date":"2026-10-11","downloads":145},{"category":"3.12","date":"2026-10-12","downloads":229},{"category":"3.12","date":"2026-10-13","downloads":305},{"category":"3.12","date":"2026-10-14","downloads":346},{"category":"3.12","date":"2026-10-15","downloads":349},{"category":"3.12","date":"2026-10-16","downloads":282},{"category":"3.12","date":"2026-10-17","downloads":202},{"category":"3.12","date":"2026-10-18","downloads":255},{"category":"3.12","date":"2026-10-19","downloads":407},{"category":"3.12","date":"2026-10-20","downloads":390},{"category":"3.12","date":"2026-10-21","downloads":339},{"category":"3.12","date":"2026-10-22","downloads":456},{"category":"3.12","date":"2026-10-23","downloads":417},{"category":"3.12","date":"2026-10-24","downloads":195},{"category":"3.12","date":"2026-10-25","downloads":196},{"category":"3.12","date":"2026-10-26","downloads":429},{"category":"3.12","date":"2026-10-27","downloads":504},{"category":"3.12","date":"2026-10-28","downloads":456},{"category":"3.13","date":"2026-09-03","downloads":1},{"category":"3.13","date":"2026-09-04","downloads":12},{"category":"3.13","date":"2026-09-05","downloads":5},{"category":"3.13","date":"2026-09-06","downloads":5},{"category":"3.13","date":"2026-09-07","downloads":4},{"category":"3.13","date":"2026-09-08","downloads":7},{"category":"3.13","date":"2026-09-09","downloads":6},{"category":"3.13","date":"2026-09-10","downloads":3},{"category":"3.13","date":"2026-09-11","downloads":13},{"category":"3.13","date":"2026-09-12","downloads":14},{"category":"3.13","date":"2026-09-13","downloads":10},{"category":"3.13","date":"2026-09-14","downloads":24},{"category":"3.13","date":"2026-09-15","downloads":9},{"category":"3.13","date":"2026-09-16","downloads":7},{"category":"3.13","date":"2026-09-17","downloads":2},{"category":"3.13","date":"2026-09-18","downloads":4},{"category":"3.13","date":"2026-09-19","downloads":2},{"category":"3.13","date":"2026-09-20","downloads":1},{"category":"3.13","date":"2026-09-21","downloads":5},{"category":"3.13","date":"2026-09-22","downloads":4},{"category":"3.13","date":"2026-09-23","downloads":7},{"category":"3.13","date":"2026-09-24","downloads":7},{"category":"3.13","date":"2026-09-25","downloads":3},{"category":"3.13","date":"2026-09-26","downloads":8},{"category":"3.13","date":"2026-09-27","downloads":5},{"category":"3.13","date":"2026-09-28","downloads":12},{"category":"3.13","date":"2026-09-29","downloads":7},{"category":"3.13","date":"2026-09-30","downloads":5},{"category":"3.13","date":"2026-10-01","downloads":5},{"category":"3.13","date":"2026-10-02","downloads":6},{"category":"3.13","date":"2026-10-03","downloads":10},{"category":"3.13","date":"2026-10-04","downloads":10},{"category":"3.13","date":"2026-10-05","downloads":7},{"category":"3.13","date":"2026-10-06","downloads":4},{"category":"3.13","date":"2026-10-07","downloads":4},{"category":"3.13","date":"2026-10-08","downloads":4},{"category":"3.13","date":"2026-10-09","downloads":6},{"category":"3.13","date":"2026-10-10","downloads":10},{"category":"3.13","date":"2026-10-11","downloads":2},{"category":"3.13","date":"2026-10-12","downloads":4},{"category":"3.13","date":"2026-10-13","downloads":7},{"category":"3.13","date":"2026-10-14","downloads":11},{"category":"3.13","date":"2026-10-15","downloads":3},{"category":"3.13","date":"2026-10-16","downloads":8},{"category":"3.13","date":"2026-10-17","downloads":1},{"category":"3.13","date":"2026-10-18","downloads":1},{"category":"3.13","date":"2026-10-19","downloads":7},{"category":"3.13","date":"2026-10-20","downloads":4},{"category":"3.13","date":"2026-10-21","downloads":4},{"category":"3.13","date":"2026-10-22","downloads":12},{"category":"3.13","date":"2026-10-23","downloads":4},{"category":"3.13","date":"2026-10-24","downloads":2},{"category":"3.13","date":"2026-10-25","downloads":2},{"category":"3.13","date":"2026-10-26","downloads":6},{"category":"3.13","date":"2026-10-27","downloads":5},{"category":"3.13","date":"2026-10-28","downloads":3},{"category":"3.14","date":"2026-09-22","downloads":1},{"category":"null","date":"2026-08-26","downloads":157},{"category":"null","date":"2026-08-27","downloads":4},{"category":"null","date":"2026-08-28","downloads":11},{"category":"null","date":"2026-08-29","downloads":20},{"category":"null","date":"2026-08-31","downloads":94},{"category":"null","date":"2026-09-01","downloads":6},{"category":"null","date":"2026-09-02","downloads":16},{"category":"null","date":"2026-09-03","downloads":21},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":5},{"category":"null","date":"2026-09-06","downloads":1},{"category":"null","date":"2026-09-08","downloads":12},{"category":"null","date":"2026-09-10","downloads":9},{"category":"null","date":"2026-09-11","downloads":2},{"category":"null","date":"2026-09-12","downloads":1},{"category":"null","date":"2026-09-14","downloads":7},{"category":"null","date":"2026-09-15","downloads":8},{"category":"null","date":"2026-09-16","downloads":7},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-18","downloads":1},{"category":"null","date":"2026-09-19","downloads":8},{"category":"null","date":"2026-09-20","downloads":10},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-23","downloads":3},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":2},{"category":"null","date":"2026-09-27","downloads":8},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":1},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":2},{"category":"null","date":"2026-10-09","downloads":2},{"category":"null","date":"2026-10-10","downloads":8},{"category":"null","date":"2026-10-11","downloads":6},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-14","downloads":5},{"category":"null","date":"2026-10-15","downloads":18},{"category":"null","date":"2026-10-17","downloads":3},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":4},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-23","downloads":3},{"category":"null","date":"2026-10-24","downloads":2},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-27","downloads":1}],"package":"mooncake-transfer-engine-rocm","type":"python_minor_downloads"}},"system":{"status":"ok","fetchedAt":"2026-10-29T07:13:18.598674+00:00","source":"https://pypistats.org/api/packages/mooncake-transfer-engine-rocm/system","payload":{"data":[{"category":"Darwin","date":"2026-08-27","downloads":8},{"category":"Darwin","date":"2026-08-31","downloads":20},{"category":"Darwin","date":"2026-09-24","downloads":1},{"category":"Darwin","date":"2026-10-28","downloads":1},{"category":"Linux","date":"2026-08-26","downloads":4},{"category":"Linux","date":"2026-08-29","downloads":27},{"category":"Linux","date":"2026-08-31","downloads":3},{"category":"Linux","date":"2026-09-01","downloads":5},{"category":"Linux","date":"2026-09-02","downloads":18},{"category":"Linux","date":"2026-09-03","downloads":155},{"category":"Linux","date":"2026-09-04","downloads":138},{"category":"Linux","date":"2026-09-05","downloads":94},{"category":"Linux","date":"2026-09-06","downloads":62},{"category":"Linux","date":"2026-09-07","downloads":82},{"category":"Linux","date":"2026-09-08","downloads":130},{"category":"Linux","date":"2026-09-09","downloads":272},{"category":"Linux","date":"2026-09-10","downloads":131},{"category":"Linux","date":"2026-09-11","downloads":204},{"category":"Linux","date":"2026-09-12","downloads":80},{"category":"Linux","date":"2026-09-13","downloads":79},{"category":"Linux","date":"2026-09-14","downloads":157},{"category":"Linux","date":"2026-09-15","downloads":136},{"category":"Linux","date":"2026-09-16","downloads":171},{"category":"Linux","date":"2026-09-17","downloads":210},{"category":"Linux","date":"2026-09-18","downloads":203},{"category":"Linux","date":"2026-09-19","downloads":76},{"category":"Linux","date":"2026-09-20","downloads":74},{"category":"Linux","date":"2026-09-21","downloads":167},{"category":"Linux","date":"2026-09-22","downloads":215},{"category":"Linux","date":"2026-09-23","downloads":216},{"category":"Linux","date":"2026-09-24","downloads":222},{"category":"Linux","date":"2026-09-25","downloads":205},{"category":"Linux","date":"2026-09-26","downloads":166},{"category":"Linux","date":"2026-09-27","downloads":152},{"category":"Linux","date":"2026-09-28","downloads":228},{"category":"Linux","date":"2026-09-29","downloads":316},{"category":"Linux","date":"2026-09-30","downloads":296},{"category":"Linux","date":"2026-10-01","downloads":245},{"category":"Linux","date":"2026-10-02","downloads":426},{"category":"Linux","date":"2026-10-03","downloads":171},{"category":"Linux","date":"2026-10-04","downloads":144},{"category":"Linux","date":"2026-10-05","downloads":237},{"category":"Linux","date":"2026-10-06","downloads":230},{"category":"Linux","date":"2026-10-07","downloads":233},{"category":"Linux","date":"2026-10-08","downloads":269},{"category":"Linux","date":"2026-10-09","downloads":277},{"category":"Linux","date":"2026-10-10","downloads":200},{"category":"Linux","date":"2026-10-11","downloads":148},{"category":"Linux","date":"2026-10-12","downloads":233},{"category":"Linux","date":"2026-10-13","downloads":312},{"category":"Linux","date":"2026-10-14","downloads":357},{"category":"Linux","date":"2026-10-15","downloads":360},{"category":"Linux","date":"2026-10-16","downloads":295},{"category":"Linux","date":"2026-10-17","downloads":204},{"category":"Linux","date":"2026-10-18","downloads":256},{"category":"Linux","date":"2026-10-19","downloads":414},{"category":"Linux","date":"2026-10-20","downloads":402},{"category":"Linux","date":"2026-10-21","downloads":354},{"category":"Linux","date":"2026-10-22","downloads":468},{"category":"Linux","date":"2026-10-23","downloads":423},{"category":"Linux","date":"2026-10-24","downloads":197},{"category":"Linux","date":"2026-10-25","downloads":199},{"category":"Linux","date":"2026-10-26","downloads":435},{"category":"Linux","date":"2026-10-27","downloads":509},{"category":"Linux","date":"2026-10-28","downloads":459},{"category":"Windows","date":"2026-09-22","downloads":1},{"category":"Windows","date":"2026-09-24","downloads":1},{"category":"Windows","date":"2026-10-04","downloads":1},{"category":"Windows","date":"2026-10-27","downloads":1},{"category":"null","date":"2026-08-26","downloads":157},{"category":"null","date":"2026-08-27","downloads":4},{"category":"null","date":"2026-08-28","downloads":11},{"category":"null","date":"2026-08-29","downloads":20},{"category":"null","date":"2026-08-31","downloads":94},{"category":"null","date":"2026-09-01","downloads":6},{"category":"null","date":"2026-09-02","downloads":16},{"category":"null","date":"2026-09-03","downloads":21},{"category":"null","date":"2026-09-04","downloads":7},{"category":"null","date":"2026-09-05","downloads":5},{"category":"null","date":"2026-09-06","downloads":1},{"category":"null","date":"2026-09-08","downloads":12},{"category":"null","date":"2026-09-10","downloads":9},{"category":"null","date":"2026-09-11","downloads":2},{"category":"null","date":"2026-09-12","downloads":1},{"category":"null","date":"2026-09-14","downloads":7},{"category":"null","date":"2026-09-15","downloads":8},{"category":"null","date":"2026-09-16","downloads":7},{"category":"null","date":"2026-09-17","downloads":1},{"category":"null","date":"2026-09-18","downloads":1},{"category":"null","date":"2026-09-19","downloads":8},{"category":"null","date":"2026-09-20","downloads":10},{"category":"null","date":"2026-09-21","downloads":4},{"category":"null","date":"2026-09-23","downloads":3},{"category":"null","date":"2026-09-24","downloads":1},{"category":"null","date":"2026-09-25","downloads":2},{"category":"null","date":"2026-09-27","downloads":8},{"category":"null","date":"2026-09-28","downloads":2},{"category":"null","date":"2026-10-01","downloads":1},{"category":"null","date":"2026-10-02","downloads":4},{"category":"null","date":"2026-10-03","downloads":1},{"category":"null","date":"2026-10-04","downloads":2},{"category":"null","date":"2026-10-05","downloads":3},{"category":"null","date":"2026-10-06","downloads":1},{"category":"null","date":"2026-10-07","downloads":2},{"category":"null","date":"2026-10-09","downloads":2},{"category":"null","date":"2026-10-10","downloads":8},{"category":"null","date":"2026-10-11","downloads":6},{"category":"null","date":"2026-10-12","downloads":20},{"category":"null","date":"2026-10-14","downloads":5},{"category":"null","date":"2026-10-15","downloads":18},{"category":"null","date":"2026-10-17","downloads":3},{"category":"null","date":"2026-10-18","downloads":10},{"category":"null","date":"2026-10-19","downloads":6},{"category":"null","date":"2026-10-20","downloads":4},{"category":"null","date":"2026-10-21","downloads":2},{"category":"null","date":"2026-10-23","downloads":3},{"category":"null","date":"2026-10-24","downloads":2},{"category":"null","date":"2026-10-25","downloads":1},{"category":"null","date":"2026-10-27","downloads":1}],"package":"mooncake-transfer-engine-rocm","type":"system_downloads"}}},"version":"0.3.13.post1","summary":"A KVCache-centric Disaggregated Architecture for large-scale LLM inference and training. (AMD ROCm version)"}]} diff --git a/docs/source/api-reference/python/ep-backend.md b/docs/source/api-reference/python/ep-backend.md index 5bd5c61d9f..200e4684ab 100644 --- a/docs/source/api-reference/python/ep-backend.md +++ b/docs/source/api-reference/python/ep-backend.md @@ -158,7 +158,7 @@ Arguments: | Function | Purpose | Notes | | --- | --- | --- | -| `pg.set_host_ip(host_ip)` | Override the host IP used by the backend. | Call before `init_process_group()`. | +| `pg.set_host_ip(host_ip)` | Override the host IP used by the backend. | Call before `init_process_group()`. Single-node (`MASTER_ADDR` unset or loopback, including the whole `127.0.0.0/8` range such as Debian/Ubuntu `127.0.1.1`) keeps the default `127.0.0.1`. On multi-node jobs (`MASTER_ADDR` is a non-loopback host or hostname) Mooncake PG derives a routable address toward `MASTER_ADDR` (hostname fallback only when that result is also non-loopback); if derivation fails, call `set_host_ip()` or `set_transfer_engine(...)` or initialization fails fast instead of hanging on rank 0. | | `pg.set_device_filter(filters)` | Restrict NIC/HCA selection. | Call before `init_process_group()`. | | `pg.set_transfer_engine(engine)` | Reuse an external `TransferEngine`. | The engine must outlive all process groups. | | `pg.get_active_ranks(backend)` | Return the backend active-rank tensor. | Used by EP fallback and recovery paths. | diff --git a/docs/source/community/pypi-downloads.md b/docs/source/community/pypi-downloads.md new file mode 100644 index 0000000000..89c2012a7c --- /dev/null +++ b/docs/source/community/pypi-downloads.md @@ -0,0 +1,28 @@ +# PyPI downloads + +Download statistics for `mooncake-transfer-engine` and its platform variants. + +:::{raw} html +
+

Loading download statistics…

+ +
+::: + +:::{dropdown} Data notes + +- The snapshot is refreshed during documentation deployment, at most once per day + for each endpoint. Collection times and missing data are shown above. +- **Day / week / month** preserve the `recent` API values and exclude mirrors. + They do not follow the date filter and can differ from sums of daily records. + **Selected range** is calculated from the daily `overall` series. +- **All dates** means available history, not lifetime downloads. PyPIStats + documents roughly 180 days of retention; all dates actually returned are kept. + Dates and timestamps use UTC. +- Missing values are shown as **—**, not zero. Totals include returned records + only. Downloads include repeated installations and CI activity, not unique users. +- Python and operating system statistics always exclude mirrors. They are + separate breakdowns of the same downloads and must not be added together. +- The package list is discovered from the [PyPI index](https://pypi.org/simple/). + TestPyPI is excluded. Source: [PyPIStats API](https://pypistats.org/api/). +::: diff --git a/docs/source/conf.py b/docs/source/conf.py index dfb0ae5df7..250347ae8e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -319,5 +319,12 @@ def add_pr_dashboard_assets(app, pagename, templatename, context, doctree): app.add_js_file("pr-dashboard.js", loading_method="defer") +def add_pypi_downloads_assets(app, pagename, templatename, context, doctree): + if pagename == "community/pypi-downloads": + app.add_css_file("pypi-downloads.css") + app.add_js_file("pypi-downloads.js", loading_method="defer") + + def setup(app): app.connect("html-page-context", add_pr_dashboard_assets) + app.connect("html-page-context", add_pypi_downloads_assets) diff --git a/docs/source/design/tent/overview.md b/docs/source/design/tent/overview.md index 6b37df87e7..30c5e1fdf9 100644 --- a/docs/source/design/tent/overview.md +++ b/docs/source/design/tent/overview.md @@ -139,4 +139,5 @@ failover :maxdepth: 1 testing +ub-native-validation ::: diff --git a/docs/source/design/tent/ub-native-validation.md b/docs/source/design/tent/ub-native-validation.md new file mode 100644 index 0000000000..c7294823b8 --- /dev/null +++ b/docs/source/design/tent/ub-native-validation.md @@ -0,0 +1,322 @@ +# TENT Native UB Hardware Validation Guide + +This guide validates the TENT-native UB transport on machines with a real +UMDK/URMA provider. It covers build linkage, two-node READ/WRITE correctness, +multi-rail scheduling, fault recovery, retryable shutdown, and a performance +baseline. The repository's CI uses an injected adapter; the hardware cases in +this guide are therefore required before declaring native UB production-ready. + +## 1. Record the Test Environment + +Run every case on the same Mooncake commit and record the following on both +hosts: + +```bash +git rev-parse HEAD +uname -a +gcc --version | head -1 +cmake --version | head -1 +urma_admin -l +ldconfig -p | grep liburma +``` + +The two hosts must have mutually reachable EIDs and TCP reachability for the +TENT P2P control plane. Use the vendor-supported tool to verify each active UB +port, for example: + +```bash +urma_admin -p urma0 +``` + +If UMDK is installed outside the system search paths, export its library path +before configuring and running Mooncake: + +```bash +export LD_LIBRARY_PATH=/path/to/umdk/lib:${LD_LIBRARY_PATH} +``` + +Unset `MC_TENT_CONF` for the commands below unless the referenced file +explicitly enables `transports/ub`. The environment config replaces, rather +than merges with, the configuration assembled by `tebench`. + +```bash +unset MC_TENT_CONF +``` + +## 2. Build and Verify Real URMA Linkage + +Configure a Debug build first. CMake discovers standard UMDK installations +automatically; pass `URMA_SYSTEM_INCLUDE_DIR` and `URMA_LIBRARY` only for a +non-standard installation. + +```bash +cmake -S . -B build-ub-debug \ + -DUSE_TENT=ON \ + -DUSE_UB=ON \ + -DUSE_CUDA=OFF \ + -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BENCHMARK=ON \ + -DCMAKE_BUILD_TYPE=Debug + +cmake --build build-ub-debug -j"$(nproc)" --target \ + tent_ub_core_test \ + tent_ub_teardown_test \ + tent_ub_native_data_path_test \ + tent_transport_selector_test \ + tebench +``` + +Confirm that CMake found a real library and that `tebench` has a runtime +dependency on it: + +```bash +grep '^URMA_LIBRARY:' build-ub-debug/CMakeCache.txt +ldd build-ub-debug/mooncake-transfer-engine/benchmark/tebench | grep urma +``` + +Both commands must show the intended UMDK installation. A header-only fallback +is sufficient for mock tests but **not** for hardware validation. + +Run the deterministic mock suite before touching the devices: + +```bash +ctest --test-dir build-ub-debug --output-on-failure \ + -R 'tent_(ub_core|ub_teardown|ub_native_data_path|transport_selector)_test' +``` + +Expected result: all selected tests pass. In particular, the teardown test +must demonstrate that a failed unregister/delete retains ownership and that a +second cleanup call releases the same handle. It also verifies that a failed +device unpublishes every old endpoint and cannot recover until every JFC has +reported healthy in the current failure epoch. + +## 3. Two-Node Correctness + +Use `HOST_A` as the target and `HOST_B` as the initiator. The examples use a +fixed control-plane port so peer-restart tests can reuse the same segment name. + +On `HOST_A`: + +```bash +build-ub-debug/mooncake-transfer-engine/benchmark/tebench \ + --backend=tent \ + --metadata_type=p2p \ + --rpc_server_port=18001 \ + --xport_type=ub \ + --seg_type=DRAM \ + --total_buffer_size=1073741824 2>&1 | tee target-ub.log +``` + +Copy the printed segment name. On `HOST_B`, run a consistency-checked mixed +transfer first: + +```bash +TARGET_SEG='' + +build-ub-debug/mooncake-transfer-engine/benchmark/tebench \ + --backend=tent \ + --metadata_type=p2p \ + --target_seg_name="${TARGET_SEG}" \ + --xport_type=ub \ + --tent_transport_hint=ub \ + --seg_type=DRAM \ + --op_type=mix \ + --check_consistency=true \ + --total_buffer_size=1073741824 \ + --start_block_size=4096 \ + --max_block_size=1048576 \ + --start_batch_size=1 \ + --max_batch_size=8 \ + --start_num_threads=1 \ + --max_num_threads=4 \ + --duration=10 2>&1 | tee initiator-mix.log +``` + +Then repeat with `--op_type=read` and `--op_type=write`. All rows must finish, +consistency checking must report no mismatch, and neither host may log a +fallback to a non-UB transport. This covers both upward-facing TENT APIs and +native URMA READ/WRITE without converting requests to the Classic TE path. + +## 4. Multi-Device and Pressure Scheduling + +Leave `transports/ub/device_filter` empty so every discovered active EID is +eligible. Confirm that the startup topology lists at least two `ub::eid` +entries on each host, then run a fixed high-concurrency case: + +```bash +build-ub-debug/mooncake-transfer-engine/benchmark/tebench \ + --backend=tent \ + --metadata_type=p2p \ + --target_seg_name="${TARGET_SEG}" \ + --xport_type=ub \ + --tent_transport_hint=ub \ + --seg_type=DRAM \ + --op_type=write \ + --total_buffer_size=1073741824 \ + --start_block_size=65536 \ + --max_block_size=65536 \ + --start_batch_size=16 \ + --max_batch_size=16 \ + --start_num_threads=16 \ + --max_num_threads=16 \ + --duration=60 2>&1 | tee initiator-multirail.log +``` + +Monitor per-device traffic with the platform's supported counters. Every +healthy eligible rail should make progress; saturating one path must not stop +posting on another path with free quota. Record aggregate bandwidth and the +per-device byte deltas. + +## 5. Fault and Recovery Cases + +Run these cases with a long-lived embedding harness that keeps the initiator's +TENT engine alive across each fault. Preserve both host logs, note exact fault +timestamps, and record endpoint generations before and after recovery. A new +`tebench` process may be used for basic reconnect smoke testing, but it is not +sufficient evidence for generation isolation. + +### Peer restart and endpoint rebuild + +1. Stop `HOST_A` with `SIGINT` while transfers are active. +2. Start it again with the same `--rpc_server_port=18001` command. +3. In the still-running `HOST_B` harness, discover the republished segment and + submit READ and WRITE again without reinstalling the local TENT transport. + +The old endpoint generation must not be reused. The live engine must establish +a new generation and complete new transfers; stale completions must not change +a new task. Save both generation values as part of the PASS evidence. + +### Remote segment republish + +Using the embedding application's TENT API, unregister and re-register the +target buffer at the same virtual address while the engines remain alive. +Publish the returned `BufferDesc`, then submit another READ and WRITE. The new +metadata generation must be imported and the stale imported segment must be +released. Do not use `tebench` for this case because it registers its buffers +only once at process startup. + +### UB port down/up + +1. Disable one UB port using the deployment's vendor-approved administration + procedure; do not power-cycle a host as the first test. +2. Confirm that traffic continues on another healthy rail. +3. Re-enable the port and wait at least the configured endpoint cooldown. +4. Confirm that the recovered rail resumes traffic and that a new endpoint is + built before posting. + +The current recovery loop first unpublishes every endpoint backed by the failed +device. It reactivates the context only after every provider JFC has polled +successfully in the latest failure epoch, the cooldown has elapsed, and all old +WRs have drained. This supports a transient port/link failure when the provider +keeps Context/JFC handles valid. A destructive device reset that invalidates +those handles still requires transport reinstall and must be reported as +unsupported for in-place recovery. + +### Cancellation and timeout + +Exercise cancellation through the embedding application's TENT API while a +large multi-slice transfer is active. For timeout testing, drop the UB path +without stopping the control-plane TCP connection. The transfer must either +complete on another rail or reach a terminal timeout/failure; it must not hang +and shutdown must still drain safely. + +### Transport fallback + +The `tebench --xport_type=ub` commands above deliberately disable every other +transport, so they validate UB path failover but **cannot** validate transport +fallback. Use an embedding test harness with both UB and TCP (or RDMA) enabled, +and a selector policy that prefers UB but permits the second transport. Submit +the request with `transport_hint=UNSPEC`; do not pin it to UB. + +1. Verify the first consistency-checked READ and WRITE select UB. +2. Disable every usable UB path while requests remain active. +3. Verify the runtime reaches a terminal result for the failed UB attempt and + resubmits through TCP/RDMA rather than leaving the task pending. +4. Restore UB, wait for its cooldown, and verify later policy-selected traffic + can use UB again. + +Capture selector/failover logs or transport metrics that identify both the +failed UB attempt and the replacement transport. A successful copy without +this evidence does not prove transport fallback. + +## 6. Repeated Shutdown and Sanitizers + +Configure a separate sanitizer build: + +```bash +cmake -S . -B build-ub-asan \ + -DUSE_TENT=ON \ + -DUSE_UB=ON \ + -DUSE_CUDA=OFF \ + -DBUILD_UNIT_TESTS=ON \ + -DBUILD_BENCHMARK=ON \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS='-fsanitize=address,undefined -fno-omit-frame-pointer' \ + -DCMAKE_EXE_LINKER_FLAGS='-fsanitize=address,undefined' \ + -DCMAKE_SHARED_LINKER_FLAGS='-fsanitize=address,undefined' + +cmake --build build-ub-asan -j"$(nproc)" --target tebench +export ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 +export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 +``` + +Use an embedding test harness to perform at least 100 +install-register-transfer-unregister-uninstall cycles **in one process**. +Include ten cycles where a UB port or peer is unavailable during uninstall. If +unregister or uninstall returns a provider `busy`/error, keep the engine alive +and retry the same cleanup operation; the retry must act on the retained native +handle and eventually succeed after the fault is removed. Also run 100 +target/initiator start-transfer-`SIGINT` process cycles as a supplementary +check. + +A cycle is successful only if shutdown returns, the next cycle can reuse the +same buffers and control-plane port, and ASan/UBSan reports no UAF, double free, +or leaked native ownership graph. Record the first and retry return codes plus +the provider log around every injected cleanup failure. + +## 7. Performance Baseline + +Use a `RelWithDebInfo` build and keep CPU affinity, NUMA placement, block size, +batch size, concurrency, UMDK version, and device set fixed. Record: + +| Case | Required measurements | +| --- | --- | +| 4 KiB, batch 1, one thread | average, P99, and P999 latency | +| 64 KiB and 1 MiB, one thread | single-stream bandwidth and CPU usage | +| 64 KiB and 1 MiB, 16 threads | aggregate bandwidth and CPU usage | +| Multi-device, 16 threads | aggregate and per-device bandwidth | +| First transfer to a peer | endpoint connection time | + +Run the same matrix against Classic TE UB using the existing Classic UB +benchmark on the same hosts. Any material regression needs a profile and an +explanation; do not compare results from different UMDK versions or topology. + +## 8. Result Checklist + +Attach the following to the review: + +```text +Mooncake commit: +Host/OS/kernel: +UMDK package and liburma path: +Devices/EIDs: +Debug USE_TENT+USE_UB build: PASS/FAIL +Mock UB tests: PASS/FAIL +Two-node READ: PASS/FAIL +Two-node WRITE: PASS/FAIL +Consistency-checked mix: PASS/FAIL +Multi-device pressure scheduling: PASS/FAIL +Peer restart / endpoint rebuild: PASS/FAIL +Segment republish: PASS/FAIL +Port down/up: PASS/FAIL +Cancellation and timeout: PASS/FAIL +UB to TCP/RDMA transport fallback: PASS/FAIL +100-cycle ASan/UBSan shutdown: PASS/FAIL +Classic UB baseline artifact: +TENT UB baseline artifact: +Known provider errors or unsupported cases: +``` + +Do not mark the hardware gate complete from bandwidth alone. Correctness, +generation isolation, retryable resource release, and sanitizer-clean shutdown +are independent acceptance requirements. diff --git a/docs/source/design/transfer-engine/adaptive-congestion-control-startup-config.md b/docs/source/design/transfer-engine/adaptive-congestion-control-startup-config.md index ca6fdde13d..5c1b70d01a 100644 --- a/docs/source/design/transfer-engine/adaptive-congestion-control-startup-config.md +++ b/docs/source/design/transfer-engine/adaptive-congestion-control-startup-config.md @@ -4,12 +4,11 @@ orphan: true # Adaptive Congestion Control: Shared Core and Startup Configuration -This change provides a shared pressure/window controller, an RDMA error -classifier, and environment-variable parsing for future Classic TE and TENT -adapters. It does not yet connect admission or completion feedback to either -transport, so enabling this build option alone does not throttle transfers. +Classic TE RDMA connects the shared pressure/window controller to worker +admission and completion feedback. TENT transport integration remains a separate +follow-up. The build option and runtime mode both default to `off`. -The core tracks in-flight bytes, adapts byte windows using pressure epochs, +The core tracks in-flight bytes, adapts byte windows using completion feedback, and fences permits by generation. It does not own failure detection, quarantine, cooldowns, or recovery probes. Those decisions remain with the existing transport health mechanisms. Error classification describes evidence; @@ -18,22 +17,57 @@ it does not create a second health/recovery state machine. ## Modes and configuration `MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL` defaults to `OFF`. When enabled, -the shared core target is available to Classic TE and TENT builds. The runtime -loader snapshots the environment when `loadConfigFromEnvironment()` is called. +the shared core target is available to Classic TE and TENT builds. Each Classic +RDMA worker pool snapshots the environment during construction. + +Installed headers read this build setting from `mooncake_te_build_config.h` so +their public layouts match the library. Consumers do not need to define the +macro themselves; the generated header overrides conflicting definitions. | Variable | Default | Description | | --- | ---: | --- | | `MC_ADAPTIVE_CONGESTION_CONTROL_MODE` | `off` | `off` bypasses core accounting and admission; `observe` records pressure without deferring transfers; `enforce` applies byte-window admission. | | `MC_ADAPTIVE_CONGESTION_CONTROL_MIN_WINDOW_BYTES` | 4194304 | Minimum byte window. | | `MC_ADAPTIVE_CONGESTION_CONTROL_MAX_WINDOW_BYTES` | 67108864 | Maximum byte window; must be at least the minimum. | -| `MC_ADAPTIVE_CONGESTION_CONTROL_TARGET_DRAIN_US` | 2000 | Target drain interval, converted to nanoseconds with overflow checking. | -| `MC_ADAPTIVE_CONGESTION_CONTROL_HIGH_PRESSURE_EPOCHS` | 2 | Consecutive high-pressure epochs before shrinking the window. | -| `MC_ADAPTIVE_CONGESTION_CONTROL_LOW_PRESSURE_EPOCHS` | 3 | Consecutive low-pressure epochs before growing the window. | +| `MC_ADAPTIVE_CONGESTION_CONTROL_TARGET_DRAIN_US` | 2000 | Retained and validated for configuration compatibility; not used by window decisions. | +| `MC_ADAPTIVE_CONGESTION_CONTROL_HIGH_PRESSURE_EPOCHS` | 2 | Retained and validated for configuration compatibility; explicit receiver pressure reduces the window in the current epoch. | +| `MC_ADAPTIVE_CONGESTION_CONTROL_LOW_PRESSURE_EPOCHS` | 3 | Retained and validated for configuration compatibility; clean completions allow growth in the current epoch. | Numeric variables accept positive decimal integers only. Epoch counts must fit in an unsigned 32-bit integer. Invalid input returns an invalid result with -adaptive control disabled; future adapters must report the initialization -error and preserve their original transfer behavior. +adaptive control disabled. Classic RDMA logs the initialization error once and +preserves its original transfer behavior. + +## Classic RDMA integration + +Each worker pool owns a device window and a window for each peer NIC path. +`observe` accounts admitted bytes and updates pressure windows without limiting +posts. `enforce` admits a queue prefix that fits both windows; the remaining +slices stay queued on their existing posting owner. At most the admitted prefix +is submitted to the endpoint. + +In enforce mode, a partially admitted queue on an already-active single route +replenishes in groups bounded by the configured minimum window, without +splitting slices. Unsubmitted permits are returned. Idle routes, fully admitted +tails, mixed-route queues, and off/observe modes do not wait for a group. This +only changes posting granularity; it adds no configuration or recovery gate. + +Successful and failed completions release permits. Unposted work also releases +its permit before retry or local-worker handoff. Endpoint and device generations +fence feedback from retired attempts. Window updates run in the existing monitor +worker. Existing rail/context health continues to control failure recovery. + +Ordinary backlog and estimated drain time do not reduce windows or veto growth. +Each monitor epoch with successful completions and no explicit error grows the +window by one quarter of the configured minimum, up to the configured maximum. +Explicit receiver pressure halves it, bounded by the minimum. Root errors veto +growth for the addressed domain in that epoch; peer-local errors do not veto +healthy sibling routes on the device. Errors do not create another recovery +state machine. Idle epochs do not grow the window. + +With the build option disabled, adapter state and branches are compiled out. +With runtime mode `off`, workers use the original posting path. No task-status +API or TENT admission behavior is added by this integration. ## Health ownership and draft migration @@ -44,6 +78,6 @@ existing transport's health/recovery configuration instead. Legacy health signal fields and path-state enum values remain source-compatible, but the shared controller does not use them to quarantine, probe, or recover a path. -Transport adapters, task-scoped status APIs, and physical congestion-benefit +TENT integration, task-scoped status APIs, and physical congestion-benefit validation are separate follow-ups. The core microbenchmark measures only controller overhead and is not evidence of end-to-end RDMA performance. diff --git a/docs/source/index.md b/docs/source/index.md index 4dde051c98..d1a94cc33a 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -241,4 +241,5 @@ troubleshooting/store-rs-key-access community/governance community/ecosystem-prs +community/pypi-downloads ::: diff --git a/docs/source/performance/mooncake/tebench.md b/docs/source/performance/mooncake/tebench.md index a0d2978be1..20400dec06 100644 --- a/docs/source/performance/mooncake/tebench.md +++ b/docs/source/performance/mooncake/tebench.md @@ -555,9 +555,9 @@ gpu_id + thread_id **Transport (TENT only)** * `--xport_type` : selects the benchmark transport, such as `tcp`, `hp_tcp`, - `rdma`, `shm`, or `nvlink`, and disables the other selectable transports. - MLU VRAM runs also enable the local-copy transport needed for host staging. - Empty means no transport + `rdma`, `shm`, `nvlink`, `ub`, or `sunrise_link`, and disables the other + selectable transports. MLU VRAM runs also enable the local-copy transport + needed for host staging. Empty means no transport is explicitly enabled or disabled by tebench — the engine reads the transport enable list from the `MC_TENT_CONF` config file (see Section 5.8 for multi-transport scenarios). diff --git a/docs/tests/pypi-downloads.test.cjs b/docs/tests/pypi-downloads.test.cjs new file mode 100644 index 0000000000..fa54e9df5d --- /dev/null +++ b/docs/tests/pypi-downloads.test.cjs @@ -0,0 +1,42 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { sum, recent, rangeTotal, series, breakdown } = require('../source/_static/pypi-downloads.js'); +const pkg = { name: 'mooncake-transfer-engine', endpoints: { + overall: { payload: { data: [ + {date:'2026-10-01', category:'without_mirrors', downloads:10}, + {date:'2026-10-01', category:'with_mirrors', downloads:12}, + {date:'2026-10-03', category:'without_mirrors', downloads:0}, + ]}}, + recent: { payload: { data: {last_day:0, last_week:10, last_month:50}}}, + system: { payload: { data: [ + {date:'2026-10-01', category:'Linux', downloads:8}, + {date:'2026-10-01', category:null, downloads:2}, + ]}}, +}}; +const missing = {name:'mooncake-transfer-engine-cuda13', endpoints:{}}; +test('missing values, zeros and mirror categories stay distinct', () => { + assert.equal(sum([null, null]), null); + assert.equal(sum([0, null]), 0); + assert.equal(recent(pkg, 'last_day'), 0); + assert.equal(recent(missing, 'last_day'), null); + assert.equal(rangeTotal(pkg, '2026-10-01', '2026-10-03', 'without_mirrors'), 10); + assert.equal(rangeTotal(pkg, '2026-10-01', '2026-10-03', 'with_mirrors'), 12); +}); +test('daily series retains gaps and partial coverage', () => { + const data = series([pkg, missing], '2026-10-01', '2026-10-03', 'without_mirrors'); + assert.deepEqual(data.map(d=>d.total), [10,null,0]); + assert.deepEqual(data.map(d=>d.reporting), [1,0,1]); + assert.ok(data.every(d=>d.values[1]===null)); +}); +test('environment categories retain unknown values and honor dates', () => { + assert.deepEqual(breakdown([pkg], 'system', '2026-10-01', '2026-10-03'), [['Linux',8],['Unknown',2]]); + assert.deepEqual(breakdown([pkg], 'system', '2026-10-02', '2026-10-03'), []); +}); +test('real snapshot chart totals match an independent sum', () => { + const data = JSON.parse(fs.readFileSync(path.join(__dirname, '../source/_static/pypi-downloads.json'))); + const expected = data.packages.flatMap(p=>p.endpoints.overall.payload.data).filter(r=>r.category==='without_mirrors').reduce((n,r)=>n+r.downloads,0); + const dates = data.packages.flatMap(p=>p.endpoints.overall.payload.data.map(r=>r.date)).sort(); + assert.equal(sum(series(data.packages,dates[0],dates.at(-1),'without_mirrors').map(d=>d.total)),expected); +}); diff --git a/docs/tests/test_pypi_downloads.py b/docs/tests/test_pypi_downloads.py new file mode 100644 index 0000000000..36a3adec5f --- /dev/null +++ b/docs/tests/test_pypi_downloads.py @@ -0,0 +1,92 @@ +"""Offline checks for snapshot refresh and upstream failure handling.""" + +import copy +import datetime as dt +import importlib.util +import io +import json +import unittest +import urllib.error +from pathlib import Path +from unittest.mock import patch + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/update_pypi_downloads.py" +spec = importlib.util.spec_from_file_location("downloads", SCRIPT) +downloads = importlib.util.module_from_spec(spec) +spec.loader.exec_module(downloads) + + +class SnapshotTests(unittest.TestCase): + def setUp(self): + self.snapshot = json.loads(downloads.DEFAULT_PATH.read_text()) + stamp = downloads.now().isoformat() + self.snapshot["catalog"].update(fetchedAt=stamp, status="ok") + for package in self.snapshot["packages"]: + for result in package["endpoints"].values(): + result.update(fetchedAt=stamp, status="ok") + + def test_fresh_snapshot_makes_no_network_requests(self): + with patch.object(downloads, "open_url") as request: + result = downloads.update(self.snapshot) + request.assert_not_called() + self.assertEqual(result["packages"], self.snapshot["packages"]) + + def test_failed_endpoint_preserves_payload_and_collection_time(self): + endpoint = self.snapshot["packages"][0]["endpoints"]["overall"] + endpoint["fetchedAt"] = "2020-01-01T00:00:00+00:00" + before = copy.deepcopy(endpoint) + with patch.object(downloads, "open_url", side_effect=OSError("offline")): + with patch.object(downloads.time, "sleep"): + updated = downloads.update(self.snapshot) + result = updated["packages"][0]["endpoints"]["overall"] + self.assertEqual(result["payload"], before["payload"]) + self.assertEqual(result["fetchedAt"], before["fetchedAt"]) + self.assertEqual(result["status"], "stale") + self.assertEqual(endpoint, before) + + def test_rate_limit_stops_further_requests(self): + for package in self.snapshot["packages"]: + for result in package["endpoints"].values(): + result["fetchedAt"] = "2020-01-01T00:00:00+00:00" + error = urllib.error.HTTPError("https://pypistats.org", 429, "limit", {}, None) + with patch.object(downloads, "open_url", side_effect=error) as request: + with patch.object(downloads.time, "sleep"): + updated = downloads.update(self.snapshot) + self.assertEqual(request.call_count, 1) + self.assertTrue( + all( + r["status"] == "stale" + for p in updated["packages"] + for r in p["endpoints"].values() + ) + ) + + def test_discovery_finds_variants_and_excludes_near_matches(self): + html = b"mooncake-transfer-engine\nmooncake-transfer-engine-cuda13\nmooncake-transfer-engineering\n" + with patch.object(downloads, "open_url", return_value=io.BytesIO(html)): + self.assertEqual( + downloads.discover(), [downloads.PREFIX, downloads.PREFIX + "-cuda13"] + ) + + def test_snapshot_and_validation(self): + self.assertEqual( + self.snapshot["catalog"]["names"], + [p["name"] for p in self.snapshot["packages"]], + ) + for package in self.snapshot["packages"]: + for endpoint in downloads.ENDPOINTS: + downloads.validate( + package["endpoints"][endpoint]["payload"], package["name"], endpoint + ) + invalid = { + "package": downloads.PREFIX, + "data": {"last_day": -1, "last_week": 1, "last_month": 2}, + } + with self.assertRaises(ValueError): + downloads.validate(invalid, downloads.PREFIX, "recent") + future = (downloads.now() + dt.timedelta(days=1)).isoformat() + self.assertFalse(downloads.fresh({"status": "ok", "fetchedAt": future})) + + +if __name__ == "__main__": + unittest.main() diff --git a/mooncake-conductor/include/conductor/prefixindex/prefix_indexer.h b/mooncake-conductor/include/conductor/prefixindex/prefix_indexer.h index 870c6689f2..58b1d3aa76 100644 --- a/mooncake-conductor/include/conductor/prefixindex/prefix_indexer.h +++ b/mooncake-conductor/include/conductor/prefixindex/prefix_indexer.h @@ -51,6 +51,8 @@ struct ContextState { mutable std::shared_mutex mutex; const HashProfile profile; std::map> instance_ranks; + std::map> + instance_publishers; std::unordered_map blocks; // Store order: newest prefix at the front, oldest at the back. Query does diff --git a/mooncake-conductor/include/conductor/prefixindex/types.h b/mooncake-conductor/include/conductor/prefixindex/types.h index 730467940b..f72a708c48 100644 --- a/mooncake-conductor/include/conductor/prefixindex/types.h +++ b/mooncake-conductor/include/conductor/prefixindex/types.h @@ -58,6 +58,7 @@ struct EngineRegistration { int64_t dp_rank = 0; int64_t effective_block_size = 0; std::optional cache_group; + common::PublisherKind publisher_kind = common::PublisherKind::kVllm; }; struct EngineMutation { diff --git a/mooncake-conductor/src/kvevent/conductor_service.cpp b/mooncake-conductor/src/kvevent/conductor_service.cpp index 83bb6f2ad5..7a21ac7656 100644 --- a/mooncake-conductor/src/kvevent/conductor_service.cpp +++ b/mooncake-conductor/src/kvevent/conductor_service.cpp @@ -27,7 +27,8 @@ prefixindex::EngineRegistration RegistrationFromService( .instance_id = service.instance_id, .dp_rank = service.dp_rank, .effective_block_size = service.block_size, - .cache_group = service.cache_group}; + .cache_group = service.cache_group, + .publisher_kind = service.publisher_kind}; } std::string ValidateServiceConfig(const common::ServiceConfig& service) { diff --git a/mooncake-conductor/src/kvevent/event_handler.cpp b/mooncake-conductor/src/kvevent/event_handler.cpp index ee3a6ae440..2d8bba173a 100644 --- a/mooncake-conductor/src/kvevent/event_handler.cpp +++ b/mooncake-conductor/src/kvevent/event_handler.cpp @@ -50,6 +50,18 @@ std::optional SharedTier( return std::nullopt; } +std::optional VllmTier( + const std::optional& medium) { + const std::string normalized = AsciiToLower(medium.value_or("")); + if (normalized == "gpu") { + return prefixindex::StorageTier::kDevice; + } + if (normalized == "cpu") { + return prefixindex::StorageTier::kCpuLocal; + } + return std::nullopt; +} + bool IsSglangDeviceMedium(const std::optional& medium) { // SGLang's base radix caches omit medium for their engine-local cache. // The upstream wire name remains GPU on both GPU and NPU platforms. @@ -764,11 +776,10 @@ std::string KVEventHandler::HandleVllmStored( !error.empty()) { return error; } - const std::string medium = AsciiToLower(event.medium.value_or("")); - if (medium != "gpu") { - // Upstream calls device memory GPU; other tiers are expected no-ops. - VLOG(1) << "Ignoring vLLM non-GPU event endpoint=" << service_.endpoint - << " publisher_kind=" + const auto tier = VllmTier(event.medium); + if (!tier.has_value()) { + VLOG(1) << "Ignoring vLLM unsupported-medium event endpoint=" + << service_.endpoint << " publisher_kind=" << common::PublisherKindName(metadata.publisher_kind) << " instance=" << service_.instance_id << " dp_rank=" << service_.dp_rank @@ -790,6 +801,7 @@ std::string KVEventHandler::HandleVllmStored( .owner = EngineOwnerFromService(service_), .effective_block_size = service_.block_size, .cache_group = service_.cache_group, + .tier = *tier, }); } @@ -799,10 +811,10 @@ std::string KVEventHandler::HandleVllmRemoved( !error.empty()) { return error; } - const std::string medium = AsciiToLower(event.medium.value_or("")); - if (medium != "gpu") { - VLOG(1) << "Ignoring vLLM non-GPU event endpoint=" << service_.endpoint - << " publisher_kind=" + const auto tier = VllmTier(event.medium); + if (!tier.has_value()) { + VLOG(1) << "Ignoring vLLM unsupported-medium event endpoint=" + << service_.endpoint << " publisher_kind=" << common::PublisherKindName(metadata.publisher_kind) << " instance=" << service_.instance_id << " dp_rank=" << service_.dp_rank @@ -824,6 +836,7 @@ std::string KVEventHandler::HandleVllmRemoved( .owner = EngineOwnerFromService(service_), .effective_block_size = service_.block_size, .cache_group = service_.cache_group, + .tier = *tier, }); } @@ -840,6 +853,7 @@ std::string KVEventHandler::HandleVllmCleared( .owner = EngineOwnerFromService(service_), .effective_block_size = service_.block_size, .cache_group = service_.cache_group, + .tier = std::nullopt, }); } diff --git a/mooncake-conductor/src/prefixindex/prefix_indexer.cpp b/mooncake-conductor/src/prefixindex/prefix_indexer.cpp index 0da1892325..f39691df8a 100644 --- a/mooncake-conductor/src/prefixindex/prefix_indexer.cpp +++ b/mooncake-conductor/src/prefixindex/prefix_indexer.cpp @@ -228,6 +228,10 @@ RegistrationResult PrefixCacheTable::ValidateRegistration( if (registration.dp_rank < 0) { return {.error = "dp_rank must be non-negative"}; } + if (registration.publisher_kind != common::PublisherKind::kVllm && + registration.publisher_kind != common::PublisherKind::kSglang) { + return {.error = "engine publisher must be vLLM or SGLang"}; + } if (auto error = ValidateHashProfile(registration.profile); !error.empty()) { return {.error = std::move(error)}; @@ -246,6 +250,8 @@ RegistrationResult PrefixCacheTable::Register( std::make_shared(registration.profile, block_limit_); candidate->instance_ranks[registration.instance_id].insert( registration.dp_rank); + candidate->instance_publishers[registration.instance_id].emplace( + registration.dp_rank, registration.publisher_kind); std::shared_ptr state; { @@ -263,9 +269,21 @@ RegistrationResult PrefixCacheTable::Register( return {.error = "registration conflicts with the ContextKey hash profile"}; } + auto publisher = state->instance_publishers.find(registration.instance_id); + if (publisher != state->instance_publishers.end()) { + auto rank = publisher->second.find(registration.dp_rank); + if (rank != publisher->second.end() && + rank->second != registration.publisher_kind) { + return { + .error = + "registration conflicts with the engine publisher kind"}; + } + } const bool inserted = state->instance_ranks[registration.instance_id] .insert(registration.dp_rank) .second; + state->instance_publishers[registration.instance_id].emplace( + registration.dp_rank, registration.publisher_kind); return {.inserted = inserted, .error = ""}; } @@ -322,6 +340,13 @@ std::string PrefixCacheTable::Unregister(const ContextKey& context, state->instance_ranks.erase(instance); } } + auto publisher = state->instance_publishers.find(instance_id); + if (publisher != state->instance_publishers.end()) { + publisher->second.erase(dp_rank); + if (publisher->second.empty()) { + state->instance_publishers.erase(publisher); + } + } for (auto& [unused_prefix, presence] : state->blocks) { (void)unused_prefix; @@ -357,7 +382,18 @@ std::string PrefixCacheTable::StoreEngine(const EngineMutation& mutation) { !instance->second.contains(mutation.owner.dp_rank)) { return "engine owner instance/rank is not registered"; } + const bool vllm_local = + mutation.tier == StorageTier::kCpuLocal && + state->instance_publishers.at(mutation.owner.instance_id) + .at(mutation.owner.dp_rank) == common::PublisherKind::kVllm; for (ProjectedPrefix prefix : mutation.prefixes) { + if (vllm_local) { + auto existing = state->blocks.find(prefix); + if (existing != state->blocks.end() && + !existing->second.cpu_share_owners.empty()) { + continue; + } + } auto& presence = state->blocks[prefix]; if (mutation.tier == StorageTier::kDevice) { presence.device_owners.insert(mutation.owner); @@ -442,6 +478,17 @@ std::string PrefixCacheTable::StoreShared(const SharedMutation& mutation) { auto& presence = state->blocks[prefix]; if (mutation.tier == StorageTier::kCpuShare) { presence.cpu_share_owners.insert(mutation.owner); + std::erase_if( + presence.cpu_local_owners, [&](const EngineOwner& owner) { + auto instance = + state->instance_publishers.find(owner.instance_id); + if (instance == state->instance_publishers.end()) { + return false; + } + auto rank = instance->second.find(owner.dp_rank); + return rank != instance->second.end() && + rank->second == common::PublisherKind::kVllm; + }); } else { presence.disk_owners.insert(mutation.owner); } diff --git a/mooncake-conductor/tests/client_server_e2e_test.cpp b/mooncake-conductor/tests/client_server_e2e_test.cpp index a69a9904a6..7d9fbd2935 100644 --- a/mooncake-conductor/tests/client_server_e2e_test.cpp +++ b/mooncake-conductor/tests/client_server_e2e_test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -938,6 +939,90 @@ class RouterEventPublisher { uint64_t sequence_ = 0; }; +void PublishVllmCpu(RouterEventPublisher& publisher, uint64_t hash, + bool removed = false) { + msgpack::sbuffer buffer; + msgpack::packer packer(&buffer); + packer.pack_array(3); + packer.pack(1.25); + packer.pack_array(1); + packer.pack_map(removed ? 3 : 8); + packer.pack("type"); + packer.pack(removed ? "BlockRemoved" : "BlockStored"); + packer.pack("block_hashes"); + packer.pack(std::vector{hash}); + packer.pack("medium"); + packer.pack("CPU"); + if (!removed) { + packer.pack("parent_block_hash"); + packer.pack_nil(); + packer.pack("token_ids"); + packer.pack_nil(); + packer.pack("block_size"); + packer.pack(16); + packer.pack("lora_id"); + packer.pack_nil(); + packer.pack("lora_name"); + packer.pack_nil(); + } + packer.pack(0); + publisher.Publish(std::string(buffer.data(), buffer.size())); +} + +void PublishMooncakeCpu(RouterEventPublisher& publisher, uint64_t hash, + bool removed = false) { + constexpr int64_t timestamp = 1700000000123; + msgpack::sbuffer buffer; + msgpack::packer packer(&buffer); + packer.pack_array(3); + packer.pack(timestamp); + packer.pack_array(1); + packer.pack_map(removed ? 16 : 18); + packer.pack("event_type"); + packer.pack(removed ? "removed" : "stored"); + packer.pack("event_id"); + packer.pack(1); + packer.pack("timestamp"); + packer.pack(timestamp); + packer.pack("model_name"); + packer.pack(E2EContext().model_name); + packer.pack("block_size"); + packer.pack(E2EContext().block_size); + packer.pack("additional_salt"); + packer.pack_nil(); + packer.pack("lora_name"); + packer.pack_nil(); + packer.pack("tenant_id"); + packer.pack(E2EContext().tenant_id); + packer.pack("backend_id"); + packer.pack("e2e-backend"); + packer.pack("medium"); + packer.pack("CPU"); + packer.pack("dp_rank"); + packer.pack(0); + packer.pack("group_id"); + packer.pack_nil(); + packer.pack("seq_hashes"); + packer.pack(std::vector{hash}); + packer.pack("base_block_idx"); + packer.pack_nil(); + packer.pack("object_key"); + packer.pack("e2e-object"); + char low64[17]; + std::snprintf(low64, sizeof(low64), "%016llx", + static_cast(hash)); + packer.pack("connector_block_hash"); + packer.pack(std::string(48, '0') + low64); + if (!removed) { + packer.pack("parent_hash"); + packer.pack_nil(); + packer.pack("token_ids"); + packer.pack_nil(); + } + packer.pack(0); + publisher.Publish(std::string(buffer.data(), buffer.size())); +} + enum class QueryProtocol { kHttp, kRpc }; class RouterQueryTest : public ::testing::TestWithParam { @@ -1180,6 +1265,87 @@ TEST_P(RouterQueryTest, FallsBackOnMissAndQueryFailure) { ASSERT_NO_FATAL_FAILURE(ExpectRoute(0, RouteReason::kQueryFailed)); } +TEST(ClientServerE2ETest, VllmCpuLocalYieldsToMooncakeAcrossPublicQueries) { + RouterEventPublisher a; + RouterEventPublisher b; + RouterEventPublisher pool; + EventManager manager({}, 0, kRpcPort); + ASSERT_TRUE(manager.StartHTTPServer()); + ASSERT_TRUE(manager.StartRPCServer()); + manager.Start(); + const uint16_t http_port = EventManagerTestPeer::HttpPort(manager); + ConductorClient client; + ASSERT_NO_FATAL_FAILURE( + SetupWithRetry(client, "127.0.0.1:" + std::to_string(kRpcPort))); + auto engine_a = E2EService("engine-a", a.Endpoint()); + auto engine_b = E2EService("engine-b", b.Endpoint()); + auto shared_pool = E2EService("shared-pool", pool.Endpoint()); + shared_pool.publisher_kind = common::PublisherKind::kMooncake; + ASSERT_TRUE(client.Register(engine_a).has_value()); + ASSERT_TRUE(a.WaitForSubscriber()); + ASSERT_TRUE(client.Register(engine_b).has_value()); + ASSERT_TRUE(b.WaitForSubscriber()); + ASSERT_TRUE(client.Register(shared_pool).has_value()); + ASSERT_TRUE(pool.WaitForSubscriber()); + + QueryRequest request{.context = E2EContext(), .token_ids = Sequence(1, 16)}; + const auto hashes = + ProjectedFor(request.context, E2EProfile(), request.token_ids); + ASSERT_EQ(hashes.size(), 1u); + coro_http::coro_http_client http; + auto query = + [&](const QueryRequest& requested) -> std::optional { + const auto rpc = client.Query(requested); + if (!rpc) return std::nullopt; + return QueryResultToJson(*rpc)["instances"]; + }; + auto wait_for = [&](int64_t local_b, int64_t shared_a, + int64_t shared_b) -> ::testing::AssertionResult { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + Json::Value last_rpc; + Json::Value last_http; + do { + const auto result = query(request); + if (result) last_rpc = *result; + const auto response = + HttpPost(http, http_port, "/query", QueryMsgpackBody(request)); + if (response.status == 200) { + last_http = DecodeMsgpackBody(response.body)["instances"]; + } + if (result && last_rpc == last_http && + last_rpc["engine-b"]["cpu_local"].asInt64() == local_b && + last_rpc["engine-a"]["cpu_share"].asInt64() == shared_a && + last_rpc["engine-b"]["cpu_share"].asInt64() == shared_b) { + return ::testing::AssertionSuccess(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } while (std::chrono::steady_clock::now() < deadline); + return ::testing::AssertionFailure() + << "query state did not converge: rpc=" << last_rpc + << " http=" << last_http; + }; + std::array backends{ + {{"engine-a", 0, {}}, {"engine-b", 0, {}}}}; + MockRouter router(query, backends); + + PublishVllmCpu(b, hashes[0].value); + ASSERT_TRUE(wait_for(16, 0, 16)); + EXPECT_EQ(router.Route(request).backend, 1u); + PublishMooncakeCpu(pool, hashes[0].value); + ASSERT_TRUE(wait_for(0, 16, 16)); + EXPECT_EQ(router.Route(request).backend, 0u); + PublishMooncakeCpu(pool, hashes[0].value, true); + ASSERT_TRUE(wait_for(0, 0, 0)); + EXPECT_EQ(router.Route(request).reason, RouteReason::kMiss); + PublishVllmCpu(b, hashes[0].value); + ASSERT_TRUE(wait_for(16, 0, 16)); + EXPECT_EQ(router.Route(request).backend, 1u); + + client.Close(); + manager.Stop(); +} + INSTANTIATE_TEST_SUITE_P( PublicInterfaces, RouterQueryTest, ::testing::Values(QueryProtocol::kHttp, QueryProtocol::kRpc), diff --git a/mooncake-conductor/tests/event_manager_test.cpp b/mooncake-conductor/tests/event_manager_test.cpp index 812155ef6a..c3c3857989 100644 --- a/mooncake-conductor/tests/event_manager_test.cpp +++ b/mooncake-conductor/tests/event_manager_test.cpp @@ -114,7 +114,8 @@ EngineRegistration RegistrationFor(const ServiceConfig& service) { .instance_id = service.instance_id, .dp_rank = service.dp_rank, .effective_block_size = service.block_size, - .cache_group = service.cache_group}; + .cache_group = service.cache_group, + .publisher_kind = service.publisher_kind}; } ServiceConfig VllmService(const std::string& instance_id = "instance-1", @@ -666,7 +667,7 @@ class QueryHttpTest : public ::testing::Test { std::vector tokens_; }; -TEST_F(QueryHttpTest, FourTierResponseSeparatesLocalAndSharedCpu) { +TEST_F(QueryHttpTest, VllmSharedCpuSuppressesLocalInHttpResponse) { const auto context = ContextFor(instance_one_); const auto prefixes = ProjectedFor(context, instance_one_.hash_profile, tokens_); @@ -685,6 +686,9 @@ TEST_F(QueryHttpTest, FourTierResponseSeparatesLocalAndSharedCpu) { const auto result = ParseMsgpackResponse(response); const auto& one = result["instances"]["1"]; const auto& two = result["instances"]["2"]; + EXPECT_TRUE(PrefixCacheTableTestPeer::Presence(*manager_->GetIndexer(), + context, prefixes[2]) + ->cpu_local_owners.empty()); for (const auto* value : {&one, &two}) { EXPECT_FALSE(value->isMember("gpu")); EXPECT_FALSE(value->isMember("cpu")); @@ -694,9 +698,9 @@ TEST_F(QueryHttpTest, FourTierResponseSeparatesLocalAndSharedCpu) { EXPECT_EQ((*value)["cpu_share"].asInt64(), 48); EXPECT_EQ((*value)["disk"].asInt64(), 48); } - EXPECT_EQ(one["cpu_local"].asInt64(), 48); + EXPECT_EQ(one["cpu_local"].asInt64(), 32); EXPECT_EQ(two["cpu_local"].asInt64(), 0); - EXPECT_EQ(one["rank_matches"]["0"]["cpu_local"].asInt64(), 48); + EXPECT_EQ(one["rank_matches"]["0"]["cpu_local"].asInt64(), 32); EXPECT_EQ(two["rank_matches"]["1"]["cpu_share"].asInt64(), 48); } @@ -2207,7 +2211,7 @@ TEST(KVEventHandlerTest, VllmBinaryHashUsesFinalEightBytesWithoutRehashing) { 4); } -TEST(KVEventHandlerTest, VllmCpuDiskAreNoOpsAndGpuSiblingContinues) { +TEST(KVEventHandlerTest, VllmCpuAndGpuAreIndexedWhileDiskIsIgnored) { EventManager manager({}, 0); const auto service = VllmService("engine", "default", 0, 4); ASSERT_TRUE( @@ -2217,28 +2221,21 @@ TEST(KVEventHandlerTest, VllmCpuDiskAreNoOpsAndGpuSiblingContinues) { const auto prefixes = ProjectedFor(ContextFor(service), TestProfile(), tokens); - const std::array ignored_media = {"CPU", "cpu", "DISK", - "disk"}; - std::vector events; + const std::array ignored_media = {"DISK", "disk"}; + std::vector events{VllmStored(prefixes[0].value, 4, "CPU"), + VllmStored(prefixes[0].value, 4, "cpu"), + VllmStored(prefixes[0].value, 4, "GPU")}; for (const auto& medium : ignored_media) { events.emplace_back(VllmStored(prefixes[0].value, 4, medium)); } - events.emplace_back(VllmStored(prefixes[0].value, 4, "GPU")); - for (const auto& medium : ignored_media) { - events.emplace_back( - VllmRemovedEvent{.block_hashes = {prefixes[0].value}, - .medium = medium, - .group_idx = 0}); - } - // Expected non-GPU traffic is traced at VLOG(1) (reported to sinks as - // GLOG_INFO), not at WARNING: this test would emit eight warnings. InfoSink info; WarningSink warnings; EXPECT_TRUE(DispatchVllm(handler, service, std::move(events), 0).empty()); - const auto result = + auto result = manager.GetIndexer()->Query(ContextFor(service), tokens).at("engine"); EXPECT_EQ(result.device, 4); + EXPECT_EQ(result.cpu_local, 4); EXPECT_EQ(result.cpu_share, 4); EXPECT_EQ(result.disk, 4); @@ -2247,8 +2244,31 @@ TEST(KVEventHandlerTest, VllmCpuDiskAreNoOpsAndGpuSiblingContinues) { const auto& presence = snapshot.contexts.at(ContextFor(service)).blocks.at(prefixes[0]); EXPECT_FALSE(presence.device_owners.empty()); + EXPECT_EQ(presence.cpu_local_owners.size(), 1u); EXPECT_TRUE(presence.cpu_share_owners.empty()); EXPECT_TRUE(presence.disk_owners.empty()); + + std::vector removals{ + VllmRemovedEvent{.block_hashes = {prefixes[0].value}, + .medium = "CPU", + .group_idx = 0}, + VllmRemovedEvent{.block_hashes = {prefixes[0].value}, + .medium = "cpu", + .group_idx = 0}}; + for (const auto& medium : ignored_media) { + removals.emplace_back( + VllmRemovedEvent{.block_hashes = {prefixes[0].value}, + .medium = medium, + .group_idx = 0}); + } + EXPECT_TRUE(DispatchVllm(handler, service, std::move(removals), 0).empty()); + result = + manager.GetIndexer()->Query(ContextFor(service), tokens).at("engine"); + EXPECT_EQ(result.device, 4); + EXPECT_EQ(result.cpu_local, 4); + EXPECT_TRUE(PrefixCacheTableTestPeer::Presence( + *manager.GetIndexer(), ContextFor(service), prefixes[0]) + ->cpu_local_owners.empty()); if (VLOG_IS_ON(1)) { EXPECT_TRUE(info.Contains("endpoint=" + service.endpoint)); EXPECT_TRUE(info.Contains("publisher_kind=vLLM")); @@ -2262,7 +2282,6 @@ TEST(KVEventHandlerTest, VllmCpuDiskAreNoOpsAndGpuSiblingContinues) { } EXPECT_TRUE(info.Contains("hash_count=1")); } - // Non-GPU events never reach the tier-ignore warning path. for (const auto& medium : ignored_media) { EXPECT_FALSE( warnings.Contains("event_type=BlockStored medium=" + medium)); @@ -2271,6 +2290,70 @@ TEST(KVEventHandlerTest, VllmCpuDiskAreNoOpsAndGpuSiblingContinues) { } } +TEST(KVEventHandlerTest, VllmCpuLocalYieldsToMooncakeAndClearPreservesShared) { + EventManager manager({}, 0); + const auto engine_a = VllmService("engine-a", "default", 0, 4); + const auto engine_b = VllmService("engine-b", "default", 0, 4); + for (const auto& service : {engine_a, engine_b}) { + ASSERT_TRUE(manager.GetIndexer() + ->Register(RegistrationFor(service)) + .error.empty()); + } + KVEventHandler handler_a(&manager, engine_a); + KVEventHandler handler_b(&manager, engine_b); + const auto pool = MooncakeService(engine_a); + KVEventHandler pool_handler(&manager, pool); + const auto tokens = Sequence(1, 4); + const auto prefix = + ProjectedFor(ContextFor(engine_a), TestProfile(), tokens).front(); + const auto local = VllmStored(prefix.value, 4, "CPU"); + const auto shared = MooncakeStored(engine_a, prefix.value, "shared-object"); + + ASSERT_TRUE(DispatchVllm(handler_a, engine_a, + {local, VllmStored(prefix.value, 4)}, 0) + .empty()); + ASSERT_TRUE(DispatchVllm(handler_b, engine_b, {local}, 0).empty()); + EXPECT_EQ(manager.GetIndexer() + ->Query(ContextFor(engine_a), tokens) + .at("engine-b") + .cpu_local, + 4); + ASSERT_TRUE(DispatchMooncake(pool_handler, pool, {shared}).empty()); + auto presence = PrefixCacheTableTestPeer::Presence( + *manager.GetIndexer(), ContextFor(engine_a), prefix); + ASSERT_TRUE(presence.has_value()); + EXPECT_TRUE(presence->cpu_local_owners.empty()); + EXPECT_EQ(presence->device_owners.size(), 1u); + EXPECT_EQ(presence->cpu_share_owners.size(), 1u); + + ASSERT_TRUE(DispatchVllm(handler_b, engine_b, {local}, 0).empty()); + ASSERT_TRUE( + DispatchVllm(handler_a, engine_a, {VllmClearedEvent{}}, 0).empty()); + auto hits = manager.GetIndexer()->Query(ContextFor(engine_a), tokens); + EXPECT_EQ(hits.at("engine-a").device, 0); + EXPECT_EQ(hits.at("engine-a").cpu_local, 0); + EXPECT_EQ(hits.at("engine-b").cpu_local, 0); + EXPECT_EQ(hits.at("engine-b").cpu_share, 4); + + ASSERT_TRUE(DispatchMooncake(pool_handler, pool, {MooncakeRemoved(shared)}) + .empty()); + hits = manager.GetIndexer()->Query(ContextFor(engine_a), tokens); + EXPECT_EQ(hits.at("engine-b").cpu_local, 0); + EXPECT_EQ(hits.at("engine-b").cpu_share, 0); + ASSERT_TRUE(DispatchVllm(handler_b, engine_b, {local}, 0).empty()); + EXPECT_EQ(manager.GetIndexer() + ->Query(ContextFor(engine_a), tokens) + .at("engine-b") + .cpu_local, + 4); + handler_b.OnSourceStale("test", MetadataFor(engine_b), "source stale"); + EXPECT_EQ(manager.GetIndexer() + ->Query(ContextFor(engine_a), tokens) + .at("engine-b") + .cpu_local, + 0); +} + TEST(KVEventHandlerTest, VllmClearPreservesOtherEngineAndSharedOwners) { EventManager manager({}, 0); const auto engine_a = VllmService("engine-a", "default", 0, 4); diff --git a/mooncake-conductor/tests/prefix_indexer_test.cpp b/mooncake-conductor/tests/prefix_indexer_test.cpp index bac68f3408..753b85894f 100644 --- a/mooncake-conductor/tests/prefix_indexer_test.cpp +++ b/mooncake-conductor/tests/prefix_indexer_test.cpp @@ -1,11 +1,13 @@ #include +#include #include #include #include #include #include #include +#include #include #include @@ -15,6 +17,7 @@ namespace { +using mooncake::conductor::common::PublisherKind; using mooncake::conductor::prefixindex::BlockPresenceSnapshot; using mooncake::conductor::prefixindex::CacheHitResult; using mooncake::conductor::prefixindex::ContextKey; @@ -93,14 +96,16 @@ HashProfile SglangBigramProfile() { } EngineRegistration Registration(const std::string& instance_id = "instance-a", - int64_t dp_rank = 0) { + int64_t dp_rank = 0, + PublisherKind kind = PublisherKind::kVllm) { const ContextKey context = TestContext(); return {.context = context, .profile = TestProfile(), .instance_id = instance_id, .dp_rank = dp_rank, .effective_block_size = context.block_size, - .cache_group = 0}; + .cache_group = 0, + .publisher_kind = kind}; } EngineOwner DeviceOwner(const std::string& instance_id = "instance-a", @@ -281,6 +286,10 @@ TEST(Registration, InvalidInputsDoNotCreateContextState) { negative_rank.dp_rank = -1; invalid.push_back(negative_rank); + auto pool_publisher = Registration(); + pool_publisher.publisher_kind = PublisherKind::kMooncake; + invalid.push_back(pool_publisher); + auto malformed_profile = Registration(); malformed_profile.profile.root_digest = "not-a-digest"; invalid.push_back(malformed_profile); @@ -330,9 +339,11 @@ TEST(Registration, SglangSeedVariantsShareContextAndBinding) { PrefixCacheTable table; auto first = Registration(); first.profile = profile; + first.publisher_kind = PublisherKind::kSglang; ASSERT_TRUE(table.Register(first).error.empty()); auto second = Registration("instance-b", 1); second.profile = profile; + second.publisher_kind = PublisherKind::kSglang; second.profile.python_hash_seed = "different-unused-seed"; ASSERT_TRUE(table.Register(second).error.empty()); EXPECT_TRUE(table.ValidateProfileBinding(TestContext(), second.profile) @@ -382,9 +393,35 @@ TEST(Registration, TracksEveryInstanceAndRankIdempotently) { EXPECT_EQ(state.profile, TestProfile()); EXPECT_EQ(state.instance_ranks.at("instance-a"), (std::set{0, 2})); EXPECT_EQ(state.instance_ranks.at("instance-b"), (std::set{1})); + EXPECT_EQ(state.instance_publishers.at("instance-a").at(0), + PublisherKind::kVllm); + EXPECT_EQ(state.instance_publishers.at("instance-a").at(2), + PublisherKind::kVllm); + EXPECT_EQ(state.instance_publishers.at("instance-b").at(1), + PublisherKind::kVllm); EXPECT_TRUE(state.blocks.empty()); } +TEST(Registration, PublisherKindConflictPreservesStateAndUnregisterRemovesIt) { + PrefixCacheTable table; + RegisterOrFail(table, Registration("instance-a", 0)); + const auto before = PrefixCacheTableTestPeer::Snapshot(table); + const auto conflict = + table.Register(Registration("instance-a", 0, PublisherKind::kSglang)); + EXPECT_FALSE(conflict.inserted); + EXPECT_NE(conflict.error.find("publisher kind"), std::string::npos); + EXPECT_EQ(PrefixCacheTableTestPeer::Snapshot(table), before); + + ASSERT_EQ(table.Unregister(TestContext(), "instance-a", 0), ""); + EXPECT_TRUE(PrefixCacheTableTestPeer::Snapshot(table) + .contexts.at(TestContext()) + .instance_publishers.empty()); + auto replacement = + table.Register(Registration("instance-a", 0, PublisherKind::kSglang)); + EXPECT_TRUE(replacement.error.empty()); + EXPECT_TRUE(replacement.inserted); +} + TEST(Registration, ConflictingProfilePreservesCompleteState) { PrefixCacheTable table; RegisterOrFail(table, Registration()); @@ -724,6 +761,7 @@ TEST(Query, TrailingPartialBlockNeverReportsMoreThanPromptTokens) { PrefixCacheTable table; auto registration = Registration(); registration.profile = SglangProfile(); + registration.publisher_kind = PublisherKind::kSglang; RegisterOrFail(table, registration); // 40 tokens over a 16-token block size: two whole blocks plus a trailing @@ -750,6 +788,7 @@ TEST(Query, WholeBlockRunIsUnaffectedByThePromptLengthClamp) { PrefixCacheTable table; auto registration = Registration(); registration.profile = SglangProfile(); + registration.publisher_kind = PublisherKind::kSglang; RegisterOrFail(table, registration); const auto tokens = Tokens(40); @@ -772,6 +811,7 @@ TEST(Query, BigramFullMatchReportsOneFewerPositionThanPromptTokens) { PrefixCacheTable table; auto registration = Registration(); registration.profile = SglangBigramProfile(); + registration.publisher_kind = PublisherKind::kSglang; RegisterOrFail(table, registration); // 40 raw tokens over a 16-token block size. The bigram chain covers 39 @@ -799,6 +839,7 @@ TEST(Query, BigramWholeBlockRunIsUnaffectedByTheLogicalLengthClamp) { PrefixCacheTable table; auto registration = Registration(); registration.profile = SglangBigramProfile(); + registration.publisher_kind = PublisherKind::kSglang; RegisterOrFail(table, registration); const auto tokens = Tokens(40); @@ -928,7 +969,7 @@ TEST(Query, CumulativeTierIncludesHigherTierAfterLowerTierBlocks) { EXPECT_EQ(result.disk, 64); } -TEST(Query, LocalAndSharedTiersRespectRankAndContinuousPrefix) { +TEST(Query, VllmSharedCpuSupersedesLocalAcrossRanksAndPrefix) { PrefixCacheTable table; for (const auto& registration : {Registration(), Registration("instance-a", 1), @@ -954,9 +995,12 @@ TEST(Query, LocalAndSharedTiersRespectRankAndContinuousPrefix) { ASSERT_EQ(table.StoreShared(shared), ""); EXPECT_EQ(table.Query(TestContext(), tokens).at("instance-b").cpu_share, 80); + EXPECT_TRUE(Presence(table, hashes[0]).cpu_local_owners.empty()); + EXPECT_EQ( + table.Query(TestContext(), tokens).at("instance-a").rank_matches.at(0), + RankMatch(0, 0, 80, 80)); ASSERT_EQ(table.RemoveShared(shared), ""); - EXPECT_EQ(table.Query(TestContext(), tokens).at("instance-a").cpu_share, - 48); + EXPECT_EQ(table.Query(TestContext(), tokens).at("instance-a").cpu_share, 0); EXPECT_EQ(table.Query(TestContext(), tokens).at("instance-b").cpu_share, 0); ASSERT_EQ(table.Unregister(TestContext(), "instance-a", 0), ""); RegisterOrFail(table, Registration()); @@ -1013,11 +1057,16 @@ TEST(Query, FourTierInterleavedPrefixesAndGaps) { TEST(Query, FourTierOwnersRemainIndependentAcrossRanksAndLifecycle) { PrefixCacheTable table; - RegisterOrFail(table, Registration()); - RegisterOrFail(table, Registration("instance-a", 1)); - RegisterOrFail(table, Registration("instance-b")); + for (const auto& [instance, rank] : + std::vector>{ + {"instance-a", 0}, {"instance-a", 1}, {"instance-b", 0}}) { + auto registration = + Registration(instance, rank, PublisherKind::kSglang); + registration.profile = SglangProfile(); + RegisterOrFail(table, registration); + } const auto tokens = Tokens(48); - const auto hashes = Hashes(tokens); + const auto hashes = SglangHashes(tokens); auto local = Device(hashes); local.tier = StorageTier::kCpuLocal; const auto shared = Shared(hashes, StorageTier::kCpuShare); @@ -1048,12 +1097,111 @@ TEST(Query, FourTierOwnersRemainIndependentAcrossRanksAndLifecycle) { RankMatch(0, 0, 48, 48)); ASSERT_EQ(table.StoreEngine(local), ""); ASSERT_EQ(table.Unregister(TestContext(), "instance-a", 0), ""); - RegisterOrFail(table, Registration()); + auto replacement = Registration("instance-a", 0, PublisherKind::kSglang); + replacement.profile = SglangProfile(); + RegisterOrFail(table, replacement); EXPECT_EQ( table.Query(TestContext(), tokens).at("instance-a").rank_matches.at(0), RankMatch(0, 0, 48, 48)); } +TEST(Mutations, VllmCpuLocalYieldsToSharedAcrossInstancesAndRanks) { + PrefixCacheTable table; + RegisterOrFail(table, Registration("instance-a", 0)); + RegisterOrFail(table, Registration("instance-a", 1)); + RegisterOrFail(table, Registration("instance-b", 0)); + const auto tokens = Tokens(32); + const auto hashes = Hashes(tokens); + ASSERT_EQ(hashes.size(), 2u); + const auto a0 = DeviceOwner("instance-a", 0, "stream-a0"); + const auto a1 = DeviceOwner("instance-a", 1, "stream-a1"); + const auto b0 = DeviceOwner("instance-b", 0, "stream-b0"); + auto local_a0 = Device({hashes[0]}, a0); + local_a0.tier = StorageTier::kCpuLocal; + auto local_a1 = Device({hashes[0]}, a1); + local_a1.tier = StorageTier::kCpuLocal; + auto local_b0 = Device({hashes[0]}, b0); + local_b0.tier = StorageTier::kCpuLocal; + ASSERT_EQ(table.StoreEngine(local_a0), ""); + ASSERT_EQ(table.StoreEngine(local_a1), ""); + ASSERT_EQ(table.StoreEngine(local_b0), ""); + ASSERT_EQ(table.StoreEngine(Device({hashes[0]}, a0)), ""); + ASSERT_EQ(table.StoreShared(Shared({hashes[0]}, StorageTier::kDisk)), ""); + EXPECT_EQ(Presence(table, hashes[0]).cpu_local_owners.size(), 3u); + + const auto shared_a = Shared({hashes[0]}, StorageTier::kCpuShare); + const auto shared_b = + Shared({hashes[0]}, StorageTier::kCpuShare, SharedOwner("object-b")); + ASSERT_EQ(table.StoreShared(shared_a), ""); + EXPECT_TRUE(Presence(table, hashes[0]).cpu_local_owners.empty()); + EXPECT_EQ(Presence(table, hashes[0]).device_owners, + (std::set{a0})); + EXPECT_EQ(Presence(table, hashes[0]).disk_owners.size(), 1u); + + const auto before = PrefixCacheTableTestPeer::Snapshot(table); + const auto order = PrefixCacheTableTestPeer::Order(table, TestContext()); + ASSERT_EQ(table.StoreEngine(local_a0), ""); + ASSERT_EQ(table.StoreEngine(local_b0), ""); + EXPECT_EQ(PrefixCacheTableTestPeer::Snapshot(table), before); + EXPECT_EQ(PrefixCacheTableTestPeer::Order(table, TestContext()).write_order, + order.write_order); + ASSERT_EQ(table.StoreShared(shared_b), ""); + ASSERT_EQ(table.RemoveShared(shared_a), ""); + EXPECT_TRUE(Presence(table, hashes[0]).cpu_local_owners.empty()); + ASSERT_EQ(table.RemoveShared(shared_b), ""); + EXPECT_TRUE(Presence(table, hashes[0]).cpu_local_owners.empty()); + ASSERT_EQ(table.StoreEngine(local_b0), ""); + EXPECT_EQ(Presence(table, hashes[0]).cpu_local_owners, + (std::set{b0})); + ASSERT_EQ(table.RemoveEngine(local_b0), ""); + EXPECT_TRUE(Presence(table, hashes[0]).cpu_local_owners.empty()); + + const auto shared_first = Shared({hashes[1]}, StorageTier::kCpuShare); + auto later_local = Device({hashes[1]}, a0); + later_local.tier = StorageTier::kCpuLocal; + ASSERT_EQ(table.StoreShared(shared_first), ""); + ASSERT_EQ(table.StoreEngine(later_local), ""); + EXPECT_TRUE(Presence(table, hashes[1]).cpu_local_owners.empty()); + ASSERT_EQ(table.RemoveShared(shared_first), ""); + EXPECT_FALSE( + PrefixCacheTableTestPeer::Presence(table, TestContext(), hashes[1]) + .has_value()); + ASSERT_EQ(table.StoreEngine(later_local), ""); + EXPECT_EQ(Presence(table, hashes[1]).cpu_local_owners, + (std::set{a0})); +} + +TEST(Mutations, ConcurrentVllmLocalAndSharedStoresKeepSharedOnly) { + PrefixCacheTable table; + RegisterOrFail(table, Registration()); + for (uint64_t value = 100; value < 132; ++value) { + const auto prefix = Prefix(value); + auto local = Device({prefix}); + local.tier = StorageTier::kCpuLocal; + const auto shared = Shared({prefix}, StorageTier::kCpuShare); + std::barrier start(3); + std::string local_error; + std::string shared_error; + std::thread local_writer([&] { + start.arrive_and_wait(); + local_error = table.StoreEngine(local); + }); + std::thread shared_writer([&] { + start.arrive_and_wait(); + shared_error = table.StoreShared(shared); + }); + start.arrive_and_wait(); + local_writer.join(); + shared_writer.join(); + ASSERT_TRUE(local_error.empty()) << local_error; + ASSERT_TRUE(shared_error.empty()) << shared_error; + const auto presence = Presence(table, prefix); + EXPECT_TRUE(presence.cpu_local_owners.empty()); + EXPECT_EQ(presence.cpu_share_owners, + (std::set{shared.owner})); + } +} + TEST(Mutations, FourTierMutationsRejectWrongOwnerFamily) { PrefixCacheTable table; RegisterOrFail(table, Registration()); @@ -1080,6 +1228,7 @@ TEST(Query, FourTierLocalPartialTailUsesProfileLogicalPositions) { PrefixCacheTable table; auto registration = Registration(); registration.profile = bigram ? SglangBigramProfile() : SglangProfile(); + registration.publisher_kind = PublisherKind::kSglang; RegisterOrFail(table, registration); const auto hashes = bigram ? SglangBigramHashes(tokens) : SglangHashes(tokens); diff --git a/mooncake-conductor/tests/prefix_indexer_test_peer.h b/mooncake-conductor/tests/prefix_indexer_test_peer.h index 3420790c43..71cc5f006c 100644 --- a/mooncake-conductor/tests/prefix_indexer_test_peer.h +++ b/mooncake-conductor/tests/prefix_indexer_test_peer.h @@ -26,6 +26,8 @@ struct BlockPresenceSnapshot { struct ContextStateSnapshot { HashProfile profile; std::map> instance_ranks; + std::map> + instance_publishers; std::unordered_map blocks; bool operator==(const ContextStateSnapshot&) const = default; @@ -115,6 +117,7 @@ class PrefixCacheTableTestPeer { ContextStateSnapshot state_snapshot; state_snapshot.profile = state->profile; state_snapshot.instance_ranks = state->instance_ranks; + state_snapshot.instance_publishers = state->instance_publishers; for (const auto& [prefix, presence] : state->blocks) { state_snapshot.blocks.emplace( prefix, BlockPresenceSnapshot{presence.device_owners, diff --git a/mooncake-pg/CMakeLists.txt b/mooncake-pg/CMakeLists.txt index 8feaffb7c2..8840da0e04 100644 --- a/mooncake-pg/CMakeLists.txt +++ b/mooncake-pg/CMakeLists.txt @@ -3,3 +3,6 @@ project(mooncake-pg) include_directories(include) add_subdirectory(src) +if(BUILD_UNIT_TESTS) + add_subdirectory(tests) +endif() diff --git a/mooncake-pg/include/pg_host_ip_resolve.h b/mooncake-pg/include/pg_host_ip_resolve.h new file mode 100644 index 0000000000..396c8ebe3b --- /dev/null +++ b/mooncake-pg/include/pg_host_ip_resolve.h @@ -0,0 +1,135 @@ +#ifndef MOONCAKE_PG_HOST_IP_RESOLVE_H +#define MOONCAKE_PG_HOST_IP_RESOLVE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mooncake { +namespace detail { + +inline bool IsLoopbackIpv4(const in_addr& addr) { + // 127.0.0.0/8 (covers Debian/Ubuntu 127.0.1.1 hostname mappings). + return (ntohl(addr.s_addr) >> 24) == 127; +} + +inline bool IsLoopbackIpv6(const in6_addr& addr) { + static const in6_addr kLoopback = IN6ADDR_LOOPBACK_INIT; + if (std::memcmp(&addr, &kLoopback, sizeof(addr)) == 0) return true; + // IPv4-mapped ::ffff:127.x.x.x + if (IN6_IS_ADDR_V4MAPPED(&addr)) { + in_addr v4{}; + std::memcpy(&v4, &addr.s6_addr[12], sizeof(v4)); + return IsLoopbackIpv4(v4); + } + return false; +} + +} // namespace detail + +inline bool IsLoopbackHost(std::string_view host) { + if (host.empty() || host == "localhost") return true; + + const std::string host_str(host); + in_addr v4{}; + if (inet_pton(AF_INET, host_str.c_str(), &v4) == 1) { + return detail::IsLoopbackIpv4(v4); + } + in6_addr v6{}; + if (inet_pton(AF_INET6, host_str.c_str(), &v6) == 1) { + return detail::IsLoopbackIpv6(v6); + } + // Non-IP hostnames are not treated as loopback. + return false; +} + +struct HostIpResolveHooks { + // Optional: return a routable local address toward peer_host. + std::function(const std::string& peer_host)> + local_ip_toward_peer; + // Optional: return a routable address derived from the local hostname. + std::function()> local_ip_from_hostname; +}; + +enum class HostIpResolveOutcome { + Unchanged, + Resolved, + FailFast, +}; + +struct HostIpResolveResult { + HostIpResolveOutcome outcome = HostIpResolveOutcome::Unchanged; + std::string host_ip; + std::string error_message; +}; + +// Resolve a routable host_ip when the current value is still loopback. +// +// Hostname fallback and fail-fast only run when master_host is a confirmed +// multi-node signal (non-empty and non-loopback). Single-node torchrun with +// MASTER_ADDR=127.0.0.1 / unset keeps the loopback default. +// Discovery hooks that return loopback (e.g. 127.0.1.1) are ignored so the +// next fallback / fail-fast can run. +inline HostIpResolveResult ResolveRoutableHostIp( + std::string host_ip, std::string_view master_host, + const HostIpResolveHooks& hooks) { + HostIpResolveResult result; + result.host_ip = std::move(host_ip); + if (!IsLoopbackHost(result.host_ip)) { + result.outcome = HostIpResolveOutcome::Unchanged; + return result; + } + + const bool multi_node = + !master_host.empty() && !IsLoopbackHost(master_host); + if (!multi_node) { + // Preserve the documented single-node default (127.0.0.1). + result.outcome = HostIpResolveOutcome::Unchanged; + return result; + } + + if (hooks.local_ip_toward_peer) { + if (auto ip = hooks.local_ip_toward_peer(std::string(master_host))) { + if (!IsLoopbackHost(*ip)) { + result.host_ip = std::move(*ip); + result.outcome = HostIpResolveOutcome::Resolved; + return result; + } + } + } + + if (hooks.local_ip_from_hostname) { + if (auto ip = hooks.local_ip_from_hostname()) { + if (!IsLoopbackHost(*ip)) { + result.host_ip = std::move(*ip); + result.outcome = HostIpResolveOutcome::Resolved; + return result; + } + } + } + + result.outcome = HostIpResolveOutcome::FailFast; + result.error_message = + "Mooncake PG host_ip is still loopback while MASTER_ADDR=" + + std::string(master_host) + + " looks multi-node. Call mooncake.pg.set_host_ip() before init_process_group(), or pass a TransferEngine " + "via set_transfer_engine()."; + return result; +} + +// Production helpers implemented in pg_host_ip_resolve.cpp. +std::optional LocalIpTowardPeer(const std::string& peer_host); +std::optional LocalIpFromHostname(); + +// Reads MASTER_ADDR and applies ResolveRoutableHostIp with real socket hooks. +HostIpResolveResult MaybeResolveRoutableHostIpFromEnv(std::string host_ip); + +} // namespace mooncake + +#endif // MOONCAKE_PG_HOST_IP_RESOLVE_H diff --git a/mooncake-pg/src/CMakeLists.txt b/mooncake-pg/src/CMakeLists.txt index fbf3d452be..3c1ba5cb96 100644 --- a/mooncake-pg/src/CMakeLists.txt +++ b/mooncake-pg/src/CMakeLists.txt @@ -1,13 +1,14 @@ set(_pg_device_source "${CMAKE_CURRENT_SOURCE_DIR}/mooncake_worker.cu") if(USE_CUDA) - # Preserve the existing sm_80/sm_90 coverage on CUDA 12 unless the caller - # selects architectures through CMake's standard variable or CUDAARCHS - # environment. + # Unless the caller selects architectures through CMake's standard variable or + # CUDAARCHS environment, build the same list as mooncake-ep: SASS plus PTX for + # each entry. Without PTX, GPUs outside the list (sm_100, sm_120, ...) cannot + # load any PG kernel ("no kernel image is available"). if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES AND "$ENV{CUDAARCHS}" STREQUAL "") if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "13.0") - set(CMAKE_CUDA_ARCHITECTURES 80-real 90-real 103-real) + set(CMAKE_CUDA_ARCHITECTURES 80 90 103) else() - set(CMAKE_CUDA_ARCHITECTURES 80-real 90-real) + set(CMAKE_CUDA_ARCHITECTURES 80 90) endif() endif() enable_language(CUDA) @@ -148,6 +149,7 @@ add_library( mooncake_pg.cpp mooncake_worker_host.cpp mooncake_worker_thread.cpp + pg_host_ip_resolve.cpp p2p_proxy.cpp) # The initial collective v2 runtime implementation is CUDA-only. @@ -175,9 +177,9 @@ target_include_directories( mooncake_pg PUBLIC $ $) if(USE_MACA) - target_include_directories(mooncake_pg PRIVATE - "${MACA_ROOT}/tools/cu-bridge/include" - "${MACA_ROOT}/include/mcr") + target_include_directories( + mooncake_pg PRIVATE "${MACA_ROOT}/tools/cu-bridge/include" + "${MACA_ROOT}/include/mcr") endif() target_link_libraries(mooncake_pg PRIVATE transfer_engine mooncake_pg_device) diff --git a/mooncake-pg/src/mooncake_communicator.cpp b/mooncake-pg/src/mooncake_communicator.cpp index 205491c7c4..2c2d3d4d4f 100644 --- a/mooncake-pg/src/mooncake_communicator.cpp +++ b/mooncake-pg/src/mooncake_communicator.cpp @@ -16,6 +16,7 @@ #include "error_types.h" #include "gpu_runtime.h" #include "memory_location.h" +#include "pg_host_ip_resolve.h" #if MOONCAKE_PG_HAS_COLLECTIVE_V2 #include "device_comm/device_collective/device_collective.h" #include "device_comm/device_collective/device_collective_workspace.h" @@ -28,6 +29,16 @@ namespace { // A non-zero operation size is required to ensure that the worker creates a // task for the barrier. constexpr size_t kBarrierDummySize = 1; + +PGResult ApplyResolvedHostIp(std::string& host_ip) { + auto resolved = MaybeResolveRoutableHostIpFromEnv(host_ip); + if (resolved.outcome == HostIpResolveOutcome::FailFast) { + return makePGError(PGErrorCode::InvalidArgument, + std::move(resolved.error_message)); + } + host_ip = std::move(resolved.host_ip); + return {}; +} #if MOONCAKE_PG_HAS_COLLECTIVE_V2 constexpr size_t kDefaultDeviceCollectiveBufferSize = 32ull * 1024 * 1024; // 32 MiB @@ -217,6 +228,11 @@ PGResult MooncakePGContext::initialize(int rank, int world_size) { PG_VALIDATE_ARG(std::getenv("MC_MACA_HOST_TRANSPORT") != nullptr, "MACA PG requires MC_MACA_HOST_TRANSPORT=1"); #endif + // P2PHANDSHAKE publishes host_ip: as the segment name. The + // default host_ip is 127.0.0.1, which hangs multi-node jobs that forget + // set_host_ip(); resolve a routable address when MASTER_ADDR indicates + // multi-node (#4981). + PG_TRY(ApplyResolvedHostIp(host_ip)); PG_TRY_TE(engine->init(P2PHANDSHAKE, host_ip)); engine_initialized = true; } @@ -1810,23 +1826,15 @@ void MooncakeCommunicator::applyGroupState( } break; case CollectiveExtensionState::Normal: - // Build the new mask before publishing it. Clearing the shared - // array first creates a window in which a rank that remains active - // can observe its own slot as inactive and reject a collective. - // Write each slot's final value once instead. - { - std::vector next_active_ranks(meta_->maxGroupSize, false); - for (size_t local_rank = 0; local_rank < view.rank_order.size(); - ++local_rank) { - const auto global_rank = view.rank_order[local_rank]; - next_active_ranks[local_rank] = - view.members[global_rank].isActive(); - } - for (int local_rank = 0; local_rank < meta_->maxGroupSize; - ++local_rank) { - meta_->activeRanks[local_rank] = - next_active_ranks[local_rank]; - } + for (int local_rank = 0; local_rank < meta_->maxGroupSize; + ++local_rank) { + meta_->activeRanks[local_rank] = false; + } + for (size_t local_rank = 0; local_rank < view.rank_order.size(); + ++local_rank) { + const auto global_rank = view.rank_order[local_rank]; + meta_->activeRanks[local_rank] = + view.members[global_rank].isActive(); } break; } diff --git a/mooncake-pg/src/pg_host_ip_resolve.cpp b/mooncake-pg/src/pg_host_ip_resolve.cpp new file mode 100644 index 0000000000..fc536543bf --- /dev/null +++ b/mooncake-pg/src/pg_host_ip_resolve.cpp @@ -0,0 +1,116 @@ +#include "pg_host_ip_resolve.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" + +namespace mooncake { + +std::optional LocalIpTowardPeer(const std::string& peer_host) { + if (IsLoopbackHost(peer_host)) return std::nullopt; + + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; + addrinfo* result = nullptr; + if (getaddrinfo(peer_host.c_str(), "1", &hints, &result) != 0 || + result == nullptr) { + return std::nullopt; + } + + std::optional local_ip; + for (addrinfo* rp = result; rp != nullptr; rp = rp->ai_next) { + const int fd = socket(rp->ai_family, SOCK_DGRAM, 0); + if (fd < 0) continue; + if (connect(fd, rp->ai_addr, rp->ai_addrlen) != 0) { + close(fd); + continue; + } + sockaddr_storage local_addr{}; + socklen_t local_len = sizeof(local_addr); + if (getsockname(fd, reinterpret_cast(&local_addr), + &local_len) == 0) { + char buf[INET6_ADDRSTRLEN] = {}; + if (local_addr.ss_family == AF_INET) { + auto* sin = reinterpret_cast(&local_addr); + if (inet_ntop(AF_INET, &sin->sin_addr, buf, sizeof(buf))) { + local_ip = buf; + } + } else if (local_addr.ss_family == AF_INET6) { + auto* sin6 = reinterpret_cast(&local_addr); + if (inet_ntop(AF_INET6, &sin6->sin6_addr, buf, sizeof(buf))) { + local_ip = buf; + } + } + } + close(fd); + if (local_ip && !IsLoopbackHost(*local_ip)) break; + local_ip.reset(); + } + freeaddrinfo(result); + return local_ip; +} + +std::optional LocalIpFromHostname() { + char hostname[256] = {}; + if (gethostname(hostname, sizeof(hostname)) != 0) return std::nullopt; + + addrinfo hints{}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* result = nullptr; + if (getaddrinfo(hostname, nullptr, &hints, &result) != 0 || + result == nullptr) { + return std::nullopt; + } + + std::optional local_ip; + for (addrinfo* rp = result; rp != nullptr; rp = rp->ai_next) { + char buf[INET6_ADDRSTRLEN] = {}; + const void* addr_ptr = nullptr; + if (rp->ai_family == AF_INET) { + addr_ptr = &reinterpret_cast(rp->ai_addr)->sin_addr; + } else if (rp->ai_family == AF_INET6) { + addr_ptr = &reinterpret_cast(rp->ai_addr)->sin6_addr; + } + if (addr_ptr == nullptr) continue; + if (!inet_ntop(rp->ai_family, addr_ptr, buf, sizeof(buf))) continue; + if (IsLoopbackHost(buf)) continue; + local_ip = buf; + if (rp->ai_family == AF_INET) break; + } + freeaddrinfo(result); + return local_ip; +} + +HostIpResolveResult MaybeResolveRoutableHostIpFromEnv(std::string host_ip) { + const char* master_addr = std::getenv("MASTER_ADDR"); + std::string master_host; + if (master_addr && *master_addr) { + master_host = getHostNameWithoutPort(master_addr); + } + + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string& peer) { + return LocalIpTowardPeer(peer); + }; + hooks.local_ip_from_hostname = []() { return LocalIpFromHostname(); }; + + auto result = ResolveRoutableHostIp(std::move(host_ip), master_host, hooks); + if (result.outcome == HostIpResolveOutcome::Resolved) { + LOG(INFO) << "Mooncake PG: derived host_ip=" << result.host_ip + << " (MASTER_ADDR host=" + << (master_host.empty() ? "" : master_host) << ")"; + } + return result; +} + +} // namespace mooncake diff --git a/mooncake-pg/tests/CMakeLists.txt b/mooncake-pg/tests/CMakeLists.txt new file mode 100644 index 0000000000..0da26f61d3 --- /dev/null +++ b/mooncake-pg/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +# Header-only host_ip resolve policy tests (no CUDA / mooncake_pg link). +add_executable(pg_host_ip_resolve_test pg_host_ip_resolve_test.cpp) +target_include_directories( + pg_host_ip_resolve_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_SOURCE_DIR}/../../mooncake-transfer-engine/include) +target_compile_features(pg_host_ip_resolve_test PRIVATE cxx_std_20) +target_link_libraries(pg_host_ip_resolve_test PUBLIC gtest gtest_main) +add_test(NAME pg_host_ip_resolve_test COMMAND pg_host_ip_resolve_test) diff --git a/mooncake-pg/tests/pg_host_ip_resolve_test.cpp b/mooncake-pg/tests/pg_host_ip_resolve_test.cpp new file mode 100644 index 0000000000..2ab3bf35da --- /dev/null +++ b/mooncake-pg/tests/pg_host_ip_resolve_test.cpp @@ -0,0 +1,158 @@ +#include +#include + +#include + +#include "pg_host_ip_resolve.h" + +namespace mooncake { +namespace { + +TEST(PgHostIpResolveTest, IsLoopbackHostTreatsWhole127Slash8) { + EXPECT_TRUE(IsLoopbackHost("")); + EXPECT_TRUE(IsLoopbackHost("localhost")); + EXPECT_TRUE(IsLoopbackHost("127.0.0.1")); + EXPECT_TRUE(IsLoopbackHost("127.0.1.1")); + EXPECT_TRUE(IsLoopbackHost("127.255.255.255")); + EXPECT_TRUE(IsLoopbackHost("::1")); + EXPECT_TRUE(IsLoopbackHost("::ffff:127.0.0.1")); + EXPECT_TRUE(IsLoopbackHost("::ffff:127.0.1.1")); + + EXPECT_FALSE(IsLoopbackHost("10.0.0.1")); + EXPECT_FALSE(IsLoopbackHost("192.168.1.10")); + EXPECT_FALSE(IsLoopbackHost("8.8.8.8")); + EXPECT_FALSE(IsLoopbackHost("::2")); + EXPECT_FALSE(IsLoopbackHost("node0")); + EXPECT_FALSE(IsLoopbackHost("master.example.com")); +} + +TEST(PgHostIpResolveTest, PreservesSingleNodeLoopbackDefault) { + int toward_calls = 0; + int hostname_calls = 0; + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [&](const std::string&) { + ++toward_calls; + return std::optional{"10.0.0.8"}; + }; + hooks.local_ip_from_hostname = [&]() { + ++hostname_calls; + return std::optional{"10.0.0.9"}; + }; + + // Torchrun single-node: MASTER_ADDR is loopback. + auto result = ResolveRoutableHostIp("127.0.0.1", "127.0.0.1", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Unchanged); + EXPECT_EQ(result.host_ip, "127.0.0.1"); + EXPECT_EQ(toward_calls, 0); + EXPECT_EQ(hostname_calls, 0); + + // MASTER_ADDR unset. + result = ResolveRoutableHostIp("127.0.0.1", "", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Unchanged); + EXPECT_EQ(result.host_ip, "127.0.0.1"); + EXPECT_EQ(toward_calls, 0); + EXPECT_EQ(hostname_calls, 0); + + // Debian/Ubuntu /etc/hosts style 127.0.1.1 is still single-node. + result = ResolveRoutableHostIp("127.0.0.1", "127.0.1.1", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Unchanged); + EXPECT_EQ(result.host_ip, "127.0.0.1"); + EXPECT_EQ(toward_calls, 0); + EXPECT_EQ(hostname_calls, 0); +} + +TEST(PgHostIpResolveTest, PrefersRouteTowardMasterAddr) { + int hostname_calls = 0; + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string& peer) { + EXPECT_EQ(peer, "192.168.1.10"); + return std::optional{"192.168.1.20"}; + }; + hooks.local_ip_from_hostname = [&]() { + ++hostname_calls; + return std::optional{"10.0.0.1"}; + }; + + auto result = ResolveRoutableHostIp("127.0.0.1", "192.168.1.10", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Resolved); + EXPECT_EQ(result.host_ip, "192.168.1.20"); + EXPECT_EQ(hostname_calls, 0); +} + +TEST(PgHostIpResolveTest, HostnameFallbackWhenRouteUnavailable) { + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string&) { + return std::nullopt; + }; + hooks.local_ip_from_hostname = []() { + return std::optional{"10.1.2.3"}; + }; + + auto result = ResolveRoutableHostIp("127.0.0.1", "10.9.8.7", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Resolved); + EXPECT_EQ(result.host_ip, "10.1.2.3"); +} + +TEST(PgHostIpResolveTest, ExplicitHostIpTakesPrecedence) { + int toward_calls = 0; + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [&](const std::string&) { + ++toward_calls; + return std::optional{"192.168.0.2"}; + }; + + // set_host_ip() already replaced the default before init. + auto result = ResolveRoutableHostIp("10.0.0.5", "192.168.1.10", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Unchanged); + EXPECT_EQ(result.host_ip, "10.0.0.5"); + EXPECT_EQ(toward_calls, 0); +} + +TEST(PgHostIpResolveTest, FailFastWhenMultiNodeCannotResolve) { + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string&) { + return std::nullopt; + }; + hooks.local_ip_from_hostname = []() { return std::nullopt; }; + + auto result = ResolveRoutableHostIp("127.0.0.1", "203.0.113.1", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::FailFast); + EXPECT_EQ(result.host_ip, "127.0.0.1"); + EXPECT_NE(result.error_message.find("set_host_ip"), std::string::npos); + EXPECT_NE(result.error_message.find("203.0.113.1"), std::string::npos); +} + +TEST(PgHostIpResolveTest, FailFastWhenHostnameFallbackIsLoopback) { + // Ubuntu/Debian /etc/hosts often maps the hostname to 127.0.1.1. That + // must not be published as host_ip; fall through to FailFast instead. + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string&) { + return std::nullopt; + }; + hooks.local_ip_from_hostname = []() { + return std::optional{"127.0.1.1"}; + }; + + auto result = ResolveRoutableHostIp("127.0.0.1", "node0", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::FailFast); + EXPECT_EQ(result.host_ip, "127.0.0.1"); + EXPECT_NE(result.error_message.find("set_host_ip"), std::string::npos); + EXPECT_NE(result.error_message.find("node0"), std::string::npos); +} + +TEST(PgHostIpResolveTest, IgnoresLoopbackRouteThenUsesHostname) { + HostIpResolveHooks hooks; + hooks.local_ip_toward_peer = [](const std::string&) { + return std::optional{"127.0.1.1"}; + }; + hooks.local_ip_from_hostname = []() { + return std::optional{"10.2.3.4"}; + }; + + auto result = ResolveRoutableHostIp("127.0.0.1", "node0", hooks); + EXPECT_EQ(result.outcome, HostIpResolveOutcome::Resolved); + EXPECT_EQ(result.host_ip, "10.2.3.4"); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 26325aab47..aa53a1613e 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -63,14 +63,12 @@ namespace mooncake { -class EtcdOpLogStore; class ShardAllocator; class DfsAllocatorInterface; class ImmutableBucketAllocator; // Forward declarations class AllocationStrategy; -class EvictionStrategy; class HaKvBackend; class HttpMetadataServer; class OpLogBatchStorage; @@ -360,22 +358,15 @@ class MasterService { * @param segment_name The name of the segment (storage device) to clear * from. If empty, clears replicas from all segments for the given * client_id. + * @param tenant_id The tenant that owns the keys. * @return An expected object containing a vector of successfully cleared * keys on success, or an ErrorCode on failure. Only successfully * cleared keys are included in the result. */ - // Existing key-only overload (signature unchanged): kept for legacy - // callers; delegates with "default". - auto BatchReplicaClear(const std::vector& object_keys, - const UUID& client_id, - const std::string& segment_name) - -> tl::expected, ErrorCode>; - - // New: tenant-aware overload - auto BatchReplicaClear(const std::vector& object_keys, - const UUID& client_id, - const std::string& segment_name, - const std::string& tenant_id) + auto BatchReplicaClear( + const std::vector& object_keys, const UUID& client_id, + const std::string& segment_name, + const std::string& tenant_id = std::string(TenantId::kDefaultValue)) -> tl::expected, ErrorCode>; /** @@ -880,13 +871,6 @@ class MasterService { const std::string& key, const TenantId& tenant_id, const std::vector& targets); - /** - * @brief Submit a dynamic replica action proposal after Master-side - * hotness admission. - */ - tl::expected SubmitReplicaActionProposal( - const ReplicaActionProposal& proposal); - /** * @brief Create a move task to move an object's replica from source segment * to target segment @@ -1696,11 +1680,10 @@ class MasterService { } // Holds the key only while it holds an object. - MetadataAccessorRW(MasterService* service, ObjectIdentity object_id, - bool cleanup_invalid = true) + MetadataAccessorRW(MasterService* service, ObjectIdentity object_id) : MetadataAccessorRW(service, std::move(object_id), AsIs{}) { // Invalid memory replicas go first, which may tear the object down. - if (cleanup_invalid && IsPublished()) { + if (IsPublished()) { service_->CleanupInvalidMemoryReplicas(*this); } } @@ -1873,9 +1856,6 @@ class MasterService { std::optional guard_; }; - friend class MetadataAccessorRW; - friend class MetadataAccessorRO; - // Calls `visit` with each namespace that holds an object when the walk // starts, holding no lock, so `visit` may reach any key. A namespace that // gains its first object meanwhile may or may not be visited. Its objects @@ -1949,13 +1929,9 @@ class MasterService { const uint32_t nof_heartbeat_failures_threshold_; struct NoFHeartbeatState { - UUID owner_client_id{0, 0}; - std::string segment_name; - std::string te_endpoint; std::chrono::steady_clock::time_point next_probe_at{}; std::chrono::steady_clock::time_point last_success_at{}; uint32_t consecutive_failures{0}; - std::string last_error_reason; }; std::mutex nof_heartbeat_mutex_; std::unordered_map> @@ -2195,8 +2171,6 @@ class MasterService { static constexpr uint32_t kMaxDrainUnitRetries = 3; - tl::expected ValidateDrainRequest( - const CreateDrainJobRequest& request); tl::expected ValidateDrainRequestLocked( ScopedSegmentAccess& segment_access, const CreateDrainJobRequest& request); @@ -2332,8 +2306,6 @@ class MasterService { // OpLog publishing helpers std::string SerializeMetadataForOpLog(const ObjectMetadata& metadata) const; - std::string SerializeMetadataForOpLogWithoutMemReplicas( - const ObjectMetadata& metadata) const; std::string SerializeMetadataForOpLogFromReplicaDescriptors( const ObjectMetadata& metadata, const std::vector& replicas) const; @@ -2382,27 +2354,20 @@ class MasterService { bool IsEvictableMemoryReplica(const Replica& replica) const; /** - * Segment lifecycle persist helper. Tries to durably persist the - * SEGMENT_MOUNT / SEGMENT_UNMOUNT entry up-front; on failure enqueues - * the same OpLogEntry (with its already-allocated sequence_id) for - * background retry so the standby segment registry eventually - * converges. Suitable for paths where the local segment commit has - * already happened (UnmountSegment) and rolling back is impossible. + * Appends a SEGMENT_MOUNT / SEGMENT_UNMOUNT entry without waiting for it + * to become durable. The local segment change has already happened and + * cannot be rolled back, so a failed append is only logged; nothing + * retries it. */ - void PersistSegmentOpForHAOrEnqueue(const char* why, OpType type, - const std::string& key, - const std::string& payload); - void PersistSegmentOpForHAOrEnqueue(const char* why, OpType type, - const TenantId& tenant_id, - const std::string& key, - const std::string& payload); + void PersistSegmentOpForHA(const char* why, OpType type, + const std::string& key, + const std::string& payload); /** - * Helper to persist REMOVE OpLog for a key with strong-consistency. - * @return OK on success, error on persist failure (caller must skip erase) + * Appends a REMOVE entry for the key, without waiting for it to become + * durable. + * @return OK on success, error on append failure (caller must skip erase) */ - tl::expected PersistRemoveForHA(const char* why, - const std::string& key); tl::expected PersistRemoveForHA(const char* why, const TenantId& tenant_id, const std::string& key); diff --git a/mooncake-store/include/object_runtime_state.h b/mooncake-store/include/object_runtime_state.h index ff61ae6a02..30a0c9f76b 100644 --- a/mooncake-store/include/object_runtime_state.h +++ b/mooncake-store/include/object_runtime_state.h @@ -88,12 +88,6 @@ struct PromotionCandidate { // consumption; reset only when a genuinely new chain starts (fresh // insert with 0, e.g. after a give-up or a success). uint32_t execution_failures{0}; - // True when the task was registered by a prefetch caller - // (RegisterPrefetchTask) rather than promotion-on-hit admission. Same - // execution chain, but failure handling differs: a prefetch failure is - // retried by the client-side throttle, not re-recorded as an on-hit - // candidate. - bool from_prefetch{false}; }; // NotifyPromotionSuccess should commit, so a concurrent Put on the diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 8f03a71621..fdc34d8cb4 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -63,13 +62,6 @@ namespace mooncake { namespace { -uint64_t SaturatingAdd(uint64_t lhs, uint64_t rhs) { - if (lhs > std::numeric_limits::max() - rhs) { - return std::numeric_limits::max(); - } - return lhs + rhs; -} - uint64_t SaturatingMultiply(uint64_t lhs, uint64_t rhs) { if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) { return std::numeric_limits::max(); @@ -975,9 +967,9 @@ auto MasterService::MountSegment(const Segment& segment, const UUID& client_id) op.is_memory_segment = true; op.file_path.clear(); auto bytes = struct_pack::serialize(op); - PersistSegmentOpForHAOrEnqueue("MountSegment", OpType::SEGMENT_MOUNT, - segment.te_endpoint, - std::string(bytes.begin(), bytes.end())); + PersistSegmentOpForHA("MountSegment", OpType::SEGMENT_MOUNT, + segment.te_endpoint, + std::string(bytes.begin(), bytes.end())); } if (mount_result == ErrorCode::OK) { UpdateClientHostId(client_id, segment.host_id); @@ -1372,9 +1364,9 @@ auto MasterService::ReMountSegment(const std::vector& segments, op.is_memory_segment = true; op.file_path.clear(); auto bytes = struct_pack::serialize(op); - PersistSegmentOpForHAOrEnqueue( - "ReMountSegment", OpType::SEGMENT_MOUNT, seg.name, - std::string(bytes.begin(), bytes.end())); + PersistSegmentOpForHA("ReMountSegment", OpType::SEGMENT_MOUNT, + seg.name, + std::string(bytes.begin(), bytes.end())); } } NotifyPolicyCapacityChanged(); @@ -2733,9 +2725,9 @@ auto MasterService::UnmountSegment(const UUID& segment_id, if (enable_oplog_ && ordered_oplog_writer_ && !te_endpoint.empty()) { SegmentUnmountOp op{te_endpoint}; auto bytes = struct_pack::serialize(op); - PersistSegmentOpForHAOrEnqueue("UnmountSegment", - OpType::SEGMENT_UNMOUNT, te_endpoint, - std::string(bytes.begin(), bytes.end())); + PersistSegmentOpForHA("UnmountSegment", OpType::SEGMENT_UNMOUNT, + te_endpoint, + std::string(bytes.begin(), bytes.end())); } NotifyPolicyCapacityChanged(); return {}; @@ -3793,12 +3785,7 @@ tl::expected MasterService::RestoreFromStandbyStateCore( } } - const bool bounded_restore = metadata_store != nullptr; - // Batch restore spans chunks, so it needs an ordered cross-chunk index; - // legacy restore keeps its contiguous vector-and-sort validation path. - std::unordered_map>> - legacy_memory_ranges; + // Batch restore spans chunks, so it needs an ordered cross-chunk index. std::unordered_map> bounded_memory_ranges; std::unordered_map restored_accounted_memory_bytes; @@ -3900,29 +3887,21 @@ tl::expected MasterService::RestoreFromStandbyStateCore( return tl::make_unexpected( ErrorCode::INVALID_PARAMS); } - if (bounded_restore) { - auto& ranges = bounded_memory_ranges[segment]; - auto next = - ranges.lower_bound(buffer.buffer_address_); - const uintptr_t end = - buffer.buffer_address_ + buffer.size_; - if ((next != ranges.end() && end > next->first) || - (next != ranges.begin() && - std::prev(next)->first + - std::prev(next)->second > - buffer.buffer_address_)) { - LOG(ERROR) << "RestoreFromStandbySnapshot: " - "overlapping " - "memory descriptors"; - return tl::make_unexpected( - ErrorCode::INVALID_PARAMS); - } - ranges.emplace(buffer.buffer_address_, - buffer.size_); - } else { - legacy_memory_ranges[segment].emplace_back( - buffer.buffer_address_, buffer.size_); + auto& ranges = bounded_memory_ranges[segment]; + auto next = ranges.lower_bound(buffer.buffer_address_); + const uintptr_t end = + buffer.buffer_address_ + buffer.size_; + if ((next != ranges.end() && end > next->first) || + (next != ranges.begin() && + std::prev(next)->first + std::prev(next)->second > + buffer.buffer_address_)) { + LOG(ERROR) << "RestoreFromStandbySnapshot: " + "overlapping " + "memory descriptors"; + return tl::make_unexpected( + ErrorCode::INVALID_PARAMS); } + ranges.emplace(buffer.buffer_address_, buffer.size_); bytes += buffer.size_; } @@ -4001,22 +3980,6 @@ tl::expected MasterService::RestoreFromStandbyStateCore( std::move(user_key), std::move(replicas)}); } - if (!bounded_restore) { - for (auto& [segment, ranges] : legacy_memory_ranges) { - (void)segment; - std::sort(ranges.begin(), ranges.end()); - for (size_t i = 1; i < ranges.size(); ++i) { - if (ranges[i].first < - ranges[i - 1].first + ranges[i - 1].second) { - LOG(ERROR) - << "RestoreFromStandbySnapshot: overlapping memory " - "descriptors"; - return tl::make_unexpected(ErrorCode::INVALID_PARAMS); - } - } - } - } - if (auto published = PublishRestoredObjects(prepared, installed); !published) { return tl::make_unexpected(published.error()); @@ -4305,13 +4268,6 @@ auto MasterService::BatchQueryIp(const std::vector& client_ids) return results; } -auto MasterService::BatchReplicaClear( - const std::vector& object_keys, const UUID& client_id, - const std::string& segment_name) - -> tl::expected, ErrorCode> { - return BatchReplicaClear(object_keys, client_id, segment_name, "default"); -} - auto MasterService::BatchReplicaClear( const std::vector& object_keys, const UUID& client_id, const std::string& segment_name, const std::string& tenant_id) @@ -8652,13 +8608,6 @@ void MasterService::CleanupExpiredDynamicReplicationState() { dynamic_replication_.EraseExpiredLeases(std::chrono::system_clock::now()); } -tl::expected -MasterService::SubmitReplicaActionProposal( - const ReplicaActionProposal& proposal) { - std::shared_lock shared_lock(master_state_mutex_); - return SubmitReplicaActionProposalLocked(proposal); -} - tl::expected MasterService::SubmitReplicaActionProposalLocked( const ReplicaActionProposal& proposal) { @@ -10221,9 +10170,6 @@ void MasterService::NofHeartbeatThreadFunc() { auto [it, inserted] = nof_heartbeat_states_.try_emplace(snapshot.segment_id); auto& state = it->second; - state.owner_client_id = snapshot.client_id; - state.segment_name = snapshot.segment.name; - state.te_endpoint = snapshot.segment.te_endpoint; if (inserted) { int64_t spread_ms = 0; if (!ok_segments.empty()) { @@ -10291,7 +10237,6 @@ void MasterService::NofHeartbeatThreadFunc() { if (it != nof_heartbeat_states_.end()) { it->second.consecutive_failures = 0; it->second.last_success_at = success_time; - it->second.last_error_reason.clear(); it->second.next_probe_at = success_time + nof_heartbeat_interval_sec_; } @@ -10321,7 +10266,6 @@ void MasterService::NofHeartbeatThreadFunc() { if (it != nof_heartbeat_states_.end()) { it->second.consecutive_failures++; failure_count = it->second.consecutive_failures; - it->second.last_error_reason = error_reason; it->second.next_probe_at = failure_time + nof_heartbeat_interval_sec_; should_unmount = @@ -10663,12 +10607,6 @@ tl::expected MasterService::MarkTaskToComplete( return {}; } -tl::expected MasterService::ValidateDrainRequest( - const CreateDrainJobRequest& request) { - ScopedSegmentAccess segment_access = segment_manager_.getSegmentAccess(); - return ValidateDrainRequestLocked(segment_access, request); -} - tl::expected MasterService::ValidateDrainRequestLocked( ScopedSegmentAccess& segment_access, const CreateDrainJobRequest& request) { if (request.segments.empty() || request.max_concurrency == 0) { @@ -11533,59 +11471,20 @@ void MasterService::HttpMetadataCleanupThreadFunc() { std::string MasterService::SerializeMetadataForOpLog( const ObjectMetadata& metadata) const { - MetadataPayload payload; - payload.client_id = metadata.client_id; - payload.size = metadata.TotalLength(); - payload.group_id = metadata.group_id; - payload.session_id = metadata.session_id; - payload.data_type = metadata.data_type; - payload.hard_pinned = metadata.IsHardPinned(); - payload.object_checksum = metadata.GetPart(0).checksum(); - - // Extract replica descriptors - get them all at once - const auto& replicas = metadata.GetAllReplicas(); - payload.replicas.reserve(replicas.size()); - for (const auto& replica : replicas) { - payload.replicas.push_back(replica.get_descriptor()); - } - - // NOTE: Lease information is NOT serialized because: - // 1. Standby does not perform eviction, so lease info is not used - // 2. After promotion, new Primary should grant fresh leases, not restore - // old ones - - // Serialize using struct_pack (msgpack binary format) - auto result = struct_pack::serialize(payload); - return std::string(result.begin(), result.end()); -} - -std::string MasterService::SerializeMetadataForOpLogWithoutMemReplicas( - const ObjectMetadata& metadata) const { - MetadataPayload payload; - payload.client_id = metadata.client_id; - payload.size = metadata.TotalLength(); - payload.group_id = metadata.group_id; - payload.session_id = metadata.session_id; - payload.data_type = metadata.data_type; - payload.hard_pinned = metadata.IsHardPinned(); - payload.object_checksum = metadata.GetPart(0).checksum(); - - const auto& replicas = metadata.GetAllReplicas(); - payload.replicas.reserve(replicas.size()); - for (const auto& replica : replicas) { - if (replica.type() == ReplicaType::MEMORY) { - continue; - } - payload.replicas.push_back(replica.get_descriptor()); + const auto all_replicas = metadata.GetAllReplicas(); + std::vector replicas; + replicas.reserve(all_replicas.size()); + for (const auto& replica : all_replicas) { + replicas.push_back(replica.get_descriptor()); } - - auto result = struct_pack::serialize(payload); - return std::string(result.begin(), result.end()); + return SerializeMetadataForOpLogFromReplicaDescriptors(metadata, replicas); } std::string MasterService::SerializeMetadataForOpLogFromReplicaDescriptors( const ObjectMetadata& metadata, const std::vector& replicas) const { + // Leases are not serialized: a standby does not evict, and a promoted + // primary grants fresh leases instead of restoring old ones. MetadataPayload payload; payload.client_id = metadata.client_id; payload.size = metadata.TotalLength(); @@ -11787,11 +11686,6 @@ MasterService::AppendReservedOpLogWithDurableFinalize( return entry; } -tl::expected MasterService::PersistRemoveForHA( - const char* why, const std::string& key) { - return PersistRemoveForHA(why, TenantId::Default(), key); -} - tl::expected MasterService::PersistRemoveForHA( const char* why, const TenantId& tenant_id, const std::string& key) { auto result = AppendOpLogWithDurableFinalize( @@ -11804,19 +11698,11 @@ tl::expected MasterService::PersistRemoveForHA( return {}; } -void MasterService::PersistSegmentOpForHAOrEnqueue(const char* why, OpType type, - const std::string& key, - const std::string& payload) { - PersistSegmentOpForHAOrEnqueue(why, type, TenantId::Default(), key, - payload); -} - -void MasterService::PersistSegmentOpForHAOrEnqueue(const char* why, OpType type, - const TenantId& tenant_id, - const std::string& key, - const std::string& payload) { - auto result = - AppendOpLogVisibleBeforeDurable(type, tenant_id.value(), key, payload); +void MasterService::PersistSegmentOpForHA(const char* why, OpType type, + const std::string& key, + const std::string& payload) { + auto result = AppendOpLogVisibleBeforeDurable( + type, TenantId::Default().value(), key, payload); if (!result) { LOG(WARNING) << why << ": segment OpLog queue failed for key=" << key << ", type=" << static_cast(type) diff --git a/mooncake-store/tests/dfs_sync_client_test.cpp b/mooncake-store/tests/dfs_sync_client_test.cpp index 0be16160c9..165fa827a5 100644 --- a/mooncake-store/tests/dfs_sync_client_test.cpp +++ b/mooncake-store/tests/dfs_sync_client_test.cpp @@ -627,6 +627,9 @@ TEST_F(DfsSyncClientTest, DfsOnlyBatchFailureOnlyRevokesFailedKey) { ExpectDfsOnlyValue(keys[0], values[0]); ExpectNotFound(keys[1]); + // Restore writes after the persistent failure before retrying the key. + adapter_->FailWriteCall(-1); + // The revoked key and its extent can be reused. std::vector retry{{values[1].data(), values[1].size()}}; ASSERT_TRUE(writer_->Put(keys[1], retry, DfsOnlyConfig()).has_value()); diff --git a/mooncake-store/tests/dynamic_replication_test.cpp b/mooncake-store/tests/dynamic_replication_test.cpp index 06d6da7fee..34b373caf1 100644 --- a/mooncake-store/tests/dynamic_replication_test.cpp +++ b/mooncake-store/tests/dynamic_replication_test.cpp @@ -302,8 +302,8 @@ TEST_F(DynamicReplicationTest, ASSERT_TRUE(no_task.has_value()); EXPECT_TRUE(no_task->empty()); - auto lease = - service.SubmitReplicaActionProposal(BuildProposal(service, "hot-key")); + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( + BuildProposal(service, "hot-key")); ASSERT_TRUE(lease.has_value()); auto tasks = WaitForTasks(service, source.client_id, 1); ASSERT_EQ(tasks.size(), 1u); @@ -413,8 +413,8 @@ TEST_F(DynamicReplicationTest, MaxReplicaLimitSuppressesFanout) { (void)PrepareSegment(service, "segment_1", 0x800000000); PutObject(service, source.client_id, "max-key", source.segment_name); - auto lease = - service.SubmitReplicaActionProposal(BuildProposal(service, "max-key")); + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( + BuildProposal(service, "max-key")); ASSERT_FALSE(lease.has_value()); auto tasks = service.FetchTasks(source.client_id, 16); ASSERT_TRUE(tasks.has_value()); @@ -434,7 +434,8 @@ TEST_F(DynamicReplicationTest, BelowFrequencyThresholdDoesNotQueueCopy) { PutObject(service, source.client_id, "warm-key", source.segment_name); auto proposal = BuildProposal(service, "warm-key", std::nullopt, false); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_FALSE(lease.has_value()); auto tasks = service.FetchTasks(source.client_id, 16); @@ -458,7 +459,8 @@ TEST_F(DynamicReplicationTest, RejectDomainHintsBeforeDomainAwarePlacement) { auto proposal = BuildProposal(service, "domain-hint-key"); proposal.target_domain = "domain-a"; - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_FALSE(lease.has_value()); EXPECT_EQ(lease.error(), ErrorCode::INVALID_PARAMS); } @@ -484,7 +486,8 @@ TEST_F(DynamicReplicationTest, LeaseDeadlineDoesNotExceedProposalDeadline) { .count()) + std::chrono::seconds(1).count() * 1000; - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); EXPECT_LE(lease->expire_at_ms_epoch, proposal.expire_at_ms_epoch); } @@ -500,9 +503,11 @@ TEST_F(DynamicReplicationTest, ProposalIdempotencyReturnsSameLease) { PutObject(service, source.client_id, "proposal-key", source.segment_name); auto proposal = BuildProposal(service, "proposal-key", target.segment_name); - auto first = service.SubmitReplicaActionProposal(proposal); + auto first = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(first.has_value()); - auto second = service.SubmitReplicaActionProposal(proposal); + auto second = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(second.has_value()); EXPECT_EQ(second->proposal_id, first->proposal_id); @@ -529,12 +534,14 @@ TEST_F(DynamicReplicationTest, ProposalIdempotencyRejectsConflictingRequest) { PutObject(service, source.client_id, "proposal-key", source.segment_name); auto proposal = BuildProposal(service, "proposal-key", target.segment_name); - auto first = service.SubmitReplicaActionProposal(proposal); + auto first = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(first.has_value()); auto conflicting = proposal; conflicting.preferred_target_segment = other_target.segment_name; - auto second = service.SubmitReplicaActionProposal(conflicting); + auto second = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(conflicting); ASSERT_FALSE(second.has_value()); EXPECT_EQ(second.error(), ErrorCode::INVALID_PARAMS); } @@ -550,7 +557,8 @@ TEST_F(DynamicReplicationTest, CopyLifecycleMarksDynamicReplicaComplete) { PutObject(service, source.client_id, "copy-key", source.segment_name); auto proposal = BuildProposal(service, "copy-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart(source.client_id, "copy-key", TenantId::Default(), @@ -585,7 +593,8 @@ TEST_F(DynamicReplicationTest, auto proposal = BuildProposal(service, "invalid-target-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart( @@ -660,7 +669,7 @@ TEST_F(DynamicReplicationTest, PutObject(service, source.client_id, "stale-dynamic-task-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "stale-dynamic-task-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); auto tasks = service.FetchTasks(source.client_id, 16); @@ -699,7 +708,7 @@ TEST_F(DynamicReplicationTest, DynamicCopyStartRejectsNonSourceClient) { PutObject(service, source.client_id, "source-client-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "source-client-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); @@ -733,7 +742,7 @@ TEST_F(DynamicReplicationTest, DynamicCopyStartFailureClearsPending) { PutObject(service, source.client_id, "copy-start-fail-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "copy-start-fail-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); ASSERT_TRUE(service.UnmountSegment(target.segment_id, target.client_id) @@ -768,8 +777,10 @@ TEST_F(DynamicReplicationTest, PutObject(service, source.client_id, "stale-vs-current-key", source.segment_name); - auto stale_lease = service.SubmitReplicaActionProposal(BuildProposal( - service, "stale-vs-current-key", stale_target.segment_name)); + auto stale_lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal( + BuildProposal(service, "stale-vs-current-key", + stale_target.segment_name)); ASSERT_TRUE(stale_lease.has_value()); auto stale_tasks = service.FetchTasks(source.client_id, 16); ASSERT_TRUE(stale_tasks.has_value()); @@ -779,8 +790,10 @@ TEST_F(DynamicReplicationTest, ClearDynamicReplicationState(service, "stale-vs-current-key"); - auto current_lease = service.SubmitReplicaActionProposal(BuildProposal( - service, "stale-vs-current-key", current_target.segment_name)); + auto current_lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal( + BuildProposal(service, "stale-vs-current-key", + current_target.segment_name)); ASSERT_TRUE(current_lease.has_value()); auto stale_copy_start = service.CopyStart( @@ -816,7 +829,7 @@ TEST_F(DynamicReplicationTest, ExpiredDynamicCopyTaskClearsDynamicState) { PutObject(service, source.client_id, "expired-copy-task-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "expired-copy-task-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart( @@ -833,7 +846,7 @@ TEST_F(DynamicReplicationTest, ExpiredDynamicCopyTaskClearsDynamicState) { EXPECT_FALSE( HasDynamicState(service, "expired-copy-task-key", lease->proposal_id)); - auto retry = service.SubmitReplicaActionProposal( + auto retry = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "expired-copy-task-key", target.segment_name)); ASSERT_FALSE(retry.has_value()); EXPECT_EQ(retry.error(), ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS); @@ -850,7 +863,7 @@ TEST_F(DynamicReplicationTest, DynamicCopyEndRejectsMismatchedLease) { PutObject(service, source.client_id, "copy-end-fence-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "copy-end-fence-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart( @@ -886,7 +899,7 @@ TEST_F(DynamicReplicationTest, DynamicCopyRevokeRejectsMismatchedLease) { PutObject(service, source.client_id, "copy-revoke-fence-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "copy-revoke-fence-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart( @@ -923,7 +936,8 @@ TEST_F(DynamicReplicationTest, CopyStartRejectsVersionMismatch) { PutObject(service, source.client_id, "version-key", source.segment_name); auto proposal = BuildProposal(service, "version-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); BumpVersionEpoch(service, "version-key"); @@ -948,7 +962,7 @@ TEST_F(DynamicReplicationTest, ExpiredDynamicPendingDoesNotBlockRegularCopy) { PutObject(service, source.client_id, "expired-pending-regular-copy-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "expired-pending-regular-copy-key", expired_target.segment_name)); ASSERT_TRUE(lease.has_value()); @@ -978,7 +992,8 @@ TEST_F(DynamicReplicationTest, ExpiredLeaseRejectsCopyStart) { PutObject(service, source.client_id, "expired-key", source.segment_name); auto proposal = BuildProposal(service, "expired-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); ExpireDynamicPending(service, "expired-key"); @@ -1001,7 +1016,7 @@ TEST_F(DynamicReplicationTest, CleanupFailsTheTaskOfAnExpiredPendingProposal) { auto source = PrepareSegment(service, "segment_0", 0xf00000000); auto target = PrepareSegment(service, "segment_1", 0x1000000000); PutObject(service, source.client_id, "cleanup-key", source.segment_name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "cleanup-key", target.segment_name)); ASSERT_TRUE(lease.has_value()); @@ -1033,7 +1048,8 @@ TEST_F(DynamicReplicationTest, EvictedDynamicReplicaBlocksImmediateRecreate) { auto proposal = BuildProposal(service, "evicted-dynamic-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); auto copy_start = service.CopyStart( source.client_id, "evicted-dynamic-key", TenantId::Default(), @@ -1053,7 +1069,8 @@ TEST_F(DynamicReplicationTest, EvictedDynamicReplicaBlocksImmediateRecreate) { auto immediate_recreate = BuildProposal(service, "evicted-dynamic-key", next_target.segment_name); - auto rejected = service.SubmitReplicaActionProposal(immediate_recreate); + auto rejected = MasterServiceTestPeer(service).SubmitReplicaActionProposal( + immediate_recreate); ASSERT_FALSE(rejected.has_value()); EXPECT_EQ(rejected.error(), ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS); } @@ -1080,7 +1097,8 @@ TEST_F(DynamicReplicationTest, SourceSelectionUsesStableTieBreak) { auto proposal = BuildProposal(service, "source-fanout-key", target.segment_name); - auto lease = service.SubmitReplicaActionProposal(proposal); + auto lease = + MasterServiceTestPeer(service).SubmitReplicaActionProposal(proposal); ASSERT_TRUE(lease.has_value()); auto stable_score = [](std::string_view key, std::string_view segment) { @@ -1141,7 +1159,7 @@ TEST_F(DynamicReplicationTest, TargetSelectionPrefersDifferentHost) { service.MountSegment(other_host_target, generate_uuid()).has_value()); PutObject(service, source_client, "target-host-key", source_segment.name); - auto lease = service.SubmitReplicaActionProposal( + auto lease = MasterServiceTestPeer(service).SubmitReplicaActionProposal( BuildProposal(service, "target-host-key")); ASSERT_TRUE(lease.has_value()); EXPECT_EQ(lease->target_segment, other_host_target.name); diff --git a/mooncake-store/tests/master_service/master_service_test_peer.cpp b/mooncake-store/tests/master_service/master_service_test_peer.cpp index 605f4441f8..37e9e2a0af 100644 --- a/mooncake-store/tests/master_service/master_service_test_peer.cpp +++ b/mooncake-store/tests/master_service/master_service_test_peer.cpp @@ -27,12 +27,6 @@ std::string MasterServiceTestPeer::SerializeMetadataForOpLogForTesting( return service.SerializeMetadataForOpLog(metadata); } -std::string -MasterServiceTestPeer::SerializeMetadataForOpLogWithoutMemReplicasForTesting( - MasterService& service, const ObjectMetadata& metadata) { - return service.SerializeMetadataForOpLogWithoutMemReplicas(metadata); -} - std::string MasterServiceTestPeer:: SerializeMetadataForOpLogFromReplicaDescriptorsForTesting( MasterService& service, const ObjectMetadata& metadata, diff --git a/mooncake-store/tests/master_service/master_service_test_peer.h b/mooncake-store/tests/master_service/master_service_test_peer.h index fcb2fbd9f3..0f89f56ec2 100644 --- a/mooncake-store/tests/master_service/master_service_test_peer.h +++ b/mooncake-store/tests/master_service/master_service_test_peer.h @@ -53,8 +53,6 @@ class MasterServiceTestPeer { // the fields that payload carries. static std::string SerializeMetadataForOpLogForTesting( MasterService& service, const ObjectMetadata& metadata); - static std::string SerializeMetadataForOpLogWithoutMemReplicasForTesting( - MasterService& service, const ObjectMetadata& metadata); static std::string SerializeMetadataForOpLogFromReplicaDescriptorsForTesting( MasterService& service, const ObjectMetadata& metadata, @@ -445,6 +443,14 @@ class MasterServiceTestPeer { return DynamicReplicationController::VersionEpoch(metadata); } + // Submits a dynamic replica proposal under the shared state lock that + // SubmitReplicaActionProposalLocked expects. + tl::expected SubmitReplicaActionProposal( + const ReplicaActionProposal& proposal) { + std::shared_lock lock(service_.master_state_mutex_); + return service_.SubmitReplicaActionProposalLocked(proposal); + } + size_t EraseReplicasWithCacheTotalAccounting( ObjectMetadata& metadata, const std::function& pred_fn, diff --git a/mooncake-store/tests/object_part_test.cpp b/mooncake-store/tests/object_part_test.cpp index 245f3c2db3..5d33c1196a 100644 --- a/mooncake-store/tests/object_part_test.cpp +++ b/mooncake-store/tests/object_part_test.cpp @@ -214,9 +214,6 @@ TEST_F(OplogPayloadTest, EveryOplogMetadataSerializerCarriesTheChecksum) { const std::vector serialized_payloads{ test::MasterServiceTestPeer::SerializeMetadataForOpLogForTesting( service, object_metadata), - test::MasterServiceTestPeer:: - SerializeMetadataForOpLogWithoutMemReplicasForTesting( - service, object_metadata), test::MasterServiceTestPeer:: SerializeMetadataForOpLogFromReplicaDescriptorsForTesting( service, object_metadata, {})}; diff --git a/mooncake-transfer-engine/include/CMakeLists.txt b/mooncake-transfer-engine/include/CMakeLists.txt index 8634964243..7b4e6f25dc 100644 --- a/mooncake-transfer-engine/include/CMakeLists.txt +++ b/mooncake-transfer-engine/include/CMakeLists.txt @@ -1,3 +1,5 @@ +configure_file(mooncake_te_build_config.h.in mooncake_te_build_config.h @ONLY) + install(FILES transfer_engine_c.h DESTINATION include) install(FILES buffer_range_index.h DESTINATION include) install(FILES common.h DESTINATION include) @@ -11,6 +13,15 @@ install(FILES transfer_metadata.h DESTINATION include) install(FILES ub_allocator.h DESTINATION include) install(FILES common/base/status.h DESTINATION include/common/base) install(FILES transport/transport.h DESTINATION include/transport) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mooncake_te_build_config.h + DESTINATION include) + +if(MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL) + install( + FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../src/common/adaptive_congestion_control/adaptive_congestion_control.h + DESTINATION include) +endif() # Device API headers (header-only, consumed by EP kernel) install(FILES transport/device/device_transport.h diff --git a/mooncake-transfer-engine/include/mooncake_te_build_config.h.in b/mooncake-transfer-engine/include/mooncake_te_build_config.h.in new file mode 100644 index 0000000000..c30d45d7ed --- /dev/null +++ b/mooncake-transfer-engine/include/mooncake_te_build_config.h.in @@ -0,0 +1,22 @@ +// Copyright 2026 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MOONCAKE_TE_BUILD_CONFIG_H_ +#define MOONCAKE_TE_BUILD_CONFIG_H_ + +// Public layouts must follow the library, not consumer compile definitions. +#undef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +#cmakedefine MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL 1 + +#endif // MOONCAKE_TE_BUILD_CONFIG_H_ diff --git a/mooncake-transfer-engine/include/transport/rdma_transport/rdma_endpoint.h b/mooncake-transfer-engine/include/transport/rdma_transport/rdma_endpoint.h index 20245ec8a7..965200d0a4 100644 --- a/mooncake-transfer-engine/include/transport/rdma_transport/rdma_endpoint.h +++ b/mooncake-transfer-engine/include/transport/rdma_transport/rdma_endpoint.h @@ -187,6 +187,11 @@ class RdmaEndPoint : public std::enable_shared_from_this { // Failed tasks (which must be submitted) are inserted in failed_slice_list int submitPostSend(std::vector &slice_list, std::vector &failed_slice_list); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + int submitPostSendLimited( + std::vector &slice_list, + std::vector &failed_slice_list, size_t max_count); +#endif // Map a CQE onto the posting-order FIFO for that QP. Retires every WR // up to `completed` inclusive (RC order). The tail stays queued because diff --git a/mooncake-transfer-engine/include/transport/rdma_transport/worker_pool.h b/mooncake-transfer-engine/include/transport/rdma_transport/worker_pool.h index e3fc9bd312..8661c72525 100644 --- a/mooncake-transfer-engine/include/transport/rdma_transport/worker_pool.h +++ b/mooncake-transfer-engine/include/transport/rdma_transport/worker_pool.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,52 @@ #include "config.h" #include "rdma_context.h" +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +#include "adaptive_congestion_control.h" +#endif + namespace mooncake { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +class ClassicRdmaCongestionControl { + public: + using SliceList = std::vector; + + explicit ClassicRdmaCongestionControl( + adaptive_congestion_control::Config config); + ~ClassicRdmaCongestionControl(); + + ClassicRdmaCongestionControl(const ClassicRdmaCongestionControl &) = delete; + ClassicRdmaCongestionControl &operator=( + const ClassicRdmaCongestionControl &) = delete; + + bool enabled() const { + return mode_ != adaptive_congestion_control::Mode::kOff; + } + void prepare(Transport::Slice *slice, const std::string &peer_nic_path); + void prepare(SliceList &slices, const std::string &peer_nic_path); + void bindEndpoint(Transport::Slice *slice, RdmaEndPoint *endpoint); + void retireEndpoint(const std::string &peer_nic_path, + RdmaEndPoint *endpoint); + size_t gate(SliceList &queued); + void complete(Transport::Slice *slice, ibv_wc_status status, + uint32_t vendor_error = 0); + void releaseUnposted(Transport::Slice *slice); + void tick(uint64_t now_ns); + void resetDevice(); + + private: + struct RouteState; + + RouteState *route(const std::string &peer_nic_path); + + adaptive_congestion_control::Config config_; + adaptive_congestion_control::Mode mode_; + std::unique_ptr device_; + std::mutex routes_mutex_; + std::unordered_map> routes_; +}; +#endif + class WorkerPoolTestPeer; class WorkerPool { friend class WorkerPoolTestPeer; @@ -191,6 +237,9 @@ class WorkerPool { std::atomic context_failure_count_{0}; uint64_t breaker_reactivate_after_ns_{0}; static constexpr int kLocalCompletionFailureThreshold = 32; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + ClassicRdmaCongestionControl adaptive_congestion_control_; +#endif }; } // namespace mooncake diff --git a/mooncake-transfer-engine/include/transport/transport.h b/mooncake-transfer-engine/include/transport/transport.h index 6c4c64683f..818fcbcc93 100644 --- a/mooncake-transfer-engine/include/transport/transport.h +++ b/mooncake-transfer-engine/include/transport/transport.h @@ -31,8 +31,12 @@ #include #include +#include "mooncake_te_build_config.h" #include "common/base/status.h" #include "transfer_metadata.h" +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +#include "adaptive_congestion_control.h" +#endif namespace mooncake { @@ -206,6 +210,12 @@ class Transport { // delete the slice. using CleanupCallback = void (*)(Slice *); CleanupCallback cleanup_callback = nullptr; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control::Permit rdma_congestion_control_permit; + void *rdma_congestion_control_route = nullptr; + RdmaEndPoint *rdma_congestion_control_endpoint = nullptr; + uint64_t rdma_congestion_control_endpoint_generation = 0; +#endif union { struct { @@ -444,6 +454,12 @@ class Transport { slice->cleanup_callback = nullptr; if (cleanup) cleanup(slice); slice->interned_peer_nic_path = nullptr; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + Transport::releaseCongestionPermit(slice); + slice->rdma_congestion_control_route = nullptr; + slice->rdma_congestion_control_endpoint = nullptr; + slice->rdma_congestion_control_endpoint_generation = 0; +#endif if (head_ - tail_ == kLazyDeleteSliceCapacity) { delete slice; @@ -612,6 +628,10 @@ class Transport { static ThreadLocalSliceCache &getSliceCache(); private: +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + static void releaseCongestionPermit(Slice *slice); +#endif + virtual int registerLocalMemory(void *addr, size_t length, const std::string &location, bool remote_accessible, diff --git a/mooncake-transfer-engine/src/CMakeLists.txt b/mooncake-transfer-engine/src/CMakeLists.txt index cd997638aa..2092cc5f35 100644 --- a/mooncake-transfer-engine/src/CMakeLists.txt +++ b/mooncake-transfer-engine/src/CMakeLists.txt @@ -1,3 +1,5 @@ +include_directories(${CMAKE_CURRENT_BINARY_DIR}/../include) + file(GLOB ENGINE_SOURCES "*.cpp" "shared_segment/*.cpp") set(TRANSFER_METADATA_PLUGIN_SOURCES config.cpp transfer_metadata_plugin.cpp) list(REMOVE_ITEM ENGINE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/config.cpp" @@ -71,6 +73,7 @@ set_target_properties(transfer_engine PROPERTIES INSTALL_RPATH "$ORIGIN") target_include_directories( transfer_engine PUBLIC $ + $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/shared_segment) target_compile_definitions(transfer_engine PRIVATE MOONCAKE_USE_ETCD diff --git a/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.cpp b/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.cpp index 3fb228ba94..004229905d 100644 --- a/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.cpp +++ b/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.cpp @@ -56,25 +56,6 @@ struct AcquireResult { AdmissionReason reason; }; -uint64_t drainTimeNs(uint64_t backlog_bytes, - uint64_t delivery_rate_bytes_per_sec) { - if (delivery_rate_bytes_per_sec == 0) return 0; - const long double value = - static_cast(backlog_bytes) * 1'000'000'000.0L / - static_cast(delivery_rate_bytes_per_sec); - return value > static_cast( - std::numeric_limits::max()) - ? std::numeric_limits::max() - : static_cast(value); -} - -uint32_t addBounded(uint32_t current, uint64_t value) { - const uint64_t sum = static_cast(current) + value; - return sum > std::numeric_limits::max() - ? std::numeric_limits::max() - : static_cast(sum); -} - } // namespace struct CoreAccess { @@ -145,29 +126,24 @@ struct CoreAccess { } if (generation == 0) return; - if (signals.delivery_rate_bytes_per_sec != 0) { - state.backlog_bytes_.store(signals.backlog_bytes, - std::memory_order_relaxed); - state.delivery_rate_bytes_per_sec_.store( - signals.delivery_rate_bytes_per_sec, std::memory_order_relaxed); - state.has_rate_sample_.store(true, std::memory_order_relaxed); - } - addSaturating(state.completed_bytes_, signals.completed_bytes); addSaturating(state.successful_completions_, signals.successful_completions); addSaturating(state.receiver_pressure_, signals.receiver_pressure); + if (signals.route_timeouts != 0 || signals.hard_errors != 0 || + signals.fatal_failures != 0) { + state.epoch_error_.store(true, std::memory_order_relaxed); + } state.feedback_writers_.fetch_sub(1, std::memory_order_release); } static void recordOutcome(DomainState* state, uint32_t generation, - uint64_t bytes, OutcomeClass outcome, - FailureScope, bool route) { + OutcomeClass outcome, FailureScope scope, + bool route) { if (state == nullptr || state->config_.mode == Mode::kOff) return; Signals signals; switch (outcome) { case OutcomeClass::kSuccess: - signals.completed_bytes = bytes; signals.successful_completions = 1; break; case OutcomeClass::kCongestion: @@ -180,22 +156,33 @@ struct CoreAccess { signals.receiver_pressure = 1; break; case OutcomeClass::kRouteTimeout: + signals.route_timeouts = 1; + break; case OutcomeClass::kFatal: + signals.fatal_failures = 1; + break; case OutcomeClass::kLocalConfiguration: case OutcomeClass::kRemoteMetadata: + signals.hard_errors = 1; + break; case OutcomeClass::kDerivedFlush: return; } + // Peer-local failures do not veto healthy siblings on the device. + if (!route && + (signals.route_timeouts != 0 || signals.hard_errors != 0 || + signals.fatal_failures != 0) && + scope != FailureScope::kCq && scope != FailureScope::kPort && + scope != FailureScope::kDevice) { + return; + } addSignals(*state, generation, signals); } static void clearSignals(DomainState& state) { - state.backlog_bytes_.store(0, std::memory_order_relaxed); - state.delivery_rate_bytes_per_sec_.store(0, std::memory_order_relaxed); - state.has_rate_sample_.store(false, std::memory_order_relaxed); - state.completed_bytes_.store(0, std::memory_order_relaxed); state.successful_completions_.store(0, std::memory_order_relaxed); state.receiver_pressure_.store(0, std::memory_order_relaxed); + state.epoch_error_.store(false, std::memory_order_relaxed); } static bool pauseFeedback(DomainState& state, uint32_t generation) { @@ -334,9 +321,9 @@ bool complete(Permit& permit, OutcomeClass outcome, FailureScope scope) { } CoreAccess::recordOutcome(permit.device_, permit.device_generation_, - permit.bytes_, outcome, scope, false); - CoreAccess::recordOutcome(permit.route_, permit.route_generation_, - permit.bytes_, outcome, scope, true); + outcome, scope, false); + CoreAccess::recordOutcome(permit.route_, permit.route_generation_, outcome, + scope, true); if (permit.device_reserved_) CoreAccess::releaseBytes(*permit.device_, permit.bytes_); if (permit.route_reserved_) @@ -385,59 +372,35 @@ void controlTick(DomainState& state, uint64_t) { if (!CoreAccess::pauseFeedback(state, generation)) return; FeedbackResumeGuard feedback_guard(state, generation); - const uint64_t backlog = - state.backlog_bytes_.exchange(0, std::memory_order_relaxed); - const uint64_t rate = state.delivery_rate_bytes_per_sec_.exchange( - 0, std::memory_order_relaxed); - const bool has_rate = - state.has_rate_sample_.exchange(false, std::memory_order_relaxed); - state.completed_bytes_.exchange(0, std::memory_order_relaxed); const uint64_t successes = state.successful_completions_.exchange(0, std::memory_order_relaxed); const uint64_t receiver_pressure = state.receiver_pressure_.exchange(0, std::memory_order_relaxed); - const uint64_t drain_ns = has_rate ? drainTimeNs(backlog, rate) : 0; - const bool high_pressure = - receiver_pressure != 0 || - (has_rate && drain_ns > state.config_.target_drain_time_ns); - const bool low_pressure = - has_rate && drain_ns < state.config_.target_drain_time_ns / 2; - - if (high_pressure) { - state.low_pressure_streak_ = 0; - state.high_pressure_streak_ = - addBounded(state.high_pressure_streak_, 1); - if (state.high_pressure_streak_ >= state.config_.high_pressure_epochs) { - const uint64_t current = - state.window_bytes_.load(std::memory_order_relaxed); - state.window_bytes_.store( - std::max(state.config_.min_window_bytes, current / 2), - std::memory_order_relaxed); - state.high_pressure_streak_ = 0; - state.state_.store(PathState::kCongested, - std::memory_order_release); - } + const bool epoch_error = + state.epoch_error_.exchange(false, std::memory_order_relaxed); + if (receiver_pressure != 0) { + const uint64_t current = + state.window_bytes_.load(std::memory_order_relaxed); + state.window_bytes_.store( + std::max(state.config_.min_window_bytes, current / 2), + std::memory_order_relaxed); + state.state_.store(PathState::kCongested, std::memory_order_release); return; } - state.high_pressure_streak_ = 0; - if (successes != 0 && low_pressure) { - state.low_pressure_streak_ = addBounded(state.low_pressure_streak_, 1); - if (state.low_pressure_streak_ >= state.config_.low_pressure_epochs) { - const uint64_t current = - state.window_bytes_.load(std::memory_order_relaxed); - const uint64_t increment = - std::max(1, state.config_.min_window_bytes / 4); - const uint64_t next = - current > state.config_.max_window_bytes - increment - ? state.config_.max_window_bytes - : current + increment; - state.window_bytes_.store(next, std::memory_order_relaxed); - state.low_pressure_streak_ = 0; - state.state_.store(PathState::kHealthy, std::memory_order_release); - } - } else { - state.low_pressure_streak_ = 0; + if (epoch_error) return; + + if (successes != 0) { + const uint64_t current = + state.window_bytes_.load(std::memory_order_relaxed); + const uint64_t increment = + std::max(1, state.config_.min_window_bytes / 4); + const uint64_t next = + current > state.config_.max_window_bytes - increment + ? state.config_.max_window_bytes + : current + increment; + state.window_bytes_.store(next, std::memory_order_relaxed); + state.state_.store(PathState::kHealthy, std::memory_order_release); } } @@ -448,8 +411,6 @@ void resetGeneration(DomainState& state, uint32_t generation) { if (!CoreAccess::pauseFeedback(state, current_generation)) return; CoreAccess::clearSignals(state); - state.high_pressure_streak_ = 0; - state.low_pressure_streak_ = 0; state.deferred_bytes_.store(0, std::memory_order_relaxed); state.window_bytes_.store(state.config_.min_window_bytes, std::memory_order_relaxed); diff --git a/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.h b/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.h index 63b489d02c..6782f13b1f 100644 --- a/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.h +++ b/mooncake-transfer-engine/src/common/adaptive_congestion_control/adaptive_congestion_control.h @@ -57,21 +57,25 @@ struct Config { Mode mode = Mode::kOff; uint64_t min_window_bytes = 4ULL << 20; uint64_t max_window_bytes = 64ULL << 20; + // Retained for configuration compatibility, not used by window decisions. uint64_t target_drain_time_ns = 2'000'000; uint32_t high_pressure_epochs = 2; uint32_t low_pressure_epochs = 3; }; struct Signals { + // Compatibility inputs; adapters no longer sample or consume these values. uint64_t backlog_bytes = 0; uint64_t delivery_rate_bytes_per_sec = 0; uint64_t completed_bytes = 0; uint64_t successful_completions = 0; uint64_t receiver_pressure = 0; - // Legacy health signals are accepted but do not affect the window. + // Explicit errors veto growth for the addressed domain in this epoch. + // They do not indicate congestion or replace transport recovery. uint64_t route_timeouts = 0; uint64_t hard_errors = 0; uint64_t fatal_failures = 0; + // Poller stalls remain observational and do not veto growth. bool poller_stalled = false; }; @@ -121,14 +125,9 @@ class DomainState { std::atomic feedback_generation_{1}; std::atomic feedback_writers_{0}; - std::atomic backlog_bytes_{0}; - std::atomic delivery_rate_bytes_per_sec_{0}; - std::atomic has_rate_sample_{false}; - std::atomic completed_bytes_{0}; std::atomic successful_completions_{0}; std::atomic receiver_pressure_{0}; - uint32_t high_pressure_streak_ = 0; - uint32_t low_pressure_streak_ = 0; + std::atomic epoch_error_{false}; }; // DomainState instances must outlive their PathHandle and every active Permit. diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/CMakeLists.txt b/mooncake-transfer-engine/src/transport/rdma_transport/CMakeLists.txt index 6bb67f495d..ffc5f48b90 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/CMakeLists.txt +++ b/mooncake-transfer-engine/src/transport/rdma_transport/CMakeLists.txt @@ -6,6 +6,10 @@ target_link_libraries( PUBLIC gflags::gflags glog::glog PRIVATE JsonCpp::JsonCpp pthread) +if(MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL) + target_link_libraries(rdma_transport PUBLIC adaptive_congestion_control_core) +endif() + if(USE_CUDA) target_compile_definitions(rdma_transport PRIVATE _GNU_SOURCE) target_link_libraries(rdma_transport PUBLIC ${CMAKE_DL_LIBS}) diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp index 747e9c7942..435bbf7512 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp @@ -156,18 +156,64 @@ bool isKernelDmabufSupported() { if (found[0] && found[1]) break; } - // Fallback: probe /proc/kallsyms for the corresponding kernel symbols. + // Fallback: probe /proc/kallsyms for kernel symbols that exist ONLY + // when the corresponding config is enabled. + // + // The previous substring match on "pci_p2pdma" was a FALSE POSITIVE: + // the exported helper dma_pci_p2pdma_supported() is compiled in + // whenever CONFIG_HAS_DMA is set — regardless of CONFIG_PCI_P2PDMA + // (see include/linux/dma-mapping.h / kernel/dma/mapping.c) — so + // kernels without P2PDMA passed this gate. Since the config-file + // candidates above are typically invisible inside containers, this + // fallback is the deciding probe in containerized deployments. + // + // Consequence observed in production on MI308X + bnxt_re (Thor2) + // with kernel 6.6 lacking CONFIG_PCI_P2PDMA: the dmabuf MR path was + // allowed, and amdgpu silently migrated the registered VRAM buffer + // to GTT/system memory (mem_info_vram_used dropped by the buffer + // size, mem_info_gtt_used grew by the same amount). GPU-side access + // to the buffer collapsed from ~1838 GB/s to 115 GB/s (16x) and did + // not recover after MR deregistration; on a KV-cache pool this + // slowed attention kernels by 5-50x. + // + // The matching below is an exact symbol-name comparison (the third + // kallsyms field, with the __pfx_ prefix stripped) against symbols + // from drivers/pci/p2pdma.c, which is only compiled when + // CONFIG_PCI_P2PDMA=y (the =n variants are static inline and never + // appear in kallsyms). if (!found[0] || !found[1]) { std::ifstream f("/proc/kallsyms"); if (f.good()) { std::string line; while (std::getline(f, line)) { - if (!found[0] && - line.find("pci_p2pdma") != std::string::npos) + // kallsyms line format: " [module]" + size_t t1 = line.find(' '); + size_t t2 = (t1 == std::string::npos) + ? std::string::npos + : line.find(' ', t1 + 1); + if (t2 == std::string::npos) continue; + std::string name = line.substr(t2 + 1); + size_t sp = name.find(' '); + if (sp != std::string::npos) name.resize(sp); + // Strip the __pfx_ prefix used by newer kernels. + if (name.rfind("__pfx_", 0) == 0) name.erase(0, 6); + + // Present in kallsyms iff CONFIG_PCI_P2PDMA=y. + if (!found[0] && (name == "pci_p2pdma_add_resource" || + name == "pci_p2pdma_distance_many" || + name == "pci_p2pdma_enable_store")) { found[0] = true; - if (!found[1] && - line.find("dma_buf_move_notify") != std::string::npos) + } + // NOTE: dma_buf_move_notify() is compiled with the + // dma-buf core (CONFIG_DMA_SHARED_BUFFER, see + // drivers/dma-buf/dma-buf.c), so this only proves the + // core exists, not that CONFIG_DMABUF_MOVE_NOTIFY=y. + // No single =y-only symbol exists for MOVE_NOTIFY; the + // overall gate is effectively decided by the P2PDMA + // check above. + if (!found[1] && name == "dma_buf_move_notify") { found[1] = true; + } if (found[0] && found[1]) break; } } diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_endpoint.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_endpoint.cpp index b0bb64466d..9fce1198a7 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/rdma_endpoint.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/rdma_endpoint.cpp @@ -1314,20 +1314,44 @@ size_t RdmaEndPoint::collectPostedCompletions( return n; } +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL int RdmaEndPoint::submitPostSend( std::vector &slice_list, std::vector &failed_slice_list) { + return submitPostSendLimited(slice_list, failed_slice_list, + slice_list.size()); +} + +int RdmaEndPoint::submitPostSendLimited( + std::vector &slice_list, + std::vector &failed_slice_list, size_t max_count) { +#else +int RdmaEndPoint::submitPostSend( + std::vector &slice_list, + std::vector &failed_slice_list) { +#endif RWSpinlock::WriteGuard guard(lock_); if (!active_.load(std::memory_order_acquire) || status_.load(std::memory_order_relaxed) != CONNECTED) { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + const size_t requested = std::min(slice_list.size(), max_count); + failed_slice_list.insert(failed_slice_list.end(), slice_list.begin(), + slice_list.begin() + requested); + slice_list.erase(slice_list.begin(), slice_list.begin() + requested); +#else for (auto &slice : slice_list) failed_slice_list.push_back(slice); slice_list.clear(); +#endif return 0; } const size_t num_qp = qp_list_.size(); - if (slice_list.empty()) return 0; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + const size_t requested = std::min(slice_list.size(), max_count); +#else const size_t requested = slice_list.size(); +#endif + if (requested == 0) return 0; int cq_remaining = int(globalConfig().max_cqe) - cq_outstanding_->load(std::memory_order_relaxed); if (cq_remaining <= 0) return 0; diff --git a/mooncake-transfer-engine/src/transport/rdma_transport/worker_pool.cpp b/mooncake-transfer-engine/src/transport/rdma_transport/worker_pool.cpp index fb2db2d9b2..73c634d056 100644 --- a/mooncake-transfer-engine/src/transport/rdma_transport/worker_pool.cpp +++ b/mooncake-transfer-engine/src/transport/rdma_transport/worker_pool.cpp @@ -29,6 +29,10 @@ #include "config.h" #include "memory_location.h" #include "transport/rdma_transport/rdma_batch_cache.h" +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +#include "adaptive_congestion_control_config.h" +#include "rdma_error_classifier.h" +#endif #include "transport/rdma_transport/rdma_context.h" #include "transport/rdma_transport/rdma_endpoint.h" #include "transport/rdma_transport/rdma_transport.h" @@ -39,6 +43,227 @@ namespace mooncake { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +struct ClassicRdmaCongestionControl::RouteState { + RouteState(adaptive_congestion_control::Config config, std::string path, + const ClassicRdmaCongestionControl *owner) + : owner(owner), peer_nic_path(std::move(path)), domain(config) {} + + const ClassicRdmaCongestionControl *const owner; + const std::string peer_nic_path; + adaptive_congestion_control::DomainState domain; + std::atomic endpoint{nullptr}; + std::atomic endpoint_generation{1}; + std::mutex endpoint_mutex; +}; + +ClassicRdmaCongestionControl::ClassicRdmaCongestionControl( + adaptive_congestion_control::Config config) + : config_(config), + mode_(config.mode), + device_(mode_ == adaptive_congestion_control::Mode::kOff + ? nullptr + : std::make_unique( + config)) {} + +ClassicRdmaCongestionControl::~ClassicRdmaCongestionControl() = default; + +ClassicRdmaCongestionControl::RouteState *ClassicRdmaCongestionControl::route( + const std::string &peer_nic_path) { + std::lock_guard lock(routes_mutex_); + auto &entry = routes_[peer_nic_path]; + if (!entry) { + entry = std::make_unique(config_, peer_nic_path, this); + } + return entry.get(); +} + +void ClassicRdmaCongestionControl::prepare(Transport::Slice *slice, + const std::string &peer_nic_path) { + if (!enabled()) return; + auto *current_route = + static_cast(slice->rdma_congestion_control_route); + if (current_route != nullptr && current_route->owner == this && + current_route->peer_nic_path == peer_nic_path) { + return; + } + RouteState *next_route = route(peer_nic_path); + if (current_route != next_route) releaseUnposted(slice); + slice->rdma_congestion_control_route = next_route; +} + +void ClassicRdmaCongestionControl::prepare(SliceList &slices, + const std::string &peer_nic_path) { + if (!enabled() || slices.empty()) return; + auto *next_route = static_cast( + slices.front()->rdma_congestion_control_route); + if (next_route == nullptr || next_route->owner != this || + next_route->peer_nic_path != peer_nic_path) { + next_route = route(peer_nic_path); + } + for (auto *slice : slices) { + auto *current_route = + static_cast(slice->rdma_congestion_control_route); + if (current_route != next_route) releaseUnposted(slice); + slice->rdma_congestion_control_route = next_route; + } +} + +void ClassicRdmaCongestionControl::bindEndpoint(Transport::Slice *slice, + RdmaEndPoint *endpoint) { + auto *state = + static_cast(slice->rdma_congestion_control_route); + if (state == nullptr) return; + if (state->endpoint.load(std::memory_order_acquire) == endpoint) return; + std::lock_guard lock(state->endpoint_mutex); + if (state->endpoint.load(std::memory_order_relaxed) == endpoint) return; + uint64_t next = + state->endpoint_generation.load(std::memory_order_relaxed) + 1; + if (next == 0) next = 1; + state->endpoint_generation.store(next, std::memory_order_release); + state->endpoint.store(endpoint, std::memory_order_release); +} + +void ClassicRdmaCongestionControl::retireEndpoint( + const std::string &peer_nic_path, RdmaEndPoint *endpoint) { + if (!enabled()) return; + RouteState *state = nullptr; + { + std::lock_guard lock(routes_mutex_); + auto it = routes_.find(peer_nic_path); + if (it == routes_.end()) return; + state = it->second.get(); + } + std::lock_guard lock(state->endpoint_mutex); + if (state->endpoint.load(std::memory_order_relaxed) != endpoint) return; + uint64_t next = + state->endpoint_generation.load(std::memory_order_relaxed) + 1; + if (next == 0) next = 1; + state->endpoint_generation.store(next, std::memory_order_release); + state->endpoint.store(nullptr, std::memory_order_release); +} + +size_t ClassicRdmaCongestionControl::gate(SliceList &queued) { + if (!enabled() || queued.empty()) return queued.size(); + size_t admitted_count = 0; + size_t cursor = 0; + auto *first_route = static_cast( + queued.front()->rdma_congestion_control_route); + bool coalesce = mode_ == adaptive_congestion_control::Mode::kEnforce && + first_route != nullptr && + adaptive_congestion_control::snapshot(first_route->domain) + .inflight_bytes != 0; + while (cursor < queued.size()) { + auto *slice = queued[cursor]; + auto *state = + static_cast(slice->rdma_congestion_control_route); + if (state != first_route) coalesce = false; + if (state == nullptr) { + std::swap(queued[admitted_count++], queued[cursor++]); + continue; + } + adaptive_congestion_control::PathHandle path{ + device_.get(), &state->domain, + adaptive_congestion_control::generation(*device_), + adaptive_congestion_control::generation(state->domain)}; + const auto decision = adaptive_congestion_control::tryAcquire( + path, slice->length, slice->rdma_congestion_control_permit); + if (decision == adaptive_congestion_control::Decision::kAllow) { + slice->rdma_congestion_control_endpoint = + state->endpoint.load(std::memory_order_acquire); + slice->rdma_congestion_control_endpoint_generation = + state->endpoint_generation.load(std::memory_order_acquire); + std::swap(queued[admitted_count++], queued[cursor++]); + } else { + ++cursor; + } + } + size_t retained = admitted_count; + // Group an active route's partial replenishment; idle routes and fully + // admitted tails always make progress. + if (coalesce && admitted_count != queued.size()) { + retained = 0; + uint64_t bytes = 0; + for (size_t i = 0; i < admitted_count; ++i) { + bytes += std::min(queued[i]->length, + config_.min_window_bytes - bytes); + if (bytes == config_.min_window_bytes || + (i + 1 < queued.size() && + queued[i + 1]->length > config_.min_window_bytes - bytes)) { + retained = i + 1; + bytes = 0; + } + } + for (size_t i = retained; i < admitted_count; ++i) + releaseUnposted(queued[i]); + } + return retained; +} + +void ClassicRdmaCongestionControl::complete(Transport::Slice *slice, + ibv_wc_status status, + uint32_t vendor_error) { + auto *state = + static_cast(slice->rdma_congestion_control_route); + if (!slice->rdma_congestion_control_permit.active()) return; + const bool current_endpoint = + state != nullptr && + slice->rdma_congestion_control_endpoint == + state->endpoint.load(std::memory_order_acquire) && + slice->rdma_congestion_control_endpoint_generation == + state->endpoint_generation.load(std::memory_order_acquire); + const auto result = + current_endpoint + ? adaptive_congestion_control::classifyCompletion(status, + vendor_error) + : adaptive_congestion_control::Classification{ + adaptive_congestion_control::OutcomeClass::kDerivedFlush, + adaptive_congestion_control::FailureScope::kOperation, false, + vendor_error}; + adaptive_congestion_control::complete(slice->rdma_congestion_control_permit, + result.outcome, result.scope); +} + +void ClassicRdmaCongestionControl::releaseUnposted(Transport::Slice *slice) { + if (!slice->rdma_congestion_control_permit.active()) return; + adaptive_congestion_control::complete( + slice->rdma_congestion_control_permit, + adaptive_congestion_control::OutcomeClass::kDerivedFlush, + adaptive_congestion_control::FailureScope::kOperation); +} + +void ClassicRdmaCongestionControl::tick(uint64_t now_ns) { + if (!enabled()) return; + adaptive_congestion_control::controlTick(*device_, now_ns); + + std::lock_guard lock(routes_mutex_); + for (auto &entry : routes_) { + auto &state = *entry.second; + adaptive_congestion_control::controlTick(state.domain, now_ns); + } +} + +void ClassicRdmaCongestionControl::resetDevice() { + if (!enabled()) return; + const uint32_t generation = + adaptive_congestion_control::generation(*device_) + 1; + adaptive_congestion_control::resetGeneration(*device_, generation); +} + +static adaptive_congestion_control::Config +loadClassicCongestionControlConfig() { + auto result = adaptive_congestion_control::loadConfigFromEnvironment(); + if (result.valid) return result.config; + + static std::once_flag warning_once; + std::call_once(warning_once, [&] { + LOG(ERROR) << "Invalid adaptive congestion-control setting: " + << result.error << "; classic RDMA enforcement disabled"; + }); + return adaptive_congestion_control::Config{}; +} +#endif + static std::string resolveBufferLocation( const TransferMetadata::BufferDesc &buffer, uint64_t offset) { std::string location = buffer.name; @@ -234,7 +459,12 @@ WorkerPool::WorkerPool(RdmaContext &context, int numa_socket_id) worker_slice_queue_(worker_count_), worker_slice_queue_lock_(worker_count_), submitted_slice_count_(0), - processed_slice_count_(0) { + processed_slice_count_(0) +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + , + adaptive_congestion_control_(loadClassicCongestionControlConfig()) +#endif +{ collective_slice_queue_.resize(worker_count_); for (int i = 0; i < worker_count_; ++i) worker_thread_.emplace_back( @@ -719,6 +949,10 @@ void WorkerPool::performPostSend(int thread_id) { refreshPublishedLocalTopology(); redispatch_counter_++; } +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint(entry.first, + endpoint.get()); +#endif context_.deleteEndpointByPtr(endpoint.get()); for (auto &slice : entry.second) { if (!has_peer_alternative && local_context_inactive && @@ -739,6 +973,10 @@ void WorkerPool::performPostSend(int thread_id) { << ", deleting endpoint"; markRailFailed(entry.first, true); redispatch_counter_++; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint(entry.first, + endpoint.get()); +#endif context_.deleteEndpointByPtr(endpoint.get()); for (auto &slice : entry.second) failed_slice_list.push_back(slice); @@ -746,14 +984,53 @@ void WorkerPool::performPostSend(int thread_id) { } continue; } - // Set endpoint pointer for each slice before submitting - auto auto_gid_token = endpoint->autoGidCompletionToken(); - for (auto &slice : entry.second) { - slice->rdma.endpoint = endpoint.get(); - slice->rdma.auto_gid_token = auto_gid_token; +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + if (adaptive_congestion_control_.enabled()) { + adaptive_congestion_control_.prepare(entry.second, entry.first); + adaptive_congestion_control_.bindEndpoint(entry.second.front(), + endpoint.get()); + const size_t admitted_count = + adaptive_congestion_control_.gate(entry.second); + if (admitted_count != 0) { + auto auto_gid_token = endpoint->autoGidCompletionToken(); + for (size_t i = 0; i < admitted_count; ++i) { + entry.second[i]->rdma.endpoint = endpoint.get(); + entry.second[i]->rdma.auto_gid_token = auto_gid_token; + } + const size_t failed_before = failed_slice_list.size(); + const size_t queued_before = entry.second.size(); + endpoint->submitPostSendLimited(entry.second, failed_slice_list, + admitted_count); + const size_t consumed = queued_before - entry.second.size(); + for (size_t i = failed_before; i < failed_slice_list.size(); + ++i) { + adaptive_congestion_control_.releaseUnposted( + failed_slice_list[i]); + } + for (size_t i = 0; i < admitted_count - consumed; ++i) + adaptive_congestion_control_.releaseUnposted( + entry.second[i]); + } + } else { +#endif + auto auto_gid_token = endpoint->autoGidCompletionToken(); + for (auto *slice : entry.second) { + slice->rdma.endpoint = endpoint.get(); + slice->rdma.auto_gid_token = auto_gid_token; + } + endpoint->submitPostSend(entry.second, failed_slice_list); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + } +#endif + if (!endpoint->active()) { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + if (adaptive_congestion_control_.enabled()) { + adaptive_congestion_control_.retireEndpoint(entry.first, + endpoint.get()); + } +#endif + context_.deleteEndpointByPtr(endpoint.get()); } - endpoint->submitPostSend(entry.second, failed_slice_list); - if (!endpoint->active()) context_.deleteEndpointByPtr(endpoint.get()); #endif } @@ -891,6 +1168,11 @@ void WorkerPool::processCompletions(int thread_id, for (const auto &wc : wc_list) { Transport::Slice *slice = (Transport::Slice *)wc.wr_id; assert(slice); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + if (adaptive_congestion_control_.enabled()) + adaptive_congestion_control_.complete(slice, wc.status, + wc.vendor_err); +#endif if (wc.status != IBV_WC_SUCCESS) { // Flush errors are generated when QPs transition to ERR state // during normal endpoint destruction (beginDestroy). They are not @@ -964,6 +1246,10 @@ void WorkerPool::processCompletions(int thread_id, } else if (slice->rdma.endpoint && !local_failed_endpoints.count( slice->rdma.endpoint)) { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint( + slice->peerNicPath(), slice->rdma.endpoint); +#endif context_.deleteEndpointByPtr(slice->rdma.endpoint); local_failed_endpoints.insert(slice->rdma.endpoint); } @@ -979,6 +1265,10 @@ void WorkerPool::processCompletions(int thread_id, redispatch_counter_++; } if (slice->rdma.endpoint) { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint( + slice->peerNicPath(), slice->rdma.endpoint); +#endif context_.deleteEndpointByPtr(slice->rdma.endpoint); } } @@ -1326,6 +1616,10 @@ int WorkerPool::doProcessContextEvents() { if (endpoint) { auto endpoint_lifecycle_lock = context_.lockEndpointLifecycle(endpoint->peerNicPath()); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint( + endpoint->peerNicPath(), endpoint.get()); +#endif context_.deleteEndpointByPtr(endpoint.get()); } else { LOG(WARNING) << "QP fatal event endpoint is no longer tracked"; @@ -1464,6 +1758,9 @@ void WorkerPool::maybeActivateRecoveredContext() { } resetContextBreaker(true); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.resetDevice(); +#endif recovery_activate_after_ns_.store(0, std::memory_order_relaxed); refreshPublishedLocalTopology(); LOG(INFO) << "Worker: Context " << context_.deviceName() @@ -1583,6 +1880,9 @@ void WorkerPool::monitorWorker() { while (workers_running_) { const uint64_t current_ts = static_cast(getCurrentTimeInNano()); +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.tick(current_ts); +#endif maybeActivateRecoveredContext(); // Check short breaker TTLs on every loop, like GID recovery. maybeReactivateContext(); @@ -1817,6 +2117,9 @@ void WorkerPool::handleLocalFailure(const std::string &peer_nic_path, // Endpoint may also be poisoned; retire it for safety. if (endpoint) { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL + adaptive_congestion_control_.retireEndpoint(peer_nic_path, endpoint); +#endif context_.deleteEndpointByPtr(endpoint); } diff --git a/mooncake-transfer-engine/src/transport/transport.cpp b/mooncake-transfer-engine/src/transport/transport.cpp index 739f399342..51521c57c6 100644 --- a/mooncake-transfer-engine/src/transport/transport.cpp +++ b/mooncake-transfer-engine/src/transport/transport.cpp @@ -18,6 +18,15 @@ #include "transfer_engine.h" namespace mooncake { +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +void Transport::releaseCongestionPermit(Slice *slice) { + adaptive_congestion_control::complete( + slice->rdma_congestion_control_permit, + adaptive_congestion_control::OutcomeClass::kDerivedFlush, + adaptive_congestion_control::FailureScope::kOperation); +} +#endif + thread_local static Transport::ThreadLocalSliceCache tl_slice_cache; Transport::ThreadLocalSliceCache &Transport::getSliceCache() { diff --git a/mooncake-transfer-engine/tent/config/transfer-engine.json b/mooncake-transfer-engine/tent/config/transfer-engine.json index feadc511f2..12ea6bf57b 100644 --- a/mooncake-transfer-engine/tent/config/transfer-engine.json +++ b/mooncake-transfer-engine/tent/config/transfer-engine.json @@ -12,6 +12,16 @@ "max_failover_attempts": 3, "enable_auto_failover_on_poll": true, "enable_progress_worker": false, + "receiver_credit": { + "enable": false, + "data_bytes": 1073741824, + "request_slots": 1024, + "freshness_ttl_ms": 1000, + "max_freshness_ttl_ms": 60000, + "max_senders": 1024, + "max_authority_entries": 4096, + "max_sender_entries": 4096 + }, "metrics": { "enabled": true, "http_port": 9100, diff --git a/mooncake-transfer-engine/tent/include/tent/common/types.h b/mooncake-transfer-engine/tent/include/tent/common/types.h index b9d758180d..64562bf9d1 100644 --- a/mooncake-transfer-engine/tent/include/tent/common/types.h +++ b/mooncake-transfer-engine/tent/include/tent/common/types.h @@ -177,6 +177,12 @@ struct Request { // and docs/source/design/tent/deadline-scheduling.md. uint64_t deadline_ns = 0; IntentType intent_type = IntentType::INTENT_UNSPEC; + // Internal runtime fence for staging sub-transfers that are covered by + // their parent request's receiver-credit reservation. Public callers + // leave these fields zero. + uint64_t receiver_credit_session_high = 0; + uint64_t receiver_credit_session_low = 0; + uint64_t receiver_credit_epoch = 0; }; enum TransferStatusEnum { diff --git a/mooncake-transfer-engine/tent/include/tent/rpc/rpc.h b/mooncake-transfer-engine/tent/include/tent/rpc/rpc.h index daeac4916d..a19f0224ff 100644 --- a/mooncake-transfer-engine/tent/include/tent/rpc/rpc.h +++ b/mooncake-transfer-engine/tent/include/tent/rpc/rpc.h @@ -49,6 +49,7 @@ enum RpcFuncID { NotifySegmentUpdated, // Appended to preserve the numeric values of the existing RPCs. BootstrapUb, + ExchangeReceiverCredit, }; class ClientPool; diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/admission_queue.h b/mooncake-transfer-engine/tent/include/tent/runtime/admission_queue.h index 5623e9194e..786686988e 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/admission_queue.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/admission_queue.h @@ -147,6 +147,12 @@ class LocalTransferAdmissionQueue { size_t max_owners, size_t max_bytes, std::vector* dropped_owner_ids = nullptr); + // Return an owner that could not be handed off to its selected transport + // from the dispatch window to the queue. Only a Dispatching owner can be + // deferred. Outstanding owner/byte accounting remains charged while the + // owner waits for a later dispatch attempt. + Status defer(QueueOwnerId owner_id); + // Install the step-3 degradation policy inputs. Optional; without it the // queue never drops (default behavior). now defaults to steady_clock. void setDegradationPolicy(BandwidthProvider bandwidth_provider, @@ -192,10 +198,16 @@ class LocalTransferAdmissionQueue { QueueOwnerKind kind{QueueOwnerKind::User}; TransportType transport{UNSPEC}; bool degradation_eligible{false}; + // EDF owners that yield because a downstream resource is unavailable + // move to the next scheduling round. This preserves progress without + // relying on a rotated deque remaining globally deadline-sorted. + uint64_t scheduling_round{0}; QueueState state{QueueState::Queued}; TransferStatusEnum terminal_status{TransferStatusEnum::PENDING}; }; + void enqueueOwner(QueueOwnerId owner_id); + QueueLimits limits_; Status limits_status_; QueueOwnerId next_owner_id_{1}; diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/control_plane.h b/mooncake-transfer-engine/tent/include/tent/runtime/control_plane.h index ab1ce5f3e1..93b439a996 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/control_plane.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/control_plane.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -77,25 +78,32 @@ struct UbBootstrapDesc { std::vector jetty_uasids; uint64_t endpoint_generation = 0; uint64_t segment_generation = 0; + uint64_t receiver_credit_session_high = 0; + uint64_t receiver_credit_session_low = 0; + uint64_t receiver_credit_epoch = 0; std::vector capabilities; std::string reply_msg; }; inline void to_json(nlohmann::json& j, const UbBootstrapDesc& desc) { - j = nlohmann::json{{"protocol_version", desc.protocol_version}, - {"segment_name", desc.segment_name}, - {"local_nic_path", desc.local_nic_path}, - {"peer_nic_path", desc.peer_nic_path}, - {"local_device_name", desc.local_device_name}, - {"local_device_id", desc.local_device_id}, - {"local_eid_index", desc.local_eid_index}, - {"local_eid", desc.local_eid}, - {"jetty_ids", desc.jetty_ids}, - {"jetty_uasids", desc.jetty_uasids}, - {"endpoint_generation", desc.endpoint_generation}, - {"segment_generation", desc.segment_generation}, - {"capabilities", desc.capabilities}, - {"reply_msg", desc.reply_msg}}; + j = nlohmann::json{ + {"protocol_version", desc.protocol_version}, + {"segment_name", desc.segment_name}, + {"local_nic_path", desc.local_nic_path}, + {"peer_nic_path", desc.peer_nic_path}, + {"local_device_name", desc.local_device_name}, + {"local_device_id", desc.local_device_id}, + {"local_eid_index", desc.local_eid_index}, + {"local_eid", desc.local_eid}, + {"jetty_ids", desc.jetty_ids}, + {"jetty_uasids", desc.jetty_uasids}, + {"endpoint_generation", desc.endpoint_generation}, + {"segment_generation", desc.segment_generation}, + {"receiver_credit_session_high", desc.receiver_credit_session_high}, + {"receiver_credit_session_low", desc.receiver_credit_session_low}, + {"receiver_credit_epoch", desc.receiver_credit_epoch}, + {"capabilities", desc.capabilities}, + {"reply_msg", desc.reply_msg}}; } inline void from_json(const nlohmann::json& j, UbBootstrapDesc& desc) { @@ -114,14 +122,44 @@ inline void from_json(const nlohmann::json& j, UbBootstrapDesc& desc) { desc.jetty_uasids = j.value("jetty_uasids", std::vector{}); desc.endpoint_generation = j.value("endpoint_generation", uint64_t{0}); desc.segment_generation = j.value("segment_generation", uint64_t{0}); + desc.receiver_credit_session_high = + j.value("receiver_credit_session_high", uint64_t{0}); + desc.receiver_credit_session_low = + j.value("receiver_credit_session_low", uint64_t{0}); + desc.receiver_credit_epoch = j.value("receiver_credit_epoch", uint64_t{0}); desc.capabilities = j.value("capabilities", std::vector{}); desc.reply_msg = j.value("reply_msg", ""); } +// SendData/RecvData v2 keeps the legacy address/length prefix, then adds a +// magic discriminator and the receiver-credit session fence. A new receiver +// can still parse legacy non-credit requests, while credit-required receivers +// reject any request without an exact current fence. +constexpr uint64_t kXferDataProtocolMagic = 0x54454e5458465232ULL; +constexpr uint64_t kXferDataV2LegacyRejectAddress = + std::numeric_limits::max(); +struct LegacyXferDataDesc { + uint64_t peer_mem_addr; + uint64_t length; +}; struct XferDataDesc { + // An old receiver interprets the first two fields as the legacy header. + // The impossible address makes it reject v2 instead of copying the + // appended fence bytes into user memory. + uint64_t legacy_reject_addr; + uint64_t length; + uint64_t protocol_magic; uint64_t peer_mem_addr; - size_t length; + uint64_t receiver_credit_session_high; + uint64_t receiver_credit_session_low; + uint64_t receiver_credit_epoch; }; +struct XferDataReplyHeader { + uint64_t protocol_magic; +}; +static_assert(sizeof(LegacyXferDataDesc) == 16); +static_assert(sizeof(XferDataDesc) == 56); +static_assert(sizeof(XferDataReplyHeader) == 8); // Standard TCP RPC payload boundaries. READ retains the server's 1 GiB cap; // WRITE uses a 32-bit RPC attachment that includes XferDataDesc. @@ -139,6 +177,10 @@ using OnReceiveBootstrap = using OnReceiveUbBootstrap = std::function; +using OnExchangeReceiverCredit = + std::function; + using OnNotify = std::function; class ControlClient { @@ -164,13 +206,24 @@ class ControlClient { const UbBootstrapDesc& request, UbBootstrapDesc& response); + static Status exchangeReceiverCredit( + const std::string& server_addr, + const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& response); + static Status sendData(const std::string& server_addr, uint64_t peer_mem_addr, void* local_mem_addr, - size_t length); + size_t length, + uint64_t receiver_credit_session_high = 0, + uint64_t receiver_credit_session_low = 0, + uint64_t receiver_credit_epoch = 0); static Status recvData(const std::string& server_addr, uint64_t peer_mem_addr, void* local_mem_addr, - size_t length); + size_t length, + uint64_t receiver_credit_session_high = 0, + uint64_t receiver_credit_session_low = 0, + uint64_t receiver_credit_epoch = 0); static Status notify(const std::string& server_addr, const Notification& message); @@ -224,6 +277,11 @@ class ControlService { ub_bootstrap_callback_ = callback; } + void setReceiverCreditCallback(const OnExchangeReceiverCredit& callback) { + std::lock_guard lock(receiver_credit_callback_mutex_); + receiver_credit_callback_ = callback; + } + void setNotifyCallback(const OnNotify& callback); Status start(uint16_t& port, bool ipv6_ = false, size_t threads = 1); @@ -237,6 +295,9 @@ class ControlService { void onBootstrapUb(const std::string_view& request, std::string& response); + void onExchangeReceiverCredit(const std::string_view& request, + std::string& response); + void onSendData(const std::string_view& request, std::string& response); void onRecvData(const std::string_view& request, std::string& response); @@ -280,6 +341,8 @@ class ControlService { std::mutex notify_cb_mutex_; std::condition_variable notify_cb_cv_; size_t notify_callbacks_in_flight_ = 0; + std::mutex receiver_credit_callback_mutex_; + OnExchangeReceiverCredit receiver_credit_callback_; OnNotify notify_callback_; static thread_local const ControlService* active_notify_service_; diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/receiver_credit.h b/mooncake-transfer-engine/tent/include/tent/runtime/receiver_credit.h index 09b243ab13..76ad717327 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/receiver_credit.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/receiver_credit.h @@ -6,93 +6,352 @@ #include #include +#include #include +#include #include #include #include #include "tent/common/status.h" +#include "tent/thirdparty/nlohmann/json.h" namespace mooncake::tent { +// A receiver session changes whenever a receiver process restarts. A sender +// instance has the same representation, but remains a distinct type so the +// two identities cannot be accidentally exchanged at an API boundary. struct ReceiverSessionId { uint64_t high{0}, low{0}; - bool operator==(const ReceiverSessionId& o) const { - return high == o.high && low == o.low; - } + bool operator==(const ReceiverSessionId&) const = default; + [[nodiscard]] bool empty() const noexcept { return high == 0 && low == 0; } }; + +struct SenderInstanceId { + uint64_t high{0}, low{0}; + bool operator==(const SenderInstanceId&) const = default; + [[nodiscard]] bool empty() const noexcept { return high == 0 && low == 0; } +}; + struct CreditKey { ReceiverSessionId receiver_session; - uint64_t sender_peer{0}; + SenderInstanceId sender_instance; uint32_t qos_class{0}; - bool operator==(const CreditKey& o) const { - return receiver_session == o.receiver_session && - sender_peer == o.sender_peer && qos_class == o.qos_class; - } + bool operator==(const CreditKey&) const = default; }; + struct CreditKeyHash { size_t operator()(const CreditKey&) const noexcept; }; + enum class CreditResource : uint16_t { DataBytes = 1, RequestSlots, StagingSlots, ConsumerSlots }; + constexpr size_t kCreditResourceCount = 4; +constexpr uint16_t kReceiverCreditProtocolVersion = 1; +constexpr uint16_t kReceiverCreditRequired = 1U << 0; + +// A cumulative grant. grant_total never decreases within a session/epoch. struct CreditAmount { - CreditResource resource; + CreditResource resource{CreditResource::DataBytes}; uint64_t grant_total{0}; }; + +// A cumulative counter used for release reports and desired grant totals. +struct CreditCounter { + CreditResource resource{CreditResource::DataBytes}; + uint64_t total{0}; +}; + struct CreditCharge { std::vector> resources; }; + +struct ReceiverCreditAdvertV1 { + uint16_t schema_version{kReceiverCreditProtocolVersion}; + uint16_t flags{kReceiverCreditRequired}; + ReceiverSessionId receiver_session_id; + uint64_t epoch{0}; + uint32_t freshness_ttl_ms{0}; + std::vector resources; + // Explicit capacities let a sender reject a request that can never fit + // instead of leaving it indefinitely deferred behind the credit gate. + std::vector capacities; +}; + struct ReceiverCreditUpdateV1 { - uint16_t schema_version{1}, flags{0}; + uint16_t schema_version{kReceiverCreditProtocolVersion}, flags{0}; uint32_t qos_class{0}; ReceiverSessionId receiver_session_id; uint64_t epoch{0}, sequence{0}; uint32_t freshness_ttl_ms{0}; std::vector grants; }; + +// One cumulative exchange is sufficient for initial credit, replenishment, +// replay recovery, and terminal release reporting. desired_grant_totals is +// absolute (not a delta), making retries idempotent. +struct ReceiverCreditExchangeRequestV1 { + uint16_t schema_version{kReceiverCreditProtocolVersion}, flags{0}; + ReceiverSessionId expected_receiver_session_id; + uint64_t epoch{0}; + SenderInstanceId sender_instance_id; + uint32_t qos_class{0}; + uint64_t known_update_sequence{0}; + uint64_t report_sequence{0}; + std::vector released_totals; + std::vector desired_grant_totals; +}; + +struct ReceiverCreditExchangeReplyV1 { + uint16_t schema_version{kReceiverCreditProtocolVersion}, flags{0}; + SenderInstanceId sender_instance_id; + ReceiverCreditUpdateV1 update; + std::string reply_msg; +}; + enum class CreditUpdateDisposition : uint8_t { Applied, DuplicateOrOld, SequenceGap }; -// Private, sender-side state model. It has no network or Admission integration. +using CreditNowProvider = std::function; + +// Sender-side cumulative ledger. A successful transport handoff consumes a +// grant permanently; terminal completion reports a cumulative release to the +// receiver, which is the only authority allowed to mint a replacement grant. class SenderCreditLedger { public: - explicit SenderCreditLedger(size_t max_entries = 1024) - : max_entries_(max_entries) {} + explicit SenderCreditLedger(size_t max_entries = 1024, + uint32_t max_freshness_ttl_ms = 60000, + CreditNowProvider now_provider = {}); + Status activate(const CreditKey&, uint64_t epoch); - // Removes an exactly matched epoch after the caller has fenced and drained - // (or failed) its transport-owned work. An old cleanup cannot erase a - // reactivated, newer epoch. Status deactivate(const CreditKey&, uint64_t epoch); + Status deactivateSession(const ReceiverSessionId&); Status applyUpdate(const CreditKey&, const ReceiverCreditUpdateV1&, CreditUpdateDisposition&); + Status tryReserve(const CreditKey&, const CreditCharge&); - // Only for work not yet handed to a transport; completions need a new - // grant. + // Only for a failed submit that is proven to have handed no work to a + // transport. Consumption remains cumulative and the same amount is added + // to released, so the receiver (rather than this sender) reclaims it. Status rollbackReservation(const CreditKey&, const CreditCharge&); + // Records terminal completion of committed work. It deliberately does + // not reduce consumed; the receiver must authorize replacement credit. + Status releaseCommitted(const CreditKey&, const CreditCharge&); + + // Builds the next idempotent exchange. The desired totals request enough + // credit for `next_charge` in addition to all locally consumed credit. + Status prepareExchange(const CreditKey&, const CreditCharge& next_charge, + ReceiverCreditExchangeRequestV1&); + // Reports terminal releases without asking the receiver to replenish an + // idle sender. A later prepareExchange() requests credit on demand. + Status prepareReleaseExchange(const CreditKey&, + ReceiverCreditExchangeRequestV1&); + Status available(const CreditKey&, CreditResource, uint64_t&) const; Status consumed(const CreditKey&, CreditResource, uint64_t&) const; + Status released(const CreditKey&, CreditResource, uint64_t&) const; private: struct Entry { - uint64_t epoch{0}, last_sequence{0}; + uint64_t epoch{0}, last_sequence{0}, report_sequence{0}; + uint64_t expires_at_ns{0}; bool has_update{false}; - std::array grants{}, consumed{}; + std::array grants{}, consumed{}, + released{}; }; + static Status resourceIndex(CreditResource, size_t&); static Status normalize(const CreditCharge&, std::array&); + uint64_t nowNanos() const; + Status requireFresh(const Entry&) const; + mutable std::mutex mutex_; const size_t max_entries_; + const uint32_t max_freshness_ttl_ms_; + CreditNowProvider now_provider_; + std::unordered_map entries_; +}; + +struct ReceiverCreditLimits { + std::array capacity{}; + uint32_t freshness_ttl_ms{1000}; + size_t max_senders{1024}; + // Replay history is retained for the receiver session so an old initial + // demand cannot be accepted again after all of its grants are released. + // The bound is fail-closed; entries are reset only by a new receiver + // session, which also fences every old request on the wire. + size_t max_entries{4096}; +}; + +// Receiver-side grant authority. All resources in one desired request are +// granted atomically, and aggregate outstanding authorization can never exceed +// the configured capacity. Expiry is sender-side freshness only: this class +// conservatively does not recycle an unreachable sender without a transport +// fence proving its old DMA is gone. +class ReceiverCreditAuthority { + public: + ReceiverCreditAuthority(ReceiverSessionId session, uint64_t epoch, + ReceiverCreditLimits limits); + + Status status() const { return status_; } + ReceiverCreditAdvertV1 advert() const; + Status exchange(const ReceiverCreditExchangeRequestV1&, + ReceiverCreditExchangeReplyV1&); + Status outstanding(CreditResource, uint64_t&) const; + + private: + struct Entry { + uint64_t last_report_sequence{0}; + uint64_t update_sequence{0}; + std::array grants{}, released{}; + }; + + static Status validateLimits(const ReceiverCreditLimits&); + static Status normalizeCounters(const std::vector&, + std::array&, + std::array&); + void fillReply(const CreditKey&, const Entry&, + ReceiverCreditExchangeReplyV1&) const; + + ReceiverSessionId session_; + uint64_t epoch_{0}; + ReceiverCreditLimits limits_; + Status status_; + mutable std::mutex mutex_; + std::array outstanding_{}; std::unordered_map entries_; }; +inline void to_json(nlohmann::json& j, const ReceiverSessionId& id) { + j = nlohmann::json{{"high", id.high}, {"low", id.low}}; +} +inline void from_json(const nlohmann::json& j, ReceiverSessionId& id) { + j.at("high").get_to(id.high); + j.at("low").get_to(id.low); +} +inline void to_json(nlohmann::json& j, const SenderInstanceId& id) { + j = nlohmann::json{{"high", id.high}, {"low", id.low}}; +} +inline void from_json(const nlohmann::json& j, SenderInstanceId& id) { + j.at("high").get_to(id.high); + j.at("low").get_to(id.low); +} +inline void to_json(nlohmann::json& j, const CreditResource& resource) { + j = static_cast(resource); +} +inline void from_json(const nlohmann::json& j, CreditResource& resource) { + resource = static_cast(j.get()); +} +inline void to_json(nlohmann::json& j, const CreditAmount& amount) { + j = nlohmann::json{{"resource", amount.resource}, + {"grant_total", amount.grant_total}}; +} +inline void from_json(const nlohmann::json& j, CreditAmount& amount) { + j.at("resource").get_to(amount.resource); + j.at("grant_total").get_to(amount.grant_total); +} +inline void to_json(nlohmann::json& j, const CreditCounter& counter) { + j = nlohmann::json{{"resource", counter.resource}, + {"total", counter.total}}; +} +inline void from_json(const nlohmann::json& j, CreditCounter& counter) { + j.at("resource").get_to(counter.resource); + j.at("total").get_to(counter.total); +} +inline void to_json(nlohmann::json& j, const ReceiverCreditAdvertV1& advert) { + j = nlohmann::json{{"schema_version", advert.schema_version}, + {"flags", advert.flags}, + {"receiver_session_id", advert.receiver_session_id}, + {"epoch", advert.epoch}, + {"freshness_ttl_ms", advert.freshness_ttl_ms}, + {"resources", advert.resources}, + {"capacities", advert.capacities}}; +} +inline void from_json(const nlohmann::json& j, ReceiverCreditAdvertV1& advert) { + advert.schema_version = j.value("schema_version", uint16_t{0}); + advert.flags = j.value("flags", uint16_t{0}); + j.at("receiver_session_id").get_to(advert.receiver_session_id); + advert.epoch = j.value("epoch", uint64_t{0}); + advert.freshness_ttl_ms = j.value("freshness_ttl_ms", uint32_t{0}); + advert.resources = j.value("resources", std::vector{}); + advert.capacities = j.value("capacities", std::vector{}); +} +inline void to_json(nlohmann::json& j, const ReceiverCreditUpdateV1& update) { + j = nlohmann::json{{"schema_version", update.schema_version}, + {"flags", update.flags}, + {"qos_class", update.qos_class}, + {"receiver_session_id", update.receiver_session_id}, + {"epoch", update.epoch}, + {"sequence", update.sequence}, + {"freshness_ttl_ms", update.freshness_ttl_ms}, + {"grants", update.grants}}; +} +inline void from_json(const nlohmann::json& j, ReceiverCreditUpdateV1& update) { + update.schema_version = j.value("schema_version", uint16_t{0}); + update.flags = j.value("flags", uint16_t{0}); + update.qos_class = j.value("qos_class", uint32_t{0}); + j.at("receiver_session_id").get_to(update.receiver_session_id); + update.epoch = j.value("epoch", uint64_t{0}); + update.sequence = j.value("sequence", uint64_t{0}); + update.freshness_ttl_ms = j.value("freshness_ttl_ms", uint32_t{0}); + update.grants = j.value("grants", std::vector{}); +} +inline void to_json(nlohmann::json& j, + const ReceiverCreditExchangeRequestV1& request) { + j = nlohmann::json{ + {"schema_version", request.schema_version}, + {"flags", request.flags}, + {"expected_receiver_session_id", request.expected_receiver_session_id}, + {"epoch", request.epoch}, + {"sender_instance_id", request.sender_instance_id}, + {"qos_class", request.qos_class}, + {"known_update_sequence", request.known_update_sequence}, + {"report_sequence", request.report_sequence}, + {"released_totals", request.released_totals}, + {"desired_grant_totals", request.desired_grant_totals}}; +} +inline void from_json(const nlohmann::json& j, + ReceiverCreditExchangeRequestV1& request) { + request.schema_version = j.value("schema_version", uint16_t{0}); + request.flags = j.value("flags", uint16_t{0}); + j.at("expected_receiver_session_id") + .get_to(request.expected_receiver_session_id); + request.epoch = j.value("epoch", uint64_t{0}); + j.at("sender_instance_id").get_to(request.sender_instance_id); + request.qos_class = j.value("qos_class", uint32_t{0}); + request.known_update_sequence = + j.value("known_update_sequence", uint64_t{0}); + request.report_sequence = j.value("report_sequence", uint64_t{0}); + request.released_totals = + j.value("released_totals", std::vector{}); + request.desired_grant_totals = + j.value("desired_grant_totals", std::vector{}); +} +inline void to_json(nlohmann::json& j, + const ReceiverCreditExchangeReplyV1& reply) { + j = nlohmann::json{{"schema_version", reply.schema_version}, + {"flags", reply.flags}, + {"sender_instance_id", reply.sender_instance_id}, + {"update", reply.update}, + {"reply_msg", reply.reply_msg}}; +} +inline void from_json(const nlohmann::json& j, + ReceiverCreditExchangeReplyV1& reply) { + reply.schema_version = j.value("schema_version", uint16_t{0}); + reply.flags = j.value("flags", uint16_t{0}); + j.at("sender_instance_id").get_to(reply.sender_instance_id); + j.at("update").get_to(reply.update); + reply.reply_msg = j.value("reply_msg", ""); +} + } // namespace mooncake::tent #endif diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/segment.h b/mooncake-transfer-engine/tent/include/tent/runtime/segment.h index 63d73df312..1f6d8f846a 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/segment.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/segment.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "tent/common/concurrent/rw_spinlock.h" #include "tent/common/concurrent/thread_local_storage.h" #include "tent/runtime/metastore.h" +#include "tent/runtime/receiver_credit.h" #include "tent/rpc/rpc.h" #include "tent/runtime/topology.h" #include "tent/thirdparty/nlohmann/json.h" @@ -107,6 +109,11 @@ struct MemorySegmentDesc { // int maps to TransportType enum std::unordered_map transport_attrs; + // Optional runtime-level receiver-credit advertisement. Per-sender + // grants are never published in SegmentDesc; they are obtained through + // the credit exchange RPC. + std::optional receiver_credit; + public: // Get transport attributes for a specific transport type std::string& getTransportAttrs(int transport_type) { @@ -129,6 +136,7 @@ inline void to_json(json& j, const MemorySegmentDesc& m) { {"devices", m.devices}, {"topology", m.topology.toString()}, {"transport_attrs", m.transport_attrs}}; + if (m.receiver_credit) j["receiver_credit"] = *m.receiver_credit; } inline void from_json(const json& j, MemorySegmentDesc& m) { @@ -142,6 +150,11 @@ inline void from_json(const json& j, MemorySegmentDesc& m) { if (j.contains("transport_attrs")) { j.at("transport_attrs").get_to(m.transport_attrs); } + m.receiver_credit.reset(); + if (j.contains("receiver_credit")) { + m.receiver_credit = + j.at("receiver_credit").get(); + } } struct FileSegmentDesc { diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h index 1ae7654de1..c4f5fcf682 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/transfer_engine_impl.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "tent/common/status.h" #include "tent/common/types.h" #include "tent/runtime/admission_queue.h" +#include "tent/runtime/receiver_credit.h" #include "tent/runtime/transport.h" #include "tent/runtime/transport_selector.h" #include "tent/runtime/hp_tcp_transport_config.h" @@ -360,6 +362,16 @@ class TransferEngineImpl { } } + // Test-only hook: CI hosts have no device memory, so no request is ever + // staged. A test can force a staging policy for the requests it selects + // (return true and fill `policy`) to drive a staged owner end to end. + // Set before the first submit; not thread-safe with in-flight transfers. + using StagingPolicyForTest = + std::function&)>; + void setStagingPolicyForTest(StagingPolicyForTest hook) { + staging_policy_for_test_ = std::move(hook); + } + Status publishRuntimeConfigForTest( std::shared_ptr snapshot) { if (!snapshot) { @@ -477,7 +489,7 @@ class TransferEngineImpl { QueueOwnerKind owner_kind) const; Status prepareSubmit(Batch* batch, const std::vector& request_list, - PreparedSubmit& prepared, + QueueOwnerKind owner_kind, PreparedSubmit& prepared, bool require_post_submit_cancellation = false); Status commitPreparedSubmit(Batch* batch, const PreparedSubmit& prepared); @@ -496,6 +508,43 @@ class TransferEngineImpl { Status dispatchQueuedOwner(QueueOwnerId owner_id); + struct ReceiverCreditTarget; + struct ReceiverCreditReservation; + + Status resolveReceiverCreditTarget(const Request& request, + ReceiverCreditTarget& target) const; + + Status acquireReceiverCredit(const Request& request, + ReceiverCreditReservation& reservation, + bool& acquired); + + Status validateReceiverCreditCapacity( + const Request& request, const ReceiverCreditTarget& target) const; + + void bindReceiverCreditFence(Transport::SubBatchRef, + const ReceiverCreditReservation&) const; + + void bindReceiverCreditFence(Transport::SubBatchRef, const Request&) const; + + void rollbackReceiverCredit(ReceiverCreditReservation& reservation); + + void releaseReceiverCredit(ReceiverCreditReservation& reservation); + + void scheduleReceiverCreditRelease(const CreditKey&, SegmentID, + const std::string&, uint64_t epoch); + + bool tryReportReceiverCreditRelease(const CreditKey&); + + void progressReceiverCreditReleases(); + + void observeReceiverCreditSession(SegmentID, + const ReceiverSessionId& session); + + Status exchangeReceiverCredit( + SegmentID target_id, const std::string& rpc_server_addr, + const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& reply); + Status markQueuedOwnerSubmitted(QueueOwnerId owner_id); Status finishQueuedOwner(QueueOwnerId owner_id, @@ -614,12 +663,59 @@ class TransferEngineImpl { std::chrono::microseconds progress_fallback_interval{50000}; }; + struct ReceiverCreditConfig { + bool sender_enabled{false}; + bool receiver_enabled{false}; + ReceiverCreditLimits limits{}; + size_t max_sender_entries{4096}; + uint32_t max_freshness_ttl_ms{60000}; + }; + + struct ReceiverCreditTarget { + bool required{false}; + ReceiverCreditAdvertV1 advert{}; + std::string rpc_server_addr; + }; + + struct ReceiverCreditReservation { + bool active{false}; + bool committed{false}; + CreditKey key{}; + CreditCharge charge{}; + SegmentID target_id{LOCAL_SEGMENT_ID}; + std::string rpc_server_addr; + uint64_t epoch{0}; + }; + + struct PendingReceiverCreditRelease { + SegmentID target_id{LOCAL_SEGMENT_ID}; + std::string rpc_server_addr; + uint64_t epoch{0}; + uint32_t attempts{0}; + std::chrono::steady_clock::time_point retry_at{}; + }; + + // A demand exchange the receiver did not satisfy (or that failed). A + // deferred owner is retried on every refill, and the exchange is an RPC + // made under progress_mutex_; repeating it on every poll can starve the + // in-flight owners whose completion is what returns the credit. Until + // this sender releases credit for the key, asks for less, or the backoff + // expires, the owner is requeued without another RPC. + struct ReceiverCreditDenial { + std::array released{}; // DataBytes, RequestSlots + std::array charge{}; + uint32_t attempts{0}; + std::chrono::steady_clock::time_point retry_at{}; + }; + struct QueuedOwnerState { Batch* batch{nullptr}; size_t owner_task_id{0}; std::vector public_task_ids; size_t byte_charge{0}; bool in_dispatch_window{false}; + bool receiver_credit_exempt{false}; + ReceiverCreditReservation credit; }; private: @@ -648,12 +744,23 @@ class TransferEngineImpl { std::string local_segment_name_; std::unique_ptr staging_proxy_; + StagingPolicyForTest staging_policy_for_test_; bool merge_requests_; std::shared_ptr runtime_config_snapshot_; bool notify_rpc_fallback_{true}; bool enable_progress_worker_{false}; RuntimeQueueConfig runtime_queue_config_; + ReceiverCreditConfig receiver_credit_config_; std::unique_ptr runtime_queue_; + SenderInstanceId sender_instance_id_; + std::unique_ptr sender_credit_ledger_; + std::unique_ptr receiver_credit_authority_; + std::unordered_map + pending_credit_releases_; + std::unordered_map receiver_credit_sessions_; + std::unordered_map + receiver_credit_denials_; + uint64_t receiver_credit_demand_exchanges_{0}; std::unordered_map queued_owners_; size_t dispatch_inflight_owners_{0}; size_t dispatch_inflight_bytes_{0}; diff --git a/mooncake-transfer-engine/tent/include/tent/runtime/transport.h b/mooncake-transfer-engine/tent/include/tent/runtime/transport.h index f81fe66ec2..daa54da4da 100644 --- a/mooncake-transfer-engine/tent/include/tent/runtime/transport.h +++ b/mooncake-transfer-engine/tent/include/tent/runtime/transport.h @@ -61,6 +61,13 @@ class Transport { // no pool (default spray). Carried like device_mask, from the matched // SelectionPolicy down to each RdmaTask. std::string qp_pool; + // Runtime receiver-credit authorization bound to the next submitted + // task. Native transports copy this fence into their asynchronous + // work so a refreshed segment cannot post under a different receiver + // session than the one that granted capacity. + uint64_t receiver_credit_session_high{0}; + uint64_t receiver_credit_session_low{0}; + uint64_t receiver_credit_epoch{0}; BatchID progress_batch_id{0}; std::function notify_progress; }; diff --git a/mooncake-transfer-engine/tent/include/tent/transport/hp_tcp/hp_tcp_transport.h b/mooncake-transfer-engine/tent/include/tent/transport/hp_tcp/hp_tcp_transport.h index 302a1e41ae..e2763e9fef 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/hp_tcp/hp_tcp_transport.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/hp_tcp/hp_tcp_transport.h @@ -95,6 +95,10 @@ class HighPerformanceTcpTransport final : public Transport { // peer so periodic fan-out cannot pin a peer to one lane or rail. std::mutex lane_sequence_mutex_; std::map next_lane_sequence_; + // per target: outstanding operations per lane, so a submit can pick the + // least-loaded lane + std::map>>> + lane_load_; std::atomic installed_{false}; std::atomic stopping_{false}; mutable std::mutex lifecycle_mutex_; diff --git a/mooncake-transfer-engine/tent/include/tent/transport/rdma/slice.h b/mooncake-transfer-engine/tent/include/tent/transport/rdma/slice.h index 5a22df927e..8219bac535 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/rdma/slice.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/rdma/slice.h @@ -107,6 +107,9 @@ struct RdmaTask { // no pool selected: slices spray across all data QPs as before. Resolved // from SelectionResult.qp_pool at task creation. std::string qp_pool; + uint64_t receiver_credit_session_high{0}; + uint64_t receiver_credit_session_low{0}; + uint64_t receiver_credit_epoch{0}; volatile TransferStatusEnum status_word; volatile size_t transferred_bytes; std::atomic success_slices{0}; diff --git a/mooncake-transfer-engine/tent/include/tent/transport/tcp/tcp_transport.h b/mooncake-transfer-engine/tent/include/tent/transport/tcp/tcp_transport.h index 1f0fc77d9a..d6ed97e340 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/tcp/tcp_transport.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/tcp/tcp_transport.h @@ -55,6 +55,9 @@ struct TcpTask { std::atomic transferred_bytes{0}; bool non_replayable_failure{false}; // Published by status_word. uint64_t target_addr = 0; + uint64_t receiver_credit_session_high{0}; + uint64_t receiver_credit_session_low{0}; + uint64_t receiver_credit_epoch{0}; TcpTask() = default; TcpTask(TcpTask &&other) noexcept @@ -65,7 +68,10 @@ struct TcpTask { transferred_bytes( other.transferred_bytes.load(std::memory_order_relaxed)), non_replayable_failure(other.non_replayable_failure), - target_addr(other.target_addr) {} + target_addr(other.target_addr), + receiver_credit_session_high(other.receiver_credit_session_high), + receiver_credit_session_low(other.receiver_credit_session_low), + receiver_credit_epoch(other.receiver_credit_epoch) {} TcpTask(const TcpTask &) = delete; TcpTask &operator=(const TcpTask &) = delete; }; @@ -122,8 +128,7 @@ class TcpTransport : public Transport { Status doTransferWithRetry(TcpTask *task); - Status findRemoteSegment(uint64_t dest_addr, uint64_t length, - uint64_t target_id, std::string &rpc_server_addr); + Status findRemoteSegment(const TcpTask &task, std::string &rpc_server_addr); private: bool installed_; diff --git a/mooncake-transfer-engine/tent/include/tent/transport/ub/endpoint.h b/mooncake-transfer-engine/tent/include/tent/transport/ub/endpoint.h index 3dfe4533ba..3679bffaf3 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/ub/endpoint.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/ub/endpoint.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "tent/common/status.h" @@ -116,6 +117,11 @@ class UbEndpoint final : public std::enable_shared_from_this { [[nodiscard]] uint64_t peerGeneration() const noexcept { return peer_generation_.load(std::memory_order_acquire); } + [[nodiscard]] bool peerSupportsCapability( + std::string_view capability) const; + [[nodiscard]] bool peerMatchesReceiverCredit(uint64_t session_high, + uint64_t session_low, + uint64_t epoch) const; [[nodiscard]] uint64_t outstandingWrs() const noexcept { return outstanding_wrs_.load(std::memory_order_relaxed); } diff --git a/mooncake-transfer-engine/tent/include/tent/transport/ub/slice.h b/mooncake-transfer-engine/tent/include/tent/transport/ub/slice.h index 3b70c3dc21..7428c5b0d8 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/ub/slice.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/ub/slice.h @@ -147,6 +147,9 @@ struct UbSliceSpec { size_t length{0}; size_t request_offset{0}; uint32_t max_retries{0}; + uint64_t receiver_credit_session_high{0}; + uint64_t receiver_credit_session_low{0}; + uint64_t receiver_credit_epoch{0}; }; struct UbAttemptToken { diff --git a/mooncake-transfer-engine/tent/include/tent/transport/ub/workers.h b/mooncake-transfer-engine/tent/include/tent/transport/ub/workers.h index a2e9031fe8..7f3fc4e1dd 100644 --- a/mooncake-transfer-engine/tent/include/tent/transport/ub/workers.h +++ b/mooncake-transfer-engine/tent/include/tent/transport/ub/workers.h @@ -147,6 +147,8 @@ class UbWorkers final { Status start(); Status stop(); + Status submitBatch(const std::vector& tasks, + uint64_t device_mask = ~0ULL); Status submit(const UbTask::Ptr& task, uint64_t device_mask = ~0ULL); Status cancel(const UbTask::Ptr& task); diff --git a/mooncake-transfer-engine/tent/src/runtime/admission_queue.cpp b/mooncake-transfer-engine/tent/src/runtime/admission_queue.cpp index 74c19416ad..3a273bf953 100644 --- a/mooncake-transfer-engine/tent/src/runtime/admission_queue.cpp +++ b/mooncake-transfer-engine/tent/src/runtime/admission_queue.cpp @@ -78,6 +78,30 @@ Status validateLimits(const QueueLimits& limits) { LocalTransferAdmissionQueue::LocalTransferAdmissionQueue(QueueLimits limits) : limits_(limits), limits_status_(validateLimits(limits)) {} +void LocalTransferAdmissionQueue::enqueueOwner(QueueOwnerId owner_id) { + fifo_.push_back(owner_id); + if (!limits_.deadline_aware) return; + + // Promotion and defer can both disturb the deque's prior global order. + // Re-establish the explicit (round, EDF) ordering on admission instead of + // passing a potentially rotated range to an ordered insertion algorithm. + // stable_sort preserves FIFO order for equal deadlines. + std::stable_sort( + fifo_.begin(), fifo_.end(), [this](QueueOwnerId lhs, QueueOwnerId rhs) { + const auto lhs_it = owners_.find(lhs); + const auto rhs_it = owners_.find(rhs); + if (lhs_it == owners_.end()) return false; + if (rhs_it == owners_.end()) return true; + const auto& lhs_owner = lhs_it->second; + const auto& rhs_owner = rhs_it->second; + if (lhs_owner.scheduling_round != rhs_owner.scheduling_round) { + return lhs_owner.scheduling_round < rhs_owner.scheduling_round; + } + return deadlineKey(lhs_owner.request.deadline_ns) < + deadlineKey(rhs_owner.request.deadline_ns); + }); +} + Status LocalTransferAdmissionQueue::tryAdmit( const QueueSubmit& submit, std::vector& admitted_owner_ids) { admitted_owner_ids.clear(); @@ -181,6 +205,17 @@ Status LocalTransferAdmissionQueue::tryAdmit( owner.kind = owner_input.kind; owner.transport = owner_input.transport; owner.degradation_eligible = owner_input.degradation_eligible; + if (limits_.deadline_aware) { + for (const auto queued_id : fifo_) { + auto queued_it = owners_.find(queued_id); + if (queued_it != owners_.end() && + queued_it->second.state == QueueState::Queued) { + owner.scheduling_round = + std::max(owner.scheduling_round, + queued_it->second.scheduling_round); + } + } + } owners_.emplace(owner_id, owner); public_to_owner_[{submit.batch_token, owner_input.owner_task_id}] = @@ -189,26 +224,8 @@ Status LocalTransferAdmissionQueue::tryAdmit( public_to_owner_[{submit.batch_token, derived_task_id}] = owner_id; } // RFC #2519 step 2: keep fifo_ ordered on admission so pickForDispatch - // never has to re-sort. Default (deadline_aware == false) appends in - // strict FIFO. When deadline-aware, insert at the earliest-deadline- - // first position; upper_bound places a new owner *after* existing - // owners with the same deadline, preserving FIFO order among ties. - if (limits_.deadline_aware) { - const uint64_t key = deadlineKey(owner.request.deadline_ns); - auto pos = std::upper_bound( - fifo_.begin(), fifo_.end(), key, - [this](uint64_t k, QueueOwnerId id) { - auto it = owners_.find(id); - uint64_t d = - (it == owners_.end()) - ? std::numeric_limits::max() - : deadlineKey(it->second.request.deadline_ns); - return k < d; - }); - fifo_.insert(pos, owner_id); - } else { - fifo_.push_back(owner_id); - } + // never has to re-sort. + enqueueOwner(owner_id); admitted_owner_ids.push_back(owner_id); } @@ -271,16 +288,26 @@ std::vector LocalTransferAdmissionQueue::pickForDispatch( .count())) : 0; - // Deadline proximity promotion: partition fifo_ so owners with critical - // slack (deadline approaching within promotion_slack_ns) appear before - // owners with comfortable slack or no deadline. stable_partition preserves - // relative EDF order within each group. + // Deadline proximity promotion applies only inside the earliest + // scheduling round. A resource-starved owner that defer() moved to the + // next round must not jump back ahead of current-round waiters merely + // because its deadline is urgent. if (promotion_enabled) { + uint64_t current_round = std::numeric_limits::max(); + for (const auto owner_id : fifo_) { + auto owner_it = owners_.find(owner_id); + if (owner_it != owners_.end() && + owner_it->second.state == QueueState::Queued) { + current_round = + std::min(current_round, owner_it->second.scheduling_round); + } + } std::stable_partition(fifo_.begin(), fifo_.end(), [&](QueueOwnerId id) { auto it = owners_.find(id); if (it == owners_.end() || it->second.state != QueueState::Queued) { return false; } + if (it->second.scheduling_round != current_round) return false; const uint64_t dl = it->second.request.deadline_ns; if (dl == 0 || dl <= now_ns) return false; return (dl - now_ns) < limits_.promotion_slack_ns; @@ -383,6 +410,35 @@ std::vector LocalTransferAdmissionQueue::pickForDispatch( return picked; } +Status LocalTransferAdmissionQueue::defer(QueueOwnerId owner_id) { + if (owner_id == 0) { + return Status::InvalidArgument("invalid queue owner id" LOC_MARK); + } + + auto owner_it = owners_.find(owner_id); + if (owner_it == owners_.end()) { + return Status::InvalidEntry("queue owner not found" LOC_MARK); + } + if (owner_it->second.state != QueueState::Dispatching) { + return Status::InvalidEntry("queue owner is not dispatching" LOC_MARK); + } + + // A resource-deferred EDF owner advances one scheduling round. It yields + // to every owner already waiting in this round while retaining EDF order + // among owners that yielded into the next one. + if (limits_.deadline_aware) { + if (owner_it->second.scheduling_round == + std::numeric_limits::max()) { + return Status::InternalError( + "admission scheduling round exhausted" LOC_MARK); + } + ++owner_it->second.scheduling_round; + } + owner_it->second.state = QueueState::Queued; + enqueueOwner(owner_id); + return Status::OK(); +} + Status LocalTransferAdmissionQueue::complete( QueueOwnerId owner_id, TransferStatusEnum terminal_status) { if (owner_id == 0) { diff --git a/mooncake-transfer-engine/tent/src/runtime/control_plane.cpp b/mooncake-transfer-engine/tent/src/runtime/control_plane.cpp index fe378fea55..60bfb22d4f 100644 --- a/mooncake-transfer-engine/tent/src/runtime/control_plane.cpp +++ b/mooncake-transfer-engine/tent/src/runtime/control_plane.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,113 @@ thread_local const ControlService* ControlService::active_notify_service_ = nullptr; thread_local CoroRpcAgent tl_rpc_agent; +namespace { + +constexpr uint64_t kDelegatedCreditProtocolMagic = 0x54454e54444c4732ULL; + +struct ParsedXferData { + uint64_t peer_mem_addr{0}; + size_t length{0}; + size_t header_size{0}; + bool version_two{false}; + uint64_t receiver_credit_session_high{0}; + uint64_t receiver_credit_session_low{0}; + uint64_t receiver_credit_epoch{0}; +}; + +Status parseXferData(const std::string_view& request, bool carries_payload, + ParsedXferData& parsed) { + if (request.size() < sizeof(LegacyXferDataDesc)) { + return Status::InvalidArgument("request too short" LOC_MARK); + } + LegacyXferDataDesc legacy; + memcpy(&legacy, request.data(), sizeof(legacy)); + const uint64_t length = le64toh(legacy.length); + if (length > std::numeric_limits::max()) { + return Status::InvalidArgument("transfer length overflow" LOC_MARK); + } + + parsed = ParsedXferData{}; + parsed.peer_mem_addr = le64toh(legacy.peer_mem_addr); + parsed.length = static_cast(length); + parsed.header_size = sizeof(LegacyXferDataDesc); + + if (request.size() >= sizeof(XferDataDesc)) { + XferDataDesc current; + memcpy(¤t, request.data(), sizeof(current)); + const bool magic_matches = + le64toh(current.protocol_magic) == kXferDataProtocolMagic; + const bool size_matches = + carries_payload + ? parsed.length <= std::numeric_limits::max() - + sizeof(XferDataDesc) && + request.size() == sizeof(XferDataDesc) + parsed.length + : request.size() == sizeof(XferDataDesc); + if (magic_matches && size_matches) { + parsed.header_size = sizeof(XferDataDesc); + parsed.version_two = true; + parsed.peer_mem_addr = le64toh(current.peer_mem_addr); + parsed.receiver_credit_session_high = + le64toh(current.receiver_credit_session_high); + parsed.receiver_credit_session_low = + le64toh(current.receiver_credit_session_low); + parsed.receiver_credit_epoch = + le64toh(current.receiver_credit_epoch); + } + } + + if (carries_payload) { + if (parsed.length > + std::numeric_limits::max() - parsed.header_size || + request.size() != parsed.header_size + parsed.length) { + return Status::InvalidArgument( + "invalid transfer request size" LOC_MARK); + } + } else if (request.size() != parsed.header_size) { + return Status::InvalidArgument( + "invalid transfer request header size" LOC_MARK); + } + return Status::OK(); +} + +Status validateXferReceiverCreditFence(const SegmentDescRef& local_desc, + const ParsedXferData& request) { + const bool has_fence = request.receiver_credit_session_high != 0 || + request.receiver_credit_session_low != 0 || + request.receiver_credit_epoch != 0; + if (!local_desc || local_desc->type != SegmentType::Memory) { + return has_fence ? Status::InvalidEntry( + "receiver-credit target is not memory" LOC_MARK) + : Status::OK(); + } + + const auto& advert = local_desc->getMemory().receiver_credit; + if (!advert) { + return has_fence + ? Status::InvalidEntry( + "receiver-credit session is unavailable" LOC_MARK) + : Status::OK(); + } + if (advert->schema_version != kReceiverCreditProtocolVersion || + advert->flags != kReceiverCreditRequired || + advert->receiver_session_id.empty() || advert->epoch == 0) { + return Status::InvalidEntry( + "invalid local receiver-credit advertisement" LOC_MARK); + } + if (!request.version_two || + request.receiver_credit_session_high != + advert->receiver_session_id.high || + request.receiver_credit_session_low != + advert->receiver_session_id.low || + request.receiver_credit_epoch != advert->epoch) { + return Status::InvalidEntry( + "receiver-credit session fence mismatch" LOC_MARK); + } + return Status::OK(); +} + +} // namespace + Status ControlClient::getSegmentDesc(const std::string& server_addr, std::string& response) { std::string request; @@ -119,29 +227,83 @@ Status ControlClient::bootstrapUb(const std::string& server_addr, return Status::OK(); } +Status ControlClient::exchangeReceiverCredit( + const std::string& server_addr, + const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& response) { + std::string request_raw, response_raw; + json encoded = request; + request_raw = encoded.dump(); + CHECK_STATUS(tl_rpc_agent.call(server_addr, ExchangeReceiverCredit, + request_raw, response_raw)); + try { + response = + json::parse(response_raw).get(); + } catch (const std::exception& error) { + return Status::MalformedJson( + std::string("Malformed receiver-credit response: ") + error.what() + + LOC_MARK); + } + if (response.schema_version != kReceiverCreditProtocolVersion || + response.flags != 0) { + return Status::RpcServiceError( + "Unsupported receiver-credit response version" LOC_MARK); + } + if (!response.reply_msg.empty()) { + return Status::RpcServiceError(response.reply_msg); + } + return Status::OK(); +} + Status ControlClient::sendData(const std::string& server_addr, uint64_t peer_mem_addr, void* local_mem_addr, - size_t length) { - // Check before addition/allocation or reading the caller's source buffer. + size_t length, + uint64_t receiver_credit_session_high, + uint64_t receiver_credit_session_low, + uint64_t receiver_credit_epoch) { if (length > kTcpMaxWriteBytes) return Status::InvalidArgument("TCP WRITE exceeds RPC payload limit"); - std::string response; - XferDataDesc desc{htole64(peer_mem_addr), htole64(length)}; - std::string request; + std::string request, response; + const bool has_any_fence = receiver_credit_session_high != 0 || + receiver_credit_session_low != 0 || + receiver_credit_epoch != 0; + const bool has_complete_fence = (receiver_credit_session_high != 0 || + receiver_credit_session_low != 0) && + receiver_credit_epoch != 0; + if (has_any_fence && !has_complete_fence) { + return Status::InvalidArgument( + "partial SendData receiver-credit fence" LOC_MARK); + } + if (length > std::numeric_limits::max() - sizeof(XferDataDesc)) { + return Status::InvalidArgument( + "SendData request size overflow" LOC_MARK); + } + size_t header_size = sizeof(LegacyXferDataDesc); request.reserve(sizeof(XferDataDesc) + length); - request.resize(sizeof(XferDataDesc)); - memcpy(request.data(), &desc, sizeof(desc)); + if (has_complete_fence) { + const XferDataDesc desc{htole64(kXferDataV2LegacyRejectAddress), + htole64(static_cast(length)), + htole64(kXferDataProtocolMagic), + htole64(peer_mem_addr), + htole64(receiver_credit_session_high), + htole64(receiver_credit_session_low), + htole64(receiver_credit_epoch)}; + header_size = sizeof(desc); + request.resize(header_size); + memcpy(request.data(), &desc, sizeof(desc)); + } else { + const LegacyXferDataDesc desc{htole64(peer_mem_addr), + htole64(static_cast(length))}; + request.resize(header_size); + memcpy(request.data(), &desc, sizeof(desc)); + } auto& loader = Platform::getLoader(); if (loader.getMemoryType(local_mem_addr) == MTYPE_CPU) { - // Single copy into the RPC attachment; avoids the resize() zero-fill - // and the extra copy in call(). request.append(reinterpret_cast(local_mem_addr), length); } else { - // resize() zero-fills the payload, so an unchecked copy failure would - // ship zeros that the peer stores successfully and reports COMPLETED. - request.resize(sizeof(XferDataDesc) + length); + request.resize(header_size + length); CHECK_STATUS( - loader.copy(request.data() + sizeof(desc), local_mem_addr, length)); + loader.copy(request.data() + header_size, local_mem_addr, length)); } auto status = tl_rpc_agent.callOwned(server_addr, SendData, std::move(request), response); @@ -152,19 +314,70 @@ Status ControlClient::sendData(const std::string& server_addr, Status ControlClient::recvData(const std::string& server_addr, uint64_t peer_mem_addr, void* local_mem_addr, - size_t length) { + size_t length, + uint64_t receiver_credit_session_high, + uint64_t receiver_credit_session_low, + uint64_t receiver_credit_epoch) { if (length > kTcpMaxReadBytes) return Status::InvalidArgument("TCP READ exceeds RPC payload limit"); std::string request, response; - XferDataDesc desc{htole64(peer_mem_addr), htole64(length)}; - request.resize(sizeof(XferDataDesc)); - memcpy(&request[0], &desc, sizeof(desc)); + const bool has_any_fence = receiver_credit_session_high != 0 || + receiver_credit_session_low != 0 || + receiver_credit_epoch != 0; + const bool has_complete_fence = (receiver_credit_session_high != 0 || + receiver_credit_session_low != 0) && + receiver_credit_epoch != 0; + if (has_any_fence && !has_complete_fence) { + return Status::InvalidArgument( + "partial RecvData receiver-credit fence" LOC_MARK); + } + if (has_complete_fence) { + const XferDataDesc desc{htole64(kXferDataV2LegacyRejectAddress), + htole64(static_cast(length)), + htole64(kXferDataProtocolMagic), + htole64(peer_mem_addr), + htole64(receiver_credit_session_high), + htole64(receiver_credit_session_low), + htole64(receiver_credit_epoch)}; + request.resize(sizeof(desc)); + memcpy(request.data(), &desc, sizeof(desc)); + } else { + const LegacyXferDataDesc desc{htole64(peer_mem_addr), + htole64(static_cast(length))}; + request.resize(sizeof(desc)); + memcpy(request.data(), &desc, sizeof(desc)); + } auto status = tl_rpc_agent.call(server_addr, RecvData, request, response); if (!status.ok()) return status; - if (response.size() != length) + // A short reply is an error, not payload: surface the server's message so + // callers keep the rejection reason (e.g. "permission denied") instead of + // a generic string, matching the pre-v2 client behavior. An empty or + // unpadded reply still gets a descriptive fallback. + if (!has_complete_fence) { + if (response.size() != length) { + return Status::RpcServiceError( + response.empty() + ? "RecvData failed: invalid or rejected legacy response" + : response); + } + return Platform::getLoader().copy(local_mem_addr, response.data(), + length); + } + if (length > + std::numeric_limits::max() - sizeof(XferDataReplyHeader) || + response.size() != sizeof(XferDataReplyHeader) + length) { return Status::RpcServiceError( - response.empty() ? "RecvData failed: empty response" : response); - return Platform::getLoader().copy(local_mem_addr, response.data(), length); + response.empty() ? "RecvData failed: invalid or rejected response" + : response); + } + XferDataReplyHeader header; + memcpy(&header, response.data(), sizeof(header)); + if (le64toh(header.protocol_magic) != kXferDataProtocolMagic) { + return Status::RpcServiceError( + "RecvData failed: unsupported response version"); + } + return Platform::getLoader().copy( + local_mem_addr, response.data() + sizeof(XferDataReplyHeader), length); } inline void to_json(nlohmann::json& j, const Notification& n) { @@ -196,11 +409,26 @@ Status ControlClient::probe(const std::string& server_addr) { } inline void to_json(json& j, const Request& r) { - j = json{{"opcode", r.opcode == Request::READ ? "READ" : "WRITE"}, - {"source", reinterpret_cast(r.source)}, - {"target_id", r.target_id}, - {"target_offset", r.target_offset}, - {"length", r.length}}; + const bool has_complete_fence = (r.receiver_credit_session_high != 0 || + r.receiver_credit_session_low != 0) && + r.receiver_credit_epoch != 0; + j = json{ + {"opcode", r.opcode == Request::READ ? "READ" : "WRITE"}, + {"source", reinterpret_cast(r.source)}, + {"target_id", has_complete_fence ? std::numeric_limits::max() + : r.target_id}, + {"target_offset", has_complete_fence + ? std::numeric_limits::max() + : r.target_offset}, + {"length", r.length}, + {"receiver_credit_session_high", r.receiver_credit_session_high}, + {"receiver_credit_session_low", r.receiver_credit_session_low}, + {"receiver_credit_epoch", r.receiver_credit_epoch}}; + if (has_complete_fence) { + j["delegate_protocol_magic"] = kDelegatedCreditProtocolMagic; + j["actual_target_id"] = r.target_id; + j["actual_target_offset"] = r.target_offset; + } } inline void from_json(const json& j, Request& r) { @@ -213,13 +441,35 @@ inline void from_json(const json& j, Request& r) { throw std::runtime_error("Invalid opcode"); r.source = reinterpret_cast(j.at("source").get()); - r.target_id = j.at("target_id").get(); - r.target_offset = j.at("target_offset").get(); + const bool fenced_delegate = + j.value("delegate_protocol_magic", uint64_t{0}) == + kDelegatedCreditProtocolMagic; + r.target_id = fenced_delegate ? j.at("actual_target_id").get() + : j.at("target_id").get(); + r.target_offset = fenced_delegate + ? j.at("actual_target_offset").get() + : j.at("target_offset").get(); r.length = j.at("length").get(); + r.receiver_credit_session_high = + j.value("receiver_credit_session_high", uint64_t{0}); + r.receiver_credit_session_low = + j.value("receiver_credit_session_low", uint64_t{0}); + r.receiver_credit_epoch = j.value("receiver_credit_epoch", uint64_t{0}); } Status ControlClient::delegate(const std::string& server_addr, const Request& request) { + const bool has_any_fence = request.receiver_credit_session_high != 0 || + request.receiver_credit_session_low != 0 || + request.receiver_credit_epoch != 0; + const bool has_complete_fence = + (request.receiver_credit_session_high != 0 || + request.receiver_credit_session_low != 0) && + request.receiver_credit_epoch != 0; + if (has_any_fence && !has_complete_fence) { + return Status::InvalidArgument( + "partial delegated receiver-credit fence" LOC_MARK); + } std::string request_raw, response_raw; json j = request; request_raw = j.dump(); @@ -321,6 +571,11 @@ ControlService::ControlService(const std::string& type, [this](const std::string_view& request, std::string& response) { onBootstrapUb(request, response); }); + rpc_server_->registerFunction( + ExchangeReceiverCredit, + [this](const std::string_view& request, std::string& response) { + onExchangeReceiverCredit(request, response); + }); rpc_server_->registerFunction( SendData, [this](const std::string_view& request, std::string& response) { @@ -376,6 +631,10 @@ ControlService::~ControlService() { std::lock_guard lock(ub_bootstrap_callback_mutex_); ub_bootstrap_callback_ = {}; } + { + std::lock_guard lock(receiver_credit_callback_mutex_); + receiver_credit_callback_ = {}; + } rpc_server_.reset(); } @@ -519,37 +778,62 @@ void ControlService::onBootstrapUb(const std::string_view& request, response = j.dump(); } +void ControlService::onExchangeReceiverCredit(const std::string_view& request, + std::string& response) { + ReceiverCreditExchangeReplyV1 reply; + try { + auto parsed = json::parse(std::string(request)) + .get(); + Status status = Status::InvalidEntry( + "receiver credit callback is not registered" LOC_MARK); + { + // Serialize callback replacement with invocation so teardown + // cannot destroy the authority while an exchange is in flight. + std::lock_guard lock(receiver_credit_callback_mutex_); + if (receiver_credit_callback_) { + status = receiver_credit_callback_(parsed, reply); + } + } + if (!status.ok()) reply.reply_msg = status.ToString(); + } catch (const std::exception& error) { + reply.reply_msg = + std::string("Malformed receiver-credit request: ") + error.what(); + } + json encoded = reply; + response = encoded.dump(); +} + void ControlService::onSendData(const std::string_view& request, std::string& response) { - if (request.size() < sizeof(XferDataDesc)) { - response = "SendData failed: request too short"; + ParsedXferData parsed; + auto status = parseXferData(request, true, parsed); + if (!status.ok()) { + response = "SendData failed: " + status.ToString(); return; } - XferDataDesc* desc = (XferDataDesc*)request.data(); auto local_desc = manager_->getLocal(); - auto peer_mem_addr = le64toh(desc->peer_mem_addr); - auto length = le64toh(desc->length); + status = validateXferReceiverCreditFence(local_desc, parsed); + if (!status.ok()) { + response = "SendData failed: " + status.ToString(); + return; + } - // Validate request size to prevent buffer over-read - if (length > kTcpMaxWriteBytes || - length > request.size() - sizeof(XferDataDesc)) { + if (parsed.length > kTcpMaxWriteBytes) { response = "SendData failed: invalid request size"; return; } - - if (auto* buffer = local_desc->findBuffer(peer_mem_addr, length)) { + if (auto* buffer = + local_desc->findBuffer(parsed.peer_mem_addr, parsed.length)) { if (buffer->permission != kGlobalReadWrite) { response = "SendData failed: remote write permission denied"; return; } - auto status = - Platform::getLoader().copy((void*)peer_mem_addr, &desc[1], length); - if (!status.ok()) { - // A non-empty response is interpreted as an RPC error by the - // client (see ControlClient::sendData). Without this the sender's - // transfer would be reported COMPLETED even though the destination - // buffer was never written. - response = "SendData failed: copy: " + status.ToString(); + auto copy_status = Platform::getLoader().copy( + reinterpret_cast(parsed.peer_mem_addr), + const_cast(request.data() + parsed.header_size), + parsed.length); + if (!copy_status.ok()) { + response = "SendData failed: copy: " + copy_status.ToString(); } } else { response = "SendData failed: target address not in registered buffer"; @@ -558,46 +842,59 @@ void ControlService::onSendData(const std::string_view& request, void ControlService::onRecvData(const std::string_view& request, std::string& response) { - if (request.size() < sizeof(XferDataDesc)) { - response = "RecvData failed: request too short"; + ParsedXferData parsed; + auto status = parseXferData(request, false, parsed); + // Legacy readers accept any response of exactly the requested length. + // Fenced readers also require the reply magic; pad errors to keep both + // response formats unambiguous, including parse and session failures. + const size_t reply_header_size = + parsed.version_two ? sizeof(XferDataReplyHeader) : 0; + auto fail = [&response, &parsed, reply_header_size](std::string message) { + response = std::move(message); + if (parsed.length <= + std::numeric_limits::max() - reply_header_size && + response.size() == reply_header_size + parsed.length) { + response.push_back(' '); + } + }; + if (!status.ok()) { + fail("RecvData failed: " + status.ToString()); return; } - XferDataDesc* desc = (XferDataDesc*)request.data(); auto local_desc = manager_->getLocal(); - auto peer_mem_addr = le64toh(desc->peer_mem_addr); - auto length = le64toh(desc->length); - - // The client accepts any response of exactly `length` bytes as payload (see - // ControlClient::recvData), so an error of that size must be padded or it - // would be copied into the caller's buffer and reported as success. - auto fail = [&response, length](std::string message) { - response = std::move(message); - if (response.size() == length) response.push_back(' '); - }; - - // Validate length to prevent DoS via excessive memory allocation - if (length > kTcpMaxReadBytes) { + status = validateXferReceiverCreditFence(local_desc, parsed); + if (!status.ok()) { + fail("RecvData failed: " + status.ToString()); + return; + } + if (parsed.length > kTcpMaxReadBytes) { fail("RecvData failed: length exceeds maximum allowed"); return; } - - if (auto* buffer = local_desc->findBuffer(peer_mem_addr, length)) { + if (auto* buffer = + local_desc->findBuffer(parsed.peer_mem_addr, parsed.length)) { if (buffer->permission != kGlobalReadOnly && buffer->permission != kGlobalReadWrite) { fail("RecvData failed: remote read permission denied"); return; } auto& loader = Platform::getLoader(); - if (loader.getMemoryType((void*)peer_mem_addr) == MTYPE_CPU) { - // assign() skips the resize() zero-fill pass. - response.assign(reinterpret_cast(peer_mem_addr), - length); + auto* source = reinterpret_cast(parsed.peer_mem_addr); + response.clear(); + if (parsed.version_two) { + const XferDataReplyHeader reply{htole64(kXferDataProtocolMagic)}; + response.assign(reinterpret_cast(&reply), + sizeof(reply)); + } + if (loader.getMemoryType(source) == MTYPE_CPU) { + response.append(reinterpret_cast(source), + parsed.length); } else { - response.resize(length); - auto status = - loader.copy(response.data(), (void*)peer_mem_addr, length); - if (!status.ok()) { - fail("RecvData failed: copy: " + status.ToString()); + response.resize(reply_header_size + parsed.length); + auto copy_status = loader.copy(response.data() + reply_header_size, + source, parsed.length); + if (!copy_status.ok()) { + fail("RecvData failed: copy: " + copy_status.ToString()); } } } else { diff --git a/mooncake-transfer-engine/tent/src/runtime/proxy_manager.cpp b/mooncake-transfer-engine/tent/src/runtime/proxy_manager.cpp index 64cad52563..55851c94ca 100644 --- a/mooncake-transfer-engine/tent/src/runtime/proxy_manager.cpp +++ b/mooncake-transfer-engine/tent/src/runtime/proxy_manager.cpp @@ -238,6 +238,11 @@ BatchID ProxyManager::submitCrossStage(const Request& request, inter_stage.length = chunk_length; inter_stage.target_id = request.target_id; inter_stage.target_offset = remote_stage_buffer; + inter_stage.receiver_credit_session_high = + request.receiver_credit_session_high; + inter_stage.receiver_credit_session_low = + request.receiver_credit_session_low; + inter_stage.receiver_credit_epoch = request.receiver_credit_epoch; auto batch = impl_->allocateBatch(1); auto status = impl_->submitStagingTransfer(batch, {inter_stage}); if (!status.ok()) { @@ -288,6 +293,11 @@ Status ProxyManager::waitRemoteStage(const std::string& server_addr, remote_stage.length = chunk_length; remote_stage.target_id = LOCAL_SEGMENT_ID; remote_stage.target_offset = request.target_offset + offset; + remote_stage.receiver_credit_session_high = + request.receiver_credit_session_high; + remote_stage.receiver_credit_session_low = + request.receiver_credit_session_low; + remote_stage.receiver_credit_epoch = request.receiver_credit_epoch; return ControlClient::delegate(server_addr, remote_stage); } @@ -301,6 +311,11 @@ void ProxyManager::submitRemoteStage( remote_stage.length = chunk_length; remote_stage.target_id = LOCAL_SEGMENT_ID; remote_stage.target_offset = request.target_offset + offset; + remote_stage.receiver_credit_session_high = + request.receiver_credit_session_high; + remote_stage.receiver_credit_session_low = + request.receiver_credit_session_low; + remote_stage.receiver_credit_epoch = request.receiver_credit_epoch; operation = std::make_shared( server_addr, remote_stage_buffer, [server_addr, remote_stage_buffer] { ControlClient::unpinStageBufferAsync( diff --git a/mooncake-transfer-engine/tent/src/runtime/receiver_credit.cpp b/mooncake-transfer-engine/tent/src/runtime/receiver_credit.cpp index 5ae6c14a2e..00bf199c6e 100644 --- a/mooncake-transfer-engine/tent/src/runtime/receiver_credit.cpp +++ b/mooncake-transfer-engine/tent/src/runtime/receiver_credit.cpp @@ -3,176 +3,622 @@ #include "tent/runtime/receiver_credit.h" +#include +#include +#include +#include + namespace mooncake::tent { +namespace { + +uint64_t steadyNowNs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} -size_t CreditKeyHash::operator()(const CreditKey& k) const noexcept { - size_t h = std::hash{}(k.receiver_session.high); - auto mix = [&h](uint64_t v) { - h ^= std::hash{}(v) + 0x9e3779b97f4a7c15ULL + (h << 6) + - (h >> 2); - }; - mix(k.receiver_session.low); - mix(k.sender_peer); - mix(k.qos_class); - return h; -} - -Status SenderCreditLedger::resourceIndex(CreditResource r, size_t& i) { - auto raw = static_cast(r); - if (raw < 1 || raw > kCreditResourceCount) +Status indexFor(CreditResource resource, size_t& index) { + const auto raw = static_cast(resource); + if (raw < 1 || raw > kCreditResourceCount) { return Status::InvalidArgument("unknown credit resource" LOC_MARK); - i = raw - 1; + } + index = raw - 1; return Status::OK(); } +void mixHash(size_t& hash, uint64_t value) { + hash ^= std::hash{}(value) + 0x9e3779b97f4a7c15ULL + (hash << 6) + + (hash >> 2); +} + +struct SenderInstanceIdHash { + size_t operator()(const SenderInstanceId& id) const noexcept { + size_t hash = std::hash{}(id.high); + mixHash(hash, id.low); + return hash; + } +}; + +} // namespace + +size_t CreditKeyHash::operator()(const CreditKey& key) const noexcept { + size_t hash = std::hash{}(key.receiver_session.high); + mixHash(hash, key.receiver_session.low); + mixHash(hash, key.sender_instance.high); + mixHash(hash, key.sender_instance.low); + mixHash(hash, key.qos_class); + return hash; +} + +SenderCreditLedger::SenderCreditLedger(size_t max_entries, + uint32_t max_freshness_ttl_ms, + CreditNowProvider now_provider) + : max_entries_(max_entries), + max_freshness_ttl_ms_(max_freshness_ttl_ms), + now_provider_(std::move(now_provider)) {} + +Status SenderCreditLedger::resourceIndex(CreditResource resource, + size_t& index) { + return indexFor(resource, index); +} + Status SenderCreditLedger::normalize( - const CreditCharge& c, std::array& out) { - out.fill(0); - if (c.resources.empty()) + const CreditCharge& charge, + std::array& normalized) { + normalized.fill(0); + if (charge.resources.empty()) { return Status::InvalidArgument("empty credit charge" LOC_MARK); - for (auto [r, amount] : c.resources) { - size_t i = 0; - CHECK_STATUS(resourceIndex(r, i)); - if (!amount || out[i]) + } + for (const auto& [resource, amount] : charge.resources) { + size_t index = 0; + CHECK_STATUS(resourceIndex(resource, index)); + if (amount == 0 || normalized[index] != 0) { return Status::InvalidArgument( "zero or duplicate credit charge" LOC_MARK); - out[i] = amount; + } + normalized[index] = amount; } return Status::OK(); } -Status SenderCreditLedger::activate(const CreditKey& k, uint64_t epoch) { - if (!epoch) return Status::InvalidArgument("zero credit epoch" LOC_MARK); +uint64_t SenderCreditLedger::nowNanos() const { + return now_provider_ ? now_provider_() : steadyNowNs(); +} + +Status SenderCreditLedger::requireFresh(const Entry& entry) const { + if (!entry.has_update || entry.expires_at_ns == 0 || + nowNanos() >= entry.expires_at_ns) { + return Status::InvalidEntry("receiver credit is stale" LOC_MARK); + } + return Status::OK(); +} + +Status SenderCreditLedger::activate(const CreditKey& key, uint64_t epoch) { + if (key.receiver_session.empty() || key.sender_instance.empty() || + epoch == 0) { + return Status::InvalidArgument( + "invalid credit activation identity" LOC_MARK); + } std::lock_guard lock(mutex_); - auto existing = entries_.find(k); + auto existing = entries_.find(key); if (existing != entries_.end()) { - if (epoch < existing->second.epoch) + if (epoch < existing->second.epoch) { return Status::InvalidEntry("stale credit activation" LOC_MARK); + } if (epoch == existing->second.epoch) return Status::OK(); Entry replacement; replacement.epoch = epoch; existing->second = replacement; return Status::OK(); - } else if (entries_.size() >= max_entries_) { + } + if (entries_.size() >= max_entries_) { return Status::TooManyRequests("credit ledger entry limit" LOC_MARK); } - Entry e; - e.epoch = epoch; - entries_.emplace(k, e); + Entry entry; + entry.epoch = epoch; + entries_.emplace(key, entry); return Status::OK(); } -Status SenderCreditLedger::deactivate(const CreditKey& k, uint64_t epoch) { - if (!epoch) return Status::InvalidArgument("zero credit epoch" LOC_MARK); +Status SenderCreditLedger::deactivate(const CreditKey& key, uint64_t epoch) { + if (epoch == 0) { + return Status::InvalidArgument("zero credit epoch" LOC_MARK); + } std::lock_guard lock(mutex_); - auto existing = entries_.find(k); - if (existing == entries_.end()) return Status::OK(); // idempotent cleanup - if (existing->second.epoch != epoch) + auto existing = entries_.find(key); + if (existing == entries_.end()) return Status::OK(); + if (existing->second.epoch != epoch) { return Status::InvalidEntry("credit cleanup epoch mismatch" LOC_MARK); + } entries_.erase(existing); return Status::OK(); } -Status SenderCreditLedger::applyUpdate(const CreditKey& k, - const ReceiverCreditUpdateV1& u, +Status SenderCreditLedger::deactivateSession(const ReceiverSessionId& session) { + if (session.empty()) { + return Status::InvalidArgument( + "empty receiver credit session cleanup" LOC_MARK); + } + std::lock_guard lock(mutex_); + for (auto it = entries_.begin(); it != entries_.end();) { + if (it->first.receiver_session == session) { + it = entries_.erase(it); + } else { + ++it; + } + } + return Status::OK(); +} + +Status SenderCreditLedger::applyUpdate(const CreditKey& key, + const ReceiverCreditUpdateV1& update, CreditUpdateDisposition& disposition) { - if (u.schema_version != 1 || !u.epoch || !u.sequence) + if (update.schema_version != kReceiverCreditProtocolVersion || + update.flags != 0 || update.epoch == 0 || update.sequence == 0 || + update.freshness_ttl_ms == 0 || + update.freshness_ttl_ms > max_freshness_ttl_ms_) { return Status::InvalidArgument("invalid credit update header" LOC_MARK); - if (!(u.receiver_session_id == k.receiver_session) || - u.qos_class != k.qos_class || u.grants.size() > kCreditResourceCount) + } + if (!(update.receiver_session_id == key.receiver_session) || + update.qos_class != key.qos_class || + update.grants.size() > kCreditResourceCount) { return Status::InvalidArgument("credit update identity/size" LOC_MARK); + } + std::array proposed{}; std::array present{}; - for (auto a : u.grants) { - size_t i = 0; - CHECK_STATUS(resourceIndex(a.resource, i)); - if (present[i]) + for (const auto& amount : update.grants) { + size_t index = 0; + CHECK_STATUS(resourceIndex(amount.resource, index)); + if (present[index]) { return Status::InvalidArgument("duplicate grant resource" LOC_MARK); - present[i] = true; - proposed[i] = a.grant_total; + } + present[index] = true; + proposed[index] = amount.grant_total; } + std::lock_guard lock(mutex_); - auto it = entries_.find(k); - if (it == entries_.end() || it->second.epoch != u.epoch) + auto it = entries_.find(key); + if (it == entries_.end() || it->second.epoch != update.epoch) { return Status::InvalidEntry("inactive or stale credit epoch" LOC_MARK); - auto& e = it->second; - if (e.has_update && u.sequence <= e.last_sequence) { + } + auto& entry = it->second; + if (entry.has_update && update.sequence <= entry.last_sequence) { disposition = CreditUpdateDisposition::DuplicateOrOld; return Status::OK(); } - // `grants` is a partial cumulative update: each resource present in this - // message replaces that resource's cumulative grant total, while omitted - // resources retain their previous totals. This lets the receiver refresh - // only the resources whose available capacity changed. - for (size_t i = 0; i < kCreditResourceCount; ++i) - if (present[i] && - (proposed[i] < e.grants[i] || proposed[i] < e.consumed[i])) + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (present[i] && (proposed[i] < entry.grants[i] || + proposed[i] < entry.consumed[i])) { return Status::InvalidArgument( "decreasing or under-consumed grant" LOC_MARK); - bool gap = e.has_update && u.sequence > e.last_sequence + 1; - for (size_t i = 0; i < kCreditResourceCount; ++i) - if (present[i]) e.grants[i] = proposed[i]; - e.last_sequence = u.sequence; - e.has_update = true; + } + } + + const bool gap = + entry.has_update && update.sequence > entry.last_sequence + 1; + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (present[i]) entry.grants[i] = proposed[i]; + } + entry.last_sequence = update.sequence; + entry.has_update = true; + const uint64_t ttl_ns = + static_cast(update.freshness_ttl_ms) * 1000000ULL; + const uint64_t now = nowNanos(); + entry.expires_at_ns = ttl_ns > std::numeric_limits::max() - now + ? std::numeric_limits::max() + : now + ttl_ns; disposition = gap ? CreditUpdateDisposition::SequenceGap : CreditUpdateDisposition::Applied; return Status::OK(); } -Status SenderCreditLedger::tryReserve(const CreditKey& k, - const CreditCharge& c) { - std::array n; - CHECK_STATUS(normalize(c, n)); +Status SenderCreditLedger::tryReserve(const CreditKey& key, + const CreditCharge& charge) { + std::array normalized; + CHECK_STATUS(normalize(charge, normalized)); std::lock_guard lock(mutex_); - auto it = entries_.find(k); - if (it == entries_.end() || !it->second.has_update) + auto it = entries_.find(key); + if (it == entries_.end()) { return Status::InvalidEntry("credit unavailable" LOC_MARK); - auto& e = it->second; - for (size_t i = 0; i < kCreditResourceCount; ++i) - if (e.consumed[i] > e.grants[i] || n[i] > e.grants[i] - e.consumed[i]) + } + CHECK_STATUS(requireFresh(it->second)); + auto& entry = it->second; + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (entry.consumed[i] > entry.grants[i] || + normalized[i] > entry.grants[i] - entry.consumed[i]) { return Status::TooManyRequests("insufficient credit" LOC_MARK); - for (size_t i = 0; i < kCreditResourceCount; ++i) e.consumed[i] += n[i]; + } + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + entry.consumed[i] += normalized[i]; + } return Status::OK(); } -Status SenderCreditLedger::rollbackReservation(const CreditKey& k, - const CreditCharge& c) { - std::array n; - CHECK_STATUS(normalize(c, n)); +Status SenderCreditLedger::rollbackReservation(const CreditKey& key, + const CreditCharge& charge) { + std::array normalized; + CHECK_STATUS(normalize(charge, normalized)); std::lock_guard lock(mutex_); - auto it = entries_.find(k); - if (it == entries_.end()) + auto it = entries_.find(key); + if (it == entries_.end()) { return Status::InvalidEntry("credit session inactive" LOC_MARK); - for (size_t i = 0; i < kCreditResourceCount; ++i) - if (n[i] > it->second.consumed[i]) + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (it->second.released[i] > it->second.consumed[i] || + normalized[i] > it->second.consumed[i] - it->second.released[i]) { return Status::InvalidArgument( "credit rollback underflow" LOC_MARK); - for (size_t i = 0; i < kCreditResourceCount; ++i) - it->second.consumed[i] -= n[i]; + } + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (normalized[i] > + std::numeric_limits::max() - it->second.released[i]) { + return Status::InvalidArgument("credit rollback overflow" LOC_MARK); + } + it->second.released[i] += normalized[i]; + } + return Status::OK(); +} + +Status SenderCreditLedger::releaseCommitted(const CreditKey& key, + const CreditCharge& charge) { + std::array normalized; + CHECK_STATUS(normalize(charge, normalized)); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + return Status::InvalidEntry("credit session inactive" LOC_MARK); + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (it->second.released[i] > it->second.consumed[i] || + normalized[i] > it->second.consumed[i] - it->second.released[i]) { + return Status::InvalidArgument( + "credit release exceeds committed consumption" LOC_MARK); + } + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (normalized[i] > + std::numeric_limits::max() - it->second.released[i]) { + return Status::InvalidArgument("credit release overflow" LOC_MARK); + } + it->second.released[i] += normalized[i]; + } + return Status::OK(); +} + +Status SenderCreditLedger::prepareExchange( + const CreditKey& key, const CreditCharge& next_charge, + ReceiverCreditExchangeRequestV1& request) { + std::array normalized; + CHECK_STATUS(normalize(next_charge, normalized)); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + return Status::InvalidEntry("credit session inactive" LOC_MARK); + } + auto& entry = it->second; + if (entry.report_sequence == std::numeric_limits::max()) { + return Status::InternalError( + "credit report sequence exhausted" LOC_MARK); + } + + request = ReceiverCreditExchangeRequestV1{}; + request.expected_receiver_session_id = key.receiver_session; + request.epoch = entry.epoch; + request.sender_instance_id = key.sender_instance; + request.qos_class = key.qos_class; + request.known_update_sequence = entry.last_sequence; + request.report_sequence = ++entry.report_sequence; + request.released_totals.reserve(kCreditResourceCount); + request.desired_grant_totals.reserve(next_charge.resources.size()); + for (size_t i = 0; i < kCreditResourceCount; ++i) { + const auto resource = static_cast(i + 1); + request.released_totals.push_back({resource, entry.released[i]}); + if (normalized[i] == 0) continue; + if (normalized[i] > + std::numeric_limits::max() - entry.consumed[i]) { + return Status::InvalidArgument( + "desired credit total overflow" LOC_MARK); + } + request.desired_grant_totals.push_back( + {resource, entry.consumed[i] + normalized[i]}); + } + return Status::OK(); +} + +Status SenderCreditLedger::prepareReleaseExchange( + const CreditKey& key, ReceiverCreditExchangeRequestV1& request) { + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + return Status::InvalidEntry("credit session inactive" LOC_MARK); + } + auto& entry = it->second; + if (entry.report_sequence == std::numeric_limits::max()) { + return Status::InternalError( + "credit report sequence exhausted" LOC_MARK); + } + + request = ReceiverCreditExchangeRequestV1{}; + request.expected_receiver_session_id = key.receiver_session; + request.epoch = entry.epoch; + request.sender_instance_id = key.sender_instance; + request.qos_class = key.qos_class; + request.known_update_sequence = entry.last_sequence; + request.report_sequence = ++entry.report_sequence; + request.released_totals.reserve(kCreditResourceCount); + for (size_t i = 0; i < kCreditResourceCount; ++i) { + const auto resource = static_cast(i + 1); + request.released_totals.push_back({resource, entry.released[i]}); + if (entry.consumed[i] != 0) { + request.desired_grant_totals.push_back( + {resource, entry.consumed[i]}); + } + } + if (request.desired_grant_totals.empty()) { + return Status::InvalidEntry( + "credit release exchange has no consumption" LOC_MARK); + } return Status::OK(); } -Status SenderCreditLedger::available(const CreditKey& k, CreditResource r, - uint64_t& v) const { - size_t i = 0; - CHECK_STATUS(resourceIndex(r, i)); +Status SenderCreditLedger::available(const CreditKey& key, + CreditResource resource, + uint64_t& value) const { + size_t index = 0; + CHECK_STATUS(resourceIndex(resource, index)); std::lock_guard lock(mutex_); - auto it = entries_.find(k); - if (it == entries_.end() || !it->second.has_update) + auto it = entries_.find(key); + if (it == entries_.end()) { return Status::InvalidEntry("credit unavailable" LOC_MARK); - v = it->second.grants[i] - it->second.consumed[i]; + } + CHECK_STATUS(requireFresh(it->second)); + value = it->second.grants[index] - it->second.consumed[index]; + return Status::OK(); +} + +Status SenderCreditLedger::consumed(const CreditKey& key, + CreditResource resource, + uint64_t& value) const { + size_t index = 0; + CHECK_STATUS(resourceIndex(resource, index)); + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + return Status::InvalidEntry("credit session inactive" LOC_MARK); + } + value = it->second.consumed[index]; return Status::OK(); } -Status SenderCreditLedger::consumed(const CreditKey& k, CreditResource r, - uint64_t& v) const { - size_t i = 0; - CHECK_STATUS(resourceIndex(r, i)); +Status SenderCreditLedger::released(const CreditKey& key, + CreditResource resource, + uint64_t& value) const { + size_t index = 0; + CHECK_STATUS(resourceIndex(resource, index)); std::lock_guard lock(mutex_); - auto it = entries_.find(k); - if (it == entries_.end()) + auto it = entries_.find(key); + if (it == entries_.end()) { return Status::InvalidEntry("credit session inactive" LOC_MARK); - v = it->second.consumed[i]; + } + value = it->second.released[index]; + return Status::OK(); +} + +ReceiverCreditAuthority::ReceiverCreditAuthority(ReceiverSessionId session, + uint64_t epoch, + ReceiverCreditLimits limits) + : session_(session), epoch_(epoch), limits_(std::move(limits)) { + if (session_.empty() || epoch_ == 0) { + status_ = Status::InvalidArgument( + "invalid receiver credit authority identity" LOC_MARK); + } else { + status_ = validateLimits(limits_); + } +} + +Status ReceiverCreditAuthority::validateLimits( + const ReceiverCreditLimits& limits) { + if (limits.freshness_ttl_ms == 0 || limits.max_senders == 0 || + limits.max_entries < limits.max_senders) { + return Status::InvalidArgument( + "invalid receiver credit limits" LOC_MARK); + } + bool any_capacity = false; + for (const auto capacity : limits.capacity) any_capacity |= capacity != 0; + if (!any_capacity) { + return Status::InvalidArgument( + "receiver credit capacity is empty" LOC_MARK); + } + return Status::OK(); +} + +ReceiverCreditAdvertV1 ReceiverCreditAuthority::advert() const { + ReceiverCreditAdvertV1 result; + result.receiver_session_id = session_; + result.epoch = epoch_; + result.freshness_ttl_ms = limits_.freshness_ttl_ms; + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (limits_.capacity[i] != 0) { + const auto resource = static_cast(i + 1); + result.resources.push_back(resource); + result.capacities.push_back({resource, limits_.capacity[i]}); + } + } + return result; +} + +Status ReceiverCreditAuthority::normalizeCounters( + const std::vector& counters, + std::array& normalized, + std::array& present) { + normalized.fill(0); + present.fill(false); + if (counters.size() > kCreditResourceCount) { + return Status::InvalidArgument("too many credit counters" LOC_MARK); + } + for (const auto& counter : counters) { + size_t index = 0; + CHECK_STATUS(indexFor(counter.resource, index)); + if (present[index]) { + return Status::InvalidArgument( + "duplicate credit counter resource" LOC_MARK); + } + present[index] = true; + normalized[index] = counter.total; + } + return Status::OK(); +} + +void ReceiverCreditAuthority::fillReply( + const CreditKey& key, const Entry& entry, + ReceiverCreditExchangeReplyV1& reply) const { + reply = ReceiverCreditExchangeReplyV1{}; + reply.sender_instance_id = key.sender_instance; + reply.update.qos_class = key.qos_class; + reply.update.receiver_session_id = session_; + reply.update.epoch = epoch_; + reply.update.sequence = entry.update_sequence; + reply.update.freshness_ttl_ms = limits_.freshness_ttl_ms; + reply.update.grants.reserve(kCreditResourceCount); + for (size_t i = 0; i < kCreditResourceCount; ++i) { + reply.update.grants.push_back( + {static_cast(i + 1), entry.grants[i]}); + } +} + +Status ReceiverCreditAuthority::exchange( + const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& reply) { + CHECK_STATUS(status_); + if (request.schema_version != kReceiverCreditProtocolVersion || + request.flags != 0 || request.epoch != epoch_ || + !(request.expected_receiver_session_id == session_) || + request.sender_instance_id.empty() || request.qos_class > 2 || + request.report_sequence == 0 || request.desired_grant_totals.empty()) { + return Status::InvalidArgument( + "invalid receiver credit exchange identity/header" LOC_MARK); + } + + std::array released{}, desired{}; + std::array released_present{}, + desired_present{}; + CHECK_STATUS( + normalizeCounters(request.released_totals, released, released_present)); + CHECK_STATUS(normalizeCounters(request.desired_grant_totals, desired, + desired_present)); + + CreditKey key{request.expected_receiver_session_id, + request.sender_instance_id, request.qos_class}; + std::lock_guard lock(mutex_); + auto it = entries_.find(key); + if (it == entries_.end()) { + if (request.known_update_sequence != 0) { + return Status::InvalidArgument( + "new sender knows an impossible credit update" LOC_MARK); + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (released_present[i] && released[i] != 0) { + return Status::InvalidArgument( + "new sender reports impossible credit release" LOC_MARK); + } + } + if (entries_.size() >= limits_.max_entries) { + return Status::TooManyRequests( + "receiver credit replay-history limit" LOC_MARK); + } + it = entries_.emplace(key, Entry{}).first; + } + auto& entry = it->second; + if (request.known_update_sequence > entry.update_sequence) { + return Status::InvalidArgument( + "sender knows an impossible credit update" LOC_MARK); + } + if (request.report_sequence <= entry.last_report_sequence) { + fillReply(key, entry, reply); + return Status::OK(); + } + if (entry.update_sequence == std::numeric_limits::max()) { + return Status::InternalError( + "receiver credit update sequence exhausted" LOC_MARK); + } + + // Validate the entire cumulative release before mutating any resource. + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (!released_present[i]) continue; + if (released[i] < entry.released[i] || released[i] > entry.grants[i]) { + return Status::InvalidArgument( + "invalid cumulative credit release" LOC_MARK); + } + } + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (!released_present[i]) continue; + const uint64_t delta = released[i] - entry.released[i]; + if (delta > outstanding_[i]) { + return Status::InternalError( + "receiver credit accounting underflow" LOC_MARK); + } + } + + // Apply releases first, then atomically decide whether every requested + // resource can reach its absolute desired total. + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (!released_present[i]) continue; + outstanding_[i] -= released[i] - entry.released[i]; + entry.released[i] = released[i]; + } + + std::array grant_delta{}; + bool can_grant_all = true; + for (size_t i = 0; i < kCreditResourceCount; ++i) { + if (!desired_present[i] || desired[i] <= entry.grants[i]) continue; + grant_delta[i] = desired[i] - entry.grants[i]; + const uint64_t free = limits_.capacity[i] - outstanding_[i]; + if (grant_delta[i] > free) can_grant_all = false; + } + const bool requests_new_grant = + std::any_of(grant_delta.begin(), grant_delta.end(), + [](uint64_t delta) { return delta != 0; }); + if (can_grant_all && requests_new_grant) { + std::unordered_set + active_senders; + active_senders.reserve(std::min(entries_.size(), limits_.max_senders)); + for (const auto& [entry_key, tracked] : entries_) { + bool has_outstanding = false; + for (size_t i = 0; i < kCreditResourceCount; ++i) { + has_outstanding |= tracked.grants[i] > tracked.released[i]; + } + if (has_outstanding) { + active_senders.insert(entry_key.sender_instance); + } + } + if (!active_senders.count(request.sender_instance_id) && + active_senders.size() >= limits_.max_senders) { + can_grant_all = false; + } + } + if (can_grant_all) { + for (size_t i = 0; i < kCreditResourceCount; ++i) { + entry.grants[i] += grant_delta[i]; + outstanding_[i] += grant_delta[i]; + } + } + + entry.last_report_sequence = request.report_sequence; + ++entry.update_sequence; + fillReply(key, entry, reply); + return Status::OK(); +} + +Status ReceiverCreditAuthority::outstanding(CreditResource resource, + uint64_t& value) const { + size_t index = 0; + CHECK_STATUS(indexFor(resource, index)); + std::lock_guard lock(mutex_); + value = outstanding_[index]; return Status::OK(); } diff --git a/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp b/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp index e00c70f55d..6d6471127e 100644 --- a/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp +++ b/mooncake-transfer-engine/tent/src/runtime/transfer_engine_impl.cpp @@ -60,6 +60,33 @@ constexpr uint8_t kRedisDefaultDbIndex = 0; // the counter), so only a permanently failing poll or queue retire gets here. constexpr size_t kMaxReclaimAttempts = 4096; constexpr int kMaxHpTcpMetadataRefreshRetries = 1; +uint64_t randomIdentityWord() { + try { + static thread_local std::random_device random; + return (static_cast(random()) << 32) ^ random() ^ + getCurrentTimeInNano(); + } catch (const std::exception&) { + // random_device is allowed to be unavailable on unusual libstdc++ + // targets. Preserve startup while still mixing a process-local nonce + // into the clock-seeded fallback generator. + static std::atomic nonce{1}; + auto& random = SimpleRandom::Get(); + return (static_cast(random.next()) << 32) ^ random.next() ^ + nonce.fetch_add(1, std::memory_order_relaxed); + } +} + +SenderInstanceId makeSenderInstanceId() { + SenderInstanceId id{randomIdentityWord(), randomIdentityWord()}; + if (id.empty()) id.low = 1; + return id; +} + +ReceiverSessionId makeReceiverSessionId() { + ReceiverSessionId id{randomIdentityWord(), randomIdentityWord()}; + if (id.empty()) id.low = 1; + return id; +} } // namespace struct Batch { @@ -363,6 +390,11 @@ Status TransferEngineImpl::setupLocalSegment() { segment.rpc_server_addr = buildIpAddrWithPort(hostname_, port_, ipv6_); auto& detail = std::get(segment.detail); detail.topology = *(topology_.get()); + if (receiver_credit_authority_) { + detail.receiver_credit = receiver_credit_authority_->advert(); + } else { + detail.receiver_credit.reset(); + } return Status::OK(); })); local_segment_tracker_ = std::make_unique(manager); @@ -398,6 +430,15 @@ Status TransferEngineImpl::construct() { notify_rpc_fallback_ = conf_->get("notification/rpc_fallback", true); enable_progress_worker_ = conf_->get("enable_progress_worker", false); runtime_queue_config_.enabled = conf_->get("enable_runtime_queue", false); + const bool receiver_credit_legacy_enable = + conf_->get("receiver_credit/enable", false); + receiver_credit_config_.sender_enabled = conf_->get( + "receiver_credit/sender_enable", receiver_credit_legacy_enable); + receiver_credit_config_.receiver_enabled = conf_->get( + "receiver_credit/receiver_enable", receiver_credit_legacy_enable); + if (receiver_credit_config_.sender_enabled) { + runtime_queue_config_.enabled = true; + } if (runtime_queue_config_.enabled) enable_progress_worker_ = true; runtime_queue_config_.limits.max_outstanding_owners = conf_->get("runtime_queue/max_outstanding_owners", 1024UL); @@ -420,6 +461,59 @@ Status TransferEngineImpl::construct() { runtime_queue_config_.progress_fallback_interval = std::chrono::microseconds( conf_->get("runtime_queue/progress_fallback_interval_us", 50000UL)); + + receiver_credit_config_.limits.capacity[0] = + conf_->get("receiver_credit/data_bytes", + static_cast( + runtime_queue_config_.limits.max_outstanding_bytes)); + receiver_credit_config_.limits.capacity[1] = + conf_->get("receiver_credit/request_slots", + static_cast( + runtime_queue_config_.limits.max_outstanding_owners)); + receiver_credit_config_.limits.capacity[2] = 0; + receiver_credit_config_.limits.capacity[3] = 0; + receiver_credit_config_.limits.freshness_ttl_ms = + conf_->get("receiver_credit/freshness_ttl_ms", uint32_t{1000}); + receiver_credit_config_.limits.max_senders = + conf_->get("receiver_credit/max_senders", size_t{1024}); + receiver_credit_config_.limits.max_entries = + conf_->get("receiver_credit/max_authority_entries", size_t{4096}); + receiver_credit_config_.max_sender_entries = + conf_->get("receiver_credit/max_sender_entries", size_t{4096}); + receiver_credit_config_.max_freshness_ttl_ms = + conf_->get("receiver_credit/max_freshness_ttl_ms", uint32_t{60000}); + if (conf_->get("receiver_credit/staging_slots", uint64_t{0}) != 0 || + conf_->get("receiver_credit/consumer_slots", uint64_t{0}) != 0) { + return Status::InvalidArgument( + "staging/consumer receiver-credit resources are not supported " + "by the PR6 runtime gate" LOC_MARK); + } + if (receiver_credit_config_.sender_enabled) { + if (receiver_credit_config_.max_sender_entries == 0 || + receiver_credit_config_.max_freshness_ttl_ms == 0 || + runtime_queue_config_.progress_fallback_interval.count() <= 0) { + return Status::InvalidArgument( + "receiver-credit sender requires non-zero entries, TTL " + "limit, and progress fallback interval" LOC_MARK); + } + } + if (receiver_credit_config_.receiver_enabled) { + if (receiver_credit_config_.limits.capacity[0] == 0 || + receiver_credit_config_.limits.capacity[1] == 0 || + receiver_credit_config_.limits.max_senders == 0 || + receiver_credit_config_.limits.max_entries < + receiver_credit_config_.limits.max_senders) { + return Status::InvalidArgument( + "receiver credit requires non-zero bytes, request slots, " + "and a replay-history limit no smaller than the sender " + "limit" LOC_MARK); + } + if (receiver_credit_config_.limits.freshness_ttl_ms > + receiver_credit_config_.max_freshness_ttl_ms) { + return Status::InvalidArgument( + "receiver credit TTL exceeds sender safety limit" LOC_MARK); + } + } if (runtime_queue_config_.enabled && (runtime_queue_config_.max_dispatch_owners == 0 || runtime_queue_config_.max_dispatch_bytes == 0)) { @@ -428,6 +522,15 @@ Status TransferEngineImpl::construct() { } runtime_queue_ = std::make_unique( runtime_queue_config_.limits); + sender_instance_id_ = makeSenderInstanceId(); + sender_credit_ledger_ = std::make_unique( + std::max(receiver_credit_config_.max_sender_entries, 1), + std::max(receiver_credit_config_.max_freshness_ttl_ms, 1)); + if (receiver_credit_config_.receiver_enabled) { + receiver_credit_authority_ = std::make_unique( + makeReceiverSessionId(), 1, receiver_credit_config_.limits); + CHECK_STATUS(receiver_credit_authority_->status()); + } if (!hostname_.empty()) CHECK_STATUS(checkLocalIpAddress(hostname_, ipv6_)); else @@ -440,6 +543,14 @@ Status TransferEngineImpl::construct() { metadata_ = std::make_shared(metadata_type, metadata_servers, this); + if (receiver_credit_authority_) { + metadata_->setReceiverCreditCallback( + [this](const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& reply) { + return receiver_credit_authority_->exchange(request, reply); + }); + } + CHECK_STATUS(metadata_->start(port_, ipv6_, rpc_server_threads)); if (metadata_type == "p2p") @@ -611,6 +722,28 @@ Status TransferEngineImpl::deconstruct() { << status.ToString(); } } + // Only a reservation that never reached a transport can be returned here. + // A remaining committed owner is not terminal, and the UB DMA fence is + // established later while its transport is destroyed. Conservatively + // abandon that authorization in the old receiver session instead of + // making it available while DMA may still be active. + size_t abandoned_receiver_credits = 0; + for (auto& [_, queued] : queued_owners_) { + if (!queued.credit.active) continue; + if (!queued.credit.committed) { + rollbackReceiverCredit(queued.credit); + } else { + queued.credit = ReceiverCreditReservation{}; + ++abandoned_receiver_credits; + } + } + if (abandoned_receiver_credits != 0) { + LOG(WARNING) << "Conservatively abandoned " + << abandoned_receiver_credits + << " in-flight receiver-credit reservations during " + "shutdown"; + } + progressReceiverCreditReleases(); // Destroy staging_proxy_ first: its destructor calls back into // unregisterLocalMemory/freeLocalMemory, which require @@ -1790,6 +1923,10 @@ struct RequestBoundaryInfo { std::optional source_key; std::optional target_key; uint64_t max_merge_bytes{std::numeric_limits::max()}; + // A required receiver-credit advert limits the size of one physical + // owner. Request merging must not turn individually admissible requests + // into an owner that can never acquire credit. + std::optional receiver_credit_data_capacity; }; struct MergeResult { @@ -1812,6 +1949,8 @@ struct TransferEngineImpl::PreparedSubmit { std::vector derived_task_ids; Request request{}; SelectionResult route{}; + bool receiver_credit_required{false}; + bool receiver_credit_exempt{false}; bool staging{false}; std::vector staging_params; }; @@ -1912,6 +2051,15 @@ MergeResult mergeRequests(const std::vector& requests, if (last.req.length + curr.req.length > last.boundary.max_merge_bytes) { return false; } + const size_t merged_length = last.req.length + curr.req.length; + if (last.boundary.receiver_credit_data_capacity != + curr.boundary.receiver_credit_data_capacity) { + return false; + } + if (last.boundary.receiver_credit_data_capacity && + merged_length > *last.boundary.receiver_credit_data_capacity) { + return false; + } uint64_t last_source_end = 0; uint64_t last_target_end = 0; @@ -1990,6 +2138,21 @@ std::vector resolveRequestBoundaries( std::min(hp_tcp_limit, endpoint.max_transfer_bytes); } } + std::optional receiver_credit_data_capacity; + if (target_desc->type == SegmentType::Memory && + target_desc->getMemory().receiver_credit) { + // Zero deliberately disables merging for malformed + // adverts; prepareSubmit validates and rejects them with + // the protocol-specific error below. + receiver_credit_data_capacity = 0; + for (const auto& capacity : + target_desc->getMemory().receiver_credit->capacities) { + if (capacity.resource == CreditResource::DataBytes) { + receiver_credit_data_capacity = capacity.total; + break; + } + } + } bool any_missing = false; for (size_t i : idxs) { const auto& r = requests[i]; @@ -1999,6 +2162,8 @@ std::vector resolveRequestBoundaries( r.transport_hint == UNSPEC)) { boundaries[i].max_merge_bytes = hp_tcp_limit; } + boundaries[i].receiver_credit_data_capacity = + receiver_credit_data_capacity; auto* buffer = target_desc->findBuffer(r.target_offset, r.length); if (!buffer) { @@ -2096,6 +2261,8 @@ bool TransferEngineImpl::xpuIpcReady(const Request& request, void TransferEngineImpl::findStagingPolicy(const Request& request, std::vector& policy) { + if (staging_policy_for_test_ && staging_policy_for_test_(request, policy)) + return; if (request.target_id == LOCAL_SEGMENT_ID) return; SegmentDesc* desc = nullptr; @@ -2249,7 +2416,8 @@ SelectionResult TransferEngineImpl::resolveTransport(const Request& req, Status TransferEngineImpl::prepareSubmit( Batch* batch, const std::vector& request_list, - PreparedSubmit& prepared, bool require_post_submit_cancellation) { + QueueOwnerKind owner_kind, PreparedSubmit& prepared, + bool require_post_submit_cancellation) { if (!batch) return Status::InvalidArgument("Invalid batch" LOC_MARK); for (size_t i = 0; i < request_list.size(); ++i) { auto st = validateTransportHint(request_list[i], i); @@ -2330,6 +2498,49 @@ Status TransferEngineImpl::prepareSubmit( "transport" LOC_MARK); } } + owner.receiver_credit_exempt = + owner_kind == QueueOwnerKind::StagingInternal; + if (owner_kind != QueueOwnerKind::StagingInternal) { + ReceiverCreditTarget credit_target; + CHECK_STATUS( + resolveReceiverCreditTarget(owner.request, credit_target)); + owner.receiver_credit_required = credit_target.required; + // Scatter bypasses the runtime queue to preserve synchronous + // cancellation. It cannot bypass a peer's required credit gate. + if (credit_target.required && require_post_submit_cancellation) { + return Status::NotImplemented( + "scatter transfer does not support required receiver " + "credit" LOC_MARK); + } + if (credit_target.required && + !receiver_credit_config_.sender_enabled) { + return Status::InvalidArgument( + "peer requires receiver credit but the sender gate is " + "disabled" LOC_MARK); + } + CHECK_STATUS( + validateReceiverCreditCapacity(owner.request, credit_target)); + } else { + const bool has_credit_fence = + owner.request.receiver_credit_session_high != 0 || + owner.request.receiver_credit_session_low != 0 || + owner.request.receiver_credit_epoch != 0; + if (has_credit_fence) { + ReceiverCreditTarget credit_target; + CHECK_STATUS( + resolveReceiverCreditTarget(owner.request, credit_target)); + if (!credit_target.required || + credit_target.advert.receiver_session_id.high != + owner.request.receiver_credit_session_high || + credit_target.advert.receiver_session_id.low != + owner.request.receiver_credit_session_low || + credit_target.advert.epoch != + owner.request.receiver_credit_epoch) { + return Status::InvalidEntry( + "staging receiver-credit session changed" LOC_MARK); + } + } + } owner.staging_params = owner.route.staging_params; if (owner.staging_params.empty() && (owner.route.transport == TCP || owner.route.transport == HP_TCP)) { @@ -2374,6 +2585,397 @@ void TransferEngineImpl::attachProgressNotifier( }; } +Status TransferEngineImpl::resolveReceiverCreditTarget( + const Request& request, ReceiverCreditTarget& target) const { + target = ReceiverCreditTarget{}; + SegmentDescRef segment; + if (request.target_id == LOCAL_SEGMENT_ID) { + segment = metadata_->segmentManager().getLocal(); + } else { + CHECK_STATUS(metadata_->segmentManager().getRemoteCached( + segment, request.target_id)); + } + if (!segment || segment->type != SegmentType::Memory) return Status::OK(); + const auto& memory = segment->getMemory(); + if (!memory.receiver_credit) return Status::OK(); + + const auto& advert = *memory.receiver_credit; + if (advert.schema_version != kReceiverCreditProtocolVersion || + advert.flags != kReceiverCreditRequired || + advert.receiver_session_id.empty() || advert.epoch == 0 || + advert.freshness_ttl_ms == 0 || + advert.freshness_ttl_ms > + receiver_credit_config_.max_freshness_ttl_ms || + advert.resources.empty() || + advert.resources.size() > kCreditResourceCount || + advert.capacities.size() != advert.resources.size()) { + return Status::InvalidArgument( + "invalid receiver-credit advertisement" LOC_MARK); + } + + std::array resources{}; + for (const auto resource : advert.resources) { + const auto raw = static_cast(resource); + if (raw < 1 || raw > kCreditResourceCount || resources[raw - 1]) { + return Status::InvalidArgument( + "invalid receiver-credit resource advertisement" LOC_MARK); + } + resources[raw - 1] = true; + } + std::array capacities{}; + for (const auto& capacity : advert.capacities) { + const auto raw = static_cast(capacity.resource); + if (raw < 1 || raw > kCreditResourceCount || capacity.total == 0 || + capacities[raw - 1] || !resources[raw - 1]) { + return Status::InvalidArgument( + "invalid receiver-credit capacity advertisement" LOC_MARK); + } + capacities[raw - 1] = true; + } + if (capacities != resources) { + return Status::InvalidArgument( + "receiver-credit resources/capacities mismatch" LOC_MARK); + } + if (!resources[static_cast(CreditResource::DataBytes) - 1] || + !resources[static_cast(CreditResource::RequestSlots) - 1]) { + return Status::InvalidArgument( + "receiver credit lacks bytes/request resources" LOC_MARK); + } + if (request.target_id != LOCAL_SEGMENT_ID && + segment->rpc_server_addr.empty()) { + return Status::InvalidArgument( + "receiver credit peer has no RPC address" LOC_MARK); + } + target.required = true; + target.advert = advert; + target.rpc_server_addr = segment->rpc_server_addr; + return Status::OK(); +} + +Status TransferEngineImpl::validateReceiverCreditCapacity( + const Request& request, const ReceiverCreditTarget& target) const { + if (!target.required) return Status::OK(); + uint64_t data_bytes = 0; + uint64_t request_slots = 0; + for (const auto& capacity : target.advert.capacities) { + if (capacity.resource == CreditResource::DataBytes) { + data_bytes = capacity.total; + } else if (capacity.resource == CreditResource::RequestSlots) { + request_slots = capacity.total; + } + } + if (request_slots == 0 || request.length > data_bytes) { + return Status::TooManyRequests( + "request exceeds receiver-credit capacity" LOC_MARK); + } + return Status::OK(); +} + +Status TransferEngineImpl::exchangeReceiverCredit( + SegmentID target_id, const std::string& rpc_server_addr, + const ReceiverCreditExchangeRequestV1& request, + ReceiverCreditExchangeReplyV1& reply) { + if (target_id == LOCAL_SEGMENT_ID) { + if (!receiver_credit_authority_) { + return Status::InvalidEntry( + "local receiver-credit authority is unavailable" LOC_MARK); + } + return receiver_credit_authority_->exchange(request, reply); + } + return ControlClient::exchangeReceiverCredit(rpc_server_addr, request, + reply); +} + +Status TransferEngineImpl::acquireReceiverCredit( + const Request& request, ReceiverCreditReservation& reservation, + bool& acquired) { + acquired = false; + if (reservation.active) { + return Status::InternalError( + "receiver credit attempt already has a reservation" LOC_MARK); + } + + ReceiverCreditTarget target; + CHECK_STATUS(resolveReceiverCreditTarget(request, target)); + if (!target.required) { + acquired = true; + return Status::OK(); + } + CHECK_STATUS(validateReceiverCreditCapacity(request, target)); + if (!receiver_credit_config_.sender_enabled || !sender_credit_ledger_) { + return Status::InvalidEntry( + "receiver-credit sender gate is disabled" LOC_MARK); + } + + const uint32_t qos_class = static_cast( + std::clamp(request.priority, static_cast(PRIO_HIGH), + static_cast(PRIO_LOW))); + CreditKey key{target.advert.receiver_session_id, sender_instance_id_, + qos_class}; + CreditCharge charge{{{CreditResource::DataBytes, request.length}, + {CreditResource::RequestSlots, 1}}}; + observeReceiverCreditSession(request.target_id, + target.advert.receiver_session_id); + CHECK_STATUS(sender_credit_ledger_->activate(key, target.advert.epoch)); + + auto reserve_status = sender_credit_ledger_->tryReserve(key, charge); + if (!reserve_status.ok()) { + if (!reserve_status.IsTooManyRequests() && + !reserve_status.IsInvalidEntry()) { + return reserve_status; + } + + std::array released_now{}; + (void)sender_credit_ledger_->released(key, CreditResource::DataBytes, + released_now[0]); + (void)sender_credit_ledger_->released(key, CreditResource::RequestSlots, + released_now[1]); + const std::array charge_now{request.length, 1}; + const auto now = std::chrono::steady_clock::now(); + auto denial_it = receiver_credit_denials_.find(key); + if (denial_it != receiver_credit_denials_.end()) { + const auto& denial = denial_it->second; + const bool asks_less = charge_now[0] < denial.charge[0] || + charge_now[1] < denial.charge[1]; + if (denial.released == released_now && !asks_less && + now < denial.retry_at) { + return Status::OK(); // Nothing changed: requeue, no RPC. + } + } + // Back off exponentially (1 ms .. 32 ms) while the receiver keeps + // refusing. A local release retries at once; the cap bounds the + // wait for capacity that another sender returned. + auto note_denial = [&] { + auto& denial = receiver_credit_denials_[key]; + if (denial.released != released_now) denial.attempts = 0; + denial.released = released_now; + denial.charge = charge_now; + const uint32_t shift = std::min(denial.attempts, 5); + ++denial.attempts; + denial.retry_at = std::chrono::steady_clock::now() + + std::chrono::milliseconds(1U << shift); + }; + + ReceiverCreditExchangeRequestV1 exchange_request; + CHECK_STATUS(sender_credit_ledger_->prepareExchange(key, charge, + exchange_request)); + ReceiverCreditExchangeReplyV1 exchange_reply; + ++receiver_credit_demand_exchanges_; + auto exchange_status = + exchangeReceiverCredit(request.target_id, target.rpc_server_addr, + exchange_request, exchange_reply); + if (!exchange_status.ok()) { + if (request.target_id != LOCAL_SEGMENT_ID) { + (void)metadata_->segmentManager().invalidateRemote( + request.target_id); + } + LOG(WARNING) << "Receiver-credit exchange deferred: " + << exchange_status.ToString(); + note_denial(); + return Status::OK(); + } + if (!(exchange_reply.sender_instance_id == sender_instance_id_)) { + return Status::InvalidArgument( + "receiver-credit reply sender mismatch" LOC_MARK); + } + if (exchange_reply.flags != 0) { + return Status::InvalidArgument( + "receiver-credit demand reply flags" LOC_MARK); + } + CreditUpdateDisposition disposition; + CHECK_STATUS(sender_credit_ledger_->applyUpdate( + key, exchange_reply.update, disposition)); + pending_credit_releases_.erase(key); + reserve_status = sender_credit_ledger_->tryReserve(key, charge); + if (reserve_status.IsTooManyRequests() || + reserve_status.IsInvalidEntry()) { + note_denial(); + return Status::OK(); + } + CHECK_STATUS(reserve_status); + } + receiver_credit_denials_.erase(key); + + reservation.active = true; + reservation.committed = false; + reservation.key = key; + reservation.charge = std::move(charge); + reservation.target_id = request.target_id; + reservation.rpc_server_addr = std::move(target.rpc_server_addr); + reservation.epoch = target.advert.epoch; + acquired = true; + return Status::OK(); +} + +void TransferEngineImpl::rollbackReceiverCredit( + ReceiverCreditReservation& reservation) { + if (!reservation.active) return; + if (!reservation.committed) { + const auto key = reservation.key; + const auto target_id = reservation.target_id; + const auto rpc_server_addr = reservation.rpc_server_addr; + const auto epoch = reservation.epoch; + auto status = sender_credit_ledger_->rollbackReservation( + reservation.key, reservation.charge); + reservation = ReceiverCreditReservation{}; + if (!status.ok()) { + LOG(ERROR) << "Receiver-credit rollback failed: " + << status.ToString(); + return; + } + scheduleReceiverCreditRelease(key, target_id, rpc_server_addr, epoch); + (void)tryReportReceiverCreditRelease(key); + return; + } + reservation = ReceiverCreditReservation{}; +} + +void TransferEngineImpl::releaseReceiverCredit( + ReceiverCreditReservation& reservation) { + if (!reservation.active) return; + if (!reservation.committed) { + rollbackReceiverCredit(reservation); + return; + } + + const auto key = reservation.key; + const auto target_id = reservation.target_id; + const auto rpc_server_addr = reservation.rpc_server_addr; + const auto epoch = reservation.epoch; + auto status = sender_credit_ledger_->releaseCommitted(reservation.key, + reservation.charge); + reservation = ReceiverCreditReservation{}; + if (!status.ok()) { + LOG(ERROR) << "Receiver-credit terminal release failed: " + << status.ToString(); + return; + } + + scheduleReceiverCreditRelease(key, target_id, rpc_server_addr, epoch); + (void)tryReportReceiverCreditRelease(key); +} + +void TransferEngineImpl::scheduleReceiverCreditRelease( + const CreditKey& key, SegmentID target_id, + const std::string& rpc_server_addr, uint64_t epoch) { + auto& pending = pending_credit_releases_[key]; + pending.target_id = target_id; + pending.rpc_server_addr = rpc_server_addr; + pending.epoch = epoch; + pending.retry_at = std::chrono::steady_clock::now(); +} + +bool TransferEngineImpl::tryReportReceiverCreditRelease(const CreditKey& key) { + auto pending_it = pending_credit_releases_.find(key); + if (pending_it == pending_credit_releases_.end()) return true; + auto& pending = pending_it->second; + if (std::chrono::steady_clock::now() < pending.retry_at) return false; + + auto defer_retry = [&](const Status& status) { + ++pending.attempts; + const uint32_t shift = std::min(pending.attempts - 1, 4); + const auto delay = std::chrono::milliseconds(50U << shift); + pending.retry_at = std::chrono::steady_clock::now() + delay; + LOG(WARNING) << "Receiver-credit release exchange deferred (attempt " + << pending.attempts << "): " << status.ToString(); + return false; + }; + + ReceiverCreditExchangeRequestV1 request; + auto status = sender_credit_ledger_->prepareReleaseExchange(key, request); + if (!status.ok()) { + return defer_retry(status); + } + ReceiverCreditExchangeReplyV1 reply; + status = exchangeReceiverCredit(pending.target_id, pending.rpc_server_addr, + request, reply); + if (!status.ok()) { + return defer_retry(status); + } + if (!(reply.sender_instance_id == sender_instance_id_)) { + return defer_retry(Status::InvalidArgument( + "receiver-credit release reply sender mismatch" LOC_MARK)); + } + if (reply.flags != 0) { + return defer_retry(Status::InvalidArgument( + "receiver-credit release reply flags" LOC_MARK)); + } + CreditUpdateDisposition disposition; + status = sender_credit_ledger_->applyUpdate(key, reply.update, disposition); + if (!status.ok()) { + return defer_retry(status); + } + pending_credit_releases_.erase(pending_it); + notifyRuntimeQueueReady(); + return true; +} + +void TransferEngineImpl::progressReceiverCreditReleases() { + const auto now = std::chrono::steady_clock::now(); + std::vector due; + due.reserve(pending_credit_releases_.size()); + for (const auto& [key, pending] : pending_credit_releases_) { + if (pending.retry_at <= now) due.push_back(key); + } + for (const auto& key : due) (void)tryReportReceiverCreditRelease(key); +} + +void TransferEngineImpl::observeReceiverCreditSession( + SegmentID target_id, const ReceiverSessionId& session) { + auto [it, inserted] = + receiver_credit_sessions_.try_emplace(target_id, session); + if (inserted || it->second == session) return; + + const auto stale_session = it->second; + auto status = sender_credit_ledger_->deactivateSession(stale_session); + if (!status.ok()) { + LOG(ERROR) << "Receiver-credit stale session cleanup failed: " + << status.ToString(); + } + for (auto pending = pending_credit_releases_.begin(); + pending != pending_credit_releases_.end();) { + if (pending->first.receiver_session == stale_session) { + pending = pending_credit_releases_.erase(pending); + } else { + ++pending; + } + } + for (auto denial = receiver_credit_denials_.begin(); + denial != receiver_credit_denials_.end();) { + if (denial->first.receiver_session == stale_session) { + denial = receiver_credit_denials_.erase(denial); + } else { + ++denial; + } + } + it->second = session; +} + +void TransferEngineImpl::bindReceiverCreditFence( + Transport::SubBatchRef sub_batch, + const ReceiverCreditReservation& reservation) const { + if (!sub_batch) return; + sub_batch->receiver_credit_session_high = 0; + sub_batch->receiver_credit_session_low = 0; + sub_batch->receiver_credit_epoch = 0; + if (!reservation.active) return; + sub_batch->receiver_credit_session_high = + reservation.key.receiver_session.high; + sub_batch->receiver_credit_session_low = + reservation.key.receiver_session.low; + sub_batch->receiver_credit_epoch = reservation.epoch; +} + +void TransferEngineImpl::bindReceiverCreditFence( + Transport::SubBatchRef sub_batch, const Request& request) const { + if (!sub_batch) return; + sub_batch->receiver_credit_session_high = + request.receiver_credit_session_high; + sub_batch->receiver_credit_session_low = + request.receiver_credit_session_low; + sub_batch->receiver_credit_epoch = request.receiver_credit_epoch; +} + Status TransferEngineImpl::commitPreparedSubmit( Batch* batch, const PreparedSubmit& prepared) { if (!batch) return Status::InvalidArgument("Invalid batch" LOC_MARK); @@ -2648,6 +3250,8 @@ Status TransferEngineImpl::enqueuePreparedSubmit(Batch* batch, queued.batch = batch; queued.owner_task_id = prepared.owners[i].owner_task_id; queued.byte_charge = prepared.owners[i].request.length; + queued.receiver_credit_exempt = + prepared.owners[i].receiver_credit_exempt; queued.public_task_ids.push_back(prepared.owners[i].owner_task_id); queued.public_task_ids.insert( queued.public_task_ids.end(), @@ -2672,6 +3276,7 @@ Status TransferEngineImpl::finishQueuedOwner( "runtime dispatch window accounting underflow" LOC_MARK); } } + releaseReceiverCredit(queued.credit); CHECK_STATUS(runtime_queue_->complete(owner_id, terminal_status)); if (queued.in_dispatch_window) { --dispatch_inflight_owners_; @@ -2772,7 +3377,7 @@ Status TransferEngineImpl::dispatchQueuedOwner(QueueOwnerId owner_id) { if (queued_it == queued_owners_.end()) { return Status::InternalError("queued owner metadata missing" LOC_MARK); } - const auto queued = queued_it->second; + auto& queued = queued_it->second; auto* batch = queued.batch; auto& task = batch->task_list[queued.owner_task_id]; task.dispatch_time = std::chrono::steady_clock::now(); @@ -2790,13 +3395,43 @@ Status TransferEngineImpl::dispatchQueuedOwner(QueueOwnerId owner_id) { if (staging_params.empty()) findStagingPolicy(task.request, staging_params); if (!staging_params.empty() && staging_proxy_) { + bool acquired = true; + auto credit_status = Status::OK(); + if (!queued.receiver_credit_exempt) { + acquired = false; + credit_status = acquireReceiverCredit(task.request, + queued.credit, acquired); + } + if (!credit_status.ok()) { + LOG(WARNING) << "Receiver-credit gate rejected staging " + "dispatch: " + << credit_status.ToString(); + return finishQueuedOwner(owner_id, FAILED); + } + if (!acquired) { + task.type = UNSPEC; + task.sub_task_id = -1; + CHECK_STATUS(runtime_queue_->defer(owner_id)); + return Status::OK(); + } + if (queued.credit.active) { + task.request.receiver_credit_session_high = + queued.credit.key.receiver_session.high; + task.request.receiver_credit_session_low = + queued.credit.key.receiver_session.low; + task.request.receiver_credit_epoch = queued.credit.epoch; + } task.staging = true; // Orchestration only; the real transport submissions issued by // ProxyManager are counted where they recurse through the // non-staging path below. auto status = staging_proxy_->submit(&task, (BatchID)batch, staging_params); - if (!status.ok()) return finishQueuedOwner(owner_id, FAILED); + if (!status.ok()) { + rollbackReceiverCredit(queued.credit); + return finishQueuedOwner(owner_id, FAILED); + } + if (queued.credit.active) queued.credit.committed = true; task.post_time = std::chrono::steady_clock::now(); return markQueuedOwnerSubmitted(owner_id); } @@ -2818,6 +3453,30 @@ Status TransferEngineImpl::dispatchQueuedOwner(QueueOwnerId owner_id) { sub_batch->device_mask = task.device_mask; if (task.type == RDMA) sub_batch->qp_pool = task.qp_pool; } + + bool acquired = true; + auto credit_status = Status::OK(); + if (!queued.receiver_credit_exempt) { + acquired = false; + credit_status = + acquireReceiverCredit(task.request, queued.credit, acquired); + } + if (!credit_status.ok()) { + LOG(WARNING) << "Receiver-credit gate rejected dispatch: " + << credit_status.ToString(); + return finishQueuedOwner(owner_id, FAILED); + } + if (!acquired) { + task.type = UNSPEC; + task.sub_task_id = -1; + CHECK_STATUS(runtime_queue_->defer(owner_id)); + return Status::OK(); + } + if (queued.receiver_credit_exempt) { + bindReceiverCreditFence(sub_batch, task.request); + } else { + bindReceiverCreditFence(sub_batch, queued.credit); + } task.sub_task_id = sub_batch->size(); startTransportAttempt(task, task.type, std::chrono::steady_clock::now()); auto status = transport->submitTransferTasks(sub_batch, {task.request}); @@ -2830,8 +3489,12 @@ Status TransferEngineImpl::dispatchQueuedOwner(QueueOwnerId owner_id) { if (attemptSubmitStageFailover(batch, queued.owner_task_id)) { return markQueuedOwnerSubmitted(owner_id); } + rollbackReceiverCredit(queued.credit); + task.type = UNSPEC; return finishQueuedOwner(owner_id, FAILED); } + if (queued.credit.active) queued.credit.committed = true; + task.post_time = std::chrono::steady_clock::now(); return markQueuedOwnerSubmitted(owner_id); } @@ -2873,6 +3536,7 @@ Status TransferEngineImpl::progressRuntimeQueue() { std::lock_guard lk(progress_mutex_); if (!runtime_queue_config_.enabled) return Status::OK(); + progressReceiverCreditReleases(); CHECK_STATUS(refillDispatchWindow()); std::vector owner_ids; @@ -2917,7 +3581,8 @@ Status TransferEngineImpl::progressRuntimeQueue() { bool TransferEngineImpl::hasActiveRuntimeQueue() { std::lock_guard lk(progress_mutex_); - return runtime_queue_config_.enabled && !queued_owners_.empty(); + return runtime_queue_config_.enabled && + (!queued_owners_.empty() || !pending_credit_releases_.empty()); } bool TransferEngineImpl::shouldQueueSubmit(const PreparedSubmit& prepared, @@ -2925,6 +3590,12 @@ bool TransferEngineImpl::shouldQueueSubmit(const PreparedSubmit& prepared, if (!runtime_queue_config_.enabled) return false; if (prepared.require_post_submit_cancellation) return false; if (owner_kind == QueueOwnerKind::StagingInternal) return true; + if (std::any_of(prepared.owners.begin(), prepared.owners.end(), + [](const PreparedSubmit::Owner& owner) { + return owner.receiver_credit_required; + })) { + return true; + } return std::none_of( prepared.owners.begin(), prepared.owners.end(), [](const PreparedSubmit::Owner& owner) { return owner.staging; }); @@ -2944,8 +3615,8 @@ Status TransferEngineImpl::submitTransfer( std::lock_guard lk(progressLockFor(batch_id)); const size_t start_task_id = batch_ref.get()->task_list.size(); PreparedSubmit prepared; - CHECK_STATUS(prepareSubmit(batch_ref.get(), request_list, prepared, - require_post_submit_cancellation)); + CHECK_STATUS(prepareSubmit(batch_ref.get(), request_list, owner_kind, + prepared, require_post_submit_cancellation)); if (shouldQueueSubmit(prepared, owner_kind)) { CHECK_STATUS( @@ -3138,6 +3809,18 @@ Status TransferEngineImpl::resubmitTransferTask(Batch* batch, size_t task_id) { auto& task = batch->task_list[task_id]; auto prev_type = task.type; + QueueOwnerId queue_owner_id = 0; + QueuedOwnerState* queued_owner = nullptr; + if (runtime_queue_config_.enabled && batch->queue_token != 0 && + runtime_queue_ + ->resolveOwner(batch->queue_token, task_id, queue_owner_id) + .ok()) { + auto queued_it = queued_owners_.find(queue_owner_id); + if (queued_it != queued_owners_.end()) { + queued_owner = &queued_it->second; + } + } + if (++task.failover_count > task.runtime_policy.max_failover_attempts) { LOG(WARNING) << "Task failover limit reached (" << task.runtime_policy.max_failover_attempts @@ -3198,13 +3881,72 @@ Status TransferEngineImpl::resubmitTransferTask(Batch* batch, size_t task_id) { } task.sub_task_id = sub_batch->size(); task.type = type; + ReceiverCreditTarget credit_target; + const bool credit_exempt = + queued_owner && queued_owner->receiver_credit_exempt; + if (!credit_exempt) { + CHECK_STATUS(resolveReceiverCreditTarget(task.request, credit_target)); + if (credit_target.required && !queued_owner) { + return Status::InvalidEntry( + "credit-required failover needs runtime queue " + "ownership" LOC_MARK); + } + } + if (queued_owner && queued_owner->credit.active && + (!credit_target.required || + queued_owner->credit.key.receiver_session != + credit_target.advert.receiver_session_id || + queued_owner->credit.epoch != credit_target.advert.epoch)) { + // A reservation from another receiver incarnation cannot authorize + // this attempt. Do not report it to a possibly replaced authority; + // abandon it and acquire against the current session below. + queued_owner->credit = ReceiverCreditReservation{}; + } + bool acquired_this_attempt = false; + if (!credit_exempt && queued_owner && !queued_owner->credit.active) { + bool acquired = false; + CHECK_STATUS(acquireReceiverCredit(task.request, queued_owner->credit, + acquired)); + if (!acquired) { + if (!queued_owner->in_dispatch_window || + dispatch_inflight_owners_ == 0 || + dispatch_inflight_bytes_ < queued_owner->byte_charge) { + return Status::InternalError( + "runtime dispatch window accounting underflow" LOC_MARK); + } + --dispatch_inflight_owners_; + dispatch_inflight_bytes_ -= queued_owner->byte_charge; + queued_owner->in_dispatch_window = false; + CHECK_STATUS(runtime_queue_->defer(queue_owner_id)); + task.type = UNSPEC; + task.sub_task_id = -1; + return Status::OK(); + } + acquired_this_attempt = queued_owner->credit.active; + } + if (queued_owner) { + if (queued_owner->receiver_credit_exempt) { + bindReceiverCreditFence(sub_batch, task.request); + } else { + bindReceiverCreditFence(sub_batch, queued_owner->credit); + } + } startTransportAttempt(task, type, std::chrono::steady_clock::now()); - auto status = transport->submitTransferTasks(sub_batch, {task.request}); - if (!status.ok()) { + auto submit_status = + transport->submitTransferTasks(sub_batch, {task.request}); + if (!submit_status.ok()) { if (task.failure_stage < 0) task.failure_stage = 0; finishTransportAttempt(task, FAILED, std::chrono::steady_clock::now()); + if (queued_owner && acquired_this_attempt) { + rollbackReceiverCredit(queued_owner->credit); + } + return submit_status; } - return status; + if (queued_owner && queued_owner->credit.active) { + queued_owner->credit.committed = true; + } + task.post_time = std::chrono::steady_clock::now(); + return Status::OK(); } bool TransferEngineImpl::attemptSubmitStageFailover(Batch* batch, @@ -3814,9 +4556,32 @@ Status TransferEngineImpl::waitTransferCompletion(BatchID batch_id) { Status TransferEngineImpl::transferSync( const std::vector& request_list) { + auto has_any_fence = [](const Request& request) { + return request.receiver_credit_session_high != 0 || + request.receiver_credit_session_low != 0 || + request.receiver_credit_epoch != 0; + }; + auto has_complete_fence = [](const Request& request) { + return (request.receiver_credit_session_high != 0 || + request.receiver_credit_session_low != 0) && + request.receiver_credit_epoch != 0; + }; + const bool delegated_credit_fence = + !request_list.empty() && + std::all_of(request_list.begin(), request_list.end(), + has_complete_fence); + if (!delegated_credit_fence && + std::any_of(request_list.begin(), request_list.end(), has_any_fence)) { + return Status::InvalidArgument( + "mixed or partial delegated receiver-credit fence" LOC_MARK); + } + auto batch_id = allocateBatch(request_list.size()); BatchGuard batch_guard(*this, batch_id); - CHECK_STATUS(submitTransfer(batch_id, request_list)); + CHECK_STATUS(submitTransfer(batch_id, request_list, nullptr, + delegated_credit_fence + ? QueueOwnerKind::StagingInternal + : QueueOwnerKind::User)); for (uint64_t poll_count = 0;; ++poll_count) { TransferStatus xfer_status; CHECK_STATUS(progressBatch(batch_id, xfer_status)); diff --git a/mooncake-transfer-engine/tent/src/transport/hp_tcp/hp_tcp_transport.cpp b/mooncake-transfer-engine/tent/src/transport/hp_tcp/hp_tcp_transport.cpp index 978bd7d934..4355889779 100644 --- a/mooncake-transfer-engine/tent/src/transport/hp_tcp/hp_tcp_transport.cpp +++ b/mooncake-transfer-engine/tent/src/transport/hp_tcp/hp_tcp_transport.cpp @@ -2,6 +2,7 @@ #include "tent/transport/hp_tcp/hp_tcp_transport.h" #include +#include #include #include #include @@ -114,6 +115,41 @@ HighPerformanceTcpTransport::~HighPerformanceTcpTransport() { } } +namespace { + +// One outstanding operation on a peer's lane. The count is taken when the +// operation is planned. It is given back as soon as the operation completes or +// is cancelled, so the next submission already sees the lane as free, and in +// any case when the last holder lets go: the planned command, the operation +// queued on its lane and its completion callback all share the slot. A planned +// command dropped without running (a rejected admission, a later request of +// the same batch failing to plan, a rejected retry) therefore releases it too, +// exactly once, and a lane is never avoided because of a leaked count. +class LaneSlot { + public: + LaneSlot(std::shared_ptr>> load, + uint32_t lane) + : load_(std::move(load)), lane_(lane) { + (*load_)[lane_].fetch_add(1, std::memory_order_relaxed); + } + ~LaneSlot() { release(); } + LaneSlot(const LaneSlot&) = delete; + LaneSlot& operator=(const LaneSlot&) = delete; + + void release() noexcept { + if (!released_.exchange(true, std::memory_order_relaxed)) { + (*load_)[lane_].fetch_sub(1, std::memory_order_relaxed); + } + } + + private: + std::shared_ptr>> load_; + uint32_t lane_; + std::atomic released_{false}; +}; + +} // namespace + Status HighPerformanceTcpTransport::validateParams() const { if (params_.worker_count == 0 || params_.connections_per_peer == 0 || params_.max_outstanding_tasks == 0 || @@ -500,6 +536,25 @@ Status HighPerformanceTcpTransport::planTask( if (segment == nullptr || segment->type != SegmentType::Memory) { return NeedsRefresh("HP TCP target is not a memory segment"); } + const bool has_credit_fence = + batch.receiver_credit_session_high != 0 || + batch.receiver_credit_session_low != 0 || + batch.receiver_credit_epoch != 0; + if (has_credit_fence) { + const auto& advert = segment->getMemory().receiver_credit; + if (!advert || + advert->schema_version != kReceiverCreditProtocolVersion || + advert->flags != kReceiverCreditRequired || + advert->receiver_session_id.high != + batch.receiver_credit_session_high || + advert->receiver_session_id.low != + batch.receiver_credit_session_low || + advert->epoch != batch.receiver_credit_epoch) { + return Status::InvalidEntry( + "HP TCP receiver-credit session changed before " + "post" LOC_MARK); + } + } BufferDesc* buffer = segment->findBuffer(request.target_offset, request.length); if (buffer == nullptr || @@ -565,12 +620,19 @@ Status HighPerformanceTcpTransport::planTask( planned_task = task; uint64_t lane_sequence; + std::shared_ptr>> lane_load; { std::lock_guard lock(lane_sequence_mutex_); // Preserve the first lane choice, then advance independently. auto it = next_lane_sequence_.try_emplace(request.target_id, request_id) .first; lane_sequence = it->second++; + auto& load = lane_load_[request.target_id]; + if (!load || load->size() != params_.connections_per_peer) { + load = std::make_shared>>( + params_.connections_per_peer); + } + lane_load = load; } uint64_t slice_offset = 0; for (size_t slice = 0; slice < slice_count; ++slice) { @@ -579,10 +641,26 @@ Status HighPerformanceTcpTransport::planTask( (slice < request.length % slice_count ? 1 : 0); uint32_t lane_id = 0; uint32_t endpoint_id = 0; + std::shared_ptr slot; // single-slice operations only if (slice_count == 1) { - lane_id = static_cast( - lane_sequence % - static_cast(params_.connections_per_peer)); + // Least-loaded lane, ties broken from the round-robin position: a + // lane that just went idle takes the next operation instead of + // waiting for the sequence to come back around (which left lanes + // idle ~half the time under a fixed in-flight window). + const size_t n = params_.connections_per_peer; + const size_t start = static_cast(lane_sequence % n); + lane_id = static_cast(start); + int32_t best = (*lane_load)[start].load(std::memory_order_relaxed); + for (size_t k = 1; k < n && best > 0; ++k) { + const size_t cand = (start + k) % n; + const int32_t v = + (*lane_load)[cand].load(std::memory_order_relaxed); + if (v < best) { + best = v; + lane_id = static_cast(cand); + } + } + slot = std::make_shared(lane_load, lane_id); endpoint_id = static_cast(lane_id % endpoint_count); } else { endpoint_id = static_cast(slice); @@ -617,8 +695,10 @@ Status HighPerformanceTcpTransport::planTask( operation.request_id = request_id; if (slice_count == 1) { operation.complete = - [task](TransferStatusEnum terminal, size_t bytes, - std::optional remote_status) { + [task, slot]( + TransferStatusEnum terminal, size_t bytes, + std::optional remote_status) { + slot->release(); (void)task->completeOnce(terminal, bytes, remote_status); }; } else { @@ -639,12 +719,13 @@ Status HighPerformanceTcpTransport::planTask( HighPerformanceTcpWorkers::Command command; command.worker_id = owner_worker; - command.run = [this, task, slice_count, + command.run = [this, task, slice_count, slot, operation = std::move(operation)](size_t worker_id) mutable { if (task->cancelRequested() || stopping_.load(std::memory_order_acquire)) { if (slice_count == 1) { + slot->release(); (void)task->completeOnce(CANCELED, 0); } else { (void)task->completeSlice(CANCELED); @@ -653,8 +734,9 @@ Status HighPerformanceTcpTransport::planTask( } client_->enqueueOnOwner(worker_id, std::move(operation)); }; - command.cancel = [task, slice_count] { + command.cancel = [task, slice_count, slot] { if (slice_count == 1) { + slot->release(); (void)task->completeOnce(CANCELED, 0); } else { (void)task->completeSlice(CANCELED); diff --git a/mooncake-transfer-engine/tent/src/transport/io_uring/io_uring_transport.cpp b/mooncake-transfer-engine/tent/src/transport/io_uring/io_uring_transport.cpp index 4104341df6..c947634b05 100644 --- a/mooncake-transfer-engine/tent/src/transport/io_uring/io_uring_transport.cpp +++ b/mooncake-transfer-engine/tent/src/transport/io_uring/io_uring_transport.cpp @@ -19,10 +19,12 @@ #include #include +#include #include #include #include #include +#include #include "tent/runtime/slab.h" #include "tent/common/utils/os.h" @@ -244,11 +246,51 @@ Status IOUringTransport::allocateSubBatch(SubBatchRef& batch, size_t max_size) { return Status::OK(); } +// Reaps the completions of `outstanding` requests before the ring is +// destroyed: io_uring_queue_exit() does not wait for in-flight I/O. SQEs +// still in the SQ ring were never submitted and never run, so they are not +// waited for. If io_uring_wait_cqe() fails, the CQ ring is polled instead: +// it is mapped into this process and still receives completions, so the +// requests are reaped either way. +static void waitForInFlight(struct io_uring* ring, size_t outstanding) { + const size_t unsubmitted = io_uring_sq_ready(ring); + size_t in_flight = + outstanding > unsubmitted ? outstanding - unsubmitted : 0; + bool polling = false; + while (in_flight) { + struct io_uring_cqe* cqe = nullptr; + int rc = polling ? io_uring_peek_cqe(ring, &cqe) + : io_uring_wait_cqe(ring, &cqe); + if (rc < 0 && polling) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + if (rc == -EINTR) continue; + if (rc < 0) { + LOG(ERROR) << "IOUringTransport: io_uring_wait_cqe failed: " + << strerror(-rc) << "; polling for " << in_flight + << " in-flight completions"; + polling = true; + continue; + } + io_uring_cqe_seen(ring, cqe); + --in_flight; + } +} + Status IOUringTransport::freeSubBatch(SubBatchRef& batch) { auto io_uring_batch = dynamic_cast(batch); if (!io_uring_batch) return Status::InvalidArgument("Invalid IO Uring sub-batch" LOC_MARK); - io_uring_queue_exit(&io_uring_batch->ring); + auto& ring = io_uring_batch->ring; + // io_uring_queue_exit() does not wait for in-flight I/O, which would then + // land in staging buffers freed below, or in caller memory reused once the + // batch is freed: reap it first. + size_t outstanding = 0; + for (auto& task : io_uring_batch->task_list) + if (task.status_word == TransferStatusEnum::PENDING) ++outstanding; + waitForInFlight(&ring, outstanding); + io_uring_queue_exit(&ring); Slab::Get().deallocate(io_uring_batch); batch = nullptr; return Status::OK(); diff --git a/mooncake-transfer-engine/tent/src/transport/rdma/rdma_transport.cpp b/mooncake-transfer-engine/tent/src/transport/rdma/rdma_transport.cpp index 746c0fb6f2..ed02b2dc32 100644 --- a/mooncake-transfer-engine/tent/src/transport/rdma/rdma_transport.cpp +++ b/mooncake-transfer-engine/tent/src/transport/rdma/rdma_transport.cpp @@ -557,6 +557,11 @@ Status RdmaTransport::submitTransferTasks( task->request = request; task->device_mask = rdma_batch->device_mask; task->qp_pool = rdma_batch->qp_pool; // RFC #2568 step 3 + task->receiver_credit_session_high = + rdma_batch->receiver_credit_session_high; + task->receiver_credit_session_low = + rdma_batch->receiver_credit_session_low; + task->receiver_credit_epoch = rdma_batch->receiver_credit_epoch; task->num_slices = 0; task->status_word = PENDING; task->transferred_bytes = 0; diff --git a/mooncake-transfer-engine/tent/src/transport/rdma/workers.cpp b/mooncake-transfer-engine/tent/src/transport/rdma/workers.cpp index 1c8ff0a681..0d5a155550 100644 --- a/mooncake-transfer-engine/tent/src/transport/rdma/workers.cpp +++ b/mooncake-transfer-engine/tent/src/transport/rdma/workers.cpp @@ -1981,6 +1981,24 @@ Status Workers::generatePostPath(RdmaSlice* slice) { auto target_id = slice->task->request.target_id; CHECK_STATUS(getRouteHint(target, target_id, (uint64_t)slice->target_addr, slice->length)); + const bool has_credit_fence = + slice->task->receiver_credit_session_high != 0 || + slice->task->receiver_credit_session_low != 0 || + slice->task->receiver_credit_epoch != 0; + if (has_credit_fence) { + const auto& advert = target.segment->getMemory().receiver_credit; + if (!advert || + advert->schema_version != kReceiverCreditProtocolVersion || + advert->flags != kReceiverCreditRequired || + advert->receiver_session_id.high != + slice->task->receiver_credit_session_high || + advert->receiver_session_id.low != + slice->task->receiver_credit_session_low || + advert->epoch != slice->task->receiver_credit_epoch) { + return Status::InvalidEntry( + "RDMA receiver-credit session changed before post" LOC_MARK); + } + } if (slice->retry_count == 0) CHECK_STATUS(selectOptimalDevice(source, target, slice)); diff --git a/mooncake-transfer-engine/tent/src/transport/tcp/tcp_transport.cpp b/mooncake-transfer-engine/tent/src/transport/tcp/tcp_transport.cpp index 9def334c40..6a84520236 100644 --- a/mooncake-transfer-engine/tent/src/transport/tcp/tcp_transport.cpp +++ b/mooncake-transfer-engine/tent/src/transport/tcp/tcp_transport.cpp @@ -178,6 +178,9 @@ Status TcpTransport::submitTransferTasks( task.request = request; task.progress_batch_id = batch->progress_batch_id; task.notify_progress = batch->notify_progress; + task.receiver_credit_session_high = batch->receiver_credit_session_high; + task.receiver_credit_session_low = batch->receiver_credit_session_low; + task.receiver_credit_epoch = batch->receiver_credit_epoch; task.status_word.store(TransferStatusEnum::PENDING, std::memory_order_release); } @@ -264,9 +267,7 @@ Status TcpTransport::doTransferWithRetry(TcpTask *task) { return Status::InternalError("Transport shutting down"); std::string rpc_server_addr; - auto status = - findRemoteSegment(task->request.target_offset, task->request.length, - task->request.target_id, rpc_server_addr); + auto status = findRemoteSegment(*task, rpc_server_addr); if (!status.ok()) return status; Status last_error; @@ -296,11 +297,15 @@ Status TcpTransport::doTransferWithRetry(TcpTask *task) { if (task->request.opcode == Request::WRITE) { status = ControlClient::sendData( rpc_server_addr, task->request.target_offset, - task->request.source, task->request.length); + task->request.source, task->request.length, + task->receiver_credit_session_high, + task->receiver_credit_session_low, task->receiver_credit_epoch); } else { status = ControlClient::recvData( rpc_server_addr, task->request.target_offset, - task->request.source, task->request.length); + task->request.source, task->request.length, + task->receiver_credit_session_high, + task->receiver_credit_session_low, task->receiver_credit_epoch); } if (status.ok()) return Status::OK(); @@ -327,9 +332,7 @@ Status TcpTransport::doTransferWithRetry(TcpTask *task) { CHECK_STATUS(metadata_->segmentManager().invalidateRemote( task->request.target_id)); if (attempt == params_.max_retry_count) break; - auto resolve = findRemoteSegment( - task->request.target_offset, task->request.length, - task->request.target_id, rpc_server_addr); + auto resolve = findRemoteSegment(*task, rpc_server_addr); if (!resolve.ok()) return resolve; } } @@ -337,12 +340,35 @@ Status TcpTransport::doTransferWithRetry(TcpTask *task) { return last_error; } -Status TcpTransport::findRemoteSegment(uint64_t dest_addr, uint64_t length, - uint64_t target_id, +Status TcpTransport::findRemoteSegment(const TcpTask &task, std::string &rpc_server_addr) { return metadata_->segmentManager().withCachedSegment( - target_id, [&](SegmentDesc *segment) { - auto buffer = segment->findBuffer(dest_addr, length); + task.request.target_id, [&](SegmentDesc *segment) { + const bool has_credit_fence = + task.receiver_credit_session_high != 0 || + task.receiver_credit_session_low != 0 || + task.receiver_credit_epoch != 0; + if (has_credit_fence) { + if (segment->type != SegmentType::Memory) { + return Status::InvalidMetadataType( + "TCP receiver-credit target is not memory" LOC_MARK); + } + const auto &advert = segment->getMemory().receiver_credit; + if (!advert || + advert->schema_version != kReceiverCreditProtocolVersion || + advert->flags != kReceiverCreditRequired || + advert->receiver_session_id.high != + task.receiver_credit_session_high || + advert->receiver_session_id.low != + task.receiver_credit_session_low || + advert->epoch != task.receiver_credit_epoch) { + return Status::InvalidEntry( + "TCP receiver-credit session changed before " + "send" LOC_MARK); + } + } + auto buffer = segment->findBuffer(task.request.target_offset, + task.request.length); rpc_server_addr = segment->rpc_server_addr; if (!buffer) { return Status::NeedsRefreshCache( diff --git a/mooncake-transfer-engine/tent/src/transport/ub/endpoint.cpp b/mooncake-transfer-engine/tent/src/transport/ub/endpoint.cpp index 665e9204d1..e887bbf158 100644 --- a/mooncake-transfer-engine/tent/src/transport/ub/endpoint.cpp +++ b/mooncake-transfer-engine/tent/src/transport/ub/endpoint.cpp @@ -34,7 +34,11 @@ bool samePeer(const UbBootstrapDesc& lhs, const UbBootstrapDesc& rhs) { return lhs.protocol_version == rhs.protocol_version && lhs.local_eid == rhs.local_eid && lhs.jetty_ids == rhs.jetty_ids && lhs.jetty_uasids == rhs.jetty_uasids && - lhs.endpoint_generation == rhs.endpoint_generation; + lhs.endpoint_generation == rhs.endpoint_generation && + lhs.receiver_credit_session_high == + rhs.receiver_credit_session_high && + lhs.receiver_credit_session_low == rhs.receiver_credit_session_low && + lhs.receiver_credit_epoch == rhs.receiver_credit_epoch; } } // namespace @@ -340,10 +344,26 @@ Status UbEndpoint::makeBootstrapDesc(const std::string& segment_name, } output.endpoint_generation = generation_; output.segment_generation = segment_generation; - output.capabilities = {"read", "write", "endpoint_generation"}; + output.capabilities = {"read", "write", "endpoint_generation", + "receiver_credit_v1"}; return Status::OK(); } +bool UbEndpoint::peerSupportsCapability(std::string_view capability) const { + std::lock_guard lock(lifecycle_mutex_); + return std::find(peer_.capabilities.begin(), peer_.capabilities.end(), + capability) != peer_.capabilities.end(); +} + +bool UbEndpoint::peerMatchesReceiverCredit(uint64_t session_high, + uint64_t session_low, + uint64_t epoch) const { + std::lock_guard lock(lifecycle_mutex_); + return peer_.receiver_credit_session_high == session_high && + peer_.receiver_credit_session_low == session_low && + peer_.receiver_credit_epoch == epoch; +} + bool UbEndpoint::tryAcquireOutstanding(uint64_t bytes) noexcept { std::lock_guard lock(lifecycle_mutex_); if (state_.load(std::memory_order_relaxed) != State::kReady || !context_ || diff --git a/mooncake-transfer-engine/tent/src/transport/ub/ub_transport.cpp b/mooncake-transfer-engine/tent/src/transport/ub/ub_transport.cpp index 652b18c6d3..59c9c14422 100644 --- a/mooncake-transfer-engine/tent/src/transport/ub/ub_transport.cpp +++ b/mooncake-transfer-engine/tent/src/transport/ub/ub_transport.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,8 @@ namespace mooncake::tent { namespace { +constexpr std::string_view kReceiverCreditCapability = "receiver_credit_v1"; + bool filterAllows(const std::vector& filter, const ub::DeviceInfo& device) { if (filter.empty()) return true; @@ -45,6 +48,43 @@ bool filterAllows(const std::vector& filter, filter.end(); } +bool receiverCreditRequired(const SegmentDesc* segment) { + if (!segment || segment->type != SegmentType::Memory) return false; + const auto& advert = segment->getMemory().receiver_credit; + return advert && advert->schema_version == kReceiverCreditProtocolVersion && + advert->flags == kReceiverCreditRequired && + !advert->receiver_session_id.empty() && advert->epoch != 0; +} + +const ReceiverCreditAdvertV1* receiverCreditAdvert(const SegmentDesc* segment) { + if (!receiverCreditRequired(segment)) return nullptr; + return &*segment->getMemory().receiver_credit; +} + +void bindReceiverCreditIdentity(const SegmentDesc* segment, + UbBootstrapDesc& bootstrap) { + const auto* advert = receiverCreditAdvert(segment); + if (!advert) return; + bootstrap.receiver_credit_session_high = advert->receiver_session_id.high; + bootstrap.receiver_credit_session_low = advert->receiver_session_id.low; + bootstrap.receiver_credit_epoch = advert->epoch; +} + +bool matchesReceiverCredit(const UbBootstrapDesc& bootstrap, + const ReceiverCreditAdvertV1& advert) { + return bootstrap.receiver_credit_session_high == + advert.receiver_session_id.high && + bootstrap.receiver_credit_session_low == + advert.receiver_session_id.low && + bootstrap.receiver_credit_epoch == advert.epoch; +} + +bool supportsReceiverCredit(const UbBootstrapDesc& bootstrap) { + return std::find(bootstrap.capabilities.begin(), + bootstrap.capabilities.end(), + kReceiverCreditCapability) != bootstrap.capabilities.end(); +} + uint64_t generationSeed() { const auto wall = static_cast( std::chrono::duration_cast( @@ -510,7 +550,22 @@ struct UbTransport::Impl { request.remote_topology_id, peer_path}; CHECK_STATUS( endpoints->getOrCreate(key, request.local_context, endpoint)); + const auto* credit_advert = + receiverCreditAdvert(request.remote_segment); + const bool credit_required = credit_advert != nullptr; if (endpoint->ready()) { + if (credit_required && + (!endpoint->peerSupportsCapability(kReceiverCreditCapability) || + !endpoint->peerMatchesReceiverCredit( + credit_advert->receiver_session_id.high, + credit_advert->receiver_session_id.low, + credit_advert->epoch))) { + (void)endpoints->retire(endpoint); + endpoint.reset(); + return Status::InvalidArgument( + "Remote UB endpoint lacks required receiver-credit " + "support" LOC_MARK); + } recordReadyEndpoint(endpoint); return Status::OK(); } @@ -527,9 +582,20 @@ struct UbTransport::Impl { endpoint.reset(); return status; } + auto local_segment = + metadata ? metadata->segmentManager().getLocal() : SegmentDescRef{}; + bindReceiverCreditIdentity(local_segment.get(), bootstrap); UbBootstrapDesc response; status = ControlClient::bootstrapUb( request.remote_segment->rpc_server_addr, bootstrap, response); + if (status.ok() && credit_required) { + if (!supportsReceiverCredit(response) || + !matchesReceiverCredit(response, *credit_advert)) { + status = Status::InvalidArgument( + "Remote UB bootstrap receiver-credit identity " + "mismatch" LOC_MARK); + } + } if (status.ok()) status = endpoint->bind(response); if (!status.ok()) { (void)endpoints->retire(endpoint); @@ -545,6 +611,14 @@ struct UbTransport::Impl { response.reply_msg = "UB transport is shutting down"; return -1; } + auto local_segment = + metadata ? metadata->segmentManager().getLocal() : SegmentDescRef{}; + if (receiverCreditRequired(local_segment.get()) && + !supportsReceiverCredit(request)) { + response.reply_msg = + "UB bootstrap peer lacks required receiver-credit support"; + return -1; + } const std::string local_name = getNicNameFromNicPath(request.peer_nic_path); auto context_it = context_by_topology_name.find(local_name); @@ -571,6 +645,7 @@ struct UbTransport::Impl { request.local_nic_path, currentSegmentGeneration(), response); } if (status.ok()) { + bindReceiverCreditIdentity(local_segment.get(), response); recordReadyEndpoint(endpoint); } if (!status.ok()) { @@ -705,6 +780,7 @@ Status UbTransport::submitTransferTasks( const auto notify_progress = ub_batch->notify_progress; const auto progress_batch_id = ub_batch->progress_batch_id; + auto progress_armed = std::make_shared>(false); std::vector new_tasks; new_tasks.reserve(request_list.size()); for (const auto& request : request_list) { @@ -717,9 +793,12 @@ Status UbTransport::submitTransferTasks( "transports/ub/max_slices_per_task" LOC_MARK); } auto task = ub::UbTask::create( - request, - [notify_progress, progress_batch_id](const TransferStatus&) { - if (notify_progress) notify_progress(progress_batch_id); + request, [notify_progress, progress_batch_id, + progress_armed](const TransferStatus&) { + if (progress_armed->load(std::memory_order_acquire) && + notify_progress) { + notify_progress(progress_batch_id); + } }); size_t offset = 0; while (offset < request.length) { @@ -731,6 +810,11 @@ Status UbTransport::submitTransferTasks( spec.length = length; spec.request_offset = offset; spec.max_retries = impl_->params.max_retries; + spec.receiver_credit_session_high = + ub_batch->receiver_credit_session_high; + spec.receiver_credit_session_low = + ub_batch->receiver_credit_session_low; + spec.receiver_credit_epoch = ub_batch->receiver_credit_epoch; if (!task->addSlice(spec)) { return Status::InternalError( "Unable to construct UB slice" LOC_MARK); @@ -741,15 +825,15 @@ Status UbTransport::submitTransferTasks( new_tasks.push_back(std::move(task)); } - for (auto& task : new_tasks) { - ub_batch->task_list.push_back(task); - auto status = impl_->workers->submit(task, ub_batch->device_mask); - if (!status.ok()) { - for (auto& queued : new_tasks) { - if (queued) queued->requestCancellation(); - } - return status; - } + CHECK_STATUS(impl_->workers->submitBatch(new_tasks, ub_batch->device_mask)); + ub_batch->task_list.insert(ub_batch->task_list.end(), new_tasks.begin(), + new_tasks.end()); + progress_armed->store(true, std::memory_order_release); + // A very small task can finish between worker publication and the batch + // append above. Its callback intentionally stays disarmed until the task + // is queryable; this notification closes that race. + if (!new_tasks.empty() && notify_progress) { + notify_progress(progress_batch_id); } return Status::OK(); } diff --git a/mooncake-transfer-engine/tent/src/transport/ub/workers.cpp b/mooncake-transfer-engine/tent/src/transport/ub/workers.cpp index effe77fff2..28e3dc0e79 100644 --- a/mooncake-transfer-engine/tent/src/transport/ub/workers.cpp +++ b/mooncake-transfer-engine/tent/src/transport/ub/workers.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -275,44 +276,101 @@ Status UbWorkers::submit(const UbTask::Ptr& task, uint64_t device_mask) { if (!task) { return Status::InvalidArgument("Cannot submit a null UB task" LOC_MARK); } + return submitBatch({task}, device_mask); +} + +Status UbWorkers::submitBatch(const std::vector& tasks, + uint64_t device_mask) { if (!accepting_.load(std::memory_order_acquire)) { return Status::InvalidArgument( "UB workers are not accepting work" LOC_MARK); } - const auto snapshot = task->snapshot(); - if (!snapshot.sealed) { - return Status::InvalidArgument( - "UB task must be sealed before submission" LOC_MARK); - } - const int priority = - std::clamp(task->request().priority, static_cast(PRIO_HIGH), - static_cast(PRIO_LOW)); std::vector pending; - for (const auto& slice : task->slices()) { - if (slice && slice->markQueued()) { + std::unordered_set unique_tasks; + for (const auto& task : tasks) { + if (!task) { + return Status::InvalidArgument( + "Cannot submit a null UB task" LOC_MARK); + } + if (!unique_tasks.insert(task.get()).second) { + return Status::InvalidArgument( + "Cannot submit the same UB task twice in one batch" LOC_MARK); + } + const auto snapshot = task->snapshot(); + if (!snapshot.sealed) { + return Status::InvalidArgument( + "UB task must be sealed before submission" LOC_MARK); + } + const int priority = + std::clamp(task->request().priority, static_cast(PRIO_HIGH), + static_cast(PRIO_LOW)); + for (const auto& slice : task->slices()) { + if (!slice || slice->snapshot().state != UbSliceState::kInitial) { + return Status::InvalidArgument( + "UB task contains a slice that cannot be " + "submitted" LOC_MARK); + } pending.push_back(PendingSlice{task, slice, device_mask, priority, task->request().target_id, task->request().opcode}); } } - bool stopped_during_submit = false; + + Status failure = Status::OK(); + bool notify = false; + size_t marked_count = 0; { std::lock_guard lock(queue_mutex_); if (!accepting_.load(std::memory_order_relaxed)) { - stopped_during_submit = true; - } else { - for (auto& item : pending) { - queues_[item.priority].push_back(std::move(item)); + failure = Status::InvalidArgument( + "UB workers stopped during submission" LOC_MARK); + } + + if (failure.ok()) { + for (const auto& item : pending) { + if (!item.slice->markQueued()) { + failure = Status::InvalidArgument( + "UB task changed while it was being " + "submitted" LOC_MARK); + break; + } + ++marked_count; + } + } + + if (failure.ok()) { + std::array inserted_counts{}; + try { + for (const auto& item : pending) { + queues_[item.priority].push_back(item); + ++inserted_counts[item.priority]; + } + notify = !pending.empty(); + } catch (const std::bad_alloc&) { + for (size_t priority = 0; priority < queues_.size(); + ++priority) { + while (inserted_counts[priority] != 0) { + queues_[priority].pop_back(); + --inserted_counts[priority]; + } + } + failure = Status::InternalError( + "Unable to enqueue UB task batch" LOC_MARK); } } } - if (stopped_during_submit) { - for (auto& item : pending) item.slice->requestCancellation(); - return Status::InvalidArgument( - "UB workers stopped during submission" LOC_MARK); + + if (!failure.ok()) { + // A markQueued() race or allocation failure may occur after earlier + // slices changed state. None of them are visible to posting workers; + // cancel the private tasks so no slice remains orphaned in kQueued. + for (size_t index = 0; index < marked_count; ++index) { + (void)pending[index].task->requestCancellation(); + } + return failure; } - if (!pending.empty()) queue_cv_.notify_all(); + if (notify) queue_cv_.notify_all(); return Status::OK(); } @@ -472,9 +530,28 @@ Status UbWorkers::buildRoute(const PendingSlice& pending, Route& route) { return Status::InvalidMetadataType( "UB target is not a memory segment" LOC_MARK); } + const auto& spec = pending.slice->spec(); + const bool has_credit_fence = + spec.receiver_credit_session_high != 0 || + spec.receiver_credit_session_low != 0 || + spec.receiver_credit_epoch != 0; + if (has_credit_fence) { + const auto& advert = segment->getMemory().receiver_credit; + if (!advert || + advert->schema_version != kReceiverCreditProtocolVersion || + advert->flags != kReceiverCreditRequired || + advert->receiver_session_id.high != + spec.receiver_credit_session_high || + advert->receiver_session_id.low != + spec.receiver_credit_session_low || + advert->epoch != spec.receiver_credit_epoch) { + return Status::InvalidEntry( + "UB receiver-credit session changed before " + "post" LOC_MARK); + } + } auto* buffer = - segment->findBuffer(pending.slice->spec().remote_address, - pending.slice->spec().length); + segment->findBuffer(spec.remote_address, spec.length); if (!buffer) { return Status::NeedsRefreshCache( "UB target range is not registered" LOC_MARK); diff --git a/mooncake-transfer-engine/tent/tests/admission_queue_test.cpp b/mooncake-transfer-engine/tent/tests/admission_queue_test.cpp index ab404fff3c..438bb3057e 100644 --- a/mooncake-transfer-engine/tent/tests/admission_queue_test.cpp +++ b/mooncake-transfer-engine/tent/tests/admission_queue_test.cpp @@ -252,6 +252,193 @@ TEST(AdmissionQueueTest, KeepsAdmissionOrderForDispatch) { EXPECT_EQ(picked, expected_ids); } +TEST(AdmissionQueueTest, DefersDispatchingOwnerToFifoTailWithoutReaccounting) { + LocalTransferAdmissionQueue queue({4, 128, 0, 0}); + std::vector admitted_ids; + ASSERT_TRUE(queue + .tryAdmit(makeSubmit(1, 3, + {makeOwner(0, 16), makeOwner(1, 24), + makeOwner(2, 32)}), + admitted_ids) + .ok()); + ASSERT_EQ(admitted_ids.size(), 3u); + + auto picked = queue.pickForDispatch(1, 128); + ASSERT_EQ(picked, std::vector({1})); + const size_t owners_before = queue.outstandingOwners(); + const size_t bytes_before = queue.outstandingBytes(); + + EXPECT_TRUE(queue.defer(picked[0]).ok()); + EXPECT_EQ(queue.outstandingOwners(), owners_before); + EXPECT_EQ(queue.outstandingBytes(), bytes_before); + + picked = queue.pickForDispatch(3, 128); + const std::vector expected{2, 3, 1}; + EXPECT_EQ(picked, expected); +} + +TEST(AdmissionQueueTest, DeferredEdfOwnerYieldsToCurrentWaiters) { + QueueLimits limits{4, 128, 0, 0}; + limits.deadline_aware = true; + LocalTransferAdmissionQueue queue(limits); + std::vector admitted_ids; + auto first = makeOwner(0, 16); + first.request.deadline_ns = 100; + auto second = makeOwner(1, 16); + second.request.deadline_ns = 100; + auto third = makeOwner(2, 16); + third.request.deadline_ns = 200; + ASSERT_TRUE(queue + .tryAdmit(makeSubmit(1, 3, + {std::move(first), std::move(second), + std::move(third)}), + admitted_ids) + .ok()); + + auto picked = queue.pickForDispatch(1, 128); + ASSERT_EQ(picked, std::vector({1})); + EXPECT_TRUE(queue.defer(picked[0]).ok()); + + picked = queue.pickForDispatch(3, 128); + // A temporarily blocked earliest-deadline owner yields one round so it + // cannot starve later-deadline work that may have available resources. + const std::vector expected{2, 3, 1}; + EXPECT_EQ(picked, expected); +} + +TEST(AdmissionQueueTest, DeferredEdfOwnerKeepsRoundAheadOfNewAdmission) { + QueueLimits limits{4, 128, 0, 0}; + limits.deadline_aware = true; + LocalTransferAdmissionQueue queue(limits); + std::vector admitted_ids; + + auto first = makeOwner(0, 16); + first.request.deadline_ns = 100; + auto second = makeOwner(1, 16); + second.request.deadline_ns = 200; + auto third = makeOwner(2, 16); + third.request.deadline_ns = 300; + ASSERT_TRUE(queue + .tryAdmit(makeSubmit(1, 3, + {std::move(first), std::move(second), + std::move(third)}), + admitted_ids) + .ok()); + + auto picked = queue.pickForDispatch(1, 128); + ASSERT_EQ(picked, std::vector({1})); + ASSERT_TRUE(queue.defer(picked[0]).ok()); + + auto newcomer = makeOwner(0, 16); + newcomer.request.deadline_ns = 150; + ASSERT_TRUE( + queue.tryAdmit(makeSubmit(2, 1, {std::move(newcomer)}), admitted_ids) + .ok()); + ASSERT_EQ(admitted_ids, std::vector({4})); + + picked = queue.pickForDispatch(4, 128); + // The deferred owner starts the next EDF round. A newly admitted owner is + // ordered within that round instead of jumping ahead of owners 2 and 3, + // which were already waiting when owner 1 yielded. + const std::vector expected{2, 3, 1, 4}; + EXPECT_EQ(picked, expected); +} + +TEST(AdmissionQueueTest, DeferredEdfRoundsStayOrderedAcrossAdmissions) { + QueueLimits limits{6, 256, 0, 0}; + limits.deadline_aware = true; + LocalTransferAdmissionQueue queue(limits); + std::vector admitted_ids; + + auto first = makeOwner(0, 16); + first.request.deadline_ns = 100; + auto second = makeOwner(1, 16); + second.request.deadline_ns = 200; + auto third = makeOwner(2, 16); + third.request.deadline_ns = 300; + ASSERT_TRUE(queue + .tryAdmit(makeSubmit(1, 3, + {std::move(first), std::move(second), + std::move(third)}), + admitted_ids) + .ok()); + + auto picked = queue.pickForDispatch(1, 256); + ASSERT_EQ(picked, std::vector({1})); + ASSERT_TRUE(queue.defer(picked[0]).ok()); + + auto deadline_250 = makeOwner(0, 16); + deadline_250.request.deadline_ns = 250; + ASSERT_TRUE( + queue + .tryAdmit(makeSubmit(2, 1, {std::move(deadline_250)}), admitted_ids) + .ok()); + + picked = queue.pickForDispatch(1, 256); + ASSERT_EQ(picked, std::vector({2})); + ASSERT_TRUE(queue.defer(picked[0]).ok()); + + auto deadline_225 = makeOwner(0, 16); + deadline_225.request.deadline_ns = 225; + ASSERT_TRUE( + queue + .tryAdmit(makeSubmit(3, 1, {std::move(deadline_225)}), admitted_ids) + .ok()); + + picked = queue.pickForDispatch(5, 256); + const std::vector expected{3, 1, 2, 5, 4}; + EXPECT_EQ(picked, expected); +} + +TEST(AdmissionQueueTest, PromotionCannotUndoCreditDeferralRound) { + QueueLimits limits{2, 128, 0, 0}; + limits.deadline_aware = true; + limits.promotion_slack_ns = 10; + LocalTransferAdmissionQueue queue(limits); + queue.setDegradationPolicy([] { return 1e9; }, DegradationHooks{}, + [] { return uint64_t{100}; }); + + auto starved = makeOwner(0, 16); + starved.request.deadline_ns = 105; + auto waiter = makeOwner(1, 16); + waiter.request.deadline_ns = 1000; + std::vector admitted_ids; + ASSERT_TRUE( + queue + .tryAdmit(makeSubmit(1, 2, {std::move(starved), std::move(waiter)}), + admitted_ids) + .ok()); + + auto picked = queue.pickForDispatch(1, 128); + ASSERT_EQ(picked, std::vector({1})); + ASSERT_TRUE(queue.defer(picked[0]).ok()); + + // Owner 1 remains urgent, but it yielded into round 1. Promotion cannot + // move it ahead of owner 2, which is still waiting in round 0. + picked = queue.pickForDispatch(1, 128); + EXPECT_EQ(picked, std::vector({2})); +} + +TEST(AdmissionQueueTest, RejectsDeferForNonDispatchingOwner) { + LocalTransferAdmissionQueue queue({2, 128, 0, 0}); + std::vector admitted_ids; + ASSERT_TRUE( + queue.tryAdmit(makeSubmit(1, 1, {makeOwner(0, 16)}), admitted_ids) + .ok()); + ASSERT_EQ(admitted_ids.size(), 1u); + + EXPECT_TRUE(queue.defer(0).IsInvalidArgument()); + EXPECT_TRUE(queue.defer(99).IsInvalidEntry()); + EXPECT_TRUE(queue.defer(admitted_ids[0]).IsInvalidEntry()); + + auto picked = queue.pickForDispatch(1, 128); + ASSERT_EQ(picked.size(), 1u); + ASSERT_TRUE(queue.complete(picked[0], COMPLETED).ok()); + EXPECT_TRUE(queue.defer(picked[0]).IsInvalidEntry()); + EXPECT_EQ(queue.outstandingOwners(), 0u); + EXPECT_EQ(queue.outstandingBytes(), 0u); +} + TEST(AdmissionQueueTest, RequiresDispatchBeforeTerminalCompletion) { LocalTransferAdmissionQueue queue({2, 128, 0, 0}); std::vector admitted_ids; @@ -828,6 +1015,29 @@ TEST(AdmissionQueueTest, EXPECT_EQ(queue.dispatchingBytes(), 0u); } +TEST(AdmissionQueueTest, RejectsDeferForOwnerDroppedDuringDispatchPick) { + LocalTransferAdmissionQueue queue(step3Limits(1.5)); + queue.setDegradationPolicy([] { return 1e9; }, DegradationHooks{}, + [] { return uint64_t{1'000'000'000}; }); + + std::vector admitted_ids; + ASSERT_TRUE( + queue + .tryAdmit(makeSubmit(1, 1, + {makeDegradationEligibleOwnerWithDeadline( + 0, 16, 1'000'000'010)}), + admitted_ids) + .ok()); + + std::vector dropped; + EXPECT_TRUE(queue.pickForDispatch(1, 128, &dropped).empty()); + ASSERT_EQ(dropped, std::vector({1})); + EXPECT_TRUE(queue.defer(dropped[0]).IsInvalidEntry()); + EXPECT_TRUE(queue.pickForDispatch(1, 128).empty()); + EXPECT_EQ(queue.outstandingOwners(), 0u); + EXPECT_EQ(queue.outstandingBytes(), 0u); +} + TEST(AdmissionQueueTest, Step3DropsAlreadyExpiredDeadline) { LocalTransferAdmissionQueue queue(step3Limits(1.5)); queue.setDegradationPolicy([] { return 1e9; }, DegradationHooks{}, diff --git a/mooncake-transfer-engine/tent/tests/hp_tcp_transport_test.cpp b/mooncake-transfer-engine/tent/tests/hp_tcp_transport_test.cpp index 452b2e3f04..49fd732fd0 100644 --- a/mooncake-transfer-engine/tent/tests/hp_tcp_transport_test.cpp +++ b/mooncake-transfer-engine/tent/tests/hp_tcp_transport_test.cpp @@ -48,6 +48,19 @@ class HighPerformanceTcpTransportTestPeer { const HighPerformanceTcpTransport& transport) { return transport.server_->activeSessionsForTest(); } + + // Outstanding operations per lane towards `target` (empty if none planned). + static std::vector laneLoad(HighPerformanceTcpTransport& transport, + SegmentID target) { + std::lock_guard lock(transport.lane_sequence_mutex_); + std::vector out; + auto it = transport.lane_load_.find(target); + if (it == transport.lane_load_.end() || !it->second) return out; + for (const auto& lane : *it->second) { + out.push_back(lane.load(std::memory_order_relaxed)); + } + return out; + } }; namespace { @@ -251,6 +264,86 @@ Status WaitForTransportResult(HighPerformanceTcpTransport& transport, return Status::InternalError("HP TCP test transfer did not finish"); } +TEST(HighPerformanceTcpTransportTest, ReceiverCreditFenceTracksPinnedMetadata) { + auto metadata = MakeLocalMetadata(); + HighPerformanceTcpTransport transport(MakeParams()); + std::string name = "hp_transport_test"; + ASSERT_TRUE(transport.install(name, metadata, nullptr, nullptr).ok()); + std::array source{}; + std::array target{}; + source.fill(0x5a); + BufferDesc source_desc{}; + source_desc.addr = reinterpret_cast(source.data()); + source_desc.length = source.size(); + source_desc.location = "cpu:0"; + BufferDesc target_desc{}; + target_desc.addr = reinterpret_cast(target.data()); + target_desc.length = target.size(); + target_desc.location = "cpu:0"; + MemoryOptions options; + options.type = HP_TCP; + options.perm = kGlobalReadWrite; + ASSERT_TRUE(transport.addMemoryBuffer(source_desc, options).ok()); + ASSERT_TRUE(transport.addMemoryBuffer(target_desc, options).ok()); + ASSERT_TRUE(PublishBuffers(metadata, {source_desc, target_desc}).ok()); + + Request request{}; + request.opcode = Request::WRITE; + request.source = source.data(); + request.target_id = LOCAL_SEGMENT_ID; + request.target_offset = target_desc.addr; + request.length = source.size(); + ReceiverCreditAdvertV1 advert; + advert.receiver_session_id = {1, 2}; + advert.epoch = 7; + // No advert, wrong session, wrong epoch, and unsupported protocol must + // reject before worker handoff; a matching pinned advert must still work. + for (int scenario = 0; scenario < 6; ++scenario) { + SCOPED_TRACE(scenario); + auto current = advert; + if (scenario == 1) ++current.receiver_session_id.low; + if (scenario == 2) ++current.epoch; + if (scenario == 3) ++current.schema_version; + if (scenario == 4) current.flags = 0; + ASSERT_TRUE(metadata->segmentManager() + .updateLocal([&](SegmentDesc& segment) -> Status { + auto& published = + std::get(segment.detail) + .receiver_credit; + if (scenario == 0) + published.reset(); + else + published = current; + return Status::OK(); + }) + .ok()); + Transport::SubBatchRef batch = nullptr; + ASSERT_TRUE(transport.allocateSubBatch(batch, 1).ok()); + batch->receiver_credit_session_high = advert.receiver_session_id.high; + batch->receiver_credit_session_low = advert.receiver_session_id.low; + batch->receiver_credit_epoch = advert.epoch; + const Status submitted = + transport.submitTransferTasks(batch, {request}); + if (scenario < 5) { + EXPECT_TRUE(submitted.IsInvalidEntry()) << submitted.ToString(); + EXPECT_EQ(batch->size(), 0u); + EXPECT_TRUE(std::all_of(target.begin(), target.end(), + [](uint8_t byte) { return byte == 0; })); + } else { + ASSERT_TRUE(submitted.ok()) << submitted.ToString(); + TransferStatus status{}; + ASSERT_TRUE(WaitForTransportResult(transport, batch, status).ok()); + EXPECT_EQ(status.s, COMPLETED); + EXPECT_EQ(source, target); + } + EXPECT_TRUE(transport.freeSubBatch(batch).ok()); + } + EXPECT_TRUE(transport.quiesce().ok()); + EXPECT_TRUE(transport.removeMemoryBuffer(source_desc).ok()); + EXPECT_TRUE(transport.removeMemoryBuffer(target_desc).ok()); + EXPECT_TRUE(transport.uninstall().ok()); +} + // Test-only relay: observe the wire request and forward it to the real server. // Pumped by the test thread; no transport worker waits for another callback. class RailRelay { @@ -931,5 +1024,126 @@ INSTANTIATE_TEST_SUITE_P(WriteAndSlicedRead, HighPerformanceTcpLaneDistributionTest, ::testing::Bool()); +// Lane selection counts outstanding operations per lane. Commands that are +// planned but never run must give their count back, or the lane is avoided +// forever: an admission rejection (the normal backpressure path) and a batch +// whose later request fails to plan both drop already planned commands. +TEST(HighPerformanceTcpTransportTest, LaneLoadReturnsToZeroOnRejectedSubmits) { + constexpr size_t kLength = 768ULL << 10; + auto params = MakeParams(); + params.connections_per_peer = 4; + params.max_outstanding_bytes = 1ULL << 20; // two requests do not fit + params.max_transfer_bytes = 1ULL << 20; + + std::vector remote(2 * kLength, 0); + std::vector local(2 * kLength, 0x5a); + std::vector unregistered(kLength, 0x11); + + auto server_metadata = MakeLocalMetadata(); + uint16_t rpc_port = 0; + ASSERT_TRUE(server_metadata->start(rpc_port).ok()); + const std::string server_name = "127.0.0.1:" + std::to_string(rpc_port); + ASSERT_TRUE(server_metadata->segmentManager() + .updateLocal([&](SegmentDesc& segment) -> Status { + segment.name = server_name; + segment.rpc_server_addr = server_name; + return Status::OK(); + }) + .ok()); + HighPerformanceTcpTransport server(params); + std::string installed_server_name = server_name; + ASSERT_TRUE( + server.install(installed_server_name, server_metadata, nullptr, nullptr) + .ok()); + BufferDesc remote_buffer; + remote_buffer.addr = reinterpret_cast(remote.data()); + remote_buffer.length = remote.size(); + remote_buffer.location = "cpu:0"; + MemoryOptions remote_options; + remote_options.perm = kGlobalReadWrite; + ASSERT_TRUE(server.addMemoryBuffer(remote_buffer, remote_options).ok()); + ASSERT_TRUE(PublishBuffers(server_metadata, {remote_buffer}).ok()); + + auto client_metadata = MakeLocalMetadata(); + HighPerformanceTcpTransport client(params); + std::string client_name = "hp_lane_load_client"; + ASSERT_TRUE( + client.install(client_name, client_metadata, nullptr, nullptr).ok()); + SegmentID target = 0; + ASSERT_TRUE( + client_metadata->segmentManager().openRemote(target, server_name).ok()); + SegmentDescRef remote_desc; + Status resolved = + client_metadata->segmentManager().getRemoteCached(remote_desc, target); + if (resolved.IsRpcServiceError()) { + resolved = client_metadata->segmentManager().getRemoteCached( + remote_desc, target); + } + ASSERT_TRUE(resolved.ok()) << resolved.ToString(); + BufferDesc local_buffer; + local_buffer.addr = reinterpret_cast(local.data()); + local_buffer.length = local.size(); + local_buffer.location = "cpu:0"; + MemoryOptions local_options; + local_options.perm = kLocalReadWrite; + ASSERT_TRUE(client.addMemoryBuffer(local_buffer, local_options).ok()); + + const auto write = [&](void* source, size_t offset) { + Request request{}; + request.opcode = Request::WRITE; + request.source = source; + request.target_id = target; + request.target_offset = + reinterpret_cast(remote.data()) + offset; + request.length = kLength; + return request; + }; + const auto all_zero = [&] { + for (int32_t load : + HighPerformanceTcpTransportTestPeer::laneLoad(client, target)) { + if (load != 0) return false; + } + return true; + }; + + Transport::SubBatchRef batch = nullptr; + ASSERT_TRUE(client.allocateSubBatch(batch, 2).ok()); + + // Admission rejection: both requests plan (taking two lane counts), then + // the batch's 1.5 MB do not fit the 1 MB budget. + Status rejected = client.submitTransferTasks( + batch, + {write(local.data(), 0), write(local.data() + kLength, kLength)}); + ASSERT_TRUE(rejected.IsTooManyRequests()) << rejected.ToString(); + EXPECT_FALSE( + HighPerformanceTcpTransportTestPeer::laneLoad(client, target).empty()); + EXPECT_TRUE(WaitUntil(all_zero)); + + // Mid-batch planning failure: the first request plans, the second has an + // unregistered source and fails to plan. + Status failed = client.submitTransferTasks( + batch, {write(local.data(), 0), write(unregistered.data(), kLength)}); + ASSERT_FALSE(failed.ok()); + EXPECT_TRUE(WaitUntil(all_zero)); + + // The transport still works, and completed operations give their counts + // back as well. + ASSERT_TRUE( + client.submitTransferTasks(batch, {write(local.data(), 0)}).ok()); + TransferStatus status{}; + ASSERT_TRUE(WaitForTransportResult(client, batch, status).ok()); + ASSERT_EQ(status.s, COMPLETED); + EXPECT_EQ(std::memcmp(remote.data(), local.data(), kLength), 0); + EXPECT_TRUE(WaitUntil(all_zero)); + + ASSERT_TRUE(client.freeSubBatch(batch).ok()); + ASSERT_TRUE(client.removeMemoryBuffer(local_buffer).ok()); + ASSERT_TRUE(server.removeMemoryBuffer(remote_buffer).ok()); + ASSERT_TRUE(client.quiesce().ok()); + ASSERT_TRUE(server.quiesce().ok()); + ASSERT_TRUE(client.uninstall().ok()); + ASSERT_TRUE(server.uninstall().ok()); +} + } // namespace } // namespace mooncake::tent diff --git a/mooncake-transfer-engine/tent/tests/io_uring_transport_test.cpp b/mooncake-transfer-engine/tent/tests/io_uring_transport_test.cpp index f45899022b..1928491c28 100644 --- a/mooncake-transfer-engine/tent/tests/io_uring_transport_test.cpp +++ b/mooncake-transfer-engine/tent/tests/io_uring_transport_test.cpp @@ -15,15 +15,20 @@ #include #include +#include #include +#include +#include #include +#include #include #include #include #include #include #include +#include #include #include @@ -816,6 +821,129 @@ TEST(IOUringPinnedStagingTest, RejectedSubmissionAndFreeSubBatchReturnSlots) { EXPECT_EQ(pool->freeSlots(), 1u); } +// Freeing a sub-batch whose read is still in flight waits for the read: +// the kernel would otherwise complete it into the freed staging buffer. A +// FIFO keeps the read in flight until the test writes to it. +TEST(IOUringTransportTest, FreeSubBatchWaitsForInFlightIo) { + char dir_template[] = "/tmp/io_uring_fifo_XXXXXX"; + ASSERT_NE(mkdtemp(dir_template), nullptr); + const std::string path = std::string(dir_template) + "/fifo"; + ASSERT_EQ(mkfifo(path.c_str(), 0600), 0); + int writer = open(path.c_str(), O_RDWR | O_NONBLOCK); + ASSERT_GE(writer, 0); + auto metadata = std::make_shared("p2p", "", nullptr); + ASSERT_TRUE(installLocalFileSegment(*metadata, path, kPageSize).ok()); + IOUringTransport transport; + auto installed = installTransport(transport, metadata); + if (!installed.ok()) { + close(writer); + unlink(path.c_str()); + rmdir(dir_template); + GTEST_SKIP() << installed.message(); + } + + void* allocation = nullptr; + ASSERT_EQ(posix_memalign(&allocation, kPageSize, 2 * kPageSize), 0); + std::unique_ptr memory(allocation, free); + Request read{}; + read.opcode = Request::READ; + read.source = static_cast(allocation) + 1; // staged + read.target_id = LOCAL_SEGMENT_ID; + read.length = kPageSize; + Transport::SubBatchRef batch = nullptr; + ASSERT_TRUE(transport.allocateSubBatch(batch, 1).ok()); + ASSERT_TRUE(transport.submitTransferTasks(batch, {read}).ok()); + + // Free on a worker: it must stay blocked while the FIFO is unfed, and + // finish once the read is fed. Feed and join on every path, so that a + // failed check cannot leave the worker blocked. + std::atomic done{false}; + Status freed; + std::thread worker([&] { + freed = transport.freeSubBatch(batch); + done = true; + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_FALSE(done.load()) << "freeSubBatch() returned with I/O in flight"; + std::vector data(kPageSize, 0x5A); + EXPECT_EQ(write(writer, data.data(), data.size()), (ssize_t)kPageSize); + worker.join(); + EXPECT_TRUE(freed.ok()) << freed.ToString(); + close(writer); + unlink(path.c_str()); + rmdir(dir_template); +} + +// If that wait fails, freeSubBatch() keeps reaping by polling the CQ ring, +// so it still frees nothing the kernel may write into. Replacing the ring fd +// with a non-io_uring file makes io_uring_wait_cqe() fail, while the ring +// itself stays alive through its mappings. +TEST(IOUringPinnedStagingTest, FreeSubBatchDrainsWhenWaitFails) { + char dir_template[] = "/tmp/io_uring_fifo_XXXXXX"; + ASSERT_NE(mkdtemp(dir_template), nullptr); + const std::string path = std::string(dir_template) + "/fifo"; + ASSERT_EQ(mkfifo(path.c_str(), 0600), 0); + int writer = open(path.c_str(), O_RDWR | O_NONBLOCK); + ASSERT_GE(writer, 0); + auto metadata = std::make_shared("p2p", "", nullptr); + ASSERT_TRUE(installLocalFileSegment(*metadata, path, kPageSize).ok()); + PinnedStagingIOUringTransport transport; + auto installed = installPinnedStaging(transport, metadata, 4, 1); + if (!installed.ok()) { + close(writer); + unlink(path.c_str()); + rmdir(dir_template); + GTEST_SKIP() << installed.message(); + } + auto* pool = transport.stagingPool(); + ASSERT_NE(pool, nullptr); + + void* allocation = nullptr; + ASSERT_EQ(posix_memalign(&allocation, kPageSize, 2 * kPageSize), 0); + std::unique_ptr memory(allocation, free); + Request read{}; + read.opcode = Request::READ; + read.source = static_cast(allocation) + 1; // staged + read.target_id = LOCAL_SEGMENT_ID; + read.length = kPageSize; + Transport::SubBatchRef batch = nullptr; + ASSERT_TRUE(transport.allocateSubBatch(batch, 1).ok()); + ASSERT_TRUE(transport.submitTransferTasks(batch, {read}).ok()); + EXPECT_EQ(pool->freeSlots(), 0u); + + auto* sub_batch = static_cast(batch); + const auto* slot = + static_cast(sub_batch->task_list[0].buffer); + const int ring_fd = sub_batch->ring.ring_fd; + int null_fd = open("/dev/null", O_RDONLY); + ASSERT_GE(null_fd, 0); + ASSERT_EQ(dup2(null_fd, ring_fd), ring_fd); + close(null_fd); + + // As above: freeSubBatch() must stay blocked, with the slot out of the + // pool, until the read is fed. + std::atomic done{false}; + Status freed; + std::thread worker([&] { + freed = transport.freeSubBatch(batch); + done = true; + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_FALSE(done.load()) << "freeSubBatch() returned with I/O in flight"; + EXPECT_EQ(pool->freeSlots(), 0u); + std::vector data(kPageSize, 0x5A); + EXPECT_EQ(write(writer, data.data(), data.size()), (ssize_t)kPageSize); + worker.join(); + EXPECT_TRUE(freed.ok()) << freed.ToString(); + EXPECT_EQ(batch, nullptr); + // The read landed in the slot before the slot went back to the pool. + EXPECT_EQ(pool->freeSlots(), 1u); + EXPECT_TRUE(std::equal(data.begin(), data.end(), slot)); + close(writer); + unlink(path.c_str()); + rmdir(dir_template); +} + } // namespace } // namespace tent } // namespace mooncake diff --git a/mooncake-transfer-engine/tent/tests/receiver_credit_test.cpp b/mooncake-transfer-engine/tent/tests/receiver_credit_test.cpp index 33faa04a2c..86b74670a1 100644 --- a/mooncake-transfer-engine/tent/tests/receiver_credit_test.cpp +++ b/mooncake-transfer-engine/tent/tests/receiver_credit_test.cpp @@ -10,7 +10,7 @@ namespace mooncake::tent { namespace { -CreditKey key() { return {{1, 2}, 3, 4}; } +CreditKey key() { return {{1, 2}, {3, 4}, 1}; } ReceiverCreditUpdateV1 update(uint64_t epoch, uint64_t seq, std::vector grants) { ReceiverCreditUpdateV1 u; @@ -18,6 +18,7 @@ ReceiverCreditUpdateV1 update(uint64_t epoch, uint64_t seq, u.qos_class = key().qos_class; u.epoch = epoch; u.sequence = seq; + u.freshness_ttl_ms = 1000; u.grants = std::move(grants); return u; } @@ -126,7 +127,7 @@ TEST(ReceiverCredit, LedgerEntryCountIsBounded) { SenderCreditLedger l(1); ASSERT_TRUE(l.activate(key(), 7).ok()); auto other = key(); - ++other.sender_peer; + ++other.sender_instance.low; EXPECT_TRUE(l.activate(other, 7).IsTooManyRequests()); // A new epoch for an existing key does not consume another entry. EXPECT_TRUE(l.activate(key(), 8).ok()); @@ -138,7 +139,7 @@ TEST(ReceiverCredit, DeactivationReleasesCapacityAfterExactEpochFence) { grant(l, 1); ASSERT_TRUE(l.tryReserve(key(), charge(80, 1)).ok()); auto other = key(); - ++other.sender_peer; + ++other.sender_instance.low; EXPECT_TRUE(l.activate(other, 1).IsTooManyRequests()); EXPECT_TRUE(l.deactivate(key(), 6).IsInvalidEntry()); @@ -201,6 +202,27 @@ TEST(ReceiverCredit, RollbackChecksUnderflowAtomically) { ASSERT_TRUE(l.consumed(key(), CreditResource::RequestSlots, v).ok()); EXPECT_EQ(v, 1); ASSERT_TRUE(l.rollbackReservation(key(), charge(60, 1)).ok()); + ASSERT_TRUE(l.consumed(key(), CreditResource::DataBytes, v).ok()); + EXPECT_EQ(v, 60); // cumulative consumption never moves backwards + ASSERT_TRUE(l.released(key(), CreditResource::DataBytes, v).ok()); + EXPECT_EQ(v, 60); + ASSERT_TRUE(l.available(key(), CreditResource::DataBytes, v).ok()); + EXPECT_EQ(v, 40); // returned authorization is not locally re-minted +} + +TEST(ReceiverCredit, DeactivateSessionRemovesAllQosEntries) { + SenderCreditLedger ledger(2); + auto high = key(); + auto low = key(); + low.qos_class = 2; + ASSERT_TRUE(ledger.activate(high, 7).ok()); + ASSERT_TRUE(ledger.activate(low, 7).ok()); + ASSERT_TRUE(ledger.deactivateSession(key().receiver_session).ok()); + + auto replacement = key(); + replacement.receiver_session = {9, 10}; + replacement.sender_instance = {11, 12}; + ASSERT_TRUE(ledger.activate(replacement, 1).ok()); } TEST(ReceiverCredit, GrantCannotDecreaseOrFallBelowConsumption) { @@ -232,5 +254,335 @@ TEST(ReceiverCredit, ConcurrentReservationsNeverExceedGrant) { ASSERT_TRUE(l.consumed(key(), CreditResource::DataBytes, consumed).ok()); EXPECT_EQ(consumed, 100); } + +TEST(ReceiverCredit, FreshnessExpiryFailsClosedAndReplayDoesNotRenew) { + uint64_t now_ns = 100; + SenderCreditLedger ledger(8, 1000, [&] { return now_ns; }); + ASSERT_TRUE(ledger.activate(key(), 7).ok()); + auto first = update( + 7, 1, + {{CreditResource::DataBytes, 100}, {CreditResource::RequestSlots, 2}}); + first.freshness_ttl_ms = 1; + CreditUpdateDisposition disposition; + ASSERT_TRUE(ledger.applyUpdate(key(), first, disposition).ok()); + now_ns += 999999; + EXPECT_TRUE(ledger.tryReserve(key(), charge(1, 1)).ok()); + + // A replay is accepted idempotently, but must not extend the lease. + ASSERT_TRUE(ledger.applyUpdate(key(), first, disposition).ok()); + EXPECT_EQ(disposition, CreditUpdateDisposition::DuplicateOrOld); + ++now_ns; + EXPECT_TRUE(ledger.tryReserve(key(), charge(1, 1)).IsInvalidEntry()); +} + +TEST(ReceiverCredit, TerminalReleaseRequiresReceiverReplenishment) { + ReceiverCreditLimits limits; + limits.capacity[0] = 100; + limits.capacity[1] = 2; + limits.freshness_ttl_ms = 1000; + ReceiverCreditAuthority authority({9, 10}, 3, limits); + ASSERT_TRUE(authority.status().ok()); + + CreditKey sender{{9, 10}, {11, 12}, 1}; + SenderCreditLedger ledger; + ASSERT_TRUE(ledger.activate(sender, 3).ok()); + + ReceiverCreditExchangeRequestV1 request; + ASSERT_TRUE(ledger.prepareExchange(sender, charge(60, 1), request).ok()); + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(request, reply).ok()); + CreditUpdateDisposition disposition; + ASSERT_TRUE(ledger.applyUpdate(sender, reply.update, disposition).ok()); + ASSERT_TRUE(ledger.tryReserve(sender, charge(60, 1)).ok()); + ASSERT_TRUE(ledger.releaseCommitted(sender, charge(60, 1)).ok()); + + uint64_t available = 0; + ASSERT_TRUE( + ledger.available(sender, CreditResource::DataBytes, available).ok()); + EXPECT_EQ(available, 0); // local release never mints credit + + ASSERT_TRUE(ledger.prepareExchange(sender, charge(60, 1), request).ok()); + ASSERT_TRUE(authority.exchange(request, reply).ok()); + ASSERT_TRUE(ledger.applyUpdate(sender, reply.update, disposition).ok()); + EXPECT_TRUE(ledger.tryReserve(sender, charge(60, 1)).ok()); + + uint64_t outstanding = 0; + ASSERT_TRUE( + authority.outstanding(CreditResource::DataBytes, outstanding).ok()); + EXPECT_EQ(outstanding, 60); +} + +TEST(ReceiverCredit, AuthorityGrantsMultiResourceDemandAtomically) { + ReceiverCreditLimits limits; + limits.capacity[0] = 100; + limits.capacity[1] = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + + auto make_request = [](SenderInstanceId sender, uint64_t bytes) { + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = sender; + request.qos_class = 0; + request.report_sequence = 1; + request.released_totals = { + {CreditResource::DataBytes, 0}, + {CreditResource::RequestSlots, 0}, + }; + request.desired_grant_totals = { + {CreditResource::DataBytes, bytes}, + {CreditResource::RequestSlots, 1}, + }; + return request; + }; + + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(make_request({3, 4}, 60), reply).ok()); + ASSERT_EQ(reply.update.grants.size(), kCreditResourceCount); + EXPECT_EQ(reply.update.grants[0].grant_total, 60); + EXPECT_EQ(reply.update.grants[1].grant_total, 1); + + ASSERT_TRUE(authority.exchange(make_request({5, 6}, 40), reply).ok()); + // The second sender cannot get a request slot, so bytes are not stranded + // in a partial grant either. + EXPECT_EQ(reply.update.grants[0].grant_total, 0); + EXPECT_EQ(reply.update.grants[1].grant_total, 0); + uint64_t outstanding = 0; + ASSERT_TRUE( + authority.outstanding(CreditResource::DataBytes, outstanding).ok()); + EXPECT_EQ(outstanding, 60); +} + +TEST(ReceiverCredit, RollbackReleaseLetsAnotherSenderAcquireCapacity) { + ReceiverCreditLimits limits; + limits.capacity[0] = 60; + limits.capacity[1] = 1; + limits.max_senders = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + SenderCreditLedger first; + CreditKey first_key{{1, 2}, {3, 4}, 0}; + ASSERT_TRUE(first.activate(first_key, 1).ok()); + + ReceiverCreditExchangeRequestV1 request; + ASSERT_TRUE(first.prepareExchange(first_key, charge(60, 1), request).ok()); + const auto initial_demand = request; + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(request, reply).ok()); + CreditUpdateDisposition disposition; + ASSERT_TRUE(first.applyUpdate(first_key, reply.update, disposition).ok()); + ASSERT_TRUE(first.tryReserve(first_key, charge(60, 1)).ok()); + ASSERT_TRUE(first.rollbackReservation(first_key, charge(60, 1)).ok()); + ASSERT_TRUE(first.prepareReleaseExchange(first_key, request).ok()); + const auto release_report = request; + ASSERT_TRUE(authority.exchange(request, reply).ok()); + EXPECT_EQ(reply.flags, 0); + + uint64_t outstanding = 0; + ASSERT_TRUE( + authority.outstanding(CreditResource::DataBytes, outstanding).ok()); + EXPECT_EQ(outstanding, 0); + + // Both a delayed initial demand and a replayed release receive retained + // cumulative state; neither can re-mint released capacity. + ASSERT_TRUE(authority.exchange(initial_demand, reply).ok()); + EXPECT_EQ(reply.flags, 0); + EXPECT_EQ(reply.update.grants[0].grant_total, 60); + ASSERT_TRUE(authority.exchange(release_report, reply).ok()); + ASSERT_TRUE( + authority.outstanding(CreditResource::DataBytes, outstanding).ok()); + EXPECT_EQ(outstanding, 0); + + SenderCreditLedger second; + CreditKey second_key{{1, 2}, {5, 6}, 0}; + ASSERT_TRUE(second.activate(second_key, 1).ok()); + ASSERT_TRUE( + second.prepareExchange(second_key, charge(60, 1), request).ok()); + ASSERT_TRUE(authority.exchange(request, reply).ok()); + ASSERT_TRUE(second.applyUpdate(second_key, reply.update, disposition).ok()); + EXPECT_TRUE(second.tryReserve(second_key, charge(60, 1)).ok()); +} + +TEST(ReceiverCredit, SenderLimitCountsSenderNotQosEntries) { + ReceiverCreditLimits limits; + limits.capacity[0] = 100; + limits.capacity[1] = 2; + limits.max_senders = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + + auto make_request = [](SenderInstanceId sender, uint32_t qos) { + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = sender; + request.qos_class = qos; + request.report_sequence = 1; + request.released_totals = {{CreditResource::DataBytes, 0}, + {CreditResource::RequestSlots, 0}}; + request.desired_grant_totals = {{CreditResource::DataBytes, 10}, + {CreditResource::RequestSlots, 1}}; + return request; + }; + + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(make_request({3, 4}, 0), reply).ok()); + ASSERT_TRUE(authority.exchange(make_request({3, 4}, 2), reply).ok()); + ASSERT_TRUE(authority.exchange(make_request({5, 6}, 0), reply).ok()); + EXPECT_EQ(reply.update.grants[0].grant_total, 0); + EXPECT_EQ(reply.update.grants[1].grant_total, 0); +} + +TEST(ReceiverCredit, ZeroGrantHistoryDoesNotConsumeActiveSenderLimit) { + ReceiverCreditLimits limits; + limits.capacity[0] = 10; + limits.capacity[1] = 1; + limits.max_senders = 1; + limits.max_entries = 4; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + + auto make_request = [](SenderInstanceId sender) { + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = sender; + request.qos_class = 0; + request.report_sequence = 1; + request.released_totals = {{CreditResource::DataBytes, 0}, + {CreditResource::RequestSlots, 0}}; + request.desired_grant_totals = {{CreditResource::DataBytes, 10}, + {CreditResource::RequestSlots, 1}}; + return request; + }; + + auto first = make_request({3, 4}); + ReceiverCreditExchangeReplyV1 first_reply; + ASSERT_TRUE(authority.exchange(first, first_reply).ok()); + EXPECT_EQ(first_reply.update.grants[0].grant_total, 10); + + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(make_request({5, 6}), reply).ok()); + EXPECT_EQ(reply.update.grants[0].grant_total, 0); + + first.known_update_sequence = first_reply.update.sequence; + first.report_sequence = 2; + first.released_totals = {{CreditResource::DataBytes, 10}, + {CreditResource::RequestSlots, 1}}; + ASSERT_TRUE(authority.exchange(first, reply).ok()); + + ASSERT_TRUE(authority.exchange(make_request({7, 8}), reply).ok()); + EXPECT_EQ(reply.update.grants[0].grant_total, 10); + EXPECT_EQ(reply.update.grants[1].grant_total, 1); +} + +TEST(ReceiverCredit, ReplayHistoryLimitFailsClosed) { + ReceiverCreditLimits limits; + limits.capacity[0] = 10; + limits.capacity[1] = 1; + limits.max_senders = 1; + limits.max_entries = 2; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + + auto make_request = [](SenderInstanceId sender) { + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = sender; + request.report_sequence = 1; + request.released_totals = {{CreditResource::DataBytes, 0}, + {CreditResource::RequestSlots, 0}}; + request.desired_grant_totals = {{CreditResource::DataBytes, 10}, + {CreditResource::RequestSlots, 1}}; + return request; + }; + + ReceiverCreditExchangeReplyV1 reply; + ASSERT_TRUE(authority.exchange(make_request({3, 4}), reply).ok()); + ASSERT_TRUE(authority.exchange(make_request({5, 6}), reply).ok()); + EXPECT_TRUE( + authority.exchange(make_request({7, 8}), reply).IsTooManyRequests()); + + // Existing identities remain replayable after the cap is reached. + EXPECT_TRUE(authority.exchange(make_request({3, 4}), reply).ok()); +} + +TEST(ReceiverCredit, ReplayHistoryLimitMustCoverActiveSenders) { + ReceiverCreditLimits limits; + limits.capacity[0] = 10; + limits.capacity[1] = 1; + limits.max_senders = 2; + limits.max_entries = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + EXPECT_TRUE(authority.status().IsInvalidArgument()); +} + +TEST(ReceiverCredit, AdvertPublishesHardCapacityBounds) { + ReceiverCreditLimits limits; + limits.capacity[0] = 4096; + limits.capacity[1] = 8; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + const auto advert = authority.advert(); + ASSERT_EQ(advert.capacities.size(), 2u); + EXPECT_EQ(advert.capacities[0].resource, CreditResource::DataBytes); + EXPECT_EQ(advert.capacities[0].total, 4096u); + EXPECT_EQ(advert.capacities[1].resource, CreditResource::RequestSlots); + EXPECT_EQ(advert.capacities[1].total, 8u); + + const auto round_trip = + nlohmann::json(advert).get(); + EXPECT_EQ(round_trip.capacities.size(), advert.capacities.size()); + EXPECT_EQ(round_trip.capacities[0].total, 4096u); +} + +TEST(ReceiverCredit, ExchangeReplayIsIdempotent) { + ReceiverCreditLimits limits; + limits.capacity[0] = 100; + limits.capacity[1] = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = {3, 4}; + request.qos_class = 0; + request.report_sequence = 1; + request.released_totals = {{CreditResource::DataBytes, 0}, + {CreditResource::RequestSlots, 0}}; + request.desired_grant_totals = {{CreditResource::DataBytes, 50}, + {CreditResource::RequestSlots, 1}}; + + ReceiverCreditExchangeReplyV1 first, replay; + ASSERT_TRUE(authority.exchange(request, first).ok()); + ASSERT_TRUE(authority.exchange(request, replay).ok()); + EXPECT_EQ(replay.update.sequence, first.update.sequence); + EXPECT_EQ(replay.update.grants[0].grant_total, 50); + uint64_t outstanding = 0; + ASSERT_TRUE( + authority.outstanding(CreditResource::DataBytes, outstanding).ok()); + EXPECT_EQ(outstanding, 50); +} + +TEST(ReceiverCredit, InvalidNewSenderCannotConsumeAuthorityEntry) { + ReceiverCreditLimits limits; + limits.capacity[0] = 100; + limits.capacity[1] = 1; + limits.max_senders = 1; + ReceiverCreditAuthority authority({1, 2}, 1, limits); + + ReceiverCreditExchangeRequestV1 request; + request.expected_receiver_session_id = {1, 2}; + request.epoch = 1; + request.sender_instance_id = {3, 4}; + request.qos_class = 0; + request.known_update_sequence = 99; + request.report_sequence = 1; + request.released_totals = {{CreditResource::DataBytes, 0}}; + request.desired_grant_totals = {{CreditResource::DataBytes, 50}, + {CreditResource::RequestSlots, 1}}; + ReceiverCreditExchangeReplyV1 reply; + EXPECT_TRUE(authority.exchange(request, reply).IsInvalidArgument()); + + request.sender_instance_id = {5, 6}; + request.known_update_sequence = 0; + EXPECT_TRUE(authority.exchange(request, reply).ok()); + EXPECT_EQ(reply.update.grants[0].grant_total, 50); +} } // namespace } // namespace mooncake::tent diff --git a/mooncake-transfer-engine/tent/tests/request_merge_test.cpp b/mooncake-transfer-engine/tent/tests/request_merge_test.cpp index 116616e30e..638dd2ff7d 100644 --- a/mooncake-transfer-engine/tent/tests/request_merge_test.cpp +++ b/mooncake-transfer-engine/tent/tests/request_merge_test.cpp @@ -24,6 +24,7 @@ struct RequestBoundaryInfo { std::optional source_key; std::optional target_key; uint64_t max_merge_bytes{std::numeric_limits::max()}; + std::optional receiver_credit_data_capacity; }; struct MergeResult { diff --git a/mooncake-transfer-engine/tent/tests/runtime_queue_dispatch_test.cpp b/mooncake-transfer-engine/tent/tests/runtime_queue_dispatch_test.cpp index 41149ed4e1..07b3092204 100644 --- a/mooncake-transfer-engine/tent/tests/runtime_queue_dispatch_test.cpp +++ b/mooncake-transfer-engine/tent/tests/runtime_queue_dispatch_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "tent/common/types.h" #include "tent/runtime/segment.h" #include "tent/runtime/control_plane.h" +#include "tent/runtime/proxy_manager.h" #include "tent/runtime/transfer_engine_impl.h" #include "tent/runtime/transport.h" @@ -37,6 +39,12 @@ namespace mooncake { // left the normal lifecycle still holds. class TransferEngineImplTestPeer { public: + static tent::Status submitRequiringPostSubmitCancellation( + tent::TransferEngineImpl& engine, tent::BatchID batch_id, + const std::vector& requests) { + return engine.submitTransferRequiringPostSubmitCancellation(batch_id, + requests); + } static size_t queuedOwners(tent::TransferEngineImpl& engine) { return engine.queued_owners_.size(); } @@ -63,6 +71,16 @@ class TransferEngineImplTestPeer { [bytes_per_second]() { return bytes_per_second; }, tent::DegradationHooks{}, nullptr); } + // Small chunks so a short request splits into several staged chunks. + static void useSmallStagePool(tent::TransferEngineImpl& engine, + size_t chunk_size, size_t chunk_count) { + engine.staging_proxy_ = std::make_unique( + &engine, chunk_size, chunk_count); + } + static uint64_t creditDemandExchanges(tent::TransferEngineImpl& engine) { + std::lock_guard lk(engine.progress_mutex_); + return engine.receiver_credit_demand_exchanges_; + } static void adoptDeferredBatches(tent::TransferEngineImpl& engine, std::vector batches) { tent::TransferEngineImpl::DeferredStageTeardown deferred; @@ -101,6 +119,10 @@ class FakeTransport : public Transport { std::atomic submit_calls{0}; std::atomic status_calls{0}; std::atomic cancel_calls{0}; + std::atomic fail_next_submits{0}; + std::atomic last_credit_session_high{0}; + std::atomic last_credit_session_low{0}; + std::atomic last_credit_epoch{0}; bool cancellation_supported{true}; Status install(std::string&, std::shared_ptr, @@ -123,6 +145,21 @@ class FakeTransport : public Transport { Status submitTransferTasks(SubBatchRef batch, const std::vector& requests) override { ++submit_calls; + int failures = fail_next_submits.load(std::memory_order_relaxed); + while (failures > 0 && + !fail_next_submits.compare_exchange_weak( + failures, failures - 1, std::memory_order_relaxed)) { + } + if (failures > 0) { + return Status::InternalError( + "injected zero-handoff submit failure" LOC_MARK); + } + last_credit_session_high.store(batch->receiver_credit_session_high, + std::memory_order_relaxed); + last_credit_session_low.store(batch->receiver_credit_session_low, + std::memory_order_relaxed); + last_credit_epoch.store(batch->receiver_credit_epoch, + std::memory_order_relaxed); auto* fake = static_cast(batch); for (const auto& request : requests) { fake->requests.push_back(request); @@ -252,6 +289,13 @@ void installFakeRdma(TransferEngineImpl& engine, engine.swapTransportForTest(RDMA, fake_rdma); } +void installFakeUb(TransferEngineImpl& engine, + const std::shared_ptr& fake_ub) { + std::string seg_name = engine.getSegmentName(); + ASSERT_TRUE(fake_ub->install(seg_name, nullptr, nullptr).ok()); + engine.swapTransportForTest(UB, fake_ub); +} + Request makeLocalWrite(uint8_t* ptr, size_t length) { Request request; request.opcode = Request::WRITE; @@ -263,6 +307,26 @@ Request makeLocalWrite(uint8_t* ptr, size_t length) { return request; } +Request makeLocalUbWrite(uint8_t* ptr, size_t length) { + auto request = makeLocalWrite(ptr, length); + request.transport_hint = UB; + return request; +} + +Request makeRemoteUbWrite(uint8_t* source, SegmentID target_id, uint8_t* target, + size_t length) { + auto request = makeLocalUbWrite(source, length); + request.target_id = target_id; + request.target_offset = reinterpret_cast(target); + return request; +} + +Request makeLocalNetworkWrite(uint8_t* ptr, size_t length) { + auto request = makeLocalWrite(ptr, length); + request.transport_hint = UNSPEC; + return request; +} + TEST(RuntimeQueueDispatch, RejectsOverfullBatchBeforePublishingTasks) { auto cfg = makeRuntimeQueueConfig(1, 1UL << 20); TransferEngineImpl engine(cfg); @@ -339,14 +403,13 @@ TEST(RuntimeQueueDispatch, DispatchesOnlyOneWindowOnSubmit) { TransferEngineImpl engine(cfg); ASSERT_TRUE(engine.available()); - std::atomic complete_first{false}; + std::atomic complete{false}; auto fake_rdma = std::make_shared( - RDMA, [&complete_first](const Request& request, int) { - if (!complete_first.load()) { - return TransferStatus{TransferStatusEnum::PENDING, 0}; - } - return TransferStatus{TransferStatusEnum::COMPLETED, - request.length}; + RDMA, [&complete](const Request& request, int) { + return complete.load(std::memory_order_acquire) + ? TransferStatus{TransferStatusEnum::COMPLETED, + request.length} + : TransferStatus{TransferStatusEnum::PENDING, 0}; }); installFakeRdma(engine, fake_rdma); @@ -365,8 +428,7 @@ TEST(RuntimeQueueDispatch, DispatchesOnlyOneWindowOnSubmit) { .ok()); EXPECT_EQ(fake_rdma->submit_calls.load(), 1); - complete_first.store(true); - + complete.store(true, std::memory_order_release); TransferStatus first{}; ASSERT_TRUE(engine.getTransferStatus(batch, 0, first).ok()); EXPECT_EQ(first.s, TransferStatusEnum::COMPLETED); @@ -1104,6 +1166,699 @@ TEST(RuntimeQueueDispatch, TcpCapsPreservePublicMappingAndOwnerBytes) { } } +TEST(RuntimeQueueDispatch, ScatterCannotBypassRequiredReceiverCredit) { + constexpr size_t kReqLen = 4096; + auto cfg = makeRuntimeQueueConfig(2, 1UL << 20); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + auto fake_ub = std::make_shared(UB); + installFakeUb(engine, fake_ub); + std::vector buffer(kReqLen, 0x71); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(1); + ASSERT_NE(batch, (BatchID)0); + auto status = + TransferEngineImplTestPeer::submitRequiringPostSubmitCancellation( + engine, batch, {makeLocalUbWrite(buffer.data(), kReqLen)}); + EXPECT_TRUE(status.IsNotImplemented()); + EXPECT_EQ(fake_ub->submit_calls.load(), 0); + EXPECT_EQ(TransferEngineImplTestPeer::queuedOwners(engine), 0u); + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, ReceiverCreditDefersUbUntilTerminalRelease) { + constexpr size_t kReqLen = 4096; + auto cfg = makeRuntimeQueueConfig(2, 1UL << 20, true); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + cfg->set("receiver_credit/freshness_ttl_ms", uint32_t{1000}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + + std::atomic complete_first{false}; + auto fake_ub = std::make_shared( + UB, [&complete_first](const Request& request, int) { + if (!complete_first.load(std::memory_order_acquire)) { + return TransferStatus{TransferStatusEnum::PENDING, 0}; + } + return TransferStatus{TransferStatusEnum::COMPLETED, + request.length}; + }); + installFakeUb(engine, fake_ub); + + std::vector buffer(kReqLen * 2, 0x71); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(2); + ASSERT_NE(batch, (BatchID)0); + + ASSERT_TRUE( + engine + .submitTransfer( + batch, {makeLocalUbWrite(buffer.data(), kReqLen), + makeLocalUbWrite(buffer.data() + kReqLen, kReqLen)}) + .ok()); + // Both owners fit the local dispatch window, but receiver credit permits + // only one physical UB request. Merging must stop at the advertised hard + // byte capacity, so the second owner remains queued instead of making an + // inadmissible 8 KiB owner. + EXPECT_EQ(fake_ub->submit_calls.load(), 1); + TransferStatus second{}; + ASSERT_TRUE(engine.getTransferStatus(batch, 1, second).ok()); + EXPECT_EQ(second.s, TransferStatusEnum::PENDING); + EXPECT_EQ(fake_ub->submit_calls.load(), 1); + + complete_first.store(true, std::memory_order_release); + TransferStatus first{}; + ASSERT_TRUE(engine.getTransferStatus(batch, 0, first).ok()); + EXPECT_EQ(first.s, TransferStatusEnum::COMPLETED); + EXPECT_EQ(fake_ub->submit_calls.load(), 2); + ASSERT_TRUE(engine.getTransferStatus(batch, 1, second).ok()); + EXPECT_EQ(second.s, TransferStatusEnum::COMPLETED); + + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, ZeroHandoffUbFailureRollsCreditBack) { + constexpr size_t kReqLen = 4096; + auto cfg = makeRuntimeQueueConfig(2, 1UL << 20); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + + auto fake_ub = std::make_shared(UB); + fake_ub->fail_next_submits.store(1); + installFakeUb(engine, fake_ub); + std::vector buffer(kReqLen * 2, 0x72); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(2); + ASSERT_NE(batch, (BatchID)0); + + ASSERT_TRUE( + engine + .submitTransfer( + batch, {makeLocalUbWrite(buffer.data(), kReqLen), + makeLocalUbWrite(buffer.data() + kReqLen, kReqLen)}) + .ok()); + // The first submit failed before handoff. Its cumulative rollback is + // reported to the receiver before the next owner gets a fresh grant. + EXPECT_EQ(fake_ub->submit_calls.load(), 2); + TransferStatus first{}, second{}; + ASSERT_TRUE(engine.getTransferStatus(batch, 0, first).ok()); + EXPECT_EQ(first.s, TransferStatusEnum::FAILED); + ASSERT_TRUE(engine.getTransferStatus(batch, 1, second).ok()); + EXPECT_EQ(second.s, TransferStatusEnum::COMPLETED); + + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, RemoteReceiverCreditExchangeGatesUbDispatch) { + constexpr size_t kReqLen = 4096; + auto receiver_cfg = makeRuntimeQueueConfig(2, 1UL << 20); + receiver_cfg->set("receiver_credit/receiver_enable", true); + receiver_cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + receiver_cfg->set("receiver_credit/request_slots", uint64_t{1}); + auto sender_cfg = makeRuntimeQueueConfig(2, 1UL << 20); + sender_cfg->set("receiver_credit/sender_enable", true); + sender_cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + sender_cfg->set("receiver_credit/request_slots", uint64_t{1}); + + TransferEngineImpl receiver(receiver_cfg); + TransferEngineImpl sender(sender_cfg); + ASSERT_TRUE(receiver.available()); + ASSERT_TRUE(sender.available()); + + auto receiver_ub = std::make_shared(UB); + installFakeUb(receiver, receiver_ub); + std::atomic complete_first{false}; + auto sender_ub = std::make_shared( + UB, [&complete_first](const Request& request, int) { + if (!complete_first.load(std::memory_order_acquire)) { + return TransferStatus{TransferStatusEnum::PENDING, 0}; + } + return TransferStatus{TransferStatusEnum::COMPLETED, + request.length}; + }); + installFakeUb(sender, sender_ub); + + std::vector source(kReqLen * 2, 0x81); + std::vector target(kReqLen * 2, 0); + ASSERT_TRUE(sender.registerLocalMemory(source.data(), source.size()).ok()); + ASSERT_TRUE( + receiver.registerLocalMemory(target.data(), target.size()).ok()); + SegmentID remote = LOCAL_SEGMENT_ID; + ASSERT_TRUE(sender.openSegment(remote, receiver.getSegmentName()).ok()); + ASSERT_NE(remote, LOCAL_SEGMENT_ID); + + BatchID batch = sender.allocateBatch(2); + ASSERT_NE(batch, (BatchID)0); + ASSERT_TRUE( + sender + .submitTransfer( + batch, {makeRemoteUbWrite(source.data(), remote, target.data(), + kReqLen), + makeRemoteUbWrite(source.data() + kReqLen, remote, + target.data() + kReqLen, kReqLen)}) + .ok()); + EXPECT_EQ(sender_ub->submit_calls.load(), 1); + + // TCP fallback carries the same receiver-session fence on the wire. The + // receiver rejects an unfenced or stale request even if the sender's + // metadata cache still describes the old process incarnation. + const auto receiver_addr = receiver.getSegmentName(); + const uint64_t session_high = sender_ub->last_credit_session_high.load(); + const uint64_t session_low = sender_ub->last_credit_session_low.load(); + const uint64_t epoch = sender_ub->last_credit_epoch.load(); + ASSERT_NE(session_high | session_low, 0u); + ASSERT_NE(epoch, 0u); + + auto delegated = makeLocalUbWrite(target.data(), kReqLen); + delegated.receiver_credit_session_high = session_high; + delegated.receiver_credit_session_low = session_low; + delegated.receiver_credit_epoch = epoch + 1; + EXPECT_TRUE( + ControlClient::delegate(receiver_addr, delegated).IsRpcServiceError()); + delegated.receiver_credit_epoch = epoch; + ASSERT_TRUE(ControlClient::delegate(receiver_addr, delegated).ok()); + EXPECT_EQ(receiver_ub->submit_calls.load(), 1); + + delegated.receiver_credit_epoch = 0; + EXPECT_TRUE( + ControlClient::delegate(receiver_addr, delegated).IsInvalidArgument()); + + EXPECT_TRUE(ControlClient::sendData( + receiver_addr, reinterpret_cast(target.data()), + source.data(), kReqLen) + .IsRpcServiceError()); + EXPECT_EQ(target[0], 0); + EXPECT_TRUE(ControlClient::sendData( + receiver_addr, reinterpret_cast(target.data()), + source.data(), kReqLen, session_high, session_low, + epoch + 1) + .IsRpcServiceError()); + EXPECT_EQ(target[0], 0); + const auto fenced_send = ControlClient::sendData( + receiver_addr, reinterpret_cast(target.data()), source.data(), + kReqLen, session_high, session_low, epoch); + ASSERT_TRUE(fenced_send.ok()) << fenced_send.ToString(); + EXPECT_EQ(target[0], source[0]); + + std::vector readback(kReqLen, 0); + EXPECT_TRUE(ControlClient::recvData( + receiver_addr, reinterpret_cast(target.data()), + readback.data(), kReqLen, session_high, session_low, + epoch + 1) + .IsRpcServiceError()); + const auto fenced_recv = ControlClient::recvData( + receiver_addr, reinterpret_cast(target.data()), + readback.data(), kReqLen, session_high, session_low, epoch); + ASSERT_TRUE(fenced_recv.ok()) << fenced_recv.ToString(); + EXPECT_EQ(readback, std::vector(kReqLen, 0x81)); + + complete_first.store(true, std::memory_order_release); + TransferStatus first{}, second{}; + ASSERT_TRUE(sender.getTransferStatus(batch, 0, first).ok()); + EXPECT_EQ(first.s, TransferStatusEnum::COMPLETED); + EXPECT_EQ(sender_ub->submit_calls.load(), 2); + ASSERT_TRUE(sender.getTransferStatus(batch, 1, second).ok()); + EXPECT_EQ(second.s, TransferStatusEnum::COMPLETED); + + EXPECT_TRUE(sender.freeBatch(batch).ok()); + EXPECT_TRUE(sender.closeSegment(remote).ok()); + EXPECT_TRUE( + sender.unregisterLocalMemory(source.data(), source.size()).ok()); + EXPECT_TRUE( + receiver.unregisterLocalMemory(target.data(), target.size()).ok()); +} + +TEST(RuntimeQueueDispatch, RdmaFailureGatesUbFailoverWithReceiverCredit) { + constexpr size_t kReqLen = 4096; + auto cfg = makeRuntimeQueueConfig(1, 1UL << 20); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + + auto fake_rdma = + std::make_shared(RDMA, [](const Request&, int) { + return TransferStatus{TransferStatusEnum::FAILED, 0}; + }); + auto fake_ub = std::make_shared(UB); + installFakeRdma(engine, fake_rdma); + installFakeUb(engine, fake_ub); + + std::vector buffer(kReqLen, 0x91); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(1); + ASSERT_NE(batch, (BatchID)0); + ASSERT_TRUE(engine + .submitTransfer( + batch, {makeLocalNetworkWrite(buffer.data(), kReqLen)}) + .ok()); + ASSERT_EQ(fake_rdma->submit_calls.load(), 1); + ASSERT_EQ(fake_ub->submit_calls.load(), 0); + + TransferStatus status{}; + ASSERT_TRUE(engine.getTransferStatus(batch, 0, status).ok()); + EXPECT_EQ(status.s, TransferStatusEnum::PENDING); + EXPECT_EQ(fake_ub->submit_calls.load(), 1); + ASSERT_TRUE(engine.getTransferStatus(batch, 0, status).ok()); + EXPECT_EQ(status.s, TransferStatusEnum::COMPLETED); + + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, ReceiverCreditAlsoGatesInitialRdmaPath) { + constexpr size_t kReqLen = 4096; + auto cfg = makeRuntimeQueueConfig(2, 1UL << 20); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + + std::atomic complete_first{false}; + auto fake_rdma = std::make_shared( + RDMA, [&complete_first](const Request& request, int) { + if (!complete_first.load(std::memory_order_acquire)) { + return TransferStatus{TransferStatusEnum::PENDING, 0}; + } + return TransferStatus{TransferStatusEnum::COMPLETED, + request.length}; + }); + installFakeRdma(engine, fake_rdma); + + std::vector buffer(kReqLen * 2, 0x92); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(2); + ASSERT_NE(batch, (BatchID)0); + ASSERT_TRUE( + engine + .submitTransfer(batch, + {makeLocalWrite(buffer.data(), kReqLen), + makeLocalWrite(buffer.data() + kReqLen, kReqLen)}) + .ok()); + EXPECT_EQ(fake_rdma->submit_calls.load(), 1); + EXPECT_NE(fake_rdma->last_credit_session_high.load() | + fake_rdma->last_credit_session_low.load(), + 0u); + EXPECT_EQ(fake_rdma->last_credit_epoch.load(), 1u); + + complete_first.store(true, std::memory_order_release); + TransferStatus first{}, second{}; + ASSERT_TRUE(engine.getTransferStatus(batch, 0, first).ok()); + EXPECT_EQ(first.s, TransferStatusEnum::COMPLETED); + EXPECT_EQ(fake_rdma->submit_calls.load(), 2); + ASSERT_TRUE(engine.getTransferStatus(batch, 1, second).ok()); + EXPECT_EQ(second.s, TransferStatusEnum::COMPLETED); + + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, RejectsRequestLargerThanReceiverCreditCapacity) { + constexpr size_t kCapacity = 4096; + auto cfg = makeRuntimeQueueConfig(1, 1UL << 20); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kCapacity}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + + auto fake_ub = std::make_shared(UB); + installFakeUb(engine, fake_ub); + std::vector buffer(kCapacity * 2, 0x93); + ASSERT_TRUE(engine.registerLocalMemory(buffer.data(), buffer.size()).ok()); + BatchID batch = engine.allocateBatch(1); + ASSERT_NE(batch, (BatchID)0); + + const auto status = engine.submitTransfer( + batch, {makeLocalUbWrite(buffer.data(), buffer.size())}); + EXPECT_TRUE(status.IsTooManyRequests()) << status.ToString(); + EXPECT_EQ(fake_ub->submit_calls.load(), 0); + + EXPECT_TRUE(engine.freeBatch(batch).ok()); + EXPECT_TRUE( + engine.unregisterLocalMemory(buffer.data(), buffer.size()).ok()); +} + +TEST(RuntimeQueueDispatch, TcpControlDataUsesLegacyFramingWithoutCredit) { + constexpr size_t kLength = 256; + auto cfg = makeRuntimeQueueConfig(1, 1UL << 20); + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + auto fake_rdma = std::make_shared(RDMA); + installFakeRdma(engine, fake_rdma); + + std::vector source(kLength, 0xa5); + std::vector target(kLength, 0); + std::vector readback(kLength, 0); + ASSERT_TRUE(engine.registerLocalMemory(target.data(), target.size()).ok()); + const auto server_addr = engine.getSegmentName(); + + ASSERT_TRUE(ControlClient::sendData( + server_addr, reinterpret_cast(target.data()), + source.data(), source.size()) + .ok()); + EXPECT_EQ(target, source); + ASSERT_TRUE(ControlClient::recvData( + server_addr, reinterpret_cast(target.data()), + readback.data(), readback.size()) + .ok()); + EXPECT_EQ(readback, source); + + EXPECT_TRUE(ControlClient::sendData( + server_addr, reinterpret_cast(target.data()), + source.data(), source.size(), 1, 0, 0) + .IsInvalidArgument()); + EXPECT_TRUE( + engine.unregisterLocalMemory(target.data(), target.size()).ok()); +} + +// A staged owner holds the receiver credit for all of its chunks; the chunks +// themselves are credit-exempt. If one chunk fails while a sibling's cross +// stage is still writing into the receiver, the owner must not turn terminal +// (and so must not return the credit) until that write has drained. +// Otherwise the receiver could grant the same capacity to another sender +// while the old write is still landing. +TEST(RuntimeQueueDispatch, StagedFailureKeepsReceiverCreditUntilChunksDrain) { + constexpr size_t kChunk = 64; + constexpr size_t kStagedLen = 2 * kChunk; + constexpr size_t kSmallLen = 32; + auto cfg = makeRuntimeQueueConfig(8, 1UL << 20); + cfg->set("max_failover_attempts", 0); + cfg->set("receiver_credit/enable", true); + cfg->set("receiver_credit/data_bytes", uint64_t{kStagedLen}); + cfg->set("receiver_credit/request_slots", uint64_t{1}); + cfg->set("receiver_credit/freshness_ttl_ms", uint32_t{1000}); + std::vector source(kStagedLen, 0x5a); + std::vector target(kStagedLen, 0); + std::vector small_source(kSmallLen, 0x6b); + std::vector small_target(kSmallLen, 0); + // Declared after the buffers the fake transport and hook capture. + TransferEngineImpl engine(cfg); + ASSERT_TRUE(engine.available()); + auto in = [](const void* addr, const std::vector& buf) { + auto p = reinterpret_cast(addr); + auto base = reinterpret_cast(buf.data()); + return p >= base && p < base + buf.size(); + }; + + std::atomic cross_inflight{false}; + std::atomic release_cross{false}; + std::atomic failed_polls{0}; + std::atomic small_polls{0}; + auto fake_tcp = + std::make_shared(TCP, [&](const Request& request, int) { + const auto* src = static_cast(request.source); + const auto* dst = + reinterpret_cast(request.target_offset); + // Second chunk's local stage: fail only once the first chunk's + // cross stage is writing into the receiver. + if (src == source.data() + kChunk) { + if (!cross_inflight.load()) return TransferStatus{PENDING, 0}; + ++failed_polls; + return TransferStatus{FAILED, 0}; + } + // Cross stage into the receiver: held until the test lets go. + if (in(dst, target)) { + cross_inflight = true; + if (!release_cross.load()) return TransferStatus{PENDING, 0}; + } + if (in(src, small_source)) ++small_polls; + std::memcpy(const_cast(dst), request.source, request.length); + return TransferStatus{COMPLETED, request.length}; + }); + std::string seg_name = engine.getSegmentName(); + ASSERT_TRUE(fake_tcp->install(seg_name, nullptr, nullptr).ok()); + engine.swapTransportForTest(TCP, fake_tcp); + TransferEngineImplTestPeer::useSmallStagePool(engine, kChunk, 2); + // CI has no device memory, so force host staging for the large write. + engine.setStagingPolicyForTest( + [&](const Request& request, std::vector& policy) { + if (request.source != source.data() || request.length != kStagedLen) + return false; + policy = {"", kWildcardLocation, ""}; + return true; + }); + + for (auto* buf : {&source, &target, &small_source, &small_target}) + ASSERT_TRUE(engine.registerLocalMemory(buf->data(), buf->size()).ok()); + + auto make_write = [](std::vector& from, std::vector& to) { + Request request; + request.opcode = Request::WRITE; + request.source = from.data(); + request.target_id = LOCAL_SEGMENT_ID; + request.target_offset = reinterpret_cast(to.data()); + request.length = from.size(); + request.transport_hint = TCP; + return request; + }; + auto wait_until = [](const std::function& done) { + auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!done() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return done(); + }; + + BatchID staged = engine.allocateBatch(1); + ASSERT_NE(staged, (BatchID)0); + ASSERT_TRUE( + engine.submitTransfer(staged, {make_write(source, target)}).ok()); + ASSERT_TRUE(wait_until([&] { return failed_polls.load() > 0; })) + << "second chunk never failed while the first was in its cross stage"; + // Let the failure path run while the first chunk's write is held. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + TransferStatus staged_status{}; + ASSERT_TRUE(engine.getTransferStatus(staged, 0, staged_status).ok()); + EXPECT_EQ(staged_status.s, PENDING); + + // The staged owner still holds the only request slot, so a second + // credit-gated write must wait instead of reaching the transport. + BatchID small = engine.allocateBatch(1); + ASSERT_NE(small, (BatchID)0); + ASSERT_TRUE( + engine.submitTransfer(small, {make_write(small_source, small_target)}) + .ok()); + TransferStatus small_status{}; + ASSERT_TRUE(engine.getTransferStatus(small, 0, small_status).ok()); + EXPECT_EQ(small_status.s, PENDING); + EXPECT_EQ(small_polls.load(), 0); + + // Once the held write drains, the owner fails and returns the credit. + release_cross = true; + ASSERT_TRUE(wait_until([&] { + return engine.getTransferStatus(staged, 0, staged_status).ok() && + staged_status.s != PENDING; + })); + EXPECT_EQ(staged_status.s, FAILED); + ASSERT_TRUE(wait_until([&] { + return engine.getTransferStatus(small, 0, small_status).ok() && + small_status.s != PENDING; + })); + EXPECT_EQ(small_status.s, COMPLETED); + EXPECT_EQ(small_target, small_source); + + EXPECT_TRUE(engine.freeBatch(staged).ok()); + EXPECT_TRUE(engine.freeBatch(small).ok()); + for (auto* buf : {&source, &target, &small_source, &small_target}) + EXPECT_TRUE( + engine.unregisterLocalMemory(buf->data(), buf->size()).ok()); +} + +// Receiver credit lives in another engine, so each demand exchange is an RPC +// made under progress_mutex_. A deferred owner must not repeat a refused +// exchange on every poll: back-to-back RPCs hold the lock and starve the +// in-flight owner whose completion is what returns the credit (on two UB +// nodes this collapsed throughput by ~1000x once demand exceeded capacity). +TEST(RuntimeQueueDispatch, RefusedCreditIsNotReexchangedOnEveryPoll) { + constexpr size_t kReqLen = 4096; + auto receiver_cfg = makeRuntimeQueueConfig(4, 1UL << 20); + receiver_cfg->set("receiver_credit/receiver_enable", true); + receiver_cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + receiver_cfg->set("receiver_credit/request_slots", uint64_t{1}); + auto sender_cfg = makeRuntimeQueueConfig(4, 1UL << 20); + sender_cfg->set("receiver_credit/sender_enable", true); + std::vector source(kReqLen * 2, 0x91); + std::vector target(kReqLen * 2, 0); + TransferEngineImpl receiver(receiver_cfg); + TransferEngineImpl sender(sender_cfg); + ASSERT_TRUE(receiver.available()); + ASSERT_TRUE(sender.available()); + installFakeUb(receiver, std::make_shared(UB)); + std::atomic complete_first{false}; + auto sender_ub = + std::make_shared(UB, [&](const Request& request, int) { + if (request.source == source.data() && !complete_first.load()) + return TransferStatus{PENDING, 0}; + return TransferStatus{COMPLETED, request.length}; + }); + installFakeUb(sender, sender_ub); + ASSERT_TRUE(sender.registerLocalMemory(source.data(), source.size()).ok()); + ASSERT_TRUE( + receiver.registerLocalMemory(target.data(), target.size()).ok()); + SegmentID remote = LOCAL_SEGMENT_ID; + ASSERT_TRUE(sender.openSegment(remote, receiver.getSegmentName()).ok()); + + BatchID holder = sender.allocateBatch(1); + BatchID waiter = sender.allocateBatch(1); + ASSERT_NE(holder, (BatchID)0); + ASSERT_NE(waiter, (BatchID)0); + ASSERT_TRUE( + sender + .submitTransfer(holder, {makeRemoteUbWrite(source.data(), remote, + target.data(), kReqLen)}) + .ok()); + ASSERT_TRUE( + sender + .submitTransfer( + waiter, {makeRemoteUbWrite(source.data() + kReqLen, remote, + target.data() + kReqLen, kReqLen)}) + .ok()); + EXPECT_EQ(sender_ub->submit_calls.load(), 1); + + // Busy-poll the refused owner. Without the backoff every poll is another + // exchange (thousands here); with it, a handful (1, 2, 4, ... ms). + const auto before = + TransferEngineImplTestPeer::creditDemandExchanges(sender); + TransferStatus waiter_status{}; + const auto until = + std::chrono::steady_clock::now() + std::chrono::milliseconds(50); + while (std::chrono::steady_clock::now() < until) { + ASSERT_TRUE(sender.getTransferStatus(waiter, 0, waiter_status).ok()); + ASSERT_EQ(waiter_status.s, PENDING); + } + const auto refused = + TransferEngineImplTestPeer::creditDemandExchanges(sender) - before; + EXPECT_GE(refused, 1u); + EXPECT_LE(refused, 12u); + EXPECT_EQ(sender_ub->submit_calls.load(), 1); + + // Returning the credit retries at once, without waiting out the backoff. + complete_first = true; + TransferStatus holder_status{}; + ASSERT_TRUE(sender.getTransferStatus(holder, 0, holder_status).ok()); + EXPECT_EQ(holder_status.s, COMPLETED); + EXPECT_EQ(sender_ub->submit_calls.load(), 2); + ASSERT_TRUE(sender.getTransferStatus(waiter, 0, waiter_status).ok()); + EXPECT_EQ(waiter_status.s, COMPLETED); + + EXPECT_TRUE(sender.freeBatch(holder).ok()); + EXPECT_TRUE(sender.freeBatch(waiter).ok()); + EXPECT_TRUE(sender.closeSegment(remote).ok()); + EXPECT_TRUE( + sender.unregisterLocalMemory(source.data(), source.size()).ok()); + EXPECT_TRUE( + receiver.unregisterLocalMemory(target.data(), target.size()).ok()); +} + +// The refused sender is not told when another sender returns credit. The +// capped backoff must still let it through without any local release. +TEST(RuntimeQueueDispatch, RefusedCreditRecoversWhenAnotherSenderReleases) { + constexpr size_t kReqLen = 4096; + auto receiver_cfg = makeRuntimeQueueConfig(4, 1UL << 20); + receiver_cfg->set("receiver_credit/receiver_enable", true); + receiver_cfg->set("receiver_credit/data_bytes", uint64_t{kReqLen}); + receiver_cfg->set("receiver_credit/request_slots", uint64_t{1}); + auto sender_cfg = makeRuntimeQueueConfig(4, 1UL << 20); + sender_cfg->set("receiver_credit/sender_enable", true); + std::vector source_a(kReqLen, 0xa1), source_b(kReqLen, 0xb2); + std::vector target(kReqLen * 2, 0); + TransferEngineImpl receiver(receiver_cfg); + TransferEngineImpl sender_a(sender_cfg); + TransferEngineImpl sender_b(sender_cfg); + ASSERT_TRUE(receiver.available()); + ASSERT_TRUE(sender_a.available()); + ASSERT_TRUE(sender_b.available()); + installFakeUb(receiver, std::make_shared(UB)); + std::atomic complete_a{false}; + auto ub_a = + std::make_shared(UB, [&](const Request& request, int) { + if (!complete_a.load()) return TransferStatus{PENDING, 0}; + return TransferStatus{COMPLETED, request.length}; + }); + auto ub_b = std::make_shared(UB); + installFakeUb(sender_a, ub_a); + installFakeUb(sender_b, ub_b); + ASSERT_TRUE( + sender_a.registerLocalMemory(source_a.data(), source_a.size()).ok()); + ASSERT_TRUE( + sender_b.registerLocalMemory(source_b.data(), source_b.size()).ok()); + ASSERT_TRUE( + receiver.registerLocalMemory(target.data(), target.size()).ok()); + SegmentID remote_a = LOCAL_SEGMENT_ID, remote_b = LOCAL_SEGMENT_ID; + ASSERT_TRUE(sender_a.openSegment(remote_a, receiver.getSegmentName()).ok()); + ASSERT_TRUE(sender_b.openSegment(remote_b, receiver.getSegmentName()).ok()); + + BatchID batch_a = sender_a.allocateBatch(1); + BatchID batch_b = sender_b.allocateBatch(1); + ASSERT_TRUE(sender_a + .submitTransfer( + batch_a, {makeRemoteUbWrite(source_a.data(), remote_a, + target.data(), kReqLen)}) + .ok()); + ASSERT_TRUE( + sender_b + .submitTransfer( + batch_b, {makeRemoteUbWrite(source_b.data(), remote_b, + target.data() + kReqLen, kReqLen)}) + .ok()); + EXPECT_EQ(ub_a->submit_calls.load(), 1); + EXPECT_EQ(ub_b->submit_calls.load(), 0); + // Let sender B back off a few rounds while A holds the only slot. + TransferStatus status_b{}; + const auto hold = + std::chrono::steady_clock::now() + std::chrono::milliseconds(40); + while (std::chrono::steady_clock::now() < hold) { + ASSERT_TRUE(sender_b.getTransferStatus(batch_b, 0, status_b).ok()); + ASSERT_EQ(status_b.s, PENDING); + } + + complete_a = true; + TransferStatus status_a{}; + ASSERT_TRUE(sender_a.getTransferStatus(batch_a, 0, status_a).ok()); + EXPECT_EQ(status_a.s, COMPLETED); + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) { + ASSERT_TRUE(sender_b.getTransferStatus(batch_b, 0, status_b).ok()); + if (status_b.s != PENDING) break; + std::this_thread::sleep_for(std::chrono::microseconds(200)); + } + EXPECT_EQ(status_b.s, COMPLETED); + EXPECT_EQ(ub_b->submit_calls.load(), 1); + + EXPECT_TRUE(sender_a.freeBatch(batch_a).ok()); + EXPECT_TRUE(sender_b.freeBatch(batch_b).ok()); + EXPECT_TRUE(sender_a.closeSegment(remote_a).ok()); + EXPECT_TRUE(sender_b.closeSegment(remote_b).ok()); + EXPECT_TRUE( + sender_a.unregisterLocalMemory(source_a.data(), source_a.size()).ok()); + EXPECT_TRUE( + sender_b.unregisterLocalMemory(source_b.data(), source_b.size()).ok()); + EXPECT_TRUE( + receiver.unregisterLocalMemory(target.data(), target.size()).ok()); +} + } // namespace } // namespace tent } // namespace mooncake diff --git a/mooncake-transfer-engine/tent/tests/tcp_regression_test.cpp b/mooncake-transfer-engine/tent/tests/tcp_regression_test.cpp index 716ca8eaee..94b7f50c34 100644 --- a/mooncake-transfer-engine/tent/tests/tcp_regression_test.cpp +++ b/mooncake-transfer-engine/tent/tests/tcp_regression_test.cpp @@ -254,7 +254,7 @@ TEST_F(TcpRegression, ServerChecksEveryRegistrationPermission) { // Direct RPC bypasses client-side registration/routing policy. auto status = call(engine->getSegmentName(), reinterpret_cast(data.data()), - local.data(), local.size()); + local.data(), local.size(), 0, 0, 0); EXPECT_EQ(status.ok(), allowed) << status.ToString(); if (!allowed) { EXPECT_NE(status.ToString().find("permission denied"), @@ -298,7 +298,7 @@ TEST_F(TcpRegression, PayloadCapsRejectBeforeAllocationOrDataRpc) { EXPECT_NO_THROW({ EXPECT_TRUE(call(address, reinterpret_cast(data.data()), - data.data(), length) + data.data(), length, 0, 0, 0) .IsInvalidArgument()); }); } @@ -316,14 +316,20 @@ TEST_F(TcpRegression, MalformedWriteLengthCannotOverflowOrModifyTarget) { CoroRpcAgent rpc; for (size_t length : {size_t{1}, kTcpMaxWriteBytes + 1, std::numeric_limits::max()}) { - XferDataDesc header{htole64(reinterpret_cast(data.data())), - htole64(length)}; + LegacyXferDataDesc header{ + htole64(reinterpret_cast(data.data())), htole64(length)}; std::string request(reinterpret_cast(&header), sizeof(header)); std::string response; ASSERT_TRUE( rpc.call(engine->getSegmentName(), SendData, request, response) .ok()); - EXPECT_EQ(response, "SendData failed: invalid request size"); + // The v2 parser rejects the legacy header before any copy: the + // declared length can never match a request that carries no payload. + EXPECT_EQ(response.rfind("SendData failed: InvalidArgument: invalid " + "transfer request size", + 0), + 0u) + << response; EXPECT_EQ(data, original); } } diff --git a/mooncake-transfer-engine/tent/tests/transport_selector_test.cpp b/mooncake-transfer-engine/tent/tests/transport_selector_test.cpp index 7f7ad40dd8..d2709d2d30 100644 --- a/mooncake-transfer-engine/tent/tests/transport_selector_test.cpp +++ b/mooncake-transfer-engine/tent/tests/transport_selector_test.cpp @@ -411,6 +411,9 @@ TEST(ControlPlaneTest, UbBootstrapJsonRoundTripsNativeIdentity) { source.jetty_uasids = {21, 22}; source.endpoint_generation = 41; source.segment_generation = 42; + source.receiver_credit_session_high = 43; + source.receiver_credit_session_low = 44; + source.receiver_credit_epoch = 45; source.capabilities = {"read", "write"}; source.reply_msg = "ok"; @@ -427,6 +430,11 @@ TEST(ControlPlaneTest, UbBootstrapJsonRoundTripsNativeIdentity) { EXPECT_EQ(parsed.jetty_uasids, source.jetty_uasids); EXPECT_EQ(parsed.endpoint_generation, source.endpoint_generation); EXPECT_EQ(parsed.segment_generation, source.segment_generation); + EXPECT_EQ(parsed.receiver_credit_session_high, + source.receiver_credit_session_high); + EXPECT_EQ(parsed.receiver_credit_session_low, + source.receiver_credit_session_low); + EXPECT_EQ(parsed.receiver_credit_epoch, source.receiver_credit_epoch); EXPECT_EQ(parsed.capabilities, source.capabilities); EXPECT_EQ(parsed.reply_msg, source.reply_msg); } @@ -445,6 +453,9 @@ TEST(ControlPlaneTest, UbBootstrapJsonDefaultsOptionalFields) { EXPECT_TRUE(parsed.jetty_uasids.empty()); EXPECT_EQ(parsed.endpoint_generation, 0u); EXPECT_EQ(parsed.segment_generation, 0u); + EXPECT_EQ(parsed.receiver_credit_session_high, 0u); + EXPECT_EQ(parsed.receiver_credit_session_low, 0u); + EXPECT_EQ(parsed.receiver_credit_epoch, 0u); EXPECT_TRUE(parsed.capabilities.empty()); EXPECT_TRUE(parsed.reply_msg.empty()); } @@ -470,6 +481,7 @@ TEST(ControlPlaneTest, UbBootstrapRpcIdIsAppendedWithoutRenumbering) { EXPECT_EQ(static_cast(SubscribeSegmentUpdate), 10); EXPECT_EQ(static_cast(NotifySegmentUpdated), 11); EXPECT_EQ(static_cast(BootstrapUb), 12); + EXPECT_EQ(static_cast(ExchangeReceiverCredit), 13); } // --------------------------------------------------------------------------- diff --git a/mooncake-transfer-engine/tent/tests/ub_native_data_path_test.cpp b/mooncake-transfer-engine/tent/tests/ub_native_data_path_test.cpp index 4dc2694de6..2ad2b862fe 100644 --- a/mooncake-transfer-engine/tent/tests/ub_native_data_path_test.cpp +++ b/mooncake-transfer-engine/tent/tests/ub_native_data_path_test.cpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -547,6 +548,30 @@ TEST(UbNativeDataPathTest, (void)endpoints.retire(endpoint); }); ASSERT_TRUE(workers.start().ok()); + + Request invalid_batch_request{}; + invalid_batch_request.opcode = Request::WRITE; + invalid_batch_request.source = source.data(); + invalid_batch_request.target_id = LOCAL_SEGMENT_ID; + invalid_batch_request.target_offset = + reinterpret_cast(target.data()); + invalid_batch_request.length = 16; + auto valid_task = UbTask::create(invalid_batch_request); + auto valid_slice = valid_task->addSlice(UbSliceSpec{ + source.data(), reinterpret_cast(target.data()), 16, 0, 1}); + ASSERT_NE(valid_slice, nullptr); + ASSERT_TRUE(valid_task->seal()); + auto unsealed_task = UbTask::create(invalid_batch_request); + ASSERT_NE(unsealed_task->addSlice(UbSliceSpec{ + source.data(), reinterpret_cast(target.data()), 16, + 0, 1}), + nullptr); + + EXPECT_FALSE(workers.submitBatch({valid_task, unsealed_task}).ok()); + EXPECT_EQ(workers.queuedCount(), 0U); + EXPECT_EQ(workers.inflightCount(), 0U); + EXPECT_EQ(valid_slice->snapshot().state, UbSliceState::kInitial); + adapter->holdNextCompletion(); adapter->dropNextQuiescedCompletion(); @@ -762,6 +787,15 @@ TEST(UbNativeDataPathTest, EndpointStoreNeverReusesRetiredGeneration) { EXPECT_EQ(concurrent[i], concurrent[0]); } auto first = concurrent[0]; + UbBootstrapDesc local_bootstrap; + ASSERT_TRUE(first + ->makeBootstrapDesc("local", "local@ub:fake0:eid0", + key.peer_nic_path, 1, local_bootstrap) + .ok()); + EXPECT_NE( + std::find(local_bootstrap.capabilities.begin(), + local_bootstrap.capabilities.end(), "receiver_credit_v1"), + local_bootstrap.capabilities.end()); const uint64_t old_generation = first->generation(); EXPECT_TRUE(store.retire(key, old_generation)); std::shared_ptr replacement; @@ -885,8 +919,13 @@ TEST(UbNativeDataPathTest, UbTransportRunsSelfReadWriteOverNativeControlPlane) { ASSERT_TRUE(transport.addMemoryBuffer(descriptors, options).ok()); ASSERT_TRUE(control->segmentManager() .updateLocal([&](SegmentDesc& segment) { - std::get(segment.detail).buffers = - descriptors; + auto& memory = + std::get(segment.detail); + memory.buffers = descriptors; + ReceiverCreditAdvertV1 advert; + advert.receiver_session_id = {1, 2}; + advert.epoch = 1; + memory.receiver_credit = std::move(advert); return Status::OK(); }) .ok()); @@ -904,6 +943,13 @@ TEST(UbNativeDataPathTest, UbTransportRunsSelfReadWriteOverNativeControlPlane) { transport.submitTransferTasks(rejected_batch, {oversized_request}) .IsInvalidArgument()); EXPECT_TRUE(transport.freeSubBatch(rejected_batch).ok()); + UbBootstrapDesc unsupported_bootstrap; + UbBootstrapDesc rejected_bootstrap; + ASSERT_TRUE(ControlClient::bootstrapUb(segment_name, unsupported_bootstrap, + rejected_bootstrap) + .ok()); + EXPECT_NE(rejected_bootstrap.reply_msg.find("receiver-credit"), + std::string::npos); Transport::SubBatchRef batch = nullptr; ASSERT_TRUE(transport.allocateSubBatch(batch, 2).ok()); @@ -913,6 +959,11 @@ TEST(UbNativeDataPathTest, UbTransportRunsSelfReadWriteOverNativeControlPlane) { request.target_id = LOCAL_SEGMENT_ID; request.target_offset = reinterpret_cast(target.data()); request.length = source.size(); + auto invalid_request = request; + invalid_request.length = 0; + EXPECT_FALSE( + transport.submitTransferTasks(batch, {request, invalid_request}).ok()); + EXPECT_EQ(batch->size(), 0U); ASSERT_TRUE(transport.submitTransferTasks(batch, {request}).ok()); TransferStatus transfer{PENDING, 0}; diff --git a/mooncake-transfer-engine/tests/CMakeLists.txt b/mooncake-transfer-engine/tests/CMakeLists.txt index bfdb03b3d4..2a3292e48b 100644 --- a/mooncake-transfer-engine/tests/CMakeLists.txt +++ b/mooncake-transfer-engine/tests/CMakeLists.txt @@ -28,8 +28,24 @@ if(MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL) ${WORKSPACE}/adaptive_congestion_control_benchmark.cpp) target_link_libraries(adaptive_congestion_control_benchmark PRIVATE adaptive_congestion_control_core) + + add_executable(classic_rdma_congestion_control_test + ${WORKSPACE}/classic_rdma_congestion_control_test.cpp) + target_link_libraries(classic_rdma_congestion_control_test + PRIVATE transfer_engine gtest gtest_main) + add_test(NAME classic_rdma_congestion_control_test + COMMAND classic_rdma_congestion_control_test) endif() +configure_file(installed_congestion_control_header_test.cmake.in + installed_congestion_control_header_test.cmake @ONLY) +add_test( + NAME installed_congestion_control_header_test + COMMAND + ${CMAKE_COMMAND} -DTRANSFER_ENGINE_LIBRARY=$ + -P + ${CMAKE_CURRENT_BINARY_DIR}/installed_congestion_control_header_test.cmake) + if(USE_HIP) file(GLOB TEST_SOURCES "*.cpp") hipify_files(TEST_SOURCES) @@ -114,12 +130,16 @@ target_link_libraries(worker_pool_rail_state_test PUBLIC transfer_engine gtest if(CMAKE_SYSTEM_NAME STREQUAL "Linux") # Control GID I/O and port-state queries so event recovery can be exercised # without an RDMA device. + if(BUILD_SHARED_LIBS) + # --wrap only rewrites calls in objects linked into this executable. + target_sources(worker_pool_rail_state_test + PRIVATE $) + endif() target_link_options( worker_pool_rail_state_test PRIVATE "-Wl,--wrap=_ZN8mooncake11RdmaContext17refreshCurrentGidEPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES7_" - "-Wl,--wrap=_ZNK8mooncake11RdmaContext14queryPortStateEP14ibv_port_state" - ) + "-Wl,--wrap=_ZNK8mooncake11RdmaContext14queryPortStateEP14ibv_port_state") endif() add_test(NAME worker_pool_rail_state_test COMMAND worker_pool_rail_state_test) @@ -365,10 +385,9 @@ if(USE_EFA) add_executable(efa_stale_segment_desc_test ${WORKSPACE}/efa_stale_segment_desc_test.cpp) - target_link_libraries(efa_stale_segment_desc_test PUBLIC transfer_engine gtest - gtest_main) - add_test(NAME efa_stale_segment_desc_test - COMMAND efa_stale_segment_desc_test) + target_link_libraries(efa_stale_segment_desc_test PUBLIC transfer_engine + gtest gtest_main) + add_test(NAME efa_stale_segment_desc_test COMMAND efa_stale_segment_desc_test) # Compiles the real EfaEndPoint source against the test-only # RPC/address-vector fakes in efa_endpoint_handshake_fakes/ (shadowed diff --git a/mooncake-transfer-engine/tests/adaptive_congestion_control_test.cpp b/mooncake-transfer-engine/tests/adaptive_congestion_control_test.cpp index 949c3c9901..547ce1d10e 100644 --- a/mooncake-transfer-engine/tests/adaptive_congestion_control_test.cpp +++ b/mooncake-transfer-engine/tests/adaptive_congestion_control_test.cpp @@ -232,34 +232,120 @@ TEST(AdaptiveCongestionControlTest, ConcurrentOversizedSlicesAdmitOnlyOne) { EXPECT_EQ(snapshot(device).inflight_bytes, 0u); } -TEST(AdaptiveCongestionControlTest, - PressureUsesHysteresisAndAdditiveIncreaseMultiplicativeDecrease) { +TEST(AdaptiveCongestionControlTest, SuccessGrowsWithHighDrainOrNoRateSample) { + for (bool has_rate : {false, true}) { + DomainState domain(testConfig()); + Signals signals; + signals.backlog_bytes = 256; + if (has_rate) signals.delivery_rate_bytes_per_sec = 1; + signals.successful_completions = 1; + recordCurrent(domain, signals); + controlTick(domain, 1); + EXPECT_EQ(snapshot(domain).window_bytes, 80u); + EXPECT_EQ(snapshot(domain).state, PathState::kHealthy); + } +} + +TEST(AdaptiveCongestionControlTest, BacklogWithoutSuccessPreservesWindow) { DomainState domain(testConfig()); - Signals low; - low.backlog_bytes = 1; - low.delivery_rate_bytes_per_sec = 10'000'000; - low.completed_bytes = 64; - low.successful_completions = 1; - recordCurrent(domain, low); - controlTick(domain, 1'000); - recordCurrent(domain, low); - controlTick(domain, 2'000); - EXPECT_EQ(snapshot(domain).state, PathState::kHealthy); - EXPECT_GT(snapshot(domain).window_bytes, 64u); - EXPECT_LE(snapshot(domain).window_bytes, 256u); - - Signals high; - high.backlog_bytes = 256; - high.delivery_rate_bytes_per_sec = 1'000'000; - high.completed_bytes = 64; - high.successful_completions = 1; - recordCurrent(domain, high); - controlTick(domain, 3'000); - EXPECT_EQ(snapshot(domain).state, PathState::kHealthy); - recordCurrent(domain, high); - controlTick(domain, 4'000); + Signals success; + success.successful_completions = 1; + recordCurrent(domain, success); + controlTick(domain, 1); + Signals backlog; + backlog.backlog_bytes = std::numeric_limits::max(); + backlog.delivery_rate_bytes_per_sec = 1; + for (uint64_t epoch = 2; epoch < 6; ++epoch) { + recordCurrent(domain, backlog); + controlTick(domain, epoch); + EXPECT_EQ(snapshot(domain).window_bytes, 80u); + } +} + +TEST(AdaptiveCongestionControlTest, ReceiverPressureHalvesOnceBeforeErrorVeto) { + DomainState domain(testConfig()); + Signals signals; + signals.successful_completions = 1; + for (uint64_t epoch = 1; epoch <= 8; ++epoch) { + recordCurrent(domain, signals); + controlTick(domain, epoch); + } + ASSERT_EQ(snapshot(domain).window_bytes, 192u); + signals.receiver_pressure = 100; + signals.fatal_failures = 1; + recordCurrent(domain, signals); + controlTick(domain, 9); + EXPECT_EQ(snapshot(domain).window_bytes, 96u); EXPECT_EQ(snapshot(domain).state, PathState::kCongested); - EXPECT_EQ(snapshot(domain).window_bytes, 64u); + controlTick(domain, 10); + EXPECT_EQ(snapshot(domain).window_bytes, 96u); +} + +TEST(AdaptiveCongestionControlTest, RootErrorVetoUsesFailureScopeForOneEpoch) { + for (OutcomeClass outcome : + {OutcomeClass::kRouteTimeout, OutcomeClass::kFatal, + OutcomeClass::kLocalConfiguration, OutcomeClass::kRemoteMetadata}) { + for (FailureScope scope : + {FailureScope::kOperation, FailureScope::kQp, FailureScope::kCq, + FailureScope::kRoute, FailureScope::kPort, + FailureScope::kDevice}) { + DomainState device(testConfig()), route(testConfig()), + sibling(testConfig()); + Permit failed, successful; + ASSERT_EQ(tryAcquire(pathFor(device, route), 8, failed), + Decision::kAllow); + ASSERT_EQ(tryAcquire(pathFor(device, sibling), 8, successful), + Decision::kAllow); + ASSERT_TRUE(complete(successful, OutcomeClass::kSuccess, + FailureScope::kOperation)); + Signals success; + success.successful_completions = 1; + recordCurrent(route, success); + ASSERT_TRUE(complete(failed, outcome, scope)); + controlTick(device, 1); + controlTick(route, 1); + controlTick(sibling, 1); + const bool shared = scope == FailureScope::kCq || + scope == FailureScope::kPort || + scope == FailureScope::kDevice; + EXPECT_EQ(snapshot(device).window_bytes, shared ? 64u : 80u); + EXPECT_EQ(snapshot(route).window_bytes, 64u); + EXPECT_EQ(snapshot(sibling).window_bytes, 80u); + recordCurrent(device, success); + recordCurrent(route, success); + controlTick(device, 2); + controlTick(route, 2); + EXPECT_EQ(snapshot(device).window_bytes, shared ? 80u : 96u); + EXPECT_EQ(snapshot(route).window_bytes, 80u); + } + } +} + +TEST(AdaptiveCongestionControlTest, PolicyModesPreserveOffAndObserveAdmission) { + for (Mode mode : {Mode::kOff, Mode::kObserve}) { + DomainState domain(testConfig(mode)); + Signals signals; + signals.successful_completions = 1; + recordCurrent(domain, signals); + controlTick(domain, 1); + EXPECT_EQ(snapshot(domain).window_bytes, + mode == Mode::kOff ? 64u : 80u); + signals.hard_errors = 1; + recordCurrent(domain, signals); + controlTick(domain, 2); + EXPECT_EQ(snapshot(domain).window_bytes, + mode == Mode::kOff ? 64u : 80u); + signals.receiver_pressure = 1; + recordCurrent(domain, signals); + controlTick(domain, 3); + EXPECT_EQ(snapshot(domain).window_bytes, 64u); + Permit permit; + ASSERT_EQ( + tryAcquire({nullptr, &domain, 0, generation(domain)}, 1024, permit), + Decision::kAllow); + EXPECT_TRUE(complete(permit, OutcomeClass::kDerivedFlush, + FailureScope::kOperation)); + } } TEST(AdaptiveCongestionControlTest, PollerStallDoesNotBecomeRouteFailure) { diff --git a/mooncake-transfer-engine/tests/classic_rdma_congestion_control_test.cpp b/mooncake-transfer-engine/tests/classic_rdma_congestion_control_test.cpp new file mode 100644 index 0000000000..31b78b1c80 --- /dev/null +++ b/mooncake-transfer-engine/tests/classic_rdma_congestion_control_test.cpp @@ -0,0 +1,483 @@ +// Copyright 2026 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include "adaptive_congestion_control.h" +#include "transport/rdma_transport/worker_pool.h" + +namespace mooncake { +namespace { + +TEST(ClassicRdmaCongestionControlTest, RuntimeOffLeavesQueueUnchanged) { + ClassicRdmaCongestionControl adapter(adaptive_congestion_control::Config{ + .mode = adaptive_congestion_control::Mode::kOff}); + Transport::Slice slice{}; + slice.length = 64; + std::vector queued{&slice}; + + EXPECT_EQ(adapter.gate(queued), 1u); + + EXPECT_EQ(queued, std::vector({&slice})); +} + +TEST(ClassicRdmaCongestionControlTest, ObserveModeAdmitsBeyondByteWindow) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kObserve; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice first{}, second{}, third{}; + first.length = second.length = third.length = 64; + std::vector queued{&first, &second, &third}; + adapter.prepare(queued, "peer@nic"); + + EXPECT_EQ(adapter.gate(queued), 3u); + for (auto *slice : queued) { + EXPECT_TRUE(slice->rdma_congestion_control_permit.active()); + adapter.releaseUnposted(slice); + EXPECT_FALSE(slice->rdma_congestion_control_permit.active()); + } +} + +TEST(ClassicRdmaCongestionControlTest, SuccessfulCompletionReopensByteWindow) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice first{}, second{}; + first.length = second.length = 64; + auto *endpoint = reinterpret_cast(uintptr_t{1}); + std::vector first_queue{&first}, next_queue{&second}; + adapter.prepare(&first, "peer@nic"); + adapter.prepare(&second, "peer@nic"); + adapter.bindEndpoint(&first, endpoint); + adapter.bindEndpoint(&second, endpoint); + ASSERT_EQ(adapter.gate(first_queue), 1u); + ASSERT_EQ(adapter.gate(next_queue), 0u); + + adapter.complete(&first, IBV_WC_SUCCESS); + + EXPECT_FALSE(first.rdma_congestion_control_permit.active()); + EXPECT_EQ(adapter.gate(next_queue), 1u); + adapter.releaseUnposted(&second); +} + +TEST(ClassicRdmaCongestionControlTest, + DeviceResetFencesOldPermitAndPreservesOutstandingAccounting) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice old{}, next{}, other_route{}; + old.length = next.length = other_route.length = 64; + adapter.prepare(&old, "peer@nic"); + adapter.prepare(&next, "peer@nic"); + adapter.prepare(&other_route, "other-peer@nic"); + std::vector old_queue{&old}, next_queue{&next}; + std::vector other_queue{&other_route}; + ASSERT_EQ(adapter.gate(old_queue), 1u); + + adapter.resetDevice(); + + EXPECT_FALSE(adaptive_congestion_control::isCurrentGeneration( + old.rdma_congestion_control_permit)); + EXPECT_EQ(adapter.gate(other_queue), 0u); + adapter.releaseUnposted(&old); + ASSERT_EQ(adapter.gate(next_queue), 1u); + adapter.releaseUnposted(&old); + EXPECT_EQ(adapter.gate(other_queue), 0u); + adapter.releaseUnposted(&next); + ASSERT_EQ(adapter.gate(other_queue), 1u); + adapter.releaseUnposted(&other_route); + EXPECT_EQ(adapter.gate(next_queue), 1u); + adapter.releaseUnposted(&next); +} + +TEST(ClassicRdmaCongestionControlTest, + StaleDeviceSuccessDoesNotGrowCurrentWindow) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 128; + ClassicRdmaCongestionControl adapter(config); + adapter.tick(1); + Transport::Slice stale{}, outstanding{}, probe{}; + stale.length = 1'000'000; + outstanding.length = 64; + probe.length = 16; + auto *endpoint = reinterpret_cast(uintptr_t{1}); + for (auto *slice : {&stale, &outstanding, &probe}) { + adapter.prepare(slice, "peer@nic"); + adapter.bindEndpoint(slice, endpoint); + } + std::vector queued{&stale}; + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.resetDevice(); + adapter.complete(&stale, IBV_WC_SUCCESS); + EXPECT_FALSE(stale.rdma_congestion_control_permit.active()); + + queued = {&outstanding}; + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.tick(1'000'000'001); + + queued = {&probe}; + EXPECT_EQ(adapter.gate(queued), 0u); + adapter.releaseUnposted(&outstanding); + adapter.releaseUnposted(&probe); +} + +TEST(ClassicRdmaCongestionControlTest, CleanSuccessGrowsWindowOnNextTick) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 128; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice completed{}, outstanding{}, probe{}; + completed.length = outstanding.length = 64; + probe.length = 16; + for (auto *slice : {&completed, &outstanding, &probe}) + adapter.prepare(slice, "peer@nic"); + std::vector queued{&completed}; + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.complete(&completed, IBV_WC_SUCCESS); + adapter.tick(1); + + queued = {&outstanding}; + ASSERT_EQ(adapter.gate(queued), 1u); + queued = {&probe}; + EXPECT_EQ(adapter.gate(queued), 1u); + adapter.releaseUnposted(&outstanding); + adapter.releaseUnposted(&probe); +} + +TEST(ClassicRdmaCongestionControlTest, DeferredSliceStaysOnOwnerPathQueue) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice first{}; + first.length = 64; + Transport::Slice deferred{}; + deferred.length = 64; + std::vector first_queue{&first}; + std::vector second_queue{&deferred}; + + adapter.prepare(&first, "peer@nic"); + adapter.prepare(&deferred, "peer@nic"); + const size_t first_admitted = adapter.gate(first_queue); + + ASSERT_EQ(first_admitted, 1u); + adapter.gate(second_queue); + ASSERT_EQ(second_queue.size(), 1u); + EXPECT_EQ(second_queue.front(), &deferred); +} + +TEST(ClassicRdmaCongestionControlTest, BatchPrepareSharesOneRouteWindow) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice first{}; + first.length = 64; + Transport::Slice second{}; + second.length = 64; + std::vector queued{&first, &second}; + + adapter.prepare(queued, "peer@nic"); + + EXPECT_EQ(adapter.gate(queued), 1u); + EXPECT_EQ(queued.size(), 2u); +} + +TEST(ClassicRdmaCongestionControlTest, + ActiveRouteDefersPartialReplenishmentWithoutLeakingPermits) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice active{}; + active.length = 48; + std::vector initial{&active}; + adapter.prepare(initial, "peer@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + + Transport::Slice first{}, second{}; + first.length = second.length = 16; + std::vector queued{&first, &second}; + adapter.prepare(queued, "peer@nic"); + for (int retry = 0; retry < 3; ++retry) { + EXPECT_EQ(adapter.gate(queued), 0u); + EXPECT_FALSE(first.rdma_congestion_control_permit.active()); + EXPECT_FALSE(second.rdma_congestion_control_permit.active()); + } + + adapter.releaseUnposted(&active); + EXPECT_EQ(adapter.gate(queued), 2u); + adapter.releaseUnposted(&first); + adapter.releaseUnposted(&second); +} + +TEST(ClassicRdmaCongestionControlTest, ActiveRouteAdmitsCompleteShortTail) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice active{}, tail{}; + active.length = 48; + tail.length = 16; + std::vector initial{&active}, queued{&tail}; + adapter.prepare(initial, "peer@nic"); + adapter.prepare(queued, "peer@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + EXPECT_EQ(adapter.gate(queued), 1u); + adapter.releaseUnposted(&active); + adapter.releaseUnposted(&tail); +} + +TEST(ClassicRdmaCongestionControlTest, + ActiveRouteRetainsWholeReplenishmentGroups) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 128; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice completed{}; + completed.length = 16; + std::vector growth{&completed}; + adapter.prepare(growth, "peer@nic"); + for (uint64_t epoch = 1; epoch <= 4; ++epoch) { + ASSERT_EQ(adapter.gate(growth), 1u); + adapter.complete(&completed, IBV_WC_SUCCESS); + adapter.tick(epoch); + } + Transport::Slice active{}; + active.length = 48; + std::vector initial{&active}; + adapter.prepare(initial, "peer@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + Transport::Slice slices[6]{}; + std::vector queued; + for (auto &slice : slices) { + slice.length = 16; + queued.push_back(&slice); + } + adapter.prepare(queued, "peer@nic"); + EXPECT_EQ(adapter.gate(queued), 4u); + for (size_t i = 0; i < queued.size(); ++i) { + EXPECT_EQ(queued[i]->rdma_congestion_control_permit.active(), i < 4); + adapter.releaseUnposted(queued[i]); + } + adapter.releaseUnposted(&active); +} + +TEST(ClassicRdmaCongestionControlTest, + MixedRoutesDoNotCoalescePartialAdmission) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice active{}, first{}, second{}; + active.length = 32; + first.length = 16; + second.length = 32; + std::vector initial{&active}, queued{&first, &second}; + adapter.prepare(initial, "peer@nic"); + adapter.prepare(&first, "peer@nic"); + adapter.prepare(&second, "other@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + EXPECT_EQ(adapter.gate(queued), 1u); + adapter.releaseUnposted(&active); + adapter.releaseUnposted(&first); +} + +TEST(ClassicRdmaCongestionControlTest, + ReplenishmentKeepsIndivisibleSliceGroups) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice active{}, slices[3]{}; + active.length = 16; + std::vector initial{&active}, queued; + adapter.prepare(initial, "peer@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + for (auto &slice : slices) { + slice.length = 24; + queued.push_back(&slice); + } + adapter.prepare(queued, "peer@nic"); + EXPECT_EQ(adapter.gate(queued), 2u); + for (auto *slice : queued) adapter.releaseUnposted(slice); + adapter.releaseUnposted(&active); +} + +TEST(ClassicRdmaCongestionControlTest, OffAndObserveDoNotGroupReplenishment) { + for (auto mode : {adaptive_congestion_control::Mode::kOff, + adaptive_congestion_control::Mode::kObserve}) { + adaptive_congestion_control::Config config; + config.mode = mode; + config.min_window_bytes = config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice active{}, first{}, second{}; + active.length = 48; + first.length = second.length = 16; + std::vector empty, initial{&active}; + std::vector queued{&first, &second}; + EXPECT_EQ(adapter.gate(empty), 0u); + adapter.prepare(initial, "peer@nic"); + adapter.prepare(queued, "peer@nic"); + ASSERT_EQ(adapter.gate(initial), 1u); + EXPECT_EQ(adapter.gate(queued), 2u); + adapter.releaseUnposted(&active); + adapter.releaseUnposted(&first); + adapter.releaseUnposted(&second); + } +} + +TEST(ClassicRdmaCongestionControlTest, CompletionLetsRetryReacquire) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice slice{}; + slice.length = 64; + std::vector queued{&slice}; + adapter.prepare(&slice, "peer@nic"); + ASSERT_EQ(adapter.gate(queued), 1u); + + adapter.complete(&slice, IBV_WC_RETRY_EXC_ERR); + queued = {&slice}; + EXPECT_EQ(adapter.gate(queued), 1u); +} + +TEST(ClassicRdmaCongestionControlTest, LocalHandoffRebindsRouteOwner) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + const std::string path = "peer@nic"; + for (bool batch : {false, true}) { + ClassicRdmaCongestionControl failed_adapter(config), + healthy_adapter(config); + Transport::Slice slice{}; + slice.length = 64; + failed_adapter.prepare(&slice, path); + std::vector queued{&slice}; + ASSERT_EQ(failed_adapter.gate(queued), 1u); + ASSERT_TRUE(slice.rdma_congestion_control_permit.active()); + // Local failover retains the remote NIC path on the same slice. + if (batch) { + healthy_adapter.prepare(queued, path); + } else { + healthy_adapter.prepare(&slice, path); + } + EXPECT_FALSE(slice.rdma_congestion_control_permit.active()); + EXPECT_EQ(healthy_adapter.gate(queued), 1u); + healthy_adapter.releaseUnposted(&slice); + } +} + +TEST(ClassicRdmaCongestionControlTest, + CurrentEndpointFailureDoesNotCreateAnotherHealthGate) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice failed{}; + failed.length = 64; + Transport::Slice next{}; + next.length = 64; + auto *endpoint = reinterpret_cast(uintptr_t{1}); + std::vector queued{&failed}; + + adapter.prepare(&failed, "peer@nic"); + adapter.bindEndpoint(&failed, endpoint); + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.complete(&failed, IBV_WC_FATAL_ERR); + adapter.tick(1); + + adapter.prepare(&next, "peer@nic"); + queued = {&next}; + EXPECT_EQ(adapter.gate(queued), 1u); + adapter.releaseUnposted(&next); +} + +TEST(ClassicRdmaCongestionControlTest, StaleEndpointFailureIsIgnored) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice stale{}; + stale.length = 64; + Transport::Slice next{}; + next.length = 64; + auto *old_endpoint = reinterpret_cast(uintptr_t{1}); + auto *new_endpoint = reinterpret_cast(uintptr_t{2}); + std::vector queued{&stale}; + + adapter.prepare(&stale, "peer@nic"); + adapter.bindEndpoint(&stale, old_endpoint); + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.bindEndpoint(&stale, new_endpoint); + adapter.complete(&stale, IBV_WC_FATAL_ERR); + adapter.tick(1); + + adapter.prepare(&next, "peer@nic"); + queued = {&next}; + EXPECT_EQ(adapter.gate(queued), 1u); +} + +TEST(ClassicRdmaCongestionControlTest, + ReusedEndpointAddressDoesNotReviveOldFailure) { + adaptive_congestion_control::Config config; + config.mode = adaptive_congestion_control::Mode::kEnforce; + config.min_window_bytes = 64; + config.max_window_bytes = 64; + ClassicRdmaCongestionControl adapter(config); + Transport::Slice stale{}; + stale.length = 64; + Transport::Slice replacement{}; + replacement.length = 64; + Transport::Slice next{}; + next.length = 64; + auto *reused_endpoint = reinterpret_cast(uintptr_t{1}); + std::vector queued{&stale}; + + adapter.prepare(&stale, "peer@nic"); + adapter.bindEndpoint(&stale, reused_endpoint); + ASSERT_EQ(adapter.gate(queued), 1u); + adapter.retireEndpoint("peer@nic", reused_endpoint); + adapter.prepare(&replacement, "peer@nic"); + adapter.bindEndpoint(&replacement, reused_endpoint); + + adapter.complete(&stale, IBV_WC_FATAL_ERR); + adapter.tick(1); + + adapter.prepare(&next, "peer@nic"); + queued = {&next}; + EXPECT_EQ(adapter.gate(queued), 1u); +} + +} // namespace +} // namespace mooncake diff --git a/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cmake.in b/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cmake.in new file mode 100644 index 0000000000..f2092ac212 --- /dev/null +++ b/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cmake.in @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.16) + +set(prefix + "@CMAKE_CURRENT_BINARY_DIR@/installed_congestion_control_header_prefix") +file(REMOVE_RECURSE "${prefix}") + +get_filename_component(library_dir "${TRANSFER_ENGINE_LIBRARY}" DIRECTORY) +if("@BUILD_SHARED_LIBS@") + set(consumer_flags "${TRANSFER_ENGINE_LIBRARY}" "-Wl,-rpath,${library_dir}" + -o "${prefix}/consumer") +else() + set(consumer_flags -fsyntax-only) +endif() + +set(expected_adaptive_control 0) +if("@MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL@") + set(expected_adaptive_control 1) +endif() + +execute_process( + COMMAND "@CMAKE_COMMAND@" "-DCMAKE_INSTALL_PREFIX=${prefix}" + -P "@CMAKE_CURRENT_BINARY_DIR@/../include/cmake_install.cmake" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_output + ERROR_VARIABLE install_error) +if(NOT install_result EQUAL 0) + message( + FATAL_ERROR + "Installing the congestion-control build failed:\n${install_output}${install_error}" + ) +endif() + +execute_process( + COMMAND + "@CMAKE_CXX_COMPILER@" -std=c++20 + -DMC_EXPECT_ADAPTIVE_CONTROL=${expected_adaptive_control} -I${prefix}/include + "@CMAKE_CURRENT_SOURCE_DIR@/installed_congestion_control_header_test.cpp" + ${consumer_flags} + RESULT_VARIABLE compile_result + OUTPUT_VARIABLE compile_output + ERROR_VARIABLE compile_error) +if(NOT compile_result EQUAL 0) + message( + FATAL_ERROR + "Installed congestion-control consumer failed to compile or link:\n${compile_output}${compile_error}" + ) +endif() + +if(NOT "@BUILD_SHARED_LIBS@") + return() +endif() + +execute_process( + COMMAND "${prefix}/consumer" + RESULT_VARIABLE run_result + OUTPUT_VARIABLE run_output + ERROR_VARIABLE run_error) +if(NOT run_result EQUAL 0) + message(FATAL_ERROR + "Installed congestion-control consumer failed to run:\n${run_output}${run_error}") +endif() diff --git a/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cpp b/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cpp new file mode 100644 index 0000000000..f5d0a8f230 --- /dev/null +++ b/mooncake-transfer-engine/tests/installed_congestion_control_header_test.cpp @@ -0,0 +1,32 @@ +// Copyright 2026 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#if defined(MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL) != \ + MC_EXPECT_ADAPTIVE_CONTROL +#error "Installed headers do not match the library build configuration" +#endif + +struct SliceCacheConsumer : mooncake::Transport { + using Transport::getSliceCache; +}; + +int main() { + auto &cache = SliceCacheConsumer::getSliceCache(); + auto *slice = cache.allocate(); + slice->length = 64; + cache.deallocate(slice); + return 0; +} diff --git a/mooncake-transfer-engine/tests/rdma_endpoint_state_test.cpp b/mooncake-transfer-engine/tests/rdma_endpoint_state_test.cpp index ac02a5b0a1..5c2e750683 100644 --- a/mooncake-transfer-engine/tests/rdma_endpoint_state_test.cpp +++ b/mooncake-transfer-engine/tests/rdma_endpoint_state_test.cpp @@ -332,7 +332,6 @@ void initFakePeer(FakeRdmaPeer &peer, const std::string &server_name, void queueHandshakeSlice(FakeRdmaPeer &peer, const std::string &peer_nic_path, Transport::Slice &slice, Transport::TransferTask &task) { - slice = Transport::Slice(); task = Transport::TransferTask(); slice.task = &task; slice.status = Transport::Slice::PENDING; @@ -388,6 +387,22 @@ TEST_F(RdmaEndPointStateTest, ConnectedIsReadyToSend) { EXPECT_TRUE(endpoint_->readyToSend()); } +#ifdef MOONCAKE_ENABLE_ADAPTIVE_CONGESTION_CONTROL +TEST_F(RdmaEndPointStateTest, LimitedSubmitLeavesDeferredSuffixQueued) { + Transport::Slice first{}; + Transport::Slice deferred{}; + std::vector queued{&first, &deferred}; + std::vector failed; + + endpoint_->submitPostSendLimited(queued, failed, 1); + + ASSERT_EQ(failed.size(), 1u); + EXPECT_EQ(failed.front(), &first); + ASSERT_EQ(queued.size(), 1u); + EXPECT_EQ(queued.front(), &deferred); +} +#endif + TEST_F(RdmaEndPointStateTest, ReadyAckTimeoutOnlyAppliesToWaitingState) { RdmaEndPointTestPeer::setReadyWaitStartTs(*endpoint_, 1); @@ -663,8 +678,8 @@ TEST(RdmaEndpointLifecycleGateTest, ASSERT_NO_FATAL_FAILURE(installZeroQpEndpoint(peer_a, a_to_b)); ASSERT_NO_FATAL_FAILURE(installZeroQpEndpoint(peer_b, b_to_a)); - Transport::Slice slice_a; - Transport::Slice slice_b; + Transport::Slice slice_a{}; + Transport::Slice slice_b{}; Transport::TransferTask task_a; Transport::TransferTask task_b; queueHandshakeSlice(peer_a, a_to_b, slice_a, task_a); @@ -716,7 +731,7 @@ TEST(RdmaEndpointLifecycleGateTest, ActiveHandshakeWaitDrainsOwnerCq) { RdmaContextTestPeer::clearNativeCompletionQueue(*peer.context, 0); peer.context->cqOutstandingCount(0)->store(1, std::memory_order_relaxed); - Transport::Slice slice; + Transport::Slice slice{}; Transport::TransferTask task; queueHandshakeSlice(peer, peer_nic_path, slice, task); slice.rdma.max_retry_cnt = 1; diff --git a/scripts/e2e/python/test_hicache_storage_mooncake_backend.py b/scripts/e2e/python/test_hicache_storage_mooncake_backend.py index f736b60c45..0f19fe03d5 100644 --- a/scripts/e2e/python/test_hicache_storage_mooncake_backend.py +++ b/scripts/e2e/python/test_hicache_storage_mooncake_backend.py @@ -8,7 +8,9 @@ """ import os +import socket import subprocess +import sys import time import unittest @@ -72,128 +74,143 @@ def _start_mooncake_services(cls): f"Using master port: {cls.mooncake_master_port}, metadata port: {cls.mooncake_metadata_port}" ) - # Start metadata service with configurable port - try: - # Start metadata server with port configuration - cls.metadata_service_process = subprocess.Popen( - [ - "python3", - "-m", - "mooncake.http_metadata_server", - "--port", - str(cls.mooncake_metadata_port), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=os.setsid, # Create new process group - ) - print( - f"Mooncake metadata service started on port {cls.mooncake_metadata_port}" - ) - except (FileNotFoundError, subprocess.SubprocessError) as e: - print(f"Warning: Could not start Mooncake metadata service: {e}") - cls.metadata_service_process = None + # The metadata service is bound to loopback on purpose: it refuses an + # unauthenticated non-loopback bind (mooncake #4472), and every client + # used by this test runs on the same host. Child output is inherited so + # a failed launch shows up in the CI log instead of being swallowed by + # an unread pipe. + cls.metadata_service_process = cls._spawn_service( + "metadata", + [ + sys.executable, + "-m", + "mooncake.http_metadata_server", + "--host", + "127.0.0.1", + "--port", + str(cls.mooncake_metadata_port), + ], + ) # Start master service with configurable port + cls.master_service_process = cls._spawn_service( + "master", + ["mooncake_master", "--port", str(cls.mooncake_master_port)], + ) + + # Wait for services to be ready instead of fixed sleep + cls._wait_for_mooncake_services_ready() + + @classmethod + def _spawn_service(cls, name, command): + """Start a Mooncake service process, returning None if it cannot be spawned""" try: - # Start master server with port configuration - cls.master_service_process = subprocess.Popen( - ["mooncake_master", "--port", str(cls.mooncake_master_port)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + process = subprocess.Popen( + command, preexec_fn=os.setsid, # Create new process group ) - print(f"Mooncake master service started on port {cls.mooncake_master_port}") except (FileNotFoundError, subprocess.SubprocessError) as e: - print(f"Warning: Could not start Mooncake master service: {e}") - cls.master_service_process = None - - # Wait for services to be ready instead of fixed sleep - cls._wait_for_mooncake_services_ready() + print(f"Warning: Could not start Mooncake {name} service: {e}") + return None + print(f"Mooncake {name} service started (pid={process.pid})") + return process @classmethod - def _wait_for_mooncake_services_ready(cls, timeout: int = 30) -> bool: - """Wait for Mooncake services to be ready by checking their endpoints""" + def _wait_for_mooncake_services_ready(cls, timeout: int = 60) -> bool: + """Wait for Mooncake services to be ready by checking their endpoints + + Fails fast when a service exits, so a broken launch is reported here + instead of surfacing later as "Failed to setup Mooncake store". + """ print("Waiting for Mooncake services to be ready...") - start_time = time.time() - services_ready = False + deadline = time.time() + timeout + metadata_ready = False + master_ready = False - while time.time() - start_time < timeout: - try: - # Check metadata service - metadata_ready = False - if ( - cls.metadata_service_process - and cls.metadata_service_process.poll() is None - ): - try: - # Try to connect to the metadata service - metadata_url = ( - f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata" - ) - response = requests.get(metadata_url, timeout=2) - if response.status_code == 200: - metadata_ready = True - print("Mooncake metadata service is ready") - except (requests.RequestException, ConnectionError): - # Service might not be fully started yet - pass - - # Check master service (if it has a health endpoint) - master_ready = False - if ( - cls.master_service_process - and cls.master_service_process.poll() is None - ): - # For now, we'll assume master service is ready if process is running - # and it's been a few seconds since startup - if ( - time.time() - start_time > 5 - ): # Give master service time to initialize - master_ready = True - print("Mooncake master service is ready") - - # Both services should be ready - if metadata_ready and master_ready: - services_ready = True - print("All Mooncake services are ready") - break - - except Exception as e: - print(f"Error checking service readiness: {e}") - - time.sleep(2) - - if not services_ready: - print( - "Warning: Mooncake services may not be fully ready, continuing anyway..." + while time.time() < deadline: + cls._ensure_service_running("metadata", cls.metadata_service_process) + cls._ensure_service_running("master", cls.master_service_process) + + if not metadata_ready and cls._probe_metadata_endpoint(): + metadata_ready = True + print("Mooncake metadata service is ready") + + if not master_ready and cls._probe_tcp_port(cls.mooncake_master_port): + master_ready = True + print("Mooncake master service is ready") + + # Both services should be ready + if metadata_ready and master_ready: + print("All Mooncake services are ready") + return True + + time.sleep(1) + + raise RuntimeError( + f"Mooncake services were not ready within {timeout}s " + f"(metadata port {cls.mooncake_metadata_port}: " + f"{'ready' if metadata_ready else 'not ready'}, " + f"master port {cls.mooncake_master_port}: " + f"{'ready' if master_ready else 'not ready'})" + ) + + @staticmethod + def _ensure_service_running(name, process): + """Raise if a Mooncake service process has already exited""" + if process is None: + raise RuntimeError(f"Mooncake {name} service was never started") + return_code = process.poll() + if return_code is not None: + raise RuntimeError( + f"Mooncake {name} service exited with code {return_code} before " + "it became ready; check the service output above" ) - return services_ready + @classmethod + def _probe_metadata_endpoint(cls) -> bool: + """Return True once the metadata service answers requests""" + # Any HTTP status means the listener is up: a missing key is answered + # with 400/404 and a token-protected server with 401. + url = ( + f"http://127.0.0.1:{cls.mooncake_metadata_port}" + "/metadata?key=__hicache_probe__" + ) + try: + requests.get(url, timeout=2) + except (requests.RequestException, ConnectionError): + # Service might not be fully started yet + return False + return True + + @classmethod + def _probe_tcp_port(cls, port: int, host: str = "127.0.0.1") -> bool: + """Return True when something accepts connections on host:port""" + try: + with socket.create_connection((host, port), timeout=2): + return True + except OSError: + return False @classmethod def _stop_mooncake_services(cls): """Stop Mooncake services""" print("Stopping Mooncake services...") - # Stop metadata service - if hasattr(cls, "metadata_service_process") and cls.metadata_service_process: - try: - os.killpg(os.getpgid(cls.metadata_service_process.pid), 9) - cls.metadata_service_process.wait(timeout=5) - print("Mooncake metadata service stopped") - except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e: - print(f"Warning: Could not stop Mooncake metadata service: {e}") - - # Stop master service - if hasattr(cls, "master_service_process") and cls.master_service_process: + for name in ("metadata", "master"): + process = getattr(cls, f"{name}_service_process", None) + if process is None: + continue + if process.poll() is not None: + # Already gone, e.g. it exited during readiness detection. + print(f"Mooncake {name} service already exited") + continue try: - os.killpg(os.getpgid(cls.master_service_process.pid), 9) - cls.master_service_process.wait(timeout=5) - print("Mooncake master service stopped") + os.killpg(os.getpgid(process.pid), 9) + process.wait(timeout=5) + print(f"Mooncake {name} service stopped") except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e: - print(f"Warning: Could not stop Mooncake master service: {e}") + print(f"Warning: Could not stop Mooncake {name} service: {e}") @classmethod def _get_additional_server_args_and_env(cls):