From 10027777f619b5fee6837ee54fbfb9d5b1c968cb Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 7 Oct 2026 21:46:10 +0300 Subject: [PATCH 1/2] Match unquoted anchor attributes in the link checker Minified pages such as the WHATWG HTML spec and the Kafka docs write id attributes without quotes, so the checker reported their anchors as missing. Accept double-quoted, single-quoted, and unquoted values, and skip suffixed attributes such as data-id. Co-Authored-By: Claude Opus 5.5 --- scripts/check_links.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check_links.py b/scripts/check_links.py index e26371d..7eb594f 100644 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -16,7 +16,8 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent TOPICS = ROOT / "docs" / "topics" LINK = re.compile(r"\]\((https?://[^)\s]+)\)") -ANCHOR = re.compile(r'(?:id|name)="([^"]+)"') +# Minified HTML (the WHATWG spec, Hugo sites) leaves attribute values unquoted. +ANCHOR = re.compile(r"""(?]+))""") # Wikipedia rejects requests without a descriptive User-Agent. USER_AGENT = "interview-briefs-link-checker (+https://github.com/pinebit/interviews)" @@ -29,7 +30,7 @@ def fetch(url): body = response.read().decode("utf-8", "replace") except (OSError, http.client.HTTPException) as e: # URLError and timeouts are OSErrors return str(e), set() - return None, {html.unescape(a) for a in ANCHOR.findall(body)} + return None, {html.unescape("".join(groups)) for groups in ANCHOR.findall(body)} def main(): From 8023b4a1c39e69c1ff0650c95ac8d963bbbb8441 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 7 Oct 2026 21:46:11 +0300 Subject: [PATCH 2/2] Link system and distributed brief terms and remove bold Link terms in the system design and distributed systems briefs to official docs, RFCs, papers, and Wikipedia, and link concepts owned by another brief to that file. Remove bold from both briefs per the format rules. Reword the Kafka log compaction bullet: compaction runs in the background and keeps at least the latest record per key. Co-Authored-By: Claude Opus 5.5 --- docs/topics/distributed.md | 182 ++++++++++++++++---------------- docs/topics/system.md | 206 ++++++++++++++++++------------------- 2 files changed, 194 insertions(+), 194 deletions(-) diff --git a/docs/topics/distributed.md b/docs/topics/distributed.md index 7051b63..30b68fc 100644 --- a/docs/topics/distributed.md +++ b/docs/topics/distributed.md @@ -6,234 +6,234 @@ Key distributed-systems building blocks and trade-offs, grouped by subtopic. ### CAP and PACELC -- During a **network partition**, choose Consistency (reject or block — etcd, ZooKeeper) or Availability (serve possibly stale data — Cassandra, DynamoDB default). -- CAP's "C" is linearizability, not ACID's "C" (application invariants). -- **PACELC**: if Partitioned, A vs C; Else, **Latency vs Consistency** — the everyday cost when nothing is broken. +- During a [network partition](https://en.wikipedia.org/wiki/Network_partition), choose Consistency (reject or block — [etcd](https://etcd.io/docs/), [ZooKeeper](https://zookeeper.apache.org/)) or Availability (serve possibly stale data — [Cassandra](https://cassandra.apache.org/doc/latest/cassandra/architecture/dynamo.html#tunable-consistency), [DynamoDB default](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html)). +- [CAP](https://en.wikipedia.org/wiki/CAP_theorem)'s "C" is [linearizability](https://jepsen.io/consistency/models/linearizable), not [ACID's "C"](https://en.wikipedia.org/wiki/ACID#Consistency) (application invariants). +- [PACELC](https://en.wikipedia.org/wiki/PACELC_design_principle): if Partitioned, A vs C; Else, Latency vs Consistency — the everyday cost when nothing is broken. ### FLP and Two Generals -- **FLP**: no deterministic algorithm guarantees consensus in a fully asynchronous system with even one crash. Real systems use timeouts and give up liveness, never safety, in bad periods. -- **Two Generals**: over a lossy channel the last ack can always be lost, so two parties can never be sure they agree — why exactly-once *delivery* is impossible. +- [FLP](https://groups.csail.mit.edu/tds/papers/Lynch/jacm85.pdf): no deterministic algorithm guarantees consensus in a fully asynchronous system with even one crash. Real systems use timeouts and give up [liveness, never safety](https://en.wikipedia.org/wiki/Safety_and_liveness_properties), in bad periods. +- [Two Generals](https://en.wikipedia.org/wiki/Two_Generals%27_Problem): over a lossy channel the last ack can always be lost, so two parties can never be sure they agree — why exactly-once *delivery* is impossible. ### Byzantine faults -- Under partial synchrony, tolerating `f` nodes that lie needs **`3f + 1`** nodes, vs `2f + 1` for crash faults; the classic algorithm is **PBFT**. -- The bound depends on the model: with signed messages and a synchronous network, fewer nodes suffice. +- Under [partial synchrony](https://groups.csail.mit.edu/tds/papers/Lynch/jacm88.pdf), tolerating `f` [nodes that lie](https://en.wikipedia.org/wiki/Byzantine_fault) needs `3f + 1` nodes, vs `2f + 1` for crash faults; the classic algorithm is [PBFT](https://www.usenix.org/conference/osdi-99/practical-byzantine-fault-tolerance). +- The bound depends on the model: with [signed messages](https://lamport.azurewebsites.net/pubs/byz.pdf) and a synchronous network, fewer nodes suffice. - Most internal systems assume crash faults only; blockchains must assume Byzantine ones. ## Consistency models ### Consistency hierarchy -- Single-object models, strongest first: linearizable > sequential > causal; eventual only promises convergence, which causal alone doesn't guarantee. -- **Linearizability**: single-object recency — once a write completes, every later read (in real time) sees it or a newer value. -- **Serializability**: multi-object transaction isolation — equivalent to *some* serial order, not necessarily real-time order. -- **Strict serializability** adds real-time order (Spanner, FoundationDB); CockroachDB is serializable but linearizable only per key. +- [Single-object models](https://jepsen.io/consistency), strongest first: [linearizable](https://jepsen.io/consistency/models/linearizable) > [sequential](https://jepsen.io/consistency/models/sequential) > [causal](https://jepsen.io/consistency/models/causal); [eventual](https://en.wikipedia.org/wiki/Eventual_consistency) only promises convergence, which causal alone doesn't guarantee. +- Linearizability: single-object recency — once a write completes, every later read (in real time) sees it or a newer value. +- [Serializability](https://jepsen.io/consistency/models/serializable): multi-object transaction isolation — equivalent to *some* serial order, not necessarily real-time order. +- [Strict serializability](https://jepsen.io/consistency/models/strict-serializable) adds real-time order ([Spanner](https://cloud.google.com/spanner/docs/true-time-external-consistency), [FoundationDB](https://apple.github.io/foundationdb/consistency.html)); [CockroachDB](https://www.cockroachlabs.com/blog/consistency-model/) is serializable but linearizable only per key. ### Session guarantees -- **Read-your-writes**, **monotonic reads**, **monotonic writes**, writes-follow-reads — per-client guarantees layered on eventual consistency. +- [Read-your-writes](https://jepsen.io/consistency/models/read-your-writes), [monotonic reads](https://jepsen.io/consistency/models/monotonic-reads), [monotonic writes](https://jepsen.io/consistency/models/monotonic-writes), [writes-follow-reads](https://jepsen.io/consistency/models/writes-follow-reads) — per-client guarantees layered on eventual consistency. - Typical implementations: sticky routing to one replica, or reading from a replica that has caught up to the client's last-seen version. ## Replication ### Single-leader replication -- All writes go to one leader; **sync** followers are durable but add latency, **async** followers are fast but lose recent writes on failover. -- Semi-sync waits for at least one follower. -- **Replication lag** breaks read-your-writes: a user can't see their own update right after writing. +- All writes go to one leader; [sync](https://www.postgresql.org/docs/current/warm-standby.html#SYNCHRONOUS-REPLICATION) followers are durable but add latency, async followers are fast but lose recent writes on failover. +- [Semi-sync](https://mariadb.com/docs/server/ha-and-performance/standard-replication/semisynchronous-replication) waits for at least one follower. +- Replication lag breaks read-your-writes: a user can't see their own update right after writing. ### Multi-leader and leaderless -- **Multi-leader**: several nodes accept writes (one per region) — needs conflict resolution. -- **Leaderless** (Dynamo-style): the client writes to and reads from several replicas, tuned with quorums. +- [Multi-leader](https://en.wikipedia.org/wiki/Multi-master_replication): several nodes accept writes (one per region) — needs conflict resolution. +- Leaderless ([Dynamo-style](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf)): the client writes to and reads from several replicas, tuned with quorums. ### Quorums -- **`W + R > N`** makes read and write sets overlap, e.g. N=3, W=2, R=2. -- **Sloppy quorum + hinted handoff**: during an outage, accept writes on stand-in nodes and hand them back later — overlap is no longer guaranteed. +- [`W + R > N`](https://en.wikipedia.org/wiki/Quorum_%28distributed_computing%29) makes read and write sets overlap, e.g. N=3, W=2, R=2. +- [Sloppy quorum + hinted handoff](https://www.allthingsdistributed.com/files/amazon-dynamo-sosp2007.pdf): during an outage, accept writes on stand-in nodes and hand them back later — overlap is no longer guaranteed. ### Replica repair -- **Read repair** fixes stale replicas seen during a read. -- **Anti-entropy** compares **Merkle trees** in the background so only differing key ranges transfer. +- [Read repair](https://cassandra.apache.org/doc/latest/cassandra/managing/operating/read_repair.html) fixes stale replicas seen during a read. +- [Anti-entropy](https://cassandra.apache.org/doc/latest/cassandra/managing/operating/repair.html) compares [Merkle trees](https://en.wikipedia.org/wiki/Merkle_tree) in the background so only differing key ranges transfer. ## Partitioning ### Hash vs range partitioning -- **Hash** spreads load evenly but loses range scans; **range** keeps keys sorted but sequential keys (timestamps) create hot spots. -- Compound keys combine both: hash the first component to pick a partition, keep rows sorted by the rest (Cassandra partition + clustering key). +- Hash spreads load evenly but loses range scans; range keeps keys sorted but sequential keys (timestamps) create [hot spots](https://en.wikipedia.org/wiki/Partition_%28database%29). +- Compound keys combine both: hash the first component to pick a partition, keep rows sorted by the rest ([Cassandra partition + clustering key](https://cassandra.apache.org/doc/latest/cassandra/developing/cql/ddl.html#primary-key)). ### Consistent and rendezvous hashing -- **Consistent hashing**: nodes and keys on a ring; a key belongs to the next node clockwise, so adding a node moves ~**`1/N`** of keys. **Virtual nodes** smooth the load. -- Rendezvous hashing: owner = node with the highest `hash(node, key)` — no ring, same minimal movement. +- [Consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing): nodes and keys on a ring; a key belongs to the next node clockwise, so adding a node moves ~`1/N` of keys. [Virtual nodes](https://cassandra.apache.org/doc/latest/cassandra/architecture/dynamo.html) smooth the load. +- [Rendezvous hashing](https://en.wikipedia.org/wiki/Rendezvous_hashing): owner = node with the highest `hash(node, key)` — no ring, same minimal movement. ### Hot keys and secondary indexes -- A single hot key isn't fixed by more partitions — **salt** it into sub-keys and merge on read. -- **Local** secondary indexes (per partition) need scatter-gather unless the query fixes the partition key; **global** ones are partitioned by the indexed value, so a write touches several partitions — asynchronous in DynamoDB, transactional in Spanner. +- A single hot key isn't fixed by more partitions — [salt](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/bp-partition-key-sharding.html) it into sub-keys and merge on read. +- Local [secondary indexes](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html) (per partition) need scatter-gather unless the query fixes the partition key; global ones are partitioned by the indexed value, so a write touches several partitions — asynchronous in DynamoDB, transactional in [Spanner](https://cloud.google.com/spanner/docs/secondary-indexes). ## Consensus and coordination ### Quorum sizing -- Consensus needs a **majority** (`⌊N/2⌋ + 1`): `2f + 1` nodes tolerate `f` crashes — hence 3 or 5 nodes. +- [Consensus](https://en.wikipedia.org/wiki/Consensus_%28computer_science%29) needs a majority (`⌊N/2⌋ + 1`): `2f + 1` nodes tolerate `f` crashes — hence 3 or 5 nodes. - An even size adds no tolerance: 4 nodes still tolerate only 1 failure. ### Raft -- **Leader election**: randomized timeouts avoid split votes; one vote per node per term. -- Log replication: a leader commits an entry once a **majority** stores it — only for current-term entries; older ones commit indirectly with them. -- **Election restriction**: only a candidate with an up-to-date log can win, so committed entries survive. +- [Leader election](https://raft.github.io/raft.pdf): randomized timeouts avoid split votes; one vote per node per term. +- Log replication: a leader commits an entry once a majority stores it — only for current-term entries; older ones commit indirectly with them. +- Election restriction: only a candidate with an up-to-date log can win, so committed entries survive. ### Raft reads and membership -- A leader can't just read locally — a deposed leader may not know it yet. **ReadIndex** confirms leadership with a heartbeat round; **lease reads** skip it but rely on bounded clock drift. -- Membership changes go one server at a time (or via joint consensus); **snapshots** truncate the log. +- A leader can't just read locally — a deposed leader may not know it yet. [ReadIndex](https://web.stanford.edu/~ouster/cgi-bin/papers/OngaroPhD.pdf) confirms leadership with a heartbeat round; lease reads skip it but rely on bounded clock drift. +- Membership changes go one server at a time (or via [joint consensus](https://raft.github.io/raft.pdf)); snapshots truncate the log. ### Paxos -- Two phases across proposers and acceptors: **prepare/promise**, then **accept/accepted**. -- Same safety as Raft but harder to implement; **Multi-Paxos** adds a stable leader to skip the first phase. +- [Two phases](https://lamport.azurewebsites.net/pubs/paxos-simple.pdf) across proposers and acceptors: prepare/promise, then accept/accepted. +- Same safety as Raft but harder to implement; [Multi-Paxos](https://en.wikipedia.org/wiki/Paxos_%28computer_science%29#Multi-Paxos) adds a stable leader to skip the first phase. ### Leases and fencing tokens -- A lease is a lock with a TTL; if the holder dies, it expires and someone else takes over. -- A holder paused past expiry (long GC) still thinks it's the leader — **split brain**. **Fencing tokens** (increasing per lease, checked by the storage) reject its stale writes. -- **Redlock** depends on timing assumptions; use a consensus store (etcd, ZooKeeper) when correctness matters. +- A [lease](https://en.wikipedia.org/wiki/Lease_%28computer_science%29) is a lock with a TTL; if the holder dies, it expires and someone else takes over. +- A holder paused past expiry (long GC) still thinks it's the leader — [split brain](https://en.wikipedia.org/wiki/Split-brain_%28computing%29). [Fencing tokens](https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html) (increasing per lease, checked by the storage) reject its stale writes. +- [Redlock](https://redis.io/docs/latest/develop/use/patterns/distributed-locks/) depends on timing assumptions; use a consensus store ([etcd](https://etcd.io/docs/), [ZooKeeper](https://zookeeper.apache.org/)) when correctness matters. ## Time, ordering, and IDs ### Physical clocks and TrueTime -- Clocks drift and NTP can step them backward, so wall-clock timestamps can't order events across nodes — **last-write-wins** silently loses data. -- Spanner's **TrueTime** returns an uncertainty interval; a commit waits out the uncertainty (~ms) before becoming visible, giving external consistency. -- Measure durations, timeouts, and leases with the **monotonic clock** (`CLOCK_MONOTONIC`, Go's `time.Since`, `performance.now()`), which never jumps; wall-clock time steps on NTP corrections, manual changes, and VM migration. +- Clocks drift and [NTP](https://en.wikipedia.org/wiki/Network_Time_Protocol) can step them backward, so wall-clock timestamps can't order events across nodes — last-write-wins silently loses data. +- Spanner's [TrueTime](https://cloud.google.com/spanner/docs/true-time-external-consistency) returns an uncertainty interval; a commit waits out the uncertainty (~ms) before becoming visible, giving external consistency. +- Measure durations, timeouts, and leases with the monotonic clock ([`CLOCK_MONOTONIC`](https://man7.org/linux/man-pages/man2/clock_gettime.2.html), Go's [`time.Since`](https://pkg.go.dev/time#hdr-Monotonic_Clocks), [`performance.now()`](https://developer.mozilla.org/en-US/docs/Web/API/Performance/now)), which never jumps; wall-clock time steps on NTP corrections, manual changes, and VM migration. ### Logical clocks -- **Lamport clock**: increment per event; on receive, `max(local, received) + 1`. With node ID as tie-breaker, a total order consistent with causality, but can't detect concurrency. -- **Vector clock**: one counter per node; comparing two vectors tells **happened-before** from concurrent. -- Hybrid logical clock (HLC): physical time + logical counter — causal and close to wall time (CockroachDB). +- [Lamport clock](https://en.wikipedia.org/wiki/Lamport_timestamp): increment per event; on receive, `max(local, received) + 1`. With node ID as tie-breaker, a total order consistent with causality, but can't detect concurrency. +- [Vector clock](https://en.wikipedia.org/wiki/Vector_clock): one counter per node; comparing two vectors tells [happened-before](https://en.wikipedia.org/wiki/Happened-before) from concurrent. +- [Hybrid logical clock](https://cse.buffalo.edu/tech-reports/2014-04.pdf) (HLC): physical time + logical counter — causal and close to wall time ([CockroachDB](https://www.cockroachlabs.com/docs/stable/architecture/transaction-layer)). ### Distributed IDs | Scheme | Layout | Note | |---|---|---| -| **Snowflake** | 41-bit ms timestamp + 10-bit machine + 12-bit sequence | 4,096 IDs/ms/machine, fits `bigint` | -| **UUIDv7** | 48-bit ms timestamp + random | time-ordered, good B-tree locality | -| **ULID** | timestamp + random, 26-char string | sortable text form | -| UUIDv4 | 122 random bits | scatters index inserts | +| [Snowflake](https://en.wikipedia.org/wiki/Snowflake_ID) | 41-bit ms timestamp + 10-bit machine + 12-bit sequence | 4,096 IDs/ms/machine, fits `bigint` | +| [UUIDv7](https://www.rfc-editor.org/rfc/rfc9562#name-uuid-version-7) | 48-bit ms timestamp + random | time-ordered, good B-tree locality | +| [ULID](https://github.com/ulid/spec) | timestamp + random, 26-char string | sortable text form | +| [UUIDv4](https://www.rfc-editor.org/rfc/rfc9562#name-uuid-version-4) | 122 random bits | scatters index inserts | ## Transactions across services ### Two-phase commit -- Coordinator asks participants to prepare (vote and hold locks), then commit or abort. -- Atomic but **blocking**: if the coordinator dies after prepare, participants hold locks until it returns. **3PC** avoids that only without partitions, so it's rarely used. +- [Coordinator](https://en.wikipedia.org/wiki/Two-phase_commit_protocol) asks participants to prepare (vote and hold locks), then commit or abort. +- Atomic but blocking: if the coordinator dies after prepare, participants hold locks until it returns. [3PC](https://en.wikipedia.org/wiki/Three-phase_commit_protocol) avoids that only without partitions, so it's rarely used. ### Sagas -- A chain of local transactions, each with a **compensating action**; eventual consistency, **no isolation** (others see intermediate states). -- **Orchestration** (a central coordinator) vs choreography (services react to each other's events). +- A [chain of local transactions](https://microservices.io/patterns/data/saga.html), each with a [compensating action](https://learn.microsoft.com/en-us/azure/architecture/patterns/compensating-transaction); eventual consistency, no isolation (others see intermediate states). +- Orchestration (a central coordinator) vs choreography (services react to each other's events). ### Delivery guarantees -- **At-most-once**: ack (or commit the offset) before processing — a crash mid-processing loses the message. -- **At-least-once**: ack after processing — a crash between the effect and the ack redelivers it, so handlers see duplicates. -- Exactly-once *delivery* is impossible; an **exactly-once effect** = at-least-once + idempotent handling, or one transaction covering both the effect and the offset. +- [At-most-once](https://kafka.apache.org/43/design/design/#message-delivery-semantics): ack (or commit the offset) before processing — a crash mid-processing loses the message. +- At-least-once: ack after processing — a crash between the effect and the ack redelivers it, so handlers see duplicates. +- Exactly-once *delivery* is impossible; an exactly-once effect = at-least-once + [idempotent](https://en.wikipedia.org/wiki/Idempotence#Computer_science_meaning) handling, or one transaction covering both the effect and the offset. - Publish events atomically with the local write via the transactional outbox; dedupe on the consumer via an inbox — both in [backend.md](backend.md). ## Conflict resolution ### Last-write-wins and siblings -- **LWW** is simplest but silently drops concurrent writes and depends on clocks. -- **Version vectors** detect concurrent writes and keep **siblings** for the app or user to merge. +- LWW is simplest but silently drops concurrent writes and depends on clocks. +- [Version vectors](https://en.wikipedia.org/wiki/Version_vector) detect concurrent writes and keep [siblings](https://docs.riak.com/riak/kv/latest/learn/concepts/causal-context/index.html#siblings) for the app or user to merge. ### CRDTs -- State-based: merges are **commutative, associative, idempotent**, so replicas converge without coordination. -- Op-based: replicas broadcast operations that must commute; delivery must be exactly-once and usually causal. -- **G-Counter** (per-node counts, summed), PN-Counter (two G-Counters), **OR-Set** (add wins over concurrent remove), LWW-Register. +- [State-based](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#State-based_CRDTs): merges are commutative, associative, idempotent, so replicas converge without coordination. +- [Op-based](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#Operation-based_CRDTs): replicas broadcast operations that must commute; delivery must be exactly-once and usually causal. +- [G-Counter](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type#G-Counter_%28Grow-only_Counter%29) (per-node counts, summed), PN-Counter (two G-Counters), OR-Set (add wins over concurrent remove), LWW-Register. ### OT vs CRDT for collaborative editing -- **OT** transforms concurrent operations against each other, usually via a central server (Google Docs). -- **CRDTs** merge peer-to-peer with larger metadata (Automerge, Yjs). +- [OT](https://en.wikipedia.org/wiki/Operational_transformation) transforms concurrent operations against each other, usually via a central server (Google Docs). +- [CRDTs](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type) merge peer-to-peer with larger metadata ([Automerge](https://automerge.org/), [Yjs](https://docs.yjs.dev/)). ## Failure handling ### Timeouts and retries - Put a timeout on every remote call. -- Retry only **idempotent** operations, with **exponential backoff + jitter** so retries don't synchronize. -- A **retry budget** (e.g. retries ≤ 10% of requests) stops retries from multiplying load during an outage. +- Retry only idempotent operations, with [exponential backoff + jitter](https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/) so retries don't synchronize. +- A [retry budget](https://sre.google/sre-book/handling-overload/) (e.g. retries ≤ 10% of requests) stops retries from multiplying load during an outage. ### Circuit breakers -- **Closed** (normal) → **open** after an error threshold (fail fast) → **half-open** (let a probe through, then close or reopen). +- [Closed](https://martinfowler.com/bliki/CircuitBreaker.html) (normal) → open after an error threshold (fail fast) → half-open (let a probe through, then close or reopen). - Keep one breaker per dependency and pair it with a fallback (cached or default response). ### Bulkheads, backpressure, load shedding -- **Bulkheads** give each dependency its own pool (threads, connections), so one slow dependency can't exhaust everything. -- **Backpressure** slows producers; **load shedding** rejects excess work early rather than queueing it without bound. +- [Bulkheads](https://learn.microsoft.com/en-us/azure/architecture/patterns/bulkhead) give each dependency its own pool (threads, connections), so one slow dependency can't exhaust everything. +- [Backpressure](https://www.reactivemanifesto.org/glossary#Back-Pressure) slows producers; [load shedding](https://aws.amazon.com/builders-library/using-load-shedding-to-avoid-overload/) rejects excess work early rather than queueing it without bound. ### Tail latency and hedging -- Waiting on all `n` servers, each independently slow with probability `p`, a request is slow with probability **`1 − (1 − p)ⁿ`** — at 100 servers, a 1% tail hits 63% of requests. -- **Hedged requests**: send a second copy after ~p95 latency and take the first reply. +- Waiting on all `n` servers, each independently slow with probability `p`, a request is slow with probability `1 − (1 − p)ⁿ` — at 100 servers, [a 1% tail hits 63% of requests](https://research.google/pubs/the-tail-at-scale/). +- Hedged requests: send a second copy after ~p95 latency and take the first reply. ### Failure detection -- **Fixed-timeout heartbeats** trade detection speed against false positives. -- **Phi-accrual** (Cassandra, Akka) outputs a suspicion level from heartbeat history. -- **SWIM** (Consul, Serf) pings a random member, then asks `k` peers to probe it indirectly before suspecting it. +- Fixed-timeout [heartbeats](https://en.wikipedia.org/wiki/Heartbeat_%28computing%29) trade detection speed against false positives. +- [Phi-accrual](https://doc.akka.io/libraries/akka-core/current/typed/failure-detector.html) (Cassandra, Akka) outputs a suspicion level from heartbeat history. +- [SWIM](https://www.cs.cornell.edu/projects/Quicksilver/public_pdfs/SWIM.pdf) ([Consul](https://developer.hashicorp.com/consul/docs/architecture/gossip), [Serf](https://github.com/hashicorp/serf)) pings a random member, then asks `k` peers to probe it indirectly before suspecting it. ## Stream processing ### Event time, watermarks, and windows -- **Event time** (when it happened) vs **processing time** (when it arrived) — late events break processing-time windows. -- A **watermark** says "no events older than T are expected", letting a window close; later stragglers go to a side output or update results. -- Windows: tumbling (fixed, non-overlapping), sliding or hopping (fixed, overlapping), session (closed by an inactivity gap). +- [Event time](https://nightlies.apache.org/flink/flink-docs-stable/docs/concepts/time/) (when it happened) vs processing time (when it arrived) — late events break processing-time windows. +- A [watermark](https://nightlies.apache.org/flink/flink-docs-stable/docs/concepts/time/#event-time-and-watermarks) says "no events older than T are expected", letting a window close; later stragglers go to a [side output](https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/datastream/side_output/) or update results. +- [Windows](https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/datastream/operators/windows/): tumbling (fixed, non-overlapping), sliding or hopping (fixed, overlapping), session (closed by an inactivity gap). ### Change data capture -- **Log-based CDC** (Debezium) tails the WAL or binlog: every committed change in commit order, deletes included, with no dual writes in the application. +- [Log-based CDC](https://debezium.io/documentation/reference/stable/features.html) (Debezium) tails the [WAL](https://www.postgresql.org/docs/current/wal-intro.html) or [binlog](https://debezium.io/documentation/reference/stable/connectors/mysql.html): every committed change in commit order, deletes included, with no dual writes in the application. - Polling an `updated_at` column misses hard deletes and rows that commit late with an older timestamp. -- Bootstrap from a **consistent snapshot**, then stream from the log position it was taken at. +- Bootstrap from a [consistent snapshot](https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-snapshots), then stream from the log position it was taken at. - Events are ordered per key (partition by primary key); connectors deliver at least once, so consumers must be idempotent. ### Kafka ordering and consumer groups -- Order is guaranteed **only within a partition**; key by entity ID to keep one entity's events in order. -- A consumer group splits partitions among its members — parallelism is capped at the **partition count**. -- Adding partitions changes `hash(key) % partitions`, so existing keys move and **per-key ordering breaks**; the count can't be decreased — size it up front. -- Consumer lag (latest offset − committed offset) is the health metric to alert on. +- Order is guaranteed only within a [partition](https://kafka.apache.org/43/getting-started/introduction/#main-concepts-and-terminology); key by entity ID to keep one entity's events in order. +- A [consumer group](https://kafka.apache.org/43/design/design/#the-consumer) splits partitions among its members — parallelism is capped at the partition count. +- Adding partitions changes `hash(key) % partitions`, so existing keys move and per-key ordering breaks; the count can't be decreased — size it up front. +- [Consumer lag](https://kafka.apache.org/43/operations/monitoring/) (latest offset − committed offset) is the health metric to alert on. ### Kafka producer semantics -- **Idempotent producer** (default since Kafka 3.0) dedupes retried sends per partition. -- **Transactions** make writes across partitions and the consumer offset commit atomic — exactly-once for read-process-write within Kafka. +- [Idempotent producer](https://kafka.apache.org/43/configuration/producer-configs/#producerconfigs_enable.idempotence) (default since [Kafka 3.0](https://cwiki.apache.org/confluence/display/KAFKA/KIP-679%3A+Producer+will+enable+the+strongest+delivery+guarantee+by+default)) dedupes retried sends per partition. +- [Transactions](https://kafka.apache.org/43/design/design/#using-transactions) make writes across partitions and the consumer offset commit atomic — exactly-once for read-process-write within Kafka. ### Kafka durability -- Each partition has a leader and followers; the **ISR** (in-sync replicas) is the leader plus followers caught up within `replica.lag.time.max.ms`. -- **`acks=all`** waits for every current ISR member; **`min.insync.replicas=2`** (with replication factor 3) rejects writes when the ISR shrinks below 2 — survives one broker loss without losing acknowledged data. -- `unclean.leader.election.enable=false` (default) refuses to elect an out-of-sync replica, choosing unavailability over data loss. +- Each partition has a leader and followers; the [ISR](https://kafka.apache.org/43/design/design/#replication) (in-sync replicas) is the leader plus followers caught up within [`replica.lag.time.max.ms`](https://kafka.apache.org/43/configuration/broker-configs/#brokerconfigs_replica.lag.time.max.ms). +- [`acks=all`](https://kafka.apache.org/43/configuration/producer-configs/#producerconfigs_acks) waits for every current ISR member; [`min.insync.replicas=2`](https://kafka.apache.org/43/configuration/broker-configs/#brokerconfigs_min.insync.replicas) (with replication factor 3) rejects writes when the ISR shrinks below 2 — survives one broker loss without losing acknowledged data. +- [`unclean.leader.election.enable=false`](https://kafka.apache.org/43/configuration/broker-configs/#brokerconfigs_unclean.leader.election.enable) (default) refuses to elect an out-of-sync replica, choosing unavailability over data loss. ### Kafka rebalancing, compaction, KRaft -- A rebalance reassigns partitions when consumers join or leave; the **KIP-848** consumer protocol (GA in Kafka 4.0) makes it incremental and broker-driven instead of stop-the-world. -- **Log compaction** keeps only the latest record per key (a delete = `null` tombstone) — for changelogs and state restoration. -- **Kafka 4.0 (March 2025) removed ZooKeeper**: metadata lives in a Raft quorum of controllers (KRaft). +- A rebalance reassigns partitions when consumers join or leave; the [KIP-848](https://kafka.apache.org/43/operations/consumer-rebalance-protocol/) consumer protocol (GA in [Kafka 4.0](https://kafka.apache.org/blog/2025/03/18/apache-kafka-4.0.0-release-announcement/)) makes it incremental and broker-driven instead of stop-the-world. +- [Log compaction](https://kafka.apache.org/43/design/design/#log-compaction) removes superseded records in the background, keeping at least the latest record per key (a delete = `null` tombstone) — for changelogs and state restoration. +- Kafka 4.0 (March 2025) removed ZooKeeper: metadata lives in a Raft quorum of controllers ([KRaft](https://kafka.apache.org/43/operations/kraft/)). ### Event sourcing and CQRS -- **Event sourcing** stores every change as an immutable event and rebuilds state by replay, with snapshots for speed. -- **CQRS** splits the write model from read models; if read models update asynchronously (often from events), reads are eventually consistent. +- [Event sourcing](https://martinfowler.com/eaaDev/EventSourcing.html) stores every change as an immutable event and rebuilds state by replay, with snapshots for speed. +- [CQRS](https://martinfowler.com/bliki/CQRS.html) splits the write model from read models; if read models update asynchronously (often from events), reads are eventually consistent. - Event schemas must stay readable forever: only add fields, or upcast old events. diff --git a/docs/topics/system.md b/docs/topics/system.md index 99549c7..b9f9417 100644 --- a/docs/topics/system.md +++ b/docs/topics/system.md @@ -6,52 +6,52 @@ Key system design building blocks and trade-offs, grouped by subtopic. For CAP, ### Structure and estimates -- Order: **requirements** (functional + scale, latency, availability, consistency) → estimates → API → data model → high-level design → **deep dive** on the hardest 1–2 parts → bottlenecks and trade-offs. +- Order: requirements (functional + scale, latency, availability, consistency) → estimates → API → data model → high-level design → deep dive on the hardest 1–2 parts → bottlenecks and trade-offs. - Latencies: RAM read ~100 ns, SSD random read ~100 µs, same-DC round trip ~0.5 ms, cross-continent ~150 ms. -- Downtime per year: 99.9% ≈ **8.8 h**, 99.99% ≈ 53 min, 99.999% ≈ 5 min. +- [Downtime per year](https://en.wikipedia.org/wiki/High_availability#Percentage_calculation): 99.9% ≈ 8.8 h, 99.99% ≈ 53 min, 99.999% ≈ 5 min. ### Back-of-envelope conversions -- 1 day ≈ 86,400 s ≈ 10⁵ s, so **1M requests/day ≈ 12 QPS**; assume peak ≈ 2–3× average unless told otherwise. +- 1 day ≈ 86,400 s ≈ 10⁵ s, so 1M requests/day ≈ 12 QPS; assume peak ≈ 2–3× average unless told otherwise. - 1 KB × 1M items = 1 GB; 1 KB × 1B = 1 TB. -- Ballpark capacity (varies with workload): a well-indexed PostgreSQL box handles **10k+ simple queries/s**; Redis ~100k ops/s per core. -- **Little's law** L = λ × W: requests in flight = throughput × latency — 1,000 RPS × 200 ms = 200 concurrent requests, which sizes thread pools, connection pools, and queues. +- Ballpark capacity (varies with workload): a well-indexed PostgreSQL box handles 10k+ simple queries/s; Redis ~100k ops/s per core. +- [Little's law](https://en.wikipedia.org/wiki/Little%27s_law) L = λ × W: requests in flight = throughput × latency — 1,000 RPS × 200 ms = 200 concurrent requests, which sizes thread pools, connection pools, and queues. ## Traffic and edge ### Load balancing -- **L4** balances by IP/port, fast, no inspection; **L7** reads HTTP (path, headers, cookies) for content routing and retries, at higher CPU cost. -- Algorithms: round robin, weighted, **least connections** (uneven request lengths), IP/consistent hash (same key → same server, good for caches). -- Sticky sessions defeat statelessness and unbalance load — move session state to a shared store instead. +- [L4](https://en.wikipedia.org/wiki/Transport_layer) balances by IP/port, fast, no inspection; [L7](https://en.wikipedia.org/wiki/Application_layer) reads HTTP (path, headers, cookies) for content routing and retries, at higher CPU cost. +- Algorithms: round robin, weighted, least connections (uneven request lengths), IP/[consistent hash](https://en.wikipedia.org/wiki/Consistent_hashing) (same key → same server, good for caches). +- [Sticky sessions](https://en.wikipedia.org/wiki/Load_balancing_%28computing%29#Persistence) defeat statelessness and unbalance load — move session state to a shared store instead. ### Balancing long-lived connections -- HTTP/2 and **gRPC** multiplex all requests over one long-lived connection, so an L4 balancer pins a client to one backend and new replicas get no traffic. -- Fix with L7 (per-request) balancing, client-side balancing, or a **maximum connection age** that forces reconnects. -- WebSockets have the same problem after scale-out: rebalance by closing connections gradually. +- [HTTP/2](https://www.rfc-editor.org/rfc/rfc9113) and [gRPC](https://grpc.io/blog/grpc-load-balancing/) multiplex all requests over one long-lived connection, so an L4 balancer pins a client to one backend and new replicas get no traffic. +- Fix with L7 (per-request) balancing, client-side balancing, or a [maximum connection age](https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md) that forces reconnects. +- [WebSockets](https://www.rfc-editor.org/rfc/rfc6455) have the same problem after scale-out: rebalance by closing connections gradually. ### Proxies and gateways -- **Reverse proxy** (Nginx, Envoy): TLS termination, load balancing, caching in front of servers. -- **API gateway**: auth, rate limiting, routing, aggregation for north–south traffic; a BFF is one gateway per client type. -- **Service mesh** (Istio, Linkerd): sidecar proxies (or per-node proxies in Istio ambient mode) handle east–west traffic — mTLS, retries, telemetry without app changes. +- [Reverse proxy](https://en.wikipedia.org/wiki/Reverse_proxy) ([Nginx](https://nginx.org/en/docs/), [Envoy](https://www.envoyproxy.io/docs/envoy/latest/intro/what_is_envoy)): TLS termination, load balancing, caching in front of servers. +- [API gateway](https://learn.microsoft.com/en-us/azure/architecture/microservices/design/gateway): auth, rate limiting, routing, aggregation for north–south traffic; a [BFF](https://learn.microsoft.com/en-us/azure/architecture/patterns/backends-for-frontends) is one gateway per client type. +- [Service mesh](https://en.wikipedia.org/wiki/Service_mesh) ([Istio](https://istio.io/latest/docs/overview/what-is-istio/), [Linkerd](https://linkerd.io/what-is-a-service-mesh/)): sidecar proxies (or per-node proxies in [Istio ambient mode](https://istio.io/latest/docs/ambient/overview/)) handle east–west traffic — [mTLS](https://en.wikipedia.org/wiki/Mutual_authentication#mTLS), retries, telemetry without app changes. ### Real-time delivery to clients | Transport | Direction | Note | |---|---|---| | Short polling | client pulls | simple, wasteful, latency = interval | -| Long polling | client pulls, server holds | works everywhere, one request per message | -| **SSE** | server → client | plain HTTP, auto-reconnect with `Last-Event-ID`, text only | -| **WebSocket** | both ways | stateful connections — needs a connection registry to find the server holding a user | +| [Long polling](https://en.wikipedia.org/wiki/Push_technology#Long_polling) | client pulls, server holds | works everywhere, one request per message | +| [SSE](https://html.spec.whatwg.org/multipage/server-sent-events.html) | server → client | plain HTTP, auto-reconnect with [`Last-Event-ID`](https://html.spec.whatwg.org/multipage/server-sent-events.html#the-last-event-id-header), text only | +| [WebSocket](https://www.rfc-editor.org/rfc/rfc6455) | both ways | stateful connections — needs a connection registry to find the server holding a user | -- Push to many clients means **stateful** servers: plan for reconnect storms after a deploy (jittered reconnects). +- Push to many clients means stateful servers: plan for reconnect storms after a deploy ([jittered](https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/) reconnects). ### CDN and object storage -- **Pull CDN** fetches from origin on first miss; **push CDN** is pre-uploaded (large, rarely changing files). Invalidate with versioned filenames, not purges. -- Object storage (S3, GCS) holds blobs cheaply and durably; keep only metadata in the database and use **pre-signed URLs** so transfers bypass app servers. +- Pull [CDN](https://en.wikipedia.org/wiki/Content_delivery_network) fetches from origin on first miss; push CDN is pre-uploaded (large, rarely changing files). Invalidate with versioned filenames, not purges. +- [Object storage](https://en.wikipedia.org/wiki/Object_storage) ([S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html), [GCS](https://cloud.google.com/storage/docs/introduction)) holds blobs cheaply and durably; keep only metadata in the database and use [pre-signed URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) so transfers bypass app servers. ## Caching @@ -59,48 +59,48 @@ Key system design building blocks and trade-offs, grouped by subtopic. For CAP, | Pattern | Write path | Read path | Trade-off | |---|---|---|---| -| Cache-aside | app writes DB, invalidates cache | miss → read DB, fill cache | most common; stale until TTL/invalidation | +| [Cache-aside](https://learn.microsoft.com/en-us/azure/architecture/patterns/cache-aside) | app writes DB, invalidates cache | miss → read DB, fill cache | most common; stale until TTL/invalidation | | Read-through | — | cache loads from DB on miss | cache owns the loading logic | -| Write-through | write cache + DB together | cache always warm | slower writes; the two writes aren't atomic, so a crash or race can still diverge them | -| Write-behind | write cache, flush DB async | fast writes | risk of data loss before flush | +| [Write-through](https://en.wikipedia.org/wiki/Cache_%28computing%29#Write_policies) | write cache + DB together | cache always warm | slower writes; the two writes aren't atomic, so a crash or race can still diverge them | +| [Write-behind](https://en.wikipedia.org/wiki/Cache_%28computing%29#Write_policies) | write cache, flush DB async | fast writes | risk of data loss before flush | ### Eviction and stampedes -- Eviction: **LRU** (most common), LFU, TTL. -- **Cache stampede**: many concurrent misses on a hot key hit the DB at once — fix with request coalescing (single-flight), a lock, or probabilistic early refresh. -- **Jitter TTLs** so keys written together don't expire together. +- [Eviction](https://en.wikipedia.org/wiki/Cache_replacement_policies): [LRU](https://en.wikipedia.org/wiki/Cache_replacement_policies#LRU) (most common), [LFU](https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_frequently_used_%28LFU%29), TTL. +- [Cache stampede](https://en.wikipedia.org/wiki/Cache_stampede): many concurrent misses on a hot key hit the DB at once — fix with request coalescing ([single-flight](https://pkg.go.dev/golang.org/x/sync/singleflight)), a lock, or [probabilistic early refresh](https://en.wikipedia.org/wiki/Cache_stampede#Probabilistic_early_expiration). +- Jitter TTLs so keys written together don't expire together. ### Cache and database consistency -- On write, **delete** the key rather than updating it, so concurrent writers can't leave an older value in place. -- Cache-aside still races: a slow reader can refill the cache with a value read before the write — bound it with a TTL, a delayed second delete, or **leases** (Facebook memcache). -- For many writers, invalidate from the database's change stream (**CDC**) instead of from application code. +- On write, delete the key rather than updating it, so concurrent writers can't leave an older value in place. +- Cache-aside still races: a slow reader can refill the cache with a value read before the write — bound it with a TTL, a delayed second delete, or [leases (Facebook memcache)](https://www.usenix.org/conference/nsdi13/technical-sessions/presentation/nishtala). +- For many writers, invalidate from the database's change stream ([CDC](distributed.md)) instead of from application code. ## Data storage choices ### SQL vs NoSQL -- NoSQL families by access pattern: key-value, document, **wide-column** (heavy writes, time series), graph; model tables around queries, not entities. +- [NoSQL](https://en.wikipedia.org/wiki/NoSQL) families by access pattern: [key-value](https://en.wikipedia.org/wiki/Key%E2%80%93value_database), [document](https://en.wikipedia.org/wiki/Document-oriented_database), [wide-column](https://en.wikipedia.org/wiki/Wide-column_store) (heavy writes, time series), [graph](https://en.wikipedia.org/wiki/Graph_database); model tables around queries, not entities. - Pick NoSQL only for a named reason (write volume, schema flexibility, access pattern). Internals in [database.md](database.md). ### Scaling ladder -- Cheapest first: query/index tuning → caching → read replicas → vertical scaling → split databases by domain → **sharding**. +- Cheapest first: query/index tuning → caching → read replicas → vertical scaling → split databases by domain → [sharding](https://en.wikipedia.org/wiki/Shard_%28database_architecture%29). - Sharding needs a high-cardinality, evenly spread key; cross-shard joins and transactions become the app's problem. -- **Denormalization** trades write complexity for read speed; CQRS separates the models entirely ([distributed.md](distributed.md)). +- [Denormalization](https://en.wikipedia.org/wiki/Denormalization) trades write complexity for read speed; [CQRS](https://martinfowler.com/bliki/CQRS.html) separates the models entirely ([distributed.md](distributed.md)). ## Asynchronous processing ### Queue semantics -- Queues decouple producers from consumers, absorb spikes, and move slow work off the request path. -- Delivery is at-least-once, so consumers must be **idempotent**; route repeatedly failing messages to a **dead-letter queue**. -- **Fan-out**: pub/sub (SNS, Kafka topics) delivers each event to many subscribers independently. +- [Queues](https://en.wikipedia.org/wiki/Message_queue) decouple producers from consumers, absorb spikes, and move slow work off the request path. +- Delivery is at-least-once, so consumers must be idempotent; route repeatedly failing messages to a [dead-letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html). +- Fan-out: [pub/sub](https://en.wikipedia.org/wiki/Publish%E2%80%93subscribe_pattern) ([SNS](https://docs.aws.amazon.com/sns/latest/dg/welcome.html), [Kafka topics](https://kafka.apache.org/43/getting-started/introduction/#main-concepts-and-terminology)) delivers each event to many subscribers independently. ### Queues vs logs -- **RabbitMQ/SQS** (queue): a message is removed once acknowledged — distributes tasks among workers. -- **Kafka** (log): a durable append-only log; each consumer group tracks its own offset, so many groups can **replay** the same data. +- [RabbitMQ](https://www.rabbitmq.com/docs/confirms)/[SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/welcome.html) (queue): a message is removed once acknowledged — distributes tasks among workers. +- [Kafka](https://kafka.apache.org/43/getting-started/introduction/#main-concepts-and-terminology) (log): a durable append-only log; each [consumer group](https://kafka.apache.org/43/design/design/#consumer-position) tracks its own offset, so many groups can replay the same data. - Partitioning and ordering: see [distributed.md](distributed.md). ## Rate limiting @@ -109,163 +109,163 @@ Key system design building blocks and trade-offs, grouped by subtopic. For CAP, | Algorithm | Behavior | Note | |---|---|---| -| Token bucket | tokens refill at a fixed rate | allows bursts up to bucket size; most common | -| Leaky bucket | requests drain at a constant rate | smooth output, no bursts | +| [Token bucket](https://en.wikipedia.org/wiki/Token_bucket) | tokens refill at a fixed rate | allows bursts up to bucket size; most common | +| [Leaky bucket](https://en.wikipedia.org/wiki/Leaky_bucket) | requests drain at a constant rate | smooth output, no bursts | | Fixed window counter | count per time window | simple; up to 2× limit at window boundary | -| Sliding window log/counter | exact or blended count | accurate; counter variant is cheap | +| [Sliding window log/counter](https://blog.cloudflare.com/counting-things-a-lot-of-different-things/) | exact or blended count | accurate; counter variant is cheap | ### Distributed rate limiting -- Enforce at the gateway, keyed by user, API key, or IP; reject with **429** and `Retry-After`. -- Share counters in **Redis** so every server sees one limit; wrap `INCR` + `EXPIRE` in `MULTI` or a Lua script, or a crash between them leaves a counter with no TTL. +- Enforce at the gateway, keyed by user, API key, or IP; reject with [429](https://www.rfc-editor.org/rfc/rfc6585#section-4) and [`Retry-After`](https://www.rfc-editor.org/rfc/rfc9110#field.retry-after). +- Share counters in Redis so every server sees one limit; wrap [`INCR`](https://redis.io/docs/latest/commands/incr/) + [`EXPIRE`](https://redis.io/docs/latest/commands/expire/) in [`MULTI`](https://redis.io/docs/latest/develop/interact/transactions/) or a [Lua script](https://redis.io/docs/latest/develop/interact/programmability/eval-intro/), or a crash between them leaves a counter with no TTL. - A local per-instance limiter skips the round trip but is only approximate (limit ÷ instances). ## Reliability and operations ### Availability math -- Components in **series** multiply: 99.9% × 99.9% ≈ 99.8%. -- Redundant components in **parallel** fail only together: 1 − 0.001² ≈ 99.9999%. -- Both formulas assume **independent failures**; a shared zone or dependency wipes out most of the parallel redundancy gain. +- Components in [series](https://en.wikipedia.org/wiki/Reliability_block_diagram) multiply: 99.9% × 99.9% ≈ 99.8%. +- Redundant components in parallel fail only together: 1 − 0.001² ≈ 99.9999%. +- Both formulas assume independent failures; a shared zone or dependency wipes out most of the parallel redundancy gain. ### Multi-region and disaster recovery -- **RPO** = acceptable data loss, **RTO** = acceptable recovery time; together they pick the tier. +- [RPO](https://en.wikipedia.org/wiki/Recovery_point_objective) = acceptable data loss, [RTO](https://en.wikipedia.org/wiki/Recovery_time_objective) = acceptable recovery time; together they pick the tier. | Tier | What runs in the standby region | RTO / RPO | |---|---|---| -| Backup and restore | nothing; restore from backups | hours / hours | -| Pilot light | replicated data, compute off | tens of minutes / minutes | -| Warm standby | a scaled-down copy | minutes / seconds | -| Active-active | full capacity everywhere | ~zero / ~zero with sync replication | +| [Backup and restore](https://docs.aws.amazon.com/whitepapers/latest/disaster-recovery-workloads-on-aws/disaster-recovery-options-in-the-cloud.html) | nothing; restore from backups | hours / hours | +| [Pilot light](https://docs.aws.amazon.com/whitepapers/latest/disaster-recovery-workloads-on-aws/disaster-recovery-options-in-the-cloud.html) | replicated data, compute off | tens of minutes / minutes | +| [Warm standby](https://docs.aws.amazon.com/whitepapers/latest/disaster-recovery-workloads-on-aws/disaster-recovery-options-in-the-cloud.html) | a scaled-down copy | minutes / seconds | +| [Active-active](https://docs.aws.amazon.com/whitepapers/latest/disaster-recovery-workloads-on-aws/disaster-recovery-options-in-the-cloud.html) | full capacity everywhere | ~zero / ~zero with sync replication | -- Async cross-region replication means RPO > 0; active-active also needs conflict handling or a home region per record, plus global routing (GeoDNS, anycast). -- A failover that was never rehearsed rarely works: run regular game days. +- Async cross-region replication means RPO > 0; active-active also needs conflict handling or a home region per record, plus global routing ([GeoDNS](https://en.wikipedia.org/wiki/GeoDNS), [anycast](https://en.wikipedia.org/wiki/Anycast)). +- A failover that was never rehearsed rarely works: run regular [game days](https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/rel_testing_resiliency_game_days_resiliency.html). ### SLIs, SLOs, error budgets -- SLI = what's measured, **SLO** = internal target, SLA = contractual promise (looser than the SLO). -- The gap to 100% is the **error budget** — spend it on releases, slow down when it's exhausted. -- Alert on SLO **burn rate** and user-felt symptoms, not on every cause (CPU at 80%). +- [SLI](https://sre.google/sre-book/service-level-objectives/) = what's measured, SLO = internal target, SLA = contractual promise (looser than the SLO). +- The gap to 100% is the [error budget](https://sre.google/workbook/error-budget-policy/) — spend it on releases, slow down when it's exhausted. +- Alert on SLO [burn rate](https://sre.google/workbook/alerting-on-slos/) and user-felt symptoms, not on every cause (CPU at 80%). ### Observability -- Metrics: numeric time series (Prometheus); golden signals latency, traffic, errors, saturation — **RED** for services, **USE** for resources. -- Logs: structured, centralized, tagged with a request ID. Traces: a request's span tree across services (OpenTelemetry). +- Metrics: numeric time series ([Prometheus](https://prometheus.io/docs/introduction/overview/)); [golden signals](https://sre.google/sre-book/monitoring-distributed-systems/#xref_monitoring_golden-signals) latency, traffic, errors, saturation — [RED](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) for services, [USE](https://www.brendangregg.com/usemethod.html) for resources. +- Logs: structured, centralized, tagged with a request ID. Traces: a request's span tree across services ([OpenTelemetry](https://opentelemetry.io/docs/concepts/signals/traces/)). - Metrics say something is wrong, traces say where, logs say why. -- Percentiles **can't be averaged** across hosts or time windows: merge the histograms (or sketches), then take p99. -- High-cardinality labels (user ID, request ID) multiply Prometheus time series; keep them in logs and traces. +- Percentiles can't be averaged across hosts or time windows: merge the [histograms](https://prometheus.io/docs/practices/histograms/) (or sketches), then take p99. +- [High-cardinality labels](https://prometheus.io/docs/practices/naming/#labels) (user ID, request ID) multiply Prometheus time series; keep them in logs and traces. ## Architecture ### Monolith vs microservices -- Default to a **modular monolith**: in-process calls and local transactions, with module boundaries that could later become services. -- Extract a service for a concrete reason (independent scaling, deploy cadence, team ownership); boundaries mirror teams (**Conway's law**). +- Default to a modular monolith: in-process calls and local transactions, with module boundaries that could later become services. +- Extract a service for a concrete reason (independent scaling, deploy cadence, team ownership); boundaries mirror teams ([Conway's law](https://en.wikipedia.org/wiki/Conway%27s_law)). ### Service discovery -- Instances register with a registry (Consul, etcd, Eureka) and drop out on failed health checks; clients or a load balancer resolve names to healthy instances. -- Kubernetes does this natively with Service DNS names — see [devops.md](devops.md). +- Instances register with a registry ([Consul](https://developer.hashicorp.com/consul/docs), [etcd](https://etcd.io/docs/), [Eureka](https://github.com/Netflix/eureka)) and drop out on failed health checks; clients or a load balancer resolve names to healthy instances. +- Kubernetes does this natively with [Service DNS names](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/) — see [devops.md](devops.md). ## Probabilistic and spatial structures ### Bloom filter -- Set membership in a bit array with `k` hash functions: "definitely not" or "probably yes" — **no false negatives**. -- Can't delete (a counting variant can); used to skip disk lookups (LSM trees) and dedupe crawled URLs. +- [Set membership](https://en.wikipedia.org/wiki/Bloom_filter) in a bit array with `k` hash functions: "definitely not" or "probably yes" — no false negatives. +- Can't delete (a [counting variant](https://en.wikipedia.org/wiki/Counting_Bloom_filter) can); used to skip disk lookups ([LSM trees](https://en.wikipedia.org/wiki/Log-structured_merge-tree)) and dedupe crawled URLs. ### Count-min sketch and HyperLogLog -- **Count-min sketch** estimates per-item frequency in fixed memory — heavy hitters, top-k in a stream. -- **HyperLogLog** estimates distinct counts in ~**12 KB** with ~0.8% error (Redis `PFADD`/`PFCOUNT`). +- [Count-min sketch](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) estimates per-item frequency in fixed memory — heavy hitters, top-k in a stream. +- [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) estimates distinct counts in ~12 KB with ~0.8% error (Redis [`PFADD`/`PFCOUNT`](https://redis.io/docs/latest/develop/data-types/probabilistic/hyperloglogs/)). ### Geospatial indexes -- **Geohash** encodes lat/lon as a string: a shared prefix means nearby, but close points can straddle a cell boundary and share none. -- Pick a precision whose cell is at least the search radius, then query the cell and its **8 neighbors**. -- A **quadtree** subdivides dense areas further, adapting to uneven density; Google S2 and Uber H3 use hierarchical cells the same way. +- [Geohash](https://en.wikipedia.org/wiki/Geohash) encodes lat/lon as a string: a shared prefix means nearby, but close points can straddle a cell boundary and share none. +- Pick a precision whose cell is at least the search radius, then query the cell and its 8 neighbors. +- A [quadtree](https://en.wikipedia.org/wiki/Quadtree) subdivides dense areas further, adapting to uneven density; Google [S2](https://s2geometry.io/) and Uber [H3](https://h3geo.org/) use hierarchical cells the same way. ## Classic designs: user-facing ### URL shortener -- Read-heavy (~100:1); **base62** in 7 characters gives ~3.5 trillion keys. +- Read-heavy (~100:1); [base62](https://en.wikipedia.org/wiki/Base62) in 7 characters gives ~3.5 trillion keys. - Keys from hash + collision check, a counter encoded in base62 (predictable unless shuffled), or a pre-generated key pool. -- **301** is cached by browsers by default (less load, lost clicks); 302 isn't cached without explicit freshness headers, so every click hits you (analytics). Collect analytics asynchronously. +- [301](https://www.rfc-editor.org/rfc/rfc9110#status.301) is [cached by browsers by default](https://www.rfc-editor.org/rfc/rfc9111#heuristic.freshness) (less load, lost clicks); [302](https://www.rfc-editor.org/rfc/rfc9110#status.302) isn't cached without explicit freshness headers, so every click hits you (analytics). Collect analytics asynchronously. ### News feed -- **Fan-out on write**: precompute followers' feeds at post time — instant reads, but a celebrity triggers massive write amplification. -- **Fan-out on read**: merge followees' posts at read time — slower reads, no amplification. -- **Hybrid**: push for normal accounts, pull for celebrities; store post IDs, hydrate from cache, rank as a separate step. +- Fan-out on write: precompute followers' feeds at post time — instant reads, but a celebrity triggers massive write amplification. +- Fan-out on read: merge followees' posts at read time — slower reads, no amplification. +- Hybrid: push for normal accounts, pull for celebrities; store post IDs, hydrate from cache, rank as a separate step. ### Chat system -- A persistent **WebSocket** per client; a connection registry (user → server) routes messages via pub/sub. -- A server-assigned **per-conversation sequence number** orders messages; a client-generated message ID dedupes retries. +- A persistent [WebSocket](https://www.rfc-editor.org/rfc/rfc6455) per client; a connection registry (user → server) routes messages via [pub/sub](https://en.wikipedia.org/wiki/Publish%E2%80%93subscribe_pattern). +- A server-assigned per-conversation sequence number orders messages; a client-generated message ID dedupes retries. - Offline users get a push notification and fetch on reconnect; large groups hit the same fan-out problem as feeds. ### Typeahead -- A **trie** (or prefix index) with precomputed **top-k suggestions** per node, built offline from query logs and cached aggressively. -- Debounce on the client; for semantic matches, query an approximate nearest-neighbor index (HNSW). +- A [trie](https://en.wikipedia.org/wiki/Trie) (or prefix index) with precomputed top-k suggestions per node, built offline from query logs and cached aggressively. +- [Debounce](https://developer.mozilla.org/en-US/docs/Glossary/Debounce) on the client; for semantic matches, query an [approximate nearest-neighbor](https://en.wikipedia.org/wiki/Nearest_neighbor_search#Approximation_methods) index ([HNSW](https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world)). ### Video streaming -- Upload to object storage → a **transcoding** pipeline splits the video into chunks and encodes them in parallel at several resolutions and bitrates. -- Package as **HLS or DASH**: 2–10 s segments plus a manifest listing the renditions, served from a CDN. -- The player picks a rendition per segment from measured bandwidth and buffer (**adaptive bitrate**), so quality drops instead of stalling. +- Upload to object storage → a [transcoding](https://en.wikipedia.org/wiki/Transcoding) pipeline splits the video into chunks and encodes them in parallel at several resolutions and bitrates. +- Package as [HLS](https://www.rfc-editor.org/rfc/rfc8216) or [DASH](https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP): 2–10 s segments plus a manifest listing the renditions, served from a CDN. +- The player picks a rendition per segment from measured bandwidth and buffer ([adaptive bitrate](https://en.wikipedia.org/wiki/Adaptive_bitrate_streaming)), so quality drops instead of stalling. - Pre-warm popular titles at the edge; the long tail streams from regional caches or the origin. ### Notification system -- One API → a queue per channel (push, SMS, email) → workers calling providers (APNs, FCM, Twilio) with retries. -- Dedupe by **notification ID**, respect user preferences and rate limits, track delivery status per message. +- One API → a queue per channel (push, SMS, email) → workers calling providers ([APNs](https://developer.apple.com/documentation/usernotifications), [FCM](https://firebase.google.com/docs/cloud-messaging), [Twilio](https://www.twilio.com/docs/messaging)) with retries. +- Dedupe by notification ID, respect user preferences and rate limits, track delivery status per message. ### Proximity service -- Index places by **geohash** prefix or quadtree cell; search the cell plus neighbors, then filter by exact distance. -- Static places are read-heavy — cache per cell; moving objects (drivers) report every few seconds into an in-memory index (Redis GEO). +- Index places by [geohash](https://en.wikipedia.org/wiki/Geohash) prefix or [quadtree](https://en.wikipedia.org/wiki/Quadtree) cell; search the cell plus neighbors, then filter by exact distance. +- Static places are read-heavy — cache per cell; moving objects (drivers) report every few seconds into an in-memory index ([Redis GEO](https://redis.io/docs/latest/develop/data-types/geospatial/)). ## Classic designs: data and infrastructure ### Web crawler -- A **URL frontier** of per-host queues enforces politeness (one connection per host, `robots.txt`, crawl delay) and priority. -- Dedupe URLs with a seen-set (Bloom filter at scale) and content with checksums or **SimHash**. -- Cache DNS; guard against spider traps with URL depth and length limits. +- A [URL frontier](https://en.wikipedia.org/wiki/Crawl_frontier) of per-host queues enforces politeness (one connection per host, [`robots.txt`](https://www.rfc-editor.org/rfc/rfc9309), crawl delay) and priority. +- Dedupe URLs with a seen-set ([Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) at scale) and content with checksums or [SimHash](https://en.wikipedia.org/wiki/SimHash). +- Cache DNS; guard against [spider traps](https://en.wikipedia.org/wiki/Spider_trap) with URL depth and length limits. ### File sync -- Split files into **chunks** (~4 MB) addressed by content hash: upload only changed chunks and dedupe identical ones across users. -- A metadata service stores file → chunk list and versions; other devices get change notifications via long polling or WebSocket. -- Concurrent edits produce a **conflicted copy** rather than a silent overwrite. +- Split files into chunks (~4 MB) addressed by [content hash](https://en.wikipedia.org/wiki/Content-addressable_storage): upload only changed chunks and dedupe identical ones across users. +- A metadata service stores file → chunk list and versions; other devices get change notifications via [long polling](https://en.wikipedia.org/wiki/Push_technology#Long_polling) or [WebSocket](https://www.rfc-editor.org/rfc/rfc6455). +- Concurrent edits produce a conflicted copy rather than a silent overwrite. ### Metrics and click aggregation -- Events flow through Kafka into a stream processor that aggregates per key per **tumbling window**, by event time with watermarks ([distributed.md](distributed.md)). -- Keep raw events so aggregates can be **recomputed** after a bug; dedupe by event ID. +- Events flow through [Kafka](https://kafka.apache.org/43/getting-started/introduction/#main-concepts-and-terminology) into a stream processor that aggregates per key per [tumbling window](https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/datastream/operators/windows/#tumbling-windows), by event time with watermarks ([distributed.md](distributed.md)). +- Keep raw events so aggregates can be recomputed after a bug; dedupe by event ID. - Pre-aggregate hot keys locally before the shuffle. ### Distributed job scheduler -- Store jobs with `next_run_at`; workers claim due rows with **`FOR UPDATE SKIP LOCKED`** or a lease, so each run has one owner. -- An expired lease hands a dead worker's job to another, so jobs must be **idempotent**. -- At high volume, partition jobs by ID or bucket them by time (a **timing wheel**). +- Store jobs with `next_run_at`; workers claim due rows with [`FOR UPDATE SKIP LOCKED`](https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE) or a lease, so each run has one owner. +- An expired lease hands a dead worker's job to another, so jobs must be idempotent. +- At high volume, partition jobs by ID or bucket them by time (a [timing wheel](https://www.confluent.io/blog/apache-kafka-purgatory-hierarchical-timing-wheels/)). ### Payments and ledgers -- A **double-entry ledger**: every transaction is balanced debits and credits; rows are append-only, corrections are new entries. -- **Idempotency keys** on every provider call; reconcile against provider reports daily. -- Store amounts as **integer minor units** with a currency, never floats. +- A [double-entry ledger](https://en.wikipedia.org/wiki/Double-entry_bookkeeping): every transaction is balanced debits and credits; rows are append-only, corrections are new entries. +- [Idempotency keys](backend.md) on every provider call; reconcile against provider reports daily. +- Store amounts as [integer minor units](https://docs.stripe.com/currencies#minor-units) with a currency, never floats. ### Leaderboard -- A Redis **sorted set**: `ZINCRBY` updates a score and `ZREVRANK` returns a user's rank in O(log n); `ZRANGE ... REV` reads the top k in O(log n + k). +- A Redis [sorted set](https://redis.io/docs/latest/develop/data-types/sorted-sets/): [`ZINCRBY`](https://redis.io/docs/latest/commands/zincrby/) updates a score and [`ZREVRANK`](https://redis.io/docs/latest/commands/zrevrank/) returns a user's rank in O(log n); [`ZRANGE ... REV`](https://redis.io/docs/latest/commands/zrange/) reads the top k in O(log n + k). - Beyond one node, shard by score range or keep per-shard top-k and merge. ### Booking and inventory contention -- Overselling comes from check-then-write races: use **conditional updates** (`UPDATE ... SET left = left - 1 WHERE id = ? AND left > 0`) or row locks. -- For seat selection, place a **temporary hold with a TTL** that expires if payment doesn't finish; a waiting room absorbs flash-sale spikes. +- Overselling comes from check-then-write races: use conditional updates (`UPDATE ... SET left = left - 1 WHERE id = ? AND left > 0`) or [row locks](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS). +- For seat selection, place a temporary hold with a TTL that expires if payment doesn't finish; a waiting room absorbs flash-sale spikes.