From 1db03b3a0468cb27e8ffbad8fa48a8b03253c52e Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 8 Oct 2026 08:47:50 +0300 Subject: [PATCH] Link infra and domain brief terms and ban bold Remove bold from the aws, devops, ai and ethereum briefs and link their terms to official docs, EIPs and papers. Update facts found in review: NLB QUIC and listener rules, MCP 2026-07-28 statelessness, Terraform count index shifts, anonymous event topics. With every brief now bold-free, lint_briefs.py rejects any bold marker outside code and link targets instead of allowing three per concept. Co-Authored-By: Claude Opus 5.5 --- docs/topics/ai.md | 149 ++++++++++++++++++------------------ docs/topics/aws.md | 146 +++++++++++++++++------------------ docs/topics/devops.md | 166 ++++++++++++++++++++-------------------- docs/topics/ethereum.md | 142 +++++++++++++++++----------------- scripts/lint_briefs.py | 24 +++--- 5 files changed, 314 insertions(+), 313 deletions(-) diff --git a/docs/topics/ai.md b/docs/topics/ai.md index 2bec062..db161c5 100644 --- a/docs/topics/ai.md +++ b/docs/topics/ai.md @@ -6,117 +6,117 @@ What experienced AI engineers forget before an interview, grouped by subtopic. F ### Tokens and context windows -- Models read **tokens** (subword pieces from BPE-style tokenizers): roughly **4 characters or ¾ of a word** of English per token; code and non-English text use more. -- The context window holds input **and** output tokens; price and latency scale with both, and output tokens cost several times more than input. +- Models read tokens (subword pieces from [BPE](https://en.wikipedia.org/wiki/Byte-pair_encoding)-style tokenizers): roughly 4 characters or ¾ of a word of English per token; code and non-English text use more. +- The [context window](https://platform.claude.com/docs/en/build-with-claude/context-windows) holds input and output tokens; price and latency scale with both, and output tokens cost several times more than input. ### Attention cost -- Self-attention compares every token with every other: prefill compute grows **O(n²)** with sequence length; each decoded token attends to all cached ones. -- **GQA/MQA** share key/value heads across query heads, shrinking the KV cache several-fold. -- **FlashAttention** computes exact attention in tiles held in on-chip memory — same result, far less memory traffic. +- [Self-attention](https://arxiv.org/abs/1706.03762) compares every token with every other: prefill compute grows O(n²) with sequence length; each decoded token attends to all cached ones. +- [GQA](https://arxiv.org/abs/2305.13245)/[MQA](https://arxiv.org/abs/1911.02150) share key/value heads across query heads, shrinking the KV cache several-fold. +- [FlashAttention](https://arxiv.org/abs/2205.14135) computes exact attention in tiles held in on-chip memory — same result, far less memory traffic. ### Mixture of experts -- An **MoE** layer routes each token to a few of many expert feed-forward networks, so **active parameters** per token are a fraction of the total. -- Compute follows active parameters, but memory follows **total** parameters (offloading experts to CPU saves GPU memory, at a speed cost); uneven routing complicates serving. +- An [MoE](https://arxiv.org/abs/1701.06538) layer routes each token to a few of many expert feed-forward networks, so active parameters per token are a fraction of the total. +- Compute follows active parameters, but memory follows total parameters (offloading experts to CPU saves GPU memory, at a speed cost); uneven routing complicates serving. ### Reasoning models -- Reasoning models spend **test-time compute**: they generate a chain of thought before answering, trading latency and cost for accuracy on math, code, and planning. -- **Thinking tokens are billed as output** and count against the context window; APIs expose a thinking budget or effort level. +- Reasoning models spend [test-time compute](https://arxiv.org/abs/2408.03314): they generate a [chain of thought](https://arxiv.org/abs/2201.11903) before answering, trading latency and cost for accuracy on math, code, and planning. +- [Thinking tokens are billed as output](https://platform.claude.com/docs/en/build-with-claude/extended-thinking) and count against the context window; APIs expose a thinking budget or [effort level](https://platform.claude.com/docs/en/build-with-claude/effort). - Prompt them with goals and constraints — "think step by step" is already built in. ### Sampling parameters -- **Temperature** scales logits before softmax: low → focused and repeatable, high → diverse. **Top-p** (nucleus) samples from the smallest set covering probability `p`; top-k from the `k` likeliest tokens. -- Temperature 0 is near-greedy but **not guaranteed deterministic** — batching and floating-point order change results. +- [Temperature](https://en.wikipedia.org/wiki/Softmax_function) scales logits before softmax: low → focused and repeatable, high → diverse. [Top-p](https://arxiv.org/abs/1904.09751) (nucleus) samples from the smallest set covering probability `p`; top-k from the `k` likeliest tokens. +- Temperature 0 is near-greedy but [not guaranteed deterministic](https://thinkingmachines.ai/blog/defeating-nondeterminism-in-llm-inference/) — batching and floating-point order change results. ### Long-context behavior -- Recall is weakest for facts in the **middle** of a long prompt ("lost in the middle"), and quality degrades as irrelevant context grows. -- Put instructions and the key evidence at the start or end; retrieving less but better beats stuffing the window. +- Recall is weakest for facts in the middle of a long prompt (["lost in the middle"](https://arxiv.org/abs/2307.03172)), and quality degrades as irrelevant context grows. +- Put instructions and the key evidence [at the start or end](https://platform.claude.com/docs/en/build-with-claude/prompt-engineering/claude-prompting-best-practices#long-context-prompting); retrieving less but better beats stuffing the window. ### Structured output -- **Constrained decoding** masks tokens so output matches a JSON schema or grammar — except when cut off by the token limit or replaced by a refusal. -- It guarantees **shape, not correctness** — still validate values in code. +- [Constrained decoding](https://platform.claude.com/docs/en/build-with-claude/structured-outputs) masks tokens so output matches a [JSON schema](https://json-schema.org/) or grammar — except when cut off by the token limit or replaced by a refusal. +- It guarantees shape, not correctness — still validate values in code. ### Tool calling -- The model emits a structured call (name + JSON arguments); **the application executes it** and appends the result, then the model continues — a loop until the model answers. +- The model emits a [structured call](https://platform.claude.com/docs/en/agents-and-tools/tool-use/overview) (name + JSON arguments); the application executes it and appends the result, then the model continues — a loop until the model answers. - The model never runs anything itself, so every side effect is under application control. ## Inference and serving ### Prefill vs decode -- Prefill processes the whole prompt in parallel — compute-bound, drives **time to first token (TTFT)**. -- Decode emits one token at a time — memory-bandwidth-bound, drives **time per output token (TPOT)**; total latency ≈ TTFT + TPOT × output tokens. +- Prefill processes the whole prompt in parallel — compute-bound, drives [time to first token (TTFT)](https://docs.nvidia.com/nim/benchmarking/llm/latest/metrics.html). +- Decode emits one token at a time — memory-bandwidth-bound, drives time per output token (TPOT); total latency ≈ TTFT + TPOT × output tokens. ### KV cache -- Attention keys and values of past tokens are cached so each new token doesn't recompute them; cache size grows with **sequence length × layers × batch**, which is what limits concurrency on a GPU. -- **Prompt caching** reuses the KV cache of an identical **prefix** — put stable content (system prompt, tool schemas, documents) first. +- Attention keys and values of past tokens are [cached](https://huggingface.co/docs/transformers/en/kv_cache) so each new token doesn't recompute them; cache size grows with sequence length × layers × batch, which is what limits concurrency on a GPU. +- [Prompt caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) reuses the KV cache of an identical prefix — put stable content (system prompt, tool schemas, documents) first. ### Batching and PagedAttention -- **Continuous batching** adds and removes requests at every decode step instead of waiting for a whole batch to finish. -- **PagedAttention** (vLLM) stores the KV cache in fixed-size blocks like virtual-memory pages, cutting fragmentation so more requests fit. +- [Continuous batching](https://www.usenix.org/conference/osdi22/presentation/yu) adds and removes requests at every decode step instead of waiting for a whole batch to finish. +- [PagedAttention](https://arxiv.org/abs/2309.06180) ([vLLM](https://docs.vllm.ai/en/latest/)) stores the KV cache in fixed-size blocks like virtual-memory pages, cutting fragmentation so more requests fit. ### Speculative decoding -- A small **draft model** proposes several tokens; the large model verifies them in one pass and keeps the accepted prefix. -- Faster decode with the **same output distribution**; the gain depends on how often drafts are accepted. +- A small draft model proposes several tokens; the large model [verifies them in one pass](https://arxiv.org/abs/2211.17192) and keeps the accepted prefix. +- Faster decode with the same output distribution; the gain depends on how often drafts are accepted. ### Model parallelism and GPU memory - Serving memory ≈ weights + KV cache + activations; a model too large for one GPU must be split. -- **Tensor parallelism** splits each layer's matrices across GPUs; it needs a fast interconnect (NVLink), so it stays within a node. -- **Pipeline parallelism** puts consecutive layers on different GPUs or nodes: less communication, but pipeline bubbles leave stages idle. -- Data parallelism replicates the whole model for throughput; **disaggregated serving** runs prefill and decode on separate GPU pools, each tuned for its bottleneck. +- [Tensor parallelism](https://arxiv.org/abs/1909.08053) splits each layer's matrices across GPUs; it needs a fast interconnect ([NVLink](https://www.nvidia.com/en-us/data-center/nvlink/)), so it is usually kept within a node. +- [Pipeline parallelism](https://arxiv.org/abs/1811.06965) puts consecutive layers on different GPUs or nodes: less communication, but pipeline bubbles leave stages idle. +- Data parallelism replicates the whole model for throughput; [disaggregated serving](https://arxiv.org/abs/2401.09670) runs prefill and decode on separate GPU pools, each tuned for its bottleneck. ### Quantization -- Weights in FP16 take **2 bytes per parameter** (a 70B model ≈ 140 GB); INT8 halves that, **4-bit** (GPTQ, AWQ) quarters it, with some quality loss. +- Weights in FP16 take 2 bytes per parameter (a 70B model ≈ 140 GB); INT8 halves that, 4-bit ([GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978)) quarters it, with some quality loss. - Smaller weights also speed up decode, since decode is memory-bound. ## Retrieval and RAG ### RAG pipeline -- Ingest → parse → **chunk** → embed → index → retrieve → **rerank** → answer with citations. +- Ingest → parse → chunk → embed → index → retrieve → rerank → answer with citations ([RAG](https://arxiv.org/abs/2005.11401)). - Keep source version and access metadata on every chunk, so answers stay traceable and refreshable. -- RAG doesn't guarantee truth — the answer can be missing or stale; **abstain** when evidence is insufficient. +- RAG doesn't guarantee truth — the answer can be missing or stale; abstain when evidence is insufficient. ### Embeddings and similarity -- An embedding maps text to a vector; similarity is usually **cosine** — on normalized vectors that's just the dot product. -- Query and documents must share one **embedding space** (same model, or jointly trained query/passage encoders as in DPR); changing models means re-embedding the corpus. +- An embedding maps text to a vector; similarity is usually [cosine](https://en.wikipedia.org/wiki/Cosine_similarity) — on normalized vectors that's just the dot product. +- Query and documents must share one embedding space (same model, or jointly trained query/passage encoders as in [DPR](https://arxiv.org/abs/2004.04906)); changing models means re-embedding the corpus. ### Vector index types | Index | How | Trade-off | |---|---|---| | Flat | exact scan | perfect recall, O(n) per query | -| **HNSW** | layered proximity graph | high recall and speed, memory-heavy, slow builds | -| **IVF** | cluster, then probe `nprobe` clusters | less memory; recall depends on `nprobe` | -| **PQ** | compress vectors into codes | much smaller, lower recall; often combined as IVF-PQ | +| [HNSW](https://arxiv.org/abs/1603.09320) | layered proximity graph | high recall and speed, memory-heavy, slow builds | +| [IVF](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes) | cluster, then probe `nprobe` clusters | less memory; recall depends on `nprobe` | +| [PQ](https://ieeexplore.ieee.org/document/5432202) | compress vectors into codes | much smaller, lower recall; often combined as IVF-PQ | -- **Filtered search**: post-filtering the top-k returns too few hits when the filter is selective, and pre-filtering breaks HNSW graph connectivity — use a filter-aware engine or partition the index (per tenant). +- [Filtered search](https://qdrant.tech/documentation/search-patterns/vector-search-filtering/): post-filtering the top-k returns too few hits when the filter is selective, and pre-filtering breaks HNSW graph connectivity — use a filter-aware engine or partition the index (per tenant). ### Chunking -- Chunk along **document structure** (sections, paragraphs); a few hundred tokens with **10–20% overlap** is a common starting point — tune on evals. -- **Parent-document retrieval**: match small chunks, return the surrounding section for context. +- Chunk along document structure (sections, paragraphs); a few hundred tokens with 10–20% overlap is a common starting point — tune on evals. +- Parent-document retrieval: match small chunks, return the surrounding section for context. ### Hybrid search and reranking -- **BM25** (lexical) catches exact names and IDs; dense vectors catch paraphrases; merge with **reciprocal rank fusion**. -- A cross-encoder reranker rescores a small candidate set — but can't recover a passage that was never retrieved, so tune **recall@k** first. +- [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) (lexical) catches exact names and IDs; dense vectors catch paraphrases; merge with [reciprocal rank fusion](https://cormack.uwaterloo.ca/cormacksigir09-rrf.pdf). +- A [cross-encoder](https://sbert.net/examples/cross_encoder/applications/README.html) reranker rescores a small candidate set — but can't recover a passage that was never retrieved, so tune [recall@k](https://en.wikipedia.org/wiki/Evaluation_measures_%28information_retrieval%29) first. ### Permission-aware retrieval -- Filter by tenant, ACL, and sensitivity **before** a chunk reaches the model; a prompt saying "ignore forbidden data" is not access control. +- Filter by tenant, ACL, and sensitivity before a chunk reaches the model; a prompt saying "ignore forbidden data" is not [access control](https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/). - The same filter must apply to caches, follow-up retrieval, and agent memory; fail closed when permission metadata is missing. ### RAG vs prompting vs fine-tuning @@ -131,34 +131,34 @@ What experienced AI engineers forget before an interview, grouped by subtopic. F ### Training pipeline -- **Pretraining**: next-token prediction on trillions of tokens; Chinchilla-optimal is ≈ **20 tokens per parameter**, but modern models train far past it so a smaller model is cheaper to serve. -- Post-training: SFT on demonstrations, then preference tuning (RLHF, DPO) for helpfulness and safety. -- **RL with verifiable rewards** (math and code graded automatically, e.g. GRPO) is how reasoning models learn long chains of thought. +- Pretraining: next-token prediction on trillions of tokens; [Chinchilla](https://arxiv.org/abs/2203.15556)-optimal is ≈ 20 tokens per parameter, but modern models train far past it so a smaller model is cheaper to serve. +- Post-training: SFT on demonstrations, then preference tuning ([RLHF](https://arxiv.org/abs/2203.02155), [DPO](https://arxiv.org/abs/2305.18290)) for helpfulness and safety. +- [RL with verifiable rewards](https://arxiv.org/abs/2411.15124) (math and code graded automatically, e.g. [GRPO](https://arxiv.org/abs/2402.03300)) is how reasoning models learn long chains of thought. ### SFT, RLHF, DPO -- **SFT**: train on input → ideal-output pairs. -- **RLHF**: train a reward model on human preferences, then optimize the policy against it (PPO). -- **DPO**: learn directly from preferred/rejected pairs, no separate reward model — simpler and more stable. +- SFT: train on input → ideal-output pairs. +- [RLHF](https://arxiv.org/abs/2203.02155): train a reward model on human preferences, then optimize the policy against it ([PPO](https://arxiv.org/abs/1707.06347)). +- [DPO](https://arxiv.org/abs/2305.18290): learn directly from preferred/rejected pairs, no separate reward model — simpler and more stable. ### LoRA and QLoRA -- **LoRA** freezes the base weights and trains small low-rank adapter matrices — a tiny fraction of parameters, swappable per task. -- **QLoRA** trains LoRA adapters on top of a **4-bit** quantized base, fitting large models on one GPU. +- [LoRA](https://arxiv.org/abs/2106.09685) freezes the base weights and trains small low-rank adapter matrices — a tiny fraction of parameters, swappable per task. +- [QLoRA](https://arxiv.org/abs/2305.14314) trains LoRA adapters on top of a 4-bit quantized base, fitting large models on one GPU. ## Agents and orchestration ### Workflow vs agent vs multi-agent -- **Workflow**: steps fixed in code — for well-understood, mostly deterministic processes. -- **Agent**: the model picks tools and next steps in a loop — for tasks whose path varies. -- **Multi-agent**: only when roles need different tools or context, or work can run in parallel — each handoff adds latency, cost, and lost context. +- [Workflow](https://www.anthropic.com/engineering/building-effective-agents): steps fixed in code — for well-understood, mostly deterministic processes. +- Agent: the model picks tools and next steps in a loop — for tasks whose path varies. +- [Multi-agent](https://www.anthropic.com/engineering/multi-agent-research-system): only when roles need different tools or context, or work can run in parallel — each handoff adds latency, cost, and lost context. - Start with the simplest design that passes the evals. ### Durable execution and state -- Persist task IDs, inputs, completed steps, approvals, and side effects **outside the model**, so a run can resume from a checkpoint. -- **Durable execution** (Temporal) fits processes that wait or retry for days; none of it makes external writes exactly-once — use idempotency keys. +- Persist task IDs, inputs, completed steps, approvals, and side effects outside the model, so a run can resume from a checkpoint. +- Durable execution ([Temporal](https://docs.temporal.io/evaluate/understanding-temporal)) fits processes that wait or retry for days; none of it makes external writes exactly-once — use idempotency keys. ### Stop conditions @@ -167,68 +167,69 @@ What experienced AI engineers forget before an interview, grouped by subtopic. F ### Context management -- Long agent runs overflow the window: **compact** old turns into summaries and keep tool outputs short. -- Store bulky state in files or memory the agent can re-read; **subagents** keep exploratory work out of the main context. +- Long agent runs overflow the window: [compact](https://platform.claude.com/docs/en/build-with-claude/compaction) old turns into summaries and keep tool outputs short. +- Store bulky state in files or memory the agent can re-read; [subagents](https://code.claude.com/docs/en/sub-agents) keep exploratory work out of the main context. ## Tools and integration ### Tool design -- Narrow names and schemas, clear errors, short outputs; a tool call is a **request**, not permission — authorize and validate in code. -- Writes need timeouts and **idempotency keys**; log what was read and changed. +- [Narrow names and schemas](https://www.anthropic.com/engineering/writing-tools-for-agents), clear errors, short outputs; a tool call is a request, not permission — authorize and validate in code. +- Writes need timeouts and idempotency keys; log what was read and changed. ### Model Context Protocol -- **MCP** standardizes how apps discover and call tools, read resources, and fetch prompts — JSON-RPC 2.0 over stdio (local) or **Streamable HTTP** (remote, replaced HTTP+SSE in 2025). -- Authorization is optional; HTTP servers that support it use **OAuth 2.1**. MCP doesn't replace per-action authorization or business validation. +- [MCP](https://modelcontextprotocol.io/specification/2026-07-28) standardizes how apps discover and call tools, read resources, and fetch prompts — [JSON-RPC 2.0](https://www.jsonrpc.org/specification) over stdio (local) or [Streamable HTTP](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) (remote, replaced HTTP+SSE in 2025). +- Stateless since the 2026-07-28 revision: no `initialize` handshake or `Mcp-Session-Id`; each request carries its protocol version and capabilities in `_meta`, and servers advertise theirs via `server/discover`. +- [Authorization](https://modelcontextprotocol.io/specification/2026-07-28/basic/authorization) is optional; HTTP servers that support it use [OAuth 2.1](https://datatracker.ietf.org/doc/draft-ietf-oauth-v2-1/). MCP doesn't replace per-action authorization or business validation. ## Evaluation ### Eval sets and metrics - Build eval cases from real traffic, edge cases, and past incidents, with expected outcomes. -- Measure separately: retrieval (**recall@k**, **MRR**, **nDCG**), answer groundedness and citation accuracy, and end-to-end task success. +- Measure separately: retrieval ([recall@k](https://en.wikipedia.org/wiki/Evaluation_measures_%28information_retrieval%29), [MRR](https://en.wikipedia.org/wiki/Mean_reciprocal_rank), [nDCG](https://en.wikipedia.org/wiki/Discounted_cumulative_gain)), answer groundedness and citation accuracy, and end-to-end task success. ### LLM-as-judge -- Scales rubric grading, but has **position**, **verbosity**, and **self-preference** biases — calibrate against human labels and randomize answer order. +- Scales rubric grading, but has position, verbosity, and [self-preference biases](https://arxiv.org/abs/2306.05685) — calibrate against human labels and randomize answer order. - Prefer deterministic checks (schema, state change, test pass) wherever they exist. ### Shadow mode and prompt versioning -- **Shadow mode**: run a candidate on live inputs, log its proposed outputs without executing them, compare before rollout. -- Version prompts like code and record which **prompt + model version** produced each result, so a regression can be traced. +- Shadow mode: run a candidate on live inputs, log its proposed outputs without executing them, compare before rollout. +- Version prompts like code and record which prompt + model version produced each result, so a regression can be traced. ### Tracing -- **One trace per run** across model calls, retrieval, tool calls, approvals, and writes. +- One trace per run across model calls, retrieval, tool calls, approvals, and writes ([OpenTelemetry GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/)). - Record latency, tokens, cost, errors, and whether a side effect already happened — so a retry doesn't repeat it. ## Cost and latency ### Model routing and caching -- **Route** easy requests to a small, fast model and hard ones to a strong model. -- **Semantic caching** reuses answers to similar queries; its key must include user permissions, source version, and freshness, or it leaks. +- [Route](https://arxiv.org/abs/2406.18665) easy requests to a small, fast model and hard ones to a strong model. +- Semantic caching reuses answers to similar queries; its key must include user permissions, source version, and freshness, or it leaks. ### Batch and streaming -- **Batch APIs** run requests asynchronously (results within ~24 h) at about **half price** — for evals, backfills, and offline labeling. -- **Streaming** doesn't cut total time but shows the first tokens at TTFT, which is what users perceive. +- [Batch APIs](https://platform.claude.com/docs/en/build-with-claude/batch-processing) run requests asynchronously (results within ~24 h) at about half price — for evals, backfills, and offline labeling. +- [Streaming](https://platform.claude.com/docs/en/build-with-claude/streaming) doesn't cut total time but shows the first tokens at TTFT, which is what users perceive. ## Security and oversight ### Prompt injection -- Instructions hidden in lower-trust content (web pages, emails, tool results) can hijack the model; no filter reliably stops it. -- Treat that content as **data**, keep tool permissions narrow, and validate actions in code. +- Instructions hidden in lower-trust content (web pages, emails, tool results) can [hijack the model](https://genai.owasp.org/llmrisk/llm01-prompt-injection/); no filter reliably stops it. +- Treat that content as data, keep tool permissions narrow, and validate actions in code. ### Lethal trifecta -- **Private data access + untrusted content + an exfiltration channel** in one agent enables data theft. +- [Private data access + untrusted content + an exfiltration channel](https://simonwillison.net/2025/Jun/16/the-lethal-trifecta/) in one agent enables data theft. - Removing any one of the three closes most of the risk — e.g. no outbound network from an agent that reads private data. ### Human approval gates -- Require approval before high-impact actions (money, legal, external sends, irreversible form submits by a browser agent); show the action, evidence, and affected records. -- An approval covers **one specific action**; re-check that data hasn't changed since the proposal, and record who approved what and when. +- Require approval before [high-impact actions](https://genai.owasp.org/llmrisk/llm062025-excessive-agency/) (money, legal, external sends, irreversible form submits by a browser agent); show the action, evidence, and affected records. +- An approval covers one specific action; re-check that data hasn't changed since the proposal, and record who approved what and when. diff --git a/docs/topics/aws.md b/docs/topics/aws.md index c95b9a4..cf36cef 100644 --- a/docs/topics/aws.md +++ b/docs/topics/aws.md @@ -6,64 +6,64 @@ What experienced AWS developers and DevOps engineers forget before an interview, ### Policy evaluation -- Requests start with an implicit deny; any applicable **explicit deny** wins over every allow. -- Identity and resource policies grant; permissions boundaries, session policies, and SCPs/RCPs only **cap** what can be granted. -- A role needs both a **trust policy** allowing the principal to assume it and permissions for the resulting session to perform actions. +- Requests start with an [implicit deny](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html); any applicable explicit deny wins over every allow. +- [Identity and resource policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_identity-vs-resource.html) grant; [permissions boundaries](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html), [session policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session), and [SCPs](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html)/[RCPs](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_rcps.html) only cap what can be granted. +- A role needs both a [trust policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html#id_roles_terms-and-concepts) allowing the principal to assume it and permissions for the resulting session to perform actions. ### Workload credentials -- Use **temporary role credentials** for EC2, ECS tasks, and Lambda; use OIDC federation for external CI so pipelines do not store long-lived AWS keys. -- In ECS, the **task role** grants the application access to AWS APIs; the **execution role** lets the ECS agent pull images, fetch configured secrets, and send logs. +- Use [temporary role credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) for EC2, ECS tasks, and Lambda; use [OIDC federation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html) for external CI so pipelines do not store long-lived AWS keys. +- In ECS, the [task role](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) grants the application access to AWS APIs; the [execution role](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html) lets the ECS agent pull images, fetch configured secrets, and send logs. ### Cross-account access -- Within one account, an allow in the identity policy **or** the resource policy suffices (KMS key policies and role trust policies are exceptions). -- Across accounts, **both** sides must allow: the resource policy trusts the other account or principal, and that principal's identity policy grants the action. -- Third parties assume your role with an **`sts:ExternalId`** condition, preventing the confused-deputy problem. -- `aws:PrincipalOrgID` in a resource policy limits access to accounts in your organization. +- Within one account, an allow in the identity policy or the resource policy suffices ([KMS key policies](https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html) and role trust policies are exceptions). +- [Across accounts](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic-cross-account.html), both sides must allow: the resource policy trusts the other account or principal, and that principal's identity policy grants the action. +- [Third parties](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html) assume your role with an `sts:ExternalId` condition, preventing the [confused-deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html). +- [`aws:PrincipalOrgID`](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_condition-keys.html#condition-keys-principalorgid) in a resource policy limits access to accounts in your organization. ### KMS and envelope encryption -- `GenerateDataKey` returns a data key in plaintext (use it, then discard it) and wrapped under the KMS key (store it with the data) — the envelope pattern in [security.md](security.md); KMS `Encrypt` itself takes at most **4 KB**. -- Every KMS key has a **key policy**; IAM policies grant access only if the key policy delegates to the account. +- [`GenerateDataKey`](https://docs.aws.amazon.com/kms/latest/APIReference/API_GenerateDataKey.html) returns a data key in plaintext (use it, then discard it) and wrapped under the KMS key (store it with the data) — the envelope pattern in [security.md](security.md); KMS [`Encrypt`](https://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html) itself takes at most 4 KB. +- Every KMS key has a [key policy](https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html); IAM policies grant access only if the [key policy delegates to the account](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html). ## Networking ### Public and private subnets -- A subnet is **public** when its route table points internet-bound traffic to an internet gateway; an EC2 instance also needs a public address and permissive rules to receive internet traffic. -- A **NAT gateway** in a public subnet lets private-subnet instances initiate IPv4 internet connections; the private subnet routes outbound traffic to it. +- A subnet is [public](https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html) when its route table points internet-bound traffic to an [internet gateway](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html); an EC2 instance also needs a public address and permissive rules to receive internet traffic. +- A [NAT gateway](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html) in a public subnet lets private-subnet instances initiate IPv4 internet connections; the private subnet routes outbound traffic to it. ### Network filters and endpoints -- **Security groups** are stateful and attach to network interfaces; **network ACLs** are stateless and apply at subnet boundaries, so return traffic needs an explicit ACL rule. -- **Gateway VPC endpoints** route S3 and DynamoDB traffic without a NAT device or internet gateway; interface endpoints use private IPs and AWS PrivateLink for supported services. +- [Security groups](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-security-groups.html) are stateful and attach to network interfaces; [network ACLs](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-network-acls.html) are stateless and apply at subnet boundaries, so return traffic needs an explicit ACL rule. +- [Gateway VPC endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/gateway-endpoints.html) route S3 and DynamoDB traffic without a NAT device or internet gateway; [interface endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/privatelink-access-aws-services.html) use private IPs and [AWS PrivateLink](https://docs.aws.amazon.com/vpc/latest/privatelink/what-is-privatelink.html) for supported services. ### ALB vs NLB | | ALB | NLB | |---|---|---| -| Layer | L7: HTTP(S), gRPC, WebSockets | L4: TCP, UDP, TLS | -| Routing | host, path, header, query string | port only | -| Addresses | changing IPs; use the DNS name | **static IP per AZ** (Elastic IPs optional) | -| Client IP | in `X-Forwarded-For` | preserved, or sent via PROXY protocol v2 | -| Extras | OIDC/Cognito auth, WAF, Lambda targets | TLS passthrough, PrivateLink services, very high throughput | -| Cross-zone | on by default | **off** by default; cross-AZ data is charged when on | +| Layer | [L7](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html): HTTP(S), gRPC, WebSockets | [L4](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html): TCP, UDP, TLS, QUIC | +| Routing | host, path, header, query string | port; [listener rules](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-listeners.html#listener-rules) route by source IP version (since September 2026) | +| Addresses | changing IPs; use the DNS name | static IP per AZ ([Elastic IPs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html) optional) | +| Client IP | in [`X-Forwarded-For`](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/x-forwarded-headers.html) | preserved, or sent via [PROXY protocol v2](https://www.haproxy.org/download/3.2/doc/proxy-protocol.txt) | +| Extras | [OIDC/Cognito auth](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-authenticate-users.html), [WAF](https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html), [Lambda targets](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html) | TLS passthrough, PrivateLink services, very high throughput | +| [Cross-zone](https://docs.aws.amazon.com/elasticloadbalancing/latest/userguide/how-elastic-load-balancing-works.html#cross-zone-load-balancing) | on by default | off by default; cross-AZ data is charged when on | -Gateway Load Balancer inserts inline appliances such as third-party firewalls. +[Gateway Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/gateway/introduction.html) inserts inline appliances such as third-party firewalls. ### Connecting VPCs | Option | Topology | Note | |---|---|---| -| **VPC peering** | one-to-one | **not transitive**, CIDRs can't overlap, no per-GB fee within an AZ | -| **Transit Gateway** | hub and spoke | transitive routing across many VPCs and on-prem; per-attachment and per-GB fees | -| **PrivateLink** | one service, consumer → provider | overlapping CIDRs fine; exposes one endpoint, not a network | +| [VPC peering](https://docs.aws.amazon.com/vpc/latest/peering/what-is-vpc-peering.html) | one-to-one | [not transitive](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-basics.html), CIDRs can't overlap, no per-GB fee within an AZ | +| [Transit Gateway](https://docs.aws.amazon.com/vpc/latest/tgw/what-is-transit-gateway.html) | hub and spoke | transitive routing across many VPCs and on-prem; per-attachment and per-GB fees | +| [PrivateLink](https://docs.aws.amazon.com/vpc/latest/privatelink/what-is-privatelink.html) | one service, consumer → provider | overlapping CIDRs fine; exposes one endpoint, not a network | ### Route 53 routing -- Policies: simple, **weighted** (canary), latency, **failover** (with health checks), geolocation, geoproximity, multivalue. -- **Alias records** work at the zone apex (`example.com` → ALB/CloudFront), where a CNAME isn't allowed; queries are free when the target is an AWS resource, but billed when it is a plain record in the same zone. +- [Policies](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy.html): simple, [weighted](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy-weighted.html) (canary), latency, [failover](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-policy-failover.html) (with [health checks](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/dns-failover.html)), geolocation, geoproximity, multivalue. +- [Alias records](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resource-record-sets-choosing-alias-non-alias.html) work at the zone apex (`example.com` → ALB/CloudFront), where a CNAME isn't allowed; queries are free when the target is an AWS resource, but billed when it is a plain record in the same zone. ## Compute @@ -71,107 +71,107 @@ Gateway Load Balancer inserts inline appliances such as third-party firewalls. | Service | Use when | You still manage | |---|---|---| -| EC2 | OS, runtime, or host control matters | instances, patching, scaling | -| ECS on Fargate | containers without managing nodes | task definitions, services | -| EKS | Kubernetes APIs and ecosystem are required | workloads; nodes unless Auto Mode or Fargate | -| Lambda | event-driven, bounded work | concurrency, timeouts, event handling | +| [EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/concepts.html) | OS, runtime, or host control matters | instances, patching, scaling | +| [ECS on Fargate](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html) | containers without managing nodes | [task definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html), services | +| [EKS](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html) | Kubernetes APIs and ecosystem are required | workloads; nodes unless [Auto Mode](https://docs.aws.amazon.com/eks/latest/userguide/automode.html) or Fargate | +| [Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) | event-driven, bounded work | concurrency, timeouts, event handling | ### Lambda limits and concurrency -- Maximum timeout **15 minutes**; synchronous payloads 6 MB each way (streamed responses allow more). -- **Reserved concurrency** guarantees capacity and caps a function; provisioned concurrency pre-initializes environments to reduce startup latency and incurs separate charges. +- Maximum timeout 15 minutes; synchronous payloads 6 MB each way ([quotas](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html); [streamed responses](https://docs.aws.amazon.com/lambda/latest/dg/configuration-response-streaming.html) allow more). +- [Reserved concurrency](https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html) guarantees capacity and caps a function; [provisioned concurrency](https://docs.aws.amazon.com/lambda/latest/dg/provisioned-concurrency.html) pre-initializes environments to reduce startup latency and incurs separate charges. ### Lambda cold starts and scaling -- A cold start downloads code, starts the runtime, and runs init code; the **INIT phase is billed since August 2025**. -- Reduce it with provisioned concurrency or **SnapStart** (Java, Python, .NET), which restores a snapshot of an initialized environment — make sure randomness and connections are recreated after restore. -- Default 1,000 concurrent executions per account per Region; each function scales by up to **1,000 more every 10 seconds**. +- A cold start downloads code, starts the runtime, and runs [init code](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtime-environment.html); the [INIT phase is billed](https://aws.amazon.com/blogs/compute/aws-lambda-standardizes-billing-for-init-phase/) since August 2025. +- Reduce it with provisioned concurrency or [SnapStart](https://docs.aws.amazon.com/lambda/latest/dg/snapstart.html) (Java, Python, .NET), which restores a snapshot of an initialized environment — make sure [randomness](https://docs.aws.amazon.com/lambda/latest/dg/snapstart-uniqueness.html) and [connections](https://docs.aws.amazon.com/lambda/latest/dg/snapstart-best-practices.html#snapstart-networking) are recreated after restore. +- Default 1,000 concurrent executions per account per Region; each function [scales](https://docs.aws.amazon.com/lambda/latest/dg/scaling-behavior.html) by up to 1,000 more every 10 seconds. ### API Gateway vs ALB -- API Gateway REST API: auth, throttling, usage plans, request validation; integration timeout **29 s** by default (raisable for Regional and private APIs). -- **HTTP API**: cheaper, JWT/Lambda/IAM authorizers and throttling, but no usage plans or request validation; integration timeout max **30 s**. -- ALB in front of Lambda or containers can authenticate via OIDC/Cognito but has no throttling, usage plans, or validation; LCU pricing is usually cheaper at sustained high volume. +- API Gateway [REST API](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-vs-rest.html): auth, throttling, [usage plans](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-api-usage-plans.html), [request validation](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-method-request-validation.html); integration timeout 29 s by default (raisable for Regional and private APIs, see [quotas](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-execution-service-limits-table.html)). +- [HTTP API](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api.html): cheaper, [JWT](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-jwt-authorizer.html)/Lambda/IAM authorizers and throttling, but no usage plans or request validation; integration timeout max 30 s ([quotas](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-quotas.html)). +- ALB in front of Lambda or containers can [authenticate via OIDC/Cognito](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/listener-authenticate-users.html) but has no throttling, usage plans, or validation; [LCU pricing](https://aws.amazon.com/elasticloadbalancing/pricing/) is usually cheaper at sustained high volume. ## Storage and databases ### S3 consistency and versions -- S3 has **strong read-after-write consistency** for object PUT and DELETE, including list results; a successfully written object is immediately readable. -- With **versioning**, deleting an object without a version ID adds a delete marker; deleting a specific version permanently removes it. Lifecycle rules can expire noncurrent versions. -- A **presigned URL** grants temporary access using the signer's permissions and expires no later than the credentials used to sign it. +- S3 has [strong read-after-write consistency](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#ConsistencyModel) for object PUT and DELETE, including list results; a successfully written object is immediately readable. +- With [versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html), deleting an object without a version ID adds a [delete marker](https://docs.aws.amazon.com/AmazonS3/latest/userguide/DeleteMarker.html); deleting a specific version permanently removes it. [Lifecycle rules](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lifecycle-mgmt.html) can expire noncurrent versions. +- A [presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) grants temporary access using the signer's permissions and expires no later than the credentials used to sign it. ### S3 access and CloudFront -- New buckets block public access and disable ACLs (**bucket owner enforced**) by default since April 2023; objects are encrypted with SSE-S3 by default since January 2023. -- Serve a private bucket through CloudFront with **Origin Access Control**, which replaces the legacy OAI; the bucket policy allows only that distribution. -- CloudFront **signed URLs or signed cookies** gate private content at the edge; S3 presigned URLs bypass the CDN. +- New buckets [block public access](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-block-public-access.html) and disable ACLs ([bucket owner enforced](https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html)) by default since April 2023; objects are encrypted with [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html) by default since January 2023. +- Serve a private bucket through CloudFront with [Origin Access Control](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-restricting-access-to-s3.html), which replaces the legacy OAI; the bucket policy allows only that distribution. +- CloudFront [signed URLs or signed cookies](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/PrivateContent.html) gate private content at the edge; S3 presigned URLs bypass the CDN. ### S3 performance and classes -- Each **prefix** supports about **3,500 writes and 5,500 reads per second**; spread hot keys over prefixes. -- Classes trade storage price against retrieval cost and latency: Standard → Standard-IA → Glacier Instant/Flexible/Deep Archive; **Intelligent-Tiering** moves objects automatically. +- Each [prefix](https://docs.aws.amazon.com/AmazonS3/latest/userguide/optimizing-performance.html) supports about 3,500 writes and 5,500 reads per second; spread hot keys over prefixes. +- [Classes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) trade storage price against retrieval cost and latency: Standard → Standard-IA → Glacier Instant/Flexible/Deep Archive; [Intelligent-Tiering](https://docs.aws.amazon.com/AmazonS3/latest/userguide/intelligent-tiering.html) moves objects automatically. ### Block and shared files -- **EBS** is block storage attached to EC2 instances in the same Availability Zone; Multi-Attach works only for eligible volumes and instances and requires a clustered filesystem for simultaneous writes. -- **EFS** is shared file storage mountable by instances across Availability Zones in one Region. +- [EBS](https://docs.aws.amazon.com/ebs/latest/userguide/what-is-ebs.html) is block storage attached to EC2 instances in the same Availability Zone; [Multi-Attach](https://docs.aws.amazon.com/ebs/latest/userguide/ebs-volumes-multi.html) works only for eligible volumes and instances and requires a clustered filesystem for simultaneous writes. +- [EFS](https://docs.aws.amazon.com/efs/latest/ug/whatisefs.html) is shared file storage mountable by instances across Availability Zones in one Region. ### Managed database behavior -- An RDS **Multi-AZ DB instance** uses a synchronous standby for failover; that standby does not serve reads. Use a read replica or Multi-AZ DB cluster for read scaling. -- DynamoDB reads are **eventually consistent by default**; strong reads are available on tables and local secondary indexes, **not GSIs**. -- DynamoDB global tables support multi-Region eventual and multi-Region strong consistency (strong since June 2025). +- An RDS [Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) uses a synchronous standby for failover; that standby does not serve reads. Use a [read replica](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html) or [Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) for read scaling. +- DynamoDB reads are [eventually consistent by default](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.ReadConsistency.html); strong reads are available on tables and local secondary indexes, not GSIs. +- DynamoDB [global tables](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GlobalTables.html) support multi-Region eventual and multi-Region strong consistency (strong since June 2025). ### Aurora storage -- The storage volume keeps **6 copies across 3 AZs**; a write needs **4 of 6**, so writes survive losing an AZ; the **3-of-6** read quorum survives an AZ plus one more copy. +- The [storage volume](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Overview.StorageReliability.html) keeps 6 copies across 3 AZs; a write needs 4 of 6, so writes survive losing an AZ; the 3-of-6 read [quorum](https://www.amazon.science/publications/amazon-aurora-design-considerations-for-high-throughput-cloud-native-relational-databases) survives an AZ plus one more copy. - Normal reads go to one storage node known to be current; quorum reads are only needed during recovery. -- Up to 15 replicas share the same storage volume, so replica lag is usually well under 100 ms and failover doesn't copy data. +- Up to 15 [replicas](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Replication.html) share the same storage volume, so replica lag is usually well under 100 ms and failover doesn't copy data. ### DynamoDB partitions and indexes -- Each partition serves about **3,000 RCU and 1,000 WCU** and holds ~10 GB; a hot partition key throttles even if table capacity is fine. -- GSIs have their own capacity and are eventually consistent; a throttled GSI **throttles writes to the base table**. LSIs must be defined at table creation. -- Items are at most **400 KB**; transactions cover up to 100 items; TTL deletes are background work that can lag by days. +- Each [partition](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/bp-partition-key-design.html) serves about 3,000 RCU and 1,000 WCU and holds ~10 GB; a hot partition key throttles even if table capacity is fine. +- [GSIs](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html) have their own capacity and are eventually consistent; a throttled GSI [throttles writes to the base table](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/gsi-throttling.html). [LSIs](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LSI.html) must be defined at table creation. +- Items are at most 400 KB ([quotas](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html)); [transactions](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/transaction-apis.html) cover up to 100 items; [TTL](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/TTL.html) deletes are background work that can lag by days. ## Events and workflows ### SQS delivery -- A received SQS message stays in the queue but is hidden for its **visibility timeout**; delete it after successful processing. Expiry or duplicate delivery can expose it again, so consumers still need idempotency. -- FIFO queues order messages **within a message group**. A repeated deduplication ID suppresses duplicate sends only within a **5-minute window**; it is not permanent business-level deduplication. +- A received SQS message stays in the queue but is hidden for its [visibility timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html); delete it after successful processing. Expiry or duplicate delivery can expose it again, so consumers still need idempotency. +- [FIFO queues](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-fifo-queues.html) order messages within a message group. A repeated [deduplication ID](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagededuplicationid-property.html) suppresses duplicate sends only within a 5-minute window; it is not permanent business-level deduplication. ### Lambda with SQS -- Lambda polls SQS and invokes the function with a batch; by default, one failed record makes the whole batch visible again after the visibility timeout. Enable **partial batch responses** to retry only failed records. -- Set the queue visibility timeout to **6× the function timeout** plus the batching window (Lambda rejects anything below 1×); configure the **DLQ on the SQS queue**, not the Lambda asynchronous-failure destination. +- [Lambda polls SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) and invokes the function with a batch; by default, one failed record makes the whole batch visible again after the visibility timeout. Enable [partial batch responses](https://docs.aws.amazon.com/lambda/latest/dg/services-sqs-errorhandling.html) to retry only failed records. +- Set the queue visibility timeout to 6× the function timeout plus the batching window (Lambda rejects anything below 1×); configure the [DLQ on the SQS queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html), not the Lambda [asynchronous-failure destination](https://docs.aws.amazon.com/lambda/latest/dg/invocation-async-retain-records.html). ### Routing and orchestration -- SNS pushes to subscribers; EventBridge matches event patterns and routes to targets, with optional archive/replay; SQS retains work for consumers to poll. -- Step Functions **Standard** workflows use exactly-once execution unless a task has explicit retries; **asynchronous Express** is at-least-once, while **synchronous Express** is at-most-once. Retried external side effects still need idempotency. +- [SNS](https://docs.aws.amazon.com/sns/latest/dg/welcome.html) pushes to subscribers; EventBridge matches [event patterns](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) and routes to targets, with optional [archive/replay](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-archive.html); [SQS](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/welcome.html) retains work for consumers to poll. +- Step Functions [Standard](https://docs.aws.amazon.com/step-functions/latest/dg/choosing-workflow-type.html) workflows use exactly-once execution unless a task has explicit retries; asynchronous Express is at-least-once, while synchronous Express is at-most-once. Retried external side effects still need idempotency. ## Operations ### Observability and audit -- CloudWatch holds metrics, logs, and alarms; **CloudTrail** records account API activity. CloudTrail **data events** such as S3 object access are not logged by default when creating a trail. -- **VPC Flow Logs** record accepted/rejected IP traffic metadata, not packet payloads — for debugging routes and security rules. +- [CloudWatch](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/WhatIsCloudWatch.html) holds metrics, logs, and alarms; [CloudTrail](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-user-guide.html) records account API activity. CloudTrail [data events](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/logging-data-events-with-cloudtrail.html) such as S3 object access are not logged by default when creating a trail. +- [VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html) record accepted/rejected IP traffic metadata, not packet payloads — for debugging routes and security rules. ### Infrastructure changes and secrets -- CloudFormation **change sets** preview resource replacements or deletions before execution. Standard change sets compare templates; **drift-aware change sets** also compare actual resource state. -- Secrets Manager can **rotate** a secret in both its store and the backing service; use runtime retrieval or refresh so an application picks up the new value. +- CloudFormation [change sets](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-changesets.html) preview resource replacements or deletions before execution. Standard change sets compare templates; [drift-aware change sets](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/drift-aware-change-sets.html) also compare actual resource state. +- Secrets Manager can [rotate](https://docs.aws.amazon.com/secretsmanager/latest/userguide/rotating-secrets.html) a secret in both its store and the backing service; use runtime retrieval or refresh so an application picks up the new value. ### Capacity and cost guardrails -- Service **quotas** are per account and Region and often stop scaling before the application does — many are soft and raised by request. -- EC2 **Spot** capacity can be reclaimed; stop/terminate interruption notices give about **2 minutes** on a best-effort basis (hibernation starts immediately). Checkpoint work and replace capacity proactively. +- [Service quotas](https://docs.aws.amazon.com/servicequotas/latest/userguide/intro.html) are per account and Region and often stop scaling before the application does — many are soft and raised by request. +- EC2 [Spot](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-spot-instances.html) capacity can be reclaimed; stop/terminate [interruption notices](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-instance-termination-notices.html) give about 2 minutes on a best-effort basis (hibernation starts immediately). Checkpoint work and replace capacity proactively. ### Cost drivers -- **NAT gateway** data processing (~$0.045/GB in us-east-1, plus hourly) is a classic surprise; S3 and DynamoDB gateway endpoints are free and skip it. -- **Cross-AZ traffic** costs ~$0.01/GB in each direction, so chatty services and replicas spread across AZs add up. -- Internet egress and CloudWatch Logs ingestion (~$0.50/GB) are the other usual suspects. -- Discounts: Savings Plans and Reserved Instances (up to ~72% for 1–3 year commitments), Spot (up to ~90%), Graviton (roughly 20% cheaper per hour than comparable x86). +- [NAT gateway](https://aws.amazon.com/vpc/pricing/) data processing (~$0.045/GB in us-east-1, plus hourly) is a classic surprise; S3 and DynamoDB gateway endpoints are free and skip it. +- [Cross-AZ traffic](https://aws.amazon.com/ec2/pricing/on-demand/) costs ~$0.01/GB in each direction, so chatty services and replicas spread across AZs add up. +- Internet egress and [CloudWatch Logs](https://aws.amazon.com/cloudwatch/pricing/) ingestion (~$0.50/GB) are the other usual suspects. +- Discounts: [Savings Plans](https://aws.amazon.com/savingsplans/) and [Reserved Instances](https://aws.amazon.com/ec2/pricing/reserved-instances/) (up to ~72% for 1–3 year commitments), [Spot](https://aws.amazon.com/ec2/spot/) (up to ~90%), [Graviton](https://aws.amazon.com/ec2/graviton/) (roughly 20% cheaper per hour than comparable x86). diff --git a/docs/topics/devops.md b/docs/topics/devops.md index a75e3f0..66010d3 100644 --- a/docs/topics/devops.md +++ b/docs/topics/devops.md @@ -8,210 +8,210 @@ What experienced DevOps engineers forget before an interview, grouped by subtopi | Strategy | Mechanism | Trade-off | |---|---|---| -| Rolling | replace instances gradually | little spare capacity; old and new run side by side | -| Blue-green | two full environments, switch traffic at once | instant rollback; double capacity | -| Canary | small traffic share first, then expand or revert | smallest blast radius; needs good metrics | +| [Rolling](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#rolling-update-deployment) | replace instances gradually | little spare capacity; old and new run side by side | +| [Blue-green](https://martinfowler.com/bliki/BlueGreenDeployment.html) | two full environments, switch traffic at once | instant rollback; double capacity | +| [Canary](https://martinfowler.com/bliki/CanaryRelease.html) | small traffic share first, then expand or revert | smallest blast radius; needs good metrics | -- Every strategy needs health checks and an **automatic stop/rollback** condition. +- Every strategy needs health checks and an [automatic stop/rollback](https://argo-rollouts.readthedocs.io/en/stable/features/analysis/) condition. ### Feature flags -- Flags **decouple deploy from release**: code ships dark and is turned on per user, percentage, or region — and turned off without a deploy. +- [Flags](https://martinfowler.com/articles/feature-toggles.html) decouple deploy from release: code ships [dark](https://martinfowler.com/bliki/DarkLaunching.html) and is turned on per user, percentage, or region — and turned off without a deploy. - Stale flags are debt: give each one an owner and a removal date. ### Rollbacks -- Roll back by shifting traffic or redeploying the previous **immutable artifact**; it never undoes data changes already made. +- Roll back by shifting traffic or redeploying the previous [immutable artifact](https://martinfowler.com/bliki/ImmutableServer.html); it never undoes data changes already made. - So the previous version must run against the new schema — keep migrations backward compatible. ### Zero-downtime schema changes -- **Expand-and-contract**: add the new schema (backward compatible) → deploy code that handles both → backfill → remove the old schema in a later release. +- [Expand-and-contract](https://martinfowler.com/bliki/ParallelChange.html): add the new schema (backward compatible) → deploy code that handles both → backfill → remove the old schema in a later release. - A destructive migration (dropping a column old code still reads) makes rollback impossible. -- Keep each DDL short and lock-safe (`lock_timeout`, concurrent index builds) — see [database.md](database.md). +- Keep each DDL short and lock-safe ([`lock_timeout`](https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-LOCK-TIMEOUT), [concurrent index builds](https://www.postgresql.org/docs/current/sql-createindex.html#SQL-CREATEINDEX-CONCURRENTLY)) — see [database.md](database.md). ### Incident response -- Assign roles: an **incident commander** coordinates while others handle communications and operations. -- **Mitigate first** (roll back, flip a flag, fail over); find the root cause later. -- Write a blameless postmortem: timeline, contributing factors, and action items with owners. -- Track time to detect and time to recover (MTTD, MTTR); link a runbook from every alert. +- Assign roles: an [incident commander](https://sre.google/sre-book/managing-incidents/) coordinates while others handle communications and operations. +- Mitigate first (roll back, flip a flag, fail over); find the root cause later. +- Write a [blameless postmortem](https://sre.google/sre-book/postmortem-culture/): timeline, contributing factors, and action items with owners. +- Track time to detect and [time to recover](https://en.wikipedia.org/wiki/Mean_time_to_recovery) (MTTD, MTTR); link a [runbook](https://en.wikipedia.org/wiki/Runbook) from every alert. ## Containers ### How containers work -- A container is a Linux process with **namespaces** (isolated view: PID, network, mount, UTS, IPC, user) and **cgroups** (resource limits: CPU, memory, I/O). -- The filesystem is image layers stacked with OverlayFS plus a thin writable layer; all containers share the **host kernel**, unlike VMs. -- The OCI runtime (runc) sets this up; containerd/CRI-O manage images and lifecycles above it. +- A container is a Linux process with [namespaces](https://man7.org/linux/man-pages/man7/namespaces.7.html) (isolated view: PID, network, mount, UTS, IPC, user) and [cgroups](https://man7.org/linux/man-pages/man7/cgroups.7.html) (resource limits: CPU, memory, I/O). +- The filesystem is image layers stacked with [OverlayFS](https://docs.kernel.org/filesystems/overlayfs.html) plus a thin writable layer; all containers share the host kernel, unlike VMs. +- The [OCI runtime](https://github.com/opencontainers/runtime-spec) ([runc](https://github.com/opencontainers/runc)) sets this up; [containerd](https://containerd.io/docs/)/[CRI-O](https://cri-o.io/) manage images and lifecycles above it. ### Dockerfile practices -- **Multi-stage builds** keep compilers out of the final image; pin base images by digest; copy dependency manifests before source to reuse cached layers. -- Run as **non-root**; never put secrets in `ARG`, `ENV`, or copied files — layers keep them even if a later layer deletes them (use BuildKit secret mounts). +- [Multi-stage builds](https://docs.docker.com/build/building/multi-stage/) keep compilers out of the final image; [pin base images by digest](https://docs.docker.com/build/building/best-practices/#pin-base-image-versions); copy dependency manifests before source to reuse [cached layers](https://docs.docker.com/build/cache/). +- Run as [non-root](https://docs.docker.com/build/building/best-practices/#user); never put secrets in [`ARG`](https://docs.docker.com/reference/dockerfile/#arg), [`ENV`](https://docs.docker.com/reference/dockerfile/#env), or copied files — layers keep them even if a later layer deletes them (use BuildKit [secret mounts](https://docs.docker.com/build/building/secrets/)). ### PID 1 and signals -- PID 1 gets no default signal handlers, so an app not written for it may ignore **`SIGTERM`** and be killed after the grace period. -- Use the **exec form** (`CMD ["app"]`, not `CMD app`, which wraps it in a shell) and a tiny init (`tini`, `--init`) to forward signals and reap zombies. +- PID 1 gets no [default signal handlers](https://man7.org/linux/man-pages/man2/kill.2.html), so an app not written for it may ignore [`SIGTERM`](https://man7.org/linux/man-pages/man7/signal.7.html) and be killed after the grace period. +- Use the [exec form](https://docs.docker.com/reference/dockerfile/#shell-and-exec-form) (`CMD ["app"]`, not `CMD app`, which wraps it in a shell) and a tiny init ([`tini`](https://github.com/krallin/tini), [`--init`](https://docs.docker.com/reference/cli/docker/container/run/#init)) to forward signals and reap zombies. ## Kubernetes architecture ### Control plane -- The **API server** is the only component talking to **etcd**; the scheduler assigns Pods to nodes; **controllers** reconcile actual state toward desired state. -- Everything is declarative and level-triggered: `kubectl apply` succeeding means the object was stored, not that the app is healthy. -- A bare Pod has no controller, so nothing recreates it after a node failure — run workloads through a Deployment, StatefulSet, DaemonSet, or Job. +- The [API server](https://kubernetes.io/docs/concepts/architecture/#kube-apiserver) is the only component talking to [etcd](https://etcd.io/docs/); the [scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) assigns Pods to nodes; [controllers](https://kubernetes.io/docs/concepts/architecture/controller/) reconcile actual state toward desired state. +- Everything is [declarative](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/) and level-triggered: `kubectl apply` succeeding means the object was stored, not that the app is healthy. +- A [bare Pod](https://kubernetes.io/docs/concepts/workloads/pods/#working-with-pods) has no controller, so nothing recreates it after a node failure — run workloads through a [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/), [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/), [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), or [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/). ### Node components -- **kubelet** runs the node's Pods through the container runtime (CRI) and reports status. -- **kube-proxy** or an **eBPF** CNI such as Cilium implements Service virtual IPs. +- [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) runs the node's Pods through the container runtime ([CRI](https://kubernetes.io/docs/concepts/containers/cri/)) and reports status. +- [kube-proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) or an [eBPF](https://ebpf.io/what-is-ebpf/) CNI such as [Cilium](https://docs.cilium.io/en/stable/overview/intro/) implements [Service virtual IPs](https://kubernetes.io/docs/reference/networking/virtual-ips/). ### Packaging and extension -- **Helm** templates charts with values and tracks releases; **Kustomize** patches plain YAML with overlays, no templating. -- **CRDs + operators** extend the API: an operator's controller reconciles custom resources (databases, certificates) like built-in objects. +- [Helm](https://helm.sh/docs/) templates charts with values and tracks releases; [Kustomize](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/) patches plain YAML with overlays, no templating. +- [CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) + [operators](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/) extend the API: an operator's controller reconciles custom resources (databases, certificates) like built-in objects. ## Kubernetes workloads ### Rolling update tuning -- **`maxSurge`** (extra Pods above desired) and **`maxUnavailable`** (Pods allowed down) — both default **25%**. -- New Pods count as available only after readiness passes (plus `minReadySeconds`), so a broken readiness probe stalls the rollout safely. +- [`maxSurge`](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#max-surge) (extra Pods above desired) and [`maxUnavailable`](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#max-unavailable) (Pods allowed down) — both default 25%. +- New Pods count as available only after readiness passes (plus [`minReadySeconds`](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds)), so a broken readiness probe stalls the rollout safely. ### Sidecars and init containers -- **Init containers** run to completion, in order, before app containers start. -- **Native sidecars** (an init container with `restartPolicy: Always`, **GA in 1.33**) start before the app and stop after it — fixing Jobs that never finished because a sidecar kept running. +- [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) run to completion, in order, before app containers start. +- [Native sidecars](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) (an init container with `restartPolicy: Always`, GA in 1.33) start before the app and stop after it — fixing Jobs that never finished because a sidecar kept running. ### Scheduling constraints -- **Taints** on nodes repel Pods unless they have a matching toleration (dedicated GPU nodes). -- **Node affinity** attracts Pods to labeled nodes; pod anti-affinity or **topology spread constraints** spread replicas across nodes and zones. +- [Taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) on nodes repel Pods unless they have a matching toleration (dedicated GPU nodes). +- [Node affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) attracts Pods to labeled nodes; pod anti-affinity or [topology spread constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) spread replicas across nodes and zones. ### PodDisruptionBudgets -- A **PDB** (`minAvailable` or `maxUnavailable`) limits only disruptions that go through the **Eviction API** — node drains, autoscaler scale-down. +- A [PDB](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (`minAvailable` or `maxUnavailable`) limits only disruptions that go through the [Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) — [node drains](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/), autoscaler scale-down. - It doesn't stop crashes, direct Pod deletes, or Deployment rolling updates (those follow `maxUnavailable`). - A PDB that allows zero disruptions blocks node drains forever. ### Persistent volumes -- A **PVC** requests storage; a **StorageClass** provisions a matching PV dynamically. -- Access modes: `ReadWriteOnce` (one node — most block storage) vs `ReadWriteMany` (shared file systems such as EFS or NFS). -- **`WaitForFirstConsumer`** binding delays provisioning until the Pod is scheduled, so a zonal disk lands in the Pod's zone. -- Dynamic volumes default to reclaim policy `Delete` (the disk goes with the PVC); `Retain` keeps it. +- A [PVC](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) requests storage; a [StorageClass](https://kubernetes.io/docs/concepts/storage/storage-classes/) provisions a matching PV dynamically. +- [Access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes): `ReadWriteOnce` (one node — most block storage) vs `ReadWriteMany` (shared file systems such as EFS or NFS). +- [`WaitForFirstConsumer`](https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode) binding delays provisioning until the Pod is scheduled, so a zonal disk lands in the Pod's zone. +- Dynamic volumes default to [reclaim policy](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#reclaiming) `Delete` (the disk goes with the PVC); `Retain` keeps it. ## Kubernetes networking ### Services -- A **Service** gives a changing set of Pods (chosen by label selector) a stable virtual IP and DNS name: **ClusterIP** (internal), NodePort, LoadBalancer (cloud LB). -- No traffic? Check the selector and readiness first: `kubectl get endpointslices` (the Endpoints API is deprecated **since 1.33**). +- A [Service](https://kubernetes.io/docs/concepts/services-networking/service/) gives a changing set of Pods (chosen by label selector) a stable virtual IP and DNS name: [ClusterIP](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) (internal), NodePort, LoadBalancer (cloud LB). +- No traffic? Check the selector and readiness first: `kubectl get endpointslices` (the [Endpoints API is deprecated](https://kubernetes.io/blog/2025/04/24/endpoints-deprecation/) since 1.33). - General service-discovery patterns: see [system.md](system.md). ### Ingress and Gateway API -- **Ingress** and **Gateway API** route HTTP by host and path — neither does anything without a controller. -- **ingress-nginx was retired in March 2026** (no more fixes); new setups use Gateway API implementations (Envoy Gateway, Cilium, cloud controllers). +- [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) and [Gateway API](https://gateway-api.sigs.k8s.io/) route HTTP by host and path — neither does anything without a [controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/). +- [ingress-nginx was retired](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/) in March 2026 (no more fixes); new setups use Gateway API implementations ([Envoy Gateway](https://gateway.envoyproxy.io/), Cilium, cloud controllers). ### NetworkPolicy -- By default **all Pods can reach all Pods**; a NetworkPolicy selecting a Pod switches it to deny-by-default for the listed direction. -- Enforcement needs a CNI that supports it (Calico, Cilium) — otherwise policies are silently ignored. +- By default all Pods can reach all Pods; a [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) selecting a Pod switches it to deny-by-default for the listed direction. +- Enforcement needs a CNI that supports it ([Calico](https://docs.tigera.io/calico/latest/about/), [Cilium](https://docs.cilium.io/en/stable/security/policy/)) — otherwise policies are silently ignored. ## Kubernetes operations ### Probes -- **Readiness** gates Service traffic; **liveness** restarts a stuck container; startup holds off both during slow boots. -- A liveness probe that checks dependencies or times out under load causes **restart cascades** — keep it a cheap local check. +- [Readiness](https://kubernetes.io/docs/concepts/workloads/pods/probes/) gates Service traffic; liveness restarts a stuck container; startup holds off both during slow boots. +- A liveness probe that checks dependencies or times out under load causes restart cascades — keep it a cheap local check. ### Requests, limits, QoS -- Requests drive scheduling; limits cap usage — over the CPU limit the container is **throttled**. -- At the memory limit the kernel reclaims (page cache) first and **OOMKills** only if usage still can't stay under it. -- Requests vs limits set the QoS class (Guaranteed / Burstable / BestEffort), which decides eviction order under node pressure. -- In-place Pod resize (GA in **1.35**) changes CPU/memory requests without restarting the Pod. +- [Requests drive scheduling; limits cap usage](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) — over the CPU limit the container is [throttled](https://docs.kernel.org/scheduler/sched-bwc.html). +- At the memory limit the kernel reclaims (page cache) first and OOMKills only if usage still can't stay under it. +- Requests vs limits set the [QoS class](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/) (Guaranteed / Burstable / BestEffort), which decides eviction order under [node pressure](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/). +- [In-place Pod resize](https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/) (GA in 1.35) changes CPU/memory requests without restarting the Pod. ### Autoscaling -- **HPA** scales replicas from metrics; VPA adjusts requests; Cluster Autoscaler/Karpenter add nodes when Pods can't be scheduled. -- HPA utilization targets are a percentage of **requests**, so Pods without CPU requests can't scale on CPU; scale-down waits out a 5-minute stabilization window by default. -- **KEDA** scales on external events (queue length, consumer lag), down to zero; Karpenter also consolidates underused nodes. +- [HPA](https://kubernetes.io/docs/concepts/workloads/autoscaling/horizontal-pod-autoscale/) scales replicas from metrics; [VPA](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler) adjusts requests; [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler)/[Karpenter](https://karpenter.sh/docs/) add nodes when Pods can't be scheduled. +- HPA utilization targets are a percentage of requests, so Pods without CPU requests can't scale on CPU; scale-down waits out a 5-minute [stabilization window](https://kubernetes.io/docs/concepts/workloads/autoscaling/horizontal-pod-autoscale/#stabilization-window) by default. +- [KEDA](https://keda.sh/docs/latest/concepts/) scales on external events (queue length, consumer lag), down to zero; Karpenter also [consolidates](https://karpenter.sh/docs/concepts/disruption/#consolidation) underused nodes. - More replicas don't help when the bottleneck is a saturated downstream database. ### Graceful Pod shutdown -- On deletion, endpoint updates (EndpointSlice marks the Pod terminating, not ready) and the kubelet's shutdown run concurrently, so the Pod may still receive traffic briefly. -- The kubelet runs **`preStop`** first, then sends `SIGTERM` — a short `preStop` sleep covers that race. -- **`terminationGracePeriodSeconds`** (default **30 s**) covers `preStop` plus shutdown; then the kubelet sends `SIGKILL`. App-side steps are in [backend.md](backend.md). +- On [deletion](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination), endpoint updates ([EndpointSlice](https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/#conditions) marks the Pod terminating, not ready) and the kubelet's shutdown run concurrently, so the Pod may still receive traffic briefly. +- The kubelet runs [`preStop`](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) first, then sends `SIGTERM` — a short `preStop` sleep covers that race. +- `terminationGracePeriodSeconds` (default 30 s) covers `preStop` plus shutdown; then the kubelet sends `SIGKILL`. App-side steps are in [backend.md](backend.md). ### Debugging Pod states -- **`Pending`**: unschedulable (resources, quota, taints, unbound volume). **`ImagePullBackOff`**: wrong image or registry credentials. **`CrashLoopBackOff`**: the process or its liveness probe keeps failing. -- `kubectl describe pod` shows events; `kubectl logs --previous` shows the crashed container's last output. +- `Pending`: unschedulable (resources, quota, taints, unbound volume). [`ImagePullBackOff`](https://kubernetes.io/docs/concepts/containers/images/#imagepullbackoff): wrong image or registry credentials. [`CrashLoopBackOff`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-restarts): the process or its liveness probe keeps failing. +- [`kubectl describe pod`](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_describe/) shows events; [`kubectl logs --previous`](https://kubernetes.io/docs/reference/kubectl/generated/kubectl_logs/) shows the crashed container's last output. ## Kubernetes security ### Pod security -- **Pod Security Admission** enforces the privileged, baseline, or **restricted** profile per namespace via labels; PodSecurityPolicy was removed in 1.25. -- A hardened `securityContext`: `runAsNonRoot`, `readOnlyRootFilesystem`, `allowPrivilegeEscalation: false`, drop all capabilities, `seccompProfile: RuntimeDefault`. -- `privileged: true`, `hostPath` mounts, and `hostNetwork`/`hostPID` effectively hand over the node. -- Kyverno or OPA Gatekeeper enforce custom rules (allowed registries, no `latest` tags, required labels). +- [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) enforces the privileged, baseline, or [restricted](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile per namespace via labels; [PodSecurityPolicy](https://kubernetes.io/docs/concepts/security/pod-security-policy/) was removed in 1.25. +- A hardened [`securityContext`](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/): `runAsNonRoot`, `readOnlyRootFilesystem`, `allowPrivilegeEscalation: false`, drop all capabilities, `seccompProfile: RuntimeDefault`. +- `privileged: true`, [`hostPath`](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) mounts, and `hostNetwork`/`hostPID` effectively hand over the node. +- [Kyverno](https://kyverno.io/docs/introduction/) or [OPA Gatekeeper](https://open-policy-agent.github.io/gatekeeper/website/docs/) enforce custom rules (allowed registries, no `latest` tags, required labels). ### ServiceAccounts and workload identity -- Each Pod runs as a **ServiceAccount**; RBAC Roles bound to it limit what it may do through the Kubernetes API. -- Its tokens are projected, short-lived, and audience-bound; set `automountServiceAccountToken: false` when the app doesn't call the API. -- **Workload identity** maps a ServiceAccount to a cloud role (EKS IRSA or Pod Identity, GKE Workload Identity) — no static cloud keys in Secrets. +- Each Pod runs as a [ServiceAccount](https://kubernetes.io/docs/concepts/security/service-accounts/); [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) Roles bound to it limit what it may do through the Kubernetes API. +- Its tokens are [projected](https://kubernetes.io/docs/concepts/storage/projected-volumes/#serviceaccounttoken), short-lived, and audience-bound; set [`automountServiceAccountToken: false`](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#opt-out-of-api-credential-automounting) when the app doesn't call the API. +- Workload identity maps a ServiceAccount to a cloud role (EKS [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) or [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html), GKE [Workload Identity](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/workload-identity)) — no static cloud keys in Secrets. ### ConfigMaps and Secrets -- Secret values are only **base64-encoded**; enable encryption at rest (KMS), restrict access with **RBAC**, or sync from a vault (External Secrets Operator). -- Env vars from a ConfigMap don't update in a running Pod; mounted files do (eventually) — most apps still need a restart. +- [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) values are only base64-encoded; enable [encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) ([KMS](https://kubernetes.io/docs/tasks/administer-cluster/kms-provider/)), restrict access with RBAC, or sync from a vault ([External Secrets Operator](https://external-secrets.io/latest/)). +- Env vars from a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) don't update in a running Pod; [mounted files do](https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically) (eventually) — most apps still need a restart. ## Terraform ### Plan and apply -- `init` (providers, backend) → `plan` (diff) → `apply`; review every plan for **replacements** — replacing a stateful resource usually means data loss. -- `lifecycle { prevent_destroy = true }` and `create_before_destroy` guard critical resources. +- [`init`](https://developer.hashicorp.com/terraform/cli/commands/init) (providers, backend) → [`plan`](https://developer.hashicorp.com/terraform/cli/commands/plan) (diff) → [`apply`](https://developer.hashicorp.com/terraform/cli/commands/apply); review every plan for replacements — replacing a stateful resource usually means data loss. +- [`lifecycle`](https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle) `{ prevent_destroy = true }` and `create_before_destroy` guard critical resources. ### State and locking -- **State** maps resource addresses to real objects and can hold secrets — keep it in a **remote backend with locking**, never in Git. -- **Drift**: out-of-band changes show up in the next plan, which proposes reverting them. +- [State](https://developer.hashicorp.com/terraform/language/state) maps resource addresses to real objects and can [hold secrets](https://developer.hashicorp.com/terraform/language/manage-sensitive-data) — keep it in a [remote backend](https://developer.hashicorp.com/terraform/language/backend) with [locking](https://developer.hashicorp.com/terraform/language/state/locking), never in Git. +- Drift: out-of-band changes show up in the next plan, which proposes reverting them. ### count vs for_each -- `count` addresses instances by index (`aws_instance.web[2]`): removing one element shifts every later index, so Terraform **destroys and recreates** those resources. -- **`for_each`** keys instances by stable map or set keys (`aws_instance.web["api"]`), so adding or removing one touches only that one. +- [`count`](https://developer.hashicorp.com/terraform/language/meta-arguments/count) addresses instances by index (`aws_instance.web[2]`): removing one element shifts every later index, so each later instance takes its neighbor's config and is updated in place or replaced, and the last one is destroyed. +- [`for_each`](https://developer.hashicorp.com/terraform/language/meta-arguments/for_each) keys instances by stable map or set keys (`aws_instance.web["api"]`), so adding or removing one touches only that one. - `for_each` keys must be known at plan time, not computed during apply; switching between the two needs `moved` blocks. ### Refactoring resources -- **`moved`** blocks record a renamed address, so the plan doesn't destroy and recreate it. -- **`import`** blocks (1.5) adopt existing resources into state. -- **`removed`** blocks (1.7) drop a resource from config; the default destroys it — only `lifecycle { destroy = false }` keeps the real object. +- [`moved`](https://developer.hashicorp.com/terraform/language/modules/develop/refactoring) blocks record a renamed address, so the plan doesn't destroy and recreate it. +- [`import`](https://developer.hashicorp.com/terraform/language/import) blocks (1.5) adopt existing resources into state. +- [`removed`](https://developer.hashicorp.com/terraform/language/block/removed) blocks (1.7) drop a resource from config; the default destroys it — only `lifecycle { destroy = false }` keeps the real object. ### Modules, workspaces, OpenTofu -- Modules package resources behind inputs and outputs. -- Workspaces give one configuration several states but are **not an access boundary** — isolate environments with separate backends or root modules. -- **OpenTofu** is the open-source fork of Terraform with the same workflow. +- [Modules](https://developer.hashicorp.com/terraform/language/modules) package resources behind inputs and outputs. +- [Workspaces](https://developer.hashicorp.com/terraform/language/state/workspaces) give one configuration several states but are not an access boundary — isolate environments with separate backends or root modules. +- [OpenTofu](https://opentofu.org/docs/) is the open-source fork of Terraform with the same workflow. ## CI/CD and GitOps ### CI pipeline practices -- **Trunk-based development**: short-lived branches merged at least daily, unfinished work behind flags. -- **Build once, promote** the same image digest through environments — never rebuild per environment. -- Cache dependencies keyed on the lock-file hash, run fast checks first, and pin third-party CI actions by commit SHA. +- [Trunk-based development](https://trunkbaseddevelopment.com/): short-lived branches merged at least daily, unfinished work behind flags. +- Build once, promote the same image digest through environments — never rebuild per environment. +- [Cache dependencies](https://docs.github.com/en/actions/concepts/workflows-and-actions/dependency-caching) keyed on the lock-file hash, run fast checks first, and [pin third-party CI actions by commit SHA](https://docs.github.com/en/actions/reference/security/secure-use#using-third-party-actions). ### Pull-based reconciliation -- A controller (**Argo CD**, **Flux**) watches a Git repo of desired state and reconciles the cluster — the cluster pulls, so CI needs no cluster credentials. -- Manual `kubectl` changes are drift: Flux reapplies every interval, Argo CD reverts them only with **`selfHeal`** on; emergency fixes must land in Git too. +- A controller ([Argo CD](https://argo-cd.readthedocs.io/en/stable/), [Flux](https://fluxcd.io/flux/)) watches a Git repo of desired state and reconciles the cluster — the cluster pulls, so CI needs no cluster credentials. +- Manual `kubectl` changes are drift: Flux reapplies every interval, Argo CD reverts them only with [`selfHeal`](https://argo-cd.readthedocs.io/en/stable/user-guide/auto_sync/#automatic-self-healing) on; emergency fixes must land in Git too. diff --git a/docs/topics/ethereum.md b/docs/topics/ethereum.md index dda14ca..8c07dad 100644 --- a/docs/topics/ethereum.md +++ b/docs/topics/ethereum.md @@ -6,137 +6,137 @@ What experienced Ethereum engineers forget before an interview, grouped by subto ### Account nonces -- The **nonce** orders an EOA's transactions and prevents replay; an EIP-7702 authorization also increments it, even in someone else's transaction. +- The [nonce](https://ethereum.org/developers/docs/accounts/) orders an EOA's transactions and prevents replay; an [EIP-7702](https://eips.ethereum.org/EIPS/eip-7702) authorization also increments it, even in someone else's transaction. - A stuck transaction blocks every later nonce until replaced by one with the same nonce and a higher fee. ### Address derivation - EOA: last 20 bytes of `keccak256(pubkey)`. -- **`CREATE`**: last 20 bytes of `keccak256(rlp([sender, nonce]))`. -- **`CREATE2`**: derived from the deployer, a salt, and the init code hash, so an address is predictable before deployment (counterfactual wallets). +- `CREATE`: last 20 bytes of `keccak256(rlp([sender, nonce]))` ([RLP](https://ethereum.org/developers/docs/data-structures-and-encoding/rlp/)). +- [`CREATE2`](https://eips.ethereum.org/EIPS/eip-1014): derived from the deployer, a salt, and the init code hash, so an address is predictable before deployment (counterfactual wallets). ### Transaction types | Type | EIP | Adds | |---|---|---| | 0 | legacy | single `gasPrice` | -| 1 | 2930 | access lists (pre-warmed addresses and slots) | -| 2 | **1559** | base fee + priority fee | -| 3 | **4844** | blobs for rollup data | -| 4 | **7702** | set-code: an EOA delegates to contract code (Pectra, May 2025) | +| 1 | [2930](https://eips.ethereum.org/EIPS/eip-2930) | access lists (pre-warmed addresses and slots) | +| 2 | [1559](https://eips.ethereum.org/EIPS/eip-1559) | base fee + priority fee | +| 3 | [4844](https://eips.ethereum.org/EIPS/eip-4844) | blobs for rollup data | +| 4 | [7702](https://eips.ethereum.org/EIPS/eip-7702) | set-code: an EOA delegates to contract code ([Pectra](https://ethereum.org/roadmap/pectra/), May 2025) | ### Transaction lifecycle -- Signed → propagated to the **mempool** → a builder assembles a block (**PBS** via MEV-Boost separates building from proposing) → the proposer publishes it → validators attest. -- **Finalized after 2 epochs (~12.8 min)**; before that a block can be reorged, so exchanges and bridges wait for finality or several confirmations. +- Signed → propagated to the mempool → a builder assembles a block ([PBS](https://ethereum.org/roadmap/pbs/) via [MEV-Boost](https://docs.flashbots.net/flashbots-mev-boost/introduction) separates building from proposing) → the proposer publishes it → validators [attest](https://ethereum.org/developers/docs/consensus-mechanisms/pos/attestations/). +- [Finalized](https://ethereum.org/developers/docs/consensus-mechanisms/pos/#finality) after 2 epochs (~12.8 min); before that a block can be reorged, so exchanges and bridges wait for finality or several confirmations. ### RPC block tags and reorgs -- `latest` can be reorged; **`safe`** (the justified checkpoint) rarely is; **`finalized`** can't be reverted without slashing at least ⅓ of the stake. -- `eth_call` simulates against a chosen block without sending a transaction; `eth_estimateGas` binary-searches a working gas limit and can underestimate state-dependent paths, so add headroom. +- `latest` can be reorged; `safe` (the justified checkpoint) rarely is; `finalized` can't be reverted without slashing at least ⅓ of the stake ([block tags](https://ethereum.org/developers/docs/apis/json-rpc/#block-parameter)). +- [`eth_call`](https://ethereum.org/developers/docs/apis/json-rpc/#eth-call) simulates against a chosen block without sending a transaction; [`eth_estimateGas`](https://ethereum.org/developers/docs/apis/json-rpc/#eth-estimategas) binary-searches a working gas limit and can underestimate state-dependent paths, so add headroom. - Indexers store block hashes and roll back on a reorg, or index only finalized blocks. ## Gas and fees ### EIP-1559 fee market -- Fee = `gasUsed × (baseFee + priorityFee)`; users set `maxFeePerGas` and `maxPriorityFeePerGas`, and unused headroom is refunded. -- The **base fee** moves up to **±12.5%** per block toward a 50%-full target and is **burned**; the tip goes to the proposer. +- [Fee](https://ethereum.org/developers/docs/gas/) = `gasUsed × (baseFee + priorityFee)`; users set `maxFeePerGas` and `maxPriorityFeePerGas`, and unused headroom is refunded. +- The [base fee](https://eips.ethereum.org/EIPS/eip-1559) moves up to ±12.5% per block toward a 50%-full target and is burned; the tip goes to the proposer. - ETH supply shrinks whenever the burn exceeds new issuance. ### Calldata costs -- A transaction costs **21,000** gas base; calldata costs 16 gas per non-zero byte, 4 per zero byte. -- **EIP-7623** (Pectra) adds a floor of 40/10 gas per non-zero/zero byte for data-heavy transactions, pushing bulk data to blobs. +- A transaction costs 21,000 gas base; calldata costs 16 gas per non-zero byte ([EIP-2028](https://eips.ethereum.org/EIPS/eip-2028)), 4 per zero byte. +- [EIP-7623](https://eips.ethereum.org/EIPS/eip-7623) (Pectra) adds a floor of 40/10 gas per non-zero/zero byte for data-heavy transactions, pushing bulk data to blobs. ### Storage access costs -- EIP-2929: the first (**cold**) `SLOAD` of a slot costs **2,100**; the first touch of an address (`CALL`, `BALANCE`, `EXT*`) costs 2,600; any warm access costs 100. -- Clearing a slot to zero earns a refund, capped at **1/5** of the transaction's gas since London (EIP-3529, was 1/2). +- [EIP-2929](https://eips.ethereum.org/EIPS/eip-2929): the first (cold) `SLOAD` of a slot costs 2,100; the first touch of an address (`CALL`, `BALANCE`, `EXT*`) costs 2,600; any warm access costs 100. +- Clearing a slot to zero earns a refund, capped at 1/5 of the transaction's gas since London ([EIP-3529](https://eips.ethereum.org/EIPS/eip-3529), was 1/2). ### Gas limits -- **EIP-7825** (Fusaka, December 2025) caps one transaction at **2²⁴ ≈ 16.7M gas**, whatever the block limit. -- The block gas limit isn't fixed by protocol: validators move it by up to ~1/1024 per block, so any number is a snapshot. +- [EIP-7825](https://eips.ethereum.org/EIPS/eip-7825) ([Fusaka](https://ethereum.org/roadmap/fusaka/), December 2025) caps one transaction at 2²⁴ ≈ 16.7M gas, whatever the block limit. +- The [block gas limit](https://ethereum.org/developers/docs/blocks/#block-size) isn't fixed by protocol: validators move it by up to ~1/1024 per block, so any number is a snapshot. ## EVM ### Execution model -- Stack-based, **256-bit words**, stack depth 1024 — `DUP` reaches only the top **16** items (`SWAP16` the 17th), the source of "stack too deep". -- Memory expansion cost is **quadratic** in size; storage is the most expensive resource, priced per 32-byte slot. -- `DELEGATECALL` runs another contract's code on the caller's storage; any call forwards at most 63/64 of the remaining gas (EIP-150) — see [solidity.md](solidity.md). +- [Stack-based](https://ethereum.org/developers/docs/evm/), 256-bit words, stack depth 1024 — `DUP` reaches only the top 16 items (`SWAP16` the 17th), the source of "stack too deep" ([opcodes](https://ethereum.org/developers/docs/evm/opcodes/)). +- Memory expansion cost is quadratic in size; storage is the most expensive resource, priced per 32-byte slot. +- `DELEGATECALL` runs another contract's code on the caller's storage; any call forwards at most 63/64 of the remaining gas ([EIP-150](https://eips.ethereum.org/EIPS/eip-150)) — see [solidity.md](solidity.md). ### Precompiles -Built-in contracts at fixed low addresses for cryptography too expensive in bytecode; they have no code, so `extcodesize` returns 0. +Built-in [contracts at fixed low addresses](https://www.evm.codes/precompiled) for cryptography too expensive in bytecode; they have no code, so `extcodesize` returns 0. | Address | Function | |---|---| | `0x01` | `ecrecover` | | `0x02`, `0x03` | SHA-256, RIPEMD-160 | | `0x05` | modexp (RSA verification) | -| `0x06`–`0x08` | BN254 add, mul, pairing — **SNARK verifiers** | -| `0x0a` | KZG point evaluation (EIP-4844 blobs) | -| `0x0b`–`0x11` | BLS12-381 operations (Pectra, EIP-2537) | -| `0x100` | P-256 verification (Fusaka, EIP-7951) — passkeys and secure enclaves | +| `0x06`–`0x08` | BN254 add, mul, pairing ([EIP-196](https://eips.ethereum.org/EIPS/eip-196), [EIP-197](https://eips.ethereum.org/EIPS/eip-197)) — SNARK verifiers | +| `0x0a` | KZG point evaluation ([EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) blobs) | +| `0x0b`–`0x11` | BLS12-381 operations (Pectra, [EIP-2537](https://eips.ethereum.org/EIPS/eip-2537)) | +| `0x100` | P-256 verification (Fusaka, [EIP-7951](https://eips.ethereum.org/EIPS/eip-7951)) — passkeys and secure enclaves | ### Transient storage and SELFDESTRUCT -- **Transient storage** (`TSTORE`/`TLOAD`, EIP-1153, Cancun) costs 100 gas and is cleared after each transaction — cheap reentrancy locks. -- **`SELFDESTRUCT`** only sends ETH since Cancun (**EIP-6780**); code and storage are deleted only when called in the same transaction as creation. +- Transient storage (`TSTORE`/`TLOAD`, [EIP-1153](https://eips.ethereum.org/EIPS/eip-1153), [Cancun](https://ethereum.org/roadmap/dencun/)) costs 100 gas and is cleared after each transaction — cheap reentrancy locks. +- `SELFDESTRUCT` only sends ETH since Cancun ([EIP-6780](https://eips.ethereum.org/EIPS/eip-6780)); code and storage are deleted only when called in the same transaction as creation. ### Events and logs -- Logs live in the **receipt**, not storage — far cheaper, but contracts can't read them. -- Up to **3 indexed** parameters become searchable topics, plus the event signature hash as topic 0. +- [Logs](https://docs.soliditylang.org/en/latest/contracts.html#events) live in the receipt, not storage — far cheaper, but contracts can't read them. +- Up to 3 indexed parameters become searchable topics, plus the event signature hash as topic 0; anonymous events drop the signature topic and allow 4. ### World state and light clients -- World state (address → account) is a **Merkle Patricia Trie**; each contract's storage is its own trie. -- A block header commits to the state, transactions, and receipts roots, so a **light client** verifies a balance or receipt with a Merkle proof instead of the full state. +- World state (address → account) is a [Merkle Patricia Trie](https://ethereum.org/developers/docs/data-structures-and-encoding/patricia-merkle-trie/); each contract's storage is its own trie. +- A block header commits to the state, transactions, and receipts roots, so a [light client](https://ethereum.org/developers/docs/nodes-and-clients/light-clients/) verifies a balance or receipt with a Merkle proof instead of the full state. ## Signatures and standards ### Signing -- ECDSA over **secp256k1**, signature `(v, r, s)`; **`ecrecover`** returns `address(0)` for an invalid signature — always check. -- **EIP-191** (`personal_sign`) prefixes messages so a signed message can never double as a transaction. +- ECDSA over [secp256k1](https://www.secg.org/sec2-v2.pdf), signature `(v, r, s)`; [`ecrecover`](https://docs.soliditylang.org/en/latest/units-and-global-variables.html#mathematical-and-cryptographic-functions) returns `address(0)` for an invalid signature — always check. +- [EIP-191](https://eips.ethereum.org/EIPS/eip-191) (`personal_sign`) prefixes messages so a signed message can never double as a transaction. ### EIP-712 and permits -- **EIP-712** signs typed data with a domain separator (name, version, chainId, contract) — readable in wallets; stops cross-chain/cross-contract replay only if the verifier checks the domain, and same-contract replay still needs a nonce. -- **ERC-1271** lets contract wallets "sign" via `isValidSignature`. -- **EIP-2612 `permit`** grants an ERC-20 approval by signature, so the owner skips the `approve` transaction (anyone can submit it); Permit2 needs a one-time `approve` to its contract first. +- [EIP-712](https://eips.ethereum.org/EIPS/eip-712) signs typed data with a domain separator (name, version, chainId, contract) — readable in wallets; stops cross-chain/cross-contract replay only if the verifier checks the domain, and same-contract replay still needs a nonce. +- [ERC-1271](https://eips.ethereum.org/EIPS/eip-1271) lets contract wallets "sign" via `isValidSignature`. +- [EIP-2612](https://eips.ethereum.org/EIPS/eip-2612) `permit` grants an ERC-20 approval by signature, so the owner skips the `approve` transaction (anyone can submit it); [Permit2](https://developers.uniswap.org/docs/protocols/permit2/overview) needs a one-time `approve` to its contract first. ### Token standards | Standard | Kind | Note | |---|---|---| -| ERC-20 | fungible | `approve` + `transferFrom`; infinite approvals are a risk | -| ERC-721 | NFT, unique `tokenId` | `safeTransferFrom` calls `onERC721Received` (a reentrancy point) | -| ERC-1155 | multi-token | batch transfers | -| ERC-4626 | tokenized vault | share/asset math; first-depositor inflation attack | +| [ERC-20](https://eips.ethereum.org/EIPS/eip-20) | fungible | `approve` + `transferFrom`; infinite approvals are a risk | +| [ERC-721](https://eips.ethereum.org/EIPS/eip-721) | NFT, unique `tokenId` | `safeTransferFrom` calls `onERC721Received` (a reentrancy point) | +| [ERC-1155](https://eips.ethereum.org/EIPS/eip-1155) | multi-token | batch transfers | +| [ERC-4626](https://eips.ethereum.org/EIPS/eip-4626) | tokenized vault | share/asset math; [first-depositor inflation attack](https://docs.openzeppelin.com/contracts/5.x/erc4626#security-concern-inflation-attack) | ### Account abstraction -- **ERC-4337**: users sign UserOperations, bundlers submit them to a singleton EntryPoint, and **paymasters** can sponsor gas — no protocol change. -- **EIP-7702** lets an existing EOA delegate to smart-wallet code while keeping its address. +- [ERC-4337](https://eips.ethereum.org/EIPS/eip-4337): users sign UserOperations, bundlers submit them to a singleton EntryPoint, and paymasters can sponsor gas — no protocol change. +- [EIP-7702](https://eips.ethereum.org/EIPS/eip-7702) lets an existing EOA delegate to smart-wallet code while keeping its address. - 7702 risks: one authorization signature hands the EOA to the delegate's code, a prime phishing target; an authorization with `chain_id = 0` is valid on every chain. -- The delegate's storage lives in the EOA, so switching delegates can collide storage layouts — use namespaced storage (ERC-7201). +- The delegate's storage lives in the EOA, so switching delegates can collide storage layouts — use namespaced storage ([ERC-7201](https://eips.ethereum.org/EIPS/eip-7201)). ## Consensus (PoS) ### Slots, epochs, finality -- **12 s slots**, 32-slot (~6.4 min) epochs; each slot has a random proposer and a committee that attests. -- **Gasper** = LMD-GHOST (fork choice by attestation weight) + **Casper FFG** (checkpoints with 2/3 of stake become justified, then finalized). +- 12 s [slots](https://ethereum.org/developers/docs/consensus-mechanisms/pos/), 32-slot (~6.4 min) epochs; each slot has a random proposer and a committee that attests. +- [Gasper](https://ethereum.org/developers/docs/consensus-mechanisms/pos/gasper/) = LMD-GHOST (fork choice by attestation weight) + [Casper FFG](https://arxiv.org/abs/1710.09437) (checkpoints with 2/3 of stake become justified, then finalized). ### Slashing and penalties -- **Slashing** burns stake for provable misbehavior: double proposals or contradictory (surround or double) attestations. -- The **inactivity leak** drains offline validators while the chain can't finalize. -- EIP-7251 (Pectra) raised the **maximum effective balance to 2048 ETH**, so large operators consolidate validators. +- [Slashing](https://ethereum.org/developers/docs/consensus-mechanisms/pos/rewards-and-penalties/#slashing) burns stake for provable misbehavior: double proposals or contradictory (surround or double) attestations. +- The [inactivity leak](https://ethereum.org/developers/docs/consensus-mechanisms/pos/rewards-and-penalties/#inactivity-leak) drains offline validators while the chain can't finalize. +- [EIP-7251](https://eips.ethereum.org/EIPS/eip-7251) (Pectra) raised the maximum effective balance to 2048 ETH, so large operators consolidate validators. ## Scaling @@ -144,50 +144,50 @@ Built-in contracts at fixed low addresses for cryptography too expensive in byte | Type | Examples | Proof | L1 withdrawal | |---|---|---|---| -| **Optimistic** | Arbitrum, OP Mainnet, Base | fraud proofs in a challenge window | ~7 days | -| **ZK** | zkSync, Starknet, Scroll, Linea | validity proof per batch | once the proof verifies | +| [Optimistic](https://ethereum.org/developers/docs/scaling/optimistic-rollups/) | Arbitrum, OP Mainnet, Base | fraud proofs in a challenge window | ~7 days | +| [ZK](https://ethereum.org/developers/docs/scaling/zk-rollups/) | zkSync, Starknet, Scroll, Linea | validity proof per batch | once the proof verifies | ### Sequencers and data availability -- Rollups post transaction data to L1 for **data availability**, so anyone can rebuild state and challenge or exit. -- Most use a **centralized sequencer** for ordering, with forced inclusion through L1 as the censorship backstop. +- Rollups post transaction data to L1 for [data availability](https://ethereum.org/developers/docs/data-availability/), so anyone can rebuild state and challenge or exit. +- Most use a centralized sequencer for ordering, with forced inclusion through L1 as the censorship backstop ([L2BEAT](https://l2beat.com/) tracks each rollup's setup). ### ZK proof systems -- **SNARKs** (Groth16, PLONK): proofs of a few hundred bytes, cheap to verify on L1 via the BN254 pairing precompile; many need a **trusted setup** (per circuit for Groth16, universal for PLONK). -- **STARKs**: no trusted setup and hash-based (plausibly post-quantum), but proofs run to tens or hundreds of KB, so rollups often wrap a STARK in a SNARK for L1. -- zkEVM types 1–4 trade Ethereum equivalence against proving cost: type 1 proves unmodified Ethereum blocks, type 4 compiles Solidity to a different VM. +- SNARKs ([Groth16](https://eprint.iacr.org/2016/260), [PLONK](https://eprint.iacr.org/2019/953)): proofs of a few hundred bytes, cheap to verify on L1 via the BN254 pairing precompile; many need a [trusted setup](https://vitalik.eth.limo/general/2022/03/14/trustedsetup.html) (per circuit for Groth16, universal for PLONK). +- [STARKs](https://eprint.iacr.org/2018/046): no trusted setup and hash-based (plausibly post-quantum), but proofs run to tens or hundreds of KB, so rollups often wrap a STARK in a SNARK for L1. +- [zkEVM types 1–4](https://vitalik.eth.limo/general/2022/08/04/zkevm.html) trade Ethereum equivalence against proving cost: type 1 proves unmodified Ethereum blocks, type 4 compiles Solidity to a different VM. ### Blobs and PeerDAS -- **EIP-4844** blobs are cheap, temporary data (pruned after ~18 days), separate from calldata. -- **PeerDAS** (Fusaka) lets nodes check blob availability by **sampling**; Blob Parameter Only (BPO) forks then raise blob capacity without other changes. +- [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) blobs are cheap, temporary data (pruned after ~18 days), separate from calldata. +- [PeerDAS](https://eips.ethereum.org/EIPS/eip-7594) (Fusaka) lets nodes check blob availability by sampling; [Blob Parameter Only](https://eips.ethereum.org/EIPS/eip-7892) (BPO) forks then raise blob capacity without other changes. ## DeFi and MEV ### AMMs -- Uniswap v2: constant product **`x·y = k`**; larger trades move the price more, hence slippage limits. -- **Impermanent loss**: the pool sells the rising asset, so LPs can end up worse than holding, offset by fees. -- v3 adds **concentrated liquidity** in chosen price ranges; v4 uses a singleton contract with hooks. +- [Uniswap v2](https://developers.uniswap.org/docs/get-started/concepts/how-uniswap-works): constant product `x·y = k`; larger trades move the price more, hence slippage limits. +- [Impermanent loss](https://developers.uniswap.org/docs/protocols/v2/concepts/understanding-returns): the pool sells the rising asset, so LPs can end up worse than holding, offset by fees. +- v3 adds [concentrated liquidity](https://developers.uniswap.org/docs/get-started/concepts/liquidity-providers/concentrated-liquidity) in chosen price ranges; [v4](https://developers.uniswap.org/docs/protocols/v4/overview) uses a singleton contract with hooks. ### Lending and flash loans -- Lending (Aave, Compound) is over-collateralized; a **health factor** below 1 allows liquidation at a discount. -- **Flash loans** are uncollateralized but must be repaid in the same transaction or everything reverts — turning manipulable prices into large exploits. +- Lending ([Aave](https://aave.com/docs), [Compound](https://docs.compound.xyz/)) is over-collateralized; a health factor below 1 allows liquidation at a discount. +- [Flash loans](https://aave.com/docs/aave-v3/guides/flash-loans) are uncollateralized but must be repaid in the same transaction or everything reverts — turning manipulable prices into large exploits. ### Oracle manipulation -- A DEX **spot price** can be moved within one transaction using a flash loan, then read by a victim protocol. -- Use **TWAPs** or a push oracle such as Chainlink, and check staleness. +- A DEX spot price can be moved within one transaction using a flash loan, then read by a victim protocol. +- Use [TWAPs](https://developers.uniswap.org/docs/protocols/v3/concepts/price-oracles) or a push oracle such as [Chainlink](https://docs.chain.link/data-feeds), and check staleness. ### MEV -- **Sandwich attacks** (front-run and back-run a victim's swap), arbitrage back-running, and liquidations. -- Mitigations: private order flow (Flashbots Protect), tight **slippage limits**, and PBS so small validators earn MEV too. +- [Sandwich attacks](https://ethereum.org/developers/docs/mev/) (front-run and back-run a victim's swap), arbitrage back-running, and liquidations. +- Mitigations: private order flow ([Flashbots Protect](https://docs.flashbots.net/flashbots-protect/overview)), tight slippage limits, and [PBS](https://ethereum.org/roadmap/pbs/) so small validators earn MEV too. ### Bridge exploits -- A bridge is only as secure as whoever attests to cross-chain messages, and its locked pool makes it a prime target. -- **Ronin** (2022): 5 of 9 validator keys compromised — a small validator set is just a multisig. -- **Wormhole** (2022): a signature-verification bypass minted unbacked tokens; **Nomad** (2022): an upgrade marked the zero root as trusted, so any message passed and copycats drained it. +- A [bridge](https://ethereum.org/developers/docs/bridges/) is only as secure as whoever attests to cross-chain messages, and its locked pool makes it a prime target. +- [Ronin](https://rekt.news/ronin-rekt) (2022): 5 of 9 validator keys compromised — a small validator set is just a multisig. +- [Wormhole](https://rekt.news/wormhole-rekt) (2022): a signature-verification bypass minted unbacked tokens; [Nomad](https://rekt.news/nomad-rekt) (2022): an upgrade marked the zero root as trusted, so any message passed and copycats drained it. diff --git a/scripts/lint_briefs.py b/scripts/lint_briefs.py index 02a9804..0d33b20 100644 --- a/scripts/lint_briefs.py +++ b/scripts/lint_briefs.py @@ -8,8 +8,8 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent TOPICS = ROOT / "docs" / "topics" MAX_BULLETS = 6 -MAX_BOLD = 3 # per concept, not counting tables -BOLD = re.compile(r"\*\*[^*]+\*\*") +BOLD = re.compile(r"\*\*|__") # either marker, outside code and link targets +NOT_PROSE = re.compile(r"(`+).+?\1|\]\([^)]*\)") BACKREF = re.compile(r"\b(as|see) (above|below)\b", re.IGNORECASE) @@ -27,17 +27,16 @@ def err(lineno, msg): err(3, "missing one-line intro after the title") section = None # (name, lineno, concept count) - concept = None # (name, lineno, bullets, bold) + concept = None # (name, lineno, bullets) seen = set() + in_fence = False def close_concept(): if concept is None: return - name, lineno, bullets, bold = concept + name, lineno, bullets = concept if bullets > MAX_BULLETS: err(lineno, f"'{name}' has {bullets} bullets (max {MAX_BULLETS})") - if bold > MAX_BOLD: - err(lineno, f"'{name}' has {bold} bold terms outside tables (max {MAX_BOLD})") def close_section(): if section is not None and section[2] < 2: @@ -46,6 +45,10 @@ def close_section(): for i, line in enumerate(lines, 1): if i > 1 and not line and not lines[i - 2]: err(i, "consecutive blank lines") + if line.startswith("```"): + in_fence = not in_fence + elif not in_fence and BOLD.search(NOT_PROSE.sub("", line)): + err(i, "no bold in briefs; let the wording carry emphasis") if BACKREF.search(line): err(i, "concepts must be self-contained; avoid 'as/see above/below'") if line.startswith("## "): @@ -65,12 +68,9 @@ def close_section(): err(i, "concept outside a section") else: section[2] += 1 - concept = [name, i, 0, 0] - elif concept is not None: - if line.startswith("- "): - concept[2] += 1 - if not line.startswith("|"): - concept[3] += len(BOLD.findall(line)) + concept = [name, i, 0] + elif concept is not None and line.startswith("- "): + concept[2] += 1 close_concept() close_section() return errors