76일차 자료 추가 (답 포함): 실습 15개 · 의도 · 생활 속 예 · 실행 결과, LAB-A · 정답 워크플로우와 25단계 실행 기록, 교안 개요 30장 · 4단계 정리, 바이브 코딩 tretry
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mcchaeandClaude Opus 5.5 committed
1 parent
3f36605043
commit
bbf17e4696
141 files changed
+34072
No files matched your search
@@ -0,0 +1 @@
|
||||
"""tretry: retry, backoff, deadline, fallback and a log for any command or HTTP call."""
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Textual screen (R11): targets ranked by failed tries on the left, the runs of the chosen target on the right."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.containers import Horizontal
|
||||
from textual.widgets import DataTable, Footer, Header, Static
|
||||
|
||||
from . import log
|
||||
|
||||
|
||||
class LogApp(App):
|
||||
TITLE = "tretry"
|
||||
BINDINGS = [("r", "reload", "Reload"), ("f", "failed_only", "Failed only"), ("q", "quit", "Quit")]
|
||||
CSS = "#targets { width: 55%; } #tries { width: 45%; } #note { height: auto; margin: 0 1; }"
|
||||
|
||||
def __init__(self, path: Path):
|
||||
super().__init__()
|
||||
self.path = Path(path)
|
||||
self.rows: list[dict] = []
|
||||
self.failed_only = False
|
||||
self.chosen: str | None = None
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Header()
|
||||
with Horizontal():
|
||||
yield DataTable(id="targets", cursor_type="row")
|
||||
yield DataTable(id="tries", cursor_type="row")
|
||||
yield Static("", id="note", markup=False)
|
||||
yield Footer()
|
||||
|
||||
def on_mount(self):
|
||||
self.sub_title = str(self.path)
|
||||
self.action_reload()
|
||||
|
||||
def action_reload(self):
|
||||
self.rows = log.read(self.path)
|
||||
table = self.query_one("#targets", DataTable)
|
||||
table.clear(columns=True)
|
||||
table.add_columns("target", "runs", "ok%", "tries/run", "failed", "fallback", "p95 s")
|
||||
for r in log.stats(self.rows):
|
||||
table.add_row(r["target"][:60], r["runs"], f"{r['success_rate'] * 100:.0f}%", r["attempts_per_run"],
|
||||
r["failed_attempts"], r["fallbacks"], f"{r['p95_seconds']:.2f}", key=r["target"])
|
||||
total = len(log.runs(self.rows))
|
||||
self.query_one("#note", Static).update(f"{total} run(s) · {len(self.rows)} tr(ies) in {self.path} · "
|
||||
"[enter] show tries · [f] failed only · [r] reload")
|
||||
if self.chosen is None and table.row_count:
|
||||
self.chosen = str(table.coordinate_to_cell_key((0, 0)).row_key.value)
|
||||
self.show_tries()
|
||||
|
||||
def action_failed_only(self):
|
||||
self.failed_only = not self.failed_only
|
||||
self.show_tries()
|
||||
|
||||
def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted):
|
||||
if event.data_table.id == "targets" and event.row_key is not None:
|
||||
self.chosen = str(event.row_key.value)
|
||||
self.show_tries()
|
||||
|
||||
def show_tries(self):
|
||||
table = self.query_one("#tries", DataTable)
|
||||
table.clear(columns=True)
|
||||
table.add_columns("run", "try", "at +s", "result", "error", "s")
|
||||
for number, group in enumerate(log.runs(self.rows), 1):
|
||||
if group[-1]["target"] != self.chosen:
|
||||
continue
|
||||
for r in group:
|
||||
if self.failed_only and r["ok"]:
|
||||
continue
|
||||
table.add_row(number, "fallback" if r.get("fallback") else r["attempt"], f"{r['started']:.2f}",
|
||||
"✔" if r["ok"] else "✘", (r.get("error") or "")[:40], f"{r['seconds']:.2f}")
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Command line. Exit codes: 0 worked (maybe by the fallback) · 1 all tries and the fallback failed ·
|
||||
2 usage error · 3 not retryable (e.g. HTTP 400: the request itself is wrong)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from . import log, runners
|
||||
from .retry import Policy, Result, run_with_retry
|
||||
|
||||
SUBCOMMANDS = ("http", "stats", "log")
|
||||
|
||||
|
||||
def options(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--tries", type=int, default=3, help="how many tries (default 3)")
|
||||
parser.add_argument("--backoff", type=float, default=0.5, help="wait before try 2, doubled each time (default 0.5s)")
|
||||
parser.add_argument("--max-wait", type=float, default=8.0, help="longest single wait (default 8s)")
|
||||
parser.add_argument("--jitter", type=float, default=0.0, help="make each wait up to this share shorter, e.g. 0.2")
|
||||
parser.add_argument("--timeout", type=float, help="seconds for one try")
|
||||
parser.add_argument("--deadline", type=float, help="seconds for all tries together")
|
||||
parser.add_argument("--fallback", help='shell command to run once when every try failed, e.g. "cat cached.json"')
|
||||
parser.add_argument("--breaker", type=int, default=5,
|
||||
help="skip straight to the fallback when the last N tries at this target in 60s all failed (0 = off)")
|
||||
parser.add_argument("--log", type=Path, help="log file (default $TRETRY_LOG or ~/.tretry/log.jsonl)")
|
||||
parser.add_argument("--json", action="store_true", help="print the result as JSON for other programs")
|
||||
parser.add_argument("-q", "--quiet", action="store_true", help="print only the output of the command that worked")
|
||||
|
||||
|
||||
def policy(args) -> Policy:
|
||||
if args.tries < 1 or args.backoff < 0 or args.max_wait < 0 or not 0 <= args.jitter < 1:
|
||||
raise SystemExit(usage_error("--tries must be ≥ 1, --backoff/--max-wait ≥ 0, --jitter in [0, 1)"))
|
||||
return Policy(args.tries, args.backoff, args.max_wait, args.jitter, args.deadline)
|
||||
|
||||
|
||||
def usage_error(message: str) -> int:
|
||||
print(f"tretry: {message}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def report(args, kind: str, target: str, result: Result, path: Path) -> int:
|
||||
log.write(path, kind, target, result)
|
||||
code = 0 if result.ok else (3 if result.stopped == "not-retryable" else 1)
|
||||
if args.json:
|
||||
print(json.dumps({"ok": result.ok, "exit_code": code, "stopped": result.stopped, "used_fallback": result.used_fallback,
|
||||
"seconds": result.seconds, "attempts": [a.__dict__ for a in result.attempts], "output": result.output},
|
||||
ensure_ascii=False, indent=2))
|
||||
return code
|
||||
if not args.quiet:
|
||||
for a in result.attempts:
|
||||
mark = "✔" if a.ok else "✘"
|
||||
label = "fallback" if a.fallback else f"#{a.attempt}"
|
||||
wait = f" (waited {a.waited:.2f}s)" if a.waited else ""
|
||||
print(f"{label:<9}{a.started:>7.2f}s {mark} {a.error or 'ok'} · {a.seconds:.2f}s{wait}", file=sys.stderr)
|
||||
how = "by the fallback" if result.ok and result.used_fallback else ("" if result.ok else f"({result.stopped})")
|
||||
tries = sum(1 for a in result.attempts if not a.fallback)
|
||||
state = " ".join(part for part in ("ok" if result.ok else "FAILED", how) if part)
|
||||
print(f"result: {state} · {tries} tr{'y' if tries == 1 else 'ies'} · {result.seconds:.2f}s"
|
||||
+ (" · breaker open: tries skipped" if result.stopped == "breaker" else ""), file=sys.stderr)
|
||||
if result.output:
|
||||
sys.stdout.write(result.output if result.output.endswith("\n") else result.output + "\n")
|
||||
return code
|
||||
|
||||
|
||||
def run(args, kind: str, target: str, attempt) -> int:
|
||||
path = args.log or log.default_path()
|
||||
skip = args.breaker > 0 and log.breaker_open(log.read(path), target, failures=args.breaker)
|
||||
if skip and not args.fallback:
|
||||
print(f"tretry: breaker open for {log.clean(target)} (last {args.breaker} tries failed); no --fallback, so stopping",
|
||||
file=sys.stderr)
|
||||
fallback = runners.command(args.fallback, shell=True) if args.fallback else None
|
||||
result = run_with_retry(attempt, policy(args), fallback=fallback, per_try_timeout=args.timeout, skip_tries=skip)
|
||||
return report(args, kind, target, result, path)
|
||||
|
||||
|
||||
def main_command(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(prog="tretry", usage="tretry [options] -- <command ...>",
|
||||
description="Run a command with timeout, retry (backoff), deadline and fallback.",
|
||||
epilog="also: tretry http METHOD URL · tretry stats · tretry log (screen)")
|
||||
options(parser)
|
||||
if "--" not in argv:
|
||||
if "-h" in argv or "--help" in argv:
|
||||
parser.print_help()
|
||||
return 0
|
||||
return usage_error("put the command after --, e.g. tretry --tries 3 -- curl -sf http://localhost:8800/health")
|
||||
split = argv.index("--")
|
||||
args = parser.parse_args(argv[:split])
|
||||
command = argv[split + 1:]
|
||||
if not command:
|
||||
return usage_error("no command after --")
|
||||
return run(args, "command", shlex.join(command), runners.command(command))
|
||||
|
||||
|
||||
def main_http(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(prog="tretry http", description="HTTP call: 5xx, 408/425/429, timeout and connection errors are "
|
||||
"tried again; other 4xx are not.")
|
||||
parser.add_argument("method")
|
||||
parser.add_argument("url")
|
||||
parser.add_argument("--body", help="JSON body, or @file.json")
|
||||
parser.add_argument("--header", action="append", default=[], help="K: V (values are never logged)")
|
||||
parser.add_argument("--idem-key", help="idempotency key sent with every try (header Idempotency-Key); 'auto' makes one")
|
||||
parser.add_argument("--body-key", help="also put the idempotency key in this JSON body field if it is missing, e.g. key")
|
||||
options(parser)
|
||||
args = parser.parse_args(argv)
|
||||
body = args.body
|
||||
if body and body.startswith("@"):
|
||||
body = Path(body[1:]).read_text(encoding="utf-8")
|
||||
if body:
|
||||
try:
|
||||
json.loads(body)
|
||||
except json.JSONDecodeError as error:
|
||||
return usage_error(f"--body is not JSON: {error}")
|
||||
headers = {}
|
||||
for item in args.header:
|
||||
name, _, value = item.partition(":")
|
||||
headers[name.strip()] = value.strip()
|
||||
key = uuid.uuid4().hex if args.idem_key == "auto" else args.idem_key
|
||||
if args.body_key and not key:
|
||||
return usage_error("--body-key needs --idem-key")
|
||||
target = f"{args.method.upper()} {args.url}"
|
||||
return run(args, "http", target, runners.http(args.method, args.url, body, headers, key, args.body_key))
|
||||
|
||||
|
||||
def main_stats(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(prog="tretry stats", description="Per target: runs, success rate, tries per run, p95 time.")
|
||||
parser.add_argument("--log", type=Path)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
rows = log.stats(log.read(args.log or log.default_path()))
|
||||
if args.json:
|
||||
print(json.dumps(rows, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
if not rows:
|
||||
print("no log yet")
|
||||
return 0
|
||||
print(f"{'target':<48} {'runs':>4} {'ok%':>5} {'tries/run':>9} {'failed':>6} {'fallback':>8} {'p95 s':>7}")
|
||||
for r in rows:
|
||||
print(f"{r['target'][:48]:<48} {r['runs']:>4} {r['success_rate'] * 100:>4.0f}% {r['attempts_per_run']:>9} "
|
||||
f"{r['failed_attempts']:>6} {r['fallbacks']:>8} {r['p95_seconds']:>7.2f}")
|
||||
return 0
|
||||
|
||||
|
||||
def main_log(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(prog="tretry log", description="Screen: targets by failures, and the tries of one target.")
|
||||
parser.add_argument("--log", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
from .app import LogApp
|
||||
LogApp(args.log or log.default_path()).run()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
argv = sys.argv[1:] if argv is None else argv
|
||||
try:
|
||||
if argv and argv[0] in SUBCOMMANDS:
|
||||
return {"http": main_http, "stats": main_stats, "log": main_log}[argv[0]](argv[1:])
|
||||
return main_command(argv)
|
||||
except SystemExit as stop:
|
||||
return stop.code if isinstance(stop.code, int) else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,95 @@
|
||||
"""The attempt log as JSON Lines (R7), stats from it (R9) and the circuit breaker that reads it (R10)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from .retry import Result
|
||||
|
||||
SECRET = re.compile(r"(sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S+|(api[_-]?key|token|password)=[^&\s]+)", re.I)
|
||||
|
||||
|
||||
def default_path() -> Path:
|
||||
return Path(os.environ.get("TRETRY_LOG") or Path.home() / ".tretry" / "log.jsonl")
|
||||
|
||||
|
||||
def clean(text: str) -> str:
|
||||
"""Never keep a key or a token in the log, even when it is part of a URL or an error message."""
|
||||
return SECRET.sub("***", text)
|
||||
|
||||
|
||||
def write(path: Path, kind: str, target: str, result: Result, now: float | None = None) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
at = now if now is not None else time.time()
|
||||
with path.open("a", encoding="utf-8") as out:
|
||||
first = at - result.seconds # wall clock time of the first try
|
||||
for item in result.attempts:
|
||||
row = {"at": round(first + item.started, 3), "kind": kind, "target": clean(target), **asdict(item)}
|
||||
row["error"] = clean(row["error"])
|
||||
row["final"] = item is result.attempts[-1]
|
||||
row["result"] = "ok" if result.ok else "failed"
|
||||
row["stopped"] = result.stopped
|
||||
out.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def read(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue # a half-written line from a killed run
|
||||
return rows
|
||||
|
||||
|
||||
def runs(rows: list[dict]) -> list[list[dict]]:
|
||||
"""Group attempt rows into runs (one tretry call = the rows up to and including its final row)."""
|
||||
groups, current = [], []
|
||||
for row in rows:
|
||||
current.append(row)
|
||||
if row.get("final"):
|
||||
groups.append(current)
|
||||
current = []
|
||||
return groups
|
||||
|
||||
|
||||
def p95(values: list[float]) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
return ordered[min(len(ordered) - 1, round(0.95 * (len(ordered) - 1)))]
|
||||
|
||||
|
||||
def stats(rows: list[dict]) -> list[dict]:
|
||||
"""Per target: runs, success rate, attempts per run, failed attempts, fallbacks, p95 run time. Most failures first."""
|
||||
table: dict[str, dict] = {}
|
||||
for group in runs(rows):
|
||||
target = group[-1]["target"]
|
||||
row = table.setdefault(target, {"target": target, "kind": group[-1]["kind"], "runs": 0, "ok": 0, "attempts": 0,
|
||||
"failed_attempts": 0, "fallbacks": 0, "times": []})
|
||||
row["runs"] += 1
|
||||
row["ok"] += group[-1]["result"] == "ok"
|
||||
row["attempts"] += len(group)
|
||||
row["failed_attempts"] += sum(1 for r in group if not r["ok"])
|
||||
row["fallbacks"] += any(r.get("fallback") for r in group)
|
||||
row["times"].append(group[-1]["started"] + group[-1]["seconds"])
|
||||
out = []
|
||||
for row in table.values():
|
||||
times = row.pop("times")
|
||||
out.append({**row, "success_rate": round(row["ok"] / row["runs"], 3), "attempts_per_run": round(row["attempts"] / row["runs"], 2),
|
||||
"p95_seconds": round(p95(times), 3)})
|
||||
return sorted(out, key=lambda r: (-r["failed_attempts"], r["target"]))
|
||||
|
||||
|
||||
def breaker_open(rows: list[dict], target: str, failures: int = 5, window: float = 60.0, now: float | None = None) -> bool:
|
||||
"""Open when the last `failures` attempts at this target (not fallbacks) all failed within `window` seconds."""
|
||||
now = now if now is not None else time.time()
|
||||
recent = [r for r in rows if r.get("target") == clean(target) and not r.get("fallback") and now - r["at"] <= window]
|
||||
last = recent[-failures:]
|
||||
return len(last) >= failures and not any(r["ok"] for r in last)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""The retry engine (R1 · R2 · R6 · R13). Knows nothing about commands or HTTP: it calls attempt() and reads its Outcome.
|
||||
|
||||
sleep, clock and rand are arguments so tests never wait and always get the same numbers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
|
||||
@dataclass
|
||||
class Outcome:
|
||||
"""What one attempt gave. retryable=False means trying again cannot help (e.g. HTTP 400)."""
|
||||
ok: bool
|
||||
retryable: bool = True
|
||||
error: str = ""
|
||||
output: str = ""
|
||||
status: int | None = None
|
||||
retry_after: float | None = None # seconds the server asked us to wait (HTTP 429 / 503 Retry-After)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Policy:
|
||||
tries: int = 3
|
||||
backoff: float = 0.5 # wait before try 2; doubles each time
|
||||
max_wait: float = 8.0 # one wait is never longer than this
|
||||
jitter: float = 0.0 # 0.2 = each wait is randomly 0..20% shorter, so many clients do not retry at the same moment
|
||||
deadline: float | None = None # seconds for all tries together; no new try that could not finish in time
|
||||
|
||||
def wait(self, attempt: int, rand: Callable[[], float] = random.random) -> float:
|
||||
"""Wait after attempt number `attempt` (1-based) and before the next one."""
|
||||
base = min(self.backoff * 2 ** (attempt - 1), self.max_wait)
|
||||
return round(base * (1 - self.jitter * rand()), 3)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attempt:
|
||||
attempt: int
|
||||
started: float # seconds since the first try started
|
||||
seconds: float
|
||||
ok: bool
|
||||
error: str = ""
|
||||
status: int | None = None
|
||||
waited: float = 0.0 # wait before this attempt
|
||||
fallback: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
ok: bool
|
||||
output: str = ""
|
||||
attempts: list[Attempt] = field(default_factory=list)
|
||||
stopped: str = "" # why it stopped: success · tries · not-retryable · deadline · breaker
|
||||
used_fallback: bool = False
|
||||
seconds: float = 0.0
|
||||
|
||||
|
||||
def run_with_retry(attempt: Callable[[float | None], Outcome], policy: Policy, *,
|
||||
fallback: Callable[[float | None], Outcome] | None = None,
|
||||
per_try_timeout: float | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
rand: Callable[[], float] = random.random,
|
||||
skip_tries: bool = False) -> Result:
|
||||
"""Call attempt(timeout) until it works, up to policy.tries. Then fallback once, if given.
|
||||
|
||||
skip_tries=True goes straight to the fallback (the circuit breaker is open)."""
|
||||
start = clock()
|
||||
result = Result(ok=False)
|
||||
waited = 0.0
|
||||
number = 0
|
||||
while not skip_tries and number < policy.tries:
|
||||
number += 1
|
||||
timeout = per_try_timeout
|
||||
if policy.deadline is not None:
|
||||
left = policy.deadline - (clock() - start)
|
||||
if left <= 0:
|
||||
result.stopped = "deadline"
|
||||
break
|
||||
timeout = min(timeout, left) if timeout else left
|
||||
began = clock()
|
||||
outcome = attempt(timeout)
|
||||
result.attempts.append(Attempt(number, round(began - start, 3), round(clock() - began, 3), outcome.ok,
|
||||
outcome.error, outcome.status, waited))
|
||||
if outcome.ok:
|
||||
result.ok, result.output, result.stopped = True, outcome.output, "success"
|
||||
break
|
||||
if not outcome.retryable:
|
||||
result.output, result.stopped = outcome.output, "not-retryable"
|
||||
break
|
||||
if number == policy.tries:
|
||||
result.stopped = "tries"
|
||||
break
|
||||
waited = policy.wait(number, rand)
|
||||
if outcome.retry_after is not None:
|
||||
waited = min(max(waited, outcome.retry_after), policy.max_wait)
|
||||
if policy.deadline is not None and clock() - start + waited >= policy.deadline:
|
||||
result.stopped = "deadline"
|
||||
break
|
||||
sleep(waited)
|
||||
if skip_tries:
|
||||
result.stopped = "breaker"
|
||||
# a request that cannot be right (4xx) gets no fallback: the fallback would hide the caller's mistake
|
||||
if not result.ok and fallback is not None and result.stopped != "not-retryable":
|
||||
began = clock()
|
||||
outcome = fallback(per_try_timeout)
|
||||
result.attempts.append(Attempt(len(result.attempts) + 1, round(began - start, 3), round(clock() - began, 3),
|
||||
outcome.ok, outcome.error, outcome.status, 0.0, fallback=True))
|
||||
result.used_fallback = True
|
||||
if outcome.ok:
|
||||
result.ok, result.output = True, outcome.output
|
||||
result.seconds = round(clock() - start, 3)
|
||||
return result
|
||||
@@ -0,0 +1,71 @@
|
||||
"""One attempt of a command or of an HTTP call, as an Outcome (R1 · R3 · R4 · R5)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .retry import Outcome
|
||||
|
||||
# 408 timeout · 425 too early · 429 too many requests: waiting can help. Other 4xx: the request itself is wrong.
|
||||
RETRY_4XX = {408, 425, 429}
|
||||
|
||||
|
||||
def command(argv: list[str] | str, shell: bool = False):
|
||||
"""attempt(timeout) for a command: exit code 0 is success; a timeout is retryable."""
|
||||
def attempt(timeout: float | None) -> Outcome:
|
||||
try:
|
||||
done = subprocess.run(argv, shell=shell, capture_output=True, text=True, timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
return Outcome(False, True, f"timeout {timeout:.1f}s")
|
||||
except FileNotFoundError as error:
|
||||
return Outcome(False, False, f"not found: {error.filename}")
|
||||
if done.returncode == 0:
|
||||
return Outcome(True, output=done.stdout)
|
||||
tail = (done.stderr or done.stdout).strip().splitlines()[-1:] or [""]
|
||||
return Outcome(False, True, f"exit {done.returncode} {tail[0][:120]}".strip(), output=done.stdout)
|
||||
return attempt
|
||||
|
||||
|
||||
def retry_after(value: str | None) -> float | None:
|
||||
try:
|
||||
return max(0.0, float(value)) if value else None
|
||||
except ValueError:
|
||||
return None # an HTTP date: ignore, use our own backoff
|
||||
|
||||
|
||||
def http(method: str, url: str, body: str | None = None, headers: dict[str, str] | None = None,
|
||||
idem_key: str | None = None, body_key: str | None = None):
|
||||
"""attempt(timeout) for an HTTP call: 2xx success · 5xx, 408/425/429, timeout, connection error retryable ·
|
||||
other 4xx not retryable. The same idempotency key goes with every try (header, and the body field if asked)."""
|
||||
data = None
|
||||
send = dict(headers or {})
|
||||
if body is not None:
|
||||
if body_key and idem_key:
|
||||
parsed = json.loads(body)
|
||||
parsed.setdefault(body_key, idem_key)
|
||||
body = json.dumps(parsed, ensure_ascii=False)
|
||||
data = body.encode("utf-8")
|
||||
send.setdefault("Content-Type", "application/json")
|
||||
if idem_key:
|
||||
send["Idempotency-Key"] = idem_key
|
||||
|
||||
def attempt(timeout: float | None) -> Outcome:
|
||||
request = urllib.request.Request(url, data=data, method=method.upper(), headers=send)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout or 30) as response:
|
||||
return Outcome(True, output=response.read().decode("utf-8", "replace"), status=response.status)
|
||||
except urllib.error.HTTPError as error:
|
||||
text = error.read().decode("utf-8", "replace")
|
||||
retryable = error.code >= 500 or error.code in RETRY_4XX
|
||||
return Outcome(False, retryable, f"HTTP {error.code} {text[:120]}".strip(), output=text, status=error.code,
|
||||
retry_after=retry_after(error.headers.get("Retry-After")))
|
||||
except (TimeoutError, socket.timeout):
|
||||
return Outcome(False, True, f"timeout {timeout or 30:.1f}s")
|
||||
except urllib.error.URLError as error:
|
||||
if isinstance(error.reason, (TimeoutError, socket.timeout)):
|
||||
return Outcome(False, True, f"timeout {timeout or 30:.1f}s")
|
||||
return Outcome(False, True, f"connection: {error.reason}")
|
||||
return attempt
|
||||
Reference in new issue
Block a user