343 lines
8.5 KiB
Plaintext
343 lines
8.5 KiB
Plaintext
|
$ uv run tools/flow.py test cases/lab01-period.json -v
|
|||
|
|
✔ before: POST http://localhost:8800/ops-reset · HTTP 200 · 0.0s · {"seeded_at": "2026-11-04T08:30:00", "note": "79일차 운영 아침 상태 (연습 데이터, seed 79)", "counts": {"alerts": 6, "dlq": 3, "quali
|
|||
|
|
→ request POST http://localhost:8800/ops-reset
|
|||
|
|
{}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"seeded_at": "2026-11-04T08:30:00",
|
|||
|
|
"note": "79일차 운영 아침 상태 (연습 데이터, seed 79)",
|
|||
|
|
"counts": {
|
|||
|
|
"alerts": 6,
|
|||
|
|
"dlq": 3,
|
|||
|
|
"quality_log": 4,
|
|||
|
|
"quarantine": 1,
|
|||
|
|
"usage": 192
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
POST http://localhost:5678/webhook/my-ops-facts · 5 case(s) × 1
|
|||
|
|
✔ 두 주 (10/19~10/30) · 0.1s
|
|||
|
|
→ request POST /webhook/my-ops-facts
|
|||
|
|
{
|
|||
|
|
"from": "2026-10-19",
|
|||
|
|
"to": "2026-10-30"
|
|||
|
|
}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"period": {
|
|||
|
|
"from": "2026-10-19",
|
|||
|
|
"to": "2026-10-30"
|
|||
|
|
},
|
|||
|
|
"runs": {
|
|||
|
|
"total": 666,
|
|||
|
|
"errors": 21,
|
|||
|
|
"error_rate": 0.0315,
|
|||
|
|
"p95_ms": 2537,
|
|||
|
|
"unanswered_rate": 0.0511,
|
|||
|
|
"top_failure": {
|
|||
|
|
"node": "Search FAQ",
|
|||
|
|
"message": "timeout of 1000ms exceeded",
|
|||
|
|
"count": 16
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"alerts": {
|
|||
|
|
"count": 6,
|
|||
|
|
"high": 3,
|
|||
|
|
"by_reason": {
|
|||
|
|
"timeout": 1,
|
|||
|
|
"rate_limit": 1,
|
|||
|
|
"batch_rejected": 1,
|
|||
|
|
"low_quality": 2,
|
|||
|
|
"api_error": 1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dlq": {
|
|||
|
|
"size": 3,
|
|||
|
|
"parked": 2,
|
|||
|
|
"waiting": 1
|
|||
|
|
},
|
|||
|
|
"quality": {
|
|||
|
|
"batches": 4,
|
|||
|
|
"avg_score": 0.6583,
|
|||
|
|
"last_score": 1,
|
|||
|
|
"rejected_batches": 1,
|
|||
|
|
"quarantine": 1
|
|||
|
|
},
|
|||
|
|
"cost": {
|
|||
|
|
"requests": 156,
|
|||
|
|
"calls": 192,
|
|||
|
|
"cost_usd": 0.073604,
|
|||
|
|
"daily_usd": 0.08,
|
|||
|
|
"used_ratio": 0.9201,
|
|||
|
|
"top_user": {
|
|||
|
|
"user": "test-bot",
|
|||
|
|
"cost_usd": 0.027228,
|
|||
|
|
"share": 0.3699
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"release": {
|
|||
|
|
"active": "1.1.0",
|
|||
|
|
"canary_release": "1.2.0",
|
|||
|
|
"canary_percent": 20,
|
|||
|
|
"active_repeat_rate": 0.0773,
|
|||
|
|
"canary_repeat_rate": 0.625
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
✔ 하루 (10/26) · 0.1s
|
|||
|
|
→ request POST /webhook/my-ops-facts
|
|||
|
|
{
|
|||
|
|
"from": "2026-10-26",
|
|||
|
|
"to": "2026-10-26"
|
|||
|
|
}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"period": {
|
|||
|
|
"from": "2026-10-26",
|
|||
|
|
"to": "2026-10-26"
|
|||
|
|
},
|
|||
|
|
"runs": {
|
|||
|
|
"total": 70,
|
|||
|
|
"errors": 16,
|
|||
|
|
"error_rate": 0.2286,
|
|||
|
|
"p95_ms": 2468,
|
|||
|
|
"unanswered_rate": 0.0571,
|
|||
|
|
"top_failure": {
|
|||
|
|
"node": "Search FAQ",
|
|||
|
|
"message": "timeout of 1000ms exceeded",
|
|||
|
|
"count": 16
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"alerts": {
|
|||
|
|
"count": 6,
|
|||
|
|
"high": 3,
|
|||
|
|
"by_reason": {
|
|||
|
|
"timeout": 1,
|
|||
|
|
"rate_limit": 1,
|
|||
|
|
"batch_rejected": 1,
|
|||
|
|
"low_quality": 2,
|
|||
|
|
"api_error": 1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dlq": {
|
|||
|
|
"size": 3,
|
|||
|
|
"parked": 2,
|
|||
|
|
"waiting": 1
|
|||
|
|
},
|
|||
|
|
"quality": {
|
|||
|
|
"batches": 4,
|
|||
|
|
"avg_score": 0.6583,
|
|||
|
|
"last_score": 1,
|
|||
|
|
"rejected_batches": 1,
|
|||
|
|
"quarantine": 1
|
|||
|
|
},
|
|||
|
|
"cost": {
|
|||
|
|
"requests": 156,
|
|||
|
|
"calls": 192,
|
|||
|
|
"cost_usd": 0.073604,
|
|||
|
|
"daily_usd": 0.08,
|
|||
|
|
"used_ratio": 0.9201,
|
|||
|
|
"top_user": {
|
|||
|
|
"user": "test-bot",
|
|||
|
|
"cost_usd": 0.027228,
|
|||
|
|
"share": 0.3699
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"release": {
|
|||
|
|
"active": "1.1.0",
|
|||
|
|
"canary_release": "1.2.0",
|
|||
|
|
"canary_percent": 20,
|
|||
|
|
"active_repeat_rate": null,
|
|||
|
|
"canary_repeat_rate": null
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
✔ 빈 기간 (11/02~11/06) · 0건이어도 200 · 0.1s
|
|||
|
|
→ request POST /webhook/my-ops-facts
|
|||
|
|
{
|
|||
|
|
"from": "2026-11-02",
|
|||
|
|
"to": "2026-11-06"
|
|||
|
|
}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"period": {
|
|||
|
|
"from": "2026-11-02",
|
|||
|
|
"to": "2026-11-06"
|
|||
|
|
},
|
|||
|
|
"runs": {
|
|||
|
|
"total": 0,
|
|||
|
|
"errors": 0,
|
|||
|
|
"error_rate": 0,
|
|||
|
|
"p95_ms": 0,
|
|||
|
|
"unanswered_rate": 0,
|
|||
|
|
"top_failure": null
|
|||
|
|
},
|
|||
|
|
"alerts": {
|
|||
|
|
"count": 6,
|
|||
|
|
"high": 3,
|
|||
|
|
"by_reason": {
|
|||
|
|
"timeout": 1,
|
|||
|
|
"rate_limit": 1,
|
|||
|
|
"batch_rejected": 1,
|
|||
|
|
"low_quality": 2,
|
|||
|
|
"api_error": 1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dlq": {
|
|||
|
|
"size": 3,
|
|||
|
|
"parked": 2,
|
|||
|
|
"waiting": 1
|
|||
|
|
},
|
|||
|
|
"quality": {
|
|||
|
|
"batches": 4,
|
|||
|
|
"avg_score": 0.6583,
|
|||
|
|
"last_score": 1,
|
|||
|
|
"rejected_batches": 1,
|
|||
|
|
"quarantine": 1
|
|||
|
|
},
|
|||
|
|
"cost": {
|
|||
|
|
"requests": 156,
|
|||
|
|
"calls": 192,
|
|||
|
|
"cost_usd": 0.073604,
|
|||
|
|
"daily_usd": 0.08,
|
|||
|
|
"used_ratio": 0.9201,
|
|||
|
|
"top_user": {
|
|||
|
|
"user": "test-bot",
|
|||
|
|
"cost_usd": 0.027228,
|
|||
|
|
"share": 0.3699
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"release": {
|
|||
|
|
"active": "1.1.0",
|
|||
|
|
"canary_release": "1.2.0",
|
|||
|
|
"canary_percent": 20,
|
|||
|
|
"active_repeat_rate": null,
|
|||
|
|
"canary_repeat_rate": null
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
✔ 기간을 비우면 지난주 (기본) · 0.1s
|
|||
|
|
→ request POST /webhook/my-ops-facts
|
|||
|
|
{}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"period": {
|
|||
|
|
"from": "2026-10-26",
|
|||
|
|
"to": "2026-10-30"
|
|||
|
|
},
|
|||
|
|
"runs": {
|
|||
|
|
"total": 357,
|
|||
|
|
"errors": 21,
|
|||
|
|
"error_rate": 0.0588,
|
|||
|
|
"p95_ms": 2598,
|
|||
|
|
"unanswered_rate": 0.0476,
|
|||
|
|
"top_failure": {
|
|||
|
|
"node": "Search FAQ",
|
|||
|
|
"message": "timeout of 1000ms exceeded",
|
|||
|
|
"count": 16
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"alerts": {
|
|||
|
|
"count": 6,
|
|||
|
|
"high": 3,
|
|||
|
|
"by_reason": {
|
|||
|
|
"timeout": 1,
|
|||
|
|
"rate_limit": 1,
|
|||
|
|
"batch_rejected": 1,
|
|||
|
|
"low_quality": 2,
|
|||
|
|
"api_error": 1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dlq": {
|
|||
|
|
"size": 3,
|
|||
|
|
"parked": 2,
|
|||
|
|
"waiting": 1
|
|||
|
|
},
|
|||
|
|
"quality": {
|
|||
|
|
"batches": 4,
|
|||
|
|
"avg_score": 0.6583,
|
|||
|
|
"last_score": 1,
|
|||
|
|
"rejected_batches": 1,
|
|||
|
|
"quarantine": 1
|
|||
|
|
},
|
|||
|
|
"cost": {
|
|||
|
|
"requests": 156,
|
|||
|
|
"calls": 192,
|
|||
|
|
"cost_usd": 0.073604,
|
|||
|
|
"daily_usd": 0.08,
|
|||
|
|
"used_ratio": 0.9201,
|
|||
|
|
"top_user": {
|
|||
|
|
"user": "test-bot",
|
|||
|
|
"cost_usd": 0.027228,
|
|||
|
|
"share": 0.3699
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"release": {
|
|||
|
|
"active": "1.1.0",
|
|||
|
|
"canary_release": "1.2.0",
|
|||
|
|
"canary_percent": 20,
|
|||
|
|
"active_repeat_rate": 0.0773,
|
|||
|
|
"canary_repeat_rate": 0.625
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
✔ 기간과 상관없는 '지금' 숫자는 그대로 · 0.1s
|
|||
|
|
→ request POST /webhook/my-ops-facts
|
|||
|
|
{
|
|||
|
|
"from": "2026-11-02",
|
|||
|
|
"to": "2026-11-06"
|
|||
|
|
}
|
|||
|
|
← response HTTP 200
|
|||
|
|
{
|
|||
|
|
"period": {
|
|||
|
|
"from": "2026-11-02",
|
|||
|
|
"to": "2026-11-06"
|
|||
|
|
},
|
|||
|
|
"runs": {
|
|||
|
|
"total": 0,
|
|||
|
|
"errors": 0,
|
|||
|
|
"error_rate": 0,
|
|||
|
|
"p95_ms": 0,
|
|||
|
|
"unanswered_rate": 0,
|
|||
|
|
"top_failure": null
|
|||
|
|
},
|
|||
|
|
"alerts": {
|
|||
|
|
"count": 6,
|
|||
|
|
"high": 3,
|
|||
|
|
"by_reason": {
|
|||
|
|
"timeout": 1,
|
|||
|
|
"rate_limit": 1,
|
|||
|
|
"batch_rejected": 1,
|
|||
|
|
"low_quality": 2,
|
|||
|
|
"api_error": 1
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"dlq": {
|
|||
|
|
"size": 3,
|
|||
|
|
"parked": 2,
|
|||
|
|
"waiting": 1
|
|||
|
|
},
|
|||
|
|
"quality": {
|
|||
|
|
"batches": 4,
|
|||
|
|
"avg_score": 0.6583,
|
|||
|
|
"last_score": 1,
|
|||
|
|
"rejected_batches": 1,
|
|||
|
|
"quarantine": 1
|
|||
|
|
},
|
|||
|
|
"cost": {
|
|||
|
|
"requests": 156,
|
|||
|
|
"calls": 192,
|
|||
|
|
"cost_usd": 0.073604,
|
|||
|
|
"daily_usd": 0.08,
|
|||
|
|
"used_ratio": 0.9201,
|
|||
|
|
"top_user": {
|
|||
|
|
"user": "test-bot",
|
|||
|
|
"cost_usd": 0.027228,
|
|||
|
|
"share": 0.3699
|
|||
|
|
}
|
|||
|
|
},
|
|||
|
|
"release": {
|
|||
|
|
"active": "1.1.0",
|
|||
|
|
"canary_release": "1.2.0",
|
|||
|
|
"canary_percent": 20,
|
|||
|
|
"active_repeat_rate": null,
|
|||
|
|
"canary_repeat_rate": null
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
Passed 5/5
|
|||
|
|
report: runs/lab01-period-20261101-185126.json
|