671 lines
15 KiB
JSON
671 lines
15 KiB
JSON
{
|
|
"cases": "lab02-challenge.json",
|
|
"webhook": "my-log-report",
|
|
"passed": 1,
|
|
"total": 5,
|
|
"results": [
|
|
{
|
|
"case": "두 주 · 다시 묻는 비율이 가장 높은 주제",
|
|
"webhook": "my-log-report",
|
|
"run": 1,
|
|
"passed": false,
|
|
"problems": [
|
|
"repeat_topics.0.topic = null (expected 프린터)",
|
|
"repeat_topics.0.answers = null (expected 81)",
|
|
"repeat_topics.0.repeats = null (expected 8)",
|
|
"repeat_topics.0.repeat_rate = null (expected 0.0988)"
|
|
],
|
|
"seconds": 0.1,
|
|
"status": 200,
|
|
"request": {
|
|
"from": "2026-10-19",
|
|
"to": "2026-10-30"
|
|
},
|
|
"response": {
|
|
"total": 666,
|
|
"errors": 21,
|
|
"error_rate": 0.0315,
|
|
"p50_ms": 1205,
|
|
"p95_ms": 2537,
|
|
"by_route": {
|
|
"자동 답변": 591,
|
|
"담당자 연결": 34,
|
|
"오류": 21,
|
|
"잠시 후": 20
|
|
},
|
|
"by_day": {
|
|
"2026-10-19": {
|
|
"total": 73,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-20": {
|
|
"total": 57,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-21": {
|
|
"total": 60,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-22": {
|
|
"total": 55,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-23": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-26": {
|
|
"total": 70,
|
|
"errors": 16,
|
|
"error_rate": 0.2286
|
|
},
|
|
"2026-10-27": {
|
|
"total": 62,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-28": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-29": {
|
|
"total": 97,
|
|
"errors": 5,
|
|
"error_rate": 0.0515
|
|
},
|
|
"2026-10-30": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
}
|
|
},
|
|
"by_release": {
|
|
"1.0.0": {
|
|
"answers": 402,
|
|
"repeats": 19,
|
|
"repeat_rate": 0.0473
|
|
},
|
|
"1.1.0": {
|
|
"answers": 181,
|
|
"repeats": 14,
|
|
"repeat_rate": 0.0773
|
|
},
|
|
"1.2.0": {
|
|
"answers": 8,
|
|
"repeats": 5,
|
|
"repeat_rate": 0.625
|
|
}
|
|
},
|
|
"steps": {
|
|
"faq_ms": {
|
|
"runs": 646,
|
|
"p50": 27,
|
|
"p95": 39
|
|
},
|
|
"answer_ms": {
|
|
"runs": 596,
|
|
"p50": 1179,
|
|
"p95": 2509
|
|
}
|
|
},
|
|
"cost_usd": 0.07836,
|
|
"cost_per_run": 0.000118,
|
|
"llm_calls": 596,
|
|
"failures": [
|
|
{
|
|
"node": "Search FAQ",
|
|
"message": "timeout of 1000ms exceeded",
|
|
"count": 16
|
|
},
|
|
{
|
|
"node": "Answer",
|
|
"message": "429 Rate limit reached for requests",
|
|
"count": 5
|
|
}
|
|
],
|
|
"unanswered_topics": [
|
|
{
|
|
"topic": "와이파이",
|
|
"count": 34
|
|
}
|
|
],
|
|
"anomalies": [
|
|
{
|
|
"hour": "2026-10-26T09",
|
|
"total": 16,
|
|
"errors": 16,
|
|
"error_rate": 1
|
|
},
|
|
{
|
|
"hour": "2026-10-29T15",
|
|
"total": 7,
|
|
"errors": 5,
|
|
"error_rate": 0.7143
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"case": "두 주 · 수로는 비밀번호가 가장 많지만 비율은 2위",
|
|
"webhook": "my-log-report",
|
|
"run": 1,
|
|
"passed": false,
|
|
"problems": [
|
|
"repeat_topics.1.topic = null (expected 비밀번호)",
|
|
"repeat_topics.1.repeats = null (expected 9)",
|
|
"repeat_topics.1.repeat_rate = null (expected 0.0647)"
|
|
],
|
|
"seconds": 0.1,
|
|
"status": 200,
|
|
"request": {
|
|
"from": "2026-10-19",
|
|
"to": "2026-10-30"
|
|
},
|
|
"response": {
|
|
"total": 666,
|
|
"errors": 21,
|
|
"error_rate": 0.0315,
|
|
"p50_ms": 1205,
|
|
"p95_ms": 2537,
|
|
"by_route": {
|
|
"자동 답변": 591,
|
|
"담당자 연결": 34,
|
|
"오류": 21,
|
|
"잠시 후": 20
|
|
},
|
|
"by_day": {
|
|
"2026-10-19": {
|
|
"total": 73,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-20": {
|
|
"total": 57,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-21": {
|
|
"total": 60,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-22": {
|
|
"total": 55,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-23": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-26": {
|
|
"total": 70,
|
|
"errors": 16,
|
|
"error_rate": 0.2286
|
|
},
|
|
"2026-10-27": {
|
|
"total": 62,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-28": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-29": {
|
|
"total": 97,
|
|
"errors": 5,
|
|
"error_rate": 0.0515
|
|
},
|
|
"2026-10-30": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
}
|
|
},
|
|
"by_release": {
|
|
"1.0.0": {
|
|
"answers": 402,
|
|
"repeats": 19,
|
|
"repeat_rate": 0.0473
|
|
},
|
|
"1.1.0": {
|
|
"answers": 181,
|
|
"repeats": 14,
|
|
"repeat_rate": 0.0773
|
|
},
|
|
"1.2.0": {
|
|
"answers": 8,
|
|
"repeats": 5,
|
|
"repeat_rate": 0.625
|
|
}
|
|
},
|
|
"steps": {
|
|
"faq_ms": {
|
|
"runs": 646,
|
|
"p50": 27,
|
|
"p95": 39
|
|
},
|
|
"answer_ms": {
|
|
"runs": 596,
|
|
"p50": 1179,
|
|
"p95": 2509
|
|
}
|
|
},
|
|
"cost_usd": 0.07836,
|
|
"cost_per_run": 0.000118,
|
|
"llm_calls": 596,
|
|
"failures": [
|
|
{
|
|
"node": "Search FAQ",
|
|
"message": "timeout of 1000ms exceeded",
|
|
"count": 16
|
|
},
|
|
{
|
|
"node": "Answer",
|
|
"message": "429 Rate limit reached for requests",
|
|
"count": 5
|
|
}
|
|
],
|
|
"unanswered_topics": [
|
|
{
|
|
"topic": "와이파이",
|
|
"count": 34
|
|
}
|
|
],
|
|
"anomalies": [
|
|
{
|
|
"hour": "2026-10-26T09",
|
|
"total": 16,
|
|
"errors": 16,
|
|
"error_rate": 1
|
|
},
|
|
{
|
|
"hour": "2026-10-29T15",
|
|
"total": 7,
|
|
"errors": 5,
|
|
"error_rate": 0.7143
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"case": "두 주 · 다시 물은 주제는 6개",
|
|
"webhook": "my-log-report",
|
|
"run": 1,
|
|
"passed": false,
|
|
"problems": [
|
|
"repeat_topics.5.topic = null (expected 설치)",
|
|
"repeat_topics.5.repeat_rate = null (expected 0.0328)"
|
|
],
|
|
"seconds": 0.1,
|
|
"status": 200,
|
|
"request": {
|
|
"from": "2026-10-19",
|
|
"to": "2026-10-30"
|
|
},
|
|
"response": {
|
|
"total": 666,
|
|
"errors": 21,
|
|
"error_rate": 0.0315,
|
|
"p50_ms": 1205,
|
|
"p95_ms": 2537,
|
|
"by_route": {
|
|
"자동 답변": 591,
|
|
"담당자 연결": 34,
|
|
"오류": 21,
|
|
"잠시 후": 20
|
|
},
|
|
"by_day": {
|
|
"2026-10-19": {
|
|
"total": 73,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-20": {
|
|
"total": 57,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-21": {
|
|
"total": 60,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-22": {
|
|
"total": 55,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-23": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-26": {
|
|
"total": 70,
|
|
"errors": 16,
|
|
"error_rate": 0.2286
|
|
},
|
|
"2026-10-27": {
|
|
"total": 62,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-28": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-29": {
|
|
"total": 97,
|
|
"errors": 5,
|
|
"error_rate": 0.0515
|
|
},
|
|
"2026-10-30": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
}
|
|
},
|
|
"by_release": {
|
|
"1.0.0": {
|
|
"answers": 402,
|
|
"repeats": 19,
|
|
"repeat_rate": 0.0473
|
|
},
|
|
"1.1.0": {
|
|
"answers": 181,
|
|
"repeats": 14,
|
|
"repeat_rate": 0.0773
|
|
},
|
|
"1.2.0": {
|
|
"answers": 8,
|
|
"repeats": 5,
|
|
"repeat_rate": 0.625
|
|
}
|
|
},
|
|
"steps": {
|
|
"faq_ms": {
|
|
"runs": 646,
|
|
"p50": 27,
|
|
"p95": 39
|
|
},
|
|
"answer_ms": {
|
|
"runs": 596,
|
|
"p50": 1179,
|
|
"p95": 2509
|
|
}
|
|
},
|
|
"cost_usd": 0.07836,
|
|
"cost_per_run": 0.000118,
|
|
"llm_calls": 596,
|
|
"failures": [
|
|
{
|
|
"node": "Search FAQ",
|
|
"message": "timeout of 1000ms exceeded",
|
|
"count": 16
|
|
},
|
|
{
|
|
"node": "Answer",
|
|
"message": "429 Rate limit reached for requests",
|
|
"count": 5
|
|
}
|
|
],
|
|
"unanswered_topics": [
|
|
{
|
|
"topic": "와이파이",
|
|
"count": 34
|
|
}
|
|
],
|
|
"anomalies": [
|
|
{
|
|
"hour": "2026-10-26T09",
|
|
"total": 16,
|
|
"errors": 16,
|
|
"error_rate": 1
|
|
},
|
|
{
|
|
"hour": "2026-10-29T15",
|
|
"total": 7,
|
|
"errors": 5,
|
|
"error_rate": 0.7143
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"case": "첫 주 · 설치는 다시 묻지 않음",
|
|
"webhook": "my-log-report",
|
|
"run": 1,
|
|
"passed": false,
|
|
"problems": [
|
|
"repeat_topics.0.topic = null (expected 프린터)"
|
|
],
|
|
"seconds": 0.0,
|
|
"status": 200,
|
|
"request": {
|
|
"from": "2026-10-19",
|
|
"to": "2026-10-23"
|
|
},
|
|
"response": {
|
|
"total": 309,
|
|
"errors": 0,
|
|
"error_rate": 0,
|
|
"p50_ms": 1252,
|
|
"p95_ms": 2525,
|
|
"by_route": {
|
|
"자동 답변": 292,
|
|
"담당자 연결": 17
|
|
},
|
|
"by_day": {
|
|
"2026-10-19": {
|
|
"total": 73,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-20": {
|
|
"total": 57,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-21": {
|
|
"total": 60,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-22": {
|
|
"total": 55,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-23": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
}
|
|
},
|
|
"by_release": {
|
|
"1.0.0": {
|
|
"answers": 292,
|
|
"repeats": 16,
|
|
"repeat_rate": 0.0548
|
|
}
|
|
},
|
|
"steps": {
|
|
"faq_ms": {
|
|
"runs": 309,
|
|
"p50": 27,
|
|
"p95": 39
|
|
},
|
|
"answer_ms": {
|
|
"runs": 292,
|
|
"p50": 1200,
|
|
"p95": 2470
|
|
}
|
|
},
|
|
"cost_usd": 0.03855,
|
|
"cost_per_run": 0.000125,
|
|
"llm_calls": 292,
|
|
"failures": [],
|
|
"unanswered_topics": [
|
|
{
|
|
"topic": "와이파이",
|
|
"count": 17
|
|
}
|
|
],
|
|
"anomalies": []
|
|
}
|
|
},
|
|
{
|
|
"case": "앞의 지표는 그대로",
|
|
"webhook": "my-log-report",
|
|
"run": 1,
|
|
"passed": true,
|
|
"problems": [],
|
|
"seconds": 0.1,
|
|
"status": 200,
|
|
"request": {
|
|
"from": "2026-10-19",
|
|
"to": "2026-10-30"
|
|
},
|
|
"response": {
|
|
"total": 666,
|
|
"errors": 21,
|
|
"error_rate": 0.0315,
|
|
"p50_ms": 1205,
|
|
"p95_ms": 2537,
|
|
"by_route": {
|
|
"자동 답변": 591,
|
|
"담당자 연결": 34,
|
|
"오류": 21,
|
|
"잠시 후": 20
|
|
},
|
|
"by_day": {
|
|
"2026-10-19": {
|
|
"total": 73,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-20": {
|
|
"total": 57,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-21": {
|
|
"total": 60,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-22": {
|
|
"total": 55,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-23": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-26": {
|
|
"total": 70,
|
|
"errors": 16,
|
|
"error_rate": 0.2286
|
|
},
|
|
"2026-10-27": {
|
|
"total": 62,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-28": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
},
|
|
"2026-10-29": {
|
|
"total": 97,
|
|
"errors": 5,
|
|
"error_rate": 0.0515
|
|
},
|
|
"2026-10-30": {
|
|
"total": 64,
|
|
"errors": 0,
|
|
"error_rate": 0
|
|
}
|
|
},
|
|
"by_release": {
|
|
"1.0.0": {
|
|
"answers": 402,
|
|
"repeats": 19,
|
|
"repeat_rate": 0.0473
|
|
},
|
|
"1.1.0": {
|
|
"answers": 181,
|
|
"repeats": 14,
|
|
"repeat_rate": 0.0773
|
|
},
|
|
"1.2.0": {
|
|
"answers": 8,
|
|
"repeats": 5,
|
|
"repeat_rate": 0.625
|
|
}
|
|
},
|
|
"steps": {
|
|
"faq_ms": {
|
|
"runs": 646,
|
|
"p50": 27,
|
|
"p95": 39
|
|
},
|
|
"answer_ms": {
|
|
"runs": 596,
|
|
"p50": 1179,
|
|
"p95": 2509
|
|
}
|
|
},
|
|
"cost_usd": 0.07836,
|
|
"cost_per_run": 0.000118,
|
|
"llm_calls": 596,
|
|
"failures": [
|
|
{
|
|
"node": "Search FAQ",
|
|
"message": "timeout of 1000ms exceeded",
|
|
"count": 16
|
|
},
|
|
{
|
|
"node": "Answer",
|
|
"message": "429 Rate limit reached for requests",
|
|
"count": 5
|
|
}
|
|
],
|
|
"unanswered_topics": [
|
|
{
|
|
"topic": "와이파이",
|
|
"count": 34
|
|
}
|
|
],
|
|
"anomalies": [
|
|
{
|
|
"hour": "2026-10-26T09",
|
|
"total": 16,
|
|
"errors": 16,
|
|
"error_rate": 1
|
|
},
|
|
{
|
|
"hour": "2026-10-29T15",
|
|
"total": 7,
|
|
"errors": 5,
|
|
"error_rate": 0.7143
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
} |