模型评估：不只是 BLEU 和 ROUGE

from collections import Counter

def bleu_score(candidate, reference):
    cand_tokens = candidate.split()
    ref_tokens = reference.split()
    overlap = sum((Counter(cand_tokens) & Counter(ref_tokens)).values())
    return overlap / max(len(cand_tokens), 1)

candidate = "Nike Air Max 270 provides excellent cushioning for runners with its air unit"
reference = "Nike Air Max 270 跑步鞋采用气垫缓震技术，适合跑步爱好者"

bleu = bleu_score(candidate, reference)
print(f"BLEU 分数: {bleu:.3f}")
print("结论: 语义相关但中文参考下 BLEU 接近0 -> 不能反映真实质量")

high_quality = "Air Max 270 uses a large Air unit in the heel for cushioning"
ref1 = "Nike Air Max 270 缓震性能好，气垫踩上去很舒服"
low_quality = "Nike Air Max 270 is a popular Nike sneaker"

print(f"\n高质量答案 BLEU: {bleu_score(high_quality, ref1):.3f}")
print(f"低质量答案 BLEU:  {bleu_score(low_quality, ref1):.3f}")

def g_eval_score(question, answer):
    scoring_criteria = """
评分维度（每项1-5分）：
1. 准确性：回答中的事实是否正确
2. 相关性：回答是否针对用户问题
3. 完整性：回答是否覆盖了问题的各个方面
4. 可读性：表达是否清晰、简洁

评分规则：
- 4分：基本满意，有小缺陷
- 5分：优秀，全面准确
- 3分：有明显不足
- 1-2分：严重错误或完全不相关
"""
    prompt = f"""你是一个专业的 AI 评估专家。请评估以下回答的质量。

问题：{question}

回答：{answer}

{scoring_criteria}

请先说明每个维度的评分理由，最后给出总分（1-5）。
格式：
准确性：X/5 - 理由
相关性：X/5 - 理由
完整性：X/5 - 理由
可读性：X/5 - 理由
总分：X/5
"""
    return prompt

question = "Nike Air Max 270 的气垫能跑马拉松吗？"
answer_good = "Air Max 270 的气垫设计更适合日常跑步和训练，马拉松等专业长距离跑步建议选择专用的竞速跑鞋（如 Nike Vaporfly 系列），因为气垫缓震在长距离上可能偏硬。"
answer_bad = "Nike Air Max 270 是一款很流行的运动鞋，很舒适。"

print("G-Eval Prompt 示例：")
print(g_eval_score(question, answer_good))

benchmarks = {
    "MMLU": {
        "GPT-4o": 88.0, "Claude 3.5 Sonnet": 88.7,
        "Gemini-1.5-Pro": 85.9, "Qwen2.5-72B": 86.1
    },
    "HumanEval": {
        "GPT-4o": 90.2, "Claude 3.5 Sonnet": 92.0,
        "Gemini-1.5-Pro": 84.1, "Qwen2.5-72B": 79.3
    },
    "MATH": {
        "GPT-4o": 45.8, "Claude 3.5 Sonnet": 43.8,
        "Gemini-1.5-Pro": 42.3, "Qwen2.5-72B": 52.0
    }
}

print("主流模型 Benchmark 对比：")
for benchmark, scores in benchmarks.items():
    print(f"\n{benchmark}:")
    for model, score in sorted(scores.items(), key=lambda x: x[1], reverse=True):
        bar = "=" * int(score / 2)
        print(f"  {model:<25}: {score:>5.1f} {bar}")

def context_precision_at_k(retrieved_chunks, relevant_chunks, k=5):
    if k == 0:
        return 0.0
    relevant_retrieved = sum(1 for chunk in retrieved_chunks[:k] if chunk in relevant_chunks)
    return relevant_retrieved / k

def context_recall_at_k(retrieved_chunks, relevant_chunks, k=5):
    if not relevant_chunks:
        return 1.0
    relevant_retrieved = sum(1 for chunk in retrieved_chunks[:k] if chunk in relevant_chunks)
    return relevant_retrieved / len(relevant_chunks)

retrieved = ["chunk_A", "chunk_B", "chunk_C", "chunk_D", "chunk_E"]
relevant = ["chunk_A", "chunk_C"]


precision = context_precision_at_k(retrieved, relevant, k=5)
recall = context_recall_at_k(retrieved, relevant, k=5)

print(f"Context Precision@5: {precision:.2f}")
print(f"Context Recall@5: {recall:.2f}")

print("\n不同 k下的 Precision/Recall：")
for k in [1, 3, 5, 10]:
    p = context_precision_at_k(retrieved, relevant, k)
    r = context_recall_at_k(retrieved, relevant, k)
    print(f"  k={k:>2}: Precision={p:.2f}, Recall={r:.2f}")

def faithfullness_score(context, answer):
    context_set = set(context.lower().split())
    answer_set = set(answer.lower().split())
    coverage = len(context_set & answer_set) / max(len(answer_set), 1)
    return min(coverage * 1.2, 1.0)

def answer_relevance_score(question, answer):
    q_words = set(question.lower().split())
    a_words = set(answer.lower().split())
    overlap = len(q_words & a_words) / max(len(q_words), 1)
    return min(overlap * 1.5, 1.0)

context = "Nike Air Max 270 has Air Max cushioning, weighs 280g, uses mesh upper for breathability"
answer_good = "Air Max 270 uses Air Max cushioning and mesh upper for breathability, weighs 280g"
answer_bad = "Nike Air Max 270 is a great shoe. It looks nice and many people like it."

for label, ans in [("好的回答", answer_good), ("差的回答", answer_bad)]:
    fs = faithfullness_score(context, ans)
    ar = answer_relevance_score("Nike Air Max 270 specs?", ans)
    print(f"{label}:")
    print(f"  Faithfulness: {fs:.2f}, Answer Relevance: {ar:.2f}")

import numpy as np
from scipy import stats

def ab_test(control, treatment, metric_name="转化率"):
    c_mean = np.mean(control)
    t_mean = np.mean(treatment)
    lift = (t_mean - c_mean) / c_mean * 100
    t_stat, p_value = stats.ttest_ind(treatment, control)
    significant = p_value < 0.05
    return {
        "control_mean": c_mean,
        "treatment_mean": t_mean,
        "lift_pct": lift,
        "p_value": p_value,
        "significant": significant,
        "conclusion": "显著提升" if significant and t_mean > c_mean else
                     "显著下降" if significant and t_mean < c_mean else "无显著差异"
    }

np.random.seed(42)
control_responses = np.random.normal(0.12, 0.03, 1000)
treatment_responses = np.random.normal(0.135, 0.03, 1000)

result = ab_test(control_responses, treatment_responses, "产品推荐采纳率")
print("A/B 测试结果：")
print(f"  控制组均值: {result['control_mean']:.3f}")
print(f"  实验组均值: {result['treatment_mean']:.3f}")
print(f"  提升: {result['lift_pct']:+.2f}%")
print(f"  p-value: {result['p_value']:.4f}")
print(f"  结论: {result['conclusion']}")

def ecommerce_eval_design():
    layers = [
        {
            "layer": "L1: 底层能力",
            "what": "模型在标准任务上的表现",
            "how": "MMLU、HumanEval、Math benchmark",
            "metric": "准确率/正确率",
            "cadence": "每次换模型时测一次"
        },
        {
            "layer": "L2: 任务能力",
            "what": "在电商任务上的端到端表现",
            "how": "G-Eval + 人工抽样评估",
            "metric": "任务完成率/格式准确率",
            "cadence": "每周抽100条"
        },
        {
            "layer": "L3: RAG 检索",
            "what": "召回质量（检索是否精准）",
            "how": "Context Precision@5, Recall@5",
            "metric": "Precision/Recall",
            "cadence": "每次知识库更新后测"
        },
        {
            "layer": "L4: 业务指标",
            "what": "实际业务产出（用户是否满意）",
            "how": "A/B 测试",
            "metric": "转化率/采纳率/投诉率",
            "cadence": "持续运行"
        }
    ]
    return layers

for layer in ecommerce_eval_design():
    print(f"【{layer['layer']}】")
    print(f"  评估什么: {layer['what']}")
    print(f"  怎么测: {layer['how']}")
    print(f"  指标: {layer['metric']}")
    print(f"  频率: {layer['cadence']}\n")