第 19 章：评估与监控

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

def evaluate_response_accuracy(agent_output: str, expected_output: str) -> float:
    """计算 Agent 响应的简单准确率。"""
    # 仅做严格匹配，实际应用需更复杂的指标
    return 1.0 if agent_output.strip().lower() == expected_output.strip().lower() else 0.0

# 示例
agent_response = "The capital of France is Paris."
ground_truth = "Paris is the capital of France."
score = evaluate_response_accuracy(agent_response, ground_truth)
print(f"Response accuracy: {score}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# 概念示例，实际 Token 统计需用 LLM API
class LLMInteractionMonitor:
    def __init__(self):
         self.total_input_tokens = 0
         self.total_output_tokens = 0

    def record_interaction(self, prompt: str, response: str):
         # 实际应用需用 LLM API 的 Token 计数器
         input_tokens = len(prompt.split()) # 占位
         output_tokens = len(response.split()) # 占位
         self.total_input_tokens += input_tokens
         self.total_output_tokens += output_tokens
         print(f"Recorded interaction: Input tokens={input_tokens}, Output tokens={output_tokens}")

    def get_total_tokens(self):
         return self.total_input_tokens, self.total_output_tokens

# 示例
monitor = LLMInteractionMonitor()
monitor.record_interaction("What is the capital of France?", "The capital of France is Paris.")
monitor.record_interaction("Tell me a joke.", "Why don't scientists trust atoms? Because they make up everything!")
input_t, output_t = monitor.get_total_tokens()
print(f"Total input tokens: {input_t}, Total output tokens: {output_t}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

import google.generativeai as genai
import os
import json
import logging
from typing import Optional

# --- 配置 ---
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# 需设置 GOOGLE_API_KEY 环境变量
try:
    genai.configure(api_key=os.environ["GOOGLE_API_KEY"])
except KeyError:
    logging.error("Error: GOOGLE_API_KEY environment variable not set.")
    exit(1)

# --- 法律调查问卷质量评审标准 ---
LEGAL_SURVEY_RUBRIC = """
你是一名法律调查方法专家和严谨的法律评审员。你的任务是评估给定法律调查问题的质量。

请针对以下标准分别打分（1-5），并给出详细理由和具体反馈：
1. 清晰与精确
2. 中立与无偏
3. 相关性与聚焦
4. 完整性
5. 受众适配性

输出格式为 JSON，包括 overall_score、rationale、detailed_feedback、concerns、recommended_action。
"""

class LLMJudgeForLegalSurvey:
    """使用生成式 AI 评估法律调查问题的类。"""

    def __init__(self, model_name: str = 'gemini-1.5-flash-latest', temperature: float = 0.2):
         self.model = genai.GenerativeModel(model_name)
         self.temperature = temperature

    def _generate_prompt(self, survey_question: str) -> str:
         return f"{LEGAL_SURVEY_RUBRIC}\n\n---\n**待评估法律调查问题：**\n{survey_question}\n---"

    def judge_survey_question(self, survey_question: str) -> Optional[dict]:
         full_prompt = self._generate_prompt(survey_question)
         try:
              logging.info(f"请求模型 '{self.model.model_name}' 评审...")
              response = self.model.generate_content(
                    full_prompt,
                    generation_config=genai.types.GenerationConfig(
                         temperature=self.temperature,
                         response_mime_type="application/json"
                    )
              )
              if not response.parts:
                    safety_ratings = response.prompt_feedback.safety_ratings
                    logging.error(f"LLM 响应为空或被拦截。安全评级：{safety_ratings}")
                    return None
              return json.loads(response.text)
         except json.JSONDecodeError:
              logging.error(f"LLM 响应 JSON 解码失败。原始响应：{response.text}")
              return None
         except Exception as e:
              logging.error(f"LLM 评审异常：{e}")
              return None

# --- 示例 ---
if __name__ == "__main__":
    judge = LLMJudgeForLegalSurvey()
    good_legal_survey_question = """
    你是否同意瑞士现行知识产权法已充分保护符合联邦最高法院原创性标准的 AI 生成内容？
    （选项：强烈不同意、不同意、中立、同意、强烈同意）
    """
    print("\n--- 评审优质法律调查问题 ---")
    judgment_good = judge.judge_survey_question(good_legal_survey_question)
    if judgment_good:
         print(json.dumps(judgment_good, indent=2))

    biased_legal_survey_question = """
    你是否认为像 FADP 这样过于严格的数据隐私法正在阻碍瑞士的技术创新和经济增长？
    （选项：是、否）
    """
    print("\n--- 评审有偏/较差法律调查问题 ---")
    judgment_biased = judge.judge_survey_question(biased_legal_survey_question)
    if judgment_biased:
         print(json.dumps(judgment_biased, indent=2))

    vague_legal_survey_question = """
    你对法律科技有何看法？
    """
    print("\n--- 评审模糊法律调查问题 ---")
    judgment_vague = judge.judge_survey_question(vague_legal_survey_question)
    if judgment_vague:
         print(json.dumps(judgment_vague, indent=2))