(LLM系列)RAG系统性能优化实战：从评估不达标到全面通关的完整历程

1
class LLMEvaluator:
2
    """基于LLM的RAG评估器"""
3

4
    def __init__(
5
        self,
6
        rag_service=None,
7
        llm_base_url: str = "https://dashscope.aliyuncs.com/compatible-mode/v1",
8
        llm_api_key: str = None,
9
        judge_model: str = "qwen-turbo"
10
    ):
11
        self.rag_service = rag_service
12
        self.llm_base_url = llm_base_url
13
        self.llm_api_key = llm_api_key
14
        self.judge_model = judge_model
15

16
    async def _call_judge_llm(self, prompt: str, system_prompt: str = None) -> Dict:
17
        """调用Judge LLM进行评估"""
18
        try:
19
            async with httpx.AsyncClient(timeout=30.0) as client:
20
                response = await client.post(
21
                    f"{self.llm_base_url}/chat/completions",
22
                    headers={
23
                        "Authorization": f"Bearer {self.llm_api_key}",
24
                        "Content-Type": "application/json"
25
                    },
26
                    json={
27
                        "model": self.judge_model,
28
                        "messages": [
29
                            {"role": "system", "content": system_prompt or LLM_JUDGE_SYSTEM_PROMPT},
30
                            {"role": "user", "content": prompt}
31
                        ],
32
                        "temperature": 0.0
33
                    }
34
                )
35

36
                response.raise_for_status()
37
                result = response.json()
38
                content = result["choices"][0]["message"]["content"]
39

40
                # 解析JSON结果
41
                import re
42
                json_match = re.search(r'\{[^}]+\}', content, re.DOTALL)
43
                if json_match:
44
                    return json.loads(json_match.group())
45

46
                return {"score": 0.5, "reason": "评估失败，使用默认分数"}
47

48
        except Exception as e:
49
            logger.error(f"Judge LLM调用失败: {e}")
50
            return {"score": 0.5, "reason": f"调用失败: {str(e)}"}

1
class RerankService:
2
    def __init__(self):
3
        self.api_key = settings.DASHSCOPE_API_KEY
4
       # 错误端点
5
       self.base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
6
       # ✅ 修复：正确使用 compatible-api 端点
7
       self.base_url = "https://dashscope.aliyuncs.com/compatible-api/v1"
8
        self.model = "qwen3-rerank"
9

10
    async def rerank(self, query: str, documents: List[str], top_k: int = 5) -> List[Dict]:
11
        if not documents:
12
            return []
13

14
        try:
15
            async with httpx.AsyncClient() as client:
16
                response = await client.post(
17
                    f"{self.base_url}/reranks",
18
                    headers={
19
                        "Authorization": f"Bearer {self.api_key}",
20
                        "Content-Type": "application/json"
21
                    },
22
                    json={
23
                        "model": self.model,
24
                        "query": query,
25
                        "documents": documents,
26
                        "top_n": top_k
27
                    },
28
                    timeout=30.0
29
                )
30

31
                response.raise_for_status()
32
                data = response.json()
33

34
               # 错误格式
35
               results = data.get('output', {}).get('results', [])
36
               # ✅ 修复：正确解析响应格式
37
               results = data.get('results', [])
38

39
                if not results:
40
                    logger.warning("rerank_returned_empty_results", query=query[:30])
41
                    return []
42

43
                return [
44
                    {'index': item['index'], 'relevance_score': item['relevance_score']}
45
                    for item in results
46
                ]

1
class Settings(BaseSettings):
2
    # RAG 配置
3
   CHUNK_SIZE: int = 800
4
   CHUNK_OVERLAP: int = 150
5
   RAG_TOP_K: int = 30
6
   RERANK_TOP_K: int = 10
7
   RELEVANCE_THRESHOLD: float = 0.15
8
   CHUNK_SIZE: int = 600           # 减小chunk大小
9
   CHUNK_OVERLAP: int = 200         # 增加重叠区域
10
   RAG_TOP_K: int = 15              # 平衡检索数量
11
   RERANK_TOP_K: int = 6             # 平衡重排序数量
12
   RELEVANCE_THRESHOLD: float = 0.08  # 降低阈值
13

14
    # 模型配置
15
   LLM_MODEL: str = "qwen-max"
16
   LLM_MODEL: str = "qwen-turbo"    # 使用平衡模型
17
    EMBEDDING_MODEL: str = "text-embedding-v4"
18
    RERANK_MODEL: str = "qwen3-rerank"
19

20
   # 默认超时60秒
21
   # LLM 超时配置
22
   LLM_TIMEOUT_SECONDS: int = 4     # ✅ 添加4秒超时控制

1
async def _generate_stream(self, prompt: str) -> AsyncGenerator[str, None]:
2
    try:
3
       # 固定60秒超时
4
       async with httpx.AsyncClient(timeout=60.0) as client:
5
       # ✅ 优化1：使用配置的超时而非固定60秒
6
       async with httpx.AsyncClient(timeout=settings.LLM_TIMEOUT_SECONDS) as client:
7
            response = await client.post(
8
                f"{self.llm_base_url}/chat/completions",
9
                headers={
10
                    "Authorization": f"Bearer {self.llm_api_key}",
11
                    "Content-Type": "application/json"
12
                },
13
                json={
14
                    "model": self.llm_model,
15
                    "messages": [{"role": "user", "content": prompt}],
16
                   "stream": True
17
                   "stream": True,
18
                   # ✅ 优化2：限制生成token数量
19
                   "max_tokens": 200,
20
                   # ✅ 优化3：降低随机性
21
                   "temperature": 0.3
22
                }
23
            )
24

25
            # 处理SSE流式响应...

1
import asyncio
2
import time
3
import httpx
4
from typing import List
5

6
async def benchmark_retrieval(
7
    query: str,
8
    embedding_svc,
9
    vector_svc,
10
    iterations: int = 100
11
) -> dict:
12
    """检索延迟基准测试"""
13
    latencies = []
14

15
    for _ in range(iterations):
16
        start = time.time()
17
        query_vector = await embedding_svc.embed_text(query)
18
        results = await vector_svc.similarity_search(query_vector, top_k=15)
19
        latency = (time.time() - start) * 1000
20
        latencies.append(latency)
21

22
    latencies.sort()
23
    return {
24
        "avg": sum(latencies) / len(latencies),
25
        "p50": latencies[len(latencies) // 2],
26
        "p95": latencies[int(len(latencies) * 0.95)],
27
        "p99": latencies[int(len(latencies) * 0.99)]
28
    }