30 · 失败分析、延迟口径与统一服务

from pathlib import Path
import os, sys
root = Path.cwd()
if not (root / 'reclab').exists(): root = root.parent
os.chdir(root); sys.path.insert(0, str(root))
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from reclab.search_lessons import context, query_vectors, score_query, labels
from reclab.search import BM25, rrf
from reclab.retrieval import topk, metrics
ctx = context()
print('Shared search run:', ctx['receipt']['run_id'])

Shared search run: 20261030T124927-d1467622

rows=[]
for r in ctx['per_query']:
    a=r['metrics']['bm25']['ndcg@10']; b=r['metrics']['linear_rerank']['ndcg@10']
    rows.append({'qid':r['qid'],'bm25':a,'linear':b,'delta':b-a})
errors=pd.DataFrame(rows)
print('300 queries: wins/ties/losses',int((errors.delta>1e-10).sum()),int((errors.delta.abs()<=1e-10).sum()),int((errors.delta < -1e-10).sum()))
display(errors.sort_values('delta').head(8))
errors.delta.plot.hist(bins=20,title='Linear rerank minus BM25: all 300 test queries');plt.show()

Notebook 保存的输出 · 本页未执行

300 queries: wins/ties/losses 63 211 26

	qid	bm25	linear	delta
223	660	1.000000	0.000000	-1.000000
139	303	0.630930	0.000000	-0.630930
76	1303	1.000000	0.430677	-0.569323
7	1024	1.000000	0.500000	-0.500000
264	830	0.386853	0.000000	-0.386853
226	690	0.386853	0.000000	-0.386853
105	163	1.000000	0.630930	-0.369070
177	507	1.000000	0.630930	-0.369070

<Figure size 640x480 with 1 Axes>

failure_ids=[r['qid'] for r in ctx['result']['failures']]
failure_vectors=query_vectors(failure_ids); diagnostics=[]
by_id={r['qid']:r for r in ctx['per_query']}
for q,v in zip(failure_ids,failure_vectors):
    candidates=topk(score_query(q,v)[2],20); truth=ctx['truth']['test'][q]
    diagnostics.append({'qid':q,'relevant_in_candidates20':len(set(candidates)&truth), 'known_relevant':len(truth),
                        'hybrid_ndcg10':by_id[q]['metrics']['hybrid_rrf']['ndcg@10'],
                        'tinybert_ndcg10':by_id[q]['metrics']['tinybert_rerank']['ndcg@10']})
display(pd.DataFrame(diagnostics))
print('Local ctx contains query text for inspection; raw text is not saved to Git.')

/Users/lynnlee/Documents/Codex/2026-10-30/task/soutui-lab/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

Local ctx contains query text for inspection; raw text is not saved to Git.

import json, time, urllib.request
base='http://127.0.0.1:18580'
def call_search(query,method):
    request=urllib.request.Request(base+'/api/search',data=json.dumps({'query':query,'method':method,'limit':10}).encode(),headers={'Content-Type':'application/json'})
    with urllib.request.urlopen(request,timeout=90) as response: return json.load(response)
qid=ctx['per_query'][0]['qid']; query=ctx['queries'][qid]
call_search(query,'hybrid_rrf')  # explicit warmup, not included below
latency=[]
for _ in range(5):
    start=time.perf_counter(); answer=call_search(query,'hybrid_rrf'); wall=(time.perf_counter()-start)*1000
    latency.append({'http_wall_ms':wall,'reported_search_ms':answer['latency_ms']})
display(pd.DataFrame(latency))

expected=by_id[qid]
checks=[]
for method in ['bm25','dense','hybrid_rrf','linear_rerank','tinybert_rerank']:
    response=call_search(query,method)
    ids=[x['document_id'] for x in response['items']]
    assert response['run_id']==ctx['receipt']['run_id']
    assert ids==expected['top10'][method]
    checks.append({'method':method,'documents':len(ids),'matches_offline':True})
display(pd.DataFrame(checks))