DeepSeek Harness 本地聊天记忆：按来源保存聊天原文，关键词 + 向量检索，可选大模型重排与检索摘要。源码公开，仅限非商用
浏览文件
main
分支
run.js
111 行 · 7.8 KB历史下载| 1 | /** |
| 2 | * Retrieval evaluation over the synthetic chat corpus, through the same recall path the host uses. |
| 3 | * node test/eval/run.js prints a report |
| 4 | * import { evaluate } used by the regression test |
| 5 | */ |
| 6 | import { mkdtempSync, rmSync } from 'node:fs'; |
| 7 | import { tmpdir } from 'node:os'; |
| 8 | import { join } from 'node:path'; |
| 9 | import { RetrievalMemory } from '../../src/store.js'; |
| 10 | import { viewOf } from '../../src/origin.js'; |
| 11 | import * as recallModule from '../../src/recall.js'; |
| 12 | import { facts, queries, filler, BOILER, TAIL } from './corpus.js'; |
| 13 | |
| 14 | const SHARE = { shareGroups: true, shareOwner: true }; |
| 15 | const access = scope => ({ namespace: 'eval', userId: 'u', scope, ...viewOf(scope, SHARE) }); |
| 16 | |
| 17 | /** |
| 18 | * @param {{ topK?: number, maxChars?: number, embedding?: object }} options |
| 19 | * `embedding`: provider settings (as in config) to evaluate hybrid retrieval; omitted = keywords only. |
| 20 | */ |
| 21 | export async function evaluate({ topK = 6, maxChars = 4000, embedding, reranker, summaries = true } = {}) { |
| 22 | const dir = mkdtempSync(join(tmpdir(), 'dsh-eval-')); |
| 23 | const memory = new RetrievalMemory({ dir, ...(embedding ? { embedding } : {}), ...(reranker ? { reranker } : {}) }); |
| 24 | const usage = { calls: 0, ms: [], prompt: 0, cached: 0, completion: 0 }; |
| 25 | try { |
| 26 | const keyOf = new Map(); |
| 27 | for (const row of [...filler(), ...facts]) { |
| 28 | const { id } = memory.put({ namespace: 'eval', userId: 'u', scope: row.scope, sourceKey: row.key, createdAt: row.time, content: row.content }); |
| 29 | keyOf.set(id, row.key); |
| 30 | } |
| 31 | const summaryUsage = { calls: 0, prompt: 0, completion: 0, records: 0 }; |
| 32 | if (memory.summarizer && summaries) for (;;) { |
| 33 | const batch = await memory.summarize({ limit: 10 }); |
| 34 | const spent = memory.summarizer.usage; |
| 35 | if (spent) { summaryUsage.calls++; summaryUsage.prompt += spent.prompt_tokens ?? 0; summaryUsage.completion += spent.completion_tokens ?? 0; memory.summarizer.usage = null; } |
| 36 | summaryUsage.records += batch.summarized ?? 0; |
| 37 | if (batch.disabled || !batch.remaining) break; |
| 38 | } |
| 39 | if (embedding) for (;;) { |
| 40 | const batch = await memory.index({ limit: 64 }); |
| 41 | if (batch.disabled || !batch.remaining) break; |
| 42 | } |
| 43 | // The bridge has sent its fixed instructions on earlier turns. |
| 44 | for (let i = 0; i < 3; i++) memory.stripBoilerplate(`${BOILER}\n【新消息】\n路人${i}：第${i}次唤醒的一条普通消息\n${TAIL}`, { count: true }); |
| 45 | const rows = []; |
| 46 | for (const query of queries) { |
| 47 | const raw = query.q.replace('__BOILER__', BOILER).replace('__TAIL__', TAIL); |
| 48 | const text = memory.stripBoilerplate(raw); |
| 49 | const started = performance.now(); |
| 50 | const { hits, text: evidence } = await recallModule.recall(memory, text, access(query.scope), { topK, maxChars, here: query.scope }); |
| 51 | usage.ms.push(performance.now() - started); |
| 52 | const spent = memory.ranker?.usage; |
| 53 | if (spent) { |
| 54 | usage.calls++; usage.prompt += spent.prompt_tokens ?? 0; usage.completion += spent.completion_tokens ?? 0; |
| 55 | usage.cached += spent.prompt_cache_hit_tokens ?? spent.prompt_tokens_details?.cached_tokens ?? 0; |
| 56 | memory.ranker.usage = null; |
| 57 | } |
| 58 | const ranked = hits.map(hit => keyOf.get(hit.record_id)); |
| 59 | const ranks = query.expect.map(key => ranked.indexOf(key)).filter(rank => rank >= 0); |
| 60 | const first = ranks.length ? Math.min(...ranks) : -1; |
| 61 | rows.push({ |
| 62 | kind: query.kind, q: query.q.includes('__BOILER__') ? '(长提示) ' + raw.split('\n').find(line => line.includes('：') && !line.startsWith('【')) : query.q, |
| 63 | hit1: first === 0, hit3: first >= 0 && first < 3, hitK: first >= 0, rr: first >= 0 ? 1 / (first + 1) : 0, |
| 64 | answer: evidence.includes(query.answer), |
| 65 | // Evidence is chronological: the newer fact must be present and read after the older one. |
| 66 | newerLast: query.older ? evidence.includes(query.answer) && (!evidence.includes(query.olderAnswer) || evidence.lastIndexOf(query.answer) > evidence.lastIndexOf(query.olderAnswer)) : undefined, |
| 67 | leak: hits.some(hit => hit.scope.startsWith('private:') && hit.scope !== query.scope), |
| 68 | top: ranked.slice(0, 3), chars: evidence.length, |
| 69 | }); |
| 70 | } |
| 71 | const summary = kind => { |
| 72 | const set = rows.filter(row => kind === 'all' ? row.kind !== 'semantic' : row.kind === kind); |
| 73 | const mean = field => set.length ? set.reduce((sum, row) => sum + Number(row[field]), 0) / set.length : 0; |
| 74 | return { n: set.length, hit1: mean('hit1'), hit3: mean('hit3'), hitK: mean('hitK'), mrr: mean('rr'), answer: mean('answer') }; |
| 75 | }; |
| 76 | const kinds = ['exact', 'paraphrase', 'context', 'update', 'noisy', 'semantic', 'all']; |
| 77 | return { rows, byKind: Object.fromEntries(kinds.map(kind => [kind, summary(kind)])), usage, summaryUsage, |
| 78 | evidenceChars: Math.round(rows.reduce((sum, row) => sum + row.chars, 0) / rows.length), |
| 79 | newerLast: rows.filter(row => row.newerLast !== undefined).every(row => row.newerLast), |
| 80 | leaks: rows.filter(row => row.leak).length }; |
| 81 | } finally { memory.close(); rmSync(dir, { recursive: true, force: true }); } |
| 82 | } |
| 83 | |
| 84 | if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith('run.js')) { |
| 85 | // node test/eval/run.js [--embed <ollama model>] [--url http://127.0.0.1:11434] |
| 86 | // [--rerank <chat model> --rerank-url https://api.deepseek.com --rerank-key-env DEEPSEEK_API_KEY] |
| 87 | const arg = name => { const i = process.argv.indexOf(name); return i > 0 ? process.argv[i + 1] : undefined; }; |
| 88 | const model = arg('--embed'), judge = arg('--rerank'); |
| 89 | const embedding = model ? { enabled: true, protocol: 'ollama', baseUrl: arg('--url') || 'http://127.0.0.1:11434', model, timeoutMs: 30000 } : undefined; |
| 90 | const reranker = judge ? { enabled: true, protocol: 'chat', baseUrl: arg('--rerank-url') || 'https://api.deepseek.com', model: judge, |
| 91 | apiKeyEnv: arg('--rerank-key-env') || 'DEEPSEEK_API_KEY', allowRemote: true, timeoutMs: 15000 } : undefined; |
| 92 | const summaries = !process.argv.includes('--no-summary'); |
| 93 | const started = performance.now(); |
| 94 | const result = await evaluate({ embedding, reranker, summaries }); |
| 95 | const mode = ['关键词', model && '向量 ' + model, judge && '重排 ' + judge, judge && summaries && '摘要'].filter(Boolean).join(' + '); |
| 96 | console.log(`模式：${mode}，用时 ${((performance.now() - started) / 1000).toFixed(1)}s\n`); |
| 97 | const pct = value => (value * 100).toFixed(0).padStart(4) + '%'; |
| 98 | console.log('类型 条数 命中@1 命中@3 命中@6 MRR 答案在证据里'); |
| 99 | for (const [kind, s] of Object.entries(result.byKind)) |
| 100 | console.log(`${kind.padEnd(11)} ${String(s.n).padStart(3)} ${pct(s.hit1)} ${pct(s.hit3)} ${pct(s.hitK)} ${s.mrr.toFixed(2)} ${pct(s.answer)}`); |
| 101 | console.log(`变化过的事实：新说法在证据里且排在旧说法之后：${result.newerLast ? '是' : '否'}；跨私聊泄露：${result.leaks}`); |
| 102 | const ms = [...result.usage.ms].sort((a, b) => a - b); |
| 103 | console.log(`每次检索耗时 p50=${ms[Math.floor(ms.length / 2)].toFixed(0)}ms max=${ms.at(-1).toFixed(0)}ms；给模型的证据平均 ${result.evidenceChars} 字`); |
| 104 | const s = result.summaryUsage; |
| 105 | if (s.calls) console.log(`摘要 ${s.records} 条记录、${s.calls} 次请求：每条平均输入 ${Math.round(s.prompt / s.records)} token，输出 ${Math.round(s.completion / s.records)} token`); |
| 106 | if (result.usage.calls) console.log(`重排 ${result.usage.calls} 次：平均输入 ${Math.round(result.usage.prompt / result.usage.calls)} token` |
| 107 | + `（其中缓存命中 ${Math.round(result.usage.cached / result.usage.calls)}），输出 ${Math.round(result.usage.completion / result.usage.calls)} token`); |
| 108 | console.log('\n未命中或答案缺失：'); |
| 109 | for (const row of result.rows.filter(row => !row.hit1 || !row.answer)) |
| 110 | console.log(` [${row.kind}] ${row.q} → 前三：${row.top.join(', ') || '（空）'}${row.answer ? '' : ' ✗答案不在证据里'}`); |
| 111 | } |
