从零搭建我的本地 AI 知识库：5000 文件、双索引 RAG 与自动化流水线

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

┌─────────────────────────────────────────┐
│  Layer 1 · 记忆层（WorkBuddy 4 层记忆）   │
│  SOUL / IDENTITY / USER / MEMORY + 每日日志 │
├─────────────────────────────────────────┤
│  Layer 2 · 知识库层（PARA + 领域 wiki）    │
│  10-PROJECTS / 20-AREAS / 30-RESOURCES   │
│  + 业务真实数据（软链穿透）               │
├─────────────────────────────────────────┤
│  Layer 3 · 索引层（双 RAG 索引）          │
│  A 索引：含 xlsx 聚合（业务问题）          │
│  B 索引：零 xlsx（通用检索）              │
├─────────────────────────────────────────┤
│  Layer 4 · 自动化层（7 个 automation）     │
│  每日增量同步 / 月度兜底重建 / 死链扫描   │
└─────────────────────────────────────────┘

1

scan → embed → merge → verify → sync_obsidian

1
2
3

export KB_ROOT="$HOME/Documents/MyKB"
mkdir -p "$KB_ROOT"/{00-INBOX/2026,10-PROJECTS,20-AREAS,30-RESOURCES,40-ARCHIVE,99-SYSTEM/SOPs}
mkdir -p "$KB_ROOT/.workbuddy"/{scripts,memory/incremental_sync,memory/incremental_sync_logs,memory/state/rag_history}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

cat > "$KB_ROOT/00-INBOX/2026/2026-07-18-hello-kb.md" <<'EOF'
---
title: "我的知识库第一页"
date: 2026-07-18
tags: ["kb", "start"]
status: active
---

# 我的知识库第一页

这是我知识库的第一条笔记。后续所有思考、项目、阅读材料都会从这里开始，再归类到 PARA 的四个域。
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

#!/bin/bash
set -e
KB_ROOT="${KB_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}"
QUERY="${1:-知识库}"
TOPK="${2:-5}"

cd "$KB_ROOT"
python3 - <<PY
import os, json, pickle, numpy as np, sys, re
from sentence_transformers import SentenceTransformer

q = """${QUERY}"""
topk = int("""${TOPK}""")
model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
emb = model.encode(q, normalize_embeddings=True)

for name in ["full_workspace", "workspace_summary"]:
    npy = f"{os.environ['KB_ROOT']}/.workbuddy/memory/{name}.npy"
    pkl = f"{os.environ['KB_ROOT']}/.workbuddy/memory/{name}_meta.pkl"
    if not os.path.exists(npy):
        print(f"[{name}] 索引不存在，先跑 build"); continue
    vecs = np.load(npy)
    meta = pickle.load(open(pkl, 'rb'))
    scores = vecs @ emb
    top = np.argsort(scores)[::-1][:topk]
    print(f"\n=== {name} ===")
    for i, idx in enumerate(top, 1):
        print(f"{i}. {scores[idx]:.3f} | {meta[idx]['source']}")
        print(meta[idx]['text'][:200].replace('\n',' '))
PY

1

chmod +x "$KB_ROOT/.workbuddy/scripts/kb_search.sh"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#!/bin/bash
set -e
KB_ROOT="${KB_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}"
cd "$KB_ROOT"
python3 - <<PY
import os, pickle, numpy as np
from sentence_transformers import SentenceTransformer

KB_ROOT = os.environ.get('KB_ROOT', os.getcwd())
model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')

files = []
for root, _, fs in os.walk(KB_ROOT):
    # 排除这些目录
    if any(x in root for x in ['.workbuddy','.obsidian','.git','40-ARCHIVE','00-INBOX']):
        continue
    for f in fs:
        if f.endswith('.md'):
            files.append(os.path.join(root, f))

chunks, meta = [], []
for path in files:
    text = open(path, encoding='utf-8').read()
    # 简单按 800 字切块
    for i in range(0, len(text), 800):
        chunk = text[i:i+1200]
        if len(chunk) < 100: continue
        chunks.append(chunk)
        meta.append({'source': os.path.relpath(path, KB_ROOT), 'text': chunk})

print(f"索引 {len(files)} 文件 -> {len(chunks)} chunks")
vecs = model.encode(chunks, normalize_embeddings=True, show_progress_bar=True)
mem = f"{KB_ROOT}/.workbuddy/memory"
np.save(f"{mem}/full_workspace.npy", vecs)
pickle.dump(meta, open(f"{mem}/full_workspace_meta.pkl", 'wb'))
np.save(f"{mem}/workspace_summary.npy", vecs)
pickle.dump(meta, open(f"{mem}/workspace_summary_meta.pkl", 'wb'))
print("done")
PY

1

chmod +x "$KB_ROOT/.workbuddy/scripts/kb_build.sh"

1
2

export KB_ROOT="$HOME/Documents/MyKB"
"$KB_ROOT/.workbuddy/scripts/kb_build.sh"

1

"$KB_ROOT/.workbuddy/scripts/kb_search.sh" "知识库第一页" 3