替换了ragflow，完善了现在的es/opensearch数据库的流程，清除了静态缓存

ruoyi-fastapi-backend/sub_applications/kb_es/ingest_md_file.py

@@ -0,0 +1,191 @@
+import argparse
+import hashlib
+import json
+import os
+import re
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List
+
+import httpx
+
+
+def _bool_env(name: str, default: bool) -> bool:
+    val = os.getenv(name)
+    if val is None:
+        return default
+    return val.strip().lower() in {"1", "true", "yes", "y", "on"}
+
+
+def _cfg() -> Dict[str, Any]:
+    es_url = os.getenv("KB_ES_URL") or os.getenv("KB_OS_URL")
+    if not es_url:
+        raise SystemExit("Missing KB_ES_URL (or KB_OS_URL)")
+
+    return {
+        "es_url": es_url.rstrip("/"),
+        "index": os.getenv("KB_ES_INDEX", "kb_chunks_v1"),
+        "username": os.getenv("KB_ES_USERNAME", ""),
+        "password": os.getenv("KB_ES_PASSWORD", ""),
+        "verify_ssl": _bool_env("KB_ES_VERIFY_SSL", True),
+        "version": os.getenv("KB_ES_VERSION", "v1"),
+    }
+
+
+def read_text_file(path: Path) -> str:
+    data = path.read_text(encoding="utf-8", errors="ignore")
+    return data.replace("\ufeff", "").strip()
+
+
+def _slug_id(s: str) -> str:
+    return hashlib.sha1(s.encode("utf-8")).hexdigest()[:16]
+
+
+_QA_Q_RE = re.compile(r"^\s*(\d+)\s*[\.．、)]\s*(.+?)\s*$")
+_QA_A_RE = re.compile(r"^\s*[•\-*]\s*答案\s*[:：]\s*(.*)$")
+
+
+def parse_markdown_qa(text: str) -> List[Dict[str, str]]:
+    lines = [ln.rstrip() for ln in (text or "").splitlines()]
+    items: List[Dict[str, str]] = []
+
+    cur_q: str | None = None
+    cur_a_lines: List[str] = []
+
+    def flush() -> None:
+        nonlocal cur_q, cur_a_lines
+        if cur_q:
+            ans = "\n".join([a for a in cur_a_lines if a.strip()]).strip()
+            if ans:
+                items.append({"question": cur_q.strip(), "answer": ans})
+        cur_q = None
+        cur_a_lines = []
+
+    for ln in lines:
+        if not ln.strip():
+            continue
+        m_q = _QA_Q_RE.match(ln)
+        if m_q:
+            flush()
+            cur_q = m_q.group(2)
+            continue
+        m_a = _QA_A_RE.match(ln)
+        if m_a and cur_q:
+            first = m_a.group(1).strip()
+            if first:
+                cur_a_lines.append(first)
+            continue
+        if cur_q:
+            cur_a_lines.append(ln.strip())
+
+    flush()
+    return items
+
+
+def build_docs(file_path: Path, *, doc_id: str) -> List[Dict[str, Any]]:
+    content = read_text_file(file_path)
+    title = file_path.stem
+    source = str(file_path).replace("\\", "/")
+
+    qa_items = parse_markdown_qa(content)
+    if qa_items:
+        docs: List[Dict[str, Any]] = []
+        for i, qa in enumerate(qa_items, 1):
+            qa_id = f"{doc_id}_qa_{i}"
+            q = qa["question"].strip()
+            a = qa["answer"].strip()
+            docs.append(
+                {
+                    "doc_id": doc_id,
+                    "chunk_id": qa_id,
+                    "qa_id": qa_id,
+                    "title": title,
+                    "question": q,
+                    "answer": a,
+                    "content": f"{q}\n{a}",
+                    "source": source,
+                }
+            )
+        return docs
+
+    # 非 Q&A 格式：整体作为一个 chunk 写入
+    if not content:
+        return []
+    return [
+        {
+            "doc_id": doc_id,
+            "chunk_id": f"{doc_id}_0",
+            "title": title,
+            "content": content,
+            "source": source,
+        }
+    ]
+
+
+def to_bulk_ndjson(index: str, docs: List[Dict[str, Any]], *, version: str) -> str:
+    lines: List[str] = []
+    for d in docs:
+        d = dict(d)
+        d["version"] = version
+        _id = d.get("chunk_id")
+        lines.append(json.dumps({"index": {"_index": index, "_id": _id}}, ensure_ascii=False))
+        lines.append(json.dumps(d, ensure_ascii=False))
+    return "\n".join(lines) + "\n"
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description="将单个 Markdown 文件增量写入现有 ES/OpenSearch 索引（不创建/不清空）")
+    parser.add_argument("--file", required=True, help="md 文件路径")
+    parser.add_argument(
+        "--doc-id",
+        default="",
+        help="可选：指定 doc_id（不传则自动生成一个新的，保证每次都是新增）",
+    )
+    args = parser.parse_args()
+
+    cfg = _cfg()
+    file_path = Path(args.file).resolve()
+    if not file_path.exists() or not file_path.is_file():
+        raise SystemExit(f"File not found: {file_path}")
+    if file_path.suffix.lower() != ".md":
+        raise SystemExit("Only .md is supported by this script")
+
+    if args.doc_id:
+        doc_id = _slug_id(args.doc_id.strip())
+    else:
+        # 默认每次生成新 doc_id，确保不会覆盖旧数据
+        doc_id = uuid.uuid4().hex[:16]
+
+    docs = build_docs(file_path, doc_id=doc_id)
+    if not docs:
+        raise SystemExit("No content to ingest")
+
+    auth = None
+    if cfg["username"] and cfg["password"]:
+        auth = (cfg["username"], cfg["password"])
+
+    bulk_url = f"{cfg['es_url']}/_bulk"
+    headers = {"Content-Type": "application/x-ndjson"}
+    body = to_bulk_ndjson(cfg["index"], docs, version=cfg["version"])
+
+    async with httpx.AsyncClient(timeout=30.0, verify=cfg["verify_ssl"]) as client:
+        resp = await client.post(bulk_url, content=body.encode("utf-8"), headers=headers, auth=auth)
+        if resp.status_code >= 300:
+            raise SystemExit(f"Bulk failed HTTP {resp.status_code}: {resp.text}")
+        payload = resp.json()
+        if payload.get("errors"):
+            first_err = None
+            for item in payload.get("items", []):
+                v = item.get("index") or item.get("create") or item.get("update")
+                if v and v.get("error"):
+                    first_err = v.get("error")
+                    break
+            raise SystemExit(f"Bulk had errors: {first_err}")
+
+    print(f"OK index={cfg['index']} doc_id={doc_id} chunks={len(docs)} file={file_path}")
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

ruoyi-fastapi-backend/命令.md

@@ -0,0 +1,41 @@
+### 创建数据库
+```
+  
+bash ./load_env.sh dev python sub_applications/kb_es/create_index.py
+bash ./load_env.sh dev python sub_applications/kb_es/ingest_folder.py --folder ./kb_data
+```
+
+### 删除数据库
+```
+curl -XDELETE "http://127.0.0.1:1200/kb_qa_v1  #（kb_qa_v1是env中数据库的名字）
+```
+
+### 新增数据
+新增文件：sub_applications/kb_es/ingest_md_file.py
+
+   用法
+
+   它会读取你当前环境变量里的：
+   •  KB_ES_URL（或 KB_OS_URL）
+   •  KB_ES_INDEX
+   然后直接往这个 index 里新增文档。
+
+   新增导入（默认每次都新增，不会覆盖旧数据）
+
+   bash
+     python sub_applications/kb_es/ingest_md_file.py --file "路径/你的文件.md"
+
+   默认会生成一个全新的 doc_id（UUID），因此同一个 md 反复执行也会不断新增。
+
+   如果你希望同一个 doc_id（可能覆盖同 ID 的旧数据），可指定 doc-id
+
+   bash
+     python sub_applications/kb_es/ingest_md_file.py --file "路径/你的文件.md" --doc-id "my-doc-id"
+
+### 查询日志
+journalctl -u ruoyi-fastapi-backend.service --since today | grep -E "\[StaticQA\]|\[SemanticCache\]|\[RAG_SOURCE\]|\[KB_ES\]"
+
+
+### 清空缓存
+redis-cli -h 10.0.0.58 -p 6379 -n 3 --scan --pattern "rag:semantic:cache:*" \
+     | while read -r k; do redis-cli -h 10.0.0.58 -p 6379 -n 3 del "$k" >/dev/null; done