diff --git a/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_folder.py b/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_folder.py index 6963346..13757f7 100644 --- a/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_folder.py +++ b/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_folder.py @@ -66,6 +66,14 @@ def chunk_text(text: str, chunk_size: int, overlap: int) -> List[str]: _QA_Q_RE = re.compile(r"^\s*(\d+)\s*[\.．、)]\s*(.+?)\s*$") _QA_A_RE = re.compile(r"^\s*[•\-*]\s*答案\s*[:：]\s*(.*)$") +# 兼容：markdown Q&A 版（常见：**Q5：...** **A：** ...） +_MD_Q_RE = re.compile(r"^\s*(?:\*\*)?Q\s*(\d+)?\s*[:：]\s*(.+?)\s*(?:\*\*)?\s*$", re.IGNORECASE) +_MD_A_RE = re.compile(r"^\s*(?:\*\*)?A\s*[:：]\s*(?:\*\*)?\s*(.*)\s*$", re.IGNORECASE) +_MD_INLINE_QA_RE = re.compile( + r"\*\*\s*Q\s*\d*\s*[:：]\s*(.+?)\s*\*\*\s*(?:\*\*)?\s*A\s*[:：]\s*(?:\*\*)?\s*(.+)\s*$", + re.IGNORECASE, +) + def parse_markdown_qa(text: str) -> List[Dict[str, str]]: """解析形如： @@ -78,33 +86,66 @@ def parse_markdown_qa(text: str) -> List[Dict[str, str]]: cur_q: str | None = None cur_a_lines: List[str] = [] + in_answer = False def flush(): - nonlocal cur_q, cur_a_lines + nonlocal cur_q, cur_a_lines, in_answer if cur_q: ans = "\n".join([a for a in cur_a_lines if a.strip()]).strip() if ans: items.append({"question": cur_q.strip(), "answer": ans}) cur_q = None cur_a_lines = [] + in_answer = False for ln in lines: if not ln.strip(): continue + + m_inline = _MD_INLINE_QA_RE.match(ln) + if m_inline: + flush() + q = m_inline.group(1).strip() + a = m_inline.group(2).strip() + if q and a: + items.append({"question": q, "answer": a}) + continue + m_q = _QA_Q_RE.match(ln) if m_q: flush() cur_q = m_q.group(2) continue + + m_md_q = _MD_Q_RE.match(ln) + if m_md_q: + flush() + cur_q = m_md_q.group(2) + continue + m_a = _QA_A_RE.match(ln) if m_a and cur_q: first = m_a.group(1).strip() if first: cur_a_lines.append(first) + in_answer = True + continue + + m_md_a = _MD_A_RE.match(ln) + if m_md_a and cur_q: + first = (m_md_a.group(1) or "").strip() + if first: + cur_a_lines.append(first) + in_answer = True continue if cur_q: - # 支持答案多行（没有“•答案：”前缀的续行） - cur_a_lines.append(ln.strip()) + # 支持答案多行（没有“A：/•答案：”前缀的续行） + if in_answer: + s = ln.strip() + # 跳过与答案无关的章节标题/分隔线 + if s.startswith("#") or s in {"---", "***"} or re.fullmatch(r"[-*_]{3,}", s): + continue + cur_a_lines.append(s) flush() return items diff --git a/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_md_file.py b/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_md_file.py index 9a4b656..36007df 100644 --- a/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_md_file.py +++ b/ruoyi-fastapi-backend/sub_applications/kb_es/ingest_md_file.py @@ -44,6 +44,14 @@ def _slug_id(s: str) -> str: _QA_Q_RE = re.compile(r"^\s*(\d+)\s*[\.．、)]\s*(.+?)\s*$") _QA_A_RE = re.compile(r"^\s*[•\-*]\s*答案\s*[:：]\s*(.*)$") +# 兼容：markdown Q&A 版（常见：**Q5：...** **A：** ...） +_MD_Q_RE = re.compile(r"^\s*(?:\*\*)?Q\s*(\d+)?\s*[:：]\s*(.+?)\s*(?:\*\*)?\s*$", re.IGNORECASE) +_MD_A_RE = re.compile(r"^\s*(?:\*\*)?A\s*[:：]\s*(?:\*\*)?\s*(.*)\s*$", re.IGNORECASE) +_MD_INLINE_QA_RE = re.compile( + r"\*\*\s*Q\s*\d*\s*[:：]\s*(.+?)\s*\*\*\s*(?:\*\*)?\s*A\s*[:：]\s*(?:\*\*)?\s*(.+)\s*$", + re.IGNORECASE, +) + def parse_markdown_qa(text: str) -> List[Dict[str, str]]: lines = [ln.rstrip() for ln in (text or "").splitlines()] @@ -51,32 +59,64 @@ def parse_markdown_qa(text: str) -> List[Dict[str, str]]: cur_q: str | None = None cur_a_lines: List[str] = [] + in_answer = False def flush() -> None: - nonlocal cur_q, cur_a_lines + nonlocal cur_q, cur_a_lines, in_answer if cur_q: ans = "\n".join([a for a in cur_a_lines if a.strip()]).strip() if ans: items.append({"question": cur_q.strip(), "answer": ans}) cur_q = None cur_a_lines = [] + in_answer = False for ln in lines: if not ln.strip(): continue + + m_inline = _MD_INLINE_QA_RE.match(ln) + if m_inline: + flush() + q = m_inline.group(1).strip() + a = m_inline.group(2).strip() + if q and a: + items.append({"question": q, "answer": a}) + continue + m_q = _QA_Q_RE.match(ln) if m_q: flush() cur_q = m_q.group(2) continue + + m_md_q = _MD_Q_RE.match(ln) + if m_md_q: + flush() + cur_q = m_md_q.group(2) + continue + m_a = _QA_A_RE.match(ln) if m_a and cur_q: first = m_a.group(1).strip() if first: cur_a_lines.append(first) + in_answer = True continue - if cur_q: - cur_a_lines.append(ln.strip()) + + m_md_a = _MD_A_RE.match(ln) + if m_md_a and cur_q: + first = (m_md_a.group(1) or "").strip() + if first: + cur_a_lines.append(first) + in_answer = True + continue + + if cur_q and in_answer: + s = ln.strip() + if s.startswith("#") or s in {"---", "***"} or re.fullmatch(r"[-*_]{3,}", s): + continue + cur_a_lines.append(s) flush() return items diff --git a/ruoyi-fastapi-backend/命令.md b/ruoyi-fastapi-backend/命令.md index 91ae26d..570ecb9 100644 --- a/ruoyi-fastapi-backend/命令.md +++ b/ruoyi-fastapi-backend/命令.md @@ -5,10 +5,7 @@ bash ./load_env.sh dev python sub_applications/kb_es/create_index.py bash ./load_env.sh dev python sub_applications/kb_es/ingest_folder.py --folder ./kb_data ``` -### 删除数据库 -``` -curl -XDELETE "http://127.0.0.1:1200/kb_qa_v1 #（kb_qa_v1是env中数据库的名字） -``` + ### 新增数据 新增文件：sub_applications/kb_es/ingest_md_file.py @@ -40,7 +37,10 @@ journalctl -u ruoyi-fastapi-backend.service --since today | grep -E "\[StaticQA\ redis-cli -h 10.0.0.58 -p 6379 -n 3 --scan --pattern "rag:semantic:cache:*" \ | while read -r k; do redis-cli -h 10.0.0.58 -p 6379 -n 3 del "$k" >/dev/null; done ### 删除数据库 + +``` bash ./load_env.sh dev bash -c 'curl -u "$KB_ES_USERNAME:$KB_ES_PASSWORD" -XDELETE "$KB_ES_URL/$KB_ES_INDEX"' +``` ``` #验证删除