updateMD规则
This commit is contained in:
parent
0dab7ae7c1
commit
879f94f720
@ -66,6 +66,14 @@ def chunk_text(text: str, chunk_size: int, overlap: int) -> List[str]:
|
||||
_QA_Q_RE = re.compile(r"^\s*(\d+)\s*[\.．、)]\s*(.+?)\s*$")
|
||||
_QA_A_RE = re.compile(r"^\s*[•\-*]\s*答案\s*[:：]\s*(.*)$")
|
||||
|
||||
# 兼容：markdown Q&A 版（常见：**Q5：...** **A：** ...）
|
||||
_MD_Q_RE = re.compile(r"^\s*(?:\*\*)?Q\s*(\d+)?\s*[:：]\s*(.+?)\s*(?:\*\*)?\s*$", re.IGNORECASE)
|
||||
_MD_A_RE = re.compile(r"^\s*(?:\*\*)?A\s*[:：]\s*(?:\*\*)?\s*(.*)\s*$", re.IGNORECASE)
|
||||
_MD_INLINE_QA_RE = re.compile(
|
||||
r"\*\*\s*Q\s*\d*\s*[:：]\s*(.+?)\s*\*\*\s*(?:\*\*)?\s*A\s*[:：]\s*(?:\*\*)?\s*(.+)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def parse_markdown_qa(text: str) -> List[Dict[str, str]]:
|
||||
"""解析形如：
|
||||
@ -78,33 +86,66 @@ def parse_markdown_qa(text: str) -> List[Dict[str, str]]:
|
||||
|
||||
cur_q: str | None = None
|
||||
cur_a_lines: List[str] = []
|
||||
in_answer = False
|
||||
|
||||
def flush():
|
||||
nonlocal cur_q, cur_a_lines
|
||||
nonlocal cur_q, cur_a_lines, in_answer
|
||||
if cur_q:
|
||||
ans = "\n".join([a for a in cur_a_lines if a.strip()]).strip()
|
||||
if ans:
|
||||
items.append({"question": cur_q.strip(), "answer": ans})
|
||||
cur_q = None
|
||||
cur_a_lines = []
|
||||
in_answer = False
|
||||
|
||||
for ln in lines:
|
||||
if not ln.strip():
|
||||
continue
|
||||
|
||||
m_inline = _MD_INLINE_QA_RE.match(ln)
|
||||
if m_inline:
|
||||
flush()
|
||||
q = m_inline.group(1).strip()
|
||||
a = m_inline.group(2).strip()
|
||||
if q and a:
|
||||
items.append({"question": q, "answer": a})
|
||||
continue
|
||||
|
||||
m_q = _QA_Q_RE.match(ln)
|
||||
if m_q:
|
||||
flush()
|
||||
cur_q = m_q.group(2)
|
||||
continue
|
||||
|
||||
m_md_q = _MD_Q_RE.match(ln)
|
||||
if m_md_q:
|
||||
flush()
|
||||
cur_q = m_md_q.group(2)
|
||||
continue
|
||||
|
||||
m_a = _QA_A_RE.match(ln)
|
||||
if m_a and cur_q:
|
||||
first = m_a.group(1).strip()
|
||||
if first:
|
||||
cur_a_lines.append(first)
|
||||
in_answer = True
|
||||
continue
|
||||
|
||||
m_md_a = _MD_A_RE.match(ln)
|
||||
if m_md_a and cur_q:
|
||||
first = (m_md_a.group(1) or "").strip()
|
||||
if first:
|
||||
cur_a_lines.append(first)
|
||||
in_answer = True
|
||||
continue
|
||||
if cur_q:
|
||||
# 支持答案多行（没有“•答案：”前缀的续行）
|
||||
cur_a_lines.append(ln.strip())
|
||||
# 支持答案多行（没有“A：/•答案：”前缀的续行）
|
||||
if in_answer:
|
||||
s = ln.strip()
|
||||
# 跳过与答案无关的章节标题/分隔线
|
||||
if s.startswith("#") or s in {"---", "***"} or re.fullmatch(r"[-*_]{3,}", s):
|
||||
continue
|
||||
cur_a_lines.append(s)
|
||||
|
||||
flush()
|
||||
return items
|
||||
|
||||
@ -44,6 +44,14 @@ def _slug_id(s: str) -> str:
|
||||
_QA_Q_RE = re.compile(r"^\s*(\d+)\s*[\.．、)]\s*(.+?)\s*$")
|
||||
_QA_A_RE = re.compile(r"^\s*[•\-*]\s*答案\s*[:：]\s*(.*)$")
|
||||
|
||||
# 兼容：markdown Q&A 版（常见：**Q5：...** **A：** ...）
|
||||
_MD_Q_RE = re.compile(r"^\s*(?:\*\*)?Q\s*(\d+)?\s*[:：]\s*(.+?)\s*(?:\*\*)?\s*$", re.IGNORECASE)
|
||||
_MD_A_RE = re.compile(r"^\s*(?:\*\*)?A\s*[:：]\s*(?:\*\*)?\s*(.*)\s*$", re.IGNORECASE)
|
||||
_MD_INLINE_QA_RE = re.compile(
|
||||
r"\*\*\s*Q\s*\d*\s*[:：]\s*(.+?)\s*\*\*\s*(?:\*\*)?\s*A\s*[:：]\s*(?:\*\*)?\s*(.+)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def parse_markdown_qa(text: str) -> List[Dict[str, str]]:
|
||||
lines = [ln.rstrip() for ln in (text or "").splitlines()]
|
||||
@ -51,32 +59,64 @@ def parse_markdown_qa(text: str) -> List[Dict[str, str]]:
|
||||
|
||||
cur_q: str | None = None
|
||||
cur_a_lines: List[str] = []
|
||||
in_answer = False
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal cur_q, cur_a_lines
|
||||
nonlocal cur_q, cur_a_lines, in_answer
|
||||
if cur_q:
|
||||
ans = "\n".join([a for a in cur_a_lines if a.strip()]).strip()
|
||||
if ans:
|
||||
items.append({"question": cur_q.strip(), "answer": ans})
|
||||
cur_q = None
|
||||
cur_a_lines = []
|
||||
in_answer = False
|
||||
|
||||
for ln in lines:
|
||||
if not ln.strip():
|
||||
continue
|
||||
|
||||
m_inline = _MD_INLINE_QA_RE.match(ln)
|
||||
if m_inline:
|
||||
flush()
|
||||
q = m_inline.group(1).strip()
|
||||
a = m_inline.group(2).strip()
|
||||
if q and a:
|
||||
items.append({"question": q, "answer": a})
|
||||
continue
|
||||
|
||||
m_q = _QA_Q_RE.match(ln)
|
||||
if m_q:
|
||||
flush()
|
||||
cur_q = m_q.group(2)
|
||||
continue
|
||||
|
||||
m_md_q = _MD_Q_RE.match(ln)
|
||||
if m_md_q:
|
||||
flush()
|
||||
cur_q = m_md_q.group(2)
|
||||
continue
|
||||
|
||||
m_a = _QA_A_RE.match(ln)
|
||||
if m_a and cur_q:
|
||||
first = m_a.group(1).strip()
|
||||
if first:
|
||||
cur_a_lines.append(first)
|
||||
in_answer = True
|
||||
continue
|
||||
if cur_q:
|
||||
cur_a_lines.append(ln.strip())
|
||||
|
||||
m_md_a = _MD_A_RE.match(ln)
|
||||
if m_md_a and cur_q:
|
||||
first = (m_md_a.group(1) or "").strip()
|
||||
if first:
|
||||
cur_a_lines.append(first)
|
||||
in_answer = True
|
||||
continue
|
||||
|
||||
if cur_q and in_answer:
|
||||
s = ln.strip()
|
||||
if s.startswith("#") or s in {"---", "***"} or re.fullmatch(r"[-*_]{3,}", s):
|
||||
continue
|
||||
cur_a_lines.append(s)
|
||||
|
||||
flush()
|
||||
return items
|
||||
|
||||
@ -5,10 +5,7 @@ bash ./load_env.sh dev python sub_applications/kb_es/create_index.py
|
||||
bash ./load_env.sh dev python sub_applications/kb_es/ingest_folder.py --folder ./kb_data
|
||||
```
|
||||
|
||||
### 删除数据库
|
||||
```
|
||||
curl -XDELETE "http://127.0.0.1:1200/kb_qa_v1 #（kb_qa_v1是env中数据库的名字）
|
||||
```
|
||||
|
||||
|
||||
### 新增数据
|
||||
新增文件：sub_applications/kb_es/ingest_md_file.py
|
||||
@ -40,7 +37,10 @@ journalctl -u ruoyi-fastapi-backend.service --since today | grep -E "\[StaticQA\
|
||||
redis-cli -h 10.0.0.58 -p 6379 -n 3 --scan --pattern "rag:semantic:cache:*" \
|
||||
| while read -r k; do redis-cli -h 10.0.0.58 -p 6379 -n 3 del "$k" >/dev/null; done
|
||||
### 删除数据库
|
||||
|
||||
```
|
||||
bash ./load_env.sh dev bash -c 'curl -u "$KB_ES_USERNAME:$KB_ES_PASSWORD" -XDELETE "$KB_ES_URL/$KB_ES_INDEX"'
|
||||
```
|
||||
|
||||
```
|
||||
#验证删除
|
||||
|
||||
Loading…
Reference in New Issue
Block a user