增加语气词过滤

This commit is contained in:
Tian jianyong 2025-12-29 12:28:08 +08:00
parent 6902a46b7b
commit 2e393b80af

View File

@ -78,7 +78,7 @@ class MatchService:
'我们', '你们', '他们', '她们', '它们',
'这个', '那个', '这些', '那些',
'这种', '那种', '这类', '那类',
'你好', '您好', '大家好', '', '哈喽', '哈罗',
'', '哈喽', '哈罗',
'早上好', '上午好', '下午好', '晚上好',
'hi', 'hello', 'hey', '', '哈喽',
'怎么样', '如何', '怎样',
@ -111,7 +111,18 @@ class MatchService:
return ''
text = text.strip()
text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9""''()【】《》\s]', '', text)
colloquial_fillers = ['', '', '']
for filler in colloquial_fillers:
text = re.sub(f'{filler}+', filler, text)
trailing_fillers = ['', '', '', '', '', '', '', '', '', '', '', '']
for filler in trailing_fillers:
text = re.sub(f'{filler}\\s*$', '', text)
text = re.sub(f'{filler}$', '', text)
text = re.sub(r'\s+', ' ', text)
for colloquial, standard in self._colloquial_map.items():
@ -166,8 +177,8 @@ class MatchService:
if n1 == n2:
return 1.0
kw1 = set(self.extract_keywords(q1))
kw2 = set(self.extract_keywords(q2))
kw1 = set(self.extract_keywords(n1))
kw2 = set(self.extract_keywords(n2))
if not kw1 or not kw2:
return 0.0