增加语气词过滤
This commit is contained in:
parent
6902a46b7b
commit
2e393b80af
@ -78,7 +78,7 @@ class MatchService:
|
||||
'我们', '你们', '他们', '她们', '它们',
|
||||
'这个', '那个', '这些', '那些',
|
||||
'这种', '那种', '这类', '那类',
|
||||
'你好', '您好', '大家好', '嗨', '哈喽', '哈罗',
|
||||
'嗨', '哈喽', '哈罗',
|
||||
'早上好', '上午好', '下午好', '晚上好',
|
||||
'hi', 'hello', 'hey', '嗨', '哈喽',
|
||||
'怎么样', '如何', '怎样',
|
||||
@ -111,7 +111,18 @@ class MatchService:
|
||||
return ''
|
||||
|
||||
text = text.strip()
|
||||
|
||||
text = re.sub(r'[^\u4e00-\u9fa5a-zA-Z0-9,。?!、;:""''()【】《》\s]', '', text)
|
||||
|
||||
colloquial_fillers = ['嗯', '呃', '额']
|
||||
for filler in colloquial_fillers:
|
||||
text = re.sub(f'{filler}+', filler, text)
|
||||
|
||||
trailing_fillers = ['嗯', '哦', '啊', '呀', '啦', '呗', '嘛', '吧', '噢', '呃', '额', '哈']
|
||||
for filler in trailing_fillers:
|
||||
text = re.sub(f'{filler}\\s*$', '', text)
|
||||
text = re.sub(f'{filler}$', '', text)
|
||||
|
||||
text = re.sub(r'\s+', ' ', text)
|
||||
|
||||
for colloquial, standard in self._colloquial_map.items():
|
||||
@ -166,8 +177,8 @@ class MatchService:
|
||||
if n1 == n2:
|
||||
return 1.0
|
||||
|
||||
kw1 = set(self.extract_keywords(q1))
|
||||
kw2 = set(self.extract_keywords(q2))
|
||||
kw1 = set(self.extract_keywords(n1))
|
||||
kw2 = set(self.extract_keywords(n2))
|
||||
|
||||
if not kw1 or not kw2:
|
||||
return 0.0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user