kanshan/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-01-14 06:07:33 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

fix 29184 (#29188)

Browse Source 
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
呆萌闷油瓶 2025-12-18 17:11:52 +08:00 committed by GitHub
parent 46e0548731
commit 5067e4f255
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/core/rag/splitter/fixed_text_splitter.py
View File

@ -95,7 +95,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
splits = re.split(r" +", text) splits = re.split(r" +", text)
else: else:
splits = text.split(separator) splits = text.split(separator)
splits = [item + separator if i < len(splits) else item for i, item in enumerate(splits)] if self._keep_separator:
splits = [s + separator for s in splits[:-1]] + splits[-1:]
else: else:
splits = list(text) splits = list(text)
if separator == "\n": if separator == "\n":
@ -104,7 +105,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
splits = [s for s in splits if (s not in {"", "\n"})] splits = [s for s in splits if (s not in {"", "\n"})]
_good_splits = [] _good_splits = []
_good_splits_lengths = [] # cache the lengths of the splits _good_splits_lengths = [] # cache the lengths of the splits
_separator = separator if self._keep_separator else "" _separator = "" if self._keep_separator else separator
s_lens = self._length_function(splits) s_lens = self._length_function(splits)
if separator != "": if separator != "":
for s, s_len in zip(splits, s_lens): for s, s_len in zip(splits, s_lens):