Optimize citation removal in sanitize_ai_text (#650)

Signed-off-by: Glenn Jocher <[email protected]>
This commit is contained in:
Glenn Jocher
2025-11-09 22:20:40 +00:00
committed by GitHub
parent 2d05a2524c
commit 95942d5d75
2 changed files with 5 additions and 4 deletions
+1 -1
View File
@@ -27,4 +27,4 @@
# ├── test_summarize_pr.py
# └── ...
__version__ = "0.2.5"
__version__ = "0.2.6"
+4 -3
View File
@@ -32,13 +32,14 @@ SYSTEM_PROMPT_ADDITION = """Guidance:
- Use the "@" symbol to refer to GitHub users, e.g. @glenn-jocher.
- Tone: Adopt a professional, friendly, and concise tone.
"""
_CITATION_PATTERN = re.compile(
r"[\uE000-\uF8FF]*\bcite[\uE000-\uF8FF]*(turn\d+(?:search|view)\d+|[\w\d]+)[\uE000-\uF8FF]*"
)
def sanitize_ai_text(s: str) -> str:
"""Strip private-use citation tokens like '' and normalize whitespace."""
return re.sub(
r"[\uE000-\uF8FF]*\bcite[\uE000-\uF8FF]*(turn\d+(?:search|view)\d+|[\w\d]+)[\uE000-\uF8FF]*", "", s or ""
)
return _CITATION_PATTERN.sub("", s) if s else ""
def remove_outer_codeblocks(string):