27 lines
753 B
Python
27 lines
753 B
Python
"""Text normalization for accent-insensitive search.
|
|
|
|
Strips diacritics so "phở" -> "pho" and "ñandú" -> "nandu".
|
|
Used now for display/utility; drives `title_norm` shadow column in Phase 2 listings.
|
|
"""
|
|
import unicodedata
|
|
|
|
# Vietnamese đ/Đ do not decompose via NFKD, map explicitly.
|
|
_EXPLICIT = {
|
|
"đ": "d", "Đ": "d",
|
|
"ð": "d", "Ð": "d",
|
|
}
|
|
|
|
|
|
def normalize(text: str) -> str:
|
|
if not text:
|
|
return ""
|
|
out = []
|
|
for ch in text:
|
|
if ch in _EXPLICIT:
|
|
out.append(_EXPLICIT[ch])
|
|
continue
|
|
decomposed = unicodedata.normalize("NFKD", ch)
|
|
stripped = "".join(c for c in decomposed if not unicodedata.combining(c))
|
|
out.append(stripped)
|
|
return "".join(out).lower().strip()
|