text_processing.py

"""
Shared text cleaning and light filtering helpers.

Used by ``cppa_slack_tracker`` and ``discord_activity_tracker`` for normalizing
message text. ``SLACK_*`` phrase lists feed :func:`filter_sentence` (Slack) and
:func:`clean_discord_text` (Discord markup strip + same filler removal).
"""

from __future__ import annotations

import html
import re
from typing import Iterable, FrozenSet, Optional

# Default greeting/unessential words for filter_sentence (Slack message cleaning)
SLACK_GREETING_WORDS: FrozenSet[str] = frozenset(
    {
        "hi",
        "hello",
        "hey",
        "good morning",
        "good afternoon",
        "good evening",
        "greetings",
        "howdy",
        "sup",
        "what's up",
        "yo",
        "hii",
        "helloo",
        "thanks",
        "thank you",
        "thx",
        "ty",
        "appreciate it",
        "cheers",
        "nice to meet you",
        "happy to be here",
        "happy to have you here",
        "glad to see you",
        "glad to see you here",
        "glad to be here",
        "bye",
        "goodbye",
        "see you later",
        "see you soon",
        "see you tomorrow",
        "see you next week",
        "see you next month",
        "see you next year",
        "see you in the future",
    }
)

SLACK_UNESSENTIAL_WORDS: FrozenSet[str] = frozenset(
    {
        "ok",
        "okay",
        "sure",
        "yeah",
        "yep",
        "yup",
        "nope",
        "nah",
        "lol",
        "haha",
        "hahaha",
        "hehe",
        "lmao",
        "rofl",
        "👍",
        "👎",
        "😊",
        "😄",
        "😀",
        "👍🏻",
        "👌",
        "got it",
        "gotcha",
        "nice",
        "awesome",
        "great",
        "uhm",
        "um",
        "uh",
        "erm",
        "of course",
    }
)

# Discord message / export plaintext: user, role, channel mentions and custom emoji tokens.
_DISCORD_USER_MENTION_RE = re.compile(r"<@!?(\d+)>")
_DISCORD_ROLE_MENTION_RE = re.compile(r"<@&(\d+)>")
_DISCORD_CHANNEL_MENTION_RE = re.compile(r"<#(\d+)>")
_DISCORD_CUSTOM_EMOJI_RE = re.compile(r"<a?:(\w+):\d+>")
_DISCORD_COLLAPSE_WHITESPACE_RE = re.compile(r"\s+")


def clean_discord_text(
    text: str,
    *,
    greeting_words: Optional[Iterable[str]] = None,
    unessential_words: Optional[Iterable[str]] = None,
    min_words_after: int = 0,
) -> str:
    """
    Strip Discord markup, then greeting / unessential phrases (``SLACK_*`` lists).

    User mentions ``<@123>`` / ``<@!123>``, roles ``<@&id>``, channels ``<#id>``
    are removed. Custom emoji ``<:name:id>`` and animated ``<a:name:id>`` become
    ``:name:``. Whitespace is collapsed to single spaces, then :func:`filter_sentence`
    removes filler phrases (same defaults as Slack). Output is **lowercased**
    because ``filter_sentence`` lowercases for matching.

    Args:
        text: Raw Discord message content.
        greeting_words: Optional override for ``filter_sentence`` (default:
            ``SLACK_GREETING_WORDS``).
        unessential_words: Optional override for ``filter_sentence`` (default:
            ``SLACK_UNESSENTIAL_WORDS``).
        min_words_after: Passed to ``filter_sentence`` (default ``0`` so short
            messages are not blanked by word-count rules after phrase removal).

    Returns:
        Plaintext suitable for search / embedding pipelines.
    """
    if not text:
        return ""
    text = _DISCORD_USER_MENTION_RE.sub("", text)
    text = _DISCORD_ROLE_MENTION_RE.sub("", text)
    text = _DISCORD_CHANNEL_MENTION_RE.sub("", text)
    text = _DISCORD_CUSTOM_EMOJI_RE.sub(r":\1:", text)
    text = _DISCORD_COLLAPSE_WHITESPACE_RE.sub(" ", text).strip()
    return filter_sentence(
        text,
        greeting_words=greeting_words,
        unessential_words=unessential_words,
        min_words_after=min_words_after,
    )


def clean_text(text: str | None, remove_extra_spaces: bool = True) -> str:
    """
    Clean and normalize text content.

    Removes invisible characters, decodes HTML character references (e.g.
    ``&amp;``, ``&#39;``, ``&#x2f;``), fixes a few common bare entities without
    ``;``, normalizes line breaks, and optionally removes extra whitespace.

    Args:
        text: Input text to clean
        remove_extra_spaces: Whether to remove extra whitespace

    Returns:
        Cleaned text

    Examples:
        >>> clean_text("  Hello   world  ")
        'Hello world'
        >>> clean_text("Text\\n\\n\\nMore text")
        'Text\\n\\nMore text'
    """
    if not text:
        return ""

    # Remove soft hyphens and other invisible characters
    text = (
        text.replace("\xad", "")
        .replace("\u200b", "")
        .replace("\u200c", "")
        .replace("\u200d", "")
        .replace("\xa0", " ")
        .replace("\u2002", " ")
        .replace("\u2003", " ")
        .replace("\u2026", "...")
        .replace("\u202f", " ")
    )

    text = html.unescape(text)

    # Normalize line breaks
    text = re.sub(r"\r\n", "\n", text)  # Windows line breaks
    text = re.sub(r"\r", "\n", text)  # Old Mac line breaks

    if remove_extra_spaces:
        text = re.sub(r" +", " ", text)
        text = re.sub(r"\n{3,}", "\n\n", text)
        text = "\n".join(line.strip() for line in text.split("\n"))

    return text.strip()


def filter_sentence(
    sentence: str,
    greeting_words: Optional[Iterable[str]] = None,
    unessential_words: Optional[Iterable[str]] = None,
    min_words_after: int = 3,
) -> str:
    """
    Filter a single sentence by removing greeting/unessential words.

    Removes phrases from greeting_words and unessential_words (case-insensitive),
    then returns the stripped sentence, or "" if too few words remain.

    Args:
        sentence: Input sentence to filter.
        greeting_words: Phrases to remove (e.g. "hi", "thank you"). Default: SLACK_GREETING_WORDS.
        unessential_words: Phrases to remove (e.g. "ok", "lol"). Default: SLACK_UNESSENTIAL_WORDS.
        min_words_after: Minimum word count to keep (inclusive); return "" if fewer. Default: 3.

    Returns:
        Filtered sentence (lowercased, stripped), or "" if empty or fewer than min_words_after words.

    Examples:
        >>> filter_sentence("Hi there, can you help?")
        'there, can you help?'
        >>> filter_sentence("ok sure")
        ''
    """
    sentence = sentence.strip()
    if not sentence:
        return ""

    greeting = (
        {word.lower() for word in greeting_words}
        if greeting_words is not None
        else SLACK_GREETING_WORDS
    )
    unessential = (
        {word.lower() for word in unessential_words}
        if unessential_words is not None
        else SLACK_UNESSENTIAL_WORDS
    )

    sentence_lower = sentence.lower()
    removable_phrases = sorted(greeting | unessential, key=len, reverse=True)
    for phrase in removable_phrases:
        pattern = rf"(?<!\w){re.escape(phrase)}(?!\w)"
        sentence_lower = re.sub(pattern, "", sentence_lower)

    sentence_lower = re.sub(r"\s{2,}", " ", sentence_lower).strip()

    if len(sentence_lower.strip().split()) < min_words_after:
        return ""

    return sentence_lower.strip()


def truncate_content(content: str, max_length: int = 100) -> str:
    """Return ``content`` truncated to ``max_length`` characters with ``...`` when longer."""
    if max_length < 0:
        raise ValueError("max_length must be non-negative")
    if len(content) <= max_length:
        return content
    if max_length <= 3:
        return content[:max_length]
    return content[: max_length - 3] + "..."


def validate_content_length(content: str | None, min_length: int = 50) -> bool:
    """
    Validate that content meets minimum length requirement.

    Args:
        content: Content string to validate
        min_length: Minimum required length (default: 50)

    Returns:
        True if content is valid, False otherwise

    Examples:
        >>> validate_content_length("This is a short text")
        False
        >>> validate_content_length("This is a much longer text that exceeds the minimum length requirement")
        True
    """
    if not content:
        return False

    cleaned = content.strip()
    return len(cleaned) >= min_length