"""OE number normalization, category synonyms, and offer deduplication.""" from __future__ import annotations import re from agent_samochodowy.models import Offer # --------------------------------------------------------------------------- # Category synonym groups — any member matches any other member. # Keys are canonical lowercased tokens; values are sets of synonyms. # Both PL and EN forms are included so that query categories (from LLM, # often descriptive Polish) can match offer categories/titles (often EN). # --------------------------------------------------------------------------- _CATEGORY_SYNONYM_GROUPS: list[set[str]] = [ {"sprężarka", "sprężarka powietrza", "sprezarka", "compressor", "kompresor", "luftkompressor", "air compressor"}, {"hamulec górski", "hamulec silnikowy", "engine brake", "brake valve", "motorbremsventil"}, {"stabilizator", "łącznik stabilizatora", "cukierek", "tuleja stabilizatora", "buchse stabilizer"}, {"licznik", "zegary", "licznik zegary", "licznik / zegary", "instrument cluster", "kombiinstrument"}, {"lampa", "lampa tylna", "lampa przednia", "reflektor", "rear lamp", "rearlight", "rückleuchte"}, {"drążek", "drążek reakcyjny", "drążek kierowniczy", "reaction rod", "steering rod", "achsstrebe", "lenkstange"}, {"zawór", "valve", "ventil", "steuerventil", "control valve"}, {"siłownik", "siłownik sprzęgła", "siłownik hamulcowy", "siłownik podnoszenia kabiny", "cylinder", "clutch servo", "kupplungsservo", "spring brake cylinder", "cabin tilt cylinder", "kippzylinder"}, {"zacisk", "zacisk hamulcowy", "brake caliper", "bremssattel"}, {"uszczelka", "gasket", "uszczelka pokrywy rozrządu", "head cover gasket"}, # Pumps — split by type (pompa wody ≠ pompa paliwa ≠ pompa adblue) {"pompa wody", "water pump", "wasserpumpe"}, {"pompa paliwa", "fuel pump", "kraftstoffpumpe"}, {"pompa adblue", "adblue pump"}, {"pompa wtryskowa", "injection pump", "einspritzpumpe", "einspritzdüse"}, {"pompa", "pump"}, # generic fallback # Filters — split by type (filtr paliwa ≠ filtr powietrza ≠ filtr oleju) {"filtr paliwa", "fuel filter", "kraftstofffilter"}, {"filtr powietrza", "air filter", "luftfilter"}, {"filtr oleju", "oil filter", "ölfilter"}, {"filtr", "filter"}, # generic fallback {"turbo", "turbosprężarka", "turbina", "turbinka", "turbolader", "turbocharger"}, {"sensor", "czujnik", "sensor ciśnienia", "sensor ciśnienia doładowania", "pressure sensor", "drucksensor"}, {"klocki hamulcowe", "klocki", "brake pad", "bremsbelag", "bremsbeläge"}, {"alternator", "lichtmaschine", "generator"}, {"rozrusznik", "starter", "anlasser"}, {"sprzęgło", "clutch", "kupplung"}, {"chłodnica", "radiator", "kühler"}, {"łożysko", "bearing", "radlager"}, {"przełącznik", "switch", "schalter"}, {"sworzeń", "pin", "federbolzen", "spring pin"}, {"śruba", "szpilka", "bolt", "schraube"}, # Doors — split by component (zawias ≠ przedłużenie ≠ klamka) {"drzwi", "door", "tür"}, # generic / whole door panel {"zawias drzwi", "door hinge", "scharnier", "türscharnier"}, {"klamka drzwi", "door handle", "türgriff"}, {"szyba drzwi", "szyba boczna", "door glass", "türscheibe", "glasscheibe"}, {"lusterko", "lustro", "mirror", "spiegelglas"}, {"tłumik", "silencer", "schalldämpfer", "auspuff", "exhaust"}, # Air dryer — commonly called "osuszacz" in PL, stored as "zawór" in catalog {"osuszacz", "osuszacz powietrza", "air dryer", "lufttrockner", "air dryer valve", "zawór osuszacza", "podstawa osuszacza"}, # Catalytic converter / DPF {"katalizator", "dpf", "katalysator", "catalytic converter", "partikelfilter", "filtr cząstek"}, # Windshield / glass {"szyba", "szyba przednia", "szyba czołowa", "windscreen", "windshield", "frontscheibe"}, # Hose / pipe (cooling, steering, etc.) {"przewód", "wąż", "hose", "pipe", "schlauch", "leitung"}, # Pulley {"koło pasowe", "pasek napędowy", "pulley", "riemenscheibe"}, # Fuel injector / nozzle {"wtryskiwacz", "wtrysk", "injector", "nozzle", "einspritzdüse", "injektor"}, # Steering {"wspomaganie", "pompa wspomagania", "power steering", "servolenkung", "steering pump"}, # Separator / fuel separator {"separator", "separator paliwa", "fuel separator"}, # Hub (wheel hub) {"piasta", "piasta koła", "hub", "wheel hub", "radnabe"}, # Air cushion / suspension bag {"poduszka", "poduszka powietrzna", "miech", "air spring", "luftfeder", "air bag suspension"}, ] # Pre-built lookup: lowered token → set of all synonyms (also lowered) _SYNONYM_LOOKUP: dict[str, set[str]] = {} for _group in _CATEGORY_SYNONYM_GROUPS: _lowered = {s.lower() for s in _group} for _member in _lowered: _SYNONYM_LOOKUP[_member] = _lowered def expand_category_synonyms(category: str) -> list[str]: """Return all synonym forms for a category string (including itself). Tries longest-match first: "sprężarka powietrza" before "sprężarka". Single-word keys only match multi-word queries if they are the leading word, preventing "drzwi" from matching inside "przedłużenie drzwi". """ cat_lower = category.lower().strip() if cat_lower in _SYNONYM_LOOKUP: return sorted(_SYNONYM_LOOKUP[cat_lower]) # Try substring match: longest key first for key in sorted(_SYNONYM_LOOKUP, key=len, reverse=True): if key in cat_lower: # Guard: single-word key inside multi-word query must be the leading word # "drzwi" must NOT match "przedłużenie drzwi" (drzwi is modifier, not head) if len(key.split()) == 1 and len(cat_lower.split()) > 1: if not cat_lower.startswith(key): continue return sorted(_SYNONYM_LOOKUP[key]) if cat_lower in key: return sorted(_SYNONYM_LOOKUP[key]) return [cat_lower] # Tokens that look like OE numbers but are actually model identifiers MODEL_TOKENS = frozenset({ "FH12", "FH13", "FH16", "FM12", "FM13", "F2000", "F90", "TGA", "TGX", "TGS", "TGL", "TGM", "DAF95", "DAF105", "XF95", "XF105", "XF106", "ACTROS", "ATEGO", "AXOR", "R500", "R580", "G400", }) # Patterns for values commonly mis-extracted as OE numbers _VIN_RE = re.compile(r"^[A-HJ-NPR-Z0-9]{17}$") # ISO 3779 VIN: 17 alphanum, no I/O/Q _POSTAL_DE_RE = re.compile(r"^DE\d{4,5}$") # German postal code with country prefix def normalize_oe(raw: str) -> str: """Normalize OE/catalog number: strip whitespace, dots, dashes, slashes; uppercase.""" return re.sub(r"[\s.\-/]", "", raw).upper() def is_plausible_oe(normalized: str) -> bool: """Reject values that are clearly not OE numbers (VINs, postal codes, etc.).""" if len(normalized) < 4: return False if _VIN_RE.match(normalized): return False if _POSTAL_DE_RE.match(normalized): return False return True def is_model_token(token: str) -> bool: """Check if a normalized token is a known model identifier, not an OE number.""" return token in MODEL_TOKENS def deduplicate_offers(offers: list[Offer]) -> list[Offer]: """Deduplicate offers by normalized OE set. Same OE across platforms = one logical match.""" seen_oe_sets: list[frozenset[str]] = [] result: list[Offer] = [] for offer in offers: if not offer.oe_numbers: # No OE — deduplicate by offer_id if not any(o.offer_id == offer.offer_id for o in result): result.append(offer) continue oe_set = frozenset(offer.oe_numbers) # Check for overlap with already-seen OE sets is_dup = False for seen in seen_oe_sets: if oe_set & seen: # any overlap → duplicate is_dup = True break if not is_dup: seen_oe_sets.append(oe_set) result.append(offer) return result