Ë
    þ',j�  ã                  ó  — d Z ddlmZ ddlZddlZddlmZmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ej>                  e «      Z!e G d„ d«      «       Z" G d„ d«      Z#y)z4Pipeline orchestrator: wires all modules end-to-end.é    )ÚannotationsN)Ú	dataclassÚfield)ÚdatetimeÚ	timedeltaÚtimezone)ÚPath)ÚSettings)ÚDgredClient)Ú
LeadWriter)Úextract)Ú
PostSource)Úmatch)ÚCatalogIndex)ÚPost)Ú
map_statusc                  ó”   — e Zd ZU dZded<   dZded<   dZded<   dZded<   dZded<   dZ	ded<   dZ
ded	<   dZded
<   dZded<   dd„Zy)ÚPipelineStatsr   ÚintÚtotal_postsÚskipped_too_oldÚskipped_prefilterÚskipped_not_requestÚmatchedÚreviewÚno_matchÚerrorsÚ
duplicatesc                óð   — d| j                   › d| j                  › d| j                  › d| j                  › d| j                  › d| j
                  › d| j                  › d| j                  › d	| j                  › �S )
NzPosts: z | Too old: z | Skipped (filter): z | Skipped (not request): z | Duplicates: z | Dopasowano: z | Do weryfikacji: z | Brak dopasowania: z | Errors: )	r   r   r   r   r   r   r   r   r   )Úselfs    úE/var/www/html/agent.samochodowy/src/agent_samochodowy/orchestrator.pyÚsummaryzPipelineStats.summary$   s•   € à�d×&Ñ&Ð'ð (Ø×,Ñ,Ð-ð .!Ø!%×!7Ñ!7Ð 8ð 9&Ø&*×&>Ñ&>Ð%?ð @ØŸ?™?Ð+ð ,ØŸ<™<˜.ð )Ø#Ÿ{™{˜mð ,!Ø!%§¡ ð 0Ø—{‘{�mð%ð
	
ó    N)ÚreturnÚstr)Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__r   r   r   r   r   r   r   r   r"   © r#   r!   r   r      sa   … à€K�ÓØ€O�SÓØÐ�sÓØ Ð˜Ó Ø€GˆSÓØ€FˆCƒOØ€HˆcÓØ€FˆCƒOØ€J�Óô
r#   r   c                  óT   — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d	d„Zd
d„Zdd„Zdd„Zdd„Zdd„Z	y)ÚPipelineu<   End-to-end pipeline: ingest â†’ extract â†’ match â†’ write.c                óz   — || _         || _        || _        t        ||«      | _        | j                  «       | _        y )N)ÚsettingsÚsourceÚindexr   ÚwriterÚ_load_seen_idsÚ	_seen_ids)r    r.   r/   r0   Údgreds        r!   Ú__init__zPipeline.__init__5   s7   € ð !ˆŒØˆŒØˆŒ
Ü  ¨Ó1ˆŒØ×,Ñ,Ó.ˆ�r#   c                óê  — t        «       }| j                  «       }| j                  j                  «       D �]  }|xj                  dz  c_        |ra|j
                  j                  d ¬«      |k  rB|xj                  dz  c_        t        j                  d|j                  |j
                  |«       Œ||j                  | j                  v r6|xj                  dz  c_        t        j                  d|j                  «       ŒÊ	 | j                  ||«       | j                  j#                  |j                  «       �Œ | j%                  «        |j                  r5t        j'                  d|j                  | j(                  j*                  «       t        j'                  d|j-                  «       «       |S # t        $ r8 |xj                  dz  c_        t        j!                  d|j                  «       Y Œßw xY w)Né   )Útzinfoz#Post %s too old (%s < %s), skippingzDuplicate post %s, skippingzError processing post %sz4Freshness filter: %d posts rejected (older than %dh)zPipeline complete. %s)r   Ú_freshness_cutoffr/   Úfetch_postsr   Ú	timestampÚreplacer   ÚloggerÚdebugÚpost_idr3   r   Ú_process_postÚ	Exceptionr   Ú	exceptionÚaddÚ_save_seen_idsÚinfor.   Úpost_max_age_hoursr"   )r    ÚstatsÚcutoffÚposts       r!   ÚrunzPipeline.runB   sˆ  € Ü“ˆØ×'Ñ'Ó)ˆà—K‘K×+Ñ+Ó-ó 	-ˆDØ×Ò Ñ"Õñ ˜$Ÿ.™.×0Ñ0¸Ð0Ó=ÀÒFØ×%Ò%¨Ñ*Õ%Ü—‘ÐBØ!Ÿ\™\¨4¯>©>¸6ôCàð �|‰|˜tŸ~™~Ñ-Ø× Ò  AÑ%Õ Ü—‘Ð:¸D¿L¹LÔIØðKØ×"Ñ" 4¨Ô/ð
 �N‰N×Ñ˜tŸ|™|Ö,ð-	-ð0 	×ÑÔØ× Ò Ü�K‰KÐNØ×-Ñ-¨t¯}©}×/OÑ/OôQä�‰Ð+¨U¯]©]«_Ô=Øˆøô ò KØ—’ Ñ!•Ü× Ñ Ð!;¸T¿\¹\ÖJðKús   Ä F1Æ1>G2Ç1G2c                ó|   — | j                   j                  }|dk  ryt        j                  «       t	        |¬«      z
  S )zACompute the oldest acceptable post timestamp, or None to disable.r   N)Úhours)r.   rF   r   Únowr   )r    rL   s     r!   r9   zPipeline._freshness_cutoffe   s1   € à—‘×0Ñ0ˆØ�AŠ:ØÜ�|‰|‹~¤	°Ô 6Ñ6Ð6r#   c                óÌ  — t        || j                  «      }|€|xj                  dz  c_        y |j                  s|xj                  dz  c_        y t        || j                  «      }t        || j                  «      }| j                  j                  |||«       |dk(  r|xj                  dz  c_
        y |dk(  r|xj                  dz  c_        y |xj                  dz  c_        y )Nr7   Ú
DopasowanozDo weryfikacji)r   r.   r   Úis_part_requestr   r   r0   r   r1   Ú
write_leadr   r   r   )r    rI   rG   ÚqueryÚresultÚstatuss         r!   r@   zPipeline._process_postl   sÁ   € ä˜˜dŸm™mÓ,ˆØˆ=Ø×#Ò# qÑ(Õ#Øà×$Ò$Ø×%Ò%¨Ñ*Õ%Øô �u˜dŸj™jÓ)ˆô ˜F D§M¡MÓ2ˆð 	�‰×Ñ˜t U¨FÔ3ð �\Ò!Ø�MŠM˜QÑŽMØÐ'Ò'Ø�LŠL˜AÑŽLà�NŠN˜aÑŽNr#   c                óB  — t        | j                  j                  «      j                  dz  }|j	                  «       r-	 t        t        j                  |j                  «       «      «      S t        «       S # t        $ r! t        j                  d«       Y t        «       S w xY w)z-Load previously processed post IDs from disk.úseen_post_ids.jsonz'Could not load seen IDs, starting fresh)r	   r.   Údb_pathÚparentÚexistsÚsetÚjsonÚloadsÚ	read_textrA   r=   Úwarning©r    Úpaths     r!   r2   zPipeline._load_seen_idsˆ   s|   € ä�D—M‘M×)Ñ)Ó*×1Ñ1Ð4HÑHˆØ�;‰;Œ=ðJÜœ4Ÿ:™: d§n¡nÓ&6Ó7Ó8Ð8ô ‹uˆøô ò JÜ—‘ÐHÕIÜ‹uˆðJús   ¾+A4 Á4BÂBc                ó  — t        | j                  j                  «      j                  dz  }|j                  j	                  dd¬«       |j                  t        j                  t        | j                  «      «      «       y)z#Persist processed post IDs to disk.rV   T)ÚparentsÚexist_okN)
r	   r.   rW   rX   ÚmkdirÚ
write_textr[   ÚdumpsÚsortedr3   r_   s     r!   rD   zPipeline._save_seen_ids’   sW   € ä�D—M‘M×)Ñ)Ó*×1Ñ1Ð4HÑHˆØ�‰×Ñ $°ÐÔ6Ø�‰œŸ
™
¤6¨$¯.©.Ó#9Ó:Õ;r#   N)
r.   r
   r/   r   r0   r   r4   r   r$   ÚNone)r$   r   )r$   zdatetime | None)rI   r   rG   r   r$   rh   )r$   zset[str])r$   rh   )
r&   r'   r(   Ú__doc__r5   rJ   r9   r@   r2   rD   r*   r#   r!   r,   r,   2   sT   „ ÙFð/àð/ð ð/ð ð	/ð
 ð/ð 
ó/ó!óF7ó ó8ô<r#   r,   )$ri   Ú
__future__r   r[   ÚloggingÚdataclassesr   r   r   r   r   Úpathlibr	   Úagent_samochodowy.configr
   Úagent_samochodowy.dgred.clientr   Úagent_samochodowy.dgred.writerr   Úagent_samochodowy.extractionr   Úagent_samochodowy.ingestor.baser   Ú agent_samochodowy.matcher.enginer   Úagent_samochodowy.matcher.indexr   Úagent_samochodowy.modelsr   Úagent_samochodowy.statusr   Ú	getLoggerr&   r=   r   r,   r*   r#   r!   ú<module>rx      so   ðÙ :å "ã Û ß (ß 2Ñ 2Ý å -Ý 6Ý 5Ý 0Ý 6Ý 2Ý 8Ý )Ý /à	ˆ×	Ñ	˜8Ó	$€ð ÷
ð 
ó ð
÷2d<ò d<r#   