Ë
    XÖi¤  ã                  ó  — d Z ddlmZ ddlZddlZddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZmZ dd„Z ej*                  d«      Zdd„Zddd„Z	 	 	 d	 	 	 	 	 	 	 dd„Zdd„Zdd„Zedk(  r e«        yy)u2   Orchestrator scrapera leadÃ³w kancelarii prawnych.é    )ÚannotationsN)ÚdatetimeÚtimezone)ÚTypeé   )Úconfig)Ú	CRMClientÚCRMError)ÚDeduplicator)ÚLeadÚScraperSourcec                 ó&  — t         j                  j                  dd¬«       d} t        j                  t         j
                  d¬«      t        j                  t        j                  «      g}t        j                  t        j                  | |¬«       y )NT)ÚparentsÚexist_okz4%(asctime)s | %(name)s | %(levelname)s | %(message)szutf-8)Úencoding)ÚlevelÚformatÚhandlers)r   ÚLOG_DIRÚmkdirÚloggingÚFileHandlerÚLOG_FILEÚStreamHandlerÚsysÚstdoutÚbasicConfigÚINFO)Úfmtr   s     ú5/var/www/html/scraper-prawny.klono.ai/scraper/main.pyÚ_setup_loggingr!      sc   € Ü
‡N�N×Ñ °ÐÔ5Ø
@€Cä×ÑœFŸO™O°gÔ>Ü×ÑœcŸj™jÓ)ð€Hô ×ÑœgŸl™l°3ÀÖJó    Úscraperc                 óN   — ddl m}  ddlm} ddlm} ddlm} ddlm	} | ||||dœS )Nr   )Ú
OraScraper)ÚOirpScraper)ÚPanoramafirmScraper)ÚGoogleMapsScraper)ÚLinkedinScraper)ÚoraÚoirpÚpanoramafirmÚgoogle_mapsÚlinkedin)
Úsources.orar%   Úsources.oirpr&   Úsources.panoramafirmr'   Úsources.google_mapsr(   Úsources.linkedinr)   )r%   r&   r'   r(   r)   s        r    Ú_load_sourcesr4      s,   € Ý'Ý)Ý9Ý6Ý1ð ØØ+Ø(Ø#ñð r"   c           	     óÆ   — |r|j                   nd}|r|j                  ndxs d}|r|j                  ndxs d}t        j	                  d| j                  «       |||||«       y )Nú?Ú z%s | %s | %s | %s | %s %s)ÚnameÚcityÚemailÚlogÚinfoÚupper)ÚsourceÚstatusÚleadÚextrar8   r9   r:   s          r    Ú	_log_leadrB   /   sS   € Ùˆ4�9Š9 #€DÙˆD�IŠI 2Ò,¨"€DÙˆT�ZŠZ RÒ.¨B€EÜ‡H�HÐ(¨&¯,©,«.¸&À$ÈÈeÐUZÕ[r"   c                óÊ	  — t        «        |�|nt        j                  }t        «       }| xs t        j                  }|D �cg c]	  }||v sŒ|‘Œ }}t
        j                  d|||«       t        «       }	 |j                  «       st
        j                  d«       ddiS 	 t        |«      }|s|j                  «        ddddi dœ}	d}
d}|D �]ø  }|r|
|k\  rt
        j                  d«        �nÙt        j                  j                  |d«      }|xs d|z   }|s|s|xs d}|rt!        |xs |||
z
  «      }t
        j                  d	||||«       ||   }	  |d ¬
«      }dx}x}x}}	 |j%                  «       D �]Ù  }|r||k\  r �nÏ|r|
|k\  r �nÅ|j'                  «       s|dz  }t)        |d|d«       Œ<|j+                  |j,                  |j.                  «      r|dz  }t)        |d|«       Œu|r>|dz  }|
dz  }
t)        |d|«       |j1                  |j,                  |j.                  «       Œµ	 |j3                  «       }|j5                  |«      }|j                  d«      xs i j                  d«      xs |j                  d«      }|r�	 |j7                  t9        |«      d|› dt;        j<                  t>        j@                  «      jC                  «       › �|jD                  j                  d«      rd|jD                  j                  d«      › �ndz   «       |dz  }|
dz  }
|j1                  |j,                  |j.                  «       t)        |d|«       �ŒÜ |	dxx   |z  cc<   |	dxx   |z  cc<   |	d xx   |z  cc<   |	dxx   |z  cc<   ||||d!œ|	d"   |<   tK        d||z   |z
  «      }|}t
        j                  d#||||||«       �Œû t
        j                  d$|	«       |	S c c}w # t        $ r-}t
        j                  d|«       dt        |«      icY d }~S d }~ww xY w# t"        $ r/}t
        j                  d||«       |	dxx   dz  cc<   Y d }~�Œ…d }~ww xY w# t        $ r"}t
        jG                  d||«       Y d }~�Œjd }~ww xY w# t        $ r"}|dz  }t)        |d|d|› d�«       Y d }~�Œ5d }~ww xY w# t"        $ r'}t
        jI                  d||«       |dz  }Y d }~�Œ‹d }~ww xY w)%Nz@=== Scraper start | sources=%s | daily_limit=%s | dry_run=%s ===u   CRM ping failed â€” abortingÚerrorÚcrm_ping_failedzCRM unreachable: %sr   )ÚaddedÚ	duplicateÚskippedrD   Ú	by_sourceu    Daily limit reached â€” stoppingz<--- Source: %s | target=%d new leads (base=%d, carry=%d) ---)ÚlimitzFailed to init %s: %sr   ÚSKIPPEDz(brak email/tel)Ú	DUPLICATEzDRY-RUNÚdataÚidu   Å¹rÃ³dÅ‚o: z
 | scrap: Úprofile_urlz | r7   zNote failed for %s: %sÚADDEDÚERRORú(ú)zSource %s crashed: %srF   rG   rH   )rF   rG   rH   rD   rI   u@   --- %s done: +%d added, %d dup, %d skip, %d err | carryâ†’%d ---z=== Summary: %s ===)&r!   r   ÚDAILY_LIMITr4   ÚSOURCE_PRIORITYr;   r<   r	   ÚpingrD   r
   Ústrr   ÚloadÚSOURCE_LIMITSÚgetÚminÚ	ExceptionÚfetchÚis_validrB   Úis_duplicater:   ÚphoneÚrememberÚto_crm_payloadÚadd_leadÚadd_noteÚintr   Únowr   ÚutcÚ	isoformatrA   ÚwarningÚ	exceptionÚmax)ÚsourcesÚdaily_limitÚdry_runÚsrc_classesÚchosenÚsÚcrmÚeÚdedupÚsummaryÚtotal_addedÚcarryÚsource_nameÚ
base_limitÚper_source_targetÚclsr#   Ús_addedÚs_dupÚs_skipÚs_errr@   ÚpayloadÚrespÚ	client_idÚunuseds                             r    Úrunr„   6   sû  € ô
 Ôà!,Ð!8‘+¼f×>PÑ>P€KÜ“/€KØÒ.œ×.Ñ.€FØÖ4�A 1¨Ò#3ŠaÐ4€FÐ4Ü‡H�HÐOØ�[ 'ô+ô ‹+€Cð!Ø�x‰xŒzÜ�I‰IÐ4Ô5ØÐ.Ð/Ð/ð ô ˜Ó€EÙØ�
‰
Œà¨°aÀ!ÐRTÑU€GØ€KØ€Eàó ZDˆÙ˜;¨+Ò5Ü�H‰HÐ7Ô8Úä×)Ñ)×-Ñ-¨k¸1Ó=ˆ
Ø'š_¨1°Ñ5ÐÙ¡%à +Ò 0¨qÐÙÜ #Ð$5Ò$D¸ÀkÐT_ÑF_Ó `Ðä�‰ØJØÐ*¨J¸ô	
ð ˜+Ñ&ˆð	ñ  ”oˆGð ,-Ð,ˆÐ,�%Ð,˜& 5ð.	ØŸ™›ó *D�Ù$¨Ð4EÒ)EÚÙ ;°+Ò#=Úà—}‘}”Ø˜a‘K�FÜ˜k¨9°dÐ<NÔOØà×%Ñ% d§j¡j°$·*±*Ô=Ø˜Q‘J�EÜ˜k¨;¸Ô=ØáØ˜q‘L�GØ 1Ñ$�KÜ˜k¨9°dÔ;Ø—N‘N 4§:¡:¨t¯z©zÔ:ØðDØ"×1Ñ1Ó3�GØŸ<™<¨Ó0�DØ!%§¡¨&Ó!1Ò!7°R× <Ñ <¸TÓ BÒ TÀdÇhÁhÈtÃn�IÙ ðPØŸL™LÜ # I£Ø"-¨k¨]¸*ÄXÇ\Á\ÔRZ×R^ÑR^ÓE_×EiÑEiÓEkÐDlÐ mØLPÏJÉJÏNÉNÐ[hÔLi S¨¯©¯©¸Ó)FÐ(GÑ#HÐoqñ!sôð ˜q‘L�GØ 1Ñ$�KØ—N‘N 4§:¡:¨t¯z©zÔ:Ü˜k¨7°DÖ9ðO*Dð^ 	�Ó˜GÑ#ÓØ�Ó Ñ%ÓØ�	Ó˜fÑ$ÓØ�Ó˜EÑ!Óà¨5¸VÈeñ-
ˆ�Ñ˜[Ñ)ô �Q˜ eÑ+¨wÑ6Ó7ˆØˆÜ�‰ÐSØ˜g u¨f°e¸Uö	DðsZDôx ‡H�HÐ" GÔ,Ø€Nùòe 5øô ò !Ü�	‰	Ð'¨Ô+Øœ˜Q›Ð Õ ûð!ûôD ò 	Ü�I‰IÐ-¨{¸AÔ>Ø�GÓ Ñ!ÓÝûð	ûôR  (ò PÜŸK™KÐ(@À)ÈQ×OÒOûðPûô  ò DØ˜Q‘J�EÜ˜k¨7°D¸A¸a¸SÀ¸(×CÒCûðDûô ò 	Ü�M‰MÐ1°;ÀÔBØ�Q‰JŽEûð	ús±   Á	OÁOÁ4(O" Å	PÅ%CR2È+ARÊBQÌ=RÍR2Ï"	PÏ+"PÐPÐPÐ	QÐ$$QÑQÑ	RÑQ<Ñ6RÑ<RÒRÒ	R/ÒR*Ò$R2Ò*R/Ò/R2Ò2	S"Ò;SÓS"c                 óÎ   — t        j                  d¬«      } | j                  ddd¬«       | j                  dt        d¬	«       | j                  d
dd¬«       | j	                  «       S )Nu#   Scraper leadÃ³w kancelarii prawnych)Údescriptionz--sourceÚappendu8   ogranicz do jednego ÅºrÃ³dÅ‚a (moÅ¼na podaÄ‡ kilka razy))ÚactionÚhelpz--limitznadpisz dzienny limit)Útyper‰   z	--dry-runÚ
store_trueu    nie wrzucaj do CRM, tylko pokaÅ¼)ÚargparseÚArgumentParserÚadd_argumentre   Ú
parse_args)Úps    r    Ú_parse_argsr‘   µ   sX   € Ü×ÑÐ,QÔR€AØ‡N�N�: hÐ5o€NÔpØ‡N�N�9¤3Ð-D€NÔEØ‡N�N�; |Ð:\€NÔ]Ø�<‰<‹>Ðr"   c                 óp   — t        «       } t        | j                  | j                  | j                  ¬«       y )N)rl   rm   rn   )r‘   r„   r>   rJ   rn   )Úargss    r    Úmainr”   ½   s!   € Ü‹=€DÜ�—‘¨¯©¸T¿\¹\ÖJr"   Ú__main__)ÚreturnÚNone)r–   zdict[str, Type[ScraperSource]])r7   )
r>   rW   r?   rW   r@   zLead | NonerA   rW   r–   r—   )NNF)rl   zlist[str] | Nonerm   z
int | Nonern   Úboolr–   Údict)r–   zargparse.Namespace)Ú__doc__Ú
__future__r   rŒ   r   r   r   r   Útypingr   r7   r   rr   r	   r
   Údeduplicatorr   Úsources.baser   r   r!   Ú	getLoggerr;   r4   rB   r„   r‘   r”   Ú__name__© r"   r    ú<module>r¢      s¤   ðÙ 8å "ã Û Û 
ß 'Ý å ß $Ý &ß -óKð €g×Ñ˜	Ó"€óô \ð !%Ø"Øð|Øð|àð|ð ð|ð 
ó	|ó~óKð
 ˆzÒÙ…Fð r"   