+
    @j,                     l    R t ^ RIt^ RIt^ RIt^ RIHtHt ^ RIH	t	 ^ RI
Ht R tR t ! R R]	4      tR# )	aC  
Management command to compute similarity between places using KNN.

Computes three types of similarities:
1. Structural KNN (same city) - based on categorical tags
2. Image KNN (same city) - based on image feature vectors
3. Image KNN (other cities) - based on image feature vectors

Uses multiprocessing for parallel computation.

Usage:
    python manage.py compute_similarities
    python manage.py compute_similarities --method structural
    python manage.py compute_similarities --method image_same_city
    python manage.py compute_similarities --method image_other_city
N)Pool	cpu_count)BaseCommand)
connectionc                2   V w  rr4^ RI HpHpHp . ROp\	        V4      ^8  d   . # V! W84      w  rp\
        P                  ! V	^R7      pV^ 8  p\
        P                  ! V4      ^ ,          p\	        V4      ^8  d   . # \        \
        P                  ! V	P                  ^,          4      4      p. pV Fb  pV! VWVR7      pVV,          R,          pV F>  w  ppVV8X  d   K  VV,          R,          pV! VVR7      pVP                  VVVR34       K@  	  Kd  	  V# )zDProcess structural KNN for a single city. Called by multiprocessing.)build_onehot_matrix
knn_searchdistance_to_scoreaxis)k	city_maskplace_idmax_distance
structural)historictourismamenityshopleisureman_madememorialartwork_type)processing.algorithms.knnr   r   r	   lennpsumwherefloatsqrtshapeappend)argscity_id	city_namerecordsr   r   r   r	   cat_colsmatrixfeature_names_row_sums
valid_maskvalid_indicesmax_distresultsi	neighborsr   neighbor_idxdistneighbor_place_idscores   &                       f/Users/jokubas/Desktop/KTU/big_data/engineering/processing/management/commands/compute_similarities.py_process_structural_cityr7      s   %)"G\\lH
7|a	27EF1 vvf1%HAJHHZ(+M
=A	 RWWV\\!_-.HGq&D	1:j)"+L$q  ' 5j A%dBENNH&7MN #,	  N    c                   V w  rr4rVp^ RI Hp W!,          p	WA,          p
W1,          pVR8X  d*   \        P                  ! V Uu. uF  qV
8H  NK	  	  up4      pM(\        P                  ! V Uu. uF  qV
8g  NK	  	  up4      pRW&   \        P                  ! V4      ^ ,          p\        V4      ^ 8X  d   . # W.,          pW,
          p\        P                  ! \        P                  ! V^,          ^R7      4      p\        V\        V4      4      p\        P                  ! VV^,
          4      RV pV\        P                  ! VV,          4      ,          p\        \        P                  ! VP                  ^,          4      4      p. pV F\  p\        VV,          4      pVV,          pVe   VV8X  d   K+  \        VV,          4      pV! VVR7      pVP                  VVVV34       K^  	  V# u upi u upi )z@Process image KNN for a single image. Called by multiprocessing.)r	   image_same_cityFr
   Nr   )r   r	   r   arrayr   r   r    r   minargpartitionargsortr   r!   intr"   )r#   img_idxall_featuresall_place_idsall_city_idstarget_city_idr   methodr	   	query_vec
query_cityquery_placecidr   candidate_indicescandidate_featuresdiff	distancesk_actual	top_localr.   r/   	local_idxorig_idxr4   r3   r5   s   &                          r6   _process_image_knnrR   A   s   TXQG=6;%I&J(K""HH<H<CZ/<HI	HH<H<CZ/<HI	 I+A.
"	%8)Dtqyq12I1c+,-H	8a<8(CI"**Yy%9:;IRWW\//234HG	(34)(3$(9[(HYy)*!$X>%6vFG  N? IHs   G$*G)c                   B   a  ] tR t^mt o RtR tR tR tR tR t	Rt
V tR# )	CommandzDCompute place similarities using KNN algorithms with multiprocessingc                    VP                  R \        R. RORR7       VP                  R\        ^
RR7       VP                  R\        ^RR7       VP                  R	R
RR7       R# )z--methodallz"Which similarity method to compute)typedefaultchoiceshelpz--k-structuralzNumber of structural neighbors)rW   rX   rZ   z	--k-imagezNumber of image neighborsz--clear
store_truez/Clear existing similarity data before computing)actionrZ   N)rV   r   r:   image_other_city)add_argumentstrr?   )selfparsers   &&r6   add_argumentsCommand.add_argumentsp   s    P5 	 	
 	1	 	 	
 	,	 	 	
 	B 	 	
r8   c                   ^ RI HpHpHpHp VR,          pVR,          pVR,          p	VR,          '       d   V P
                  P                  R4       VR8X  d*   VP                  P                  4       P                  4        M*VP                  P                  VR7      P                  4        V P
                  P                  R	4       \        ^\        4       4      p
V P
                  P                  R
V
 R24       VR9   d-   V P
                  P                  R4       V P                  W4       VR9   d.   V P
                  P                  R4       V P                  RW4       VR9   d.   V P
                  P                  R4       V P                  RW4       V P
                  P                  V P                  P!                  R4      4       R# )    )CityPlaceSimilarPlaceImageFeaturerE   k_structuralk_imageclearz$Clearing existing similarity data...rV   )rE   zCleared.zUsing z worker process(es)z!
=== Computing Structural KNN ===r:   z(
=== Computing Image KNN (same city) ===r]   z+
=== Computing Image KNN (other cities) ===z!
Similarity computation complete!N)rV   r   )rV   r:   )rV   r]   )places.modelsrf   rg   rh   ri   stdoutwriteobjectsrV   deletefiltermaxr   _compute_structural_compute_image_knnstyleSUCCESS)r`   r#   optionsrf   rg   rh   ri   rE   k_structk_img	n_workerss   &*,        r6   handleCommand.handle   sh   II">*	"7KKDE$$((*113$$++6+:AACKKj)9;'	F9+-@AB**KKBC$$X9//KKIJ##$5uH00KKLM##$6I$**,,-QRSr8   c                z   ^ RI HpHpHp \	        VP
                  P                  4       4      p. pV EF  p\	        VP
                  P                  VR7      P                  RRRRRRR	R
R4	      4      p	. p
V	 Fg  pRVR,          RVR,          RVR,          RVR,          RVR,          RVR,          R	VR	,          R
VR
,          RVR,          /	pV
P                  V4       Ki  	  VP                  VP                  VP                  W34       V P                  P                  RVP                   R\        V
4       R24       EK  	  \        P                   ! 4        V P                  P                  RV R24       \#        VR7      ;_uu_ 4       pVP%                  \&        V4      pRRR4       . pX F  pVP)                  V4       K  	  V P                  P                  R\        V4       R24       V P+                  VR4       V P                  P                  R\        V4       R24       R#   + '       g   i     L; i)re   )rf   rg   rh   )cityidr   r   r   r   r   r   r   r   r   z	  Queued z (z places)zRunning structural KNN with z workers...	processesNzSaving z! structural similarity records...r     Saved z structural similarities)rm   rf   rg   rh   listrp   rV   rr   valuesr"   r   namern   ro   r   r   closer   mapr7   extend_save_similarities)r`   r   r{   rf   rg   rh   cities	city_argsr   placesr&   precpoolresults_listall_resultsr/   s   &&&              r6   rt   Command._compute_structural   s   ;;dll&&()	D%--..D.9@@j)Y:z> F G$*q|q|AfIq|**"An$5
 s#  dggtyy'=>KK	$))Bs7|nHMN+ 0 	8;OPI&&$88$<iHL ' #Gw' $ 	GC$4#55VWX\:HS%5$66NOP '&s   H**H:	c           
     6   ^ RI HpHp VP                  P	                  RR7      P                  RR4      p\        V4      pV'       g   V P                  P                  R4       R# V P                  P                  R\        V4       R	24       . p. p	V Fj  p
 \        P                  ! \        P                  ! V
P                  4      \        P                  R
7      pVP!                  V4       V	P!                  V
4       Kl  	  V'       g   V P                  P                  R4       R# \        V4      p\        V^ ,          4      p\        P$                  ! W3\        P                  R
7      p\'        V4       F  w  r\        V4      V8X  g   K  WV&   K  	  V	 Uu. uF  pVP(                  NK  	  ppV	 Uu. uF  pVP*                  NK  	  pp\-        V4       Uu. uF  pWVVVV,          W!3NK  	  ppV P                  P                  RV RV R24       \.        P0                  ! 4        \3        VR7      ;_uu_ 4       pVP5                  \6        V4      pRRR4       . pX F  pVP9                  V4       K  	  / pV F:  w  ppppVe   Ve   VV8X  d   K  VVV3pVV9  g   VV,          V8  g   K5  VVV&   K<  	  VP;                  4        UUu. uF!  w  ppV^ ,          V^,          VV^,          3NK#  	  pppV P                  P                  R\        V4       RV R24       V P=                  VV4       V P                  P                  R\        V4       RV R24       R#   \"         d     EK  i ; iu upi u upi u upi   + '       g   i     EL8; iu uppi )re   )rh   ri   F)place__isnullr   placez/  No image features found. Run load_data first.Nz	  Loaded z image features)dtypez!  No valid feature vectors found.z  Computing z	 KNN for z
 images...r   z	  Saving  z similarity records...r   z similarities)rm   rh   ri   rp   rr   select_relatedr   rn   ro   r   r   r;   jsonloadsfeature_vectorfloat32r"   	Exceptionzeros	enumerater   r$   ranger   r   r   r   rR   r   itemsr   ) r`   rE   r   r{   rh   ri   features_qsfeatures_listparsed_featuresvalid_featuresfeatvecndrA   r0   frB   rC   	args_listr   r   r   r/   dedupedmain_idsim_idr5   mkeyvfinal_resultss    &&&&                            r6   ru   Command._compute_image_knn   s7   < #**11 2 

.
) 	 [)KKOPIc-&8%9IJ !Dhhtzz$*=*=>bjjQ&&s+%%d+	 " KKAB "#xxbjj90FA3x1}"%Q 1 .<<^^<+9:>a		>:
 1X
 m\<?AV 	 

 	L	!JGH 	I&&$88$6	BL ' #Gw' $ )4%GVUA&.Gv4EFA&C'!WS\E%9$ *5 <C==?K?41a!A$!a1.?KIc-&8%96(BXYZv6HS%7$8&OP_   =:
 '&&  Ls7   A&MM23M7M<;N'NM/.M/N	c           
     8   ^ RI HpHp V'       g   R# \        VP                  P                  RRR7      4      p. p\        4       pV FT  w  rrW9  g   W9  d   K  W8X  d   K  WV3pW9   d   K(  VP                  V4       VP                  V! VV	V
VR7      4       KV  	  Rp^ p\        ^ \        V4      V4       F<  pWoW,            pVP                  P                  VRR7       V\        V4      ,          pK>  	  V P                  P                  R	V R
V 24       R# )re   )rh   rg   Nr   T)flat)main_place_idsimilar_place_idr5   rE   i  )ignore_conflictsz  Bulk created z records for method=)rm   rh   rg   setrp   values_listaddr"   r   r   bulk_creatern   ro   )r`   r/   rE   rh   rg   rB   	to_createseenr   r   r5   r   r   
batch_sizecreatedr0   batchs   &&&              r6   r   Command._save_similarities   s   5 EMM55d5FG	u)0%GU+v/J A&C{HHSM\%!'	  *1" 
q#i.*5A/E  ,,UT,Js5z!G 6
 	OG94HQRr8    N)__name__
__module____qualname____firstlineno__rZ   rb   r|   rt   ru   r   __static_attributes____classdictcell__)__classdict__s   @r6   rT   rT   m   s1     QD
4T@+QZGQR#S #Sr8   rT   )__doc__osr   numpyr   multiprocessingr   r   django.core.management.baser   	django.dbr   r7   rR   rT   r   r8   r6   <module>r      s:     
   + 3  
#L)XVSk VSr8   