
    yf                     Z   S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJr  SSK	J
r
  \
" 5       r	\
" S5      rSSKJrJr  SSKJrJrJr  SS	KJr  SS
KJr  SSKJr  SSKJrJr  SSKJrJrJrJrJr  SSKJr  SSKJr  SSKJ r   SSKJ!r!  SSKJ"r"J#r#  SSKJ$r$J%r%  SSKJ&r&  SSKJ'r'  SSK(J)r)J*r*  SSK+J,r,  SSK-J.r.J/r/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK7J8r8  \" SS5      r9 " S S 5      r:S! r;S" r<S0S# jr=S$ r>S% r? " S& S'5      r@ " S( S)5      rA " S* S+5      rB " S, S-\B5      rC " S. S/\B5      rDg)1    N)
namedtuple)perf_counter   )create_loggerzborg.debug.files_cache)CACHE_READMEFILES_CACHE_MODE_DISABLED)
ChunkIndexChunkIndexEntryCacheSynchronizer)Location)Error)Manifest)get_cache_dirget_security_dir)int_to_bigintbigint_to_int
bin_to_hex
hex_to_binparse_stringified_list)format_file_size)safe_ns)yes)remove_surrogates)ProgressIndicatorPercentProgressIndicatorMessage)set_ecEXIT_WARNING)safe_unlink)msgpack)ArchiveItemChunkListEntry)PlaintextKey)IntegrityCheckedFileDetachedIntegrityCheckedFileFileIntegrityError)Lock)SaveFile)cache_if_remote)LIST_SCAN_LIMITFileCacheEntryzage inode size cmtime chunk_idsc                       \ rS rSrSrS r\SS j5       rS rS r	S r
SS	 jrSS
 jrSS jrSSSS.S jrSS jrS rSrg)SecurityManager*   a;  
Tracks repositories. Ensures that nothing bad happens (repository swaps,
replay attacks, unknown repositories etc.).

This is complicated by the Cache being initially used for this, while
only some commands actually use the Cache, which meant that other commands
did not perform these checks.

Further complications were created by the Cache being a cache, so it
could be legitimately deleted, which is annoying because Borg didn't
recognize repositories after that.

Therefore a second location, the security database (see get_security_dir),
was introduced which stores this information. However, this means that
the code has to deal with a cache existing but no security DB entry,
or inconsistencies between the security DB and the cache which have to
be reconciled, and also with no cache existing but a security DB entry.
c                 ~   Xl         [        UR                  5      U l        [	        U5      U l        [
        R                  R                  U R                  S5      U l        [
        R                  R                  U R                  S5      U l	        [
        R                  R                  U R                  S5      U l
        g )Nzkey-typelocationzmanifest-timestamp)
repositoryr   id_strdir	cache_dirospathjoinkey_type_filelocation_filemanifest_ts_file)selfr0   s     ,/usr/lib/python3/dist-packages/borg/cache.py__init__SecurityManager.__init__>   sw    $#J$5$56":.WW\\$((J?WW\\$((J? "TXX7K L    Nc                     U=(       d    [        U R                  5      n[        R                  R	                  U5      (       a  [
        R                  " U5        gg)z:destroy the security dir for ``repository`` or at ``path``N)r   r1   r4   r5   existsshutilrmtreer0   r5   s     r;   destroySecurityManager.destroyF   s>     :'
(9(9:77>>$MM$  r>   c                 h    [        S U R                  U R                  U R                  4 5       5      $ )Nc              3   `   #    U  H$  n[         R                  R                  U5      v   M&     g 7fN)r4   r5   r@   ).0fs     r;   	<genexpr>(SecurityManager.known.<locals>.<genexpr>N   s(      ][q 77>>!$$[s   ,.)allr7   r8   r9   r:   s    r;   knownSecurityManager.knownM   s9     ]!//1C1CTEZEZ[] ] 	]r>   c                 :   U R                  5       (       d  g [        U R                  5       nUR                  5       nU[	        UR
                  5      :H  sS S S 5        $ ! , (       d  f       g = f! [         a   n[        R                  SU5         S nAg S nAff = f)NFz&Could not read/parse key type file: %s)	rO   openr7   readstrTYPEOSErrorloggerwarning)r:   keyfdtypeexcs        r;   key_matchesSecurityManager.key_matchesQ   sr    zz||	Jd(()Rwwys388}, *))  	JNNCSII	Js4   A0 (A	A0 
A-)A0 -A0 0
B:BBc                 .   [         R                  SU R                  R                  U R                  5        U R                  R
                  R                  5       n[         R                  SU5        [         R                  S[        UR                  5      5        [         R                  SUR                  5        [        U R                  5       nUR                  U5        S S S 5        [        U R                  5       nUR                  [        UR                  5      5        S S S 5        [        U R                  5       nUR                  UR                  5        S S S 5        g ! , (       d  f       N= f! , (       d  f       NY= f! , (       d  f       g = f)Nz#security: saving state for %s to %szsecurity: current location   %szsecurity: key type           %szsecurity: manifest timestamp %s)rW   debugr0   r1   r2   	_locationcanonical_pathrT   rU   	timestampr'   r8   writer7   r9   )r:   manifestrY   current_locationrZ   s        r;   saveSecurityManager.save[   s   :DOO<R<RTXT\T\]??44CCE68HI6CHHF68J8JKd(()RHH%& *d(()RHHS]# *d++,HHX''( -,	 *))),,s$   E$=%E5?F$
E25
F
Fc                     [        U R                  5       nUR                  5       nS S S 5        [        R	                  SW5        U(       aB  UR                  (       a1  X1R                  :w  a"  UR                  n[        R	                  SU5        U R                  R                  R                  5       nU(       a  X5:w  a  SR                  XS5      S-   n[        USSS	S
S9(       d  [        R                  5       e[        R	                  S5        [!        U R                  5       nUR#                  U5        S S S 5        U(       a  UR%                  5         g g g g ! , (       d  f       GN7= f! [
         a&    [        R	                  SU R                  5        S n GNS[         a#  n[        R                  SU5        S n S nAGNzS nAff = f! , (       d  f       N= f)Nz#security: read previous location %rz-security: previous location file %s not found)Could not read previous location file: %sz.security: using previous_location of cache: %rzDWarning: The repository at location {} was previously located at {}
zDo you want to continue? [yN] 	Aborting.Invalid answer, aborting.F BORG_RELOCATED_REPO_ACCESS_IS_OK	false_msginvalid_msgretryenv_var_overridez<security: updating location stored in cache and security dir)rR   r8   rS   rW   r`   FileNotFoundErrorrV   rX   previous_locationr0   ra   rb   formatr   CacheRepositoryAccessAbortedr'   rd   rg   )r:   cache_configrZ   rt   r\   repository_locationmsgs          r;   assert_location_matches'SecurityManager.assert_location_matchesh   sx   		%d(()R$&GGI! *LL>@QR L::?PTrTr?r , > >LLIK\]"oo77FFH!2!IZaa#801C sk?Z"5WY3355LLWX$,,-,- .!!#  "J *) ! 	%LLH$J\J\] $ 	%NNFL $	%$ .-s?   E* EE* #G
E'"E* *,G	G"G  G
Gc                     [        U R                  5       nUR                  5       nS S S 5        [        R	                  SW5        U(       a  [        XSR                  =(       d    S5      n[        R	                  SU5        U(       aM  XQR                  :  a=  [        U[        5      (       a  [        R                  5       e[        R                  5       eg g ! , (       d  f       N= f! [
         a%    [        R	                  SU R                  5        Sn N[         a"  n[        R                  SU5        Sn S nANS nAff = f)Nz$security: read manifest timestamp %rz.security: manifest timestamp file %s not found rj   z4security: determined newest manifest timestamp as %s)rR   r9   rS   rW   r`   rs   rV   rX   maxrc   
isinstancer"   rv   RepositoryIDNotUniqueRepositoryReplay)r:   re   rY   rx   rZ   rc   r\   s          r;   assert_no_manifest_replay)SecurityManager.assert_no_manifest_replay   s    		d++,GGI	 -LL?K I'='='CDIKYW%7%77#|,,1133,,..	 89 -, ! 	LLI4K`K`aI 	NNFLI	s3   C' CC' 
C$ C' ',E 	E D;;E c                    U(       aD  UR                   b7  UR                   [        UR                  5      :w  a  [        R	                  5       eU R                  5       (       a+  U R                  U5      (       d  [        R	                  5       eg g rH   )key_typerT   rU   rv   EncryptionMethodMismatchrO   r]   )r:   rY   rx   s      r;   assert_key_typeSecurityManager.assert_key_type   sj    L11=,BWBW[^_b_g_g[hBh0022::<< 0 0 5 50022 !6<r>   T)rx   warn_if_unencrypted	lock_waitc                X   U R                  XAU5        U(       a  U R                  XU5        OX[        U R                  US9nUR	                  5       (       a  U   U R                  XU5        S S S 5        OU R                  X5        [
        R                  S5        g ! , (       d  f       N$= f)Nr   z/security: repository checks ok, allowing access)assert_access_unknown_assert_secureCacheConfigr0   r@   rW   r`   )r:   re   rY   rx   r   r   s         r;   assert_secureSecurityManager.assert_secure   s     	""#6#F|<&t)LL""$$!''|D "\ ##H2FG	 "\s   B
B)c                     U R                  U5        U R                  X#5        U R                  XU5        U R                  5       (       d'  [        R                  S5        U R                  X5        g g )Nz3security: remembering previously unknown repository)r{   r   r   rO   rW   r`   rg   )r:   re   rY   rx   s       r;   r   SecurityManager._assert_secure   sW    $$\2S/&&xlCzz||LLNOIIh$ r>   c                 J   UR                   (       d  U R                  5       (       d|  SnU(       + =(       d    [        USSSSS9nU(       aD  U(       a  [        R	                  S5        O[        R	                  S5        U R                  X#5        g [        R                  5       eg g )	NziWarning: Attempting to access a previously unknown unencrypted repository!
Do you want to continue? [yN] rk   rl   F*BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OKrn   zIsecurity: remembering unknown unencrypted repository (explicitly allowed)z-security: initializing unencrypted repository)logically_encryptedrO   r   rW   r`   rg   rv   CacheInitAbortedError)r:   r   re   rY   rz   allow_accesss         r;   r   %SecurityManager.assert_access_unknown   s     &&tzz||4C22 \c#7.Z7\L &LL!lmLL!PQ		((1133 0<&r>   )r3   r2   r7   r8   r9   r0   rH   )__name__
__module____qualname____firstlineno____doc__r<   staticmethodrD   rO   r]   rg   r{   r   r   r   r   r   __static_attributes__ r>   r;   r,   r,   *   s^    &M    ]J)$@/*3 <@UYei H%4r>   r,   c                 N    [        U 5      nUR                  XR                  US9  g )Nr   )r,   r   rY   )r0   re   r   sms       r;   r   r      s#    		$BX||yAr>   c                 >   UR                   R                  5       n[        U5      n[        U 5      nUR                  UR                  :X  a  UR                  UR                  :X  a  UR
                  UR
                  :X  a  UR                  UR                  :X  a  UR                  (       as  UR                  (       ab  UR                  R                  S5      (       aB  UR                  R                  S5      (       a"  UR                  SS  UR                  SS  :X  a  U$ U $ )Nz/~/z/./   )	ra   rb   r   protouserhostportr5   
startswith)cache_locationr0   repo_locationrlcls        r;    recanonicalize_relative_locationr      s    ((779M	-	 B	.	!B	xx288277 2rww"''7IbggY[Y`Y`N`GGGGu%%"''*<*<U*C*CPQPRWYW^W^_`_aWbHb r>   c                 x    U=(       d2    [         R                  R                  [        5       U R                  5      $ rH   )r4   r5   r6   r   r1   rC   s     r;   r3   r3      s$    C277<<1B1BCCr>   c                  ^    [         R                  R                  SS5      n U (       a  SU -   $ S$ )NBORG_FILES_CACHE_SUFFIXr~   files.files)r4   environgetsuffixs    r;   files_cache_namer      s)    ZZ^^5r:F &8f3G3r>   c                     [         R                  " U 5       Vs/ s H"  oS:X  d  UR                  S5      (       d  M   UPM$     snS   $ s  snf )Nr   r   r   )r4   listdirr   )r5   fns     r;   discover_files_cache_namer      s:    D)V)27]bmmH>UB)VWXYYVs
   AAc                   X    \ rS rSrSS jrS rS rS rS rS r	S	 r
SS
 jrS rS rSrg)r      Nc                     Xl         [        X5      U l        [        R                  R	                  U R                  S5      U l        S U l        X0l        g )Nconfig)r0   r3   r5   r4   r6   config_pathlockr   )r:   r0   r5   r   s       r;   r<   CacheConfig.__init__   s:    $j/	77<<		8<	"r>   c                 &    U R                  5         U $ rH   )rR   rN   s    r;   	__enter__CacheConfig.__enter__   s    		r>   c                 $    U R                  5         g rH   closer:   exc_typeexc_valexc_tbs       r;   __exit__CacheConfig.__exit__       

r>   c                 T    [         R                  R                  U R                  5      $ rH   )r4   r5   r@   r   rN   s    r;   r@   CacheConfig.exists   s    ww~~d..//r>   c                    U R                  5       (       a   e[        R                  " S S9nUR                  S5        UR	                  SSS5        UR	                  SSU R
                  R                  5        UR	                  SSS5        UR                  S5        UR	                  SSS5        [        U R                  5       nUR                  U5        S S S 5        g ! , (       d  f       g = f)	Ninterpolationcacheversion1r0   re   r~   	integrity)
r@   configparserConfigParseradd_sectionsetr0   r1   r'   r   rd   )r:   r   rZ   s      r;   createCacheConfig.create  s    ;;==  **>7#

7Is+

7L$//*@*@A

7J+;'

;
B/d&&'2LL (''s   C
C,c                     [        [        R                  R                  U R                  S5      SU R                  S9R                  5       U l        U R                  5         g )Nr   T)	exclusivetimeout)r&   r4   r5   r6   r   acquirer   loadrN   s    r;   rR   CacheConfig.open  s>    dii8DRVR`R`aiik			r>   c           
      P   [         R                  " S S9U l        [        U R                  5       nU R                  R                  U5        S S S 5        U R                  U R                  5        U R                  R                  SS5      U l        [        U R                  R                  SS5      5      U l
        U R                  R                  SSS S9U l        U R                  R                  SSS S9U l        [        [        U R                  R                  SSS	S95      5      U l        [        [        U R                  R                  SS
S	S95      5      U l         [#        U R                  R%                  S5      5      U l        U R                  R                  SS5      U R&                  R)                  S5      :w  a  0 U l        [*        R-                  S5        U R                  R                  SSS S9nU(       a  [3        X R4                  5      U l        OS U l        U R                  R                  SSU R4                  R8                  R;                  5       5        g ! , (       d  f       GN= f! [         R.                   a    [*        R1                  S5        0 U l         Nf = f)Nr   r   r0   re   rc   )fallbackr   ignored_featuresr~   mandatory_featuresr   zHCache integrity data not available: old Borg version modified the cache.zTCache integrity: No integrity data found (files, chunks). Cache is from old version.rt   )r   r   _configrR   r   	read_file_check_upgrader   idr   manifest_idrc   r   r   r   r   r   dictitemsr   poprW   rX   NoSectionErrorr`   r   r0   rt   ra   rb   )r:   rZ   rt   s      r;   r   CacheConfig.load  s   #00tD$""#rLL""2& $D,,-,,""7L9%dll&6&6w
&KL))';)N((*t(L #$:4<<;K;KGUgrt;K;u$v w"%&<T\\=M=MgWkvx=M=y&z"{	 !$,,"4"4["ABDN||48J8J:8VV "$ij !LL,,W6ITX,Y%EFWYhYh%iD"%)D""5t7P7P7_7_7ab9 $#( ** 	 LLopDN	 s   I A>I2  
I/20J%$J%c                    U(       Ga[  U R                   R                  SSUR                  5        U R                   R                  SSUR                  5        U R                   R                  SSSR	                  U R
                  5      5        U R                   R                  SSSR	                  U R                  5      5        U R                   R                  S5      (       d  U R                   R                  S5        U R                  R                  5        H!  u  p4U R                   R                  SX45        M#     U R                   R                  SSUR                  5        U(       a0  U R                   R                  SS[        UR                  5      5        [        U R                  5       nU R                   R                  U5        S S S 5        g ! , (       d  f       g = f)	Nr   re   rc   r   ,r   r   r   )r   r   r1   rc   r6   r   r   has_sectionr   r   r   rT   rU   r'   r   rd   )r:   re   rY   fileintegrity_datarZ   s         r;   rg   CacheConfig.save2  sE   LLWj(//BLLWk83E3EFLLW&8#((4CXCX:YZLLW&:CHHTE\E\<]^<<++K88((5(,(<(<(>$  dC )?LL[*hooFLLWj#chh-@d&&'2LLr" (''s   0G
G#c                 b    U R                   b"  U R                   R                  5         S U l         g g rH   )r   releaserN   s    r;   r   CacheConfig.closeB  s(    99 IIDI !r>   c                      U R                   R                  SS5      nSnX#:w  a   U R                  5         [        SXU4-  5      eg ! [        R
                   a     U R                  5         [        SU-  5      S ef = f)Nr   r   r   z0%s has unexpected cache version %d (wanted: %d).z#%s does not look like a Borg cache.)r   getintr   	Exceptionr   r   )r:   r   cache_versionwanted_versions       r;   r   CacheConfig._check_upgradeG  s    		[ LL//CMN.

 R!,^ L!M N N / ** 	[JJLAKOPVZZ	[s   AA 4A:)r   r   r   r   r   r   r   r   r   r   r5   rt   r0   rc   )NN)r   r   r   r   r<   r   r   r@   r   rR   r   rg   r   r   r   r   r>   r;   r   r      s7    #0
c@# 

[r>   r   c            	           \ rS rSrSr " S S\5      r " S S\5      r " S S\5      r " S	 S
\5      r	 " S S\5      r
\SS j5       r\SS j5       rSSSSSS\SS4	S jrSrg)rv   iT  zClient Side cache
    c                       \ rS rSrSrSrSrg)Cache.CacheInitAbortedErroriW  zCache initialization aborted<   r   Nr   r   r   r   r   
exit_mcoder   r   r>   r;   r   r  W  s
    *
r>   r   c                       \ rS rSrSrSrSrg)Cache.EncryptionMethodMismatchi[  zLRepository encryption method changed since last access, refusing to continue=   r   Nr	  r   r>   r;   r   r  [  s
    Z
r>   r   c                       \ rS rSrSrSrSrg)Cache.RepositoryAccessAbortedi_  zRepository access aborted>   r   Nr	  r   r>   r;   rw   r  _  s
    '
r>   rw   c                       \ rS rSrSrSrSrg)Cache.RepositoryIDNotUniqueic  z`Cache is newer than repository - do you have multiple, independently updated repos with same ID??   r   Nr	  r   r>   r;   r   r  c  s
    n
r>   r   c                       \ rS rSrSrSrSrg)Cache.RepositoryReplayig  zCache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID)@   r   Nr	  r   r>   r;   r   r  g  s     	e
r>   r   Nc                     [        X5      n[        [        R                  R	                  US5      SS9R                  5         g )Nr   T)r   )r3   r&   r4   r5   r6   
break_lockrC   s     r;   r  Cache.break_lockk  s.    *RWW\\$'48CCEr>   c                 \   U=(       d2    [         R                  R                  [        5       U R                  5      n[         R                  R                  US5      n[         R                  R                  U5      (       a-  [         R                  " U5        [        R                  " U5        gg)z3destroy the cache for ``repository`` or at ``path``r   N)	r4   r5   r6   r   r1   r@   removerA   rB   )r0   r5   r   s      r;   rD   Cache.destroyp  si     Grww||MOZ5F5FGdH-77>>&!!IIfMM$ "r>   TFc                   ^^^^^^^^^
^^ U
UUUUUUUUUU4S jnUUUUUU4S jnU	(       d  U" 5       $ [        TTT5      nUR                  5       (       aG  U   UR                  TR                  :H  nS S S 5        W(       a  [        R                  S5        U" 5       $ [        R                  S5        U" 5       $ ! , (       d  f       NM= f)Nc                  *   > [        TTTTT	T
TTTT TS9$ )N)r0   rY   re   r5   syncr   progressiecr   
cache_modeconsider_part_files)
LocalCache)r"  r#  r!  rY   r   re   r5   r   r0   r  r   s   r;   localCache.__new__.<locals>.local}  s/    xVZae2EPX^a(1j^qs sr>   c            	          > [        TTTTTT S9$ )N)r0   rY   re   r   r!  r#  )
AdHocCache)r#  r!  rY   r   re   r0   s   r;   adhocCache.__new__.<locals>.adhoc  s!    x[djm2EG Gr>   z%Cache: choosing local cache (in sync)zKCache: choosing ad-hoc cache (local cache does not exist or is not in sync))r   r@   r   r   rW   r`   )clsr0   rY   re   r5   r  r   r   r   permit_adhoc_cacher"  r#  r!  r%  r)  rx   cache_in_syncs    ```````` ```    r;   __new__Cache.__new__y  s    	s 	s
	G 	G "7N #:tY?   , 8 8HKK G  DEwbcw s   B::
Cr   rH   )r   r   r   r   r   r   r   r   rw   r   r   r   r  rD   r   r.  r   r   r>   r;   rv   rv   T  s     5 %  5  F F     6:Z^$5Un$)ur>   rv   c                   P    \ rS rSrSrSS jrS r\" S/ SQ5      rS r	S r
S	 rS
rg)CacheStatsMixini  zAll archives:   {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s}

                       Unique chunks         Total chunks
Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}c                     Xl         0 U l        g rH   r!  
pre12_meta)r:   r!  s     r;   r<   CacheStatsMixin.__init__  s    r>   c                 T    U R                   R                  U R                  5       5      $ rH   )
str_formatru   format_tuplerN   s    r;   __str__CacheStatsMixin.__str__  s     %%d&7&7&9::r>   Summary)
total_sizetotal_csizeunique_sizeunique_csizetotal_unique_chunkstotal_chunksc           	         SSK Jn  U R                  R                  5       u  p#pEpgSu  p#U R                  R
                   Ha  nU" U R                  U R                  U R                  UU R                  S9n	U	R                  U SS9n
X*R                  -  nX:R                  -  nMc     U R                  X#XEXg5      R                  5       n
U
$ )Nr   )Archive)r   r   )r#  F)want_unique)archiverC  chunks	summarizere   archivesr0   rY   r#  
calc_statsosizecsizer;  _asdict)r:   rC  r<  r=  r>  r?  r@  rA  archive_namerE  statss              r;   rN  CacheStatsMixin.stats  s    $`d`k`k`u`u`w]
<O #'
 MM22Ldootxx262J2JLG&&t&?E++%J;;&K 3 Zk0@@G	 	r>   c                     U R                  5       nS H  n[        X   U R                  S9X'   M     U R                  " S0 UD6$ )N)r<  r=  r?  r!  r   )rN  r   r!  r;  )r:   rN  fields      r;   r8  CacheStatsMixin.format_tuple  s>    

BE+ELdhhGEL C||$e$$r>   c                 (    U R                  5       S   $ )Nr?  )rN  rN   s    r;   chunks_stored_size"CacheStatsMixin.chunks_stored_size  s    zz|N++r>   r3  N)F)r   r   r   r   r7  r<   r9  r   r;  rN  r8  rU  r   r   r>   r;   r1  r1    s6    DJ;  %5 6G*%,r>   r1  c                       \ rS rSrSrSSSSS\SS4S jrS rS rS	 r	S
 r
S rS rS rS rS rS rS rS rS rS rSS jrSS jrSS jrSS jrS rS rSrg) r$  i  z(
Persistent, local (client-side) cache.
NTFc                 |   [         R                  XS9  Xl        X l        X0l        Xpl        Xl        Xl        SU l        SU l	        [        X5      U l        [        U5      U l        [        U R                  U R                  U5      U l        [         R                  R#                  U R                  5      (       d,  U R                  R%                  XcU5        U R'                  5         U R)                  5          U R                  R+                  X2U R                  S9  U R-                  5       (       d  U R/                  5         U R1                  5         U(       aP  U R                  R2                  U R                  R4                  :w  a!  U R7                  5         U R9                  5         ggg!   U R;                  5         e = f)a'  
:param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
:param lock_wait: timeout for lock acquisition (int [s] or None [wait forever])
:param sync: do :meth:`.sync`
:param cache_mode: what shall be compared in the file stat infos vs. cached stat infos comparison
rQ  NF)rx   )r1  r<   r0   rY   re   r   r"  r#  rc   
txn_activer3   r5   r,   security_managerr   rx   r4   r@   r   r   rR   r   check_cache_compatibility
wipe_cacheupdate_compatibilityr   r   r  commitr   )r:   r0   rY   re   r5   r  r   r   r   r"  r#  r!  s               r;   r<   LocalCache.__init__  sM    	   /$  $#6 j/	 /
 ;'IN ww~~dii((!!778KWZ[KKM			!!//DL]L]/^1133!%%'((D,=,=,I,II		 Jt	JJLs   7B.F( (F;c                     U $ rH   r   rN   s    r;   r   LocalCache.__enter__      r>   c                 $    U R                  5         g rH   r   r   s       r;   r   LocalCache.__exit__  r   r>   c                    [         R                  " U R                  5        [        [         R                  R	                  U R                  S5      S5       nUR                  [        5        SSS5        U R                  R                  5         [        5       R                  [         R                  R	                  U R                  S5      5        [         R                  " [         R                  R	                  U R                  S5      5        [        [         R                  R	                  U R                  [        5       5      SS9    SSS5        [        [         R                  R	                  U R                  S5      S	S9 n[        R                  " U R                  US
S9  SSS5        g! , (       d  f       GNH= f! , (       d  f       N|= f! , (       d  f       g= f)z0Create a new empty cache at `self.path`
        READMEwNrF  chunks.archive.dTbinary
pre12-metaF   indent)r4   makedirsr5   rR   r6   rd   r   rx   r   r	   r'   r   jsondumpr4  r:   rZ   s     r;   r   LocalCache.create  s    	DII"'',,tyy(3S9RHH\" :  "277<<		8<=
BGGLL,>?@bggll499.>.@A$O Pbggll499l;EJbIIdoor!4 KJ :9
 POJJs$   F,F>!G,
F;>
G
Gc                    U R                   R                  5         [        [        R                  R                  U R                  S5      SU R                   R                  R                  S5      S9 n[        R                  " U5      U l
        S S S 5        SU R                  ;   a  S U l        OU R                  5          [        [        R                  R                  U R                  S5      5       n[        R                  " U5      U l        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f! ["        [        R$                  4 a     g f = f)NrF  Fr5   rd   r   drk  )rx   r   r#   r4   r5   r6   r   r   r	   rS   rF  r"  r   _read_filesrR   rp  r4  rs   JSONDecodeErrorrr  s     r;   _do_openLocalCache._do_open  s     !rww||DIIx'HPU151B1B1L1L1P1PQY1Z\_a$//"-DK\ $//!DJ	bggll499l;<"&))B- =<\ \ =<!4#7#78 		s<   1D>3D8 1D'D8 
D$'
D51D8 5D8 8EEc                     [         R                  R                  U R                  5      (       d  [        SU R                  -  5      eU R                  R                  5         U R                  5         g )Nz"%s Does not look like a Borg cache)r4   r5   isdirr  rx   rR   rollbackrN   s    r;   rR   LocalCache.open  sI    ww}}TYY''@499LMM r>   c                 >   [        [        R                  R                  U R                  S5      S5       n[        R
                  " U R                  USS9  S S S 5        U R                  b"  U R                  R                  5         S U l        g g ! , (       d  f       N>= f)Nrk  rg  rl  rm  )	rR   r4   r5   r6   rp  rq  r4  rx   r   rr  s     r;   r   LocalCache.close$  sn    "'',,tyy,7=IIdoor!4 >(##% $D ) >=s   !B
Bc                 :   0 U l         S U l        [        R                  S5        [        R                  S5        S n [        [        R                  R                  U R                  [        5       5      SU R                  R                  R                  [        5       5      S9 n[        R                  " SS9n UR                  S5      nU(       d  OdUR!                  U5         U HJ  u  pV[#        U6 n[        R$                  " UR'                  UR(                  S-   S	95      U R                   U'   ML     M}  S S S 5        Ub1  [        R5                  U5        [        R5                  S5        0 U l         [        R                  S[7        U R                   5      5        g ! [*        [,        4 a  nS
[/        U5      -  n S nAM  S nAff = f! , (       d  f       N= f! [0         a  nS[/        U5      -  n S nANS nAf[2         a  n	S[/        U	5      -  n S n	A	NS n	A	ff = f)NzReading files cache ...zFILES-CACHE-LOAD: starting...Fru  T)use_listi   r   )agez#The files cache seems invalid. [%s]z#The files cache can't be read. [%s]z"The files cache is corrupted. [%s]z:Continuing without files cache - expect lower performance.z.FILES-CACHE-LOAD: finished, %d entries loaded.)r   _newest_cmtimerW   r`   files_cache_loggerr#   r4   r5   r6   r   rx   r   r   r   UnpackerrS   feedr*   packb_replacer  	TypeError
ValueErrorrT   rV   r%   rX   len)
r:   rz   rZ   udata	path_hashitementryr\   fies
             r;   rw  LocalCache._read_files,  s   
"./  !@A	B%277<<		CSCU+V^c595F5F5P5P5T5TUeUg5hjmo$$d3779-DFF4L/0OI$2D$9E4;MM%..UZU^U^abUb.Bc4dDJJy1 01 j( ?NN3NNWXDJ  !QSVW[WaWaSbc &z2 Cc#hNj j   	C7#c(BC! 	B6SAC	Bsn   A&G "A G#AF3G5G G+F?9G?GG
GG G 
H"G55HHHc                 >   [        SS9n[        R                  R                  U R                  S5      n[        R                  " U5        UR                  S5        [        R                  " [        R                  R                  U R                  S5      U5        UR                  S5        [        R                  " [        R                  R                  U R                  S5      U5        UR                  S5         [        R                  " [        R                  R                  U R                  [        5       5      U5        [        R                  " [        R                  R                  U R                  S5      [        R                  R                  U R                  S5      5        S	U l        UR                  5         g ! [         aP    [        [        R                  R                  U[        5       5      S	S
9    S S S 5         M  ! , (       d  f        N= ff = f)Nzcache.begin_transactionmsgidtxn.tmpz.Initializing cache transaction: Reading configr   z.Initializing cache transaction: Reading chunksrF  z-Initializing cache transaction: Reading filesTri  
txn.active)r   r4   r5   r6   mkdiroutputrA   copyr   rs   r'   renamerY  finish)r:   pitxn_dirs      r;   	begin_txnLocalCache.begin_txnM  sN   %,EF'',,tyy)4

		BCBGGLLH5w?
		BCBGGLLH5w?
		AB	KKTYY0@0BCWM 			"'',,tyy)4'',,tyy,7	9
		 ! 	"'',,w0@0BCDQ RQQQ	s+   <AG :H<H	>H	
H	HHc                    U R                   (       d  gU R                  R                  U R                  U R                  5        [        SS9nU R                  Gb  U R                  c  SU l        [        [        R                  R                  SS5      5      nUR                  S5        [        R                  S5        [        [        R                   R#                  U R                   [%        5       5      S	S
9 nSnU R                  R'                  5        H  u  pV[)        [*        R,                  " U5      6 nUR.                  S:X  a#  [1        UR2                  5      U R                  :  d$  UR.                  S:  d  Mf  UR.                  U:  d  Mx  [*        R4                  " XW4U5        US-  nM     SSS5        [        R                  SU5        [        R                  SU R                  5        [        R                  SW5        WR6                  U R8                  R:                  [%        5       '   UR                  S5        [        [        R                   R#                  U R                   S5      S	S
9 nU R<                  R?                  U5        SSS5        WR6                  U R8                  R:                  S'   UR                  S5        U R8                  R                  U R                  U R                  5        [        R@                  " [        R                   R#                  U R                   S5      [        R                   R#                  U R                   S5      5        [B        RD                  " [        R                   R#                  U R                   S5      5        SU l         URG                  5         g! , (       d  f       GN= f! , (       d  f       GNB= f)zCommit transaction
        Nzcache.commitr  l    BORG_FILES_CACHE_TTL   zSaving files cachezFILES-CACHE-SAVE: starting...Tr5   rd   r   r   z>FILES-CACHE-KILL: removed all old entries with age >= TTL [%d]zCFILES-CACHE-KILL: removed all current entries with newest cmtime %dz7FILES-CACHE-SAVE: finished, %d remaining entries saved.zSaving chunks cacherF  zSaving cache configr  r  F)$rY  rZ  rg   re   rY   r   r   r  intr4   r   r   r  r  r`   r#   r5   r6   r   r   r*   r   unpackbr  r   cmtimepackr   rx   r   rF  rd   r  rA   rB   r  )r:   r  ttlrZ   entry_countr  r  r  s           r;   r^  LocalCache.commita  s    ""4==$((;%N;::!""*&1#bjjnn%;R@ACII*+$$%DE%277<<		CSCU+V^bcgi'+zz'7'7'9OI +GOOD,ABEyyA~-*EH[H[*[yy1}Si%7<#q( (: d $$%egjk$$%jlpll  A$$%^`kl>@>O>OD''(8(:;
		'(!rww||DIIx'HPTUY[KKb! V020A0A##H-
		'(t}}dhh7
		"'',,tyy,7'',,tyy)4	6bggll499i89
		3 dc  VUs%   7A?N4:N4!N4/O4
O
Oc                    [         R                  R                  [         R                  R                  U R                  S5      5      (       a>  [        R
                  " [         R                  R                  U R                  S5      5        [         R                  R                  U R                  S5      n[         R                  R                  U5      (       Ga  [        R                  " [         R                  R                  US5      U R                  5        [        R                  " [         R                  R                  US5      U R                  5        [        R                  " [         R                  R                  U[        U5      5      U R                  5        [         R                  " U[         R                  R                  U R                  S5      5        [         R                  R                  [         R                  R                  U R                  S5      5      (       a>  [        R
                  " [         R                  R                  U R                  S5      5        SU l	        U R                  5         g)z3Roll back partial and aborted transactions
        r  r  r   rF  FN)r4   r5   r@   r6   rA   rB   r  r   r  rY  ry  )r:   r  s     r;   r}  LocalCache.rollback  sV    77>>"'',,tyy)<==MM"'',,tyy)<='',,tyy,777>>'""KKWh7CKKWh7CKKW.G.PQSWS\S\]IIgrww||DIIyABww~~bggll499i@AAbggll499i@Ar>   c                   ^ ^^^^^^^	^
^^^^^^^^^^ [         R                  R                  T R                  S5      m[        5       mSmSmSmSmSmSU4S jjmUU 4S jmU 4S jmU4S jmSU4S jjmUU	UU4S jmUUUU U4S	 jm
UU4S
 jmUUU4S jmU 4S jmUUUU	U
UUUUUUUUU 4S jnUU 4S jnT R                  R                  [        R                  R                  45        T R                  5         [        T R                  T R                  S9 m	U" 5         [         R                  R                  T5      T l        U" T R                  5      T l        SSS5        g! , (       d  f       g= f)aY  Re-synchronize chunks cache with repository.

Maintains a directory with known backup archive indexes, so it only
needs to fetch infos from repo and build a chunk index once per backup
archive.
If out of sync, missing archive indexes get added, outdated indexes
get removed and a new master chunks index is built by merging all
archive indexes.
rh  r   c                 b   > [        U 5      n[        R                  R                  TX!-   5      nU$ rH   )r   r4   r5   r6   )r   r   id_hexr5   archive_paths       r;   mkpathLocalCache.sync.<locals>.mkpath  s'    ^F77<<fo>DKr>   c            	      f  > TR                   (       a  [        R                  " T5      n U  Vs1 s H  n[        U5      S:X  d  M  [	        U5      iM!     snU  Vs1 s H:  n[        U5      S:X  d  M  UR                  S5      (       d  M,  [	        US S 5      iM<     sn-  $ [        5       $ s  snf s  snf )Nr  H   .compact)do_cacher4   r   r  r   endswithr   )fnsr   r  r:   s     r;   cached_archives(LocalCache.sync.<locals>.cached_archives  s    }}jj. 25F2B2
2F69gcSW]+r{{[eOf+
2cr7+cgh h u Ggs   B)B)B.0B.B.c                     > TR                   R                  R                  5        V s1 s H  o R                  iM     sn $ s  sn f rH   )re   rH  listr   )infor:   s    r;   repo_archives&LocalCache.sync.<locals>.repo_archives  s2    (,(>(>(C(C(EF(EGG(EFFFs   A c                 (   > U  H  nT" U5        M     g rH   r   )idsr   cleanup_cached_archives     r;   cleanup_outdated)LocalCache.sync.<locals>.cleanup_outdated  s    &r* r>   c                 B  >  [         R                  " T" U 5      5        [         R                  " T" U 5      S-   5        U(       d  g  [         R                  " T" U SS95        [         R                  " T" U SS9S-   5        g ! [         a     NOf = f! [         a     g f = f)Nz
.integrityr  r   )r4   unlinkrs   )r   cleanup_compactr  s     r;   r  /LocalCache.sync.<locals>.cleanup_cached_archive  s    		&*%		&*|34 #		&J78		&J7,FG %  % s#   ;B 9B 
BB
BBc                    > U R                  5       n/ n[        T5      (       au  U Hn  nTR                  U5      nU(       aB  X   R                  UR                  S9nUR
                  UR
                  :X  d   S5       eXPU'   M]  UR                  U5        Mp     OUn[        UTR                  R                  U5      5       H;  u  p6X   R                  [        U5      S9nXPU'   UTU'   T
S-  m
T	[        U5      -  m	M=     g)z
Archives created with AdHocCache will have csize=0 in all chunk list entries whose
chunks were already in the repository.

Scan *chunk_idx* for entries where csize=0 and fill in the correct information.
)rK  zChunk size mismatchr   N)
zero_csize_idsr  r   r  rK  sizeappendzipr0   get_many)	chunk_idxall_missing_ids	fetch_idsid_already_fetched_entryr  r  chunks_fetched_size_indexdecrypted_repositoryfetched_bytes_for_csizefetched_chunks_for_csizes          r;   fetch_missing_csize,LocalCache.sync.<locals>.fetch_missing_csize  s    (668OI,--*C,E,I,I#,N), ) 7 7>S>Y>Y 7 Z$zz-B-G-GG^I^^G).#!((- + ,	 !,@,K,K,T,TU^,_`	!//c$i/@!&#16)#.(A-('3t94' ar>   c                 :  > UR                  U 5      u  p4UR                  U S[        U5      U5        TR                  R	                  USS9u  pVn[        US9nUR                  S:w  a  [        S5      e[        U5      n[        UR                  UR                  UR                  5      5       HI  u  n	u  p4UR                  U	S[        U5      U5        T[        U5      -  mTS-  mUR                  U5        MK     TR                  (       a  T
" U5        T" X5        g g )Nr   T)force_tam_not_required)internal_dictz Unknown archive metadata version)r   addr  rY   unpack_and_verify_archiver    r   r  r   r  r   r  r  r  )
archive_idr  r  rK  r  rE  verified_r  item_idr  processed_item_metadata_bytesprocessed_item_metadata_chunksr:   write_archive_indexs             r;   fetch_and_build_idx,LocalCache.sync.<locals>.fetch_and_build_idx  s    /22:>KEMM*aTE:#'88#E#Edcg#E#h Gq!8G!# BCC$Y/D*-gmm=Q=Z=Z[b[h[h=i*j&%gq#d)U;-T:-.!3.		$	 +k
 }}#I.#J: r>   c                 0  > TUR                  5       -  mT" U SS9nT" U SS9n [        US[        U 5      S-   S9 nUR                  U5        S S S 5        [        R
                  " X25        g ! , (       d  f       N%= f! [         a    [        U5         g f = f)Nr  r   z.tmpT)r5   rd   filename)compactr$   r   rd   r4   r  r  r   )r  r  r   fn_tmprZ   "compact_chunks_archive_saved_spacer  s        r;   r  ,LocalCache.sync.<locals>.write_archive_index  s    .)2C2C2EE.
:6BJv6F&1vT;Ej;QT^;^`ceOOB'` 		&%` `  $F#$s(   A= A,A= ,
A:6A= =BBc                 >  > T" U 5      n[         R                  SU5          [        US-   SS9 n[        R                  " USS9nS S S 5        T" U SS9  W$ ! , (       d  f       N= f! [
         a>    [        USS9 n[        R                  " U5      nS S S 5         OZ! , (       d  f        OK= ff = f! [         a7  n[         R                  SX5        T" U 5        [        [        5         S nAg S nAff = f[         R                  S	U5        T" U 5        T" U W5        U$ )
Nz-Reading cached archive chunk index for %s ...r  Fr  T)permit_compact)r  z1Cached archive chunk index of %s is corrupted: %sz6Found non-compact index for %s, converting to compact.)rW   r  r$   r	   rS   rs   r%   errorr   r   r`   )	r  rM  archive_chunk_idx_pathrZ   archive_chunk_idxr  r  r  r  s	         r;   read_archive_index+LocalCache.sync.<locals>.read_archive_index  s   %+J%7"KKGV@5;QT^;^fklpr,6OOBt,T) m +:uM,, ml ) @5;QY^_ce,6OOB,?) `___@ & PR^d&z2|$ LLQS_`":.
,=>$$sd   A' AA' 
A$ A' 'B/;BB/B2 
B+	&B/)B2 +B//B2 2
C3<-C..C3c                    > 0 nTR                   R                  R                  5        H-  nUR                  U ;   d  M  UR                  XR                  '   M/     [        U5      [        U 5      :X  d   eU$ rH   )re   rH  r  r   namer  )archive_idsarchive_namesr  r:   s      r;   get_archive_ids_to_names1LocalCache.sync.<locals>.get_archive_ids_to_names3  sc    
 M..33577k)-1YYM''* 6 }%[)9999  r>   c           
        > [         R                  S5        T	" 5       nT" 5       n[         R                  S[        U5      [        U5      [        X-
  5      [        X!-
  5      5        U R                  5         T
" X-
  5        [        TR                  5      nU(       Ga  TR
                  (       d  S O[        US9n [        [        U5      SSSS9nT" U5      nUR                  5        H  u  pgUR                  [        U5      /S9  TR
                  (       ax  Xa;   a  T" Xg5      nUc  UR                  U5        Xa;  a*  [         R                  S	U5        [        5       nT" UTU5        [         R                  S
5        U R                  W5        M  U =(       d	    [        US9n [         R                  SU5        T" UTU 5        M     TR
                  (       d  T" U 5        UR                  5         [         R                  S[        T5      T5        [         R                  S[        T5      T5        [         R                  S[        T5      5        [         R                  S5        U $ )NzSynchronizing chunks cache...zIArchives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.usableg?z3%3.0f%% Syncing chunks cache. Processing archive %sz
cache.sync)totalsteprz   r  )r  z.Fetching and building archive index for %s ...z$Merging into master chunks index ...z!Fetching archive index for %s ...zlCache sync: had to fetch %s (%d chunks) because no archive had a csize set for them (due to --no-cache-sync)z0Cache sync: processed %s (%d chunks) of metadataz;Cache sync: compact chunks.archive.d storage saved %s byteszDone.)rW   r  r  clearr0   r  r	   r   r   showr   r  merger  r`   r   )r  
cached_idsr  master_index_capacityr  archive_ids_to_namesr  rM  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r:   s            r;   create_master_idx*LocalCache.sync.<locals>.create_master_idx?  s   KK78(*J'/KKKcK #j/J,-s;3K/LN OOZ56 %($8!(,D:Mb;c	-C4D32g4@B (@'L$0D0J0J0L,JGG"3L"A!BGC}}%30B:0\-08 * 1 1* =%7 #KK(XZfg0:-/
<PRcd$JK!(9:$-$YCX1Y	$GV+J8LiX% 1M& }}'	2		 8-.EFH`b O-.KLNlnZ-.PQSKK r>   c                  Z  >  [         R                  " [         R                  R                  TR                  S5      5         [         R                  " [         R                  R                  TR                  S5      5         [         R                  " T 5        g!    N[= f!    N#= f!    g= f)z?bring old cache dirs into the desired state (cleanup and adapt)zchunks.archivezchunks.archive.tmpN)r4   r  r5   r6   r  )r  r:   s   r;   legacy_cleanup'LocalCache.sync.<locals>.legacy_cleanupr  sz    		"'',,tyy2BCD		"'',,tyy2FGH&s#   >B >B B& BB#&B*)decrypted_cacheN)r~   )T)r4   r5   r6   r	   re   check_repository_compatibilityr   	OperationREADr  r(   r0   rY   r|  r  rF  )r:   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s   `  @@@@@@@@@@@@@@@@@@r;   r  LocalCache.sync  s    ww||DII/AB$.L!()%)*&-.*#$ "#	
		G	+		5 	5B	; 	;&	&	%:
	!1	 1	 1	f	$ 	44h6H6H6M6M5PQT__dhhGK_ GGMM,7DM+DKK8DK HGGs   AE%%
E3c                     [         R                  nU R                  R                  U-  (       a  gU R                  R                  U::  d  gg)NFT)r   SUPPORTED_REPO_FEATURESrx   r   r   )r:   my_featuress     r;   r[  $LocalCache.check_cache_compatibility  sB    66--;   33{B r>   c                 ^   [         R                  S5        [        R                  R	                  U R                  S5      n[        R                  R                  U5      (       a|  [        R                  " [        R                  R	                  U R                  S5      5        [        R                  " [        R                  R	                  U R                  S5      5        [        5       U l
        [        [        R                  R	                  U R                  [        5       5      SS9    S S S 5        SU R                  l        U R                  R                  R!                  SSS5        [!        5       U R                  l        [!        5       U R                  l        g ! , (       d  f       Ny= f)Nz9Discarding incompatible cache and forcing a cache rebuildrh  Tri  r~   r   re   )rW   rX   r4   r5   r6   r|  rA   rB   ro  r	   rF  r'   r   rx   r   r   r   r   r   )r:   r  s     r;   r\  LocalCache.wipe_cache  s   RSww||DII/AB77==&&MM"'',,tyy2DEFKKTYY0BCD lbggll499.>.@A$O P(*%!!%%gz2>-0U*/2u, POs   )F
F,c                 \   U R                   R                  5       n[        R                  n[	        5       nUR                  5        H  u  pEUR                  U5        M     U R                  R                  R                  X2-
  5        U R                  R                  R                  X2-  5        g rH   )
re   get_all_mandatory_featuresr   r  r   r   updaterx   r   r   )r:   operation_to_features_mapr  repo_features	operationfeaturess         r;   r]  LocalCache.update_compatibility  s    $(MM$L$L$N!66#<#B#B#DI  * $E 	**11-2MN,,33M4OPr>   c                    U R                   (       d  U R                  5         [        U5      nU R                  X5      nU(       a  U(       d  U R	                  X5      $ U R
                  R                  U5      n[        U5      n	U R                  R                  XUS9  U R                  R                  USXi5        UR                  XiU(       + 5        [        XU	5      $ )Nwaitr   )rY  r  r  
seen_chunkchunk_increfrY   encryptr0   putrF  r  r  r!   
r:   r   chunkrN  	overwriter"  r  refcountr  rK  s
             r;   	add_chunkLocalCache.add_chunk  s    NN5z??2,I$$R//xx&D	B40At+Th,/b..r>   c                     U R                   R                  U[        SS S 5      5      u  p4nUb  Ub  X$:w  a  [        SXU4-  5      eU$ )Nr   z@chunk has same id [%r], but different size (stored: %d new: %d)!)rF  r   r
   r  )r:   r   r  r*  stored_sizer  s         r;   r#  LocalCache.seen_chunk  s\    #';;??2q$PT7U#V q 7D<O ^Tb3 3 4 4r>   c                     U R                   (       d  U R                  5         U R                  R                  U5      u  pVnUR	                  XgSUS9  [        XU5      $ NFpart)rY  r  rF  increfr  r!   r:   r   rN  r  r3  count_sizerK  s           r;   r$  LocalCache.chunk_incref  sK    NN"kk004eU5t4b//r>   c                 ,   U R                   (       d  U R                  5         U R                  R                  U5      u  pVnUS:X  a;  U R                  U	 U R                  R                  XS9  UR                  U* U* SUS9  g UR                  U* U* SUS9  g Nr   r!  Tr2  F)rY  r  rF  decrefr0   deleter  r:   r   rN  r"  r3  r6  r  rK  s           r;   chunk_decrefLocalCache.chunk_decref  s    NN![[//3UA:BOO""2"1LL$4L8LL$DL9r>   c                    [         R                  " UR                  5      (       d  gU R                  nSU;   a  [        R                  S5        gSU;   a  [        R                  S5        gU R                  R                  U5      nU(       d  [        R                  SU5        g[        [        R                  " U5      6 nSU;   a1  UR                  UR                  :w  a  [        R                  SU5        g	S
U;   a1  UR                  UR                  :w  a  [        R                  SU5        g	SU;   a:  [        UR                   5      UR"                  :w  a  [        R                  SU5        g	SU;   a:  [        UR                   5      UR$                  :w  a  [        R                  SU5        g	[        R&                  " UR)                  UR                  SS95      U R                  U'   SUR*                  4$ )a*  
Check if we know the file that has this path_hash (know == it is in our files cache) and
whether it is unchanged (the size/inode number/cmtime is same for stuff we check in this cache_mode).

:param hashed_path: the file's path as we gave it to hash(hashed_path)
:param path_hash: hash(hashed_path), to save some memory in the files cache
:param st: the file's stat() result
:return: known, ids (known is True if we have infos about this file in the cache,
                     ids is the list of chunk ids IF the file has not changed, otherwise None).
FNrv  zUNKNOWN: files cache disabledrzUNKNOWN: rechunking enforcedz*UNKNOWN: no file metadata in cache for: %rsz(KNOWN-CHANGED: file size has changed: %r)TNiz0KNOWN-CHANGED: file inode number has changed: %rcz)KNOWN-CHANGED: file ctime has changed: %rmz)KNOWN-CHANGED: file mtime has changed: %rr   )inoder  T)statS_ISREGst_moder"  r  r`   r   r   r*   r   r  r  st_sizerG  st_inor   r  st_ctime_nsst_mtime_nsr  r  	chunk_ids)r:   hashed_pathr  str"  r  s         r;   file_known_and_unchanged#LocalCache.file_known_and_unchanged  s    ||BJJ''__
*$$%DE *$$%CD

y)$$%QS^_ 67*rzz!9$$%OQ\]*		!9$$%WYde*u||!<!N$$%PR]^J=#>"..#P$$%PR]^ !(enn299RSn.T U

9U__$$r>   c           	         [         R                  " UR                  5      (       d  g U R                  nSU;   a  [        R                  S5        g SU;   a  Sn[        UR                  5      nO5SU;   a  Sn[        UR                  5      nOSn[        UR                  5      n[        SUR                  UR                  [        U5      US9n[        R                  " U5      U R                  U'   [!        U R"                  =(       d    SU5      U l        [        R                  S	UR%                  S
['        UR(                  5      -  S9Xa5        g )Nrv  z*FILES-CACHE-NOUPDATE: files cache disabledrE  ctimerF  mtimer   )r  rG  r  r  rO  z)FILES-CACHE-UPDATE: put %r [has %s] <- %rz[%d entries])rO  )rH  rI  rJ  r"  r  r`   r   rM  rN  r*   rL  rK  r   r   r  r   r   r  r  r  rO  )	r:   rP  r  rQ  r  r"  cmtime_type	cmtime_nsr  s	            r;   memorize_fileLocalCache.memorize_file  s   ||BJJ''__
*$$%QR*!K/IJ!K/I!K/I1BIIBJJ}]fOgsvw 'e 4

9!$"5"5":IF  !L!&.3uK_:_!`!,	;r>   )r  rx   r"  rF  r#  r  r   rY   re   r5   r4  r   r0   rZ  rc   rY  FTrH   NFTF)r   r   r   r   r   r   r<   r   r   r   ry  rR   r   rw  r  r^  r}  r  r[  r\  r]  r+  r#  r$  r>  rR  rY  r   r   r>   r;   r$  r$    s     8<$\`4<Ukp*X5%dB('R$p9d
5Q/0	:2%h;r>   r$  c                       \ rS rSrSrSr  SS jrS rS rSr	Sr
S	 rS
 rSS jrSS jrSS jrSS jrS rS rS rSrg)r(  i)  ai  
Ad-hoc, non-persistent cache.

Compared to the standard LocalCache the AdHocCache does not maintain accurate reference count,
nor does it provide a files cache (which would require persistence). Chunks that were not added
during the current AdHocCache lifetime won't have correct size/csize set (0 bytes) and will
have an infinite reference count (MAX_VALUE).
zAll archives:                unknown              unknown              unknown

                       Unique chunks         Total chunks
Chunk index:    {0.total_unique_chunks:20d}             unknownNc                     [         R                  XS9  Xl        X l        X0l        X`l        SU l        [        U5      U l        U R                  R                  X2US9  0 U l
        [        R                  S5        g )NrQ  Fr   z1Note: --no-cache-sync is an experimental feature.)r1  r<   r0   rY   re   r#  _txn_activer,   rZ  r   r4  rW   rX   )r:   r0   rY   re   r   r   r#  r!  s           r;   r<   AdHocCache.__init__9  sj       /$ #6   /
 ;++HY+OJKr>   c                     U $ rH   r   rN   s    r;   r   AdHocCache.__enter__J  rb  r>   c                     g rH   r   r   s       r;   r   AdHocCache.__exit__M      r>   rv  c                 .    [         R                  S5        g)Nz$UNKNOWN: files cache not implementedrA  )r  r`   )r:   rP  r  rQ  s       r;   rR  #AdHocCache.file_known_and_unchangedS  s      !GHr>   c                     g rH   r   )r:   rP  r  rQ  r  s        r;   rY  AdHocCache.memorize_fileW  rf  r>   c                    U(       a   S5       eU R                   (       d  U R                  5         [        U5      nU R                  X5      nU(       a  U R	                  XUS9$ U R
                  R                  U5      n[        U5      n	U R                  R                  XUS9  U R                  R                  USXi5        UR                  XiU(       + 5        [        XU	5      $ )NuH   AdHocCache does not permit overwrites — trying to use it for recreate?r  r!  r   )r`  r  r  r#  r$  rY   r%  r0   r&  rF  r  r  r!   r'  s
             r;   r+  AdHocCache.add_chunkZ  s    hhh}NN5z??2,$$RT$::xx&D	B40At+Th,/b..r>   c                 4   U R                   (       d  U R                  5         U R                  R                  U[	        SS S 5      5      nUR
                  (       a4  U(       a-  UR                  (       d  UR                  US9U R                  U'   UR
                  $ )Nr   rl  )r`  r  rF  r   r
   r*  r  r  )r:   r   r  r  s       r;   r#  AdHocCache.seen_chunki  se    NNOAtT$BC>>d5:: $nn$n7DKKO~~r>   c                     U R                   (       d  U R                  5         U R                  R                  U5      u  pVnU=(       d    UnU(       d   eUR	                  X7SUS9  [        XU5      $ r1  )r`  r  rF  r4  r  r!   r5  s           r;   r$  AdHocCache.chunk_increft  s_    NN"kk004e }tT%d3b..r>   c                 ,   U R                   (       d  U R                  5         U R                  R                  U5      u  pVnUS:X  a;  U R                  U	 U R                  R                  XS9  UR                  U* U* SUS9  g UR                  U* U* SUS9  g r:  )r`  r  rF  r;  r0   r<  r  r=  s           r;   r>  AdHocCache.chunk_decref  s    NN![[//3UA:BOO""2"1LL$4L8LL$DL9r>   c                     U R                   (       d  g U R                  R                  U R                  U R                  5        SU l         g r\  )r`  rZ  rg   re   rY   rN   s    r;   r^  AdHocCache.commit  s4    ""4==$((; r>   c                     SU l         U ?g r\  )r`  rF  rN   s    r;   r}  AdHocCache.rollback  s     Kr>   c           
         SU l         [        U R                  5      n[        US-  S9U l        [        USSS9n[        5       nSnS n U R                  R                  [        US9nUS	-  nU(       d  OOUR                  [        U5      S
9  US   n[        [        R                  SSS9nU H  nXpR                  U'   M     Mz  [        U R                  5      U:X  d   eU R                  U R                  R                  	 [        5       U-
  =(       d    Sn	UR                  5         [        R!                  SXU[#        US-  U	-  5      5        g )NTg?r  z!Downloading chunk list... %3.0f%%zcache.download_chunks)r   rz   r  r   )limitmarkerr   )increase)r*  r  rK  g{Gz?zBAdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s"   )r`  r  r0   r	   rF  r   r   r  r)   r  r
   	MAX_VALUEre   MANIFEST_IDr  rW   r`   r   )
r:   
num_chunksr  t0num_requestsrz  result
init_entryr  durations
             r;   r  AdHocCache.begin_txn  s9    )
 
S(89%J<_,CE^__)))OFALGGS[G)BZF )*2F2FQVWXJ#-C    4;;:---KK112>B&.$
		Y<9I*WY/\dJd9e	gr>   )r`  rF  r#  rY   re   r4  r0   rZ  )TNFFr[  rH   r\  r]  )r   r   r   r   r   r7  r<   r   r   r   r"  rR  rY  r+  r#  r$  r>  r^  r}  r  r   r   r>   r;   r(  r(  )  sc    CJ qvL" EJ/		/	:!gr>   r(  rH   )Er   rp  r4   rA   rH  collectionsr   timer   rW   r   r  	constantsr   r   	hashindexr	   r
   r   helpersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r    r!   
crypto.keyr"   crypto.file_integrityr#   r$   r%   lockingr&   platformr'   remoter(   r0   r)   r*   r,   r   r   r3   r   r   r   rv   r1  r$  r(  r   r>   r;   <module>r     s      	   "  !	"#;<  > E E    4 a a %   & G )    - $ i i   # ' ,.OPa4 a4HB
 D4
Za[ a[HC CL-, -,`\	; \	;~Jg Jgr>   