
    yf                       S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
  S SKJrJr  S SKJr  S SKJr  S SKJr  S SKJrJr  S S	KJr  S
SKJrJrJrJr  S
SKJr  \" 5       rS
SK J!r!  S
SK"J#r#J$r$  S
SK%J&r&  S
SK'J(r(J)r)  S
SK*J+r+J,r,  S
SK-7  S
SK.J/r0  S
SK1J2r2J3r3  S
SK1J4r4J5r5J6r6J7r7  S
SK8J9r9J:r:J;r;  S
SK1J<r<  S
SK1J=r=  S
SK1J>r>J?r?J@r@  S
SK1JArAJ/r/JBrB  S
SKCJDrDJErEJFrFJGrG  S
SK1JHrHJIrI  S
SK1JJrJJKrKJLrLJMrMJNrN  S
SK1JOrOJPrPJQrQJRrR  S
SK1JSrS  S
S K1JTrT  S
S!K1JUrU  S
S"K1JVrVJWrWJXrX  S
S#K1JYrYJZrZJ[r[  S
S$K1J\r\  S
S%K1J]r]  S
S&K1J^r^  S
S'K1J_r_  S
S(K`Jara  S
S)KbJcrcJdrdJere  S
S*KfJgrgJhrhJiri  S
S+KCJjrjJkrkJlrlJmrmJnrnJoro  S
S,KpJqrq  S
S-KrJsrsJtrt  \u" \S.5      rv " S/ S05      rwS1 rx " S2 S35      ry\y" 5       rzS4 r{S5 r|\
SSSS6S7S8.S9 j5       r} " S: S;5      r~ " S< S=5      r " S> S?\5      rSSS S S@.SA jr " SB SC5      r " SD SE5      r\a" SFSG SH9rSI r " SJ SK5      r " SL SM5      r " SN SO5      rSP r " SQ SR5      r " SS ST5      r " SU SV5      rg)W    N)OrderedDict)contextmanager)timezone	timedelta)partial)getuser)BytesIO)groupbyzip_longest)get_terminal_size   )is_win32is_linux
is_freebsd	is_darwin)create_logger)xattr)get_chunkerChunk)ChunkListEntry)key_factoryUnsupportedPayloadError)
CompressorCompressionSpec)*)IntegrityError)BackupErrorBackupRaceConditionError)BackupOSErrorBackupPermissionErrorBackupFileNotFoundErrorBackupIOError)
ChunkIndexChunkIndexEntryCacheSynchronizer)Manifest)hardlinkable)ChunkIteratorFileWrappernormalize_chunker_params	open_item)Errorr   set_ec)uid2useruser2uid	gid2group	group2gid)parse_timestampto_localtime)OutputTimestampformat_timedeltaformat_file_sizefile_statusFileSize)safe_encodesafe_decodemake_path_saferemove_surrogates)
StableDict)
bin_to_hex)safe_ns)ellipsis_truncateProgressIndicatorPercent	log_multi)os_openflags_normal	flags_dir)os_stat)msgpack)sig_int)utcnow)LRUCache)PathPrefixPatternFnmatchPattern	IECommand)ItemArchiveItemItemDiff)acl_getacl_set	set_flags	get_flagsswidthhostname)cache_if_remote)
RepositoryLIST_SCAN_LIMITlinkc                       \ rS rSrSS jrSS jrS rSrS rS r	S r
S	 r\S
 5       r\S 5       r\S 5       r\S 5       rSS jrSrg)
Statistics;   c                     Xl         X l        S=U l        =U l        =U l        U l        S=U l        =U l        =U l        U l	        SU l
        g Nr   )output_jsoniecosizecsizeusizenfilesosize_partscsize_partsusize_partsnfiles_partslast_progress)selfr_   r`   s      ./usr/lib/python3/dist-packages/borg/archive.py__init__Statistics.__init__=   sQ    &=>>
>TZ>$*t{UVVV4+Vd.>AR    c                 0   U(       dH  U =R                   U-  sl         U =R                  U-  sl        U(       a  U =R                  U-  sl        g g U =R                  U-  sl        U =R                  U-  sl        U(       a  U =R
                  U-  sl        g g N)ra   rb   rc   re   rf   rg   )rj   sizerb   uniqueparts        rk   updateStatistics.updateD   st    JJ$JJJ%J

e#
  $%  E)  rn   c                 f   [        U[        5      (       d  [        S5      e[        U R                  U R                  5      nU R
                  UR
                  -   Ul        U R                  UR                  -   Ul        U R                  UR                  -   Ul        U R                  UR                  -   Ul        U R                  UR                  -   Ul	        U R                  UR                  -   Ul
        U R                  UR                  -   Ul        U R                  UR                  -   Ul        U$ )Nzcan only add Statistics objects)
isinstancer[   	TypeErrorr_   r`   ra   rb   rc   rd   re   rf   rg   rh   )rj   otherstatss      rk   __add__Statistics.__add__P   s    %,,=>>4++TXX6jj5;;.jj5;;.jj5;;.{{U\\1 ,,u/@/@@ ,,u/@/@@ ,,u/@/@@!..1C1CCrn   zO{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}c                 6    U R                   R                  U SS9$ )NzThis archive:)rz   label)summaryformatrj   s    rk   __str__Statistics.__str__`   s    ||""_"EErn   c                 \    SR                  [        U 5      R                  [        U 5      U S9$ )NzF<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>)clshashrj   )r   type__name__idr   s    rk   __repr__Statistics.__repr__c   s0    W^^T
##"T( _ ? 	?rn   c                     [        U R                  U R                  S9[        U R                  U R                  S9[        U R                  U R                  S9U R
                  S.$ )Nr`   )original_sizecompressed_sizededuplicated_sizerd   )r7   ra   r`   rb   rc   rd   r   s    rk   as_dictStatistics.as_dictg   sI    %djjdhh?'

A!)$**$((!Ckk	
 	
rn   c                     U R                   U R                  U R                  U R                  U R                  U R
                  S.$ )Nrq   rb   rd   
size_partsrf   rh   ra   rb   rd   re   rf   rh   r   s    rk   as_raw_dictStatistics.as_raw_dicto   s<    JJZZkk**++ --
 	
rn   c                     U " 5       nUS   Ul         US   Ul        US   Ul        US   Ul        US   Ul        US   Ul        U$ )Nrq   rb   rd   r   rf   rh   r   )r   kwrj   s      rk   from_raw_dictStatistics.from_raw_dicty   sV    uZ
[
ll+m,~.rn   c                 >    [        U R                  U R                  S9$ Nr   )r5   ra   r`   r   s    rk   	osize_fmtStatistics.osize_fmt       

99rn   c                 >    [        U R                  U R                  S9$ r   )r5   rc   r`   r   s    rk   	usize_fmtStatistics.usize_fmt   r   rn   c                 >    [        U R                  U R                  S9$ r   )r5   rb   r`   r   s    rk   	csize_fmtStatistics.csize_fmt   r   rn   Nc                    [         R                  " 5       nUb  XPR                  -
  U:  Ga8  XPl        U R                  (       a|  U(       d2  U R	                  5       n[        U(       a  UR                  OS5      US'   O0 nUR                  [         R                   " 5       SUS.5        [        R                  " U5      nSnO[        5       u  pU(       dh  SR                  U 5      nU(       a  [        UR                  5      OSnU	[        U5      -
  nUS:  a  SnU	[        U5      -
  nUS:  a  U[        X5      -  nOS	U	-  nS
n[        XxU=(       d    [        R                   SS9  g g )N patharchive_progress)timer   finished
z={0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N        T)endfileflush)r   	monotonicri   r_   r   r;   r   rt   jsondumpsr   r   rT   r?   printsysstderr)rj   itemfinalstreamdtnowdatamsgr   columnslinesr   spaces                rk   show_progressStatistics.show_progress   s)   nn:111B6!$<<>D#4$TYYB#ODLD IIK. % 
 jj&!2!4Y``aefC;?,TYY7RD#fSk1Erz  '&+ 5z0==-C#V%9szzF; 7rn   )rb   rf   r`   ri   rd   rh   ra   re   r_   rc   rg   FFF)NFNN)r   
__module____qualname____firstlineno__rl   rt   r{   r   r   r   r   r   classmethodr   propertyr   r   r   r   __static_attributes__ rn   rk   r[   r[   ;   s    
* `GF?

   : : : : : :Grn   r[   c                     [         R                  " U 5      =(       d3    [         R                  " U 5      =(       d    [         R                  " U 5      $ rp   )statS_ISBLKS_ISCHRS_ISFIFO)modes    rk   
is_specialr      s-    <<Jd!3Jt}}T7JJrn   c                   .    \ rS rSrSrSS jrS rS rSrg)	BackupIO   r   c                     Xl         U $ rp   op)rj   r   s     rk   __call__BackupIO.__call__   s    rn   c                     g rp   r   r   s    rk   	__enter__BackupIO.__enter__   s    rn   c           
      r   U(       a  [        U[        5      (       a  [        R                  [        [        R
                  [        [        R                  [        [        R                  [        [        R                  [        0nUR                  UR                  [        5      nU" U R                  U5      Ueg g rp   )
issubclassOSErrorerrnoEPERMr    EACCESEBUSYENOENTr!   EIOr"   getr   r   )rj   exc_typeexc_valexc_tbE_MAPe_clss         rk   __exit__BackupIO.__exit__   sy    
8W552325		=E IIgmm];E)w6 68rn   r   N)r   )	r   r   r   r   r   r   r   r   r   r   rn   rk   r   r      s    	B
7rn   r   c              #      #    S[         l         [             [        U 5      n S S S 5        Wv   M"  ! [         a     S S S 5        g f = f! , (       d  f       N,= f7f)Nread)	backup_ior   nextStopIteration)iteratorr   s     rk   backup_io_iterr      sS     IL
H~ 
 
  ! 	 Y Ys:   AA	1A
AA		AAA		
AAc                     [         R                  " U R                  5      [         R                  " UR                  5      :w  a  [        S5      eU R                  UR                  :w  a  [        S5      eU$ )a0  
this checks for some race conditions between the first filename-based stat()
we did before dispatching to the (hopefully correct) file type backup handler
and the (hopefully) fd-based fstat() we did in the handler.

if there is a problematic difference (e.g. file type changed), we rather
skip the file than being tricked into a security problem.

such races should only happen if:
- we are backing up a live filesystem (no snapshot, not inactive)
- if files change due to normal fs activity at an unfortunate time
- if somebody is doing an attack against us
z1file type changed (race condition), skipping filez2file inode changed (race condition), skipping file)r   S_IFMTst_moder   st_ino)st_oldst_currs     rk   stat_update_checkr     sS      {{6>>"dkk'//&BB&'Z[[}}&&'[\\Nrn   Fopen)r   	parent_fdnamenoatimer   c           	   #      #    [        U5         [        XX0US9nS S S 5         Wv   Ub  [        R                  " U5        g g ! , (       d  f       N.= f! Wb  [        R                  " U5        f f = f7f)Nr   r  r  flagsr  )r   rB   osclose)r  r   r  r  r  r   fds          rk   OsOpenr     s`     	2$$U\] 
 >HHRL  
 >HHRL s1   A2AA2A A2
AA2A//A2c                   .    \ rS rSrS rSS jrSS jrSrg)	DownloadPipelinei  c                     Xl         X l        g rp   )
repositorykey)rj   r  r  s      rk   rl   DownloadPipeline.__init__  s    $rn   Nc              #     ^ #    U 4S jn[        5       n[        R                  " SS9nT R                  U5       GH  n	UR	                  U	5        U V
s/ s H  n
[        U
S9PM     nn
U H3  n
SU
;   d  M  U
R                   Vs/ s H  n[        U6 PM     snU
l        M5     U(       a   U V
s/ s H  o" U
5      (       d  M  U
PM     nn
U(       a  U(       a  U(       a  U H  n
[        U
R                  5      (       d  M  U
R                  S5      nUcN  SU
;   a  U" U
R                  5        U
R                  SS	5      (       a  UR                  U
R                  5        M  M  X;  d  M  X]   u  pUb  U" U5        UR                  U5        M     O#U H  n
SU
;   d  M  U" U
R                  5        M     U H  n
U
v   M	     GM     gs  sn
f s  snf s  sn
f 7f)
a}  
Return iterator of items.

*ids* is a chunk ID list of an item stream. *filter* is a callable
to decide whether an item will be yielded. *preload* preloads the data chunks of every yielded item.

Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved,
otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak.
c                 z   > TR                   R                  U  Vs/ s H  oR                  PM     sn5        g s  snf rp   )r  preloadr   )chunkscrj   s     rk   _preload.DownloadPipeline.unpack_many.<locals>._preload  s)    OO##6$:6aTT6$:;$:s   8Fuse_listinternal_dictr  sourceNhardlink_masterT)setrF   Unpacker
fetch_manyfeedrM   r  r   r'   r   r   addr   )rj   idsfilterpartial_extractr  hardlink_mastersr  masters_preloadedunpackerr   r   itemser  r  _s   `               rk   unpack_manyDownloadPipeline.unpack_many  s    	<  E##U3OOC(DMM$:BC($T-(ECt#?C{{"K{!>1#5{"KDK  */@%$6$<%@o !&'		22%)XXh%7F%~#+t#3$,T[[$9#'88,=t#D#D$5$9$9$))$D $E $*#B0@0HIF'-'9(0(8$5$9$9&$A !&$ !&#t+$T[[1 !& 
 O )C #L AsI   AGG%G4GGG+G?G4G=A%G&0G8Gc              #      #    [        XR                  R                  XS95       H"  u  p4U R                  R	                  X45      v   M$     g 7f)Nis_preloaded)zipr  get_manyr  decrypt)rj   r%  r2  id_r   s        rk   r"  DownloadPipeline.fetch_manyA  s>     S//":":3":"Z[IC((""3-- \s   A
A)r  r  NFFNr   )r   r   r   r   rl   r.  r"  r   r   rn   rk   r  r    s    7r.rn   r  c                   @    \ rS rSrSr\4S jrS rS rS
S jr	S r
Srg	)ChunkBufferiF  i   c                     [        5       U l        [        R                  " 5       U l        / U l        Xl        [        X R                  R                  SS.6U l	        g )NFseedsparse)
r	   bufferrF   Packerpackerr  r  r   
chunk_seedchunker)rj   r  chunker_paramss      rk   rl   ChunkBuffer.__init__I  s>    inn&"N9L9LUZ[rn   c                     U R                   R                  U R                  R                  UR	                  5       5      5        U R                  5       (       a  U R                  5         g g rp   )r?  writerA  packr   is_fullr   )rj   r   s     rk   r$  ChunkBuffer.addP  sB    $++**4<<>:;<<>>JJL rn   c                     [         erp   )NotImplementedError)rj   chunks     rk   write_chunkChunkBuffer.write_chunkU  s    !!rn   c                    U R                   R                  5       S:X  a  g U R                   R                  S5        / nU R                  R	                  U R                   5       Hx  nUR
                  S   nU[        :X  a  [        UR                  5      nO5U[        [        4;   a  [        S UR
                  S    nO[        SU-  5      eUR                  U5        Mz     U R                   R                  S5        U R                   R                  S5        U(       d  [        U5      S:X  a  S OSnUS U  H-  nU R                   R                  U R#                  U5      5        M/     US:X  a  U R                   R%                  US   5        g g )Nr   
allocationrq   z,chunk allocation has unsupported value of %rr   )r?  tellseekrC  chunkifymetaCH_DATAbytesr   CH_ALLOCCH_HOLEzeros
ValueErrorappendtruncatelenr  rN  rG  )rj   r   r  rM  allocr   r   s          rk   r   ChunkBuffer.flushX  s>   ;;"
 \\**4;;7EJJ|,EUZZ(8W--0ejj01 !ORW!WXXMM$ 8 	Qs6{a/dRDS\EKKt//67 ""9KKfRj) rn   c                 P    U R                   R                  5       U R                  :  $ rp   )r?  rS  BUFFER_SIZEr   s    rk   rI  ChunkBuffer.is_fulls  s     {{!D$4$444rn   )r?  rC  r  r  rA  Nr   )r   r   r   r   rc  ITEMS_CHUNKER_PARAMSrl   r$  rN  r   rI  r   r   rn   rk   r:  r:  F  s$    !K+? \
"*65rn   r:  c                   4   ^  \ rS rSr\4U 4S jjrS rSrU =r$ )CacheChunkBufferiw  c                 <   > [         TU ]  X$5        Xl        X0l        g rp   )superrl   cacherz   )rj   rj  r  rz   rD  	__class__s        rk   rl   CacheChunkBuffer.__init__y  s    -

rn   c                     U R                   R                  U R                  R                  U5      XR                  SS9u  n  nU R                   R
                  R                  SS9  U$ NFwait)rj  	add_chunkr  id_hashrz   r  async_response)rj   rM  r6  r-  s       rk   rN  CacheChunkBuffer.write_chunk~  sW    JJ(()9)9%)@%Z_(`	Q

,,%,8
rn   )rj  rz   )	r   r   r   r   re  rl   rN  r   __classcell__rk  s   @rk   rg  rg  w  s    9M 
 rn   rg  )
uid_forced
gid_forceduid_defaultgid_defaultc                    Ub  UnO7U(       a  S O[        U R                  5      nUc  U R                  OUnUS:  a  UnUb  UnXg4$ U(       a  S O[        U R                  5      nUc  U R
                  OUnUS:  a  UnXg4$ r^   )r.   useruidr0   groupgid)r   numericrw  rx  ry  rz  r}  r  s           rk   get_item_uid_gidr    s    d8DII#6+dhh37C 8O	 d9TZZ#8+dhh37C8Orn   c                      \ rS rSr " S S\5      r " S S\5      r " S S\5      rSS	S
S	S	S	S	S	S	S	\SSSS	S	S	4S jr	S r
S r\S 5       r\S 5       r\S 5       r\S 5       r\S 5       rS rS rS rS%S jrS&S jrS'S jrS rS(S jrS)S jrS)S jr\S 5       r  S*S jrS+S jrS  r S! r!S,S" jr"\#S-S# j5       r$S$r%g).Archivei  c                       \ rS rSrSrSrSrg)Archive.AlreadyExistsi  zArchive {} already exists   r   Nr   r   r   r   __doc__
exit_mcoder   r   rn   rk   AlreadyExistsr    
    '
rn   r  c                       \ rS rSrSrSrSrg)Archive.DoesNotExisti  zArchive {} does not exist   r   Nr  r   rn   rk   DoesNotExistr    r  rn   r  c                       \ rS rSrSrSrSrg)+Archive.IncompatibleFilesystemEncodingErrori  zrFailed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.    r   Nr  r   rn   rk   #IncompatibleFilesystemEncodingErrorr    s     	A
rn   r  NF  c                 \   [         R                  " 5       U l        X l        Xl        XPl        X0l        0 U l        [        UUS9U l	        UU l
        Xl        X@l        X@l        S U l        SU l        Xpl        Xl        Xl        Xl        Xl        Xl        Xl        US L US L :X  d   S5       eUc  [/        5       n[0        R2                  " 5       nXl        UU l        UU l        Uc
  [/        5       nUU l        UU l        [?        U R                  U R                  5      U l         X`l!        U RB                  (       a  [E        U R
                  U R                  U R                  5      U l#        XCRH                  ;   a  U RK                  U5      eSn SRM                  UU=(       a    SU-  =(       d    S5      U l'        U RN                  URH                  ;  a  g US-  nMN  U R                  RH                  RQ                  U5      nUc  U RS                  U5      eU RU                  URV                  5        g )	Nr_   r`   FzULogic error: if start is given, start_monotonic must be given as well and vice versa.r   z{}.checkpoint{}z.%dr   r   ),r	  getcwdcwdr  r  rj  manifest
hard_linksr[   rz   r`   r   r  name_in_manifestcommenttam_verifiedcheckpoint_intervalnumeric_idsr  noctimenoflagsnoaclsnoxattrsrH   r   r   rD  startstart_monotonicr   consider_part_filesr  pipelinecreaterg  items_bufferarchivesr  r   checkpoint_namer   r  loadr   )rj   r  r  r  r  rj  r  r  r  r  r  r  r  r  progressrD  r  r  r   r  log_jsonr`   iinfos                           rk   rl   Archive.__init__  s   
 99;$
 H#>
%	 $!#6 & ?d#:;  	U  >U  	U;=HE"nn.O,
.;(C#6 ($((C;; 0TXXtzz RD(((((..A'8'?'?aFWUUVYF][]'^$''x/@/@@Q	  ==))--d3D|''--IIdggrn   c                     U R                   R                  XR                  R                  U5      5      nU R                   R	                  USS9u  o0l        n[        US9nUR                  S:w  a  [        S5      eU$ )NTforce_tam_not_requiredr  r    Unknown archive metadata version)	r  r5  r  r   unpack_and_verify_archiver  rN   version	Exception)rj   r   r   archiver-  metadatas         rk   
_load_metaArchive._load_meta  sp    xxOO$7$7$;<(,(J(J4hl(J(m%"AW5q >??rn   c                 P   Xl         U R                  U R                   5      U l        U R                  R                   Vs/ s H  n[	        U5      PM     snU R                  l        U R                  R
                  U l        U R                  R                  SS5      U l        g s  snf )Nr  r   )r   r  r  cmdliner9   r  r   r  )rj   r   args      rk   r  Archive.load  sv    0=A]]=R=R S=RcS!1=R SMM&&	}}((B7 !Ts   B#c                 D    U R                   R                  n[        U5      $ )z,Timestamp of archive creation (start) in UTC)r  r   r1   rj   tss     rk   r  
Archive.ts  s     ]]r""rn   c                     U R                   R                  S5      =(       d    U R                   R                  n[        U5      $ )z*Timestamp of archive creation (end) in UTCtime_end)r  r   r   r1   r  s     rk   ts_endArchive.ts_end  s2     ]]z*@dmm.@.@r""rn   c                 ,    [        U R                  5      $ rp   )r=   r   r   s    rk   fprArchive.fpr  s    $''""rn   c                 F    [        U R                  U R                  -
  5      $ rp   )r4   r   r  r   s    rk   durationArchive.duration  s    4:: 566rn   c                 F    [        U R                  U R                  -
  5      $ rp   )r4   r  r  r   s    rk   duration_from_metaArchive.duration_from_meta  s    dgg 566rn   c           	         U R                   (       a[  U R                  nU R                  R                  [        R
                  S9nU R                  R                  [        R
                  S9nO3U R                  U R                  5      nU R                  nU R                  nU R                  U R                  [        U5      [        U5      X2-
  R                  5       UR                  5       SU R                  R                   U R"                     R$                  [&        -  0S.nU R                   (       a  [(        R*                  US'   U$ U R,                  R/                  S5      nUb  [1        U5      OSnUR3                  U R,                  R4                  U R,                  R6                  U R,                  R8                  U R,                  R/                  SS5      US.5        U$ )	Ntzinfomax_archive_size)r  r   r  r   r  rz   limitscommand_linerD  r   r  )r  rU   usernamer  rD  )r  rz   r  replacer   utcr   
calc_statsrj  r  r  r  r  r3   total_secondsr   r  r   rb   MAX_DATA_SIZEr   argvr  r   r)   rt   r  rU   r  )rj   rz   r  r   r  cps         rk   r  Archive.info  sl   ;;JJEJJ&&hll&;E((""(,,"7COODJJ/EGGE++CII(($U+"3'335]]_"DJJ$5$5dgg$>$D$D}$T

 ;;#&88D   ""#34B13)"-RBKK $ 5 5 MM22 MM22==,,Y;"$  rn   c           
         SR                  U [        U R                  R                  [        R
                  S95      [        U R                  R                  [        R
                  S95      U R                  R                  U R                     R                  [        -  U R                  R                  R                  5       S9$ )NzRepository: {location}
Archive name: {0.name}
Archive fingerprint: {0.fpr}
Time (start): {start}
Time (end):   {end}
Duration: {0.duration}
Number of files: {0.stats.nfiles}
Utilization of max. archive size: {csize_max:.0%}
r  )r  r   	csize_maxlocation)r   r3   r  r  r   r  r   rj  r  r   rb   r  r  	_locationcanonical_pathr   s    rk   r   Archive.__str__%  s    	 F!$**"4"4HLL"4"IJ 0 0 0 EFjj''066F__..==?  	rn   c                      SU R                   -  $ )NzArchive(%r))r  r   s    rk   r   Archive.__repr__7  s    tyy((rn   c                 T    U R                   (       d  SU;   a  gU(       a  U" U5      $ S$ )Nrs   FT)r  )rj   r   r&  s      rk   item_filterArchive.item_filter:  s%    ''FdN%vd|/4/rn   c              #      ^ ^#    T(       a  U(       a  U(       a  Uc   eT R                   R                  T R                  R                  UX4UU 4S jS9 H  nUv   M	     g 7f)Nc                 (   > TR                  U T5      $ rp   )r  )r   r&  rj   s    rk   <lambda>$Archive.iter_items.<locals>.<lambda>F  s    $BRBRSWY_B`rn   )r'  r  r(  r&  )r  r.  r  r+  )rj   r&  r'  r  r(  r   s   ``    rk   
iter_itemsArchive.iter_items@  sW      7?O?[[[MM--dmm.A.ASb6=5` . bD Jbs   AA c                     U(       a0  U R                   (       a  Uc  U R                  nUR                  USS9  U R                  R                  U5        g )N皙?r   r   )r   rz   r  r$  )rj   r   r   rz   s       rk   add_itemArchive.add_itemI  sB    T//}

Tc2d#rn   c                     U R                  U R                  5        U R                  R                  U R                  	 U R                  R                  U R                  U R                  5        g rp   )saver  r  r  rj  chunk_decrefr   rz   r   s    rk   write_checkpointArchive.write_checkpointP  sH    		$&&'MM""4#7#78

4rn   c                    U=(       d    U R                   nXR                  R                  ;   a  U R                  U5      eU R                  R                  SS9  [        [        R                  " 5       U R                  -
  S9nUc  [        5       nXv-
  nOX6-   nUnXl        Xpl        SUU=(       d    SU R                  R                  [        R                  [         [#        5       UR%                  [&        5      UR%                  [&        5      U R(                  S.
n	U=(       d    U R*                  nU	R-                  UR.                  UR0                  UR2                  UR4                  UR6                  UR8                  S.5        U	R-                  U=(       d    0 5        [;        U	5      n	U R<                  R?                  U	RA                  5       SS	9n
U R<                  RC                  U
5      U l"         U RF                  RI                  U RD                  XR*                  5        U RP                  RS                  SS9b   U RP                  RS                  SS9b  M  U RD                  U	R                  4U R                  R                  U'   U R                  RU                  5         U RP                  RW                  SS9  U RF                  RW                  5         g ! [J         a%  n[M        U5      nS
U;   a  [O        SU-  5      ee S nAff = f)NTr   )secondsr   r   )
r  r  r  r+  r  rU   r  r   r  rD  r      archivecontextzMore than allowed put dataz#%s - archive too big (issue #1473)!ro  F)compact),r  r  r  r  r  r   r   r   r   r  rH   r  r   r  r   r  rU   r   strftime
ISO_FORMATrD  rz   rt   ra   rb   rd   re   rf   rh   rN   r  pack_and_authenticate_metadatar   rr  r   rj  rq  r   strr+   r  rs  rG  commit)rj   r  r  	timestamprz   additional_metadatar  r   r  r  r   errerr_msgs                rk   r  Archive.saveU  so    tyy==)))$$T**d+T^^%58L8L%LM(CNE&CE
}"&&--xx 	NN:.Z0"11
 #KK[[ll++ ,,!..0 	1 	+1r2x(xx66x7G7G7IS]6^((""4(	JJ  $

; oo,,$,7C oo,,$,7C(,'?t$u-

  	#hG+w6AGKLL	s   10K 
L K??Lc                 R   U R                   R                  S5      S Ln U(       a  [        e[        R                  " S0 UR
                  U R                     D6nU$ ! [         aA    U R                  XS9nU(       d'  UR                  5       UR
                  U R                  '    U$ f = f)Nrd   )want_uniquer   )	r  r   KeyErrorr[   r   
pre12_metar  _calc_statsr   )rj   rj  r  have_borg12_metarz   s        rk   r  Archive.calc_stats  s    ==,,X6dB	A
 ,,Ju/?/?/IJE
 	  	A$$U$DE#-2->->-@  *		As   :A AB&%B&c                 Z  ^^ U R                   R                  S5      S LnU(       a  U(       d  SnGO<UU4S jn[        5       m[        T5      nU" U R                  5        U R
                  R                  SS5      n[        [        U R                   R                  5      SU-  SS9n[        U R                   R                  U R                  R                  U R                   R                  5      5       HH  u  pUR                  S	S
9  U" U	5        U R                  R                  X5      nUR!                  U5        MJ     TR#                  TR$                  5      S   nUR'                  5         [)        U R*                  S9nXLl        U(       d  U R.                  (       a5  WR0                  Ul        UR4                  Ul        UR8                  Ul        U$ WR0                  UR<                  -
  Ul        UR4                  UR>                  -
  Ul        UR8                  UR@                  -
  Ul         U$ U R.                  (       a  U R                   RB                  U R                   R2                  -   Ul        U R                   R>                  U R                   RD                  -   Ul        U R                   R@                  U R                   R:                  -   Ul        U$ U R                   R2                  Ul        U R                   RD                  Ul        U R                   R:                  Ul        U$ )Nrd   r   c                 t   > TR                   U    nTR                  U SUR                  UR                  5        g )Nr   )r  r$  rq   rb   )r   entryarchive_indexrj  s     rk   r$   Archive._calc_stats.<locals>.add  s-    R(!!"aU[[Arn   %z%%z4Calculating statistics for archive %s ... %%3.0f%%%%zarchive.calc_statstotalr   msgidr   )increase   r   )#r  r   r#   r%   r   r  r  r@   r_  r+  r3  r  r4  showr  r5  r#  stats_againstr  finishr[   r`   rc   r  num_files_totalsrd   size_totalsra   csize_totalsrb   num_files_partsr   rf   rh   rq   )rj   rj  r  r  unique_csizer$  syncarch_name_escdpir   rM  r   rz   r  s    `           @rk   r  Archive._calc_stats  sb   ==,,X6dBKLB 'LM$]3DL!YY..sD9N)DMM4G4G0H.dgu.u0DFB !!4!4doo6N6Nt}}ObOb6cd	#Bxx''2		$	 e
 )66u||DQGLIIKtxx("''#44".."//   $44t7K7KK"..@"//$2B2BB  ''#}}99DMM<P<PP"mm669K9KK"mm77$--:M:MM
   $}}33"mm00"mm11rn   c              #      #    SnSU;   a  [         R                  R                  " U/UR                  R	                  [         R
                  5      US  Q76 nUR                  UR                  S U45      u  pU
(       a8  [        (       a-  [        S5         [         R                  " X5        SnS S S 5        O	U	b  Xl
        Uv   U(       d3  U(       a+  [        (       a  S U4XbR                  S5      =(       d    U'   g g g g ! , (       d  f       NM= f7f)NFr  rY   T)r	  r   joinr  splitsepr   has_linkr   rY   r  )rj   destr   r   stripped_componentsoriginal_pathr(  hardlink_setr  r  link_targets              rk   extract_helperArchive.extract_helper  s     tWW\\$Y):):266)BCVCW)XYF"2"6"6t{{T6N"SFxxv&GGK.#'L '& #$ 0x JNt (!3!D}E  !1| '&s   BDC=,AD=
DDc
           	      N   U=(       d    0 nSU;   n
U(       d  U(       Ga3  SU;   Ga  SnU R                   R                  UR                   Vs/ s H  oR                  PM     snSS9 Hu  nU	(       a-  U	R	                  [        U5      [        UR                  5      /S9  U(       a)  [        R                  R                  R                  U5        U[        U5      -  nMw     U(       a(  [        R                  R                  R                  5         SU;   a+  UR                  nX:w  a  [        SR                  X5      5      eU
(       a  [        S	5      eg
U=(       d    UR                  nU R                   nUR                  R#                  S5      (       a  [%        S5      e[&        R                  R)                  XR                  5      n [&        R*                  " USS9n[*        R,                  " UR.                  5      (       a  [&        R0                  " U5        O[&        R2                  " U5         S nUR<                  n[*        R>                  " U5      (       Ga  [A        S5         U" U5        S
S
S
5        U RC                  XUXxU5       nU(       a
   S
S
S
5        g
[A        S5         [E        US5      nS
S
S
5        W   UR                   Vs/ s H  oR                  PM     nnU R                   R                  USS9 H  nU	(       a-  U	R	                  [        U5      [        UR                  5      /S9  [A        S5         U(       a6  [F        R#                  U5      (       a  URI                  [        U5      S5        OUR                  U5        S
S
S
5        M     [A        S5         URK                  5       =nnURM                  U5        UR                  5         U RO                  UUURQ                  5       S9  S
S
S
5        S
S
S
5        SU;   a,  UR                  nUW:w  a  [        SR                  X5      5      eU
(       a  [        S	5      e S
S
S
5        g
[@           [*        R,                  " U5      (       a]  U" U5        [&        R                  RS                  U5      (       d  [&        RT                  " U5        U(       a  U RO                  UU5        GO[*        RV                  " U5      (       a?  U" U5        URX                  n [&        RZ                  " UU5        U RO                  UUSS9  GOM[*        R\                  " U5      (       ag  U" U5        U RC                  XUXxU5       nU(       a   S
S
S
5        S
S
S
5        g
[&        R^                  " U5        U RO                  UU5        S
S
S
5        O[*        R`                  " U5      (       d  [*        Rb                  " U5      (       a}  U" U5        U RC                  XUXxU5       nU(       a   S
S
S
5        S
S
S
5        g
[&        Rd                  " UUR<                  URf                  5        U RO                  UU5        S
S
S
5        O[%        SUR<                  -  5      eS
S
S
5        g
s  snf ! [4         a'    U R7                  U[        R8                  " 5       5      S
e[:         a     GNf = f! , (       d  f       GNd= f! , (       d  f       GN0= fs  snf ! , (       d  f       GM  = f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       g
= f! [4         a'    U R7                  U[        R8                  " 5       5      S
ef = f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       g
= f)au  
Extract archive item.

:param item: the item to extract
:param restore_attrs: restore file attributes
:param dry_run: do not write any data
:param stdout: write extracted data to stdout
:param sparse: write sparse files (chunk-granularity, independent of the original being sparse)
:param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly
:param stripped_components: stripped leading path components to correct hard link extraction
:param original_path: 'path' key as stored in archive
:param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes)
chunks_healthyr  r   Tr1  )r  r  rq   z4Size inconsistency detected: size {}, chunks size {}zDFile has damaged (all-zero) chunks. Try running borg check --repair.N)/z../z!Path should be relative and localFfollow_symlinksc                     [         R                  R                  U 5      n[         R                  R                  U5      (       d  [         R                  " U5        g g rp   )r	  r   dirnameexistsmakedirs)r   
parent_dirs     rk   make_parent)Archive.extract_item.<locals>.make_parent  s8    .J77>>*--J' .rn   r?  r  wbrG  r   truncate_and_attrsr  )symlinkzUnknown archive item type %r)4r  r"  r  r   r  r_  r;   r   r   stdoutr?  rG  r   rq   r   r   r  
startswithr  r	  r,  r   S_ISDIRr   rmdirunlinkUnicodeEncodeErrorr  getfilesystemencodingr   r   S_ISREGr   r5  r  r[  rT  rS  r^  restore_attrsfilenor>  mkdirS_ISLNKr  rF  r   mkfifor   r   mknodrdev)rj   r   rO  dry_runrG  r>  r(  r1  r2  r)  has_damaged_chunksitem_chunks_sizer  r   	item_sizer0  r   strA  r   r3  r  r%  posr  s                            rk   extract_itemArchive.extract_item  s    ,1r-5f4#$  MM44DKK5PKqddK5P_c4dDT:KDII:V9WX

))//5$D	1$ e JJ%%++-T> $		I 4)*`*g*g*3+G H H!!"hii%2xx99--?@@ww||D)),		u5B||BJJ''		$	(
 yy<<:&D! '$$T7J%57:F7 7 v&dD)B ')-5A44C5 $ 8 84 8 PGGSY>OPTPYPY>Z=[G\&w/%%*:*:4*@*@ "D	1 5 " 0/ !Q ##7813:.C(
**4"))+*F	 9   T> $		I $44)*`*g*g*3+G H H%%&lmm &77: ||D!!D!ww~~d++HHTN &&tT2d##D!rJJvt, ""4t"<t$$D!((T;N)9;>J#;% Y, IIdO&&tT2; ; d##t||D'9'9D!((T;N)9;>J#;5 Y< HHT499dii8&&tT2; ;   > JKKC YS 6Q: " 	h::4AZAZA\]cgg 		 '& '& 6 0/ 98 R7 7:  * rBB63KdKdKfgmqqr
; ;; ;5 Ys"  Z	AZ Z '	[	\-\-)[6
\- \[1$A\A[6	\(A\	;\A\-B(^\>A
^)	]22^(]2+A^	^^ >^!^9[
	[

[
[.	)\-1\6
\ 	\	
\\
\*	%\--
\;>1]//^2
^	<^
^	^
^$c                 ^   S[         l        [        X R                  S9u  pV[        (       Gd   U(       a  [
        R                  " XEU5        O[
        R                  " XUSS9   U(       a!  [
        R                  " XBR                  5        O  [
        R                  " XR                  SS9  U R                  (       d   [        XU R                  US9  U R"                  (       dE  [$        R&                  " U=(       d    UUR)                  S0 5      SS9nU(       a  [+        [,        5        UR.                  n	SU;   a  UR0                  n
OU	n
S	U;   aE  UR2                  n U(       a  [
        R4                  " USX4S
9  O[
        R4                  " USX4SS9    U(       a  [
        R4                  " USX4S
9  O[
        R4                  " USX4SS9   U R6                  (       d  SU;   a   [9        XR:                  US9  gggg! [         a     GNf = f! [         a+    U(       d   [
        R                  " XR                  5         GNf = f! [         a+  nUR                  [        R                   4;  a  e  SnAGNSnAff = f! [         a     Nf = f! [         a     Nf = f! [         a     gf = f)z^
Restore filesystem attributes on *path* (*fd*) from *item*.

Does not access the repository.
attrs)r  Fr:  rE  Nxattrsatime	birthtime)ns)rc  r;  bsdflags)r   r   r  r  r   r	  fchownchownr   fchmodr   chmodrL  r  rQ   r   ENOTSUPr  r   set_allr   r,   EXIT_WARNINGmtimera  rb  utimer  rR   rd  )rj   r   r   rF  r  r}  r  r,  warningrl  ra  rb  s               rk   rO  Archive.restore_attrsi  s+    	#D2B2BCxIIbs+HHTUC 		"ii(2HHT99eD ;;D(8(8R@ ==  --
dDHHXr4J\ab<(JJE$

 d" NN		 Tu.@At0BTYZHHR5.9HHT4UNER
 <<J$$6dMMb9 %7<}    + 2"yy12  wwu}}&77 82      s   G> G> H >I I? <I? J 5J &J >
HH1II
I< I77I<?
JJ
JJ
J,+J,c                    U R                  U R                  5      n[        X1U5        U R                  R	                  UR                  5       SS9nU R                  R                  U5      nU R                  R                  XTU R                  5        XSR                  4U R                  R                  U R                  '   U R                  R                  U R                  U R                  5        XPl        g )Nr  r   )r  r   setattrr  r  r   rr  rj  rq  rz   r   r  r  r  r  )rj   r  valuer  r   new_ids         rk   set_metaArchive.set_meta  s    ??477+u%xx66x7G7G7IS]6^!!$'

V4::6-3]],Ctyy)

4rn   c                     XR                   R                  ;   a  U R                  U5      eU R                  nXl        U R	                  SU5        U R                   R                  U	 g )Nr  )r  r  r  r  rt  )rj   r  oldnames      rk   renameArchive.rename  sQ    ==)))$$T**))	fd#MM""7+rn   c           	      h  ^ ^^^^^  " S S[         5      m[        5       mSUUUU 4S jjmSUUU 4S jjnSm [        R                  " SS9nT R                  R
                  n[        [        U5      SS	S
9n[        [        UT R                  R                  U5      5      5       H  u  nu  pU(       a  UR                  U5        T R                  R                  X5      n
UR                  U
5        U" X5         U HN  n[!        US9nSU;   d  M  T R"                  (       + =(       a    SU;   nUR$                   H  u  pnU" XUS9  M     MP     M     U(       a  UR+                  5         U" T R2                  U5        T R4                  R6                  T R8                  	 T" SS9b
   T" SS9b  M
  T(       a+  [:        R=                  S5        [:        R=                  S5        g g ! [&        [(        4 a    TS:X  a  e Sm GMY  f = f! [        R,                  [.        R0                  4 a    TS:X  a  e Sm Nf = f)Nc                       \ rS rSrSrSrg)(Archive.delete.<locals>.ChunksIndexErrori  zUChunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.r   N)r   r   r   r   r  r   r   rn   rk   ChunksIndexErrorr|    s    grn   r}  Tc                    >  TR                   R                  U S9$ ! [        R                   a    TS:X  a  e SmTs $ f = f)Nro  r   T)r  rs  rW   ObjectNotFound)rp  errorexception_ignoredforcedrj   s    rk   fetch_async_response,Archive.delete.<locals>.fetch_async_response  sL    )5545@@,, ) Q;(()s    >>Fc                    >  TR                   R                  XSUS9  T" SS9  g ! [         a    [        U 5      nT" U5      ef = f)NF)rp  rs   ro  )rj  r  r  r=   )r   rz   rs   cidr}  r  rj   s       rk   r  $Archive.delete.<locals>.chunk_decref  sP    1

''D'I
 %%0	  , n&s++,s	   % Ar  zDecrementing references %3.0f%%zarchive.deleter  r  r  rs   )rs   r   ro  zAforced deletion succeeded, but the deleted archive was corrupted.z2borg check --repair is required to free all space.Tr   )r+   objectrF   r!  r  r+  r@   r_  	enumerater3  r  r4  r  r  r5  r#  rM   r  r  rx   r\  r!  UnpackExceptionrW   r  r   r  r  r  loggerrn  )rj   rz   r  r  r  r*  	items_idsr)  r  items_idr   r   rs   chunk_idrq   rb   r}  r  r  r  s   `  `            @@@@rk   deleteArchive.delete  s   	hu 	h #H		) 		)	1 	1 	''7H++I)IDem}~B)23y$//BZBZ[dBe3f)g%$XGGAJxx''7d#X-! (#$7#t+'+'?'?#?#RFdND9= 5 ,X4 H :E	 !) *h& 		 	TWWe$MM""499-"-9  #-9 NN^_NNOP - ":. ! { E! '')B)BC 	{E		s<   C H 9G >G H  G>9H =G>>H -H10H1c              #      ^ ^^^^^^^^^^#    U4S jmS mUU4S jmS mU UUUUU4S jnUUU4S jn[        5       n[        5       n/ m0 m[        T R                  U4S j5      TR                  U4S j5      5       GH  u  pU	(       aF  U
(       a?  U	R                  U
R                  :X  a%  U" X5      (       d  U	R                  U" X5      4v   MS  U	(       aU  UR	                  U	R                  S	5      nU(       a$  U" X5      (       d  U	R                  U" X5      4v   OXU	R                  '   U
(       d  M  UR	                  U
R                  S	5      nU(       a)  U" X5      (       d  UR                  U" X5      4v   GM  GM  XU
R                  '   GM     UR                  5        H8  nUR                  n[        R                  " U5      nT" X5        X" X5      4v   M:     UR                  5        H8  nUR                  n[        R                  " U5      nT" X5        X" X5      4v   M:     T HZ  u  pT" U	5      (       d   eT" U
5      (       d   eU	R                  U
R                  :X  d   S
5       eU	R                  U" X5      4v   M\     g	7f)a  
Yields tuples with a path and an ItemDiff instance describing changes/indicating equality.

:param matcher: PatternMatcher class to restrict results to only matching paths.
:param can_compare_chunk_ids: Whether --chunker-params are the same for both archives.
c                 |   > SU ;  =(       d0    [        U R                  5      (       + =(       d    U R                  T;   $ Nr  )r'   r   r  r   r(  s    rk   hardlink_master_seen;Archive.compare_archives_iter.<locals>.hardlink_master_seen  s.    4'i|DII/F+Fi$++YiJiirn   c                 v    U R                  SS5      =(       a!    SU ;  =(       a    [        U R                  5      $ Nr  Tr  )r   r'   r   r   s    rk   is_hardlink_master9Archive.compare_archives_iter.<locals>.is_hardlink_master  s0    88-t4i9MiR^_c_h_hRiirn   c                 \   > T" U 5      (       d  T" U5      (       a  X4TU R                   '   g g rp   r   )item1item2r(  r  s     rk   update_hardlink_masters>Archive.compare_archives_iter.<locals>.update_hardlink_masters  s0    !%((,>u,E,E05~ , -Frn   c                 b    [        U R                  5      =(       a    U R                  S5      U;   $ r  r'   r   r   r  s     rk   has_hardlink_master:Archive.compare_archives_iter.<locals>.has_hardlink_master!  s%    		*Utxx/AEU/UUrn   c                   > T" U T5      (       a  TU R                      S   n T" UT5      (       a  TUR                      S   n[        XTR                  R                  U R	                  S/ 5       Vs/ s H  o"R
                  PM     sn5      TR                  R                  UR	                  S/ 5       Vs/ s H  o"R
                  PM     sn5      TTS9$ s  snf s  snf )Nr   r   r  )can_compare_chunk_idscontent_only)r  rO   r  r"  r   r   )	r  r  r  archive1archive2r  r  r(  r  s	      rk   compare_items4Archive.compare_archives_iter.<locals>.compare_items$  s    "5*:;;(6q9"5*:;;(6q9E$--88		RZ\^H_9`H_1$$H_9`a$--88		RZ\^H_9`H_1$$H_9`a2G)5	7 79`9`s   1C5Cc                    > T" X5        T" U 5      (       + =(       d    T" U5      (       + nU(       a  TR                  X45        U$ )zQAdds item tuple to deferred if necessary and returns True, if items were deferred)r]  )r  r  deferdeferredr  r  s      rk   defer_if_necessary9Archive.compare_archives_iter.<locals>.defer_if_necessary/  s=    #E1,U33V;OPU;V7VE/Lrn   c                 :   > TR                  U R                  5      $ rp   matchr   r   matchers    rk   r  /Archive.compare_archives_iter.<locals>.<lambda>=      tyy1Irn   c                 :   > TR                  U R                  5      $ rp   r  r  s    rk   r  r  >  r  rn   Nz#Deferred items have different paths)r   r   r  r   popvaluesrM   create_deleted)r  r  r  r  r  r  r  orphans_archive1orphans_archive2r  r  matching_orphanaddedr   deleted_itemdeletedr  r  r(  r  r  r  s   `````           @@@@@@rk   compare_archives_iterArchive.compare_archives_iter  s8    	j	j	>	V		7 		7	 '=&='##$IJ##$IJ
LE 5::#;)%77 ::}U'BCC"2"6"6uzz4"H"-eEE$zz=+PQQ38UZZ0u"2"6"6uzz4"H"-oEE.33]?5Z[[ F 49UZZ0+
. &,,.E::D..t4L#L8|;<<	 /
 (..0G<<D..t4L#G:w=>>	 1
 %LE'....'....::+R-RR+::}U:;;	 %s   D%J6EJ)rj  r  r  rD  r  r  r  r  r   r  r   r`   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  rz   r  rp   r8  )TN)NNNNNr  )TFFFNr   NNFNr   )NFF)&r   r   r   r   r+   r  r  r  CHUNKER_PARAMSrl   r  r  r   r  r  r  r  r  r  r   r   r  r  r  r  r  r  r  r   r5  r\  rO  rt  rx  r  staticmethodr  r   r   rn   rk   r  r    sQ    u e  ?C5%)ueUZuud\`fj%*U	4l8 # #
 # # # # 7 7 7 7 D$)0$5
7r".`  2 Z_Z^~L@KZ,DQL R< R<rn   r  c                   4    \ rS rSrS rS rSS jrSS jrSrg)	MetadataCollectorid  c                X    Xl         X l        X@l        XPl        X`l        Xpl        X0l        g rp   )r  r  r  r  r  r  nobirthtime)rj   r  r  r  r  r  r  r  s           rk   rl   MetadataCollector.__init__e  s'    & &rn   c                 J   [        UR                  UR                  UR                  [	        UR
                  5      S9nU R                  (       d  [	        UR                  5      US'   U R                  (       d  [	        UR                  5      US'   U R                  (       d5  [        US5      (       a$  [	        [        UR                  S-  5      5      US'   U R                  (       a  S =US'   US'   U$ [        UR                  5      US'   [!        UR                  5      US'   U$ )	N)r   r}  r  rl  ra  ctimest_birthtime ʚ;rb  r|  r~  )dictr   st_uidst_gidr>   st_mtime_nsr  st_atime_nsr  st_ctime_nsr  hasattrintr  r  r-   r/   )rj   rZ  r_  s      rk   stat_simple_attrs#MetadataCollector.stat_simple_attrsn  s    				"..)	
 ||$R^^4E'N||$R^^4E'NGB$?$?!(R__u-D)E!FE+-11E&ME'N  %RYY/E&M&ryy1E'Nrn   Nc           	      ~   0 nU R                   (       d)  [        S5         [        X!US9nS S S 5        W(       a  XTS'   U R                  (       dG  [        S5         [        R
                  " U=(       d    USS9nS S S 5        W(       a  [        U5      US'   U R                  (       d,  [        S5          [        X$XR                  US9  S S S 5        U$ U$ ! , (       d  f       N= f! , (       d  f       Ns= f! [         a*  nUR                  [        R                  4;  a  e  S nAN]S nAff = f! , (       d  f       U$ = f)	Nzextended stat (flags)rE  rd  zextended stat (xattrs)Fr:  r`  zextended stat (ACLs))r  r   rS   r  r   get_allr<   r  rP   r  r   r   ri  )rj   rZ  r   r  r_  r  r`  r,  s           rk   stat_ext_attrs MetadataCollector.stat_ext_attrs  s    ||23!$r2 4$)j!}}34rzT5I 5",V"4h{{12D-=-="E 3 u! 43
 54  wwu}}&77 8 32 sG   CC%1D-3C6
C"%
C36
D*  D% D-%D**D--
D<c                 f    U R                  U5      nUR                  U R                  XUS95        U$ )NrE  )r  rt   r  )rj   rZ  r   r  r_  s        rk   
stat_attrsMetadataCollector.stat_attrs  s3    &&r*T((b(9:rn   )r  r  r  r  r  r  r  rp   )	r   r   r   r   rl   r  r  r  r   r   rn   rk   r  r  d  s    '0*rn   r  
   c                     g rp   r   )r-  s    rk   r  r    s    rn   )disposec                 h   U R                   S   nU[        :X  a  U R                  nU" U5      nXC4$ U[        [        4;   aD  U R                   S   nU[        [        5      ::  d   e[        [        5      S U n [        X4   nXC4$ [        S5      e! [         a    U" U5      nU[        X4'    XC4$ f = f)NrQ  rq   zunexpected allocation type)rV  rW  r   rZ  rY  r_  r[  
memoryviewzero_chunk_idsr  r\  )rM  rr  rQ  r   r  rq   s         rk   cached_hashr    s    L)JWzz4= > 
*	*zz&!s5z!!!% $'	7%wo6H > 566	  	7t}H.6NG?+ >	7s   8
B B10B1c                   4    \ rS rSrS rS rSS jrS	S jrSrg)
ChunksProcessori  c                    Xl         X l        X0l        X@l        XPl        [
        R                  " 5       U l        X`l        g rp   )	r  rj  r  r  r  r   r   last_checkpoint
rechunkify)rj   r  rj  r  r  r  r  s          rk   rl   ChunksProcessor.__init__  s3     
  0#6 #~~/$rn   c                 .   [        UR                  5       S9n[        UR                  5      nUR                  US  Ul        UR	                  SSS9  U=R
                  SU-  -  sl        X1l        US-  nU R                  USS9  U R                  5         XC4$ )Nr  T)memorizefrom_chunksz.borg_part_%dr   F)r   )	rM   r   r_  r  get_sizer   rs   r  r  )rj   r   
from_chunknumberlengths        rk   write_part_fileChunksProcessor.write_part_file  s    $,,.1T[[!kk*+. 	t6		_v--		!d%0~rn   c                    [         =(       a    [         R                  " 5       nU(       dH  U(       dA  U R                  (       a  [        R                  " 5       U R
                  -
  U R                  :  a{  U(       a  [        R                  S5        U R                  XU5      u  p#[        R                  " 5       U l        U(       a*  [         R                  " 5         [        R                  S5        X#4$ )Nz5checkpoint requested: starting checkpoint creation...z3checkpoint requested: finished checkpoint creation!)
rG   action_triggeredr  r   r   r  r  r  r  action_completed)rj   r   r  part_numberr  sig_int_triggereds         rk   maybe_checkpoint ChunksProcessor.maybe_checkpoint  s    #B(@(@(B&$$)9D<P<P)PSWSkSk)k ST&*&:&:4[&Y#J#'>>#3D  ((*QR&&rn   Nc                   ^ ^^ U(       d  UU U4S jn/ Ul         T R                  (       a  SU;   a  U?SnSnU HN  n	UR                   R                  U" U	5      5        U(       a  TR	                  USS9  T R                  XUSS9u  pxMP     US:  ay  UR                   US  (       a  T R                  XUS	S9u  pxUR                    H)  n	TR                  U	R                  TU	R                  S	S
9  M+     T=R                  US-
  -  sl	        g g )Nc                    > [        U TR                  R                  5      u  pTR                  XTSS9nTR                  R
                  R                  SS9  U$ rn  )r  r  rr  rq  rj  r  rs  )rM  r  r   chunk_entryrj  rj   rz   s       rk   chunk_processor<ChunksProcessor.process_file_chunks.<locals>.chunk_processor  sP    !,UDHH4D4D!E#oohe%oP

%%44%4@""rn   r8  r   r   r  r  F)r  T)rq   rs   )
r  r  r8  r]  r   r  chunk_increfr   rq   rh   )
rj   r   rj  rz   r   
chunk_iterr  r  r  rM  s
   ` ``      rk   process_file_chunks#ChunksProcessor.process_file_chunks  s   #  ??/47#
EKKu56####6&*&;&;Dkbg&;&h#J	   Q;;z{+ /3.C.CDVajn.C.o+J "[[E&&uxxUZZd&S )""kAo5" rn   )r  rj  r  r  r  r  r  r   rp   )	r   r   r   r   rl   r  r  r  r   r   rn   rk   r  r    s    	%' 6rn   r  c                   h    \ rS rSrSS.S jr\SS j5       rS rS rS r	S	 r
S
 rS r\S.S jrSrg)FilesystemObjectProcessorsi  Nfile_status_printerc                    Xl         X l        X0l        X@l        XPl        Xpl        U=(       d    S U l        0 U l        [        XS9U l	        [        R                  " 5       U l        [        XcR                  US.6U l        g )Nc                      g rp   r   argss    rk   r  5FilesystemObjectProcessors.__init__.<locals>.<lambda>      trn   r  r<  )metadata_collectorrj  r  r  r  r   print_file_statusr  r[   rz   r	  r  r  r   rB  rC  )rj   r  rj  r  r  r  rD  r   r>  r  r`   r  s               rk   rl   #FilesystemObjectProcessors.__init__  sf     #5
 #6 *!4!L9KH>
99;"NPVWrn   c              #   V  #    Ubc  UR                  [        R                  5      (       a   eUR                  U[        R                  -   5      (       a  Sv   g UR	                  U5      n[        U5      n[        US9nSnU=(       a    UR                  S:  n	U	(       a?  U R                  R                  UR                  UR                  45      n
U
b	  Xl        SnOSnXsX4v   U R                  XpR                  S9  U(       a%  X`R                  UR                  UR                  4'   g g 7f)N)NxFFr  Fr   hTrz   )endswithr	  r.  rH  removeprefixr:   rM   st_nlinkr  r   r   st_devr  r  rz   )rj   r   rZ  statusr'   strip_prefix	safe_pathr   r  
hardlinkedr  s              rk   create_helper(FilesystemObjectProcessors.create_helper  s     #}}RVV,,,,&&tbff}55--$$\2D"4(	#!5bkkAo
__(("))RYY)?@F!$"&J77d**-6?OORYY		23 s   D'D)c          	          U R                  XSSUS9 u  pVpxUb)  UR                  U R                  R                  X1US95        UsS S S 5        $ ! , (       d  f       g = f)NdFr'   r  rE  )r"  rt   r  r  )	rj   r   r  rZ  r  r   r  r!  r  s	            rk   process_dir_with_fd.FilesystemObjectProcessors.process_dir_with_fd;  s_    #EP\]  bMbfpzD33>>rB>OP ^]]s   1A
Ac          
         U R                  XSSUS9 u  pgpUc  UsS S S 5        $ [        XU[        SSS9 n
U
b4  [        S5         [	        U[
        R                  " U
5      5      nS S S 5        UR                  U R                  R                  XAU
S95        UsS S S 5        sS S S 5        $ ! , (       d  f       NK= f! , (       d  f       O= f S S S 5        g ! , (       d  f       g = f)	Nr%  Fr&  Tdir_open)r   r  r  r  r  r   fstatrE  )
r"  r  rD   r   r  r	  r+  rt   r  r  rj   r   r  r  rZ  r  r   r  r!  r  r  s              rk   process_dir&FilesystemObjectProcessors.process_dirA  s    #EP\]  bMbfpz| ^] TT $58:>"7+.r288B<@ ,D33>>rB>OP5 5 ^] ,+	5 5 5 ^]]s?   CCB<!B+&2B<	C+
B95B<<
C
	C
C%c          
         U R                  XSUS9 u  pgpUc  UsS S S 5        $ [        XU[        SS9 n
[        S5         [	        U[
        R                  " U
5      5      nS S S 5        U(       a  U	(       a  Xl        UR                  U R                  R                  XAU
S95        UsS S S 5        sS S S 5        $ ! , (       d  f       N_= f! , (       d  f       O= f S S S 5        g ! , (       d  f       g = f)Nfr  Tr  r+  rE  )r"  r  rC   r   r  r	  r+  r  rt   r  r  r,  s              rk   process_fifo'FilesystemObjectProcessors.process_fifoN  s    #LIMxd\f| JI TT_cdhjw'*2rxx|<B (!_+5(D33>>rB>OP ed JI (' edd JIIs@   C&C&C !B:!AC'	C&:
CC
C	C&&
C4c                   U R                  XXVS9 u  pxpUc  UsS S S 5        $ [        S5         [        U[        XUSS95      nS S S 5        UR                  Ul        U	(       a  U
(       a  Xl        UR                  U R                  R                  XA5      5        UsS S S 5        $ ! , (       d  f       Nh= f! , (       d  f       g = f)Nr1  r   F)r   r  r  r;  )
r"  r   r  rE   st_rdevrU  r  rt   r  r  )rj   r   r  r  rZ  dev_typer  r   r  r!  r  s              rk   process_dev&FilesystemObjectProcessors.process_devZ  s    (NR}SWak| ON 6"&r7X\ns+tu #

DI'1$KK//::2DE ON #"	 ONs(   B7B7B&AB7&
B4	0B77
Cc                j   U R                  XSSUS9 u  pgpUc  UsS S S 5        $ Ub  Ub  UOUn
[        S5         [        R                  " XS9nS S S 5        WUl        UR                  U R                  R                  XA5      5        UsS S S 5        $ ! , (       d  f       NJ= f! , (       d  f       g = f)NsFr&  readlink)dir_fd)r"  r   r	  r;  r  rt   r  r  )rj   r   r  r  rZ  r  r   r  r!  r  fnamer  s               rk   process_symlink*FilesystemObjectProcessors.process_symlinkg  s     #EP\]  bMbfpz| ^] !,1FDDE:&U= ' DKKK//::2DE ^] '&	 ^]s'   B$B$B:B$
B!	B$$
B2c                :   SnU R                  Xq5        S n[        U5      nUc  [        SU-  5      e[        U5      n	U	c  [        SU-  5      e[	        [
        R
                  " 5       5      S-  n
[        UUS-  S-  XXXU
S9	nU R                  XU R                  U R                  [        U R                  R                  U5      5      5        UR                  SS	9  U R                  =R                  S
-  sl        SUl        U R!                  XR                  S9  U$ )Nr  zno such user: %szno such group: %sr  i  i   )	r   r   r}  r|  r  r~  rl  ra  r  Tr  r   Fr  )r  r.   r+   r0   r  r   rM   r  rz   r   r   rC  rU  r  rd   r  r  )rj   r   rj  r  r   r|  r~  r  r}  r  tr   s               rk   process_pipe'FilesystemObjectProcessors.process_pipeu  s   v,tn;*T122;+e344		z)8+A
 	  djj$:L:Ln]a]i]i]r]rsu]vNwxt$

Q$d**-rn   )r  c                   U R                  XS US9 u  ppUc  U	sS S S 5        $ [        XX6SS9 n[        S5         [        U[        R
                  " U5      5      nS S S 5        UR                  U R                  R                  U5      5        [        UR                  5      nU(       a6  [        R                  [        R                  " UR                  5      -  Ul        U
(       a  U(       Ga%  U(       dd  [        [        R                   R#                  U R$                  U5      5      nU R&                  R)                  U5      nUR+                  XU5      u  nnOS =pSu  nnS nUbR  U H  nUR-                  U5      (       a  M  Sn	  O<   U Vs/ s H  nUR/                  UU R0                  5      PM!     nnSn	OU(       a  SOSn	U R3                  X5        S n	Xl        Ub	  UUl        GO[        S	5         U R9                  XU R0                  U R:                  [=        U R>                  RA                  S U5      5      5        S S S 5        [B        (       a  S
nOQ[        S5         [        R
                  " U5      nS S S 5        U(       + =(       a    URD                  WRD                  :g  nU(       a  Sn	U(       d>  U(       d7  URG                  XXHR6                   Vs/ s H  nURH                  PM     sn5        U R0                  =RJ                  S-  sl%        UR                  U R                  RM                  XAUS95        URO                  SS9  U	sS S S 5        sS S S 5        $ ! , (       d  f       GN= fs  snf ! , (       d  f       GN>= f! , (       d  f       GN= fs  snf ! , (       d  f       O= f S S S 5        g ! , (       d  f       g = f)Nr1  Tr  r+  r  MUAr   Ffstat2Cr   rE  rA  )(r"  r  r   r  r	  r+  rt   r  r  r   r   r   S_IFREGS_IMODEr   r8   r   r,  r  r  rr  file_known_and_unchanged
seen_chunkr  rz   r  r  r  r  r   r   rC  rU  r   r  memorize_filer   rd   r  r  )rj   r   r  r  rZ  rj  r  r  r   r  r!  r  r  is_special_filehashed_path	path_hashknownr%  r  r6  changed_while_backupst2r  s                          rk   process_file'FilesystemObjectProcessors.process_file  s    $\JNyt]g| KJ TTX\]acw'*2rxx|<B (D33EEbIJ",RZZ"8" !%t||DII/F FDI!_*&1"'',,txx2N&O$(HH$4$4[$A	%*%C%CK\^%_
s
 376%0
s!F#&C#(#3#3C#8#8), % $'
 VY%YUXce&8&8djj&IUXF%Y%(F(-3**68!F+5()&,&v. 44T$**dN`N`bpquq}q}  rG  rG  HL  NP  rQ  cR  S /#8380!*8!4&(hhrl "5
 8G3F3l2>>]`]l]lKl0/%(F.7K
 "//[f[fLg[fVWQTT[fLghJJ%%*%D33BB2PRBSTt,} ^] KJ ('8 &Z /.
 "5!4 Mhu ^]] KJJs   N;N;N !M D$N 
N &M24A N 4AM7 !N !N	8A N N.AN 	N; 
M/*N 7
NN 	
NN  
N.	*N;;
O	)r  rj  rC  r  r  r  r  r  r  r   rz   )NTN)r   r   r   r   rl   r   r"  r'  r-  r2  r7  r>  rC  rC   rV  r   r   rn   rk   r
  r
    sT     59X" @ @8
2 GS B Brn   r
  c                   T    \ rS rSrSS.S jr\SS j5       rS rS rS r	S	 r
S
 rSrg)TarfileObjectProcessorsi  Nr  c       	             Xl         X l        X0l        X@l        X`l        U	=(       d    S U l        [        XxS9U l        [        XRR                  SS.6U l
        g )Nc                      g rp   r   r  s    rk   r  2TarfileObjectProcessors.__init__.<locals>.<lambda>  r  rn   r  Fr<  )rj  r  r  r  r   r  r[   rz   r   rB  rC  )
rj   rj  r  r  r  rD  r   r  r`   r  s
             rk   rl    TarfileObjectProcessors.__init__  sL     
 #6 *!4!L9KH>
"NPUVrn   c              #   X  #    [        [        UR                  5      UR                  U-  UR                  UR
                  UR                  =(       d    S UR                  =(       d    S [        [        UR                  S-  5      5      S9nXB4v   U R                  X@R                  S9  g 7f)Nr  )r   r   r}  r  r|  r~  rl  r  )rM   r:   r  r   r}  r  unamegnamer>   r  rl  r  rz   rj   tarinfor  r   r   s        rk   r"  %TarfileObjectProcessors.create_helper  s     5GLL4<O7==;PDX_XeXeXmim!#gmmg&=">?A ld**-s   B(B*c                d    U R                  XU5       u  pBUsS S S 5        $ ! , (       d  f       g = frp   r"  ra  s        rk   r-  #TarfileObjectProcessors.process_dir  %    6.4 766   !
/c                d    U R                  XU5       u  pBUsS S S 5        $ ! , (       d  f       g = frp   re  ra  s        rk   r2  $TarfileObjectProcessors.process_fifo  rg  rh  c                    U R                  XU5       u  pB[        R                  " UR                  UR                  5      Ul        UsS S S 5        $ ! , (       d  f       g = frp   )r"  r	  makedevdevmajordevminorrU  ra  s        rk   r7  #TarfileObjectProcessors.process_dev  sC    6.4

7#3#3W5E5EFDI 766s   4A
Ac                    U R                  XU5       u  pBUR                  Ul        UsS S S 5        $ ! , (       d  f       g = frp   )r"  linknamer  ra  s        rk   process_link$TarfileObjectProcessors.process_link  s2    6.4!**DK 766s	   2
A c                   U R                  XU5       u  pRU R                  X!R                  5        S nUR                  U5      nU R	                  XPR
                  U R                  U R                  [        U R                  R                  U5      5      5        UR                  SS9  U R                  =R                  S-  sl        UsS S S 5        $ ! , (       d  f       g = f)NTrA  r   )r"  r  r  extractfiler  rj  rz   r   r   rC  rU  r  rd   )rj   rb  r  r   tarr   r  s          rk   rV  $TarfileObjectProcessors.process_file  s    6.4""6<<8F)B$$T::tzz4CUCU%3DLL4I4I"4M%NPMM4M(JJ" 766s   B4C
C)r  rj  rC  r  r  r  r   rz   NN)r   r   r   r   rl   r   r"  r-  r2  r7  rr  rV  r   r   rn   rk   rY  rY    s:     59W . .

	rn   rY  c                    ^ [        U 5      nUS:X  a  gU S   S-  S:X  a  SnOU S   S:X  a  SnOgX#::  a  gX   S-  S	:X  a  O
X   S
;   a  OgXS m[        U4S jU 5       5      $ )z1check if the data <d> looks like a msgpacked dictr   F      r      r        )         Nc              3   F   >#    U  H  nTR                  U5      v   M     g 7frp   )rH  ).0patternkey_serializeds     rk   	<genexpr>'valid_msgpacked_dict.<locals>.<genexpr>"  s     Qg~((11s   !)r_  any)r%  keys_serializedd_lenoffsr  s       @rk   valid_msgpacked_dictr  	  s    FEztd{d	
1 }w~	
&	& uXNQQQQrn   c                   D   ^  \ rS rSrSrU 4S jrS rS rS rS r	Sr
U =r$ )	RobustUnpackeri%  zCA restartable/robust version of the streaming msgpack unpacker
    c                   > [         TU ]  5         U Vs/ s H'  n[        R                  " UR	                  5       5      PM)     snU l        Xl        / U l        SU l        [        R                  " [        S9U l        g s  snf )NFobject_hook)ri  rl   rF   packbencode	item_keys	validator_buffered_data_resyncr!  r<   	_unpacker)rj   r  r  r  rk  s       rk   rl   RobustUnpacker.__init__(  s_    CLM94'--69M"  ))jA	 Ns   .A<c                      / U l         SU l        g NT)r  r  r   s    rk   resyncRobustUnpacker.resync0  s     rn   c                     U R                   (       a  U R                  R                  U5        g U R                  R	                  U5        g rp   )r  r  r]  r  r#  )rj   r   s     rk   r#  RobustUnpacker.feed4  s/    <<&&t,NN%rn   c                     U $ rp   r   r   s    rk   __iter__RobustUnpacker.__iter__:  s    rn   c                 R   U R                   (       a  SR                  U R                  5      nU R                   (       a  U(       d  [        e[	        XR
                  5      (       d  USS  nM?  [        R                  " [        S9U l	        U R                  R                  U5         [        U R                  5      nU R                  U5      (       a	  SU l         U$  USS  nU R                   (       a  M  g g [        U R                  5      $ ! [        R                  [        4 a     NKf = f)Nrn   r   r  F)r  r,  r  r   r  r  rF   r!  r<   r  r#  r   r  r  )rj   r   r   s      rk   __next__RobustUnpacker.__next__=  s    <<88D//0D,,''+D..AA8D!(!1!1j!I##D)$/D
 ~~d++',# , ABx% ,,,( ''  //? s   %D	 	D&%D&)r  r  r  r  r  )r   r   r   r   r  rl   r  r#  r  r  r   ru  rv  s   @rk   r  r  %  s'    B&( (rn   r  c                   Z    \ rS rSrS r  SS jrS rS rS rS r	SS	 jr
S
 rSS jrSrg)ArchiveCheckeriW  c                 0    SU l         [        5       U l        g NF)error_foundr   possibly_supersededr   s    rk   rl   ArchiveChecker.__init__Y  s     #&5 rn   Nc
                 \   [         R                  S5        USL =(       a    [        XEU45      (       + U l        X l        Xl        U R                  5         U R                  (       d  [         R                  S5        gU R                  U5      U l
        U(       a  U R                  5         [        R                  U R                  ;  a2  [         R                  S5        SU l        U R                  5       U l        OB [        R"                  " U[        R$                  R&                  4U R                  S9u  U l        n
U R+                  X4XVUS	9  U R-                  5         U R/                  U	S
9  U R                  (       a  [         R                  S5        O[         R                  S5        U R                  =(       d    U R                  (       + $ ! [(         aW  n[         R                  SU5        SU l        U R                  [        R                  	 U R                  5       U l         SnANSnAff = f)a  Perform a set of checks on 'repository'

:param repair: enable repair mode, write updated or corrected data into repository
:param archive: only check this archive
:param first/last/sort_by: only check this number of first/last archives ordered by sort_by
:param glob: only check archives matching this glob
:param verify_data: integrity verification of data referenced by archives
:param save_space: Repository.commit(save_space)
z%Starting archive consistency check...NzJRepository contains no apparent data at all, cannot continue check/repair.FzRepository manifest not found!T)r  z$Repository manifest is corrupted: %s)r  firstlastsort_byglob)
save_spacez3Archive consistency check complete, problems found.z6Archive consistency check complete, no problems found.)r  r  r  	check_allrepairr  init_chunksr  r  make_keyr  verify_datar&   MANIFEST_IDr  rebuild_manifestr  r  	OperationCHECKIntegrityErrorBaserebuild_refcountsorphan_chunks_checkr!  )rj   r  r  r  r  r  r  r  r  r  r-  excs               rk   checkArchiveChecker.check]  s    	;< DIe45H1I-I${{LLef==,t{{2LL9:#D 113DM8#+==h>P>P>V>V=X^b^f^f#g q 	w$^bc  "z*LLNOKKPQ{{2$"2"222 & 8CSI#' KK 4 45 $ 5 5 7	8s   9AG
 

H+AH&&H+c                     [        [        U R                  5      S-  S9U l        Sn U R                  R	                  [
        US9nU(       d  gUS   n[        SSSS9nU H  nX0R                  U'   M     MO  )z8Fetch a list of all object keys from repository
        g?)usableNlimitmarkerrR  r   refcountrq   rb   )r#   r_  r  r  listrX   r$   )rj   r  result
init_entryr6  s        rk   r  ArchiveChecker.init_chunks  sw     !DOO(<s(BC__)))OFBZF(!!1EJ#-C   rn   c                     SnU R                   R                  5        H/  u  p4US-  nUS:  a    O!UR                  U5      n [        X5      s  $    US:X  a  SnOSU-  n[        U5      e! [         a     MY  f = f)Nr   r   i  z*make_key: repository has no chunks at all!z4make_key: failed to create the key (tried %d chunks))r  	iteritemsr   r   r   r   )rj   r  attemptchunkidr-  cdatar   s          rk   r  ArchiveChecker.make_key  s    ++//1JGqLG}NN7+E":55 2 a<>CH7RCS!! + s   
A))
A76A7c                    [         R                  S5        [        U R                  5      nSnSn/ n[	        USSSS9nS n U R
                  R                  SUS	9nU(       d  OU[        U5      -  nUS
   nU R
                  R                  U5      n[        [        U5      5      n	U	(       aj  UR                  5         U	R                  S
5      n
 [        U5      nU
[        R                  :X  a  S OU
n U R                  R!                  X5        U	(       a  Mj  M  UR3                  5         X:w  a+  [         R'                  S5        [         R'                  SX5        U(       a  U R4                  (       a  [         R7                  S5        U Hq  n U R
                  R9                  U5      nU[        R                  :X  a  S OUnU R                  R!                  X5        [         R7                  S[)        U5      5        Ms     O=[         R7                  S5        U H"  n[         R=                  S[)        U5      5        M$     U(       a  [         R&                  O[         R                  nU" SX#5        g ! ["         aH  nSU l        US-  n[         R'                  S[)        U
5      U5        UR+                  U
5         S nAGNS nAff = f! [,        R.                  ["        4 a  nSU l        US-  n[         R'                  S[)        U
5      U5        [1        U["        5      (       a  UR+                  U
5        U	(       a/  [        [        U	5      5      nU R
                  R                  U5      n S nAGN[S nAff = f! ["         aL    U R                  U	 U R
                  R;                  U5        [         R=                  S[)        U5      5         GM>  f = f)Nz5Starting cryptographic data integrity verification...r   zVerifying data %6.2f%%{Gz?zcheck.verify_datar  r   stepr  Td   r  rR  r   zchunk %s, integrity error: %szchunk %s: %szGRepo/Chunks index object count vs. segment files object count mismatch.z:Repo/Chunks index: %d objects != segment files: %d objectszmFound defect chunks. They will be deleted now, so affected files can get repaired now and maybe healed later.z0chunk %s not deleted, did not consistently fail.zchunk %s deleted.z}Found defect chunks. With --repair, they would get deleted, so affected files could get repaired then and maybe healed later.zchunk %s is defect.z`Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.)r  r  r_  r  r@   r  scanr4  r  reversedr  r  r   r&   r  r  r5  r  r  r  r=   r]  rW   r  rw   r!  r  rn  r   r  debug)rj   chunks_count_indexchunks_count_segmentserrorsdefect_chunksr)  r  	chunk_idschunk_data_iterchunk_ids_revdr  encrypted_data	_chunk_idintegrity_errorr
  defect_chunklogs                    rk   r  ArchiveChecker.verify_data  s.   KL - !%,>D\cg,?A,,3v,FI!S^3!r]F"oo66yAO!(9"56N 	)--b17%)/%:N )1H4H4H(HhI7((C% !. @ 			6LLbcLLU+D{{   J K$1L
u)-)<)<\)J,8H<P<P,PDVb	((C 'Y[efr[st! %2$   W X$1LLL!6
<8PQ %2$fll&++n!	+U . 7+/(!%DjQYFZ\kl%,,X66	7 #113EF 	N'+D$aKFLLH1EsK!#'9::%,,X6%$(.)A$B	*.//*B*B9*M	NV . T KK5..|<%8*\:RSS	TsE   J? 1I* 
AM/*
J<4=J77J<?M,BM''M,/AOOc                   ^ [        S [         5       5      mU4S jn[        R                  S5        [	        U R
                  U R                  5      n[         Vs/ s H'  n[        R                  " UR                  5       5      PM)     nn[        [        U R                  5      SSSS9nU R                  R                  5        GH=  u  pgUR                  5         U R                  R!                  U5      n U R
                  R#                  Xh5      n	[+        X5      (       d  M_  SU	;  d  SU	;  a  Mm   [        R,                  " U	5      nU" U5      (       d  M   U R
                  R1                  U	SS9u  pn[7        US9nUR8                  n[        R                  SU5        X2R:                  ;   a8  Sn SX>4-  nXR:                  ;  a  OUS-  nM  [        R=                  SX?5        UnXkR>                  4UR:                  U'   GM@     URA                  5         [        R                  S5        U$ s  snf ! [$         a)  n
[        R'                  SU
5        S	U l         S
n
A
GM  S
n
A
ff = f! [        R.                   a     GM  f = f! [2         a`  nUR!                  SS5      R5                  SS5      n[        R'                  SX=5        [        R'                  S5        S	U l         S
nAGM%  S
nAff = f)zRebuild the manifest object if it is missing

Iterates through all objects in the repository looking for archive metadata blocks.
c              3   @   #    U  H  oR                  5       v   M     g 7frp   r  r  r  s     rk   r  2ArchiveChecker.rebuild_manifest.<locals>.<genexpr>  s     )XBW3**,,BW   c                 h   > [        U [        5      (       d  g[        U 5      nTR                  U5      $ r  )rw   r  r   issubset)objkeysrequired_archive_keyss     rk   valid_archive6ArchiveChecker.rebuild_manifest.<locals>.valid_archive  s-    c4((s8D(11$77rn   z9Rebuilding missing manifest, this might take some time...zRebuilding manifest %6.2f%%r  zcheck.rebuild_manifestr  zSkipping corrupted chunk: %sTNs   cmdlines	   versionFr  s   names	   <unknown>asciir  3Archive TAM authentication issue for archive %s: %szMThis archive will *not* be added to the rebuilt manifest! It will be deleted.r  zFound archive %sr   z%s.%dz(Duplicate archive name %s, storing as %szManifest rebuild complete.)!	frozensetREQUIRED_ARCHIVE_KEYSr  r  r&   r  r  ARCHIVE_KEYSrF   r  r  r@   r_  r  r  r  r   r5  r  r  r  r  unpackbr  r  r   decoderN   r  r  rn  r   r!  )rj   r  r  r  archive_keys_serializedr)  r  r-  r  r   r  r  verifiedr  r  new_namer  s                   @rk   r  ArchiveChecker.rebuild_manifest  s   
 !*)XBW)X X	8 	OP DHHdoo6LX"YLD7==#?L"Y%C,<B_fj,DF;;002KHGGIOO''1Exx''8
 (FF%);4)G!//$/ W%%+/88+M+Mdkp+M+q(Gq &G<||.5,,,A#*dY#6#+<+<<!Q	 
 NN#Mt^#D+3\\*B!!$'a 3b 			01m #Z & ;SA#'  **  &  #;;w=DDWiXDLL!VX\nLL!pq'+D$sI   .H-6H22I(J2
I%<I  I%(J J
K.AK))K.c                 
  ^ ^^^ T R                   R                  [        R                  S5        U 4S jmUU 4S jnS$U 4S jjmUUU 4S jnUU 4S jnUc  UR	                  S5      n[        X#U45      (       a  T R                  R                  R                  XEX#S9n	U(       a  U	(       d  [        R                  S	U5        U(       a/  [        U	5      U:  a   [        R                  S
U[        U	5      5        U(       a/  [        U	5      U:  a   [        R                  SU[        U	5      5        O?T R                  R                  R                  US9n	O T R                  R                  U   /n	[        U	5      n
[        U
SSSS9n[!        T R"                  5       m[%        U	5       GH  u  pUR'                  U5        [        R)                  SUR*                   SUS-    SU
 S35        UR,                  nUT R                   ;  aI  [        R                  S[/        U5      5        ST l        T R                  R                  UR*                  	 M  T" U5        T R"                  R1                  U5      n T R2                  R5                  X5      n T R2                  R9                  USS9u  nnn[;        US9nUR<                  S:w  a  [?        S5      eUR@                   Vs/ s H  n[C        U5      PM     snUl         [E        T R2                  5      nUUl#        U" U5       H-  nS U;   a  U" UR*                  U5        URI                  U5        M/     URK                  SS!9  URL                   H  nT" U5        M     UR                   Ul&        T R2                  RO                  URQ                  5       S"US#9nT R2                  RS                  U5      nT R2                  RU                  U5      nT" U[        U5      [        U5      U5        UURV                  4T R                  R                  UR*                  '   GM     URY                  5         SSS5        g! [         a     [        R                  SU5        ST l         gf = f! [6         aT  n[        R                  S[/        U5      U5        ST l        T R                  R                  UR*                  	  SnAGM4  SnAff = f! [6         aj  n[        R                  SUR*                  U5        [        R                  S5        ST l        T R                  R                  UR*                  	  SnAGM  SnAff = fs  snf ! , (       d  f       g= f)%zqRebuild object reference counts by walking the metadata

Missing and/or incorrect data is repaired when detected
Nc                    > TR                   R                  U [        SSS5      5      R                  S:X  a  TR                  R                  U 5        g g r^   )r  r   r$   r  r  r$  )r6  rj   s    rk   mark_as_possibly_supersededEArchiveChecker.rebuild_refcounts.<locals>.mark_as_possibly_supersededQ  sC    {{sOAq!$<=FF!K((,,S1 Lrn   c                    > TR                   R                  U 5      nTR                   R                  U 5      nT" U[        U 5      [        U5      U5        U$ rp   )r  rr  encryptr_  )rM  r6  r  add_referencerj   s      rk   add_callback6ArchiveChecker.rebuild_refcounts.<locals>.add_callbackU  sF    ((""5)CHH$$U+E#s5z3u:u=Jrn   c                    >  TR                   R                  U 5        g ! [         aM    Uc   e[        SXS9TR                   U '   TR                  (       a  TR
                  R                  X5         g  g f = f)Nr   r  )r  increfr  r$   r  r  put)r6  rq   rb   r  rj   s       rk   r  7ArchiveChecker.rebuild_refcounts.<locals>.add_reference[  sh    4""3' 4(((#2AD#VC ;;OO''3 4s    AA65A6c                   > U4S jnSn/ nSnSU;   nUR                   nU(       a  UR                  OUnU(       aC  [        U5      [        U5      :w  a+  [        R	                  U  SUR
                   S35        U?SnUn[        Xx5       GH  u  pU
u  pnUTR                   ;  Ga  X:X  a[  [        R                  SR                  XR
                  X3U-   [        U5      5      5        S=Tl
        nU" U5      u  ppT" XX5        GO[        R                  S	R                  XR
                  X3U-   [        U5      5      5        U	u  pnUTR                   ;   a
  T" XU5        O[        R	                  S
R                  XR
                  X3U-   [        U5      5      5        S=Tl
        nU" U5      u  ppT" XX5        O_X:X  a
  T" XU5        OP[        R                  SR                  XR
                  X3U-   [        U5      5      5        T" XU5        T" U	S   5        UR                  XU/5        X<-  nGM     U(       a  U(       d  UR                   Ul        U(       a,  XH:X  a'  [        R                  U  SUR
                   S35        U?XAl         SU;   aT  UR                  nUR                  SSS9nUU:w  a1  [        R	                  SR                  XR
                  UU5      5        ggg)zVerifies that all file chunks are present.

Missing file chunks will be replaced with new chunks of the same length containing all zeros.
If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one.
c                    > [        S [        U S9n[        UTR                  R                  5      u  p#TR                  R                  U5      n[        U5      nX XT4$ )N)rQ  rq   )r   rY  r  r  rr  r  r_  )rq   rM  r  r   r  rb   rj   s         rk   replacement_chunkWArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunks.<locals>.replacement_chunkj  sQ    dxdC!,UDHH4D4D!E((.E
u33rn   r   Fr8  z: z*: Invalid chunks_healthy metadata removed!z^{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). Replacing with all-zero chunk.Tz|{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). It has an all-zero replacement chunk already.zm{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). Generating new replacement chunk.zE{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).z,: Completely healed previously damaged file!rq   )
compressedr  z<{}: {}: size inconsistency detected: size {}, chunks size {}N)r  r8  r_  r  rn  r   r3  r  r   r=   r  r  r]  rq   r  )archive_namer   r  offset
chunk_listchunks_replacedhas_chunks_healthychunks_currentr8  chunk_currentchunk_healthyr  rq   rb   r  rY  rX  r  r  rj   s                    rk   verify_file_chunks<ArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunksd  s   4 FJ#O!1T!9![[N4FT00NN!c.&9S=P&P,r$))<fgh'%*"!/03N0S,(5%4;;.$5 &FFLf%199ftmU_`hUiGkl >BA(?7H7N4%heC %TTZTZ$0))Vd]T^_gThUjk 1>-#t{{2)(%@"NN ,OOUv+7FUYM[efn[oPqr BFED,;LT;R8HE)(%G$5%he<$k$r$r())Vd]JW_L`%b c%he<3M!4DE!!85"9:G 1TH '9&*kk#!j&B|nBtyyk9efg'$K~ II	#'==Et=#T  00NN#a#h#h#/IGW$Y Z 1 rn   c              3     >^^^^#    [        S TR                  R                   5       5      m[        S [         5       5      m[	        S TR                  R                  5      nSmUU4S jnU4S jnS mUUU4S jnSn[        U R                  U5       H  u  pg[        U5      nUS	-  (       a  U H  nU" S
X5        US-  nM     M3  US:  a  UR                  5         [        UTR                  U5      5       Hj  u  p TR                  R                  X5      n
UR                  U
5        U H-  nU" U5      u  pU(       a  [        US9v   M!  U" SU-  X5        M/     US-  nMl     M     g! [         a  nU" [!        U5      X5         SnAN/SnAf["        R$                   a    U" SX5        UR                  5          Na[&         a    U" SX5        e f = f7f)zqIterates through all archive items

Missing item chunks will be skipped and the msgpack stream will be restarted
c              3   @   #    U  H  oR                  5       v   M     g 7frp   r  r  s     rk   r  LArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<genexpr>  s     !R:Q3**,,:Qr  c              3   @   #    U  H  oR                  5       v   M     g 7frp   r  r  s     rk   r  r    s     *VCUC::<<CUr  c                 :    [        U [        5      =(       a    SU ;   $ )Ns   path)rw   r<   r  s    rk   r  KArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<lambda>  s    :dJ3O3cT[_cTc3crn   r   c                 P   > TS-  [        U TR                  ;  5      :w  a  TS-  mT$ )N   r   )r  r  )r  _staterj   s    rk   missing_chunk_detectorYArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.missing_chunk_detector  s+    A:XT[[%@!AAaKFrn   c                 f   > [        U5      nU SX#4-  -  n STl        [        R                  U 5        g )Nz [chunk: %06d_%s]T)r=   r  r  r  )r   r  chunk_nor  rj   s       rk   reportIArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.report  s3     **h_<<#' S!rn   c                 2    SR                  S U  5       5      $ )Nz, c              3      #    U  H4  n[        U[        5      (       a  UR                  S S9O
[        U5      v   M6     g7f)r  )r  N)rw   rX  r  r  )r  ks     rk   r  dArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe.<locals>.<genexpr>  s6      nimdez!U?S?S!;Y\]^Y_!_ims   <>)r,  )r  s    rk   list_keys_safeQArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe  s    yy nim nnnrn   c                   > [        U [        5      (       d  gU R                  SS 5        [        U 5      nTR	                  U5      (       d  SST" TU-
  5      -   4$ UR	                  T5      (       d  SST" UT-
  5      -   4$ g)N)Fznot a dictionarys   aclFzmissing required keys: zinvalid keys: )Tr   )rw   r<   r  r   r  )r  r  r  r/  required_item_keyss     rk   
valid_itemMArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.valid_item  s    !#z224 %3x)22488 ";nM_bfMf>g"ggg}}Y// "2^D9DT5U"UUUrn   r#  zitem metadata chunk missingr   r  zDDid not get expected metadata dict when unpacking item metadata (%s)NzCUnpacker crashed while unpacking item metadata, trying to resync...z5Exception while decrypting or unpacking item metadata)r  r  r  REQUIRED_ITEM_KEYSr  r
   r+  r  r  r3  r4  r  r5  r#  rM   r   r  rF   r  r  )r  r*  r%  r)  r3  r  stater+  r  r  r   r   validreasonr  r$  r  r/  r2  r  rj   s                  @@@@rk   robust_iterator9ArchiveChecker.rebuild_refcounts.<locals>.robust_iterator  s    
 "!R$--:Q:Q!RRI!**VCU*V!V%&c&*mm&=&=?HF"o  A '7M NU19$)<hJQ %* 19OO%'*5*2E2Ee2L'MOH#xx//@ d+$,D,6t,<ME$&*&> > &'mpv'v  yA  !E %-" FA+ (N !O& * B s?3XAA"22 *dfnr )$ VX`ds=   C9GAE. G.
G8F
G0G?GGG,)r  r  r  r  z.--glob-archives %s does not match any archivesz+--first %d archives: only found %d archivesz*--last %d archives: only found %d archives)r  zArchive '%s' not found.TzChecking archives %3.1f%%g?zcheck.rebuild_refcountsr  zAnalyzing archive z (r   r9  )z%Archive metadata block %s is missing!z*Archive metadata block %s is corrupted: %sFr  r  zEThis archive will be *removed* from the manifest! It will be deleted.r  r  r  r  r  )r  saltrp   )-r  r  r&   r  r-  r  r  r  r  r  rn  r_  r  r  r  r@   rV   r  r  r  r  r  r   r=   r   r  r5  r   r  rN   r  r  r  r9   r:  rN  r$  r   r+  r  r   rr  r  r  r!  )rj   r  r  r  r  r  r  r  r9  archive_infosnum_archivesr)  r  r  
archive_idr  r   r  r  r=  r  r  r   previous_item_idnew_archive_idr  r  r  s   `                        @@@rk   r   ArchiveChecker.rebuild_refcountsI  sx    	,,d3	2		4J	ZXF	P ?mmC(GE&'' $ 6 6 ; ;G^c ; oNN#SUYZS/%7NN#PRWY\]jYklC.5NN#OQUWZ[hWij $ 6 6 ; ;G ; L!%!7!7!@ A
 =)%L>Y`c,EGT__-$]3
02a!eWAl^STUV!WW
T[[0LL!H*U_J`a'+D$..tyy9+J7++J788++J>D
.2hh.P.PQUns.P.t+GXt &G<??a'#$FGG?F"O;s#3"O*4884+7(+G4D4'*499d; $$T* 5 """.(/$/0@A )6 , 3 3xx>>w?PZdko>p!%!1!1$!7((.nc$iUUK5CTWW4M&&tyy1_ 4` IIKc .-  6@#' , & LL!MzZdOegvw'+D$..tyy9	 &  LL!VX\XaXacrsLL!hi'+D$..tyy9 #PC .-s|   Q CU1R S!+2UU2EU'Q=<Q= 
S
ASUSU!
U+AU	UUU
U+c                 l   U R                   (       Ga  U R                  R                  5        VVs1 s H  u  pUR                  S:X  d  M  UiM     nnnX0R                  -
  nU(       a(  [
        R                  [        U5       S35        SU l        U R                  (       ax  U(       ap  [
        R                  S[        U5      [        U R                  5      4-  5        U H  nU R                  R                  U5        M      [
        R                  S5        g g g [
        R                  S5        g s  snnf )Nr   z orphaned objects found!Tz1Deleting %d orphaned and %d superseded objects...z.Finished deleting orphaned/superseded objects.z<Orphaned objects check skipped (needs all archives checked).)r  r  r  r  r  r  r  r_  r  r  r  r  r  )rj   r6  r  unusedorphaneds        rk   r  "ArchiveChecker.orphan_chunks_checkC  s    >>>,0KK,A,A,C[,Cjcu~~YZGZc,CF[ 8 88HH.FGH#' {{vOM3t'?'?#@SB B C!COO**3/ "LM  &{ KKVW \s   D0
D0c                     U R                   (       a_  [        R                  S5        U R                  R	                  5         [        R                  S5        U R
                  R                  SUS9  g g )NzWriting Manifest.zCommitting repo.F)r  r  )r  r  r  r  rG  r  r  )rj   r  s     rk   r!  ArchiveChecker.finishS  sO    ;;KK+,MM!KK*+OO""5Z"H	 rn   )r  r  r  r  r  r  r  r  )FNr   r   r   NFF)Nr   r   r   Nr   )r   r   r   r   rl   r  r  r  r  r  r  r  r!  r   r   rn   rk   r  r  W  sB    ) _c,1)3V.""&O+bJXxtX Irn   r  c                       \ rS rSr " S S\5      r\S 5       r    SS jrSS jr	S r
S	 rS
 rS rS rSS jrS rSS jrS rS rSrg)ArchiveRecreateri[  c                       \ rS rSrSS jrSrg)ArchiveRecreater.Interruptedi\  Nc                 $    U=(       d    0 U l         g rp   r  )rj   r  s     rk   rl   %ArchiveRecreater.Interrupted.__init__]  s    $NDMrn   rO  rp   )r   r   r   r   rl   r   r   rn   rk   InterruptedrM  \  s    	+rn   rQ  c                 $    U R                  S5      $ )N	.recreate)r  )r  s    rk   is_temporary_archive%ArchiveRecreater.is_temporary_archive`  s    $$[11rn   Nc                    Xl         X0l        X l        X@l        XPl        X`l        U=(       d    / U l        Xl        U	S LU l        U R                  (       a  [        R                  SU	5        U	=(       d    [        U l        Xl        Xl        U
=(       d    [        S5      U l        [#        5       U l        UU l        Xl        Xl        Xl        U=(       d    S U l        U(       a  S U l        g UU l        g )NzRechunking archives to %snonec                      g rp   r   r  s    rk   r  +ArchiveRecreater.__init__.<locals>.<lambda>  r  rn   )r  r  r  rj  r  exclude_cachesexclude_if_presentkeep_exclude_tagsr  r  r  r  rD  
recompressalways_recompressr   compressionr   seen_chunksr  rV  rz   r  r  r  )rj   r  r  r  rj  r  rZ  r[  r\  rD  r_  r]  r^  rV  rz   r  r  r  r  s                      rk   rl   ArchiveRecreater.__init__d  s    
 % 
,"4":!2(4??LL4nE,>$!2&A/&*A5"
 !4!L9K+24 8K rn   c                    U R                  U5      (       a   eU R                  U5      nU R                  XC5      nU R                  (       d  U R                  (       a  U R                  U5        U R                  R                  5       (       a)  U R                  (       d  UR                  (       d  Uc  Uc  gU R                  XE5        US L nU R                  XEX&S9  g)NF)replace_originalT)rT  open_archivecreate_targetr[  rZ  matcher_add_tagged_dirsr  emptyr]  recreate_rechunkifyprocess_itemsr  )rj   r  r  target_namer  targetrc  s          rk   recreateArchiveRecreater.recreate  s    ,,\::::##L1##G9""d&9&9((1LL  OO..# 7+&$.		'7	Nrn   c                 d  ^
 U R                   nUR                  5       (       + m
T
(       a  0 OS nU
4S jnUR                  5        GH=  nUR                  UR                  5      (       dZ  U R                  SUR                  5        U" U5      (       a/  UR                  S5      UR                  S5      S 4XFR                  '   M~  T
(       av  [        UR                  5      (       a\  UR                  S5      U;   aG  XFR                     u  pxn	U	c-  Xvl
        Ub  Xl        S S UR                  4XFR                  '   U?	OXl	        U R                  (       a  U R                  SUR                  5        GM+  U R                  XU5        GM@     U R                  (       a  UR                  R!                  SS9  g g )	Nc                    > T=(       a:    [        U R                  5      =(       a    U R                  SS5      =(       a    SU ;  $ r  r  )r   target_is_subsets    rk   item_is_hardlink_master?ArchiveRecreater.process_items.<locals>.item_is_hardlink_master  s?    $ ) +)HH.5) D(*rn   r  r  r8  r  -T)r   )r  rg  r  r  r   r  r   r'   r   r  r  r8  rV  process_itemr  rz   r   )rj   r  rk  r  r(  rq  r   r  r8  
new_sourcerp  s             @rk   ri  ArchiveRecreater.process_items  sX   ,,&}}.!12t	* &&(D==++&&sDII6*4003788H3EtxxP`Gacg2h$YY/L$;$;@RVf@f5Ekk5R2
%"(K%1.<+5944K$[[1 #-K||&&sDII6!!'48- ). ==LL&&T&2 rn   c                 >   [        UR                  5      nSU;   aN  U R                  XCR                  5        S nU R	                  XU5        UR
                  =R                  S-  sl        UR                  X2R
                  S9  U R                  XCR                  5        g )Nr  r   r  )r6   r   r  r   process_chunksrz   rd   r  )rj   r  rk  r   r  s        rk   rt  ArchiveRecreater.process_item  sx    TYY't""6995F6LL1$LL1vyy1rn   c                    U R                   (       dX  UR                  (       dG  UR                   H+  u  pEnU R                  R	                  XBR
                  5        M-     UR                  $ U R                  X[        UR                  5      5      n[        U R                  U5      nUR                  X0R                  UR
                  U R                  Xx5        g rp   )r]  rh  r  rj  r  rz   iter_chunksr  r   r  r  r  )	rj   r  rk  r   r  rq   rb   chunk_iteratorr  s	            rk   rx  ArchiveRecreater.process_chunks  s    v'A'A)-%

'',,? *5;;))'4;LM!$"6"6?""4V\\4==R`rrn   c                 (   [        X R                  R                  5      u  p4X0R                  ;   a%  U R                  R                  X1R                  5      $ U R                  nU R                  (       a  U R                  (       d  X0R                  R                  ;   a  U R                  R                  S U R                  R                  U5      SS9n[        R                  " U5      R                  U R                  R                   R#                  U5      R                  :X  a  SnU R                  R%                  X4UR                  USS9nU R                  R                  R'                  SS9  U R                  R)                  UR*                  5        U$ )NF)
decompress)	overwriterp  ro  )r  r  rr  r`  rj  r  rz   r]  r^  r  r5  r  r   r   detectr  
compressordeciderq  rs  r$  r   )rj   rk  rM  r  r   r  	old_chunkr  s           rk   r   ArchiveRecreater.chunk_processor  s$   $UHH,<,<='''::**8\\BBOO	??4#9#9h**J[J[>[((t/B/B8/LY^(_I  +00DHH4G4G4N4Nt4T4Y4YY!	jj**86<<S\ch*i

,,%,8[^^,rn   c           	   #   D  #    UR                   R                  U VVs/ s H  u  n  oTPM
     snn5      nUR                  (       a/  [        U5      nUR                  R                  U5       S h  vN   g U H  n[        U[        U5      [        S9v   M     g s  snnf  N/7f)N)rq   rQ  )	r  r"  rh  r(   rC  rU  r   r_  rW  )	rj   r  rk  r  r  r-  r|  r   rM  s	            rk   r{  ArchiveRecreater.iter_chunks  s      ))44U[5\U[>8QPQhU[5\]%% ,N;D~~..t444'EE
wGG ( 6]
 5s   B B
AB .B/0B c           	         U R                   (       a  g Uc  UR                  R                  SS5      nU R                  (       a  UR                  nU R
                  ct  UR                  R                  UR                  R                  S5      =(       d    UR                  R                  UR                  R                  [        R                  S.nO'UR                  R                  [        R                  S.nUR                  X0R
                  US9  U(       a=  UR                  [        5       U R                  S9  UR                  UR                  5        U R                  (       ab  WUl        [!        5       Ul        [%        [&        [)        U5      [&        [)        UR                  5      [)        U R*                  5      [&        5        g g )Nr  r   r  )r   r  r  recreate_cmdline)r  r  )r  r  r	  )r  )rV  r  r   rz   r  r  r   r  r   r  r  r  r[   r  rx  r  rH   r   rA   DASHESr  rj  )rj   r  rk  r  rc  _startr	  s          rk   r  ArchiveRecreater.save  sA   <<?&&**9b9G ::\\F>>!((--#,,00<U@P@P@U@U"++33$'HH# #++33$'HH# 	G~~SfgNN:<$--N@MM',,'::!FLFJf&k&,,'$**o rn   c                   ^ ^	^
^ U U
U4S jnT R                   m	/ m
/ m0 nT R                  (       a\  UR                  S S9 HH  n[        R                  " UR
                  5      (       d  M*  SU;  d  M2  SU;   d  M:  SX4R                  '   MJ     UR                  U	4S jS9 GH  nT R                  (       a  UR                  U;   a  XCUR                  '   [        R                  R                  UR                  5      u  pVUT R                  ;   a
  U" XT5        Mx  T R                  (       d  M  U[        :X  d  M  [        R                  " UR
                  5      (       d  M  SU;   a  UOX4R                     n[        X5      nUR                  [        [        5      5      [        :X  d  GM  U" XT5        GM     T	R!                  T
["        R$                  5        T	R!                  T["        R&                  5        g)zLAdd excludes to the matcher created by exclude_cache and exclude_if_present.c                    > TR                   (       a@  TR                  [        UR                  SS95        TR                  [	        U S-   SS95        g TR                  [        U SS95        g )NF)recurse_dirr9  )r\  r]  rJ   r   rK   )dirtag_itemrj   	tag_filestagged_dirss     rk   exclude9ArchiveRecreater.matcher_add_tagged_dirs.<locals>.exclude	  sU    %%  !28==e!TU"">#)#OP""#4Se#LMrn   c                 b    [         R                  R                  U R                  5      [        :H  $ rp   )r	  r   basenameCACHE_TAG_NAMEr  s    rk   r  :ArchiveRecreater.matcher_add_tagged_dirs.<locals>.<lambda>'	  s     rww?O?OPTPYPY?Z^l?lrn   )r&  r  r  Nc                    > [         R                  R                  U R                  5      [        :H  =(       d    TR	                  U R                  5      $ rp   )r	  r   r  r  r  r  s    rk   r  r  -	  s8    BGG$4$4TYY$?>$Q$mU\UbUbcgclclUm$mrn   )r  rZ  r  r   rN  r   r  r   r	  r-  r[  r  r*   r   r_  CACHE_TAG_CONTENTSr$  rL   IncludeExcludeNoRecurse)rj   r  r  cachedir_mastersr   r  tag_filecontent_itemr   r  r  r  s   `        @@@rk   rf  (ArchiveRecreater.matcher_add_tagged_dirs	  s|   	N ,,	   **2l*m<<		**xt/CTXHX48$[[1 n
 &&m ' oD""tyy4D'D.2+GGMM$))4MC4222"$$$^)CUYU^U^H_H_'/4'7t=Mkk=Z 799S!3459KKC&o 	Iy001K!;!;<rn   c           	      p   U=(       d    UR                   S-   nU R                  U5      nUR                  R                  S5      nUb  [	        U5      OSnUR
                  nU R                  =(       a    XE:g  Ul        UR                  (       a   [        R                  SU=(       d    SU5        [        U R                  U R                  UR                  UR                  U R                  UR                  S9R                   Ul        [#        UR
                  U R                  R$                  SS.6Ul        U$ )	zCreate target archive.rS  rD  Nz Rechunking archive from %s to %sz	(unknown))rj  r  r  r  r  r  Fr<  )r  create_target_archiver  r   r)   rD  r  rh  r  r  r  rj  r  r  r  r  r  r   rB  rC  )rj   r  rj  rk  src_cpdst_cps         rk   re  ArchiveRecreater.create_target;	  s    !?W\\K%?++K8!!%%&675;5G)&1T&&%)__%I9I"%%LL;V=R{TZ[%4**$((__v7N7N $ 8 8VE_E_&a buat 	" %f&;&;$((BUBU^cdrn   c                     [        U R                  U R                  U R                  USU R                  U R
                  U R                  U R                  S9	nU$ )NT)r  r  rD  rj  r  )r  r  r  r  r  rD  rj  r  )rj   r  rk  s      rk   r  &ArchiveRecreater.create_target_archiveM	  sL    $((DMM4PT#'==ATAT\`\f\f.2.F.FH rn   c                 t    [        U R                  U R                  U R                  U4SU R                  0UD6$ )Nrj  )r  r  r  r  rj  )rj   r  kwargss      rk   rd  ArchiveRecreater.open_archiveS	  s.    t$--bTZZb[abbrn   )r^  rj  r  rD  r_  rV  rZ  r[  r\  r  r  r  r  r  r  r]  r  r`  rz   r  )FNFNNFFFFFNNr  rx  r  rp   )r   r   r   r   r  rQ  r  rT  rl   rl  ri  rt  rx  r  r{  r  rf  re  r  rd  r   r   rn   rk   rK  rK  [  su    +i + 2 2 SX\aQU59	L>(#3J2s 	H%N(=T$crn   rK  )r   r   r	  socketr   r   r   collectionsr   
contextlibr   datetimer   r   	functoolsr   getpassr   ior	   	itertoolsr
   r   shutilr   platformflagsr   r   r   r   r  r   r   r   rC  r   r   rj  r   
crypto.keyr   r   compressr   r   	constantscrypto.low_levelr   r  helpersr   r   r   r    r!   r"   	hashindexr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   platformr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   lrucacherI   patternsrJ   rK   rL   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   remoterV   r  rW   rX   r  r/  r[   r   r   r   r   r  r  r  r:  rg  r  r  r  r  r  r  r
  rY  r  r  r  rK  r   rn   rk   <module>r     s     	   
  # % (    * $ D D !	  ' ! < 1  B : a a E E  ! R R 2 2 > > 2 _ _ P P    K K 5 5      B B - - N N # 32vtG tGnK
7 7. J	4 $T5V  A. A.H.5 .5b
{ 
 374UVde $K< K<\: :B "n5&J6 J6ZH HV3 3lR8/( /(dAI AIHyc ycrn   