a
    b}fih)                     @   s   d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ e
jd eee jjjjj d dlmZ dZdZdZdZdZd	d
 Zdd Zdd Zdd Zdd ZdS )    N)Path)outputi  @
   i'     i   c                 C   sV   g }t t| |}d}td||D ]*}| |||  }|t|  ~q"t|S )Ni   r   )minlenrangeappendhashlibsha256	hexdigestcalculate_total_tree_hash)partZ	part_sizeZ	checksumsupper_boundstepZ	chunk_poschunk r   M/var/www/staging/api/app_home/main/services/satbots/scripts/glacier_upload.pycalculate_tree_hash.   s    r   c                 C   s   | d d  }t |dkrg }tdt |dD ]\}|t |d k rzt|| }t||d  }|t||   q,|||  q,|}q|d S )Nr   r      )r   r   bytesfromhexr	   r
   r   r   )Zlist_of_checksumstreeparentiZpart1Zpart2r   r   r   r   8   s    r   c              
   C   s  |t k s|tkr&tdt  dt dt| s<tdd }zt|dks`tj	|d rt
|d t }tj|dd	$}	|D ]}
|	|
 qW d    n1 s0    Y  t
|d
 nt|d dd}t
|d |dd}|dd |tk rPt
|dtdd |j| ||d}d|d  d|d  d|d  d}t
|| nt||d d  tkr|d }t }||k r|d9 }|tkrxtdqxt
|d| d |}t||| |||d d |||	 W |d ur |  n|d ur|  0 d S )Nzpart-size must be between z and z MBzpart-size must be a power of 2r   r   z*Consolidating files into a .tar archive...zw:xz)fileobjmodezFiles consolidatedrb)r   z
File foundr   zFile size is less than ,z# bytes. Uploading in one request...)	vaultNamearchiveDescriptionbodyz
Uploaded.
Glacier tree hash: checksumz
Location:          locationz
Archive ID:        	archiveId
i   l      b	 z/Archive/upload size too large (more than 40 TB)z0Maximum number of parts exceeded.  Switching to z MB part size.)MIN_PART_SIZE_MBMAX_PART_SIZE_MB
ValueErrormathlog2
is_integerr   ospathisdirr   tempfileTemporaryFiletarfileopenaddseekSINGLE_UPLOAD_THRESHOLD_BYTESupload_archiveceilMAX_NUMBER_OF_PARTSmultipart_uploadclose)
vault_name	file_namearc_descpart_size_mbnum_threads	upload_idglacier_clientsatbotfile_to_uploadtarfilenamefile_size_bytesresponsemessageZtarget_part_sizeZnew_part_sizer   r   r   r6   F   s~    
*







r6   c	                 C   s  i }	t d||D ]}
d |	|
< qt|	}|d u rrt|d | j||t|d}|d }t|d|dd|dd nt|d	| d
 z*| d}|j||d}t|d}W n: | j	j
y } zt|jd d W Y d }~n
d }~0 0 |D ]L}t|d dd }
||
 ||}t||}||d kr||	|
< qt|d tjj|d
}t }i }dd |	 D D ],}|t|||||||| ||}|||< qvtjj|tjjd\}}t|dkr2|D ]}|  q|D ]8}| }|d urdt |}t|d|  qt|d|  t!n |D ]}|| }
|" |	|
< q6W d    n1 sh0    Y  t#t|	$ }t|d | j%||t||d}d| d|d  d |d!  d"|d#  d$	}t|| d S )%Nr   zInitiating multipart upload...)r   r    ZpartSizeuploadIdzFile size is r   z bytes. Will upload in z parts.zResuming upload with id z%.  Fetching already uploaded parts...Z
list_parts)r   rI   ZPartsErrorMessageZRangeInBytes-ZSHA256TreeHashzSpawning threads...)max_workersc                 S   s   g | ]\}}|d u r|qS )Nr   ).0r   r"   r   r   r   
<listcomp>   s   z$multipart_upload.<locals>.<listcomp>)return_when zException occurred: z(Upload can still be resumed. Upload ID: zCompleting multipart upload...)r   rI   ZarchiveSizer"   z'
Uploaded.
Calculated total tree hash: z
Glacier total tree hash: r"   z
Location: r#   z
Archive ID: r$   r%   )&r   r   r   Zinitiate_multipart_uploadstrget_paginatorpaginatelistsearch
exceptionsZResourceNotFoundExceptionr(   rG   int	partitionr4   readr   
concurrentfuturesThreadPoolExecutor	threadingLockitemssubmitupload_partwaitFIRST_EXCEPTIONcancel	exceptionjoin	tracebackformat_exception	Exceptionresultr   valuesZcomplete_multipart_upload)rA   r@   r;   r=   rF   part_size_bytesrC   r?   rB   Z	part_listZ
byte_start	num_partsrG   	paginatorpartseZ	part_datar   r"   executor	fileblockZfutures_listZbyte_posfuturedonenot_doneexcZ
exc_stringZtotal_tree_hashrH   r   r   r   r9      s    


(







0
r9   c
                 C   sZ  |$ | |  ||}
W d    n1 s.0    Y  | t|
 d }d|  d| d| }| | }t|
|}t|	d|d  d| d|| d d d	 d }ttD ]x}z2|j||||
d
}||d krtdW  qTW q ty } z&t|	d| d|d   W Y d }~qd }~0 0 qt|	dt d|d  d |d urN|nt	 ~
|S )Nr   zbytes rL   /z
Uploading z of z...2   )progress)r   rI   r   r!   r"   z.Local checksum does not match Glacier checksumzUpload error: z.
Trying again. Part zAfter z@ attempts, still failed to upload part. Aborting upload of part .)
r4   rZ   r   r   r   r   MAX_UPLOAD_ATTEMPTSZupload_multipart_partrj   RuntimeError)	start_posr;   r@   rm   fprF   rn   rA   rs   rB   r   end_posZrange_headerZpart_numr"   error_rG   rq   r   r   r   rb      s6    
(
,2
rb   )concurrent.futuresr[   r)   os.pathr,   r1   r/   r^   rh   r
   syspathlibr   r-   insertrR   __file__resolver   $main.services.satbots.scripts.sharedr   r5   r|   r8   r&   r'   r   r   r6   r9   rb   r   r   r   r   <module>   s*   $
Ik