|
Server : Apache System : Linux iad1-shared-b8-43 6.6.49-grsec-jammy+ #10 SMP Thu Sep 12 23:23:08 UTC 2024 x86_64 User : dh_edsupp ( 6597262) PHP Version : 8.2.26 Disable Function : NONE Directory : /lib/python3/dist-packages/s3transfer/__pycache__/ |
Upload File : |
o
�y�`)V � @ s� d dl Z d dlmZ d dlZd dlZd dlZd dlmZ d dlm Z d dl
m Z d dl m
Z
d dlZd dlmZmZmZ d dlmZmZmZ d d lmZmZ d d
lmZm Z d d l m Z d d l m Z m!Z! d d
l"m#Z#m$Z$m%Z% d dl&m'Z'm(Z( e �)e*�Z+G dd� d�Z,ddde' d de( ddfdd�Z-G dd� de.�Z/G dd� de!�Z0G dd� de �Z1G d d � d �Z2G d d � d e2�Z3G d d!� d!e�Z4G d"d#� d#�Z5G d$d%� d%�Z6G d&d'� d'�Z7G d(d)� d)�Z8dS )*� N)�BytesIO)�UNSIGNED)�Config)�urlsplit)�NoCredentialsError)�S3Client�
S3RequestType�S3RequestTlsMode)�ClientBootstrap�DefaultHostResolver�EventLoopGroup)�ClientTlsContext�TlsContextOptions)�AwsCredentialsProvider�AwsCredentials)�TransferNotDoneError)�BaseTransferFuture�BaseTransferMeta)�CallArgs�OSUtils�
get_callbacks)�GB�MBc @ s$ e Zd Zdd� Zdd� Zdd� ZdS )� CRTCredentialProviderAdapterc C s || _ d | _t�� | _d S �N)� _botocore_credential_provider�_loaded_credentials� threading�Lock�_lock)�self� botocore_credential_provider� r" �0/usr/lib/python3/dist-packages/s3transfer/crt.py�__init__' s z%CRTCredentialProviderAdapter.__init__c C s | � � �� }t|j|j|j�S r )�_get_credentials�get_frozen_credentialsr �
access_key�
secret_key�token)r � credentialsr" r" r# �__call__, s �z%CRTCredentialProviderAdapter.__call__c C sX | j � | jd u r| j�� }|d u rt� �|| _| jW d � S 1 s%w Y d S r )r r r �load_credentialsr )r � loaded_credsr" r" r# r% 1 s
�$�z-CRTCredentialProviderAdapter._get_credentialsN)�__name__�
__module__� __qualname__r$ r+ r% r" r" r" r# r & s r � � Tc C s� t |�}t|�}t||�} d}
d} |rtjntj} |dur4t� }
|r)|
j|d� nd|
_t |
�}|�
� } |r?t |�}t �
|�}
|d t }t| | |
|| | |d�S )a�
:type region: str
:param region: The region used for signing
:type botocore_credential_provider:
Optional[botocore.credentials.CredentialResolver]
:param botocore_credential_provider: Provide credentials for CRT
to sign the request if not set, the request will not be signed
:type num_threads: Optional[int]
:param num_threads: Number of worker threads generated. Default
is the number of processors in the machine.
:type target_throughput: Optional[int]
:param target_throughput: Throughput target in Bytes.
Default is 0.625 GB/s (which translates to 5 Gb/s).
:type part_size: Optional[int]
:param part_size: Size, in Bytes, of parts that files will be downloaded
or uploaded in.
:type use_ssl: boolean
:param use_ssl: Whether or not to use SSL. By default, SSL is used.
Note that not all services support non-ssl connections.
:type verify: Optional[boolean/string]
:param verify: Whether or not to verify SSL certificates.
By default SSL certificates are verified. You can provide the
following values:
* False - do not validate SSL certificates. SSL will still be
used (unless use_ssl is False), but SSL certificates
will not be verified.
* path/to/cert/bundle.pem - A filename of the CA cert bundle to
use. Specify this argument if you want to use a custom CA cert
bundle instead of the default one on your system.
N)� ca_filepathFr2 )� bootstrap�region�credential_provider� part_size�tls_mode�tls_connection_options�throughput_target_gbps)r r r
r �ENABLED�DISABLEDr �&override_default_trust_store_from_path� verify_peerr
�new_connection_optionsr r � new_delegater r )r5 r! � num_threads�target_throughputr7 �use_ssl�verify�event_loop_group�
host_resolverr4 �providerr9 r8 �tls_ctx_options�client_tls_option� credentails_provider_adapter� target_gbpsr" r" r# �create_s3_crt_client< sD -
���� �rL c @ s� e Zd Zd dd�Zdd� Zdd� Z d dd �Z d d
d �Z d d d
�Zd dd�Z dd� Z
dd� Z dd� Z d dd�Z
dd� Zdd � ZdS ) �CRTTransferManagerNc C s@ |du rt � | _|| _t|| j�| _g | _t�d�| _d| _ dS )aF A transfer manager interface for Amazon S3 on CRT s3 client.
:type crt_s3_client: awscrt.s3.S3Client
:param crt_s3_client: The CRT s3 client, handling all the
HTTP requests and functions under then hood
:type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer
:param crt_request_serializer: Serializer, generates unsigned crt HTTP
request.
:type osutil: s3transfer.utils.OSUtils
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
N� r )
r �_osutil�_crt_s3_client�S3ClientArgsCreator�_s3_args_creator�_future_coordinatorsr � Semaphore�
_semaphore� _id_counter)r �
crt_s3_client�crt_request_serializer�osutilr" r" r# r$ � s �
zCRTTransferManager.__init__c C s | S r r" �r r" r" r# � __enter__� s z CRTTransferManager.__enter__c G s d}|rd}| � |� d S )NFT�� _shutdown)r �exc_type� exc_value�args�cancelr" r" r# �__exit__� s zCRTTransferManager.__exit__c C s6 |d u ri }|d u r i }t |||||d�}| �d|�S )N��bucket�key�fileobj�
extra_args� subscribers�
get_object�r �_submit_transfer)r rd re rf rg rh �callargsr" r" r# �download� s � zCRTTransferManager.downloadc C s6 |d u ri }|d u r i }t |||||d�}| �d|�S )Nrc �
put_objectrj )r rf rd re rg rh rl r" r" r# �upload� s � zCRTTransferManager.uploadc C s4 |d u ri }|d u r i }t ||||d�}| �d|�S )N)rd re rg rh �
delete_objectrj )r rd re rg rh rl r" r" r# �delete� s � zCRTTransferManager.deleteFc C s | � |� d S r r\ �r ra r" r" r# �shutdown� � zCRTTransferManager.shutdownc C s | j D ]
}|�� s
|�� qd S r )rS �donera �r � coordinatorr" r" r# �_cancel_transfers� s
��z$CRTTransferManager._cancel_transfersc C � | j D ]}|�� qd S r )rS �resultrv r" r" r# �_finish_transfers� �
�z$CRTTransferManager._finish_transfersc C ry r )rS �%wait_until_on_done_callbacks_completerv r" r" r# �_wait_transfers_done� r| z'CRTTransferManager._wait_transfers_donec C sz |r| � � z1z| �� W n ty | � � Y n ty" Y nw W | �� d S W | �� d S W | �� d S | �� w r )rx r{ �KeyboardInterrupt� Exceptionr~ rr r" r" r# r] � s �� � �z CRTTransferManager._shutdownc K � | j �� d S r )rU �release�r �kwargsr" r" r# �_release_semaphore� rt z%CRTTransferManager._release_semaphorec
C s | j g}t| jd�}t| j|�|d�}td i |��}t|�}|�|� z$| j�� | j �
|d�}|� | j � |||||�} | j j
d i | ��}
W n& tym } z|�| d� | j j
|d|d�} | | d� W Y d } ~ n
d } ~ ww |�|
� | j�|� | jd7 _|S )
N)� transfer_id)�metarw �queuedTru )�after_subscribers)�error� r" )r� �CRTTransferCoordinatorrV �CRTTransferMeta�CRTTransferFuture�AfterDoneHandler�appendrU �acquirerR �get_crt_callback�get_make_request_argsrP � make_requestr� �
set_exception�set_s3_requestrS )
r � request_type� call_args�on_done_after_callsrw �
components�future� afterdone� on_queued� crt_callargs�crt_s3_request�e�on_doner" r" r# rk � s>
�
�� ���
z#CRTTransferManager._submit_transferr �NN�F)r. r/ r0 r$ r[ rb rm ro rq rs rx r{ r~ r] r� rk r" r" r" r# rM � s(
�
�
�
rM c @ s> e Zd ZdZd dd�Zedd� �Zedd� �Zed d
� �ZdS ) r� z*Holds metadata about the CRTTransferFutureNc C s || _ || _i | _d S r )� _transfer_id�
_call_args�
_user_context)r r� r� r" r" r# r$ s
zCRTTransferMeta.__init__c C � | j S r )r� rZ r" r" r# r� � zCRTTransferMeta.call_argsc C r� r )r� rZ r" r" r# r� r� zCRTTransferMeta.transfer_idc C r� r )r� rZ r" r" r# � user_context r� z CRTTransferMeta.user_contextr� ) r. r/ r0 �__doc__r$ �propertyr� r� r� r" r" r" r# r� s
r� c @ sD e Zd Zddd�Zedd� �Zdd� Zddd �Zd
d � Zd d
� Z dS )r� Nc C s || _ |du r t� | _ || _dS )aZ The future associated to a submitted transfer request via CRT S3 client
:type meta: s3transfer.crt.CRTTransferMeta
:param meta: The metadata associated to the transfer future.
:type coordinator: s3transfer.crt.CRTTransferCoordinator
:param coordinator: The coordinator associated to the transfer future.
N)�_metar� � _coordinator)r r� rw r" r" r# r$ & s
zCRTTransferFuture.__init__c C r� r )r� rZ r" r" r# r� 4 r� zCRTTransferFuture.metac C s
| j �� S r )r� ru rZ r" r" r# ru 8 �
zCRTTransferFuture.donec C � | j �|� d S r )r� rz �r �timeoutr" r" r# rz ; � zCRTTransferFuture.resultc C r� r )r� ra rZ r" r" r# ra > rt zCRTTransferFuture.cancelc C s$ | � � std��| jj|dd� dS )z!Sets the exception on the future.z?set_exception can only be called once the transfer is complete.T)�overrideN)ru r r� r� )r � exceptionr" r" r# r� A s
�z CRTTransferFuture.set_exceptionr� r )
r. r/ r0 r$ r� r� ru rz ra r� r" r" r" r# r� % s
r� c @ s e Zd Zdd� ZdS )�BaseCRTRequestSerializerc C s t d��)al Serialize CRT HTTP requests.
:type transfer_type: string
:param transfer_type: the type of transfer made,
e.g 'put_object', 'get_object', 'delete_object'
:type future: s3transfer.crt.CRTTransferFuture
:rtype: awscrt.http.HttpRequest
:returns: An unsigned HTTP request to be used for the CRT S3 client
zserialize_http_request())�NotImplementedError)r �
transfer_typer� r" r" r# �serialize_http_requestK s z/BaseCRTRequestSerializer.serialize_http_requestN)r. r/ r0 r� r" r" r" r# r� J s r� c @ sV e Zd Zddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Zdd� Z dd� Z
dd� Z dS )� BotocoreCRTRequestSerializerNc C sp || _ |du r i }| �||� |jdi |��| _| jjj�d| j� | jjj�d| j� | jjj�d| j � dS )a� Serialize CRT HTTP request using botocore logic
It also takes into account configuration from both the session
and any keyword arguments that could be passed to
`Session.create_client()` when serializing the request.
:type session: botocore.session.Session
:type client_kwargs: Optional[Dict[str, str]])
:param client_kwargs: The kwargs for the botocore
s3 client initialization.
Nzrequest-created.s3.*zafter-call.s3.*zbefore-send.s3.*r" )
�_session�_resolve_client_config�
create_client�_clientr� �events�register�_capture_http_request�+_change_response_to_serialized_http_request�_make_fake_http_response)r �session�
client_kwargsr" r" r# r$ [ s
�
�
�z%BotocoreCRTRequestSerializer.__init__c C sP d }|� � r
|� � }d|v r|d }ttd�}|r |�|�}||d< d|d< d S )N�config)�signature_version�s3� service_name)�get_default_client_configr r �merge)r r� r� �user_provided_config�
client_configr" r" r# r� t s
z3BotocoreCRTRequestSerializer._resolve_client_configc
C s� t |j�}|j}|jrd||jf }g }|j�� D ]\}}t|t�r*|�||f� q|�|t|d�f� qt j
� |�}d }|j rOt
|j d�rJ|j }nt|j �}t j
j|j|||d�} | S )Nz%s?%szutf-8�seek)�method�path�headers� body_stream)r �urlr� �queryr� �items�
isinstance�strr� �awscrt�http� HttpHeaders�body�hasattrr � HttpRequestr� )
r � aws_request� url_parts�crt_path� headers_list�name�value� crt_headers�crt_body_stream� crt_requestr" r" r# � _crt_request_from_aws_request� s,
�z:BotocoreCRTRequestSerializer._crt_request_from_aws_requestc C sT | � |�}|j�d�d u rt|j�}|j�d|j� |j�d�d ur(|j�d� |S )N�hostz Content-MD5)r� r� �getr r� �set�netloc�remove)r �botocore_http_requestr� r� r" r" r# � _convert_to_crt_http_request� s �
z9BotocoreCRTRequestSerializer._convert_to_crt_http_requestc K s ||j d< d S )N� http_request)�context�r �requestr� r" r" r# r� � rt z2BotocoreCRTRequestSerializer._capture_http_requestc K s |d }|� � |d< d S )Nr� � HTTPRequest)�prepare)r r� �parsedr� r� r" r" r# r� � s zHBotocoreCRTRequestSerializer._change_response_to_serialized_http_requestc K s t j�d di td��S )N�� � )�botocore�
awsrequest� AWSResponse�FakeRawResponser� r" r" r# r� � s �z5BotocoreCRTRequestSerializer._make_fake_http_responsec C s& t | j|�d|j|jd�|j��d S )N)�Bucket�Keyr� r" )�getattrr� rd re rg )r �
client_methodr� r" r" r# �_get_botocore_http_request� s ���z7BotocoreCRTRequestSerializer._get_botocore_http_requestc C s | � ||jj�}| �|�}|S r )r� r� r� r� )r r� r� r� r� r" r" r# r� � s
�
z3BotocoreCRTRequestSerializer.serialize_http_requestr ) r. r/ r0 r$ r� r� r� r� r� r� r� r� r" r" r" r# r� Z s
r� c @ s e Zd Zddd�ZdS )r� � Nc c s � | � |�}|s d S |V qr )�read)r �amt�decode_content�chunkr" r" r# �stream� s �
�zFakeRawResponse.stream)r N)r. r/ r0 r r" r" r" r# r� � s r� c @ sd e Zd ZdZddd�Zedd� �Zdd� Zdd d
�Zdd d
�Z dd� Z
ddd�Z dd� Z dd� Z
dS )r� z-A helper class for managing CRTTransferFutureNc C s0 || _ || _t�� | _d | _d | _t�� | _d S r ) r� � _s3_requestr r r �
_exception� _crt_future�Event� _done_event)r r� �
s3_requestr" r" r# r$ � s
z CRTTransferCoordinator.__init__c C r� r )r rZ r" r" r# r � r� z!CRTTransferCoordinator.s3_requestc C r� r )r
r� rZ r" r" r# �set_done_callbacks_complete� rt z2CRTTransferCoordinator.set_done_callbacks_completec C r� r )r
�waitr� r" r" r# r} � r� z<CRTTransferCoordinator.wait_until_on_done_callbacks_completeFc C sN | j � | �� r
|r|| _W d � d S W d � d S 1 s w Y d S r )r ru r )r r� r� r" r" r# r� � s �"�z$CRTTransferCoordinator.set_exceptionc C s | j r
| j �� d S d S r )r ra rZ r" r" r# ra � s �z CRTTransferCoordinator.cancelc C sp | j r| j �z$z| j�|� W n ty | �� � w W | jr#d | _| j�|� d S | jr1d | _| j�|� w r )r r rz r ra r r� r" r" r# rz � s ���z CRTTransferCoordinator.resultc C s | j d u rdS | j �� S )NF)r ru rZ r" r" r# ru � s
zCRTTransferCoordinator.donec C s || _ | j j| _d S r )r �finished_futurer )r r r" r" r# r� � s z%CRTTransferCoordinator.set_s3_requestr� r r� )r. r/ r0 r� r$ r� r r r} r� ra rz ru r� r" r" r" r# r� � s
r� c @ s( e Zd Zdd� Zdd� Z ddd�ZdS ) rQ c C s || _ || _d S r )�_request_serializer� _os_utils)r rX �os_utilsr" r" r# r$ s
z S3ClientArgsCreator.__init__c
C s� d }d }t t|�� tj�}g } |tjkr+|j}
| j�|
�}t||
|| j�} | � | � n|tj
kr>|j}| j� |�} | |j d< | j
�||�}
|
|||| �|d| |�| �|d�d�S )N�
ContentLengthru �progress)r� �type�
recv_filepath�
send_filepathr� � on_progress)r� r �upper�DEFAULT�
GET_OBJECTrf r �get_temp_filename�RenameTempFileHandlerr� �
PUT_OBJECT�
get_file_sizerg r r� r� )r r� r� rw r� r� r r �s3_meta_request_type�on_done_before_calls�final_filepath�file_ondone_call�data_lenr� r" r" r# r� s@ �
�
��
�z)S3ClientArgsCreator.get_make_request_argsNc s � ���fdd�}|S )Nc sf g }�d ur
|�7 }|t ���7 }� d ur|� 7 }|D ]}�dkr)|| d d� q|| i |�� qd S )Nr r )�bytes_transferred)r )r` r� �callbacks_list�callback�r� �before_subscribers�
callback_typer� r" r# �invoke_all_callbacks. s �zBS3ClientArgsCreator.get_crt_callback.<locals>.invoke_all_callbacksr" )r r� r) r( r� r* r" r' r# r� + s z$S3ClientArgsCreator.get_crt_callbackr� )r. r/ r0 r$ r� r� r" r" r" r# rQ s
$�rQ c @ � e Zd Zdd� Zdd� ZdS )r c C s || _ || _|| _|| _d S r )r� �_final_filename�_temp_filenamerO )r rw �final_filename�
temp_filenamerY r" r" r# r$ B s
z RenameTempFileHandler.__init__c
K sx |d }|r| j �| j� d S z | j �| j| j� W d S ty; } z| j �| j� | j�|� W Y d }~d S d }~ww )Nr� )rO � remove_filer- � rename_filer, r� r� r� )r r� r� r� r" r" r# r+ H s
���z RenameTempFileHandler.__call__N�r. r/ r0 r$ r+ r" r" r" r# r A s r c @ r+ )r� c C s
|| _ d S r )r� rv r" r" r# r$ W r� zAfterDoneHandler.__init__c K r� r )r� r r� r" r" r# r+ Z rt zAfterDoneHandler.__call__Nr2 r" r" r" r# r� V s r� )9�logging�ior r �botocore.awsrequestr� �botocore.sessionr �botocore.configr �botocore.compatr �botocore.exceptionsr � awscrt.httpr� � awscrt.s3r r r � awscrt.ior
r r r
r � awscrt.authr r �s3transfer.exceptionsr �s3transfer.futuresr r �s3transfer.utilsr r r �s3transfer.constantsr r � getLoggerr. �loggerr rL �objectrM r� r� r� r� r� r� rQ r r� r" r" r" r# �<module> sL
�O %k 5>