o h5@sddlmZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZiZedZd d Zd Zdd d Z      dddZGddde Z dddZd ddZddZdS)!) annotationsN)Optional)AsyncFileSystem_run_coros_in_chunks sync_wrapper)DEFAULT_CALLBACK) filesystemget_filesystem_classsplit_protocol url_to_fszfsspec.genericcKst|fi|t|<dSN)r _generic_fs)protocolstorage_optionsrb/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/fsspec/generic.pyset_generic_fssrdefaultcCs|pt}|p t|d}|pi}|dkrt|S|dkr t|S|dkr,t|}|S|dkr@t|fi||i\}}|Std|)zPick instance of backend FSrrgenericcurrentoptionszUnknown FS resolution method: ) default_methodr r rr rr get ValueError)urlmethodrrclsfs_rrr _resolve_fssr Fsize differentc s|p tdi|pi}|||jddd|s&td|jdddfddD} tt| d| rV|j fdd| Dddd d Dttd fd dD} D]J\} } | } | vr|d kr| | <q||dkrt |r|| n| |}| }t |r||n||}||kr| | <q| | q|| | <q|ttdrt\}}|j||fi|tt| d|r| r|| S)aUSync files between two directory trees (experimental) Parameters ---------- source: str Root of the directory tree to take files from. This must be a directory, but do not include any terminating "/" character destination: str Root path to copy into. The contents of this location should be identical to the contents of ``source`` when done. This will be made a directory, and the terminal "/" should not be included. delete_missing: bool If there are paths in the destination that don't exist in the source and this is True, delete them. Otherwise, leave them alone. source_field: str | callable If ``update_field`` is "different", this is the key in the info of source files to consider for difference. Maybe a function of the info dict. dest_field: str | callable If ``update_field`` is "different", this is the key in the info of destination files to consider for difference. May be a function of the info dict. update_cond: "different"|"always"|"never" If "always", every file is copied, regardless of whether it exists in the destination. If "never", files that exist in the destination are not copied again. If "different" (default), only copy if the info fields given by ``source_field`` and ``dest_field`` (usually "size") are different. Other comparisons may be added in the future. inst_kwargs: dict|None If ``fs`` is None, use this set of keyword arguments to make a GenericFileSystem instance fs: GenericFileSystem|None Instance to use if explicitly given. The instance defines how to to make downstream file system instances from paths. Returns ------- dict of the copy operations that were performed, {source: destination} T)withdirsdetailzCan only rsync on a directorycs0g|]\}}|ddkr|vr|qS)type directoryreplace.0av) destination otherfilessourcerr f zrsync..z directories to createcsg|]}|qSrr')r*dirn)r-r/rrr0nsexist_okcSs"i|] \}}|ddkr||qSr%filerr)rrr ps"zrsync..z files to consider for copycs0g|]\}}|vr|ddkr|qSr5r')r*or,)allfilesr-r/rrr0rr1alwaysr"z files to copyz files to deleteNr)GenericFileSystem_strip_protocolfindisdirritemsloggerdebuglenmake_many_dirscopyr(callablepopzipcprm)r/r-delete_missing source_field dest_field update_cond inst_kwargsrkwargsdirs to_deletekr, otherfileinf1v2inf2 source_files target_filesr)r9r-r.r/rrsync+sT4         rYcseZdZdZdZd1fdd ZddZdd Zd2d d ZddZ d3ddZ ddZ ddZ ddZ d4ddZddZdefddZd3d d!ZeeZ " d5d6d/d0ZZS)7r;a^Wrapper over all other FS types This implementation is a single unified interface to be able to run FS operations over generic URLs, and dispatch to the specific implementations using the URL protocol prefix. Note: instances of this FS are always async, even if you never use it with any async backend. rrc s||_tjdi|dS)aK Parameters ---------- default_method: str (optional) Defines how to configure backend FS instances. Options are: - "default": instantiate like FSClass(), with no extra arguments; this is the default instance of that FS, and can be configured via the config system - "generic": takes instances from the `_generic_fs` dict in this module, which you must populate before use. Keys are by protocol - "current": takes the most recently instantiated version of each FS Nr)rsuper__init__)selfrrO __class__rrr[szGenericFileSystem.__init__cCt||j}|||Sr )r runstrip_protocol_parentr\pathrrrrras zGenericFileSystem._parentcCr_r )r rr`r<rbrrrr<s z!GenericFileSystem._strip_protocolNFc st||j}|jr|j|f||dd|IdH}n |j|f||dd|}i}|D]\} } | } || } | | d<| || <q.|rH|St|S)NT)maxdepthr#r$name) r r async_impl_findr=r?rDr`list) r\rcrdr#r$rOroutresultrRr,rerrrrgs4   zGenericFileSystem._findcs^t||j}|jr|j|fi|IdH}n |j|fi|}|}||d|d<|S)Nre)r rrf_infoinforDr`)r\rrOrrirrrrks zGenericFileSystem._infoTcst||j}|jr|j|fddi|IdH}n |j|fddi|}dd|D}|D] }||d|d<q-|r=|Sdd|DS)Nr$TcSsg|]}|qSr)rDr*r8rrrr0z)GenericFileSystem._ls..recSsg|]}|dqS)rerrmrrrr0rn)r rrf_lslsr`)r\rr$rOrrir8rrrros zGenericFileSystem._lscs>t||j}|jr|j|fi|IdHS|j|fi|Sr )r rrf _cat_filecat_file)r\rrOrrrrrqs  zGenericFileSystem._cat_filecsBt||j}|jr|j||fi|IdHS|j||fi|Sr )r rrf _pipe_file pipe_file)r\rcvaluerOrrrrrss  zGenericFileSystem._pipe_filecs^|}t|tr |g}t|d|j}|jr$|j|fi|IdHdS|j|fi|dS)Nr) isinstancestrr rrf_rmrI)r\rrOurlsrrrrrxs zGenericFileSystem._rmcsJtd|t||j}|jr|j||dIdHdS|j||ddS)Nz Make dir %sr3)r@rAr rrf _makedirsmakedirs)r\rcr4rrrrrzs   zGenericFileSystem._makedirscKst||fd|i|dS)z[Sync files between two directory trees See `func:rsync` for more details. rN)rY)r\r/r-rOrrrrYszGenericFileSystem.rsyncic st||j}t||j}||ur+|jr!|j||fi|IdHS|j||fi|Sddd}zt|dr?||dIdHn |j|dfi|} |t | j IdHt|drb||dIdHn |j|dfi|} | j dusx| | j krt | |IdH} | j dur| snt | | IdH|| | j dusx| | j ksxWzt | IdHt | IdHWdStyYdSwzt | IdHt | IdHWwtyYww)Nrnone) blocksize cache_type open_asyncrbwb)r rrf_cp_filecp_filehasattrropenset_size maybe_awaitr!tellreadwriteabsolute_updateclose NameError) r\rurl2r}callbackrOrfs2kwf1f2datarrrrsL     zGenericFileSystem._cp_filecsZt|d|jjrfdd|D}t|IdHdS|D] }j|dq!dS)Nrcsg|] }j|dqS)r3)rz)r*ur4rrrr0Ksz5GenericFileSystem._make_many_dirs..r3)r rrfrr{)r\ryr4corosrrrr_make_many_dirsHsz!GenericFileSystem._make_many_dirsignorepath1 list[str]path2 recursiveboolon_errorrwrd Optional[int] batch_sizetempdir Optional[str]c s|rtt|d|j} t|d|j} | | ur3| jr)| j||fi|IdHS| j||fi|St| || ||||dIdHdS)Nrr)NotImplementedErrorr rrf_copyrD copy_file_op) r\rrrrrdrrrOrrrrrrSs zGenericFileSystem._copy)r)NFF)T)F)FrNNN)rrrrrrrrwrdrrrrr)__name__ __module__ __qualname____doc__rr[rar<rgrkrorqrsrxrzrYrrrrrCr __classcell__rrr]rr;s4       + r;rc sdddl}p |z fddt||D}t||dIdHWtdStw)Nrc s4g|]\}}t||tjtjdqS)r) _copy_file_oposrcjoinuuiduuid4hex)r*u1u2fs1rrrrrr0us z copy_file_op..)r)tempfilemkdtemprGrshutilrmtree) rurl1rrrrrrrrrrrns   rc s|dkrdnt}td||z5|jr|||IdHn||||jr1|||IdHn|||t |td||WdS|y_}ztd||WYd}~dSd}~ww)Nraiserz Copy %s -> %szCopy %s -> %s; donez ignoring cp exception for %s: %s) Exceptionr@rArf _get_fileget_file _put_fileput_filerunlink)rrrrlocalrexerrrrs    rcst|r |IdHS|Sr )inspect iscoroutine)corrrrrs  r)NNN)Fr!r!r"NN)Nrr)r) __future__rrloggingrrrtypingrasynrrr callbacksrcorer r r r r getLoggerr@rrr rYr;rrrrrrrs8      h]