o h]@sddlmZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZddlmZddlmZddlmZmZdd lmZmZmZmZed ZGd d d ZGd dde Z!          d5ddZ"ddZ#ddZ$e%ddZ&       d6ddZ' d7d8d%d&Z(d'd(Z)d)d*Z*d+d,Z+d-d.Z,      d9d/d0Z-d1d2Z.Gd3d4d4ej/Z0dS):) annotationsN) has_magic)Path) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr)conf) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathfsspecc@s`eZdZdZ     dddZddZdd Zd d Zd d Ze ddZ ddZ ddZ dS)OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately rbNcCs:||_||_||_t|||_||_||_||_g|_dSN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrrr!_/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/fsspec/core.py__init__As   zOpenFile.__init__cCs$t|j|j|j|j|j|j|jffSr)rrrrrrrrr r!r!r" __reduce__TszOpenFile.__reduce__cCsd|jdS)Nz )rr$r!r!r"__repr__bszOpenFile.__repr__c Cs|jddddd}z |jj|j|d}Wnty1}z t|jr,td|j|d}~ww|g|_|jdurNt |j}|||dd}|j |d|jvrdt ||j |j |jd}|j ||jdS) Ntb)ra%s not found. The URL contains glob characters: you maybe needed to pass expand=True in fsspec.open() or the storage_options of your library. You can also set the config value 'open_expand' before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.r)rrr)rreplaceropenrFileNotFoundErrorrrrr appendPickleableTextIOWrapperrrr)r rfecompressr!r!r" __enter__es2       zOpenFile.__enter__cGs |dSr)close)r argsr!r!r"__exit__s zOpenFile.__exit__cCst|j|jSr)rrrr$r!r!r" full_nameszOpenFile.full_namecCs|S)aMaterialise this as a real open file without context The OpenFile object should be explicitly closed to avoid enclosed file instances persisting. You must, therefore, keep a reference to the OpenFile during the life of the file-like it generates. r3r$r!r!r"r,sz OpenFile.opencCs>t|jD]}d|jvr|js||q|jdS)z#Close all encapsulated file objectsrN)reversedrrclosedflushr4clear)r r0r!r!r"r4s  zOpenFile.close)rNNNN) __name__ __module__ __qualname____doc__r#r%r&r3r6propertyr7r,r4r!r!r!r"r s $    rcsLeZdZdZdddfdd ZddZd d Zfd d Zd dZZ S) OpenFilesaList of OpenFile instances Can be used in a single context, which opens and closes all of the contained files. Normal list access to get the elements works as normal. A special case is made for caching filesystems - the files will be down/uploaded together at the start or end of the context, and this may happen concurrently, if the target filesystem supports it. rNrrcs"||_||_g|_tj|dSr)rrfilessuperr#)r rrr5 __class__r!r"r#szOpenFiles.__init__cCsd|jdur td|j} t|dr|||_|jSt|dr)|jdur)|j}nnq dd|DS)NzContext has already been usedT open_manyrcSsg|]}|qSr!r8.0sr!r!r" z'OpenFiles.__enter__..)r ValueErrorhasattrrIrE)r rr!r!r"r3s    zOpenFiles.__enter__csd|j}fdd|Dd|jvr0 t|dr||jdSt|dr-|jdur-|j}ndSqdS)Ncsg|]}|jqSr!)r6rJr5r!r"rMz&OpenFiles.__exit__..r9TrIr)rrrP commit_manyrE)r r5rr!rQr"r6s   zOpenFiles.__exit__cs,t|}t|trt||j|jdS|S)NrD)rF __getitem__ isinstanceslicerCrr)r itemoutrGr!r"rTs  zOpenFiles.__getitem__cCsdt|dS)Nz )lenr$r!r!r"r&szOpenFiles.__repr__) r>r?r@rAr#r3r6rTr& __classcell__r!r!rGr"rCs  rCrutf8Tc  st|||| || d\} } jdkr| _n&dvr=| r=fdd| D}|D]}z j|ddWq(ty<Yq(wtfdd | Dd S) av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandfiler9csh|]}|qSr!)_parentrKrrr!r" 3rRzopen_files..T)exist_okc s"g|] }t|dqS))rrrrr)rrdrrrrrrr!r"rM:s zopen_files..rD)get_fs_token_pathsr` auto_mkdirmakedirsPermissionErrorrC)urlpathrrrrr^r]r`rrjrakwargsfs_tokenpathsparentsparentr!rhr" open_filess4O     rsc s8ddlm}d|vr0td}g}|dD]}d|vs!||r'||q||dqn|g}g}d}|}t|D]V}| ddpNt |dpNd} t | } | |} | | i||duri |td ifdd | D} | |}d | vrt| |r|}||| | f|}q?||S) Nr)CachingFileSystemz::z .*[^a-z]+.*://r`rbcs*i|]\}}|vs||kr||qSr!r!rKkvkwsr!r" es*z_un_chain..target_protocolr!)fsspec.implementations.cachedrtrecompilesplitmatchr.copyr:popsplit_protocolr_get_kwargs_from_urlsupdatedictitems_strip_protocol issubclassreverse) rrnrtxbitsprX previous_bitbitr`cls extra_kwargskwr!ryr" _un_chainKs@         rc  st|}hdfdd|D}t||}i}tt|D].\}}|\}}}|t|dkr;td i||}q td i|||d<||d<||d<q |d\} }} t|fi|} | | fS) a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. >r]rrrarrrr^csi|] \}}|vr||qSr!r!rv known_kwargsr!r"r{szurl_to_fs..r\target_optionsr|forNr!)rrr enumerater:rYrr ) urlrnchaininkwargsichurlsr`rrm_rr!rr" url_to_fsqs    r open_expandFc KsF|durtn|}td|g|||||||d|} | st|| dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. expand: bool or None Whether to regard file paths containing special glob characters as needing expansion (finding the first match) or absolute. Setting False allows using paths which do embed such characters. If None (default), this argument takes its value from the DEFAULT_EXPAND module variable, which takes its initial value from the "open_expand" config value at startup, which will be False if not set. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations N)rmrrrrr`rrarr!)DEFAULT_EXPANDrsr-) rmrrrrr`rrarnrXr!r!r"r,s D  r,r#str | list[str] | Path | list[Path]rstrr_rreturnstr | list[str]cKsd|vrtdt|fd|i|}t|djddstd|}dd |D}Wd n1s3wYt|trAt|rFt|trJ|dS|S) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) r9z(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=TruecSsg|]}|jqSr!)namerKr0r!r!r"rMszopen_local..N)rOrsgetattrrrUrrr)rrr_ofrErpr!r!r" open_localsrcCs4|dkrt|}|dur|tvrtd|d|S)NinferzCompression type z not supported)rr rO)rmrr!r!r"rs rcCsRt|}d|vr|dd\}}t|dkr||fS|dr%|ddSd|fS)zReturn protocol, path pairrur\zdata::N)rrrY startswith)rmr`rr!r!r"r$s   rcCst|\}}t|}||S)zCReturn only path part of full URL, according to appropriate backend)rrr)rmr`rrr!r!r"strip_protocol1s  rcCsg}t|}d|vrFtdd|Ddkrtdt|t|}|D]}d|vr2|t|||q"||q"t||krD|d|}|S|D]}t|rW|| |qH||qH|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wcss|] }d|vrdVqdS)*r\Nr!)rKrr!r!r" Lsz)expand_paths_if_needed..r\z;When writing data, only one filename mask can be specified.rN) listsumrOmaxrYextend _expand_pathsr.rglob)rprr]rr^expanded_paths curr_pathr!r!r"expand_paths_if_needed8s(     rcst|tttfr|stdttt|}nt|}pi|r%|d<t|p*i}i} t t |D].\} } | \} } }| t |dkrOt di|| } q4t di|| | d<| | d<| | d<q4|d\}}}t |fi| t|tttfrfdd |D}t d d |Ddkrtd |d d |D}n|}t|tttfr|rt||||}n1t|tst|}n'd|vsd|vr|rt|||}nd|vrfdd t|D}n|g}j|fS)aFilesystem, deterministic token, and paths from a urlpath and options. Parameters ---------- urlpath: string or iterable Absolute or relative filepath, URL (may include protocols like ``s3://``), or globstring pointing to data. mode: str, optional Mode in which to open files. num: int, optional If opening in writing mode, number of files we expect to create. name_function: callable, optional If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. storage_options: dict, optional Additional keywords to pass to the filesystem class. protocol: str or None To override the protocol specifier in the URL expand: bool Expand string paths for writing, assuming the path is a directory zempty urlpath sequencer`r\rr|rrcs"g|] }tt|p idqSr)rr)rKu)r_r!r"rMsz&get_fs_token_paths..cSsh|]}|dqS)r\r!rKpcr!r!r"rfrNz%get_fs_token_paths..z$Protocol mismatch getting fs from %scSsg|]}|dqSrr!rr!r!r"rMrNrrrcsg|] }|s|qSr!)isdirrrer!r"rMsNr!)rUrtuplesetrOrnextiterrrr:rYrr rrrsortedr _fs_token)rmrr]r^r_r`raurlpath0rrrrrnested_protocolrrprpchainsr!)rr_r"rigsP       ricsttr>ddkrtddvrtjddur%t|dfddt|D}|t |kr.zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either 1. A list of paths: ['foo.json', 'bar.json', ...] 2. A directory: 'foo/ 3. A path with a '*' in it: 'foo.*.json')rUrcountrOosrjoinrrangerloggerwarningrrrY)rr^r]rpr!rr"rs(   rcs4eZdZdZ     dfdd ZddZZS) r/zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. NFcs$||||||f|_tj|jdSr)r5rFr#)r bufferrrrline_buffering write_throughrGr!r"r#s z PickleableTextIOWrapper.__init__cCs t|jfSr)r/r5r$r!r!r"r%s z"PickleableTextIOWrapper.__reduce__)NNNFF)r>r?r@rAr#r%rZr!r!rGr"r/s  r/) rNr[NNr\NNTT)rNr[NNNN)r)rrrrr_rrr)rr\NNNT)1 __future__riologgingrr~rrpathlibrfsspec.cachingrrrrr r fsspec.compressionr fsspec.configr fsspec.registryr r fsspec.utilsrrrr getLoggerrrrrCrsrrgetrr,rrrrrrir TextIOWrapperr/r!r!r!r"sh       ~< s& 2 W ! 1 N