o h!@sddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZddZdaGd d d e Zed gd Gd ddejZGdddeZdS)N)suppress)cached_propertywraps)parse_qs)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizecstfdd}|S)Nc s^z|i|WSty.}z|js|j^}}t|tr)d|vr)ttj||d}~ww)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)r kwargs exceptionmessagefuncp/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/fsspec/implementations/arrow.pywrappers z wrap_exceptions..wrapper)r)rrrrrwrap_exceptionss rcseZdZdZdZfddZeddZeddZ e d d Z d-d d Z ddZ ddZddZeddZeddZeddZed.ddZed/dd Zed0d!d"Zed-d#d$Zed%d&Zed'd(Zd1fd)d* Zfd+d,ZZS)2ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c s$tda||_tjdi|dS)Npyarrowr)rPYARROW_VERSIONfssuper__init__)selfrr __class__rrr!3szArrowFSWrapper.__init__cCs|jjSN)r type_namer"rrrprotocol9szArrowFSWrapper.protocolcCsdt|jj|jjS)Nhdfs_)r rhostportr'rrrfsid=szArrowFSWrapper.fsidcCs*t|}|d}|dr|dd}|S)Npathz//)r startswith)clsr-opsrrr_strip_protocolAs   zArrowFSWrapper._strip_protocolFc sJ|}ddlm}fddj||D}|r|Sdd|DS)Nr) FileSelectorcsg|]}|qSr) _make_entry.0entryr'rr Nsz%ArrowFSWrapper.ls..cSsg|]}|dqS)namerr5rrrr8Us)r2 pyarrow.fsr3r get_file_info)r"r-detailrr3entriesrr'rlsJs   zArrowFSWrapper.lscKs$||}|j|g\}||Sr%)r2rr;r4)r"r-rinforrrr?Ws  zArrowFSWrapper.infocCs0||}z||WdStyYdSw)NFT)r2r?rr"r-rrrexists\s   zArrowFSWrapper.existscCslddlm}|j|jurd}n|j|jurd}n|j|jur*ttjt tj|j d}|j |j ||j dS)Nr)FileType directoryfileother)r9sizetypemtime)r:rBrG DirectoryFileNotFoundrrrosstrerrorr-rFrH)r"r?rBkindrrrr4es    zArrowFSWrapper._make_entryc Ks||d}||d}||d[}|dtd}z%||d}t||Wdn1s9wY|j ||Wn"t yht t |j |Wd1sbwYwWddS1stwYdS)Nrrbz.tmp.wb)r2rstrip_opensecrets token_hexopenshutil copyfileobjrmove BaseExceptionrr delete_file)r"path1path2rlstream tmp_fnamerstreamrrrcp_filexs&   "zArrowFSWrapper.cp_filecKs2||d}||d}|j||dS)Nr)r2rRrrY)r"r\r]rrrrmvszArrowFSWrapper.mvcC||}|j|dSr%)r2rr[r@rrrrm_file zArrowFSWrapper.rm_fileNcCsF||d}||r|r|j|dStd|j|dS)Nrz0Can't delete directories without recursive=False)r2rRisdirr delete_dir ValueErrorr[)r"r- recursivemaxdepthrrrrms  zArrowFSWrapper.rmrOTc Ks|dkr|r |jj}n|jj}n|dkr|jj}n|dkr"|jj}ntd|i}|dks1|s@ttdddkr@d|d<||fi|}t |||||fi|S) NrOrQabz'unsupported mode for Arrow filesystem: .r compression) ropen_input_fileopen_input_streamopen_output_streamopen_append_streamrhintrsplit ArrowFile) r"r-mode block_sizeseekablermethod_kwargsstreamrrrrSs     zArrowFSWrapper._opencKs4||}|r|j|dddS|jj|dddS)NT)exist_okFri)r2makedirsr create_dir)r"r-create_parentsrrrrmkdirs zArrowFSWrapper.mkdircCs||}|jj|dddS)NTr~)r2rr)r"r-r}rrrrs zArrowFSWrapper.makedirscCrcr%)r2rrgr@rrrrmdirrezArrowFSWrapper.rmdircCs||}|j|jSr%)r2rr;rHr@rrrmodifieds zArrowFSWrapper.modifiedc s&|dv|d<tj|fddd|S)N)Nrry)startend)r cat_file)r"r-rrrr#rrrs zArrowFSWrapper.cat_filec s"d|d<tj||fi|dS)NFry)r get_file)r"rpathlpathrr#rrrszArrowFSWrapper.get_file)F)FN)rONT)T)NN)__name__ __module__ __qualname____doc__ root_markerr!propertyr(rr, classmethodr2r>r?rAr4rrarbrdrkrSrrrrrr __classcell__rrr#rr(sD                rr|) readseektellwritereadablewritablecloserFryc@s&eZdZdddZddZddZdS) rvNcKs.||_||_||_||_||_|_||_dSr%)r-rwrr| blocksizerxr)r"rr|r-rwrxrrrrr!s   zArrowFile.__init__cCs|Sr%rr'rrr __enter__szArrowFile.__enter__cGs|Sr%)r)r"r rrr__exit__szArrowFile.__exit__r%)rrrr!rrrrrrrvs  rvcs>eZdZdZdZ      d fdd Zed d ZZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspechdfsdefaultrNc  s:ddlm}|||||||d} tjdd| i|dS)a; Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication replication: int set replication factor of file for write operations. default value is 3. extra_conf: None or dict Passed on to HadoopFileSystem r)r)r*r+user kerb_ticket replication extra_confrNr)r:rr r!) r"r*r+rrrrrrrr#rrr!s zHadoopFileSystem.__init__cCst|}i}|ddr|d|d<|ddr|d|d<|ddr*|d|d<|ddrFt|d}|ddrFt|dd|d<|S)Nr*usernamerr+ url_queryrr)rgetrrt)r-r1outqueriesrrr_get_kwargs_from_urls"s         z&HadoopFileSystem._get_kwargs_from_urls)rrNNrN) rrrrr(r! staticmethodrrrrr#rrs'r)riorLrTrW contextlibr functoolsrr urllib.parser fsspec.specr fsspec.utilsrrr r rrrIOBaservrrrrrs(   .