o hu @sdZddlZddlZddlZddlZddlZddlmZddlm Z m Z ddlm Z zddl Z Wne eefy>dZ YnwddlmZddlmZdd lmZmZdd lmZmZmZmZd d lmZed ZedZ e!dZ"Gdddej#j$Z%Gdddej&Z'GdddZ(GdddZ)GdddeZ*GdddeZ+edZ,ddZ-Gdd d eZ.d!d"Z/d*d$d%Z0d&d'Z1e1d(d)Z2dS)+z(This file is largely copied from http.pyN)copy)dumpsloads)urlparseF)_DEFAULT_CALLBACK)register_implementation)AbstractBufferedFileAbstractFileSystem)DEFAULT_BLOCK_SIZE isfilelike nullcontexttokenize)AllBytesz6<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)z-(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)z fsspec.httpc@ eZdZdS)JsHttpExceptionN__name__ __module__ __qualname__rrt/var/www/html/construction_image-detection-poc/venv/lib/python3.10/site-packages/fsspec/implementations/http_sync.pyrs rc@r)StreamIONrrrrrr!src@seZdZdZdddZeddZddZed d Zed d Z d dZ ddZ eddZ eddZ eddZeddZeddZddZdS) ResponseProxyzLooks like a requests responseFcCs||_||_d|_d|_dSN)requeststream_data_headers)selfreqrrrr__init__*s zResponseProxy.__init__cCs8|jdur|jj}|jrt||_|jS||_|jSr)rrresponseto_bytesrr)rbrrrraw0s   zResponseProxy.rawcCst|dr |`dSdS)Nr)hasattrrrrrrclose:s zResponseProxy.closecCs4|jdurtdd|jdD|_|jS)NcSsg|]}|dqS): )split).0_rrr Bsz)ResponseProxy.headers..z )rdictrgetAllResponseHeadersstripr*r'rrrheaders>s zResponseProxy.headerscCs t|jjSr)intrstatusr'rrr status_codeIs zResponseProxy.status_codecCs"|jst|j|j|j|jddSr)okrurlr4reasonr1r'rrrraise_for_statusMs zResponseProxy.raise_for_statuscos" |j|}|r|VndSqr)r%read)r chunksizer,__outrrr iter_contentSs zResponseProxy.iter_contentcCs|jjSr)r statusTextr'rrrr7[szResponseProxy.reasoncCs |jdkS)N)r4r'rrrr5_ zResponseProxy.okcCs |jjjSr)rr" responseURLr'rrrr6cr@zResponseProxy.urlcCs |jSr)contentdecoder'rrrtextgs zResponseProxy.textcCs d|_|jSNF)rr%r'rrrrBlszResponseProxy.contentcCs t|jSr)rrDr'rrrjsonq zResponseProxy.jsonN)F)rrr__doc__r!propertyr%r(r1r4r8r=r7r5r6rDrBrFrrrrr's.           rc@sjeZdZddZ              dddZddZdd Zd d Zd d ZddZ ddZ dS)RequestsSessionShimcCs i|_dSr)r1r'rrrr!vrGzRequestsSessionShim.__init__NcCs>ddlm}m}td|||s|s| s|s|s| rt|r%|r%td|}|r-|nd}|r<|dtj |}|j ||dg|R| rL| |_ |r]| D] \}}|||qR|dd d |_|ry|t|gtd i}||n |rt|tjr|}||gtd i}||n|dt|| d S) Nr)BlobXMLHttpRequestzJS request: %s %szUse json= or data=, not bothr?FAcceptzapplication/octet-stream arraybufferzapplication/json)r)jsrKrLloggerdebugNotImplementedError ValueErrornewurllibparse urlencodeopentimeoutitemssetRequestHeader responseTypertypesend isinstanceioIOBaser9r)rmethodr6paramsdatar1cookiesfilesauthrZallow_redirectsproxieshooksrverifycertrFrKrLr extrakvblobrrrrys8       zRequestsSessionShim.requestcK|jd|fi|S)NGETrrr6kwargsrrrgetzRequestsSessionShim.getcKrr)NHEADrtrurrrheadrxzRequestsSessionShim.headcKrr)NzPOST}rtrurrrpostrxzRequestsSessionShim.postcKrr)NPUTrtrurrrputrxzRequestsSessionShim.putcKrr)NPATCHrtrurrrpatchrxzRequestsSessionShim.patchcKrr)NDELETErtrurrrdeleterxzRequestsSessionShim.delete)NNNNNNNNNNNNNN) rrrr!rrwrzr{r}rrrrrrrJus. 4 rJcseZdZdZdZdZ       d3fdd Zed d Zd d Z e de de fddZ e fddZ d4ddZd4ddZddZd5ddZdefddZdedfd d!Zd"d#Zd$d%Zd&d'Z (     d6d)d*Zd+d,Zd-d.Zd7d/d0Zd1d2ZZS)8HTTPFileSystema< Simple File-System for fetching data via HTTP(S) This is the BLOCKING version of the normal HTTPFileSystem. It uses requests in normal python and the JS runtime in pyodide. ***This implementation is extremely experimental, do not use unless you are testing pyodide/pyscript integration*** )httphttps sync-http sync-https/TN readaheadFc stj|fi||dur|nt|_||_||_||_||_|p"i|_||_ ||_ zddl } t dt|_d|_ Wn*tyf} zddl} t d| | jd i|pUi|_d|_ WYd} ~ nd} ~ wwt|} | dd|_| dd| d d| d d| |_ dS) a Parameters ---------- block_size: int Blocks to read bytes; if 0, will default to raw requests file-like objects instead of HTTPFile instances simple_links: bool If True, will consider both HTML tags and anything that looks like a URL; if False, will consider only the former. same_scheme: True When doing ls/glob, if this is True, only consider paths that have http/https matching the input URLs. size_policy: this argument is deprecated client_kwargs: dict Passed to aiohttp.ClientSession, see https://docs.aiohttp.org/en/stable/client_reference.html For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` storage_options: key-value Any other parameters passed on to requests cache_type, cache_options: defaults used in open NrzStarting JS sessionTz'Starting cpython session because of: %sFuse_listings_cachelistings_expiry_time max_pathsskip_instance_cacher)superr!r block_size simple_links same_schema cache_type cache_options client_kwargsencodedrvrPrQrRrJsession ExceptionrequestsSessionrpopr) rrr same_schemerrrrstorage_optionsrPerrequest_options __class__rrr!s6!        zHTTPFileSystem.__init__cCsdS)Nrrr'rrrfsid szHTTPFileSystem.fsidcCstr tj||jdS|S)N)r)yarlURLrrr6rrr encode_urlszHTTPFileSystem.encode_urlpathreturncCs|dddd}|S)z-For HTTP, we always want to keep the full URLz sync-http://zhttp://z sync-https://zhttps://)replace)clsrrrr_strip_protocols zHTTPFileSystem._strip_protocolcs t|}t|dkr|SdS)N)r_parentlen)rrparrrrrs  zHTTPFileSystem._parentc Ks|j}||t||jj||fi|j}||||j }|j r9t |ddt |D}n ddt |D}t}t|} |D]d} t| trW| d} | drlt| dkrl| jd| j| } | dr|jr| |ddr|| qL| dd|ddddr|| qL| d vr|d|d| dgqL|s|dr|j|dd d }|rd d|DSt|S) NcSg|]}|dqSrrr+urrrr-/z+HTTPFileSystem._ls_real..cSrrrrrrrr-1rrz://rr)z..z../F)detailcSs&g|]}|d|dr dnddqS)Nr directoryfilenamesizer^)endswithrrrrr-Is)rvrupdaterQrRrrwr_raise_not_found_for_statusrDrex2findallexsetrr`tuple startswithrschemenetlocrrstripaddrjoinlstripr_ls_realsorted) rr6rrvkwrrDlinksr<partslrrrr&sD            zHTTPFileSystem._ls_realcKsB|jr||jvr|j|}|S|j|fd|i|}||j|<|S)Nr)rdircacher)rr6rrvr<rrrlsTs   zHTTPFileSystem.lscCs|jdkr t||dS)zU Raises FileNotFoundError for 404s, otherwise uses raise_for_status. iN)r4FileNotFoundErrorr8)rr"r6rrrr\s  z*HTTPFileSystem._raise_not_found_for_statuscKs|j}||t||dus|dur2||krdS|di}|||||d<||d<|jj| |fi|}| |||j S)Nr1Range) rvrrrQrRr_process_limitsrrwrrrB)rr6startendrvrr1rrrrcat_fileds    zHTTPFileSystem.cat_fileiPc Ks|j}||t||jj||fi|}zt|j ddp+|j dd}Wnt t t fy<d}Ynw| ||||t|sQt|d}|j|ddD]} || |t| qXdS)Ncontent-lengthContent-LengthwbF)decode_unicode)rvrrrQrRrrwrr2r1rTKeyError TypeErrorset_sizerr rYr=writerelative_updater) rrpathlpath chunk_sizecallbackrvrrrchunkrrrget_filets(        zHTTPFileSystem.get_filer{c  stfdd}|j}|||}|dvr!td|t|j|} | |fd|i|} || |dS)Nc3sttjrt}d}ntd}d}|C}|r)|dd|dn t|dd|}|rR|V t ||}|s9WddSWddS1s]wYdS)NFrbTrrr) r`rarbr rYrseekgetattrr9rr)contextuse_seekfrrrrrr gen_chunkss(     "z+HTTPFileSystem.put_file..gen_chunks)r{r}z.method has to be either 'post' or 'put', not: re)rvrrlowerrTrrr) rrrrrrcrvrrmethresprrrput_files   zHTTPFileSystem.put_filecCsd}d}|dur&|dkr&|dur| }d}d}n|p ||d}||}n|dur,d}|sW|durH|dkrH|durG|pB||d}||}n|durNd}t|trW|d8}d|d |S) z"Helper for "Range"-based _cat_fileNFrrTrrbytes=-)infor`r2)rr6rrrsuffrrrrs,  zHTTPFileSystem._process_limitscKsZ|j}||zt||jj||fi|}|jdkWSt y,YdSw)Nr?F) rvrrrQrRrrwrr4r)rrrvrrrrrexistss     zHTTPFileSystem.existscKs|j|fi|Sr)r)rrrvrrrisfileszHTTPFileSystem.isfilerc Ks|dkrt|dur |n|j}|j} | ||p%|j|fi|d}|rA|rAt||f|j||||p7|j|p;|j d| St ||f||jd| S)aMake a file-like object Parameters ---------- path: str Full URL with protocol mode: string must be "rb" block_size: int or None Bytes to download in one request; use instance value if None. If zero, will return a streaming Requests file-like instance. kwargs: key-value Any other parameters, passed to requests calls rNr)rrmoderrr)rr) rSrrvrrrHTTPFilerrrHTTPStreamFile) rrrr autocommitrrrrvrrrr_opens:    zHTTPFileSystem._opencCst||j|jS)z;Unique identifier; assume HTTP files are static, unchanging)r rvprotocolrrrrukeyszHTTPFileSystem.ukeyc Ksi}dD]E}z!|t||f||jd|j||ddur&Wn$WqtyI}z|dkr8t||t t |WYd}~qd}~ww|dd|ddiS) aHGet info of URL Tries to access location via HEAD, and then GET methods, but does not fetch the data. It is possible that the server does not supply any size information, in which case size will be given as None (and certain operations on the corresponding file will not work). )rzrw) size_policyrrNrwrrr^r) r _file_inforrrvrwrrrQrRstr)rr6rvrpolicyexcrrrrs2   zHTTPFileSystem.infoc s"ddl}|d}||}|ddkr|dnt|}|ddkr*|dnt|}t||}|dd} t|s_|} d} |rH|d 7}nY||rY| sR|gS|| |iS| s]gSiSd|d|vr|d| d} |d| d} d |vr~dn || dd dd} nd } d |vrdn ||dd dd} |j| f|p| d d d |d| dd dd dd dd dd dd dd dd dd dd  d!d" dd|d#d$|d%d&| d$d'fd(d)tD} | r | St| S)*a Find files by glob-matching. This implementation is idntical to the one in AbstractFileSystem, but "?" is not considered as a character for globbing, because it is so common in URLs, often identifying the "query" part. rNr*[rFrz/*z**rT)maxdepthwithdirsr^\z\\.z\.+z\+//(z\()z\)|z\|z\^$z\${z\{}z\}z[*]{2}z =PLACEHOLDER=z[*]z[^/]*z.*cs.i|]}|dddr||qS)rr)matchrr)r+pallpathspatternrr }s z'HTTPFileSystem.glob..)rerrfindrminr has_magicrrrindexcountrrsubcompilerlist)rrr rvrendsindstarindbraceindrrootdepthind2r<rrrglob7st        (&   zHTTPFileSystem.globc Cs*zt||WSttfyYdSwrE)boolrrrT)rrrrrisdirs zHTTPFileSystem.isdir)TNTrNNF)T)NN)rNNNNNr)rrrrHrsepr!rIrr classmethodrrrrrrrrrrrrrrrrr-r/ __classcell__rrrrrsV ?   .   -  4 PrcsTeZdZdZ      dfdd Zdfdd Zd d Zd d ZddZZ S)ra A file-like object pointing to a remove HTTP(S) resource Supports only reading, with read-ahead of a predermined block-size. In the case that the server does not supply the filesize, only reading of the complete file in one go is supported. Parameters ---------- url: str Full URL of the remote resource, including the protocol session: requests.Session or None All calls will be made within this session, to avoid restarting connections where the server allows this block_size: int or None The amount of read-ahead to do, in bytes. Default is 5MB, or the value configured for the FileSystem creating this file size: None or int If given, this is the size of the file in bytes, and we don't attempt to call the server to find the value. kwargs: all other key-values are passed to requests calls. Nrbytesc sN|dkrtd||_||_||dd|_tjd||||||d| dS)NrzFile mode not supportedrr)fsrrrrrr)rSr6rdetailsrr!) rr4r6rrrrrrrvrrrr!s  zHTTPFile.__init__csj|dkr|jdkr|jdur|j|jks||jdur&|dkr%|n t|j|j|}t|S)a5Read bytes from file Parameters ---------- length: int Read up to this many bytes. If negative, read all content to end of file. If the server has not supplied the filesize, attempting to read only part of the data will raise a ValueError. rN)locr blocksize _fetch_allrrr9)rlengthrrrr9s   z HTTPFile.readcCsrtd|t|jts7|jj|j|j fi|j }| |j }tt |dd|d|_t ||_dSdS)zRead whole file in one shot, without caching This is only called when position is still at zero, and read() is called without a byte-count. zFetch all for N)rfetcherr8re)rQrRr`cacherrrwr4rr6rvr8rBrr)rrr<rrrr9s  zHTTPFile._fetch_allcCsx|dd}td|}|sdS|ddkrd}}ndd |dd D\}}|d dkr1dnt|d }|||fS) zParse the Content-Range header Content-Rangerzbytes (\d+-\d+|\*)/(\d+|\*))NNNrrNcSsg|]}t|qSr)r2)r+xrrrr-rz1HTTPFile._parse_content_range..rr)rwrrr*r2)rr1smrrtotalrrr_parse_content_ranges     zHTTPFile._parse_content_rangec CsLtd|d|d||j}|di}d|d|d|d<td|j|d|jj|j |jfd|i|}|j d krId S| |j d |j d |d}|j d kpq| |j d|kpqt|||k}|ry|j}|S|dkrtdd}g}|ddD] } || |t| 7}qd |d||}|S)a3Download a block of data The expectation is that the server returns only the requested bytes, with HTTP code 206. If this is not the case, we first check the headers, and then stream the output - if the data size is bigger than we requested, an exception is raised. zFetch range for r)rr1rrrz%s : %sirrrrzThe HTTP server doesn't appear to support range requests. Only reading this file from the beginning is supported. Open with block_size=0 for a streaming file interface.iFN)rQrRrvrrr6rrwr4rr4r8r1rBr2rBrTr=appendrr) rrrrvr1rclresponse_is_ranger<rrrr _fetch_ranges: "   zHTTPFile._fetch_range)NNrr3NNr6) rrrrHr!r9r9rBrGr2rrrrrsrz([*[])cCst|}|duSr) magic_checksearch)r?rrrrr ,s r cs8eZdZd fdd ZddZd dd Zd d ZZS)rrNc s||_||_|dkr t|dd|_tjd |||dd||jj|j|fddi|}|j ||| dd|_ d |_ ||_ dS) Nrrr)r4rrrrTiFrr)r6rrTr5rr!rwr4rrr=itleftoverr)rr4r6rrrvrrrrr!2s   zHTTPStreamFile.__init__cOstd)NzCannot seek streaming HTTP file)rT)rargsrvrrrrAszHTTPStreamFile.seekr6cCs|jg}t|j}||ks|dkr9z|j}Wn ty!Ynw|r*||nn|t|7}||ks|dksd|}|dkrP||d|_|d|}nd|_|jt|7_|S)Nrr)rLrrK__next__ StopIterationrDrr7)rnumbufslengr<rrrr9Ds(     zHTTPStreamFile.readcCs|jd|_dS)NT)rr(closedr'rrrr(Zs  zHTTPStreamFile.close)rNrH)rrrr!rr9r(r2rrrrr1s  rcKsT|}|di}d|d|d|d<|j|fd|i|}||jS)Nr1rrrr)rrrwr8rB)rr6rrrvr1rrrr get_range_s rTrzc KsDtd||}|dd}|di}||d<i}|dkr.|j|fd|i|}n|dkr>|j|fd|i|}ntd||d|jvrXt |jd|d <n6d |jvrlt |jd  d d |d <n"d |jvr{t |jd |d <nd|jvrt |jd d d |d <dD]}|j|r|j|||<q|S)zCall HEAD on the server to get details about the file (size/checksum etc.) Default operation is to explicitly allow redirects and use encoding 'identity' (no compression) to get the true size of the target. zRetrieve file size for %sriTr1rzrwz)size_policy must be "head" or "get", got rrr=rrrz content-range)ETagz Content-MD5Digest) rQrRrrrwrzrr8r1r2r*) r6rrrvarrzrrchecksum_fieldrrrris2       rcCs<tdtddtdtddtdtddtdtdddS)NrTclobberrrr)rrrrrrregistersr[cCs,ddlm}td|ddtd|dddS)NrrrTrYr)fsspec.implementations.httprrr\rrr unregisters r^)rz)3rHraloggingr urllib.errorrV urllib.parserrFrrrr ImportErrorModuleNotFoundErrorOSErrorfsspec.callbacksrfsspec.registryr fsspec.specrr fsspec.utilsr r r r cachingrr$rr getLoggerrQerror HTTPErrorrBytesIOrrrJrrrIr rrTrr[r^rrrrsP         NKR . ,