3
Rea~ @ s d dl mZmZmZ d dlmZ d dlmZmZ d dl Z d dl
Z
d dl m Z m
Z
d dlmZ ddlmZmZmZmZ dd lmZ dd
lmZ ed d eD Zed
d eD Zedd eD ZeeddgB Z dZ ej r e dE dkre j ddkst e
j!e ddF e"d d Z#n
e
j!e Z#ddddddddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3h Z$e
j!d4Z%i Z&G d5d6 d6e'Z(d7d8 Z)G d9d: d:e'Z*G d;d< d d>e,Z-G d?d@ d@e'Z.G dAdB dBe'Z/dCdD Z0dS )G )absolute_importdivisionunicode_literals) text_type) http_clienturllibN)BytesIOStringIO) webencodings )EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utilsc C s g | ]}|j d qS )ascii)encode).0item r /builddir/build/BUILDROOT/alt-python36-pip-20.2.4-5.el8.x86_64/opt/alt/python36/lib/python3.6/site-packages/pip/_vendor/html5lib/_inputstream.py
s r c C s g | ]}|j d qS )r )r )r r r r r r s c C s g | ]}|j d qS )r )r )r r r r r r s > .)sumr )r! r r r r' X s z BufferedStream._bufferedBytesc C s< | j j|}| jj| | jd d7 < t|| jd< |S )Nr r )r r/ r appendr r# )r! r. datar r r r, [ s
zBufferedStream._readStreamc C s |}g }| j d }| j d }x|t| jk r|dkr|dks@t| j| }|t|| krn|}||| g| _ n"t|| }|t|g| _ |d7 }|j|||| ||8 }d}q W |r|j| j| dj|S )Nr r )r r# r r( r1 r, join)r! r. remainingBytesrv bufferIndex bufferOffset bufferedData bytesToReadr r r r- b s&
z BufferedStream._readFromBufferN) __name__
__module__ __qualname____doc__r" r& r+ r/ r' r, r- r r r r r 3 s r c K s t | tjs(t | tjjr.t | jtjr.d}n&t| drJt | jdt }n
t | t }|rdd |D }|rvt
d| t | f|S t | f|S d S )NFr/ r c S s g | ]}|j d r|qS ) _encoding)endswith)r xr r r r s z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r)
isinstancer HTTPResponser responseaddbasefphasattrr/ r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargs isUnicode encodingsr r r HTMLInputStream} s
rO c @ sp e Zd ZdZdZdd Zdd Zdd Zd d
Zd d Z d
d Z
dddZ dd Z dd Z
d ddZdd ZdS ) rI zProvides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
i ( c C sZ t jsd| _n tddkr$| j| _n| j| _dg| _tddf| _| j || _
| j dS )a Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
Nu r r zutf-8certain) r supports_lone_surrogatesreportCharacterErrorsr# characterErrorsUCS4characterErrorsUCS2newLineslookupEncoding charEncoding
openStream
dataStreamreset)r! rK r r r r" s
z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N r )r% chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacter)r! r r r rZ s z HTMLUnicodeInputStream.resetc C s t |dr|}nt|}|S )zvProduces a file object from source.
source can be either a file object, local filename or a string.
r/ )rG r )r! rK r r r r rX s
z!HTMLUnicodeInputStream.openStreamc C sT | j }|jdd|}| j| }|jdd|}|dkr@| j| }n ||d }||fS )N
r r r )r% countr_ rfindr` )r! r) r% nLines positionLine lastLinePospositionColumnr r r _position s
z HTMLUnicodeInputStream._positionc C s | j | j\}}|d |fS )z:Returns (line, col) of the current position in the stream.r )ri r] )r! linecolr r r r s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| j stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return
EOF when EOF is reached.
r )r] r\ readChunkr r% )r! r] charr r r rm s
zHTMLUnicodeInputStream.charNc C s |d kr| j }| j| j\| _| _d| _d| _d| _| jj|}| j rX| j | }d | _ n|s`dS t
|dkrt |d }|dksd| kodkn r|d
| _ |d d }| j r| j | |j
dd }|j
d
d }|| _t
|| _d S )Nr[ r Fr
i i z
rb
Tr r r )_defaultChunkSizeri r\ r_ r` r% r] rY r/ ra r# ordrR replace)r! r\ r2 lastvr r r rl s0
z HTMLUnicodeInputStream.readChunkc C s, x&t ttj|D ]}| jjd qW d S )Nzinvalid-codepoint)ranger# invalid_unicode_refindallr^ r1 )r! r2 _r r r rS s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s d}xt j|D ]}|rqt|j }|j }tj|||d rttj|||d }|tkrn| j j
d d}q|dkr|dkr|t |d kr| j j
d qd}| j j
d qW d S )NF zinvalid-codepointTi i r ) ru finditerrq groupstartr isSurrogatePairsurrogatePairToCodepointnon_bmp_invalid_codepointsr^ r1 r# )r! r2 skipmatch codepointr$ char_valr r r rT # s z*HTMLUnicodeInputStream.characterErrorsUCS2Fc
C s yt ||f }W nl tk
r| x |D ]}t|dk s&tq&W djdd |D }|s^d| }tjd| }t ||f< Y nX g }x||j| j| j }|dkr| j | j
krP n0|j }|| j
kr|j | j| j | || _ P |j | j| j d | j
sP qW dj|} | S )z Returns a string of characters from the stream up to but not
including any character in 'characters' or EOF. 'characters' must be
a container that supports the 'in' method and iteration over its
characters.
r[ c S s g | ]}d t | qS )z\x%02x)rq )r cr r r r H s z5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N)charsUntilRegExKeyErrorrq r( r4 recompiler r% r] r\ endr1 rl )
r!
charactersoppositecharsr regexr6 mr rr r r
charsUntil: s2
z!HTMLUnicodeInputStream.charsUntilc C sT |t k rP| jdkr.|| j | _| jd7 _n"| jd8 _| j| j |ksPtd S )Nr r )r r] r% r\ r( )r! rm r r r ungeti s
z HTMLUnicodeInputStream.unget)N)F)r; r< r= r> rp r" rZ rX ri r rm rl rS rT r r r r r r rI s
&
/rI c @ sL e Zd ZdZdddZdd Zd d
Zdd d Zd
d Zdd Z dd Z
dS )rJ zProvides a unicode stream of characters to the HTMLTokenizer.
This class takes care of character encoding and removing or replacing
incorrect byte-sequences and also provides column and line tracking.
N windows-1252Tc C sn | j || _tj| | j d| _d| _|| _|| _|| _|| _ || _
| j || _ | j d dk sbt
| j dS )a Initialises the HTMLInputStream.
HTMLInputStream(source, [encoding]) -> Normalized stream from source
for use by html5lib.
source can be either a file-object, local filename or a string.
The optional encoding parameter must be a string that indicates
the encoding. If specified, that encoding will be used,
regardless of any BOM or later declaration (such as in a meta
element)
i d r N)rX rawStreamrI r" numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrW r( rZ )r! rK r r r r r
useChardetr r r r" s z HTMLBinaryInputStream.__init__c C s& | j d jj| jd| _tj| d S )Nr rr )rW
codec_info streamreaderr rY rI rZ )r! r r r rZ s zHTMLBinaryInputStream.resetc C sL t |dr|}nt|}y|j|j W n tk
rF t|}Y nX |S )zvProduces a file object from source.
source can be either a file object, local filename or a string.
r/ )rG r r+ r& Exceptionr )r! rK r r r r rX s
z HTMLBinaryInputStream.openStreamc C s | j df}|d d k r |S t| jdf}|d d k r:|S t| jdf}|d d k rX|S | j df}|d d k rt|S t| jdf}|d d k r|d jjd r|S t| jdf}|d d k r|S |rtyddl m
} W n t k
r Y nX g }| }xF|j s>| j
j| j}t|ts t|s(P |j| |j| qW |j t|jd }| j
jd |d k rt|dfS t| jdf}|d d k r|S tddfS )NrP r tentativezutf-16)UniversalDetectorencodingz windows-1252) detectBOMrV r r detectEncodingMetar name
startswithr %pip._vendor.chardet.universaldetectorr ImportErrordoner r/ r rB r. r( r1 feedcloseresultr+ r )r! chardetrW r buffersdetectorr r r r r r sR
z'HTMLBinaryInputStream.determineEncodingc C s | j d dkstt|}|d kr&d S |jdkrFtd}|d k stnT|| j d krf| j d df| _ n4| jjd |df| _ | j td| j d |f d S ) Nr rP utf-16beutf-16lezutf-8r z Encoding changed from %s to %s)r r )rW r( rV r r r+ rZ r )r! newEncodingr r r changeEncoding s
z$HTMLBinaryInputStream.changeEncodingc
C s t jdt jdt jdt jdt jdi}| jjd}t|t s r" rZ rX r r r r r r r r rJ z s
(
>"rJ c @ s e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z e e e
Z
dd Ze eZefddZdd Zdd Zdd Zd S )
EncodingByteszString-like object with an associated position and various extra methods
If the position is ever greater than the string length then an exception is
raisedc C s t |tsttj| |j S )N)rB r. r( __new__lower)r! valuer r r r F s zEncodingBytes.__new__c C s
d| _ d S )Nr r )ri )r! r r r r r" J s zEncodingBytes.__init__c C s | S )Nr )r! r r r __iter__N s zEncodingBytes.__iter__c C s> | j d }| _ |t| kr"tn |dk r.t| ||d S )Nr r )ri r#
StopIterationrH )r! pr r r __next__Q s zEncodingBytes.__next__c C s | j S )N)r )r! r r r nextY s zEncodingBytes.nextc C sB | j }|t| krtn |dk r$t|d | _ }| ||d S )Nr r )ri r# r rH )r! r r r r previous] s zEncodingBytes.previousc C s | j t| krt|| _ d S )N)ri r# r )r! r r r r setPositionf s zEncodingBytes.setPositionc C s* | j t| krt| j dkr"| j S d S d S )Nr )ri r# r )r! r r r getPositionk s
zEncodingBytes.getPositionc C s | | j | j d S )Nr )r )r! r r r getCurrentByteu s z EncodingBytes.getCurrentBytec C sL | j }x:|t| k r@| ||d }||kr6|| _|S |d7 }qW || _dS )z Skip past a list of charactersr N)r r# ri )r! r r r r r r r z s zEncodingBytes.skipc C sL | j }x:|t| k r@| ||d }||kr6|| _|S |d7 }qW || _d S )Nr )r r# ri )r! r r r r r r skipUntil s zEncodingBytes.skipUntilc C s( | j || j}|r$| jt|7 _|S )zLook for a sequence of bytes at the start of a string. If the bytes
are found return True and advance the position to the byte after the
match. Otherwise return False and leave the position alone)r r r# )r! r. r6 r r r
matchBytes s zEncodingBytes.matchBytesc C s> y | j || jt| d | _W n tk
r8 tY nX dS )zLook for the next sequence of bytes matching a given sequence. If
a match is found advance the position to the last byte of the matchr T)indexr r# ri
ValueErrorr )r! r. r r r jumpTo s
zEncodingBytes.jumpToN)r; r< r= r> r r" r r r r r r propertyr r currentBytespaceCharactersBytesr r r r r r r r r B s
r c @ sX e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd d
Z dd Z
dd Z dd Z dS )r z?Mini parser for detecting character encoding from meta elementsc C s t || _d| _dS )z3string - the data to work on for encoding detectionN)r r2 r )r! r2 r r r r" s
zEncodingParser.__init__c C s d| j krd S d| jfd| jfd| jfd| jfd| jfd| jff}x| j D ]|}d}y| j jd W n tk
rx P Y nX xD|D ]<\}}| j j|ry | }P W q tk
r d}P Y qX qW |sJP qJW | j S ) Ns )r2 r )r! r r r r s z EncodingParser.handleCommentc C s | j jtkrdS d}d }x| j }|d kr.dS |d dkr^|d dk}|r|d k r|| _dS q|d dkr|d }t|}|d k r|| _dS q|d dkrtt|d }|j }|d k rt|}|d k r|r|| _dS |}qW d S ) NTFr s
http-equivr s content-types charsets content) r2 r r getAttributer rV ContentAttrParserr parse)r! hasPragmapendingEncodingattrtentativeEncodingcodec
contentParserr r r r s: zEncodingParser.handleMetac C s
| j dS )NF)handlePossibleTag)r! r r r r s z%EncodingParser.handlePossibleStartTagc C s t | j | jdS )NT)r r2 r )r! r r r r s
z#EncodingParser.handlePossibleEndTagc C sf | j }|jtkr(|r$|j | j dS |jt}|dkrD|j n | j }x|d k r`| j }qNW dS )NTr )r2 r asciiLettersBytesr r r spacesAngleBracketsr )r! endTagr2 r r r r r r s
z EncodingParser.handlePossibleTagc C s | j jdS )Nr )r2 r )r! r r r r s zEncodingParser.handleOtherc C s | j }|jttdgB }|dks2t|dks2t|d kr>dS g }g }xt|dkrX|rXP nX|tkrl|j }P nD|d
krdj|dfS |tkr|j|j n|dkrdS |j| t
|}qHW |dkr|j dj|dfS t
| |j }|d krT|}xt
|}||kr(t
| dj|dj|fS |tkrB|j|j n
|j| qW nJ|dkrldj|dfS |tkr|j|j n|dkrdS |j| x^t
|}|t krdj|dj|fS |tkr|j|j n|dkrdS |j| qW dS ) z_Return a name,value pair for the next attribute in the stream,
if one is found, or None /Nr r =r3 ' ")r N)r r )r r )
r2 r r frozensetr# r( r4 asciiUppercaseBytesr1 r r r r )r! r2 r attrName attrValue quoteCharr r r r sh
zEncodingParser.getAttributeN)
r; r< r= r> r" r r r r r r r r r r r r r s $r c @ s e Zd Zdd Zdd ZdS )r c C s t |tst|| _d S )N)rB r. r( r2 )r! r2 r r r r" a s zContentAttrParser.__init__c C s y| j jd | j jd7 _| j j | j jdks8d S | j jd7 _| j j | j jdkr| j j}| j jd7 _| j j}| j j|r| j || j j S d S nF| j j}y | j jt | j || j j S tk
r | j |d S X W n tk
r d S X d S )Ns charsetr r r r )r r )r2 r r r r r r r )r! quoteMark oldPositionr r r r e s.
zContentAttrParser.parseN)r; r< r= r" r r r r r r ` s r c C s` t | tr.y| jd} W n tk
r, dS X | dk rXy
tj| S tk
rT dS X ndS dS )z{Return the python codec name corresponding to an encoding or None if the
string doesn't correspond to a valid encoding.r N)rB r. decodeUnicodeDecodeErrorr
lookupAttributeError)r r r r rV s
rV r r )1
__future__r r r Zpip._vendor.sixr pip._vendor.six.movesr r r r ior r pip._vendorr
constantsr r
r r r r[ r r r r r r invalid_unicode_no_surrogaterQ rc r( r evalru r~ ascii_punctuation_rer objectr rO rI rJ r. r r r rV r r r r sP
J g Ib ='