MINI MINI MANI MO
3
\" @ sX d dl Z d dlZd dlZdgZe jddZG dd dZG dd dZG dd d Z dS )
NRobotFileParser RequestRatezrequests secondsc @ sf e Zd ZdddZdd Zdd Zdd Zd
d Zd d
Zdd Z dd Z
dd Z dd Z dd Z
dS )r c C s, g | _ d | _d| _d| _| j| d| _d S )NFr )entries
default_entry disallow_all allow_allset_url last_checked)selfurl r
*/usr/lib64/python3.6/urllib/robotparser.py__init__ s
zRobotFileParser.__init__c C s | j S )N)r
)r r
r
r mtime$ s zRobotFileParser.mtimec C s dd l }|j | _d S )Nr )timer
)r r r
r
r modified- s zRobotFileParser.modifiedc C s& || _ tjj|dd \| _| _d S )N )r urllibparseurlparsehostpath)r r r
r
r r 5 s zRobotFileParser.set_urlc C s yt jj| j}W nR t jjk
rd } z2|jdkr:d| _n|jdkrT|jdk rTd| _W Y d d }~X n X |j }| j
|j dj d S )N Ti i zutf-8)r r )
r ZrequestZurlopenr errorZ HTTPErrorcoder r readr decode
splitlines)r ferrrawr
r
r r : s
zRobotFileParser.readc C s, d|j kr | jd kr(|| _n | jj| d S )N*)
useragentsr r append)r entryr
r
r
_add_entryG s
zRobotFileParser._add_entryc C s6 d}t }| j x|D ]}|sT|dkr8t }d}n |dkrT| j| t }d}|jd}|dkrr|d | }|j }|sq|jdd}t|dkr|d j j |d< tj j
|d j |d< |d dkr |dkr| j| t }|j j |d d}q|d dkr4|dkr|j
j t|d d d}q|d d krh|dkr|j
j t|d d
d}q|d d kr|dkr|d j j rt|d |_d}q|d d kr|dkr|d jd
}t|dkr|d j j r|d j j rtt|d t|d |_d}qW |dkr2| j| d S )Nr r #:z
user-agentZdisallowFZallowTz crawl-delayz request-rate/)Entryr r( findstripsplitlenlowerr r unquoter% r& rulelinesRuleLineisdigitintdelayr req_rate)r linesstater' lineiZnumbersr
r
r r P sd
zRobotFileParser.parsec C s | j r
dS | jrdS | js dS tjjtjj|}tjjdd|j|j |j
|j f}tjj |}|sfd}x"| j
D ]}|j|rn|j|S qnW | jr| jj|S dS )NFTr r, )r r r
r r r r3
urlunparser ZparamsZqueryZfragmentquoter
applies_to allowancer )r useragentr Z
parsed_urlr' r
r
r can_fetch s$
zRobotFileParser.can_fetchc C s4 | j s d S x | jD ]}|j|r|jS qW | jjS )N)r r r@ r8 r )r rB r' r
r
r crawl_delay s
zRobotFileParser.crawl_delayc C s4 | j s d S x | jD ]}|j|r|jS qW | jjS )N)r r r@ r9 r )r rB r' r
r
r request_rate s
z RobotFileParser.request_ratec C s0 | j }| jd k r || jg }djtt|d S )N
)r r joinmapstr)r r r
r
r __str__ s
zRobotFileParser.__str__N)r )__name__
__module__ __qualname__r r r r r r( r rC rD rE rJ r
r
r
r r s
C c @ s$ e Zd Zdd Zdd Zdd ZdS )r5 c C s> |dkr| rd}t jjt jj|}t jj|| _|| _d S )Nr T)r r r> r r? r rA )r r rA r
r
r r s
zRuleLine.__init__c C s | j dkp|j| j S )Nr$ )r
startswith)r filenamer
r
r r@ s zRuleLine.applies_toc C s | j r
dndd | j S )NZAllowZDisallowz: )rA r )r r
r
r rJ s zRuleLine.__str__N)rK rL rM r r@ rJ r
r
r
r r5 s r5 c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
r- c C s g | _ g | _d | _d | _d S )N)r% r4 r8 r9 )r r
r
r r s zEntry.__init__c C s g }x | j D ]}|jd| q W | jd k r@|jd| j | jd k rj| j}|jd|j d|j |jtt| j |jd dj
|S )Nz User-agent: z
Crawl-delay: zRequest-rate: r, r rF ) r% r& r8 r9 ZrequestsZsecondsextendrH rI r4 rG )r ZretagentZrater
r
r rJ s
z
Entry.__str__c C sF |j dd j }x.| jD ]$}|dkr*dS |j }||krdS qW dS )Nr, r r$ TF)r0 r2 r% )r rB rQ r
r
r r@ s zEntry.applies_toc C s$ x | j D ]}|j|r|jS qW dS )NT)r4 r@ rA )r rO r< r
r
r rA s
zEntry.allowanceN)rK rL rM r rJ r@ rA r
r
r
r r- s
r- )
collectionsZ urllib.parser Zurllib.request__all__
namedtupler r r5 r- r
r
r
r <module>
s 2
OHA YOOOO