3
‡ŒðiÑ  ã               @   s‚   d dl mZmZmZ d dlmZ d dlmZ d dlm	Z
mZ e
e_	ee_dgZG dd„ deƒZG dd„ deƒZG d	d
„ d
eƒZdS )é    )Úabsolute_importÚdivisionÚunicode_literals)Ústr)Úurllib)ÚparseÚrequestÚRobotFileParserc               @   sZ   e Zd ZdZddd„Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ ZdS )r	   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    Ú c             C   s,   g | _ d | _d| _d| _| j|ƒ d| _d S )NFr   )ÚentriesÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked)ÚselfÚurl© r   úE/tmp/pip-build-6m2fasei/future/future/backports/urllib/robotparser.pyÚ__init__   s    
zRobotFileParser.__init__c             C   s   | j S )z·Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   )r   r   r   r   Úmtime&   s    zRobotFileParser.mtimec             C   s   ddl }|j ƒ | _dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)Útimer   )r   r   r   r   r   Úmodified/   s    zRobotFileParser.modifiedc             C   s&   || _ tjj|ƒdd… \| _| _dS )z,Sets the URL referring to a robots.txt file.é   é   N)r   r   r   ÚurlparseÚhostÚpath)r   r   r   r   r   r   7   s    zRobotFileParser.set_urlc             C   s|   yt jj| jƒ}W nH t jjk
rZ } z(|jdkr:d| _n|jdkrJd| _W Y dd}~X nX |j	ƒ }| j
|jdƒjƒ ƒ dS )z4Reads the robots.txt URL and feeds it to the parser.é‘  é“  Ti�  Nzutf-8)r   r   )r   r   Úurlopenr   ÚerrorÚ	HTTPErrorÚcoder   r   Úreadr   ÚdecodeÚ
splitlines)r   ÚfÚerrÚrawr   r   r   r$   <   s    

zRobotFileParser.readc             C   s,   d|j kr| jd kr(|| _n| jj|ƒ d S )NÚ*)Ú
useragentsr   r   Úappend)r   Úentryr   r   r   Ú
_add_entryI   s    

zRobotFileParser._add_entryc             C   st  d}t ƒ }�xN|D �]D}|sL|dkr0t ƒ }d}n|dkrL| j|ƒ t ƒ }d}|jdƒ}|dkrj|d|… }|jƒ }|sxq|jddƒ}t|ƒdkr|d jƒ jƒ |d< tjj	|d jƒ ƒ|d< |d dkrö|dkrà| j|ƒ t ƒ }|j
j|d ƒ d}q|d dk�r*|dk�rX|jjt|d d	ƒƒ d}q|d d
kr|dkr|jjt|d dƒƒ d}qW |dk�rp| j|ƒ dS )z”Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r   é   ú#Nú:z
user-agentZdisallowFZallowT)ÚEntryr.   ÚfindÚstripÚsplitÚlenÚlowerr   r   Úunquoter+   r,   Ú	rulelinesÚRuleLine)r   ÚlinesÚstater-   ÚlineÚir   r   r   r   R   sJ    





zRobotFileParser.parsec             C   s–   | j r
dS | jrdS tjjtjj|ƒƒ}tjjdd|j|j|j	|j
fƒ}tjj|ƒ}|s\d}x"| jD ]}|j|ƒrd|j|ƒS qdW | jr’| jj|ƒS dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr
   ú/)r   r   r   r   r   r8   Ú
urlunparser   ÚparamsÚqueryÚfragmentÚquoter   Ú
applies_toÚ	allowancer   )r   Ú	useragentr   Ú
parsed_urlr-   r   r   r   Ú	can_fetch…   s     
zRobotFileParser.can_fetchc             C   s   dj dd„ | jD ƒƒS )Nr
   c             S   s   g | ]}t |ƒd  ‘qS )Ú
)r   )Ú.0r-   r   r   r   ú
<listcomp>�   s    z+RobotFileParser.__str__.<locals>.<listcomp>)Újoinr   )r   r   r   r   Ú__str__œ   s    zRobotFileParser.__str__N)r
   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r$   r.   r   rI   rN   r   r   r   r   r	      s   
		3c               @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	r:   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c             C   s*   |dkr| rd}t jj|ƒ| _|| _d S )Nr
   T)r   r   rD   r   rF   )r   r   rF   r   r   r   r   £   s    zRuleLine.__init__c             C   s   | j dkp|j| j ƒS )Nr*   )r   Ú
startswith)r   Úfilenamer   r   r   rE   ª   s    zRuleLine.applies_toc             C   s   | j r
dpdd | j S )NZAllowZDisallowz: )rF   r   )r   r   r   r   rN   ­   s    zRuleLine.__str__N)rO   rP   rQ   rR   r   rE   rN   r   r   r   r   r:       s   r:   c               @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )r2   z?An entry has one or more user-agents and zero or more rulelinesc             C   s   g | _ g | _d S )N)r+   r9   )r   r   r   r   r   ³   s    zEntry.__init__c             C   sP   g }x| j D ]}|jd|dgƒ qW x | jD ]}|jt|ƒdgƒ q,W dj|ƒS )NzUser-agent: rJ   r
   )r+   Úextendr9   r   rM   )r   ÚretÚagentr=   r   r   r   rN   ·   s    zEntry.__str__c             C   sF   |j dƒd jƒ }x.| jD ]$}|dkr*dS |jƒ }||krdS qW dS )z2check if this entry applies to the specified agentr?   r   r*   TF)r5   r7   r+   )r   rG   rW   r   r   r   rE   ¿   s    zEntry.applies_toc             C   s$   x| j D ]}|j|ƒr|jS qW dS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r9   rE   rF   )r   rT   r=   r   r   r   rF   Ì   s    

zEntry.allowanceN)rO   rP   rQ   rR   r   rN   rE   rF   r   r   r   r   r2   ±   s
   r2   N)Ú
__future__r   r   r   Zfuture.builtinsr   Zfuture.backportsr   Zfuture.backports.urllibr   Ú_parser   Ú_requestÚ__all__Úobjectr	   r:   r2   r   r   r   r   Ú<module>   s    	