MINI MINI MANI MO
3
\e @ s8 d dgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z ddlm
Z
m Z dZ dZ
dZej Zdd Zd
d ZG d d
d
eZG dd dZdd ZG dd deZdf ddfddZdd ZG dd d eZG dd deZ e Z G d d d e Z G d d d eZ G d d! d!e Z G d"d deZ!dS )#Pool
ThreadPool N )util) get_context TimeoutError c C s t t| S )N)listmap)args r ,/usr/lib64/python3.6/multiprocessing/pool.pymapstar+ s r c C s t tj| d | d S )Nr r )r itertoolsstarmap)r r r r
starmapstar. s r c @ s e Zd Zdd Zdd ZdS )RemoteTracebackc C s
|| _ d S )N)tb)selfr r r r
__init__6 s zRemoteTraceback.__init__c C s | j S )N)r )r r r r
__str__8 s zRemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r
r 5 s r c @ s e Zd Zdd Zdd ZdS )ExceptionWithTracebackc C s0 t jt|||}dj|}|| _d| | _d S )N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r r r r r
r < s
z ExceptionWithTraceback.__init__c C s t | j| jffS )N) rebuild_excr r )r r r r
__reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r
r ; s r c C s t || _| S )N)r __cause__)r r r r r
r! D s
r! c s, e Zd Z fddZdd Zdd Z ZS )MaybeEncodingErrorc s. t || _t || _tt| j| j| j d S )N)reprr valuesuperr$ r )r r r& ) __class__r r
r P s
zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r
r U s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r( r )r r r r
__repr__Y s zMaybeEncodingError.__repr__)r r r r r r)
__classcell__r r )r( r
r$ L s r$ Fc ' C s |j }| j}t| dr*| jj |jj |d k r:|| d}xH|d ksX|oT||k ry
| } W n$ ttfk
r tj d P Y nX | d krtj d P | \}
} } }
}yd| |
|f}W nB t
k
r } z&|r| t k rt ||j
}d|f}W Y d d }~X nX y||
| |f W nR t
k
rd } z4t||d }tj d| ||
| d|ff W Y d d }~X nX d } }
} } }
}|d7 }qBW tj d | d S )
N_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)putgethasattrr+ close_readerEOFErrorOSErrorr debug Exception_helper_reraises_exceptionr
__traceback__r$ )inqueueoutqueue initializerinitargsZmaxtasksZwrap_exceptionr, r- Z completedtaskjobifuncr kwdsresultewrappedr r r
worker] sD
$ rC c C s | d S )Nr )Zexr r r
r5 s r5 c @ s2 e Zd ZdZdd Zddf ddfddZdd Zd d
Zd d Zd
d Z f i fddZ
d<ddZ d=ddZ d>ddZ
dd Zd?ddZd@d d Zf i ddfd d ZdAd d!ZdBd"d#Zed$d% Zed&d' Zed(d) Zed*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Z ed4d5 Z e d6d7 Z d8d9 Z d:d; Z!dS )Cr Tc O s | j j||S )N)_ctxProcess)r r r? r r r
rE s z Pool.ProcessNc C sn |pt | _| j tj | _i | _t| _|| _ || _
|| _ |d krPt j
pNd}|dk r`td|d k rzt| rztd|| _g | _| j tjtj| fd| _d| j_t| j_| jj tjtj| j| j | j | j| jfd| _ d| j _t| j _| j j tjtj | j | j | jfd| _!d| j!_t| j!_| j!j t"j#| | j$| j| j%| j | j| j| j | j!| jfdd| _&d S )Nr z&Number of processes must be at least 1z initializer must be a callable)targetr T )r Z exitpriority)'r rD
_setup_queuesqueueQueue
_taskqueue_cacheRUN_state_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorcallable TypeError
_processes_pool_repopulate_pool threadingZThreadr _handle_workers_worker_handlerdaemonstart
_handle_tasks
_quick_put _outqueue
_task_handler_handle_results
_quick_get_result_handlerr ZFinalize_terminate_pool_inqueue
_terminate)r processesr9 r: Zmaxtasksperchildcontextr r r
r sT
z
Pool.__init__c C sZ d}xPt tt| jD ]:}| j| }|jd k rtjd| |j d}| j|= qW |S )NFzcleaning up worker %dT)reversedrangelenrX exitcoder r3 r )r Zcleanedr= rC r r r
_join_exited_workers s
zPool._join_exited_workersc C s| xvt | jt| j D ]^}| jt| j| j| j| j | j
| j fd}| jj | |j
jdd|_
d|_|j tjd qW d S )N)rF r rE Z
PoolWorkerTz added worker)rl rW rm rX rE rC rg ra rP rQ rO _wrap_exceptionappendnamereplacer] r^ r r3 )r r= wr r r
rY s zPool._repopulate_poolc C s | j r| j d S )N)ro rY )r r r r
_maintain_pool s zPool._maintain_poolc C s4 | j j | _| j j | _| jjj| _| jjj| _ d S )N)
rD Z SimpleQueuerg ra r+ sendr` r0 recvrd )r r r r
rH s zPool._setup_queuesc C s | j |||j S )N) apply_asyncr- )r r> r r? r r r
apply s z
Pool.applyc C s | j ||t|j S )N)
_map_asyncr r- )r r> iterable chunksizer r r
r
s zPool.mapc C s | j ||t|j S )N)rz r r- )r r> r{ r| r r r
r s z Pool.starmapc C s | j ||t|||S )N)rz r )r r> r{ r| callbackerror_callbackr r r
starmap_async s zPool.starmap_asyncc c sn y0d}x&t |D ]\}}||||fi fV qW W n8 tk
rh } z ||d t|fi fV W Y d d }~X nX d S )Nr ) enumerater4 r5 )r Z
result_jobr> r{ r= xrA r r r
_guarded_task_generation s z Pool._guarded_task_generationr c C s | j tkrtd|dkrFt| j}| jj| j|j|||j f |S t
j |||}t| j}| jj| j|jt ||j f dd |D S d S )NzPool not runningr c s s | ]}|D ]
}|V q
qd S )Nr ).0chunkitemr r r
<genexpr>@ s z Pool.imap.<locals>.<genexpr>)
rN rM rT IMapIteratorrL rK r, r _job _set_lengthr
_get_tasksr )r r> r{ r| r@ task_batchesr r r
imap' s
z Pool.imapc C s | j tkrtd|dkrFt| j}| jj| j|j|||j f |S t
j |||}t| j}| jj| j|jt ||j f dd |D S d S )NzPool not runningr c s s | ]}|D ]
}|V q
qd S )Nr )r r r r r r
r [ s z&Pool.imap_unordered.<locals>.<genexpr>)
rN rM rT IMapUnorderedIteratorrL rK r, r r r r r r )r r> r{ r| r@ r r r r
imap_unorderedB s
zPool.imap_unorderedc C sB | j tkrtdt| j||}| jj|jd|||fgd f |S )NzPool not runningr )rN rM rT ApplyResultrL rK r, r )r r> r r? r} r~ r@ r r r
rx ] s
zPool.apply_asyncc C s | j ||t|||S )N)rz r )r r> r{ r| r} r~ r r r
map_asynch s zPool.map_asyncc
C s | j tkrtdt|ds$t|}|d krTtt|t| jd \}}|rT|d7 }t|dkrdd}tj |||}t
| j |t|||d} | j j
| j| j||d f | S )NzPool not running__len__ r r )r~ )rN rM rT r. r divmodrm rX r r MapResultrL rK r, r r )
r r> r{ Zmapperr| r} r~ Zextrar r@ r r r
rz p s&
zPool._map_asyncc C sT t j }x0|jtks$| jr8|jtkr8| j tjd q
W | j j
d t j d d S )Ng?zworker handler exiting)
rZ current_threadrN rM rL TERMINATEru timesleeprK r, r r3 )poolthreadr r r
r[ s zPool._handle_workersc
C sj t j }xt| jd D ]\}}d }zx|D ]}|jr@tjd P y || W q* tk
r } zD|d d \}
} y||
j| d| f W n t k
r Y nX W Y d d } ~ X q*X q*W |rtjd |r|d nd } || d wP W d d } }}
X qW tjd y:tjd |j
d tjd x|D ]} |d q&W W n t k
rZ tjd Y nX tjd
d S ) Nz'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr ) rZ r iterr- rN r r3 r4 _setKeyErrorr, r2 )
taskqueuer, r8 r cacher ZtaskseqZ
set_lengthr; rA r< idxpr r r
r_ sB
zPool._handle_tasksc C s t j }xy
| }W n" ttfk
r6 tjd d S X |jrJtjd P |d kr^tjd P |\}}}y|| j|| W n tk
r Y nX d } }}q
W x|o|jt kr:y
| }W n" ttfk
r tjd d S X |d krtjd q|\}}}y|| j|| W n tk
r* Y nX d } }}qW t
| drtjd y,x&t dD ]}| j j
spP | q^W W n ttfk
r Y nX tjdt||j d S ) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr0 z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)rZ r r2 r1 r r3 rN r r r r. rl r0 pollrm )r8 r- r r r; r< r= objr r r
rc sZ
zPool._handle_resultsc c s4 t |}x&ttj||}|s"d S | |fV q
W d S )N)r tupler islice)r> itsizer r r r
r s zPool._get_tasksc C s t dd S )Nz:pool objects cannot be passed between processes or pickled)NotImplementedError)r r r r
r" s zPool.__reduce__c C s& t jd | jtkr"t| _t| j_d S )Nz closing pool)r r3 rN rM CLOSEr\ )r r r r
r/ s
z
Pool.closec C s$ t jd t| _t| j_| j d S )Nzterminating pool)r r3 r rN r\ rh )r r r r
terminate s
zPool.terminatec C sD t jd | jj | jj | jj x| jD ] }|j q0W d S )Nz joining pool)r r3 r\ r rb re rX )r r r r r
r s
z Pool.joinc C sD t jd | jj x*|j r>| jj r>| jj tj d qW d S )Nz7removing tasks from inqueue until task handler finishedr )
r r3 Z_rlockacquireis_aliver0 r rw r r )r7 task_handlerr r r r
_help_stuff_finish( s
zPool._help_stuff_finishc
C s( t jd t|_t|_t jd | j||t| t|_|jd t jd tj |k r`|j |rt
|d drt jd x |D ]} | j d kr| j qW t jd tj |k r|j t jd tj |k r|j |ot
|d dr$t jd x,|D ]$} | j
rt jd
| j | j qW d S ) Nzfinalizing poolz&helping task handler/workers to finishzjoining worker handlerr r zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r3 r rN r rm r, rZ r r r. rn r r pid)
clsr r7 r8 r Zworker_handlerr Zresult_handlerr r r r r
rf 1 s6
zPool._terminate_poolc C s | S )Nr )r r r r
__enter___ s zPool.__enter__c C s | j d S )N)r )r exc_typeZexc_valZexc_tbr r r
__exit__b s z
Pool.__exit__)N)N)NNN)r )r )NNN)NNN)"r r r rp rE r ro rY ru rH ry r
r r r r r rx r rz staticmethodr[ r_ rc r r" r/ r r r classmethodrf r r r r r r
r sD 8
. < .c @ s@ e Zd Zdd Zdd Zdd Zddd Zdd
d Zd d
ZdS )r c C s4 t j | _tt| _|| _|| _|| _| || j< d S )N) rZ ZEvent_eventnext job_counterr rL _callback_error_callback)r r r} r~ r r r
r k s
zApplyResult.__init__c C s
| j j S )N)r Zis_set)r r r r
readys s zApplyResult.readyc C s | j S )N)_success)r r r r
successfulv s zApplyResult.successfulNc C s | j j| d S )N)r wait)r timeoutr r r
r z s zApplyResult.waitc C s, | j | | j st| jr"| jS | jd S )N)r r r r _value)r r r r r
r- } s
zApplyResult.getc C sV |\| _ | _| jr$| j r$| j| j | jr>| j r>| j| j | jj | j| j= d S )N)r r r r r setrL r )r r= r r r r
r s
zApplyResult._set)N)N) r r r r r r r r- r r r r r
r i s
r c @ s e Zd Zdd Zdd ZdS )r c C sf t j| |||d d| _d g| | _|| _|dkrLd| _| jj || j= n|| t || | _d S )N)r~ Tr )
r r r r
_chunksize _number_leftr r r bool)r r r| lengthr} r~ r r r
r s
zMapResult.__init__c C s | j d8 _ |\}}|rp| jrp|| j|| j |d | j < | j dkr| jrZ| j| j | j| j= | jj nH| r| jrd| _|| _| j dkr| j r| j | j | j| j= | jj d S )Nr r F)
r r r r r rL r r r r )r r= Zsuccess_resultsuccessr@ r r r
r s"
zMapResult._setN)r r r r r r r r r
r s
r c @ s: e Zd Zdd Zdd Zd ddZeZdd Zd
d ZdS )
r c C sJ t jt j | _tt| _|| _tj | _
d| _ d | _ i | _
| || j< d S )Nr )rZ Z ConditionZLock_condr r r rL collectionsdeque_items_index_length _unsorted)r r r r r
r s
zIMapIterator.__init__c C s | S )Nr )r r r r
__iter__ s zIMapIterator.__iter__Nc C s | j ~ y| jj }W nh tk
r~ | j| jkr6t| j j| y| jj }W n( tk
rx | j| jkrpttY nX Y nX W d Q R X |\}}|r|S |d S )N) r r popleft
IndexErrorr r
StopIterationr r )r r r r r& r r r
r s" zIMapIterator.nextc
C s | j | j|krr| jj| | jd7 _x8| j| jkrd| jj| j}| jj| | jd7 _q.W | j j n
|| j|< | j| jkr| j| j = W d Q R X d S )Nr )
r r r rq r popnotifyr rL r )r r= r r r r
r s
zIMapIterator._setc C s< | j , || _| j| jkr.| j j | j| j= W d Q R X d S )N)r r r r rL r )r r r r r
r s
zIMapIterator._set_length)N) r r r r r r __next__r r r r r r
r s
r c @ s e Zd Zdd ZdS )r c
C sP | j @ | jj| | jd7 _| j j | j| jkrB| j| j= W d Q R X d S )Nr )r r rq r r r rL r )r r= r r r r
r s
zIMapUnorderedIterator._setN)r r r r r r r r
r s r c @ s@ e Zd ZdZedd Zddf fddZdd Zed d
ZdS ) r Fc O s ddl m} || |S )Nr )rE )ZdummyrE )r r? rE r r r
rE s zThreadPool.ProcessNc C s t j| ||| d S )N)r r )r ri r9 r: r r r
r s zThreadPool.__init__c C s, t j | _t j | _| jj| _| jj| _d S )N)rI rJ rg ra r, r` r- rd )r r r r
rH s
zThreadPool._setup_queuesc
C s< | j , | jj | jjd g| | j j W d Q R X d S )N)Z not_emptyrI clearextendZ
notify_all)r7 r r r r r
r s
z ThreadPool._help_stuff_finish) r r r rp r rE r rH r r r r r
r s
)"__all__rZ rI r r rR r r r r r r rM r r countr r r r4 r r r! r$ rC r5 objectr r Z AsyncResultr r r r r r r r
<module>
s@ * \&)@
OHA YOOOO