Your IP : 172.28.240.42


Current Path : /usr/lib/python2.7/multiprocessing/
Upload File :
Current File : //usr/lib/python2.7/multiprocessing/pool.pyc


|_c@s8dgZddlZddlZddlZddlZddlZddlmZmZm	Z	ddl
mZmZdZ
dZdZejZdZefed	Zdefd
YZdefdYZd
efdYZdefdYZdefdYZdefdYZdS(tPooliN(tProcesst	cpu_counttTimeoutError(tFinalizetdebugiiicCs
t|S(N(tmap(targs((s*/usr/lib/python2.7/multiprocessing/pool.pytmapstar@sc
Cs}|dks0t|tkr*|dks0t|j}|j}t|drn|jj|j	jn|dk	r||nd}x|dks|rj||krjy
|}Wn"t
tfk
rtdPnX|dkrtdPn|\}	}
}}}
yt
|||
f}Wntk
rI}t|f}nX||	|
|f|d7}qWtd|dS(Nit_writers)worker got EOFError or IOError -- exitingsworker got sentinel -- exitingisworker exiting after %d tasks(tNonettypetinttAssertionErrortputtgetthasattrR	tcloset_readertEOFErrortIOErrorRtTruet	ExceptiontFalse(tinqueuetoutqueuetinitializertinitargstmaxtasksRRt	completedttasktjobtitfuncRtkwdstresultte((s*/usr/lib/python2.7/multiprocessing/pool.pytworkerGs40		

!


cBseZdZeZdddddZdZdZdZdZ	didZ
ddZdd	Zdd
Z
diddZdddZed
ZedZedZedZdZdZdZdZedZedZRS(sH
    Class which supports an async version of the `apply()` builtin
    c
Cs)|jtj|_i|_t|_||_||_||_|dkr|y
t
}Wq|tk
rxd}q|Xn|dkrtdn|dk	rt
|drtdn||_g|_|jtjdtjd|f|_t|j_t|j_|jjtjdtjd|j|j|j|jf|_t|j_t|j_|jjtjdtjd|j|j|jf|_ t|j _t|j _|j jt!||j"d|j|j#|j|j|j|j|j |jfdd|_$dS(	Nis&Number of processes must be at least 1t__call__sinitializer must be a callablettargetRtexitpriorityi(%t
_setup_queuestQueuet
_taskqueuet_cachetRUNt_statet_maxtasksperchildt_initializert	_initargsR
RtNotImplementedErrort
ValueErrorRt	TypeErrort
_processest_poolt_repopulate_poolt	threadingtThreadRt_handle_workerst_worker_handlerRtdaemontstartt
_handle_taskst
_quick_putt	_outqueuet
_task_handlert_handle_resultst
_quick_gett_result_handlerRt_terminate_poolt_inqueuet
_terminate(tselft	processesRRtmaxtasksperchild((s*/usr/lib/python2.7/multiprocessing/pool.pyt__init__qsV
					


		
		
		$
		
cCswt}xjttt|jD]M}|j|}|jdk	r"td||jt	}|j|=q"q"W|S(sCleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        scleaning up worker %dN(
RtreversedtrangetlenR6texitcodeR
RtjoinR(RHtcleanedR R%((s*/usr/lib/python2.7/multiprocessing/pool.pyt_join_exited_workerss"

c
Csxt|jt|jD]}|jdtd|j|j|j|j	|j
f}|jj||jj
dd|_t|_|jtdqWdS(sBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        R'RRt
PoolWorkersadded workerN(RMR5RNR6RR%RFR@R0R1R/tappendtnametreplaceRR<R=R(RHR tw((s*/usr/lib/python2.7/multiprocessing/pool.pyR7s#	
cCs|jr|jndS(sEClean up any exited workers and start replacements for them.
        N(RRR7(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyt_maintain_poolscCsPddlm}||_||_|jjj|_|jjj|_	dS(Ni(tSimpleQueue(
tqueuesRYRFR@R	tsendR?RtrecvRC(RHRY((s*/usr/lib/python2.7/multiprocessing/pool.pyR)s
cCs.|jtkst|j|||jS(s1
        Equivalent of `apply()` builtin
        (R.R-R
tapply_asyncR(RHR!RR"((s*/usr/lib/python2.7/multiprocessing/pool.pytapplyscCs.|jtkst|j|||jS(s/
        Equivalent of `map()` builtin
        (R.R-R
t	map_asyncR(RHR!titerablet	chunksize((s*/usr/lib/python2.7/multiprocessing/pool.pyRsics|jtkst|dkrft|j|jjfdt|DjfS|dksxtt	j
||}t|j|jjfdt|DjfdDSdS(sZ
        Equivalent of `itertools.imap()` -- can be MUCH slower than `Pool.map()`
        ic3s0|]&\}}j||fifVqdS(N(t_job(t.0R tx(R#R!(s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>sc3s0|]&\}}j|t|fifVqdS(N(RbR(RcR Rd(R#(s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>scss"|]}|D]}|Vq
qdS(N((Rctchunktitem((s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>sN(R.R-R
tIMapIteratorR,R+Rt	enumeratet_set_lengthRt
_get_tasks(RHR!R`Rattask_batches((R#R!s*/usr/lib/python2.7/multiprocessing/pool.pytimapscs|jtkst|dkrft|j|jjfdt|DjfS|dksxtt	j
||}t|j|jjfdt|DjfdDSdS(sK
        Like `imap()` method but ordering of results is arbitrary
        ic3s0|]&\}}j||fifVqdS(N(Rb(RcR Rd(R#R!(s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>sc3s0|]&\}}j|t|fifVqdS(N(RbR(RcR Rd(R#(s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>scss"|]}|D]}|Vq
qdS(N((RcReRf((s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>sN(R.R-R
tIMapUnorderedIteratorR,R+RRhRiRRj(RHR!R`RaRk((R#R!s*/usr/lib/python2.7/multiprocessing/pool.pytimap_unorderedscCsV|jtkstt|j|}|jj|jd|||fgdf|S(s>
        Asynchronous equivalent of `apply()` builtin
        N(	R.R-R
tApplyResultR,R+RRbR
(RHR!RR"tcallbackR#((s*/usr/lib/python2.7/multiprocessing/pool.pyR]	s+cs|jtkstt|ds3t|}n|dkr}tt|t|jd\}}|r}|d7}q}nt|dkrd}nt	j
|||}t|j|t|||j
jfdt|DdfS(s<
        Asynchronous equivalent of `map()` builtin
        t__len__iiic3s0|]&\}}j|t|fifVqdS(N(RbR(RcR Rd(R#(s*/usr/lib/python2.7/multiprocessing/pool.pys	<genexpr>#sN(R.R-R
RtlistR
tdivmodRNR6RRjt	MapResultR,R+RRh(RHR!R`RaRptextraRk((R#s*/usr/lib/python2.7/multiprocessing/pool.pyR_s(	cCsotj}xB|jtks6|jrP|jtkrP|jtjdqW|j	j
dtddS(Ng?sworker handler exiting(
R8tcurrent_threadR.R-R,t	TERMINATERXttimetsleepR+RR
R(tpooltthread((s*/usr/lib/python2.7/multiprocessing/pool.pyR:'s*
c
Cs1tj}xt|jdD]\}}d}xt|D]P\}}|jratdPny||Wq>tk
rtdPq>Xq>W|rtd||dqqPqWtdy@td|j	dtdx|D]}	|dqWWntk
r"td	nXtd
dS(Nis'task handler found thread._state != RUNscould not put task on queuesdoing set_length()istask handler got sentinels/task handler sending sentinel to result handlers(task handler sending sentinel to workerss/task handler got IOError when sending sentinelsstask handler exiting(
R8RvtiterRR
RhR.RRR(
t	taskqueueRRRzR{ttaskseqt
set_lengthR Rtp((s*/usr/lib/python2.7/multiprocessing/pool.pyR>4s6	


	






cCstj}xy
|}Wn"ttfk
r@tddSX|jrm|jtks_ttdPn|dkrtdPn|\}}}y||j	||Wqt
k
rqXqx|ri|jtkriy
|}Wn"ttfk
rtddSX|dkr+tdqn|\}}}y||j	||Wqt
k
reqXqWt|drtdy5x.tdD] }|j
jsPn|qWWqttfk
rqXntdt||jdS(	Ns.result handler got EOFError/IOError -- exitings,result handler found thread._state=TERMINATEsresult handler got sentinels&result handler ignoring extra sentinelRs"ensuring that outqueue is not fulli
s7result handler exiting: len(cache)=%s, thread._state=%s(R8RvRRRR.RwR
R
t_settKeyErrorRRMRtpollRN(RRtcacheR{RRR tobj((s*/usr/lib/python2.7/multiprocessing/pool.pyRB[sX

	







ccsCt|}x0ttj||}|s1dS||fVqdS(N(R|ttuplet	itertoolstislice(R!tittsizeRd((s*/usr/lib/python2.7/multiprocessing/pool.pyRjscCstddS(Ns:pool objects cannot be passed between processes or pickled(R2(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyt
__reduce__scCs5td|jtkr1t|_t|j_ndS(Nsclosing pool(RR.R-tCLOSER;(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyRs
	cCs-tdt|_t|j_|jdS(Nsterminating pool(RRwR.R;RG(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyt	terminates
	cCsntd|jttfks%t|jj|jj|jjx|j	D]}|jqVWdS(Nsjoining pool(
RR.RRwR
R;RPRARDR6(RHR((s*/usr/lib/python2.7/multiprocessing/pool.pyRPs



cCsWtd|jjx9|jrR|jjrR|jjtjdqWdS(Ns7removing tasks from inqueue until task handler finishedi(	Rt_rlocktacquiretis_aliveRRR\RxRy(Rttask_handlerR((s*/usr/lib/python2.7/multiprocessing/pool.pyt_help_stuff_finishs



c	
Csstdt|_t|_td|j||t||jsct|dksctt|_|jdtd|j	|rt
|ddrtdx-|D]"}	|	jdkr|	jqqWntd|j	dtd	|j	d|rot
|ddrotd
x;|D]0}	|	jr8td|	j
|	j	q8q8WndS(Nsfinalizing pools&helping task handler/workers to finishisjoining worker handlerRsterminating workerssjoining task handlerg}Ô%ITsjoining result handlersjoining pool workersscleaning up worker %d(RRwR.RRNRR
RR
RPRRORtpid(
tclsR}RRRztworker_handlerRtresult_handlerRR((s*/usr/lib/python2.7/multiprocessing/pool.pyREs2
		
$	










N((((t__name__t
__module__t__doc__RR
RKRRR7RXR)R^RRlRnR]R_tstaticmethodR:R>RBRjRRRRPRtclassmethodRE(((s*/usr/lib/python2.7/multiprocessing/pool.pyRks0	9					
':							RocBsDeZdZdZdZddZddZdZRS(cCsStjtj|_tj|_||_t|_	||_
|||j<dS(N(R8t	ConditiontLockt_condtjob_countertnextRbR,Rt_readyt	_callback(RHRRp((s*/usr/lib/python2.7/multiprocessing/pool.pyRKs			cCs|jS(N(R(RH((s*/usr/lib/python2.7/multiprocessing/pool.pytreadyscCs|jst|jS(N(RR
t_success(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyt
successfulscCsB|jjz |js,|jj|nWd|jjXdS(N(RRRtwaittrelease(RHttimeout((s*/usr/lib/python2.7/multiprocessing/pool.pyRs

	cCs<|j||jstn|jr/|jS|jdS(N(RRRRt_value(RHR((s*/usr/lib/python2.7/multiprocessing/pool.pyR	s
			cCs|\|_|_|jr7|jr7|j|jn|jjzt|_|jjWd|jjX|j	|j
=dS(N(RRRRRRRtnotifyRR,Rb(RHR R((s*/usr/lib/python2.7/multiprocessing/pool.pyRs
	N(	RRRKRRR
RRR(((s*/usr/lib/python2.7/multiprocessing/pool.pyRos				RtcBseZdZdZRS(cCsutj|||t|_dg||_||_|dkrVd|_t|_n||t	|||_dS(Ni(
RoRKRRR
Rt
_chunksizet_number_leftRtbool(RHRRatlengthRp((s*/usr/lib/python2.7/multiprocessing/pool.pyRK$s			cCs|\}}|r||j||j|d|j+|jd8_|jdkr|jrn|j|jn|j|j=|jjzt|_	|jj
Wd|jjXqnWt|_
||_|j|j=|jjzt|_	|jj
Wd|jjXdS(Nii(RRRRR,RbRRRRRRRR(RHR tsuccess_resulttsuccessR#((s*/usr/lib/python2.7/multiprocessing/pool.pyR/s*"	

			

	(RRRKR(((s*/usr/lib/python2.7/multiprocessing/pool.pyRt"s	RgcBs>eZdZdZddZeZdZdZRS(cCsktjtj|_tj|_||_tj	|_
d|_d|_
i|_|||j<dS(Ni(R8RRRRRRbR,tcollectionstdequet_itemst_indexR
t_lengtht	_unsorted(RHR((s*/usr/lib/python2.7/multiprocessing/pool.pyRKPs				cCs|S(N((RH((s*/usr/lib/python2.7/multiprocessing/pool.pyt__iter__ZscCs|jjzy|jj}Wntk
r|j|jkrNtn|jj|y|jj}Wqtk
r|j|jkrtnt	qXnXWd|jj
X|\}}|r|S|dS(N(RRRtpopleftt
IndexErrorRRt
StopIterationRRR(RHRRfRtvalue((s*/usr/lib/python2.7/multiprocessing/pool.pyR]s&

	
	cCs|jjz|j|kr|jj||jd7_xJ|j|jkr|jj|j}|jj||jd7_qAW|jjn
||j|<|j|jkr|j	|j
=nWd|jjXdS(Ni(RRRRRTRtpopRRR,RbR(RHR R((s*/usr/lib/python2.7/multiprocessing/pool.pyRvs

cCs^|jjz<||_|j|jkrH|jj|j|j=nWd|jjXdS(N(RRRRRR,RbR(RHR((s*/usr/lib/python2.7/multiprocessing/pool.pyRis
	
N(	RRRKRR
Rt__next__RRi(((s*/usr/lib/python2.7/multiprocessing/pool.pyRgNs	
		RmcBseZdZRS(cCst|jjzR|jj||jd7_|jj|j|jkr^|j|j=nWd|jj	XdS(Ni(
RRRRTRRRR,RbR(RHR R((s*/usr/lib/python2.7/multiprocessing/pool.pyRs

(RRR(((s*/usr/lib/python2.7/multiprocessing/pool.pyRmst
ThreadPoolcBsBeZddlmZddddZdZedZRS(i(RcCstj||||dS(N(RRK(RHRIRR((s*/usr/lib/python2.7/multiprocessing/pool.pyRKscCs@tj|_tj|_|jj|_|jj|_dS(N(R*RFR@RR?RRC(RH((s*/usr/lib/python2.7/multiprocessing/pool.pyR)scCsW|jjz5|jj|jjdg||jjWd|jjXdS(N(t	not_emptyRtqueuetcleartextendR
t
notify_allR(RRR((s*/usr/lib/python2.7/multiprocessing/pool.pyRs

N((	RRtdummyRR
RKR)RR(((s*/usr/lib/python2.7/multiprocessing/pool.pyRs	(t__all__R8R*RRRxtmultiprocessingRRRtmultiprocessing.utilRRR-RRwtcountRRR
R%tobjectRRoRtRgRmR(((s*/usr/lib/python2.7/multiprocessing/pool.pyt<module>#s(		$2,I