带有workerpool模块的多线程

Question

import os
import urllib
import workerpool

from datetime import datetime

class DownloadJob(workerpool.Job):    

    def __init__(self, fa):
       self.fa = fa 

    def run(self):        
       f = open(self.fa + '.txt','w')
       f.write('Example Note.......')
       f.close()  


pool = workerpool.WorkerPool(size=5)

def workfile():        
    range1 = 51
    range2 = 102
    fam1 = 555
    fam2 = 833

    ranges = range2 -range1
    fams = fam2 -fam1
    workname = "Python"

    path = os.getcwd()

    os.system('mkdir ' + str(workname))
    sTime = datetime.now()

    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)
            pool.shutdown()
            pool.wait()                                              

        print 'Elapsed Time: %s' % (datetime.now() - sTime)

        z = open('info.txt','w')
        z.write('Elapsed Time: %s' % (datetime.now() - sTime))
        z.close()

    os.chdir(path + '\\' + str(workname))

    tumSure = open('info.txt','w')
    tumSure.write('Elapsed All Time: %s' % (datetime.now() - sTime))
    tumSure.close()

    print 'All Time: %s' % (datetime.now() - sTime)        
    print 'Workname : %s downloaded.' % (workname)

    quit()

workfile()

大家好，

我有上面的代码，我想使用线程逻辑来创建文件。文件夹编号以range1 开头，即51。在此目录中创建名称为555.txt 到833.txt 的文本文件。但是在创建名为52 的文件夹后，它会停止，无法创建555.txt 到833.txt。

它认为它停止是因为

pool.shutdown()
pool.wait()

如何让循环不停地继续？

Answer 1

我认为你应该只在你完成后才关闭池，即在 for a 循环之后，甚至可能在 try...finally 子句中。

看起来是这样的：

try:
    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)
finally:
    pool.shutdown()
    pool.wait()

这样pool关机

a) 仅在完成所有池放置并且 b) 即使发生异常以干净地关闭也会发生。

如果池有上下文管理器，那就更简单了。但是 AFAICS，it hasn't。 否则，你可以这样做

with pool:
    for a in range(ranges + 1):
        os.chdir(path + '\\' + str(workname))
        os.system('mkdir ' + str(range1 + a))
        os.chdir(path + '\\' + str(workname) + '\\' + str(range1 + a))

        for b in range(fams + 1):
            fa = str(fam1 + b)
            job = DownloadJob(fa)
            pool.put(job)

如果你愿意，你可以做

from contextlib import contextmanager

@contextmanager
def shutdown_wait(pool):
    try:
        yield pool
    finally:
        pool.shutdown()
        pool.wait()

...
with shutdown_wait(pool):
    for a ... [as above]