python基于queue和threading实现多线程下载实例

  #download worker 
  queue_download = Queue.Queue(0) 
  DOWNLOAD_WORKERS = 20 
  for i in range(DOWNLOAD_WORKERS): 
    DownloadWorker(queue_download).start() #start a download worker 
     
  for md5 in MD5S: 
    queue_download.put(md5) 
  for i in range(DOWNLOAD_WORKERS): 
    queue_download.put(None) 

import threading 
import Queue 
import md5query 
import DOM 
import os,sys 

class DownloadWorker(threading.Thread): 
  """""" 
 

  def __init__(self,queue): 
    """Constructor""" 
    self.__queue = queue 
    threading.Thread.__init__(self) 
 
 
  def run(self): 
    while 1: 
      md5 = self.__queue.get() 
      if md5 is None: 
        break #reached end of queue 
      #this is a time-cost produce 
      self._down(md5) 
 
      print "task:",md5,"finished" 
 
  def _down(self,md5): 
    config = { 
      'input':sys.stdin,'output':'./samples','location':'xxx','has-fn':False,'options':{'connect.timeout':60,'timeout':3600},'log':file('logs.txt','w'),} 
    print 'download %s...' % (md5) 
    try: 
      data = downloadproc(config['location'],config['options'])#我的下载过程 
      if data: 
        dom,fileData = md5query.splited(data) 
        filename = md5 
        if config['has-fn']: 
          filename = '%s_%s' % (md5,dom.nodeValue2('xxxxxxx','').encode('utf-8'))#这是我的下载的方法 
        f = file(os.path.join(config['output'],filename),'w') 
        f.write(fileData) 
        f.close() 
 
        print '%stok' % (md5) 
      else: 
        print>>config['log'],'%st%s' % (md5,'failed') 
    except Exception,e: 
      print>>config['log'],str(e))