Python实现多线程下载文件的代码实例
实现简单的多线程下载，需要关注如下几点：
1.文件的大小：可以从reponseheader中提取，如“Content-Length:911”表示大小是911字节
2.任务拆分：指定各个线程下载的文件的哪一块，可以通过requestheader中添加“Range:bytes=300-400”（表示下载300~400byte的内容），注意可以请求的文件的range是[0,size-1]字节的。
3.下载文件的聚合：各个线程将自己下载的文件块保存为临时文件，所有线程都完成后，再将这些临时文件按顺序聚合写入到最终的一个文件中。
实现代码：
#!/usr/bin/python
#-*-coding:utf-8-*-
#filename:paxel.py
#FROM:http://jb51.net/code/view/58/full/
#Jaymodifieditalittleandsaveforfurtherpotentialusage.
'''Itisamulti-threaddownloadingtool
Itwasdevelopedfollowingaxel.
Author:volans
E-mail:volansw[at]gmail.com
'''
importsys
importos
importtime
importurllib
fromthreadingimportThread
#incaseyouwanttousehttp_proxy
local_proxies={'http':'http://131.139.58.200:8080'}
classAxelPython(Thread,urllib.FancyURLopener):
'''Multi-threaddownloadingclass.
run()isavituralmethodofThread.
'''
def__init__(self,threadname,url,filename,ranges=0,proxies={}):
Thread.__init__(self,name=threadname)
urllib.FancyURLopener.__init__(self,proxies)
self.name=threadname
self.url=url
self.filename=filename
self.ranges=ranges
self.downloaded=0
defrun(self):
'''vertualfunctioninThread'''
try:
self.downloaded=os.path.getsize(self.filename)
exceptOSError:
#print'neverdownloaded'
self.downloaded=0
#rebuildstartpoind
self.startpoint=self.ranges[0]+self.downloaded
#Thispartiscompleted
ifself.startpoint>=self.ranges[1]:
print'Part%shasbeendownloadedover.'%self.filename
return
self.oneTimeSize=16384 #16kByte/time
print'task%swilldownloadfrom%dto%d'%(self.name,self.startpoint,self.ranges[1])
self.addheader("Range","bytes=%d-%d"%(self.startpoint,self.ranges[1]))
self.urlhandle=self.open(self.url)
data=self.urlhandle.read(self.oneTimeSize)
whiledata:
filehandle=open(self.filename,'ab+')
filehandle.write(data)
filehandle.close()
self.downloaded+=len(data)
#print"%s"%(self.name)
#progress=u'\r...'
data=self.urlhandle.read(self.oneTimeSize)
defGetUrlFileSize(url,proxies={}):
urlHandler=urllib.urlopen(url,proxies=proxies)
headers=urlHandler.info().headers
length=0
forheaderinheaders:
ifheader.find('Length')!=-1:
length=header.split(':')[-1].strip()
length=int(length)
returnlength
defSpliteBlocks(totalsize,blocknumber):
blocksize=totalsize/blocknumber
ranges=[]
foriinrange(0,blocknumber-1):
ranges.append((i*blocksize,i*blocksize+blocksize-1))
ranges.append((blocksize*(blocknumber-1),totalsize-1))
returnranges
defislive(tasks):
fortaskintasks:
iftask.isAlive():
returnTrue
returnFalse
defpaxel(url,output,blocks=6,proxies=local_proxies):
'''paxel
'''
size=GetUrlFileSize(url,proxies)
ranges=SpliteBlocks(size,blocks)
threadname=["thread_%d"%iforiinrange(0,blocks)]
filename=["tmpfile_%d"%iforiinrange(0,blocks)]
tasks=[]
foriinrange(0,blocks):
task=AxelPython(threadname[i],url,filename[i],ranges[i])
task.setDaemon(True)
task.start()
tasks.append(task)
time.sleep(2)
whileislive(tasks):
downloaded=sum([task.downloadedfortaskintasks])
process=downloaded/float(size)*100
show=u'\rFilesize:%dDownloaded:%dCompleted:%.2f%%'%(size,downloaded,process)
sys.stdout.write(show)
sys.stdout.flush()
time.sleep(0.5)
filehandle=open(output,'wb+')
foriinfilename:
f=open(i,'rb')
filehandle.write(f.read())
f.close()
try:
os.remove(i)
pass
except:
pass
filehandle.close()
if__name__=='__main__':
url='http://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg'
output='download.file'
paxel(url,output,blocks=4,proxies={})