Python实现批量下载文件
Python实现批量下载文件
#!/usr/bin/envpython
#-*-coding:utf-8-*-
fromgeventimportmonkey
monkey.patch_all()
fromgevent.poolimportPool
importrequests
importsys
importos
defdownload(url):
chrome='Mozilla/5.0(X11;Linuxi86_64)AppleWebKit/537.36'+
'(KHTML,likeGecko)Chrome/41.0.2272.101Safari/537.36'
headers={'User-Agent':chrome}
filename=url.split('/')[-1].strip()
r=requests.get(url.strip(),headers=headers,stream=True)
withopen(filename,'wb')asf:
forchunkinr.iter_content(chunk_size=1024):
ifchunk:
f.write(chunk)
f.flush()
printfilename,"isok"
defremoveLine(key,filename):
os.system('sed-i/%s/d%s'%(key,filename))
if__name__=="__main__":
iflen(sys.argv)==2:
filename=sys.argv[1]
f=open(filename,"r")
p=Pool(4)
forlineinf.readlines():
ifline:
p.spawn(download,line.strip())
key=line.split('/')[-1].strip()
removeLine(key,filename)
f.close()
p.join()
else:
print'Usage:python%surls.txt'%sys.argv[0]
其他网友的方法：
fromos.pathimportbasename
fromurlparseimporturlsplit
defurl2name(url):
returnbasename(urlsplit(url)[2])
defdownload(url,localFileName=None):
localName=url2name(url)
req=urllib2.Request(url)
r=urllib2.urlopen(req)
ifr.info().has_key('Content-Disposition'):
#IftheresponsehasContent-Disposition,wetakefilenamefromit
localName=r.info()['Content-Disposition'].split('filename=')[1]
iflocalName[0]=='"'orlocalName[0]=="'":
localName=localName[1:-1]
elifr.url!=url:
#ifwewereredirected,therealfilenamewetakefromthefinalURL
localName=url2name(r.url)
iflocalFileName:
#wecanforcetosavethefileasspecifiedname
localName=localFileName
f=open(localName,'wb')
f.write(r.read())
f.close()
download(r'你要下载的python文件的url地址')
以上便是本文给大家分享的全部内容了，小伙伴们可以测试下哪种方法效率更高呢。