python实现批量下载文件
#!/usr/bin/env python
# -*- coding:utf-8 -*-
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
import requests
import sys
import os
def download(url):
chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' +
'(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
headers = {'User-Agent': chrome}
filename = url.split('/')[-1].strip()
r = requests.get(url.strip(), headers=headers, stream=True)
with open(filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
print filename,"is ok"
def removeLine(key, filename):
os.system('sed -i /%s/d %s' % (key, filename))
if __name__ =="__main__":
if len(sys.argv) == 2:
filename = sys.argv[1]
f = open(filename,"r")
p = Pool(4)
for line in f.readlines():
if line:
p.spawn(download, line.strip())
key = line.split('/')[-1].strip()
removeLine(key, filename)
f.close()
p.join()
else:
print 'Usage: python %s urls.txt' % sys.argv[0]
其他网友的方法：
from os.path import basename
from urlparse import urlsplit
def url2name(url):
return basename(urlsplit(url)[2])
def download(url, localFileName = None):
localName = url2name(url)
req = urllib2.Request(url)
r = urllib2.urlopen(req)
if r.info().has_key('Content-Disposition'):
# If the response has Content-Disposition, we take file name from it
localName = r.info()['Content-Disposition'].split('filename=')[1]
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
elif r.url != url:
# if we were redirected, the real file name we take from the final URL
localName = url2name(r.url)
if localFileName:
# we can force to save the file as specified name
localName = localFileName
f = open(localName, 'wb')
f.write(r.read())
f.close()
download(r'你要下载的python文件的url地址')
以上便是本文给大家分享的全部内容了，小伙伴们可以测试下哪种方法效率更高呢。
新落叶冰点万能企业网站生成系统2.1(中英文双语版)
下载
新落叶冰点万能企业网站生成系统部分功能： 菜单与栏目管理(设计系统) 栏目管理|添加栏目 配置当前模板的栏目内部链接文件 新闻中心 管理 (新闻文章类频道) 添加分类|管理分类 添加新闻文章|管理新闻文章 添加专题|管理专题 新闻文章批量移动 管理评论 生成html 管理上传文件|清理 下载中心 管理 (下载类频道) 添加分类|管理分类 添加下载|管理下载 添加专题|管理专题 批量移动 管理评
