Python实现的文轩网爬虫完整示例
本文实例讲述了Python实现的文轩网爬虫。分享给大家供大家参考，具体如下：
encoding=utf8
importpymysql
importtime
importsys
importrequests
importos
#捕获错误
importtraceback
importtypes
#将html实体化
importcgi
importwarnings
reload(sys)
sys.setdefaultencoding('utf-8')
frompyqueryimportPyQueryaspq
fromlxmlimportetree
sys.setdefaultencoding('utf-8')
#屏蔽错误
warnings.filterwarnings("ignore")
#下载图片
defdowloadPic(imageUrl,filePath):
r=requests.get(imageUrl,timeout=60)
status=r.status_code
ifstatus==404:
return404
withopen(filePath,"wb")ascode:
code.write(r.content)
#根据详情页地址抓取数据并插入数据库
defgetData(final_url):
file_open=open('./url.txt','w')
file_open.write(final_url)
file_open.close()
#链接数据库
conn=pymysql.connect(host='127.0.0.1',port=3306,user='root',passwd='root',db='bookinfo',charset='utf8')
#设置浮标
cursor=conn.cursor(cursor=pymysql.cursors.DictCursor)
#解析详情页面
try:
detail_url=final_url
c=pq(detail_url)
head=c('html').attr('xmlns')
err='http://www.w3.org/1999/xhtml'
err1='http://www.winxuan.com/cms/2016db_sh'
ifhead==errorhead==err1:
return'back'
exceptException,e:
return'back'
i=0
whilei<12:
text=c('#page').find('.cont').find('li').eq(i).text()
text=text.replace('','')
if'ISBN'intext:
isbn=text.replace('ISBN：','')
isbn=isbn.strip()
sel='selectcount(*)frombi_bookwhereisbn='+isbn
cursor.execute(sel)
result=cursor.fetchone()
count=result['count(*)']
ifcount!=0:
printu'已存在'
return'back'
if'isbn：'intext:
isbn=text.replace('isbn：','')
isbn=isbn.strip()
sel='selectcount(*)frombi_bookwhereisbn='+isbn
cursor.execute(sel)
result=cursor.fetchone()
count=result['count(*)']
ifcount!=0:
printu'已存在'
return'back'
if'作者：'intext:
author=text.replace('作者：','')
if'出版社：'intext:
press_name=text.replace('出版社：','')
if'版次：'intext:
edition=text.replace('版次：','')
if'印次：'intext:
impressions=text.replace('印次：','')
if'装帧：'intext:
packaging=text.replace('装帧：','')
if'开本：'intext:
size=text.replace('开本：','')
if'出版时间：'intext:
press_time=text.replace('出版时间：','')
press_time=press_time.strip()
ifpress_time=='无':
press_time='1970-01-01'
if'印刷时间：'intext:
print_time=text.replace('印刷时间：','')
print_time=print_time.strip()
ifprint_time=='无':
print_time='1970-01-01'
if'页数：'intext:
page_num=text.replace('页数：','')
if'字数：'intext:
word_num=text.replace('字数：','')
i+=1
if('author'inlocals().keys())==False:
author=''
if('press_time'inlocals().keys())==False:
press_time='1970-01-01'
if('print_time'inlocals().keys())==False:
print_time='1970-01-01'
if('impressions'inlocals().keys())==False:
impressions=''
if('edition'inlocals().keys())==False:
edition=''
if('page_num'inlocals().keys())==False:
page_num=''
if('word_num'inlocals().keys())==False:
word_num=''
if('packaging'inlocals().keys())==False:
packaging=''
if('size'inlocals().keys())==False:
size=''
if('press_name'inlocals().keys())==False:
press_name=''
#暂无图片地址
none_img='http://static.winxuancdn.com/goods/sml_blank.jpg'
#获取大小图地址
big_path=c('.info-side').find('.img').find('a').find('img').attr('src')
ifbig_pathisNone:
return'back'
elifbig_path==none_img:
big_path=''
small_path=''
else:
small_path=big_path.replace('_16','_11')
#获取分类
#先获取a标签html
ahtml=c('#page').find('.base-nav').eq(0).html()
#解析a标签html
cate=pq(ahtml)
#获取分类的最后一个分类
category=cate('a:last').text()
#获取书名
name=c('.info-main').find('.name').eq(0).find('h1').eq(0).text()
name=name.strip()
#获取价格
price=c('.info-main').find('.attr').eq(0).find('.price-n').eq(0).find('b').text()
price=price.replace('¥','')
#循环获取内容简介和目录信息
k=5
whilek<12:
title=c('#page').find('.title').eq(k).find('.tab').find('h4').text()
if'内容简介'intitle:
con=c('#page').find('.title').eq(k).nextAll()
det=pq(con)
content=det('.text-words-1').html()
content=content.encode("utf8","ignore");
if'目录'intitle:
con=c('#page').find('.title').eq(k).nextAll()
dry=pq(con)
directory=dry('.text-words-1').html()
directory=directory.encode("utf8","ignore");
k+=1
#如果内容简介和目录没有的时候指定为空字符串
if('content'inlocals().keys())==False:
content=''
if('directory'inlocals().keys())==False:
directory=''
details='内容简介
'+content+'
目录
'+directory
details=cgi.escape(details)
#录入时间
add_time=time.strftime('%Y-%m-%d',time.localtime(time.time()))
#下载小图
#文件根目录
root_path=sys.path[0]
#创建isbn文件夹路径
root_path=root_path.replace('\\','/')
isbn_path=root_path+'/download/'+isbn
ifbig_path!=''andsmall_path!='':
#创建isbn目录
ifos.path.isdir(isbn_path)==False:
os.mkdir(isbn_path)
#组合下载后图片保存路径
down_img_small=isbn_path+"/small"+isbn+".jpg"
down_img_big=isbn_path+'/big'+isbn+".jpg"
#调用下载图片方法
small_res=dowloadPic(small_path,down_img_small)
#大图保存数据库路径
big_res=dowloadPic(big_path,down_img_big)
#小图保存数据库路径
ifsmall_res==404:
img_small='none-picture/none-small.jpg'
else:
img_small='download/'+isbn+'/small'+isbn+'.jpg'
ifbig_res==404:
img_big='none-picture/none-big.jpg'
else:
img_big='download/'+isbn+'/big'+isbn+'.jpg'
else:
#组合保存数据库中的图片路径
img_small='download/'+isbn+'/small'+isbn+'.jpg'
img_big='download/'+isbn+'/big'+isbn+'.jpg'
else:
img_big='none-picture/none-big.jpg'
img_small='none-picture/none-small.jpg'
source_type=3
try:
#要插入的列表
li=[0,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,word_num,img_big,img_small]
#执行sql
sql="insertintobi_book(book_id,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,word_num,img_big,img_small)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
aaa=cursor.execute(sql,li)
ifaaa==1:
printu'插入成功'
conn.commit()
exceptException,e:
return'back'
defwinxuan(n):
#首页解析
home_url='http://www.winxuan.com/'
h=pq(home_url)
#分类导航链接
menu=h('.mod-mainmenu').find('dd').find('a').eq(n).attr('href')
#printmenu
#分类书籍首页
try:
mh=pq(menu)
exceptException,e:
return'backs'
#text=mh('.main').find('a').text()
#text=text.encode("GBK","ignore");
li=[]
u=0
whileu<248:
detail_urls=mh('.main').find('a').eq(u).attr('href')
#将取到所有地址放入到列表当中
li.append(detail_urls)
u+=1
#进行列表去重
li=list(set(li))
forfinal_urlinli:
try:
result=getData(final_url)
exceptException,e:
continue
ifresult=='back':
continue
print'OK,finished'
n=0
whilen<58:
whilen<58:
printn
string=str(n)
file_open=open('./number.txt','w')
file_open.write(string)
file_open.close()
res=winxuan(n)
n+=1
ifres=='backs':
continue
更多关于Python相关内容可查看本站专题：《PythonSocket编程技巧总结》、《Python正则表达式用法总结》、《Python数据结构与算法教程》、《Python函数使用技巧总结》、《Python字符串操作技巧汇总》、《Python入门与进阶经典教程》及《Python文件与目录操作技巧汇总》
希望本文所述对大家Python程序设计有所帮助。