Python实现的文轩网爬虫完整示例
本文实例讲述了Python实现的文轩网爬虫。分享给大家供大家参考,具体如下:
encoding=utf8 importpymysql importtime importsys importrequests importos #捕获错误 importtraceback importtypes #将html实体化 importcgi importwarnings reload(sys) sys.setdefaultencoding('utf-8') frompyqueryimportPyQueryaspq fromlxmlimportetree sys.setdefaultencoding('utf-8') #屏蔽错误 warnings.filterwarnings("ignore") #下载图片 defdowloadPic(imageUrl,filePath): r=requests.get(imageUrl,timeout=60) status=r.status_code ifstatus==404: return404 withopen(filePath,"wb")ascode: code.write(r.content) #根据详情页地址抓取数据并插入数据库 defgetData(final_url): file_open=open('./url.txt','w') file_open.write(final_url) file_open.close() #链接数据库 conn=pymysql.connect(host='127.0.0.1',port=3306,user='root',passwd='root',db='bookinfo',charset='utf8') #设置浮标 cursor=conn.cursor(cursor=pymysql.cursors.DictCursor) #解析详情页面 try: detail_url=final_url c=pq(detail_url) head=c('html').attr('xmlns') err='http://www.w3.org/1999/xhtml' err1='http://www.winxuan.com/cms/2016db_sh' ifhead==errorhead==err1: return'back' exceptException,e: return'back' i=0 whilei<12: text=c('#page').find('.cont').find('li').eq(i).text() text=text.replace('','') if'ISBN'intext: isbn=text.replace('ISBN:','') isbn=isbn.strip() sel='selectcount(*)frombi_bookwhereisbn='+isbn cursor.execute(sel) result=cursor.fetchone() count=result['count(*)'] ifcount!=0: printu'已存在' return'back' if'isbn:'intext: isbn=text.replace('isbn:','') isbn=isbn.strip() sel='selectcount(*)frombi_bookwhereisbn='+isbn cursor.execute(sel) result=cursor.fetchone() count=result['count(*)'] ifcount!=0: printu'已存在' return'back' if'作者:'intext: author=text.replace('作者:','') if'出版社:'intext: press_name=text.replace('出版社:','') if'版次:'intext: edition=text.replace('版次:','') if'印次:'intext: impressions=text.replace('印次:','') if'装帧:'intext: packaging=text.replace('装帧:','') if'开本:'intext: size=text.replace('开本:','') if'出版时间:'intext: press_time=text.replace('出版时间:','') press_time=press_time.strip() ifpress_time=='无': press_time='1970-01-01' if'印刷时间:'intext: print_time=text.replace('印刷时间:','') print_time=print_time.strip() ifprint_time=='无': print_time='1970-01-01' if'页数:'intext: page_num=text.replace('页数:','') if'字数:'intext: word_num=text.replace('字数:','') i+=1 if('author'inlocals().keys())==False: author='' if('press_time'inlocals().keys())==False: press_time='1970-01-01' if('print_time'inlocals().keys())==False: print_time='1970-01-01' if('impressions'inlocals().keys())==False: impressions='' if('edition'inlocals().keys())==False: edition='' if('page_num'inlocals().keys())==False: page_num='' if('word_num'inlocals().keys())==False: word_num='' if('packaging'inlocals().keys())==False: packaging='' if('size'inlocals().keys())==False: size='' if('press_name'inlocals().keys())==False: press_name='' #暂无图片地址 none_img='http://static.winxuancdn.com/goods/sml_blank.jpg' #获取大小图地址 big_path=c('.info-side').find('.img').find('a').find('img').attr('src') ifbig_pathisNone: return'back' elifbig_path==none_img: big_path='' small_path='' else: small_path=big_path.replace('_16','_11') #获取分类 #先获取a标签html ahtml=c('#page').find('.base-nav').eq(0).html() #解析a标签html cate=pq(ahtml) #获取分类的最后一个分类 category=cate('a:last').text() #获取书名 name=c('.info-main').find('.name').eq(0).find('h1').eq(0).text() name=name.strip() #获取价格 price=c('.info-main').find('.attr').eq(0).find('.price-n').eq(0).find('b').text() price=price.replace('¥','') #循环获取内容简介和目录信息 k=5 whilek<12: title=c('#page').find('.title').eq(k).find('.tab').find('h4').text() if'内容简介'intitle: con=c('#page').find('.title').eq(k).nextAll() det=pq(con) content=det('.text-words-1').html() content=content.encode("utf8","ignore"); if'目录'intitle: con=c('#page').find('.title').eq(k).nextAll() dry=pq(con) directory=dry('.text-words-1').html() directory=directory.encode("utf8","ignore"); k+=1 #如果内容简介和目录没有的时候指定为空字符串 if('content'inlocals().keys())==False: content='' if('directory'inlocals().keys())==False: directory='' details='内容简介
'+content+'
目录
'+directory details=cgi.escape(details) #录入时间 add_time=time.strftime('%Y-%m-%d',time.localtime(time.time())) #下载小图 #文件根目录 root_path=sys.path[0] #创建isbn文件夹路径 root_path=root_path.replace('\\','/') isbn_path=root_path+'/download/'+isbn ifbig_path!=''andsmall_path!='': #创建isbn目录 ifos.path.isdir(isbn_path)==False: os.mkdir(isbn_path) #组合下载后图片保存路径 down_img_small=isbn_path+"/small"+isbn+".jpg" down_img_big=isbn_path+'/big'+isbn+".jpg" #调用下载图片方法 small_res=dowloadPic(small_path,down_img_small) #大图保存数据库路径 big_res=dowloadPic(big_path,down_img_big) #小图保存数据库路径 ifsmall_res==404: img_small='none-picture/none-small.jpg' else: img_small='download/'+isbn+'/small'+isbn+'.jpg' ifbig_res==404: img_big='none-picture/none-big.jpg' else: img_big='download/'+isbn+'/big'+isbn+'.jpg' else: #组合保存数据库中的图片路径 img_small='download/'+isbn+'/small'+isbn+'.jpg' img_big='download/'+isbn+'/big'+isbn+'.jpg' else: img_big='none-picture/none-big.jpg' img_small='none-picture/none-small.jpg' source_type=3 try: #要插入的列表 li=[0,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,word_num,img_big,img_small] #执行sql sql="insertintobi_book(book_id,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,word_num,img_big,img_small)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" aaa=cursor.execute(sql,li) ifaaa==1: printu'插入成功' conn.commit() exceptException,e: return'back' defwinxuan(n): #首页解析 home_url='http://www.winxuan.com/' h=pq(home_url) #分类导航链接 menu=h('.mod-mainmenu').find('dd').find('a').eq(n).attr('href') #printmenu #分类书籍首页 try: mh=pq(menu) exceptException,e: return'backs' #text=mh('.main').find('a').text() #text=text.encode("GBK","ignore"); li=[] u=0 whileu<248: detail_urls=mh('.main').find('a').eq(u).attr('href') #将取到所有地址放入到列表当中 li.append(detail_urls) u+=1 #进行列表去重 li=list(set(li)) forfinal_urlinli: try: result=getData(final_url) exceptException,e: continue ifresult=='back': continue print'OK,finished' n=0 whilen<58: whilen<58: printn string=str(n) file_open=open('./number.txt','w') file_open.write(string) file_open.close() res=winxuan(n) n+=1 ifres=='backs': continue
更多关于Python相关内容可查看本站专题:《PythonSocket编程技巧总结》、《Python正则表达式用法总结》、《Python数据结构与算法教程》、《Python函数使用技巧总结》、《Python字符串操作技巧汇总》、《Python入门与进阶经典教程》及《Python文件与目录操作技巧汇总》
希望本文所述对大家Python程序设计有所帮助。