python抓取最新博客内容并生成Rss
osc的rss不是全文输出的,不开心，所以就有了python抓取osc最新博客生成Rss
#-*-coding:utf-8-*-
frombs4importBeautifulSoup
importurllib2
importdatetime
importtime
importPyRSS2Gen
fromemail.Utilsimportformatdate
importre
importsys
importos
reload(sys)
sys.setdefaultencoding('utf-8')
classRssSpider():
def__init__(self):
self.myrss=PyRSS2Gen.RSS2(title='OSChina',
link='http://my.oschina.net',
description=str(datetime.date.today()),
pubDate=datetime.datetime.now(),
lastBuildDate=datetime.datetime.now(),
items=[]
)
self.xmlpath=r'/var/www/myrss/oschina.xml'
self.baseurl="http://www.oschina.net/blog"
#ifos.path.isfile(self.xmlpath):
#os.remove(self.xmlpath)
defuseragent(self,url):
i_headers={"User-Agent":"Mozilla/5.0(WindowsNT6.1;WOW64)
AppleWebKit/537.36(KHTML,likeGecko)Chrome/36.0.1985.125Safari/537.36",
"Referer":'http://baidu.com/'}
req=urllib2.Request(url,headers=i_headers)
html=urllib2.urlopen(req).read()
returnhtml
defenterpage(self,url):
pattern=re.compile(r'd{4}Sd{2}Sd{2}sd{2}Sd{2}')
rsp=self.useragent(url)
soup=BeautifulSoup(rsp)
timespan=soup.find('div',{'class':'BlogStat'})
timespan=str(timespan).strip().replace('n','').decode('utf-8')
match=re.search(r'd{4}Sd{2}Sd{2}sd{2}Sd{2}',timespan)
timestr=str(datetime.date.today())
ifmatch:
timestr=match.group()
#printtimestr
ititle=soup.title.string
div=soup.find('div',{'class':'BlogContent'})
rss=PyRSS2Gen.RSSItem(
title=ititle,
link=url,
description=str(div),
pubDate=timestr
)
returnrss
defgetcontent(self):
rsp=self.useragent(self.baseurl)
soup=BeautifulSoup(rsp)
ul=soup.find('div',{'id':'RecentBlogs'})
forliinul.findAll('li'):
div=li.find('div')
ifdivisnotNone:
alink=div.find('a')
ifalinkisnotNone:
link=alink.get('href')
printlink
html=self.enterpage(link)
self.myrss.items.append(html)
defSaveRssFile(self,filename):
finallxml=self.myrss.to_xml(encoding='utf-8')
file=open(self.xmlpath,'w')
file.writelines(finallxml)
file.close()
if__name__=='__main__':
rssSpider=RssSpider()
rssSpider.getcontent()
rssSpider.SaveRssFile('oschina.xml')
以上所述就是本文的全部内容了，希望大家能够喜欢。