python实现批量下载新浪博客的方法

本文实例讲述了python实现批量下载新浪博客的方法。分享给大家供大家参考。具体实现方法如下：

# coding=utf-8 
import urllib2
import sys, os
import re
import string
from BeautifulSoup import BeautifulSoup
def encode(s):
 return s.decode('utf-8').encode(sys.stdout.encoding, 'ignore')
def getHTML(url):
 #proxy_handler = urllib2.ProxyHandler({'http':'http://211.138.124.211:80'})
 #opener = urllib2.build_opener(proxy_handler)
 #urllib2.install_opener(opener)
 req = urllib2.Request(url)
 response = urllib2.urlopen(req, timeout=15)
 return BeautifulSoup(response, convertEntities=BeautifulSoup.HTML_ENTITIES)
def visible(element):
 '''抓取可见的文本元素'''
 if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
 return False
 elif re.match('', str(element)):
 return False
 elif element == u'xa0':
 return False
 return True
def delReturn(element):
 '''删除元素内的换行'''
 return re.sub('(?"|xa0]', '', filename)
def writeToFile(text, filename, dirname):
 if not os.path.exists(dirname):
 os.makedirs(dirname)
 print encode('保存到目录'), dirname
 filename = validFilename(filename)
 print encode('保存文章'), filename
 path = os.path.join(dirname, filename)
 if not os.path.exists(path):
 f = open(path, 'w')
 f.write(text)
 f.close()
 else:
 print filename, encode('已经存在')
def formatContent(url, title=''):
 '''格式化文章内容'''
 page = getHTML(url)
 content = page.find('div', {'class':'articalContent'})
 art_id = re.search('blog_(w+).html', url).group(1)
 blog_name = page.find('span', id='blognamespan').string
 if title == '':
 title = page.find('h2', id=re.compile('^t_')).string
 temp_data = filter(visible, content.findAll(text=True)) # 去掉不可见元素
 temp_data = ''.join(map(delReturn, temp_data)) # 删除元素内的换行符
 temp_data = temp_data.strip() # 删除文章首尾的空行
 temp_data = re.sub('
{2,}', '

', temp_data) # 删除文章内过多的空行
 # 

输出到文件 # 编码问题 temp_data = '本文地址:'.decode('utf-8') + url + ' ' + temp_data op_text = temp_data.encode('utf-8') op_file = title + '_' + art_id +'.txt' writeToFile(op_text, op_file, blog_name) def articlelist(url): articles = {} page = getHTML(url) pages = page.find('ul', {'class':'SG_pages'}).span.string page_num = int(re.search('(d+)', pages).group(1)) for i in range(1, page_num+1): print encode('生成第%d页文章索引'%i) if i != 1: url = re.sub('(_)d+(.html)$', 'g<1>'+str(i)+'g<2>', url) page = getHTML(url) article = page.findAll('span', {'class':'atc_title'}) for art in article: art_title = art.a['title'] art_href = art.a['href'] articles[art_title] = art_href return articles def blog_dld(articles): if not isinstance(articles, dict): return False print encode('开始下载文章') for art_title, art_href in articles.items(): formatContent(art_href, art_title) if __name__ == '__main__': sel = raw_input(encode('你要下载的是(1)全部文章还是(2)单篇文章，输入1或者2: ')) if sel == '1': #articlelist_url = 'http://blog.sina.com.cn/s/articlelist_1303481411_0_1.html' articlelist_url = raw_input(encode('请输入博客文章目录链接: ')) articles = articlelist(articlelist_url) blog_dld(articles) else: #article_url = 'http://blog.sina.com.cn/s/blog_4db18c430100gxc5.html' article_url = raw_input(encode('请输入博客文章链接: ')) formatContent(article_url)

希望本文所述对大家的Python程序设计有所帮助。