分享个Python 脚本
908
- hdwan.net2016-10-20金牌会员91764
- #!/usr/bin/env python3
- import requests,re,json,html2text,sys,time
- from bs4 import BeautifulSoup
- import time
- from urllib.request import urlretrieve
- - url="https://www.douban.com/group/haixiuzu/"
- - def getTopicList():
- #循环分页抓取 括号里的换成需要抓取的页数区间,从0开始,间隔越大,时间越长
- for x in range(16,20):
- page = x * 25
- get_url = requests.get(url+"discussion?start="+str(page))
- - soup = BeautifulSoup(get_url.text,"html.parser")
- tdList = soup.find_all("td",class_='title')
- for i in tdList:
- title = i.a.get("title")
- if len(i.contents) > 1:
- #因为大家晒照片都喜欢在前面加一个【晒】
- if '晒' in title:
- i_href = i.a.get('href')
- getTopicContext(i_href)
- time.sleep(1)
- #获取帖子里的内容
- def getTopicContext(topicUrl):
- url = requests.get(topicUrl)
- soup = BeautifulSoup(url.text,"html.parser")
- topicDiv = soup.find_all("div",class_='topic-figure cc')
- for div in topicDiv:
- if len(div.contents) > 1:
- img = div.img
- saveImage(img.get("src"))
- - #下载图片到本地
- def saveImage(imgUrl):
- fileName = imgUrl[imgUrl.rfind("/")+1:]
- path = r"你的路径"+fileName
- urlretrieve(imgUrl,path)
- - if __name__=="__main__":
- getTopicList()复制代码
就这样吧。。。