-
Notifications
You must be signed in to change notification settings - Fork 4
/
douban_page_by_spider.py
57 lines (48 loc) · 1.73 KB
/
douban_page_by_spider.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# !/usr/bin/env python
from talospider import AttrField, Item, Request, Spider, TextField
from talospider.utils import get_random_user_agent
class DoubanItem(Item):
"""
定义继承自item的Item类
"""
target_item = TextField(css_select='div.item')
title = TextField(css_select='span.title')
cover = AttrField(css_select='div.pic>a>img', attr='src')
abstract = TextField(css_select='span.inq')
def tal_title(self, title):
if isinstance(title, str):
return title
else:
return ''.join([i.text.strip().replace('\xa0', '') for i in title])
class DoubanSpider(Spider):
start_urls = ['https://movie.douban.com/top250']
request_config = {
'RETRIES': 3,
'DELAY': 0,
'TIMEOUT': 20
}
def parse(self, res):
# 将html转化为etree
etree = self.e_html(res.html)
# 提取目标值生成新的url
pages = [i.get('href') for i in etree.cssselect('.paginator>a')]
pages.insert(0, '?start=0&filter=')
headers = {
"User-Agent": get_random_user_agent()
}
for page in pages:
url = self.start_urls[0] + page
yield Request(url, request_config=self.request_config, headers=headers, callback=self.parse_item)
def parse_item(self, res):
items_data = DoubanItem.get_items(html=res.html)
# result = []
for item in items_data:
# result.append({
# 'title': item.title,
# 'cover': item.cover,
# 'abstract': item.abstract,
# })
with open('douban250.txt', 'a+') as f:
f.writelines(item.title + '\n')
if __name__ == '__main__':
DoubanSpider.start()