Python之Scrapy爬虫框架安装及使用详解

pip install Scrapy

pip --version

pip install Scrapy

H:\python\scrapyDemo>scrapy startproject tutorial
New Scrapy project 'tutorial', using template directory 'f:\\python27\\lib\\site-packages\\scrapy\\templates\\project', created in:
 H:\python\scrapyDemo\tutorial

You can start your first spider with:
 cd tutorial
 scrapy genspider example example.com

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy
from scrapy.item import Item, Field

class TutorialItem(Item):
 title = Field()
 author = Field()
 releasedate = Field() 

# -*-coding:utf-8-*-
import sys
from scrapy.linkextractors.sgml import SgmlLinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from tutorial.items import TutorialItem

reload(sys)
sys.setdefaultencoding("utf-8")


class ListSpider(CrawlSpider):
 # 爬虫名称
 name = "tutorial"
 # 设置下载延时
 download_delay = 1
 # 允许域名
 allowed_domains = ["news.cnblogs.com"]
 # 开始URL
 start_urls = [
 "https://news.cnblogs.com"
 ]
 # 爬取规则,不带callback表示向该类url递归爬取
 rules = (
 Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/page/\d',))),
 Rule(SgmlLinkExtractor(allow=(r'https://news.cnblogs.com/n/\d+',)), callback='parse_content'),
 )

 # 解析内容函数
 def parse_content(self, response):
 item = TutorialItem()

 # 当前URL
 title = response.selector.xpath('//div[@id="news_title"]')[0].extract().decode('utf-8')
 item['title'] = title

 author = response.selector.xpath('//div[@id="news_info"]/span/a/text()')[0].extract().decode('utf-8')
 item['author'] = author

 releasedate = response.selector.xpath('//div[@id="news_info"]/span[@class="time"]/text()')[0].extract().decode(
  'utf-8')
 item['releasedate'] = releasedate

 yield item

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import codecs


class TutorialPipeline(object):
 def __init__(self):
 self.file = codecs.open('data.json', mode='wb', encoding='utf-8')#数据存储到data.json

 def process_item(self, item, spider):
 line = json.dumps(dict(item)) + "\n"
 self.file.write(line.decode("unicode_escape"))

 return item

# -*- coding: utf-8 -*-

BOT_NAME = 'tutorial'

SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'

# 禁止cookies,防止被ban
COOKIES_ENABLED = False
COOKIES_ENABLES = False

# 设置Pipeline,此处实现数据写入文件
ITEM_PIPELINES = {
 'tutorial.pipelines.TutorialPipeline': 300
}

# 设置爬虫爬取的最大深度
DEPTH_LIMIT = 100

from scrapy import cmdline
cmdline.execute("scrapy crawl tutorial".split())