scrapy处理python爬虫调度详解

import requests
def main():
  for i in range(100):
    rsp = requests.get(f"http://www.example.com/{i}.html")
    with open("example-{i}.txt", "w") as f:
      f.write(rsp.text)
if __name__ == "__main__":
  main()

import scrapy
class QuotesSpider(scrapy.Spider):
  name = 'quotes'
  def start_requests(self):
    urls = [
      'http://quotes.toscrape.com/page/1/',
      'http://quotes.toscrape.com/page/2/'
    ]
    for url in urls:
      yield scrapy.Request(url=url, callback=self.parse)
  def parse(self, response):
    page = response.url.split('/')[-2]
    filename = 'quotes-%s.html' % page
    with open(filename, 'wb') as f:
      f.write(response.body)
    self.log('Save file %s' % filename)

from twisted.internet import reactor, defer
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
import time
import logging
from scrapy.utils.project import get_project_settings
 
 
#在控制台打印日志
configure_logging()
#CrawlerRunner获取settings.py里的设置信息
runner = CrawlerRunner(get_project_settings())
 
@defer.inlineCallbacks
def crawl():
 while True:
  logging.info("new cycle starting")
  yield runner.crawl("xxxxx")
  #1s跑一次
  time.sleep(1)
 reactor.stop()
 
crawl()
reactor.run()