关于python scrapy中添加cookie踩坑记录

class CookieMiddleware(object):
  """
  每次请求都随机从账号池中选择一个账号去访问
  """

  def __init__(self):
    client = pymongo.MongoClient(MONGO_URI)
    self.account_collection = client[MONGO_DATABASE][ACCOUNT_COLLECTION]

  def process_request(self, request, spider):
    if 'target' in request.meta: 
      logging.debug('进入到process_request了')
      flag = request.meta['target']
      if flag != 'no':
        all_count = self.account_collection.find({'status': 'success'}).count()
        if all_count == 0:
          raise Exception('当前账号池为空')
        random_index = random.randint(0, all_count - 1)
        random_account = self.account_collection.find({'status': 'success'})[random_index]
        
        request.cookies = json.loads(random_account['cookie'])
      else:
        logging.debug('对XXX的请求不做处理')
    else:
      all_count = self.account_collection.find({'status': 'success'}).count()
      if all_count == 0:
        raise Exception('当前账号池为空')
      random_index = random.randint(0, all_count - 1)
      random_account = self.account_collection.find({'status': 'success'})[random_index]
      
      request.cookies = json.loads(random_account['cookie'])

DOWNLOADER_MIDDLEWARES = {
  'eyny.middlewares.CookieMiddleware': 550,
}

DOWNLOADER_MIDDLEWARES_BASE = {
  # Engine side
  'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
  'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
  'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
  'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
  'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
  'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
  'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
  'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
  'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
  'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
  'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
  'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
  'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
  'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
  # Downloader side
}

class CookiesMiddleware(object):
  """This middleware enables working with sites that need cookies"""

  def __init__(self, debug=False):
    self.jars = defaultdict(CookieJar)
    self.debug = debug

  @classmethod
  def from_crawler(cls, crawler):
    if not crawler.settings.getbool('COOKIES_ENABLED'):
      raise NotConfigured
    return cls(crawler.settings.getbool('COOKIES_DEBUG'))

  def process_request(self, request, spider):
    if request.meta.get('dont_merge_cookies', False):
      return

    cookiejarkey = request.meta.get("cookiejar")
    jar = self.jars[cookiejarkey]
    cookies = self._get_request_cookies(jar, request)
    for cookie in cookies:
      jar.set_cookie_if_ok(cookie, request)

    # set Cookie header
    request.headers.pop('Cookie', None)
    jar.add_cookie_header(request)
    self._debug_cookie(request, spider)

	def process_response(self, request, response, spider):
    if request.meta.get('dont_merge_cookies', False):
      return response

    # extract cookies from Set-Cookie and drop invalid/expired cookies
    cookiejarkey = request.meta.get("cookiejar")
    jar = self.jars[cookiejarkey]
    jar.extract_cookies(response, request)
    self._debug_set_cookie(response, spider)

    return response