您应该改用scrapy.Request中的errback处理程序。
下面是一个例子：```

def start_requests(self):
    for url in self.start_urls:
        yield scrapy.Request(
            url=url,
            dont_filter=True,
            callback=self.apply_filter,
            errback=self.handle_failure)

def handle_failure(self, failure):
    self.log(failure, level=logging.ERROR)
    # try with a new proxy
    self.log('restart from the failed url {}'.format(failure.request.url))
    yield scrapy.Request(
        url=failure.request.url,
        callback=self.parse,
        errback=self.handle_failure)

下面是我最终实现我的解决方案的过程。

def parse(self, response):
    meta = response.meta
    retries = meta.get(MISSING_RATINGS_RETRY_COUNT, 0)
    if retries < MAX_RETRIES:
        throw_on_failure = True
    else:
        throw_on_failure = False
    try:
        #do something 
        #use throw_on_failure variable to thorw the exception based on missing data from the response.
        yield result
    except specificException:
        meta[MISSING_RATINGS_RETRY_COUNT] = retries + 1
        yield Request(response.url, callback=self.parse, meta=meta, dont_filter=True)