python - Scrapy catch None response. AssertionError: Received None in process_response -


scrapy 1.3.3. happened response none.

traceback (most recent call last):
file "/home/vetos/venvs/aliber/lib/python3.5/site-packages/twisted/internet/defer.py", line 1301, in _inlinecallbacks result = g.send(result)

file "/home/vetos/venvs/aliber/lib/python3.5/site-packages/scrapy/core/downloader/middleware.py", line 47, in process_response

assert response not none, 'received none in process_response'

assertionerror: received none in process_response

i have middleware:

class customprocessproxyresponse(object): exceptions_to_retry = (defer.timeouterror, timeouterror, dnslookuperror,                        connectionrefusederror, connectiondone, connecterror,                        connectionlost, tcptimedouterror, responsefailed,                        ioerror, tunnelerror) retry_http_codes = [500, 502, 503, 504, 408, 403, 401, 400, 404, 408]  def __init__(self, settings):     pass  @classmethod def from_crawler(cls, crawler):     return cls(crawler.settings)  def process_response(self, request, response, spider):     if response none:         reason = ('response none; request {}'.format(request))         return self._retry(request, reason, spider) or response     if response.status in self.retry_http_codes:         reason = response_status_message(response.status)         return self._retry(request, reason, spider) or response     return response  def process_exception(self, request, exception, spider):     return self._retry(request, exception, spider)  def _retry(self, request, reason, spider):     proxy = request.meta['proxy']     logger.debug("gave fail %(request)s (failed %(proxy)s ): %(reason)s",                  {'request': request, 'proxy': proxy, 'reason': reason},                  extra={'spider': spider})     proxymanager().remove_from_databаse(proxy)     retryreq = request.copy()     retryreq.dont_filter = true     retryreq.productid = request.productid     return retryreq 

settings.py:

downloader_middlewares = { 'aliexpress_affiliate_api.middlewares.customproxymiddleware': 100, 'aliexpress_affiliate_api.middlewares.customprocessproxyresponse': 990,}   class customproxymiddleware(object): def process_request(self, request, spider):     if spider.name == aliproductdetailextender.name:         proxy_manager = proxymanager()         proxy_server, country = proxy_manager.get_random_proxy_from_database()         request.meta['proxy'] = proxy_server         request.meta['proxy_country'] = country         print(request.meta['proxy']) 

errors interception , retrying requests performed occurs error . how intercept answer when none?


Comments

Popular posts from this blog

commonjs - How to write a typescript definition file for a node module that exports a function? -

openid - Okta: Failed to get authorization code through API call -

thorough guide for profiling racket code -