详解scrapy内置中间件的顺序
1.内置下载器中间件顺序
{'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware':560, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':700, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':400, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':350, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware':300, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware':900, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':590, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':750, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware':580, 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware':600, 'scrapy.downloadermiddlewares.retry.RetryMiddleware':550, 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware':100, 'scrapy.downloadermiddlewares.stats.DownloaderStats':850, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':500}
2.内置爬虫中间件顺序
{'scrapy.spidermiddlewares.depth.DepthMiddleware':900, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware':50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware':500, 'scrapy.spidermiddlewares.referer.RefererMiddleware':700, 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware':800}
3.内置scrapy的settings
{'AJAXCRAWL_ENABLED':False, 'AUTOTHROTTLE_DEBUG':False, 'AUTOTHROTTLE_ENABLED':False, 'AUTOTHROTTLE_MAX_DELAY':60.0, 'AUTOTHROTTLE_START_DELAY':5.0, 'AUTOTHROTTLE_TARGET_CONCURRENCY':1.0, 'BOT_NAME':'scrapybot', 'CLOSESPIDER_ERRORCOUNT':0, 'CLOSESPIDER_ITEMCOUNT':0, 'CLOSESPIDER_PAGECOUNT':0, 'CLOSESPIDER_TIMEOUT':0, 'COMMANDS_MODULE':'', 'COMPRESSION_ENABLED':True, 'CONCURRENT_ITEMS':100, 'CONCURRENT_REQUESTS':16, 'CONCURRENT_REQUESTS_PER_DOMAIN':8, 'CONCURRENT_REQUESTS_PER_IP':0, 'COOKIES_DEBUG':False, 'COOKIES_ENABLED':True, 'DEFAULT_ITEM_CLASS':'scrapy.item.Item', 'DEFAULT_REQUEST_HEADERS':{'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language':'en'}, 'DEPTH_LIMIT':0, 'DEPTH_PRIORITY':0, 'DEPTH_STATS_VERBOSE':False, 'DNSCACHE_ENABLED':True, 'DNSCACHE_SIZE':10000, 'DNS_TIMEOUT':60, 'DOWNLOADER':'scrapy.core.downloader.Downloader', 'DOWNLOADER_CLIENTCONTEXTFACTORY':'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory', 'DOWNLOADER_CLIENT_TLS_METHOD':'TLS', 'DOWNLOADER_HTTPCLIENTFACTORY':'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory', 'DOWNLOADER_MIDDLEWARES':{}, 'DOWNLOADER_MIDDLEWARES_BASE':{'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware':560, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':700, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':400, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':350, 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware':300, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware':900, 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':590, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':750, 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware':580, 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware':600, 'scrapy.downloadermiddlewares.retry.RetryMiddleware':550, 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware':100, 'scrapy.downloadermiddlewares.stats.DownloaderStats':850, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':500}, 'DOWNLOADER_STATS':True, 'DOWNLOAD_DELAY':0, 'DOWNLOAD_FAIL_ON_DATALOSS':True, 'DOWNLOAD_HANDLERS':{}, 'DOWNLOAD_HANDLERS_BASE':{'data':'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file':'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'ftp':'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler', 'http':'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https':'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 's3':'scrapy.core.downloader.handlers.s3.S3DownloadHandler'}, 'DOWNLOAD_MAXSIZE':1073741824, 'DOWNLOAD_TIMEOUT':180, 'DOWNLOAD_WARNSIZE':33554432, 'DUPEFILTER_CLASS':'scrapy.dupefilters.BaseDupeFilter', 'EDITOR':'D:\\ProgramFiles(x86)\\Notepad++\\notepad++.exe', 'EXTENSIONS':{}, 'EXTENSIONS_BASE':{'scrapy.extensions.closespider.CloseSpider':0, 'scrapy.extensions.corestats.CoreStats':0, 'scrapy.extensions.feedexport.FeedExporter':0, 'scrapy.extensions.logstats.LogStats':0, 'scrapy.extensions.memdebug.MemoryDebugger':0, 'scrapy.extensions.memusage.MemoryUsage':0, 'scrapy.extensions.spiderstate.SpiderState':0, 'scrapy.extensions.telnet.TelnetConsole':0, 'scrapy.extensions.throttle.AutoThrottle':0}, 'FEED_EXPORTERS':{}, 'FEED_EXPORTERS_BASE':{'csv':'scrapy.exporters.CsvItemExporter', 'jl':'scrapy.exporters.JsonLinesItemExporter', 'json':'scrapy.exporters.JsonItemExporter', 'jsonlines':'scrapy.exporters.JsonLinesItemExporter', 'marshal':'scrapy.exporters.MarshalItemExporter', 'pickle':'scrapy.exporters.PickleItemExporter', 'xml':'scrapy.exporters.XmlItemExporter'}, 'FEED_EXPORT_ENCODING':None, 'FEED_EXPORT_FIELDS':None, 'FEED_EXPORT_INDENT':0, 'FEED_FORMAT':'jsonlines', 'FEED_STORAGES':{}, 'FEED_STORAGES_BASE':{'':'scrapy.extensions.feedexport.FileFeedStorage', 'file':'scrapy.extensions.feedexport.FileFeedStorage', 'ftp':'scrapy.extensions.feedexport.FTPFeedStorage', 's3':'scrapy.extensions.feedexport.S3FeedStorage', 'stdout':'scrapy.extensions.feedexport.StdoutFeedStorage'}, 'FEED_STORE_EMPTY':False, 'FEED_TEMPDIR':None, 'FEED_URI':None, 'FEED_URI_PARAMS':None, 'FILES_STORE_GCS_ACL':'', 'FILES_STORE_S3_ACL':'private', 'FTP_PASSIVE_MODE':True, 'FTP_PASSWORD':'guest', 'FTP_USER':'anonymous', 'HTTPCACHE_ALWAYS_STORE':False, 'HTTPCACHE_DBM_MODULE':'dbm', 'HTTPCACHE_DIR':'httpcache', 'HTTPCACHE_ENABLED':False, 'HTTPCACHE_EXPIRATION_SECS':0, 'HTTPCACHE_GZIP':False, 'HTTPCACHE_IGNORE_HTTP_CODES':[], 'HTTPCACHE_IGNORE_MISSING':False, 'HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS':[], 'HTTPCACHE_IGNORE_SCHEMES':['file'], 'HTTPCACHE_POLICY':'scrapy.extensions.httpcache.DummyPolicy', 'HTTPCACHE_STORAGE':'scrapy.extensions.httpcache.FilesystemCacheStorage', 'HTTPPROXY_AUTH_ENCODING':'latin-1', 'HTTPPROXY_ENABLED':True, 'IMAGES_STORE_GCS_ACL':'', 'IMAGES_STORE_S3_ACL':'private', 'ITEM_PIPELINES':{}, 'ITEM_PIPELINES_BASE':{}, 'ITEM_PROCESSOR':'scrapy.pipelines.ItemPipelineManager', 'LOGSTATS_INTERVAL':0, 'LOG_DATEFORMAT':'%Y-%m-%d%H:%M:%S', 'LOG_ENABLED':True, 'LOG_ENCODING':'utf-8', 'LOG_FILE':None, 'LOG_FORMAT':'%(asctime)s[%(name)s]%(levelname)s:%(message)s', 'LOG_FORMATTER':'scrapy.logformatter.LogFormatter', 'LOG_LEVEL':'DEBUG', 'LOG_SHORT_NAMES':False, 'LOG_STDOUT':False, 'MAIL_FROM':'scrapy@localhost', 'MAIL_HOST':'localhost', 'MAIL_PASS':None, 'MAIL_PORT':25, 'MAIL_USER':None, 'MEMDEBUG_ENABLED':False, 'MEMDEBUG_NOTIFY':[], 'MEMUSAGE_CHECK_INTERVAL_SECONDS':60.0, 'MEMUSAGE_ENABLED':True, 'MEMUSAGE_LIMIT_MB':0, 'MEMUSAGE_NOTIFY_MAIL':[], 'MEMUSAGE_WARNING_MB':0, 'METAREFRESH_ENABLED':True, 'METAREFRESH_MAXDELAY':100, 'NEWSPIDER_MODULE':'', 'RANDOMIZE_DOWNLOAD_DELAY':True, 'REACTOR_THREADPOOL_MAXSIZE':10, 'REDIRECT_ENABLED':True, 'REDIRECT_MAX_TIMES':20, 'REDIRECT_PRIORITY_ADJUST':2, 'REFERER_ENABLED':True, 'REFERRER_POLICY':'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy', 'RETRY_ENABLED':True, 'RETRY_HTTP_CODES':[500,502,503,504,522,524,408], 'RETRY_PRIORITY_ADJUST':-1, 'RETRY_TIMES':2, 'ROBOTSTXT_OBEY':False, 'SCHEDULER':'scrapy.core.scheduler.Scheduler', 'SCHEDULER_DEBUG':False, 'SCHEDULER_DISK_QUEUE':'scrapy.squeues.PickleLifoDiskQueue', 'SCHEDULER_MEMORY_QUEUE':'scrapy.squeues.LifoMemoryQueue', 'SCHEDULER_PRIORITY_QUEUE':'queuelib.PriorityQueue', 'SPIDER_CONTRACTS':{}, 'SPIDER_CONTRACTS_BASE':{'scrapy.contracts.default.ReturnsContract':2, 'scrapy.contracts.default.ScrapesContract':3, 'scrapy.contracts.default.UrlContract':1}, 'SPIDER_LOADER_CLASS':'scrapy.spiderloader.SpiderLoader', 'SPIDER_LOADER_WARN_ONLY':False, 'SPIDER_MIDDLEWARES':{}, 'SPIDER_MIDDLEWARES_BASE':{'scrapy.spidermiddlewares.depth.DepthMiddleware':900, 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware':50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware':500, 'scrapy.spidermiddlewares.referer.RefererMiddleware':700, 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware':800}, 'SPIDER_MODULES':[], 'STATSMAILER_RCPTS':[], 'STATS_CLASS':'scrapy.statscollectors.MemoryStatsCollector', 'STATS_DUMP':True, 'TELNETCONSOLE_ENABLED':1, 'TELNETCONSOLE_HOST':'127.0.0.1', 'TELNETCONSOLE_PASSWORD':None, 'TELNETCONSOLE_PORT':[6023,6073], 'TELNETCONSOLE_USERNAME':'scrapy', 'TEMPLATES_DIR':'d:\\python36\\lib\\site-packages\\scrapy\\templates', 'URLLENGTH_LIMIT':2083, 'USER_AGENT':'Scrapy/1.6.0(+https://scrapy.org)', 'KEEP_ALIVE':True}
到此这篇关于详解scrapy内置中间件的顺序的文章就介绍到这了,更多相关scrapy中间件顺序内容请搜索毛票票以前的文章或继续浏览下面的相关文章希望大家以后多多支持毛票票!