Closes scrapy#6342. Setting METAREFRESH_IGNORE_TAGS to [‘noscript’] b…

aisha-partha · May 8, 2024 · 180bc9b · 180bc9b
1 parent 4300a1d
commit 180bc9b
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
@@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled.
 METAREFRESH_IGNORE_TAGS
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Default: ``[]``
+Default: ``['noscript']``
 
 Meta tags within these tags are ignored.
 
 .. versionchanged:: 2.0
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
-   ``['script', 'noscript']`` to ``[]``.
+   ``[]`` to ``['noscript']``.
 
 .. setting:: METAREFRESH_MAXDELAY
 

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
@@ -239,7 +239,7 @@
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
-METAREFRESH_IGNORE_TAGS = []
+METAREFRESH_IGNORE_TAGS = ["noscript"]
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ""

diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
@@ -395,9 +395,8 @@ def test_ignore_tags_default(self):
             """content="0;URL='http://example.org/newpage'"></noscript>"""
         )
         rsp = HtmlResponse(req.url, body=body.encode())
-        req2 = self.mw.process_response(req, rsp, self.spider)
-        assert isinstance(req2, Request)
-        self.assertEqual(req2.url, "http://example.org/newpage")
+        response = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(response, Response)
 
     def test_ignore_tags_1_x_list(self):
         """Test that Scrapy 1.x behavior remains possible"""