Added three more tests to check for edge cases in URL validation (#4441)

Co-authored-by: Ryan Johns <rkjohns@verisk.com>
Co-authored-by: k-boikov <64261260+k-boikov@users.noreply.github.com>

autogpt/url_utils/validators.py

@@ -31,6 +31,9 @@ def validate_url(func: Callable[..., Any]) -> Any:
         # Restrict access to local files
         if check_local_file_access(url):
             raise ValueError("Access to local files is restricted")
+        # Check URL length
+        if len(url) > 2000:
+            raise ValueError("URL is too long")
 
         return func(sanitize_url(url), *args, **kwargs)
 

tests/unit/test_url_validation.py

@@ -156,3 +156,19 @@ class TestValidateUrl:
 
         with pytest.raises(ValueError):
             test_func("https:www.google.com")
+
+    # Tests that the function can handle URLs that contain unusual but valid characters.
+    def test_url_with_special_chars(self):
+        url = "https://example.com/path%20with%20spaces"
+        assert dummy_method(url) == url
+
+    # Tests that the function raises a ValueError if the URL is over 2000 characters.
+    def test_extremely_long_url(self):
+        url = "http://example.com/" + "a" * 2000
+        with raises(ValueError, match="URL is too long"):
+            dummy_method(url)
+
+    # Tests that the function can handle internationalized URLs, which contain non-ASCII characters.
+    def test_internationalized_url(self):
+        url = "http://例子.测试"
+        assert dummy_method(url) == url