feat(search): add Keenable as a search engine, keyless by default

scrapegraphai/graphs/search_graph.py

@@ -71,6 +71,7 @@ def _create_graph(self) -> BaseGraph:
                 "storage_state": self.copy_config.get("storage_state"),
                 "search_engine": self.copy_config.get("search_engine"),
                 "serper_api_key": self.copy_config.get("serper_api_key"),
+                "keenable_api_key": self.copy_config.get("keenable_api_key"),
             },
         )
 

scrapegraphai/nodes/search_internet_node.py

@@ -55,6 +55,8 @@ def __init__(
             node_config["serper_api_key"] if node_config.get("serper_api_key") else None
         )
 
+        self.keenable_api_key = node_config.get("keenable_api_key")
+
         self.max_results = node_config.get("max_results", 3)
 
     def execute(self, state: dict) -> dict:
@@ -108,6 +110,7 @@ def execute(self, state: dict) -> dict:
             search_engine=self.search_engine,
             proxy=self.proxy,
             serper_api_key=self.serper_api_key,
+            keenable_api_key=self.keenable_api_key,
         )
 
         if len(answer) == 0:

scrapegraphai/utils/research_web.py

@@ -3,6 +3,7 @@
 error handling, validation, and security features.
 """
 
+import os
 import random
 import re
 import time
@@ -56,13 +57,16 @@ class SearchConfig(BaseModel):
         None, description="Proxy configuration"
     )
     serper_api_key: Optional[str] = Field(None, description="API key for Serper")
+    keenable_api_key: Optional[str] = Field(
+        None, description="Optional API key for Keenable (works without one)"
+    )
     region: Optional[str] = Field(None, description="Country/region code")
     language: str = Field("en", description="Language code")
 
     @validator("search_engine")
     def validate_search_engine(cls, v):
         """Validate search engine."""
-        valid_engines = {"duckduckgo", "bing", "searxng", "serper"}
+        valid_engines = {"duckduckgo", "bing", "searxng", "serper", "keenable"}
         if v.lower() not in valid_engines:
             raise ValueError(
                 f"Search engine must be one of: {', '.join(valid_engines)}"
@@ -167,6 +171,7 @@ def search_on_web(
     serper_api_key: Optional[str] = None,
     region: Optional[str] = None,
     language: str = "en",
+    keenable_api_key: Optional[str] = None,
 ) -> List[str]:
     """
     Search web function with improved error handling, validation, and security features.
@@ -181,6 +186,9 @@ def search_on_web(
         serper_api_key (str): API key for Serper
         region (str): Country/region code (e.g., 'mx' for Mexico)
         language (str): Language code (e.g., 'es' for Spanish)
+        keenable_api_key (str): Optional API key for Keenable. Falls back to the
+            KEENABLE_API_KEY environment variable; without either, the keyless
+            public endpoint is used (rate-limited per IP)
 
     Returns:
         List[str]: List of URLs from search results
@@ -203,6 +211,7 @@ def search_on_web(
             timeout=timeout,
             proxy=proxy,
             serper_api_key=serper_api_key,
+            keenable_api_key=keenable_api_key,
             region=region,
             language=language,
         )
@@ -233,6 +242,15 @@ def search_on_web(
                 config.query, config.max_results, config.serper_api_key, config.timeout
             )
 
+        elif config.search_engine == "keenable":
+            results = _search_keenable(
+                config.query,
+                config.max_results,
+                config.keenable_api_key,
+                config.timeout,
+                formatted_proxy,
+            )
+
         return filter_pdf_links(results)
 
     except requests.Timeout:
@@ -416,6 +434,72 @@ def _search_serper(
         raise SearchRequestError(f"Serper search failed: {str(e)}")
 
 
+KEENABLE_API_URL = "https://api.keenable.ai/v1/search"
+KEENABLE_PUBLIC_API_URL = "https://api.keenable.ai/v1/search/public"
+# Keenable returns at most 50 results per call.
+KEENABLE_MAX_RESULTS = 50
+
+
+def _search_keenable(
+    query: str,
+    max_results: int,
+    api_key: Optional[str],
+    timeout: int,
+    proxy: Optional[str] = None,
+) -> List[str]:
+    """
+    Helper function for Keenable search (https://docs.keenable.ai).
+
+    Works without an API key: with no key (argument or KEENABLE_API_KEY
+    environment variable) it calls Keenable's public endpoint, which is
+    rate-limited per IP and needs an ``X-Keenable-Title`` header naming the app.
+    A key selects the authenticated endpoint and lifts the hourly limit.
+
+    Args:
+        query (str): Search query
+        max_results (int): Maximum number of results to return (capped at 50)
+        api_key (str, optional): API key for Keenable
+        timeout (int): Request timeout in seconds
+        proxy (str, optional): Proxy configuration
+
+    Returns:
+        List[str]: List of URLs from search results
+    """
+    api_key = api_key or os.environ.get("KEENABLE_API_KEY") or None
+
+    headers = {"Content-Type": "application/json", "X-Keenable-Title": "scrapegraphai"}
+    if api_key:
+        headers["X-API-Key"] = api_key
+        url = KEENABLE_API_URL
+    else:
+        url = KEENABLE_PUBLIC_API_URL
+
+    data = {"query": query, "max_results": min(max_results, KEENABLE_MAX_RESULTS)}
+    proxies = {"http": proxy, "https": proxy} if proxy else None
+
+    try:
+        response = requests.post(
+            url,
+            json=data,
+            headers=headers,
+            proxies=proxies,
+            timeout=timeout,
+        )
+        response.raise_for_status()
+
+        json_data = response.json()
+        results = []
+        for item in json_data.get("results", []):
+            if item.get("url"):
+                results.append(item["url"])
+                if len(results) >= max_results:
+                    break
+
+        return results
+    except Exception as e:
+        raise SearchRequestError(f"Keenable search failed: {str(e)}")
+
+
 def format_proxy(proxy_config: Union[str, Dict, ProxyConfig]) -> str:
     """
     Format proxy configuration into a string.

tests/utils/research_web_keenable_test.py

@@ -0,0 +1,101 @@
+"""Mock-based tests for the Keenable engine in search_on_web (no network)."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from scrapegraphai.utils.research_web import (
+    KEENABLE_API_URL,
+    KEENABLE_PUBLIC_API_URL,
+    SearchRequestError,
+    search_on_web,
+)
+
+# search_on_web is rate limited (10 calls/minute), so call the undecorated function.
+search = search_on_web.__wrapped__
+
+RESPONSE = {
+    "query": "q",
+    "mode": "pro",
+    "results": [
+        {"title": "A", "url": "https://a.example/", "snippet": "a"},
+        {"title": "Doc", "url": "https://b.example/report.pdf", "snippet": "pdf"},
+        {"title": "C", "url": "https://c.example/", "snippet": "c"},
+        {"title": "No URL", "snippet": "skipped"},
+    ],
+}
+
+
+def _response(payload=RESPONSE):
+    response = MagicMock()
+    response.json.return_value = payload
+    response.raise_for_status.return_value = None
+    return response
+
+
+@pytest.fixture(autouse=True)
+def _no_env_key(monkeypatch):
+    monkeypatch.delenv("KEENABLE_API_KEY", raising=False)
+
+
+def test_keyless_uses_public_endpoint_with_app_title():
+    with patch(
+        "scrapegraphai.utils.research_web.requests.post", return_value=_response()
+    ) as post:
+        results = search("q", search_engine="keenable", max_results=5)
+
+    assert results == ["https://a.example/", "https://c.example/"]
+    args, kwargs = post.call_args
+    assert args[0] == KEENABLE_PUBLIC_API_URL
+    assert kwargs["headers"]["X-Keenable-Title"] == "scrapegraphai"
+    assert "X-API-Key" not in kwargs["headers"]
+    assert kwargs["json"] == {"query": "q", "max_results": 5}
+
+
+def test_api_key_uses_authenticated_endpoint():
+    with patch(
+        "scrapegraphai.utils.research_web.requests.post", return_value=_response()
+    ) as post:
+        search("q", search_engine="Keenable", keenable_api_key="keen_test")
+
+    args, kwargs = post.call_args
+    assert args[0] == KEENABLE_API_URL
+    assert kwargs["headers"]["X-API-Key"] == "keen_test"
+
+
+def test_api_key_from_environment(monkeypatch):
+    monkeypatch.setenv("KEENABLE_API_KEY", "keen_env")
+    with patch(
+        "scrapegraphai.utils.research_web.requests.post", return_value=_response()
+    ) as post:
+        search("q", search_engine="keenable")
+
+    args, kwargs = post.call_args
+    assert args[0] == KEENABLE_API_URL
+    assert kwargs["headers"]["X-API-Key"] == "keen_env"
+
+
+def test_max_results_is_capped_at_keenables_limit():
+    with patch(
+        "scrapegraphai.utils.research_web.requests.post", return_value=_response()
+    ) as post:
+        search("q", search_engine="keenable", max_results=100)
+
+    assert post.call_args.kwargs["json"]["max_results"] == 50
+
+
+def test_results_are_trimmed_to_max_results():
+    with patch(
+        "scrapegraphai.utils.research_web.requests.post", return_value=_response()
+    ):
+        results = search("q", search_engine="keenable", max_results=1)
+
+    assert results == ["https://a.example/"]
+
+
+def test_http_error_is_a_search_request_error():
+    response = _response()
+    response.raise_for_status.side_effect = Exception("429 Too Many Requests")
+    with patch("scrapegraphai.utils.research_web.requests.post", return_value=response):
+        with pytest.raises(SearchRequestError, match="Keenable search failed"):
+            search("q", search_engine="keenable")