From 172b31f17c353236adee8754a82e62270e0a24cf Mon Sep 17 00:00:00 2001 From: Karl-Gustav Kallasmaa <29797627+KGKallasmaa@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:59:51 +0100 Subject: [PATCH] feat(search): add Keenable as a search engine, keyless by default --- scrapegraphai/graphs/search_graph.py | 1 + scrapegraphai/nodes/search_internet_node.py | 3 + scrapegraphai/utils/research_web.py | 86 ++++++++++++++++- tests/utils/research_web_keenable_test.py | 101 ++++++++++++++++++++ 4 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 tests/utils/research_web_keenable_test.py diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py index 2458c1d8b..3331ae77c 100644 --- a/scrapegraphai/graphs/search_graph.py +++ b/scrapegraphai/graphs/search_graph.py @@ -71,6 +71,7 @@ def _create_graph(self) -> BaseGraph: "storage_state": self.copy_config.get("storage_state"), "search_engine": self.copy_config.get("search_engine"), "serper_api_key": self.copy_config.get("serper_api_key"), + "keenable_api_key": self.copy_config.get("keenable_api_key"), }, ) diff --git a/scrapegraphai/nodes/search_internet_node.py b/scrapegraphai/nodes/search_internet_node.py index b9ccb0ece..6669d3ce5 100644 --- a/scrapegraphai/nodes/search_internet_node.py +++ b/scrapegraphai/nodes/search_internet_node.py @@ -55,6 +55,8 @@ def __init__( node_config["serper_api_key"] if node_config.get("serper_api_key") else None ) + self.keenable_api_key = node_config.get("keenable_api_key") + self.max_results = node_config.get("max_results", 3) def execute(self, state: dict) -> dict: @@ -108,6 +110,7 @@ def execute(self, state: dict) -> dict: search_engine=self.search_engine, proxy=self.proxy, serper_api_key=self.serper_api_key, + keenable_api_key=self.keenable_api_key, ) if len(answer) == 0: diff --git a/scrapegraphai/utils/research_web.py b/scrapegraphai/utils/research_web.py index 7efb497cb..a52b7e914 100644 --- a/scrapegraphai/utils/research_web.py +++ b/scrapegraphai/utils/research_web.py @@ -3,6 +3,7 @@ error handling, validation, and security features. """ +import os import random import re import time @@ -56,13 +57,16 @@ class SearchConfig(BaseModel): None, description="Proxy configuration" ) serper_api_key: Optional[str] = Field(None, description="API key for Serper") + keenable_api_key: Optional[str] = Field( + None, description="Optional API key for Keenable (works without one)" + ) region: Optional[str] = Field(None, description="Country/region code") language: str = Field("en", description="Language code") @validator("search_engine") def validate_search_engine(cls, v): """Validate search engine.""" - valid_engines = {"duckduckgo", "bing", "searxng", "serper"} + valid_engines = {"duckduckgo", "bing", "searxng", "serper", "keenable"} if v.lower() not in valid_engines: raise ValueError( f"Search engine must be one of: {', '.join(valid_engines)}" @@ -167,6 +171,7 @@ def search_on_web( serper_api_key: Optional[str] = None, region: Optional[str] = None, language: str = "en", + keenable_api_key: Optional[str] = None, ) -> List[str]: """ Search web function with improved error handling, validation, and security features. @@ -181,6 +186,9 @@ def search_on_web( serper_api_key (str): API key for Serper region (str): Country/region code (e.g., 'mx' for Mexico) language (str): Language code (e.g., 'es' for Spanish) + keenable_api_key (str): Optional API key for Keenable. Falls back to the + KEENABLE_API_KEY environment variable; without either, the keyless + public endpoint is used (rate-limited per IP) Returns: List[str]: List of URLs from search results @@ -203,6 +211,7 @@ def search_on_web( timeout=timeout, proxy=proxy, serper_api_key=serper_api_key, + keenable_api_key=keenable_api_key, region=region, language=language, ) @@ -233,6 +242,15 @@ def search_on_web( config.query, config.max_results, config.serper_api_key, config.timeout ) + elif config.search_engine == "keenable": + results = _search_keenable( + config.query, + config.max_results, + config.keenable_api_key, + config.timeout, + formatted_proxy, + ) + return filter_pdf_links(results) except requests.Timeout: @@ -416,6 +434,72 @@ def _search_serper( raise SearchRequestError(f"Serper search failed: {str(e)}") +KEENABLE_API_URL = "https://api.keenable.ai/v1/search" +KEENABLE_PUBLIC_API_URL = "https://api.keenable.ai/v1/search/public" +# Keenable returns at most 50 results per call. +KEENABLE_MAX_RESULTS = 50 + + +def _search_keenable( + query: str, + max_results: int, + api_key: Optional[str], + timeout: int, + proxy: Optional[str] = None, +) -> List[str]: + """ + Helper function for Keenable search (https://docs.keenable.ai). + + Works without an API key: with no key (argument or KEENABLE_API_KEY + environment variable) it calls Keenable's public endpoint, which is + rate-limited per IP and needs an ``X-Keenable-Title`` header naming the app. + A key selects the authenticated endpoint and lifts the hourly limit. + + Args: + query (str): Search query + max_results (int): Maximum number of results to return (capped at 50) + api_key (str, optional): API key for Keenable + timeout (int): Request timeout in seconds + proxy (str, optional): Proxy configuration + + Returns: + List[str]: List of URLs from search results + """ + api_key = api_key or os.environ.get("KEENABLE_API_KEY") or None + + headers = {"Content-Type": "application/json", "X-Keenable-Title": "scrapegraphai"} + if api_key: + headers["X-API-Key"] = api_key + url = KEENABLE_API_URL + else: + url = KEENABLE_PUBLIC_API_URL + + data = {"query": query, "max_results": min(max_results, KEENABLE_MAX_RESULTS)} + proxies = {"http": proxy, "https": proxy} if proxy else None + + try: + response = requests.post( + url, + json=data, + headers=headers, + proxies=proxies, + timeout=timeout, + ) + response.raise_for_status() + + json_data = response.json() + results = [] + for item in json_data.get("results", []): + if item.get("url"): + results.append(item["url"]) + if len(results) >= max_results: + break + + return results + except Exception as e: + raise SearchRequestError(f"Keenable search failed: {str(e)}") + + def format_proxy(proxy_config: Union[str, Dict, ProxyConfig]) -> str: """ Format proxy configuration into a string. diff --git a/tests/utils/research_web_keenable_test.py b/tests/utils/research_web_keenable_test.py new file mode 100644 index 000000000..a54a01791 --- /dev/null +++ b/tests/utils/research_web_keenable_test.py @@ -0,0 +1,101 @@ +"""Mock-based tests for the Keenable engine in search_on_web (no network).""" + +from unittest.mock import MagicMock, patch + +import pytest + +from scrapegraphai.utils.research_web import ( + KEENABLE_API_URL, + KEENABLE_PUBLIC_API_URL, + SearchRequestError, + search_on_web, +) + +# search_on_web is rate limited (10 calls/minute), so call the undecorated function. +search = search_on_web.__wrapped__ + +RESPONSE = { + "query": "q", + "mode": "pro", + "results": [ + {"title": "A", "url": "https://a.example/", "snippet": "a"}, + {"title": "Doc", "url": "https://b.example/report.pdf", "snippet": "pdf"}, + {"title": "C", "url": "https://c.example/", "snippet": "c"}, + {"title": "No URL", "snippet": "skipped"}, + ], +} + + +def _response(payload=RESPONSE): + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +@pytest.fixture(autouse=True) +def _no_env_key(monkeypatch): + monkeypatch.delenv("KEENABLE_API_KEY", raising=False) + + +def test_keyless_uses_public_endpoint_with_app_title(): + with patch( + "scrapegraphai.utils.research_web.requests.post", return_value=_response() + ) as post: + results = search("q", search_engine="keenable", max_results=5) + + assert results == ["https://a.example/", "https://c.example/"] + args, kwargs = post.call_args + assert args[0] == KEENABLE_PUBLIC_API_URL + assert kwargs["headers"]["X-Keenable-Title"] == "scrapegraphai" + assert "X-API-Key" not in kwargs["headers"] + assert kwargs["json"] == {"query": "q", "max_results": 5} + + +def test_api_key_uses_authenticated_endpoint(): + with patch( + "scrapegraphai.utils.research_web.requests.post", return_value=_response() + ) as post: + search("q", search_engine="Keenable", keenable_api_key="keen_test") + + args, kwargs = post.call_args + assert args[0] == KEENABLE_API_URL + assert kwargs["headers"]["X-API-Key"] == "keen_test" + + +def test_api_key_from_environment(monkeypatch): + monkeypatch.setenv("KEENABLE_API_KEY", "keen_env") + with patch( + "scrapegraphai.utils.research_web.requests.post", return_value=_response() + ) as post: + search("q", search_engine="keenable") + + args, kwargs = post.call_args + assert args[0] == KEENABLE_API_URL + assert kwargs["headers"]["X-API-Key"] == "keen_env" + + +def test_max_results_is_capped_at_keenables_limit(): + with patch( + "scrapegraphai.utils.research_web.requests.post", return_value=_response() + ) as post: + search("q", search_engine="keenable", max_results=100) + + assert post.call_args.kwargs["json"]["max_results"] == 50 + + +def test_results_are_trimmed_to_max_results(): + with patch( + "scrapegraphai.utils.research_web.requests.post", return_value=_response() + ): + results = search("q", search_engine="keenable", max_results=1) + + assert results == ["https://a.example/"] + + +def test_http_error_is_a_search_request_error(): + response = _response() + response.raise_for_status.side_effect = Exception("429 Too Many Requests") + with patch("scrapegraphai.utils.research_web.requests.post", return_value=response): + with pytest.raises(SearchRequestError, match="Keenable search failed"): + search("q", search_engine="keenable")