Skip to main content

mojentic/llm/tools/
web_search_tool.rs

1use crate::error::Result;
2use crate::llm::tools::{FunctionDescriptor, LlmTool, ToolDescriptor};
3use async_trait::async_trait;
4use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use serde_json::{json, Value};
8use std::collections::HashMap;
9
10const BASE_URL: &str = "https://lite.duckduckgo.com/lite/";
11const MAX_RESULTS: usize = 10;
12const TIMEOUT_SECONDS: u64 = 10;
13
14/// A web search result from DuckDuckGo
15#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
16pub struct SearchResult {
17    /// The title of the search result
18    pub title: String,
19    /// The URL of the search result
20    pub url: String,
21    /// A snippet/description of the search result
22    pub snippet: String,
23}
24
25/// Tool for searching the web using DuckDuckGo
26///
27/// This tool searches DuckDuckGo's lite endpoint and returns organic search results.
28/// It does not require an API key, making it a free alternative to paid search APIs.
29///
30/// # Examples
31///
32/// ```ignore
33/// use mojentic::llm::tools::web_search_tool::WebSearchTool;
34/// use std::collections::HashMap;
35///
36/// let tool = WebSearchTool::new();
37/// let mut args = HashMap::new();
38/// args.insert("query".to_string(), serde_json::json!("Rust programming"));
39///
40/// let results = tool.run(&args, &crate::llm::tools::ToolRunCtx::default()).await?;
41/// // results contains an array of search results with title, url, and snippet
42/// ```
43#[derive(Clone)]
44pub struct WebSearchTool {
45    client: reqwest::Client,
46}
47
48impl WebSearchTool {
49    /// Creates a new WebSearchTool instance
50    pub fn new() -> Self {
51        let client = reqwest::Client::builder()
52            .timeout(std::time::Duration::from_secs(TIMEOUT_SECONDS))
53            .build()
54            .expect("Failed to create HTTP client");
55
56        Self { client }
57    }
58
59    /// Creates a new WebSearchTool with a custom HTTP client (for testing)
60    #[cfg(test)]
61    pub fn with_client(client: reqwest::Client) -> Self {
62        Self { client }
63    }
64
65    /// Perform the web search
66    async fn perform_search(&self, query: &str) -> Result<Vec<SearchResult>> {
67        let url = format!("{}?q={}", BASE_URL, utf8_percent_encode(query, NON_ALPHANUMERIC));
68
69        let response = self
70            .client
71            .get(&url)
72            .send()
73            .await
74            .map_err(crate::error::MojenticError::HttpError)?;
75
76        if !response.status().is_success() {
77            return Err(crate::error::MojenticError::ApiError(format!(
78                "HTTP request failed with status {}",
79                response.status()
80            )));
81        }
82
83        let html = response.text().await.map_err(crate::error::MojenticError::HttpError)?;
84
85        self.parse_results(&html)
86    }
87
88    /// Parse HTML results from DuckDuckGo lite using regex-based extraction
89    fn parse_results(&self, html: &str) -> Result<Vec<SearchResult>> {
90        let link_re =
91            Regex::new(r#"<a[^>]+class="result-link"[^>]*href="([^"]+)"[^>]*>([^<]+)</a>"#)
92                .expect("Invalid link regex");
93        let snippet_re = Regex::new(r#"<td[^>]+class="result-snippet"[^>]*>([^<]*)<"#)
94            .expect("Invalid snippet regex");
95
96        let mut results = Vec::new();
97        let links: Vec<_> = link_re.captures_iter(html).collect();
98        let snippets: Vec<_> = snippet_re.captures_iter(html).collect();
99
100        for (i, caps) in links.iter().take(MAX_RESULTS).enumerate() {
101            let href = &caps[1];
102            let title = &caps[2];
103            let url = Self::decode_url(href);
104
105            let snippet = snippets.get(i).map(|s| Self::clean_text(&s[1])).unwrap_or_default();
106
107            results.push(SearchResult {
108                title: Self::clean_text(title),
109                url,
110                snippet,
111            });
112        }
113
114        Ok(results)
115    }
116
117    /// Decode DuckDuckGo redirect URLs
118    fn decode_url(url: &str) -> String {
119        // DuckDuckGo uses redirect URLs like //duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com
120        if url.contains("uddg=") {
121            url.split("uddg=")
122                .nth(1)
123                .and_then(|s| s.split('&').next())
124                .map(|s| percent_encoding::percent_decode_str(s).decode_utf8_lossy().to_string())
125                .unwrap_or_else(|| url.to_string())
126        } else {
127            url.to_string()
128        }
129    }
130
131    /// Clean text by removing extra whitespace and decoding HTML entities
132    fn clean_text(text: &str) -> String {
133        let text = text.trim().replace(|c: char| c.is_whitespace(), " ");
134        let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
135
136        // Decode common HTML entities
137        text.replace("&amp;", "&")
138            .replace("&lt;", "<")
139            .replace("&gt;", ">")
140            .replace("&quot;", "\"")
141            .replace("&#39;", "'")
142    }
143}
144
145impl Default for WebSearchTool {
146    fn default() -> Self {
147        Self::new()
148    }
149}
150
151#[async_trait]
152impl LlmTool for WebSearchTool {
153    async fn run(
154        &self,
155        args: &HashMap<String, Value>,
156        _ctx: &crate::llm::tools::ToolRunCtx,
157    ) -> Result<Value> {
158        let query = args.get("query").and_then(|v| v.as_str()).ok_or_else(|| {
159            crate::error::MojenticError::InvalidArgument("query parameter is required".to_string())
160        })?;
161
162        if query.is_empty() {
163            return Err(crate::error::MojenticError::InvalidArgument(
164                "query parameter cannot be empty".to_string(),
165            ));
166        }
167
168        // Since we're in a sync context, we need to run the async function
169        let rt = tokio::runtime::Runtime::new().map_err(|e| {
170            crate::error::MojenticError::RuntimeError(format!("Failed to create runtime: {}", e))
171        })?;
172
173        let results = rt.block_on(self.perform_search(query)).map_err(|e| {
174            crate::error::MojenticError::ToolExecutionError(format!("Search failed: {}", e))
175        })?;
176
177        Ok(json!(results))
178    }
179
180    fn descriptor(&self) -> ToolDescriptor {
181        ToolDescriptor {
182            r#type: "function".to_string(),
183            function: FunctionDescriptor {
184                name: "web_search".to_string(),
185                description: "Search the web for information using DuckDuckGo. Returns organic search results including title, URL, and snippet for each result.".to_string(),
186                parameters: json!({
187                    "type": "object",
188                    "properties": {
189                        "query": {
190                            "type": "string",
191                            "description": "The search query"
192                        }
193                    },
194                    "required": ["query"]
195                }),
196            },
197        }
198    }
199
200    fn clone_box(&self) -> Box<dyn LlmTool> {
201        Box::new(self.clone())
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208    use mockito::Server;
209
210    fn sample_html() -> String {
211        r#"
212        <!DOCTYPE html>
213        <html>
214        <body>
215            <table>
216                <tr>
217                    <td>
218                        <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.rust-lang.org%2F">The Rust Programming Language</a>
219                    </td>
220                </tr>
221                <tr>
222                    <td class="result-snippet">A language empowering everyone to build reliable and efficient software.</td>
223                </tr>
224                <tr>
225                    <td>
226                        <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2F">Rust Documentation</a>
227                    </td>
228                </tr>
229                <tr>
230                    <td class="result-snippet">The official Rust documentation and learning resources.</td>
231                </tr>
232                <tr>
233                    <td>
234                        <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fcrates.io%2F">crates.io: Rust Package Registry</a>
235                    </td>
236                </tr>
237                <tr>
238                    <td class="result-snippet">The Rust community's crate registry.</td>
239                </tr>
240            </table>
241        </body>
242        </html>
243        "#.to_string()
244    }
245
246    #[test]
247    fn test_descriptor() {
248        let tool = WebSearchTool::new();
249        let descriptor = tool.descriptor();
250
251        assert_eq!(descriptor.r#type, "function");
252        assert_eq!(descriptor.function.name, "web_search");
253        assert!(descriptor
254            .function
255            .description
256            .contains("Search the web for information using DuckDuckGo"));
257
258        let params = descriptor.function.parameters;
259        assert_eq!(params["type"], "object");
260        assert!(params["properties"]["query"].is_object());
261        assert_eq!(params["required"][0], "query");
262    }
263
264    #[test]
265    fn test_parse_results() {
266        let tool = WebSearchTool::new();
267        let html = sample_html();
268
269        let results = tool.parse_results(&html).unwrap();
270
271        assert_eq!(results.len(), 3);
272
273        assert_eq!(results[0].title, "The Rust Programming Language");
274        assert_eq!(results[0].url, "https://www.rust-lang.org/");
275        assert!(results[0].snippet.contains("A language empowering everyone"));
276
277        assert_eq!(results[1].title, "Rust Documentation");
278        assert_eq!(results[1].url, "https://doc.rust-lang.org/");
279        assert!(results[1].snippet.contains("official Rust documentation"));
280
281        assert_eq!(results[2].title, "crates.io: Rust Package Registry");
282        assert_eq!(results[2].url, "https://crates.io/");
283        assert!(results[2].snippet.contains("crate registry"));
284    }
285
286    #[test]
287    fn test_decode_url() {
288        let url = "//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpath";
289        let decoded = WebSearchTool::decode_url(url);
290        assert_eq!(decoded, "https://example.com/path");
291
292        // Test URL without encoding
293        let url = "https://example.com/direct";
294        let decoded = WebSearchTool::decode_url(url);
295        assert_eq!(decoded, "https://example.com/direct");
296    }
297
298    #[test]
299    fn test_clean_text() {
300        assert_eq!(WebSearchTool::clean_text("  Multiple   spaces  "), "Multiple spaces");
301        assert_eq!(WebSearchTool::clean_text("Text&amp;more"), "Text&more");
302        assert_eq!(WebSearchTool::clean_text("&lt;tag&gt;"), "<tag>");
303        assert_eq!(WebSearchTool::clean_text("&quot;quoted&quot;"), "\"quoted\"");
304        assert_eq!(WebSearchTool::clean_text("it&#39;s"), "it's");
305    }
306
307    #[test]
308    fn test_tool_matches() {
309        let tool = WebSearchTool::new();
310        assert!(tool.matches("web_search"));
311        assert!(!tool.matches("other_tool"));
312    }
313
314    #[tokio::test]
315    async fn test_run_missing_query() {
316        let tool = WebSearchTool::new();
317        let args = HashMap::new();
318
319        let result = tool.run(&args, &crate::llm::tools::ToolRunCtx::default()).await;
320        assert!(result.is_err());
321        assert!(result.unwrap_err().to_string().contains("query parameter is required"));
322    }
323
324    #[tokio::test]
325    async fn test_run_empty_query() {
326        let tool = WebSearchTool::new();
327        let mut args = HashMap::new();
328        args.insert("query".to_string(), json!(""));
329
330        let result = tool.run(&args, &crate::llm::tools::ToolRunCtx::default()).await;
331        assert!(result.is_err());
332        assert!(result.unwrap_err().to_string().contains("query parameter cannot be empty"));
333    }
334
335    #[tokio::test]
336    async fn test_perform_search_success() {
337        let mut server = Server::new_async().await;
338        let mock = server
339            .mock("GET", mockito::Matcher::Any)
340            .with_status(200)
341            .with_body(sample_html())
342            .create_async()
343            .await;
344
345        let client = reqwest::Client::builder()
346            .timeout(std::time::Duration::from_secs(10))
347            .build()
348            .unwrap();
349
350        let tool = WebSearchTool::with_client(client);
351
352        // Override BASE_URL for testing by constructing URL directly
353        let url = format!("{}?q=rust", server.url());
354        let response = tool.client.get(&url).send().await.unwrap();
355        let html = response.text().await.unwrap();
356        let results = tool.parse_results(&html).unwrap();
357
358        assert_eq!(results.len(), 3);
359        assert_eq!(results[0].title, "The Rust Programming Language");
360
361        mock.assert_async().await;
362    }
363
364    #[tokio::test]
365    async fn test_perform_search_http_error() {
366        let mut server = Server::new_async().await;
367        let mock = server.mock("GET", mockito::Matcher::Any).with_status(500).create_async().await;
368
369        let client = reqwest::Client::builder()
370            .timeout(std::time::Duration::from_secs(10))
371            .build()
372            .unwrap();
373
374        let tool = WebSearchTool::with_client(client);
375
376        let url = format!("{}?q=test", server.url());
377        let response = tool.client.get(&url).send().await.unwrap();
378
379        assert_eq!(response.status(), 500);
380
381        mock.assert_async().await;
382    }
383
384    #[test]
385    fn test_max_results_limit() {
386        let tool = WebSearchTool::new();
387
388        // Create HTML with more than MAX_RESULTS entries
389        let mut html = String::from("<html><body><table>");
390        for i in 0..15 {
391            html.push_str(&format!(
392                r#"<tr><td><a class="result-link" href="https://example.com/{}">Result {}</a></td></tr>"#,
393                i, i
394            ));
395            html.push_str(&format!(r#"<tr><td class="result-snippet">Snippet {}</td></tr>"#, i));
396        }
397        html.push_str("</table></body></html>");
398
399        let results = tool.parse_results(&html).unwrap();
400
401        assert_eq!(results.len(), MAX_RESULTS);
402    }
403
404    #[test]
405    fn test_clone_box() {
406        let tool = WebSearchTool::new();
407        let cloned = tool.clone_box();
408
409        assert_eq!(cloned.descriptor().function.name, tool.descriptor().function.name);
410    }
411
412    #[test]
413    fn test_search_result_serialization() {
414        let result = SearchResult {
415            title: "Test Title".to_string(),
416            url: "https://example.com".to_string(),
417            snippet: "Test snippet".to_string(),
418        };
419
420        let json = serde_json::to_string(&result).unwrap();
421        assert!(json.contains("Test Title"));
422        assert!(json.contains("https://example.com"));
423        assert!(json.contains("Test snippet"));
424
425        let deserialized: SearchResult = serde_json::from_str(&json).unwrap();
426        assert_eq!(deserialized, result);
427    }
428
429    #[test]
430    fn test_parse_empty_html() {
431        let tool = WebSearchTool::new();
432        let html = "<html><body></body></html>";
433
434        let results = tool.parse_results(html).unwrap();
435        assert_eq!(results.len(), 0);
436    }
437
438    #[test]
439    fn test_parse_malformed_html() {
440        let tool = WebSearchTool::new();
441        let html = "<html><body><a class=\"result-link\">No href</a></body></html>";
442
443        let results = tool.parse_results(html).unwrap();
444        assert_eq!(results.len(), 0);
445    }
446}