mojentic/llm/tools/
web_search_tool.rs1use crate::error::Result;
2use crate::llm::tools::{FunctionDescriptor, LlmTool, ToolDescriptor};
3use async_trait::async_trait;
4use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use serde_json::{json, Value};
8use std::collections::HashMap;
9
10const BASE_URL: &str = "https://lite.duckduckgo.com/lite/";
11const MAX_RESULTS: usize = 10;
12const TIMEOUT_SECONDS: u64 = 10;
13
14#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
16pub struct SearchResult {
17 pub title: String,
19 pub url: String,
21 pub snippet: String,
23}
24
25#[derive(Clone)]
44pub struct WebSearchTool {
45 client: reqwest::Client,
46}
47
48impl WebSearchTool {
49 pub fn new() -> Self {
51 let client = reqwest::Client::builder()
52 .timeout(std::time::Duration::from_secs(TIMEOUT_SECONDS))
53 .build()
54 .expect("Failed to create HTTP client");
55
56 Self { client }
57 }
58
59 #[cfg(test)]
61 pub fn with_client(client: reqwest::Client) -> Self {
62 Self { client }
63 }
64
65 async fn perform_search(&self, query: &str) -> Result<Vec<SearchResult>> {
67 let url = format!("{}?q={}", BASE_URL, utf8_percent_encode(query, NON_ALPHANUMERIC));
68
69 let response = self
70 .client
71 .get(&url)
72 .send()
73 .await
74 .map_err(crate::error::MojenticError::HttpError)?;
75
76 if !response.status().is_success() {
77 return Err(crate::error::MojenticError::ApiError(format!(
78 "HTTP request failed with status {}",
79 response.status()
80 )));
81 }
82
83 let html = response.text().await.map_err(crate::error::MojenticError::HttpError)?;
84
85 self.parse_results(&html)
86 }
87
88 fn parse_results(&self, html: &str) -> Result<Vec<SearchResult>> {
90 let link_re =
91 Regex::new(r#"<a[^>]+class="result-link"[^>]*href="([^"]+)"[^>]*>([^<]+)</a>"#)
92 .expect("Invalid link regex");
93 let snippet_re = Regex::new(r#"<td[^>]+class="result-snippet"[^>]*>([^<]*)<"#)
94 .expect("Invalid snippet regex");
95
96 let mut results = Vec::new();
97 let links: Vec<_> = link_re.captures_iter(html).collect();
98 let snippets: Vec<_> = snippet_re.captures_iter(html).collect();
99
100 for (i, caps) in links.iter().take(MAX_RESULTS).enumerate() {
101 let href = &caps[1];
102 let title = &caps[2];
103 let url = Self::decode_url(href);
104
105 let snippet = snippets.get(i).map(|s| Self::clean_text(&s[1])).unwrap_or_default();
106
107 results.push(SearchResult {
108 title: Self::clean_text(title),
109 url,
110 snippet,
111 });
112 }
113
114 Ok(results)
115 }
116
117 fn decode_url(url: &str) -> String {
119 if url.contains("uddg=") {
121 url.split("uddg=")
122 .nth(1)
123 .and_then(|s| s.split('&').next())
124 .map(|s| percent_encoding::percent_decode_str(s).decode_utf8_lossy().to_string())
125 .unwrap_or_else(|| url.to_string())
126 } else {
127 url.to_string()
128 }
129 }
130
131 fn clean_text(text: &str) -> String {
133 let text = text.trim().replace(|c: char| c.is_whitespace(), " ");
134 let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
135
136 text.replace("&", "&")
138 .replace("<", "<")
139 .replace(">", ">")
140 .replace(""", "\"")
141 .replace("'", "'")
142 }
143}
144
145impl Default for WebSearchTool {
146 fn default() -> Self {
147 Self::new()
148 }
149}
150
151#[async_trait]
152impl LlmTool for WebSearchTool {
153 async fn run(
154 &self,
155 args: &HashMap<String, Value>,
156 _ctx: &crate::llm::tools::ToolRunCtx,
157 ) -> Result<Value> {
158 let query = args.get("query").and_then(|v| v.as_str()).ok_or_else(|| {
159 crate::error::MojenticError::InvalidArgument("query parameter is required".to_string())
160 })?;
161
162 if query.is_empty() {
163 return Err(crate::error::MojenticError::InvalidArgument(
164 "query parameter cannot be empty".to_string(),
165 ));
166 }
167
168 let rt = tokio::runtime::Runtime::new().map_err(|e| {
170 crate::error::MojenticError::RuntimeError(format!("Failed to create runtime: {}", e))
171 })?;
172
173 let results = rt.block_on(self.perform_search(query)).map_err(|e| {
174 crate::error::MojenticError::ToolExecutionError(format!("Search failed: {}", e))
175 })?;
176
177 Ok(json!(results))
178 }
179
180 fn descriptor(&self) -> ToolDescriptor {
181 ToolDescriptor {
182 r#type: "function".to_string(),
183 function: FunctionDescriptor {
184 name: "web_search".to_string(),
185 description: "Search the web for information using DuckDuckGo. Returns organic search results including title, URL, and snippet for each result.".to_string(),
186 parameters: json!({
187 "type": "object",
188 "properties": {
189 "query": {
190 "type": "string",
191 "description": "The search query"
192 }
193 },
194 "required": ["query"]
195 }),
196 },
197 }
198 }
199
200 fn clone_box(&self) -> Box<dyn LlmTool> {
201 Box::new(self.clone())
202 }
203}
204
205#[cfg(test)]
206mod tests {
207 use super::*;
208 use mockito::Server;
209
210 fn sample_html() -> String {
211 r#"
212 <!DOCTYPE html>
213 <html>
214 <body>
215 <table>
216 <tr>
217 <td>
218 <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.rust-lang.org%2F">The Rust Programming Language</a>
219 </td>
220 </tr>
221 <tr>
222 <td class="result-snippet">A language empowering everyone to build reliable and efficient software.</td>
223 </tr>
224 <tr>
225 <td>
226 <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fdoc.rust-lang.org%2F">Rust Documentation</a>
227 </td>
228 </tr>
229 <tr>
230 <td class="result-snippet">The official Rust documentation and learning resources.</td>
231 </tr>
232 <tr>
233 <td>
234 <a class="result-link" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fcrates.io%2F">crates.io: Rust Package Registry</a>
235 </td>
236 </tr>
237 <tr>
238 <td class="result-snippet">The Rust community's crate registry.</td>
239 </tr>
240 </table>
241 </body>
242 </html>
243 "#.to_string()
244 }
245
246 #[test]
247 fn test_descriptor() {
248 let tool = WebSearchTool::new();
249 let descriptor = tool.descriptor();
250
251 assert_eq!(descriptor.r#type, "function");
252 assert_eq!(descriptor.function.name, "web_search");
253 assert!(descriptor
254 .function
255 .description
256 .contains("Search the web for information using DuckDuckGo"));
257
258 let params = descriptor.function.parameters;
259 assert_eq!(params["type"], "object");
260 assert!(params["properties"]["query"].is_object());
261 assert_eq!(params["required"][0], "query");
262 }
263
264 #[test]
265 fn test_parse_results() {
266 let tool = WebSearchTool::new();
267 let html = sample_html();
268
269 let results = tool.parse_results(&html).unwrap();
270
271 assert_eq!(results.len(), 3);
272
273 assert_eq!(results[0].title, "The Rust Programming Language");
274 assert_eq!(results[0].url, "https://www.rust-lang.org/");
275 assert!(results[0].snippet.contains("A language empowering everyone"));
276
277 assert_eq!(results[1].title, "Rust Documentation");
278 assert_eq!(results[1].url, "https://doc.rust-lang.org/");
279 assert!(results[1].snippet.contains("official Rust documentation"));
280
281 assert_eq!(results[2].title, "crates.io: Rust Package Registry");
282 assert_eq!(results[2].url, "https://crates.io/");
283 assert!(results[2].snippet.contains("crate registry"));
284 }
285
286 #[test]
287 fn test_decode_url() {
288 let url = "//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpath";
289 let decoded = WebSearchTool::decode_url(url);
290 assert_eq!(decoded, "https://example.com/path");
291
292 let url = "https://example.com/direct";
294 let decoded = WebSearchTool::decode_url(url);
295 assert_eq!(decoded, "https://example.com/direct");
296 }
297
298 #[test]
299 fn test_clean_text() {
300 assert_eq!(WebSearchTool::clean_text(" Multiple spaces "), "Multiple spaces");
301 assert_eq!(WebSearchTool::clean_text("Text&more"), "Text&more");
302 assert_eq!(WebSearchTool::clean_text("<tag>"), "<tag>");
303 assert_eq!(WebSearchTool::clean_text(""quoted""), "\"quoted\"");
304 assert_eq!(WebSearchTool::clean_text("it's"), "it's");
305 }
306
307 #[test]
308 fn test_tool_matches() {
309 let tool = WebSearchTool::new();
310 assert!(tool.matches("web_search"));
311 assert!(!tool.matches("other_tool"));
312 }
313
314 #[tokio::test]
315 async fn test_run_missing_query() {
316 let tool = WebSearchTool::new();
317 let args = HashMap::new();
318
319 let result = tool.run(&args, &crate::llm::tools::ToolRunCtx::default()).await;
320 assert!(result.is_err());
321 assert!(result.unwrap_err().to_string().contains("query parameter is required"));
322 }
323
324 #[tokio::test]
325 async fn test_run_empty_query() {
326 let tool = WebSearchTool::new();
327 let mut args = HashMap::new();
328 args.insert("query".to_string(), json!(""));
329
330 let result = tool.run(&args, &crate::llm::tools::ToolRunCtx::default()).await;
331 assert!(result.is_err());
332 assert!(result.unwrap_err().to_string().contains("query parameter cannot be empty"));
333 }
334
335 #[tokio::test]
336 async fn test_perform_search_success() {
337 let mut server = Server::new_async().await;
338 let mock = server
339 .mock("GET", mockito::Matcher::Any)
340 .with_status(200)
341 .with_body(sample_html())
342 .create_async()
343 .await;
344
345 let client = reqwest::Client::builder()
346 .timeout(std::time::Duration::from_secs(10))
347 .build()
348 .unwrap();
349
350 let tool = WebSearchTool::with_client(client);
351
352 let url = format!("{}?q=rust", server.url());
354 let response = tool.client.get(&url).send().await.unwrap();
355 let html = response.text().await.unwrap();
356 let results = tool.parse_results(&html).unwrap();
357
358 assert_eq!(results.len(), 3);
359 assert_eq!(results[0].title, "The Rust Programming Language");
360
361 mock.assert_async().await;
362 }
363
364 #[tokio::test]
365 async fn test_perform_search_http_error() {
366 let mut server = Server::new_async().await;
367 let mock = server.mock("GET", mockito::Matcher::Any).with_status(500).create_async().await;
368
369 let client = reqwest::Client::builder()
370 .timeout(std::time::Duration::from_secs(10))
371 .build()
372 .unwrap();
373
374 let tool = WebSearchTool::with_client(client);
375
376 let url = format!("{}?q=test", server.url());
377 let response = tool.client.get(&url).send().await.unwrap();
378
379 assert_eq!(response.status(), 500);
380
381 mock.assert_async().await;
382 }
383
384 #[test]
385 fn test_max_results_limit() {
386 let tool = WebSearchTool::new();
387
388 let mut html = String::from("<html><body><table>");
390 for i in 0..15 {
391 html.push_str(&format!(
392 r#"<tr><td><a class="result-link" href="https://example.com/{}">Result {}</a></td></tr>"#,
393 i, i
394 ));
395 html.push_str(&format!(r#"<tr><td class="result-snippet">Snippet {}</td></tr>"#, i));
396 }
397 html.push_str("</table></body></html>");
398
399 let results = tool.parse_results(&html).unwrap();
400
401 assert_eq!(results.len(), MAX_RESULTS);
402 }
403
404 #[test]
405 fn test_clone_box() {
406 let tool = WebSearchTool::new();
407 let cloned = tool.clone_box();
408
409 assert_eq!(cloned.descriptor().function.name, tool.descriptor().function.name);
410 }
411
412 #[test]
413 fn test_search_result_serialization() {
414 let result = SearchResult {
415 title: "Test Title".to_string(),
416 url: "https://example.com".to_string(),
417 snippet: "Test snippet".to_string(),
418 };
419
420 let json = serde_json::to_string(&result).unwrap();
421 assert!(json.contains("Test Title"));
422 assert!(json.contains("https://example.com"));
423 assert!(json.contains("Test snippet"));
424
425 let deserialized: SearchResult = serde_json::from_str(&json).unwrap();
426 assert_eq!(deserialized, result);
427 }
428
429 #[test]
430 fn test_parse_empty_html() {
431 let tool = WebSearchTool::new();
432 let html = "<html><body></body></html>";
433
434 let results = tool.parse_results(html).unwrap();
435 assert_eq!(results.len(), 0);
436 }
437
438 #[test]
439 fn test_parse_malformed_html() {
440 let tool = WebSearchTool::new();
441 let html = "<html><body><a class=\"result-link\">No href</a></body></html>";
442
443 let results = tool.parse_results(html).unwrap();
444 assert_eq!(results.len(), 0);
445 }
446}