From 51b72eb6c2faf45031fe0fdcc406162d11d25589 Mon Sep 17 00:00:00 2001
From: Miter <miter.me@outlook.com>
Date: Fri, 25 Apr 2025 12:49:37 +0800
Subject: [PATCH] =?UTF-8?q?=E8=8E=B7=E5=8F=96searxng=E6=90=9C=E7=B4=A2?=
 =?UTF-8?q?=E7=BB=93=E6=9E=9Curl=E7=9A=84=E5=86=85=E5=AE=B9=E4=BD=9C?=
 =?UTF-8?q?=E4=B8=BA=E5=A4=A7=E6=A8=A1=E5=9E=8B=E5=9B=9E=E7=AD=94=E7=9A=84?=
 =?UTF-8?q?=E5=8F=82=E8=80=83=EF=BC=8C=E8=80=8C=E4=B8=8D=E6=98=AF=E4=BD=BF?=
 =?UTF-8?q?=E7=94=A8searxng=E6=90=9C=E7=B4=A2=E7=BB=93=E6=9E=9C=E7=9A=84?=
 =?UTF-8?q?=E6=91=98=E8=A6=81=E5=86=85=E5=AE=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../WebSearchProvider/SearxngProvider.ts      | 37 ++++++++++++-------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts b/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts
index 0e95c12cc7..0fb35d33e8 100644
--- a/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts
+++ b/src/renderer/src/providers/WebSearchProvider/SearxngProvider.ts
@@ -1,6 +1,7 @@
 import { SearxngClient } from '@agentic/searxng'
 import { WebSearchState } from '@renderer/store/websearch'
-import { WebSearchProvider, WebSearchResponse } from '@renderer/types'
+import { WebSearchProvider, WebSearchResponse, WebSearchResult } from '@renderer/types'
+import { fetchWebContent, noContent } from '@renderer/utils/fetch'
 import axios from 'axios'
 import ky from 'ky'
 
@@ -112,20 +113,28 @@ export default class SearxngProvider extends BaseWebSearchProvider {
       if (!result || !Array.isArray(result.results)) {
         throw new Error('Invalid search results from SearxNG')
       }
-      return {
-        query: result.query,
-        results: result.results.slice(0, websearch.maxResults).map((result) => {
-          let content = result.content || ''
-          if (websearch.contentLimit && content.length > websearch.contentLimit) {
-            content = content.slice(0, websearch.contentLimit) + '...'
-          }
 
-          return {
-            title: result.title || 'No title',
-            content: content,
-            url: result.url || ''
-          }
-        })
+      const validItems = result.results
+        .filter((item) => item.url.startsWith('http') || item.url.startsWith('https'))
+        .slice(0, websearch.maxResults)
+      // console.log('Valid search items:', validItems)
+
+      // Fetch content for each URL concurrently
+      const fetchPromises = validItems.map(async (item) => {
+        // console.log(`Fetching content for ${item.url}...`)
+        const result = await fetchWebContent(item.url, 'markdown', this.provider.usingBrowser)
+        if (websearch.contentLimit && result.content.length > websearch.contentLimit) {
+          result.content = result.content.slice(0, websearch.contentLimit) + '...'
+        }
+        return result
+      })
+
+      // Wait for all fetches to complete
+      const results: WebSearchResult[] = await Promise.all(fetchPromises)
+      
+      return {
+        query: query,
+        results: results.filter((result) => result.content != noContent)
       }
     } catch (error) {
       console.error('Searxng search failed:', error)