返回提交历史
Added
g4f/Provider/SearXNG.py
+69
-0
Modified
g4f/tools/web_search.py
+243
-142
XFEstudio/gpt4free
Add SearXNG provider and support in web_search
9790c955
代码差异
2 个文件
+312
-142
@@ -0,0 +1,69 @@
1
import os
2
import aiohttp
3
import asyncio
4
from ..typing import Messages, AsyncResult
5
from ..providers.base_provider import AsyncGeneratorProvider
6
from ..providers.response import FinishReason
7
from ..tools.web_search import fetch_and_scrape
8
9
class SearXNG(AsyncGeneratorProvider):
10
default_url = os.environ.get("SEARXNG_URL", "http://searxng:8080")
11
label = "SearXNG (configurable)"
12
models = ["searx"]
13
14
@classmethod
15
async def create_async_generator(
16
cls,
17
model: str,
18
messages: Messages,
19
proxy: str = None,
20
timeout: int = 30,
21
max_results: int = 5,
22
max_words: int = 2500,
23
add_text: bool = True,
24
**kwargs
25
) -> AsyncResult:
26
url = cls.default_url
27
query = messages[-1]["content"] if isinstance(messages[-1], dict) else getattr(messages[-1], "content", "")
28
29
30
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
31
params = {
32
"q": query,
33
"format": "json",
34
"language": "it",
35
"safesearch": 0,
36
"categories": "general",
37
}
38
39
async with session.get(f"{url}/search", params=params) as resp:
40
print(f"Request URL on SearXNG: {resp.url}")
41
data = await resp.json()
42
results = data.get("results", [])
43
44
if not results:
45
yield "Nessun risultato trovato."
46
yield FinishReason("stop")
47
return
48
49
if add_text:
50
requests = []
51
for r in results[:max_results]:
52
requests.append(fetch_and_scrape(session, r["url"], int(max_words / max_results), False))
53
texts = await asyncio.gather(*requests)
54
for i, r in enumerate(results[:max_results]):
55
r["text"] = texts[i]
56
57
formatted = ""
58
used_words = 0
59
for i, r in enumerate(results[:max_results]):
60
title = r.get("title", "Senza titolo")
61
url = r.get("url", "#")
62
content = r.get("text") or r.get("snippet") or ""
63
formatted += f"Title: {title}\n\n{content}\n\nSource: [[{i}]]({url})\n\n"
64
used_words += content.count(" ")
65
if max_words and used_words >= max_words:
66
break
67
68
yield formatted.strip()
69
yield FinishReason("stop")
@@ -5,10 +5,10 @@ import json
5
5
import hashlib
6
6
from pathlib import Path
7
7
from urllib.parse import urlparse, quote_plus
8
from datetime import datetime
9
import datetime
8
from datetime import datetime, date
10
9
import asyncio
11
10
11
# Optional dependencies
12
12
try:
13
13
from duckduckgo_search import DDGS
14
14
from duckduckgo_search.exceptions import DuckDuckGoSearchException
@@ -17,13 +17,14 @@ try:
17
17
has_requirements = True
18
18
except ImportError:
19
19
has_requirements = False
20
20
21
try:
21
22
import spacy
22
23
has_spacy = True
23
except:
24
except ImportError:
24
25
has_spacy = False
25
26
26
from typing import Iterator
27
from typing import Iterator, List, Optional
27
28
from ..cookies import get_cookies_dir
28
29
from ..providers.response import format_link, JsonMixin, Sources
29
30
from ..errors import MissingRequirementsError
@@ -35,86 +36,104 @@ Make sure to add the sources of cites using [[Number]](Url) notation after the r
35
36
"""
36
37
37
38
class SearchResults(JsonMixin):
38
def __init__(self, results: list, used_words: int):
39
"""
40
Represents a collection of search result entries along with the count of used words.
41
"""
42
def __init__(self, results: List[SearchResultEntry], used_words: int):
39
43
self.results = results
40
44
self.used_words = used_words
41
45
42
46
@classmethod
43
def from_dict(cls, data: dict):
47
def from_dict(cls, data: dict) -> SearchResults:
44
48
return cls(
45
49
[SearchResultEntry(**item) for item in data["results"]],
46
50
data["used_words"]
47
51
)
48
52
49
def __iter__(self):
53
def __iter__(self) -> Iterator[SearchResultEntry]:
50
54
yield from self.results
51
55
52
def __str__(self):
53
search = ""
56
def __str__(self) -> str:
57
# Build a string representation of the search results with markdown formatting.
58
output = []
54
59
for idx, result in enumerate(self.results):
55
if search:
56
search += "\n\n\n"
57
search += f"Title: {result.title}\n\n"
58
if result.text:
59
search += result.text
60
else:
61
search += result.snippet
62
search += f"\n\nSource: [[{idx}]]({result.url})"
63
return search
60
parts = [
61
f"Title: {result.title}",
62
"",
63
result.text if result.text else result.snippet,
64
"",
65
f"Source: [[{idx}]]({result.url})"
66
]
67
output.append("\n".join(parts))
68
return "\n\n\n".join(output)
64
69
65
70
def __len__(self) -> int:
66
71
return len(self.results)
67
72
68
69
73
def get_sources(self) -> Sources:
70
74
return Sources([{"url": result.url, "title": result.title} for result in self.results])
71
75
72
def get_dict(self):
76
def get_dict(self) -> dict:
73
77
return {
74
78
"results": [result.get_dict() for result in self.results],
75
79
"used_words": self.used_words
76
80
}
77
81
78
82
class SearchResultEntry(JsonMixin):
79
def __init__(self, title: str, url: str, snippet: str, text: str = None):
83
"""
84
Represents a single search result entry.
85
"""
86
def __init__(self, title: str, url: str, snippet: str, text: Optional[str] = None):
80
87
self.title = title
81
88
self.url = url
82
89
self.snippet = snippet
83
90
self.text = text
84
91
85
def set_text(self, text: str):
92
def set_text(self, text: str) -> None:
86
93
self.text = text
87
94
88
def scrape_text(html: str, max_words: int = None, add_source=True, count_images: int = 2) -> Iterator[str]:
89
source = BeautifulSoup(html, "html.parser")
90
soup = source
95
def scrape_text(html: str, max_words: Optional[int] = None, add_source: bool = True, count_images: int = 2) -> Iterator[str]:
96
"""
97
Parses the provided HTML and yields text fragments.
98
99
Args:
100
html (str): HTML content to scrape.
101
max_words (int, optional): Maximum words allowed. Defaults to None.
102
add_source (bool): Whether to append source link at the end.
103
count_images (int): Maximum number of images to include.
104
105
Yields:
106
str: Text or markdown image links extracted from the HTML.
107
"""
108
soup = BeautifulSoup(html, "html.parser")
109
# Try to narrow the parsing scope using common selectors.
91
110
for selector in [
92
"main",
93
".main-content-wrapper",
94
".main-content",
95
".emt-container-inner",
96
".content-wrapper",
97
"#content",
98
"#mainContent",
99
]:
100
select = soup.select_one(selector)
101
if select:
102
soup = select
111
"main", ".main-content-wrapper", ".main-content", ".emt-container-inner",
112
".content-wrapper", "#content", "#mainContent",
113
]:
114
selected = soup.select_one(selector)
115
if selected:
116
soup = selected
103
117
break
104
# Zdnet
105
for remove in [".c-globalDisclosure"]:
106
select = soup.select_one(remove)
107
if select:
108
select.extract()
109
110
image_select = "img[alt][src^=http]:not([alt='']):not(.avatar):not([width])"
111
image_link_select = f"a:has({image_select})"
112
yield_words = []
113
for paragraph in soup.select(f"h1, h2, h3, h4, h5, h6, p, pre, table:not(:has(p)), ul:not(:has(p)), {image_link_select}"):
118
119
# Remove unwanted elements.
120
for remove_selector in [".c-globalDisclosure"]:
121
unwanted = soup.select_one(remove_selector)
122
if unwanted:
123
unwanted.extract()
124
125
image_selector = "img[alt][src^=http]:not([alt='']):not(.avatar):not([width])"
126
image_link_selector = f"a:has({image_selector})"
127
seen_texts = []
128
129
# Iterate over paragraphs and other elements.
130
for element in soup.select(f"h1, h2, h3, h4, h5, h6, p, pre, table:not(:has(p)), ul:not(:has(p)), {image_link_selector}"):
131
# Process images if available and allowed.
114
132
if count_images > 0:
115
image = paragraph.select_one(image_select)
133
image = element.select_one(image_selector)
116
134
if image:
117
title = str(paragraph.get("title", paragraph.text))
135
# Use the element's title attribute if available, otherwise use its text.
136
title = str(element.get("title", element.text))
118
137
if title:
119
138
yield f"!{format_link(image['src'], title)}\n"
120
139
if max_words is not None:
@@ -122,118 +141,221 @@ def scrape_text(html: str, max_words: int = None, add_source=True, count_images:
122
141
count_images -= 1
123
142
continue
124
143
125
for line in paragraph.get_text(" ").splitlines():
144
# Split the element text into lines and yield non-duplicate lines.
145
for line in element.get_text(" ").splitlines():
126
146
words = [word for word in line.split() if word]
127
count = len(words)
128
if not count:
147
if not words:
129
148
continue
130
words = " ".join(words)
131
if words in yield_words:
149
joined_line = " ".join(words)
150
if joined_line in seen_texts:
132
151
continue
133
if max_words:
134
max_words -= count
152
if max_words is not None:
153
max_words -= len(words)
135
154
if max_words <= 0:
136
155
break
137
yield words + "\n"
138
yield_words.append(words)
156
yield joined_line + "\n"
157
seen_texts.append(joined_line)
139
158
159
# Add a canonical link as source info if requested.
140
160
if add_source:
141
canonical_link = source.find("link", rel="canonical")
161
canonical_link = soup.find("link", rel="canonical")
142
162
if canonical_link and "href" in canonical_link.attrs:
143
163
link = canonical_link["href"]
144
164
domain = urlparse(link).netloc
145
165
yield f"\nSource: [{domain}]({link})"
146
166
147
async def fetch_and_scrape(session: ClientSession, url: str, max_words: int = None, add_source: bool = False) -> str:
167
async def fetch_and_scrape(session: ClientSession, url: str, max_words: Optional[int] = None, add_source: bool = False) -> str:
168
"""
169
Fetches a URL and returns the scraped text, using caching to avoid redundant downloads.
170
171
Args:
172
session (ClientSession): An aiohttp client session.
173
url (str): URL to fetch.
174
max_words (int, optional): Maximum words for the scraped text.
175
add_source (bool): Whether to append source link.
176
177
Returns:
178
str: The scraped text or an empty string in case of errors.
179
"""
148
180
try:
149
bucket_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "fetch_and_scrape"
150
bucket_dir.mkdir(parents=True, exist_ok=True)
181
cache_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "fetch_and_scrape"
182
cache_dir.mkdir(parents=True, exist_ok=True)
151
183
md5_hash = hashlib.md5(url.encode(errors="ignore")).hexdigest()
152
cache_file = bucket_dir / f"{quote_plus(url.split('?')[0].split('//')[1].replace('/', ' ')[:48])}.{datetime.date.today()}.{md5_hash[:16]}.cache"
184
# Build cache filename using a portion of the URL and current date.
185
cache_file = cache_dir / f"{quote_plus(url.split('?')[0].split('//')[1].replace('/', ' ')[:48])}.{date.today()}.{md5_hash[:16]}.cache"
153
186
if cache_file.exists():
154
187
return cache_file.read_text()
188
155
189
async with session.get(url) as response:
156
190
if response.status == 200:
157
191
html = await response.text(errors="replace")
158
text = "".join(scrape_text(html, max_words, add_source))
192
scraped_text = "".join(scrape_text(html, max_words, add_source))
159
193
with open(cache_file, "wb") as f:
160
f.write(text.encode(errors="replace"))
161
return text
194
f.write(scraped_text.encode(errors="replace"))
195
return scraped_text
162
196
except (ClientError, asyncio.TimeoutError):
163
return
197
return ""
198
return ""
199
200
async def search(
201
query: str,
202
max_results: int = 5,
203
max_words: int = 2500,
204
backend: str = "auto",
205
add_text: bool = True,
206
timeout: int = 5,
207
region: str = "wt-wt",
208
provider: str = "DDG" # Default fallback to DuckDuckGo
209
) -> SearchResults:
210
"""
211
Performs a web search and returns search results.
212
213
Args:
214
query (str): The search query.
215
max_results (int): Maximum number of results.
216
max_words (int): Maximum words for textual results.
217
backend (str): Backend type.
218
add_text (bool): Whether to fetch and add full text to each result.
219
timeout (int): Timeout for HTTP requests.
220
region (str): Region parameter for the search engine.
221
provider (str): The search provider to use.
222
223
Returns:
224
SearchResults: The collection of search results and used words.
225
"""
226
# If using SearXNG provider.
227
if provider == "SearXNG":
228
from ..Provider.SearXNG import SearXNG
229
230
debug.log(f"[SearXNG] Using local container for query: {query}")
231
232
results_texts = []
233
async for chunk in SearXNG.create_async_generator(
234
"SearXNG",
235
[{"role": "user", "content": query}],
236
max_results=max_results,
237
max_words=max_words,
238
add_text=add_text
239
):
240
if isinstance(chunk, str):
241
results_texts.append(chunk)
242
243
used_words = sum(text.count(" ") for text in results_texts)
244
245
return SearchResults([
246
SearchResultEntry(
247
title=f"Result {i + 1}",
248
url="",
249
snippet=text,
250
text=text
251
) for i, text in enumerate(results_texts)
252
], used_words=used_words)
253
254
# -------------------------
255
# Default: DuckDuckGo logic
256
# -------------------------
257
debug.log(f"[DuckDuckGo] Using local container for query: {query}")
164
258
165
async def search(query: str, max_results: int = 5, max_words: int = 2500, backend: str = "auto", add_text: bool = True, timeout: int = 5, region: str = "wt-wt") -> SearchResults:
166
259
if not has_requirements:
167
raise MissingRequirementsError('Install "duckduckgo-search" and "beautifulsoup4" package | pip install -U g4f[search]')
260
raise MissingRequirementsError('Install "duckduckgo-search" and "beautifulsoup4" | pip install -U g4f[search]')
168
261
169
results = []
262
results: List[SearchResultEntry] = []
170
263
for result in ddgs.text(
171
query,
172
region=region,
173
safesearch="moderate",
174
timelimit="y",
175
max_results=max_results,
176
backend=backend,
177
):
264
query,
265
region=region,
266
safesearch="moderate",
267
timelimit="y",
268
max_results=max_results,
269
backend=backend,
270
):
178
271
if ".google." in result["href"]:
179
272
continue
180
273
results.append(SearchResultEntry(
181
result["title"],
182
result["href"],
183
result["body"]
274
title=result["title"],
275
url=result["href"],
276
snippet=result["body"]
184
277
))
185
278
279
# Optionally add full text for each result.
186
280
if add_text:
187
requests = []
281
tasks = []
188
282
async with ClientSession(timeout=ClientTimeout(timeout)) as session:
189
283
for entry in results:
190
requests.append(fetch_and_scrape(session, entry.url, int(max_words / (max_results - 1)), False))
191
texts = await asyncio.gather(*requests)
284
# Divide available words among results
285
tasks.append(fetch_and_scrape(session, entry.url, int(max_words / (max_results - 1)), False))
286
texts = await asyncio.gather(*tasks)
192
287
193
formatted_results = []
288
formatted_results: List[SearchResultEntry] = []
194
289
used_words = 0
195
290
left_words = max_words
196
291
for i, entry in enumerate(results):
197
292
if add_text:
198
293
entry.text = texts[i]
199
if max_words:
200
left_words -= entry.title.count(" ") + 5
201
if entry.text:
202
left_words -= entry.text.count(" ")
203
else:
204
left_words -= entry.snippet.count(" ")
205
if 0 > left_words:
206
break
294
# Deduct word counts for title and text/snippet.
295
left_words -= entry.title.count(" ") + 5
296
if entry.text:
297
left_words -= entry.text.count(" ")
298
else:
299
left_words -= entry.snippet.count(" ")
300
if left_words < 0:
301
break
207
302
used_words = max_words - left_words
208
303
formatted_results.append(entry)
209
304
210
305
return SearchResults(formatted_results, used_words)
211
306
212
async def do_search(prompt: str, query: str = None, instructions: str = DEFAULT_INSTRUCTIONS, **kwargs) -> tuple[str, Sources]:
307
async def do_search(
308
prompt: str,
309
query: Optional[str] = None,
310
instructions: str = DEFAULT_INSTRUCTIONS,
311
**kwargs
312
) -> tuple[str, Optional[Sources]]:
313
"""
314
Combines search results with the user prompt, using caching for improved efficiency.
315
316
Args:
317
prompt (str): The user prompt.
318
query (str, optional): The search query. If None the first line of prompt is used.
319
instructions (str): Additional instructions to append.
320
**kwargs: Additional parameters for the search.
321
322
Returns:
323
tuple[str, Optional[Sources]]: A tuple containing the new prompt with search results and the sources.
324
"""
325
# If the prompt already includes the instructions, do not perform a search.
213
326
if instructions and instructions in prompt:
214
return prompt, None # We have already added search results
327
return prompt, None
328
215
329
if prompt.startswith("##") and query is None:
216
return prompt, None # We have no search query
330
return prompt, None
331
332
# Use the first line of the prompt as the query if not provided.
217
333
if query is None:
218
query = prompt.strip().splitlines()[0] # Use the first line as the search query
334
query = prompt.strip().splitlines()[0]
335
336
# Prepare a cache key.
219
337
json_bytes = json.dumps({"query": query, **kwargs}, sort_keys=True).encode(errors="ignore")
220
338
md5_hash = hashlib.md5(json_bytes).hexdigest()
221
bucket_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / f"web_search" / f"{datetime.date.today()}"
222
bucket_dir.mkdir(parents=True, exist_ok=True)
223
cache_file = bucket_dir / f"{quote_plus(query[:20])}.{md5_hash}.cache"
224
search_results = None
339
cache_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "web_search" / f"{date.today()}"
340
cache_dir.mkdir(parents=True, exist_ok=True)
341
cache_file = cache_dir / f"{quote_plus(query[:20])}.{md5_hash}.cache"
342
343
search_results: Optional[SearchResults] = None
344
# Load cached search results if available.
225
345
if cache_file.exists():
226
346
with cache_file.open("r") as f:
227
search_results = f.read()
228
try:
229
search_results = SearchResults.from_dict(json.loads(search_results))
230
except json.JSONDecodeError:
231
search_results = None
347
try:
348
search_results = SearchResults.from_dict(json.loads(f.read()))
349
except json.JSONDecodeError:
350
search_results = None
351
352
# Otherwise perform the search.
232
353
if search_results is None:
233
354
search_results = await search(query, **kwargs)
234
355
if search_results.results:
235
356
with cache_file.open("w") as f:
236
357
f.write(json.dumps(search_results.get_dict()))
358
237
359
if instructions:
238
360
new_prompt = f"""
239
361
{search_results}
@@ -249,49 +371,28 @@ User request:
249
371
250
372
{prompt}
251
373
"""
374
252
375
debug.log(f"Web search: '{query.strip()[:50]}...'")
253
376
debug.log(f"with {len(search_results.results)} Results {search_results.used_words} Words")
254
return new_prompt, search_results.get_sources()
255
256
def get_search_message(prompt: str, raise_search_exceptions=False, **kwargs) -> str:
377
return new_prompt.strip(), search_results.get_sources()
378
379
def get_search_message(prompt: str, raise_search_exceptions: bool = False, **kwargs) -> str:
380
"""
381
Synchronously obtains the search message by wrapping the async search call.
382
383
Args:
384
prompt (str): The original prompt.
385
raise_search_exceptions (bool): Whether to propagate search exceptions.
386
**kwargs: Additional search parameters.
387
388
Returns:
389
str: The new prompt including search results.
390
"""
257
391
try:
258
return asyncio.run(do_search(prompt, **kwargs))[0]
392
result, _ = asyncio.run(do_search(prompt, **kwargs))
393
return result
259
394
except (DuckDuckGoSearchException, MissingRequirementsError) as e:
260
395
if raise_search_exceptions:
261
396
raise e
262
397
debug.error(f"Couldn't do web search: {e.__class__.__name__}: {e}")
263
398
return prompt
264
265
def spacy_get_keywords(text: str):
266
if not has_spacy:
267
return text
268
269
# Load the spaCy language model
270
nlp = spacy.load("en_core_web_sm")
271
272
# Process the query
273
doc = nlp(text)
274
275
# Extract keywords based on POS and named entities
276
keywords = []
277
for token in doc:
278
# Filter for nouns, proper nouns, and adjectives
279
if token.pos_ in {"NOUN", "PROPN", "ADJ"} and not token.is_stop:
280
keywords.append(token.lemma_)
281
282
# Add named entities as keywords
283
for ent in doc.ents:
284
keywords.append(ent.text)
285
286
# Remove duplicates and print keywords
287
keywords = list(set(keywords))
288
#print("Keyword:", keywords)
289
290
#keyword_freq = Counter(keywords)
291
#keywords = keyword_freq.most_common()
292
#print("Keyword Frequencies:", keywords)
293
294
keywords = [chunk.text for chunk in doc.noun_chunks if not chunk.root.is_stop]
295
#print("Phrases:", keywords)
296
297
return keywords