gpt4free

公开

关注 0 Fork 0 Star 0

返回提交历史

XFEstudio/gpt4free

Refactor search and response handling; introduce CachedSearch and DDGS classes for improved web search functionality and response management. Add PlainTextResponse for handling plain text responses. Update requirements and setup for new dependencies.

6b210f44

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-10-03 11:38

父提交 305f473

代码差异

17 个文件 +505 -313

Modified g4f/Provider/__init__.py +2 -3

@@ -12,9 +12,6 @@ try:

12 12 from .needs_auth.mini_max import HailuoAI, MiniMax

13 13 except ImportError as e:

14 14 debug.error("MiniMax providers not loaded:", e)

16 from .template import OpenaiTemplate, BackendApi

17 from .qwen.QwenCode import QwenCode

18 15 try:

19 16 from .not_working import *

20 17 except ImportError as e:

@@ -36,6 +33,8 @@ try:

36 33 except ImportError as e:

37 34 debug.error("Search providers not loaded:", e)

38 35

36 from .template import OpenaiTemplate, BackendApi

37 from .qwen.QwenCode import QwenCode

39 38 from .deprecated.ARTA import ARTA

40 39 from .deprecated.Blackbox import Blackbox

41 40 from .deprecated.DuckDuckGo import DuckDuckGo

Modified g4f/Provider/needs_auth/LMArena.py +6 -1

@@ -25,7 +25,7 @@ except ImportError:

25 25 from ...typing import AsyncResult, Messages, MediaListType

26 26 from ...requests import StreamSession, get_args_from_nodriver, raise_for_status, merge_cookies

27 27 from ...errors import ModelNotFoundError, CloudflareError, MissingAuthError, MissingRequirementsError

28 from ...providers.response import FinishReason, Usage, JsonConversation, ImageResponse, Reasoning

from ...providers.response import FinishReason, Usage, JsonConversation, ImageResponse, Reasoning, PlainTextResponse, JsonRequest

29 29 from ...tools.media import merge_media

30 30 from ..base_provider import AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin

31 31 from ..helper import get_last_user_message

@@ -675,6 +675,7 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

675 675 ],

676 676 "modality": "image" if is_image_model else "chat"

677 677 }

678 yield JsonRequest.from_dict(data)

678 679 try:

679 680 async with StreamSession(**args, timeout=timeout) as session:

680 681 async with session.post(

@@ -686,6 +687,7 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

686 687 args["cookies"] = merge_cookies(args["cookies"], response)

687 688 async for chunk in response.iter_lines():

688 689 line = chunk.decode()

690 yield PlainTextResponse(line)

689 691 if line.startswith("af:"):

690 692 yield JsonConversation(message_ids=[modelAMessageId])

691 693 elif line.startswith("a0:"):

@@ -693,6 +695,9 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

693 695 if chunk == "hasArenaError":

694 696 raise ModelNotFoundError("LMArena Beta encountered an error: hasArenaError")

695 697 yield chunk

698 elif line.startswith("ag:"):

699 chunk = json.loads(line[3:])

700 yield Reasoning(chunk)

696 701 elif line.startswith("a2:"):

697 702 yield ImageResponse([image.get("image") for image in json.loads(line[3:])], prompt)

698 703 elif line.startswith("ad:"):

Added g4f/Provider/search/CachedSearch.py +103 -0

@@ -0,0 +1,103 @@

1 from __future__ import annotations

3 import json

4 import hashlib

5 from pathlib import Path

6 from urllib.parse import quote_plus

7 from datetime import date

9 from ...typing import AsyncResult, Messages, Optional

10 from ..base_provider import AsyncGeneratorProvider, AuthFileMixin

11 from ...cookies import get_cookies_dir

12 from ..helper import format_media_prompt

13 from .DDGS import DDGS, SearchResults, SearchResultEntry

14 from .SearXNG import SearXNG

15 from ... import debug

17 async def search(

18 query: str,

19 max_results: int = 5,

20 max_words: int = 2500,

21 backend: str = "auto",

22 add_text: bool = True,

23 timeout: int = 5,

24 region: str = "us-en",

25 provider: str = "DDG"

26 ) -> SearchResults:

27 """

28 Performs a web search and returns search results.

29 """

30 if provider == "SearXNG":

31 debug.log(f"[SearXNG] Using local container for query: {query}")

32 results_texts = []

33 async for chunk in SearXNG.create_async_generator(

34 "SearXNG",

35 [{"role": "user", "content": query}],

36 max_results=max_results,

37 max_words=max_words,

38 add_text=add_text

39 ):

40 if isinstance(chunk, str):

41 results_texts.append(chunk)

42 used_words = sum(text.count(" ") for text in results_texts)

43 return SearchResults([

44 SearchResultEntry(

45 title=f"Result {i + 1}",

46 url="",

47 snippet=text,

48 text=text

49 ) for i, text in enumerate(results_texts)

50 ], used_words=used_words)

52 return await anext(DDGS.create_async_generator(

53 provider,

54 [],

55 prompt=query,

56 max_results=max_results,

57 max_words=max_words,

58 add_text=add_text,

59 timeout=timeout,

60 region=region,

61 backend=backend

62 ))

64 class CachedSearch(AsyncGeneratorProvider, AuthFileMixin):

65 working = True

67 @classmethod

68 async def create_async_generator(

69 cls,

70 model: str,

71 messages: Messages,

72 prompt: str = None,

73 **kwargs

74 ) -> AsyncResult:

75 """

76 Combines search results with the user prompt, using caching for improved efficiency.

77 """

78 prompt = format_media_prompt(messages, prompt)

79 search_parameters = ["max_results", "max_words", "add_text", "timeout", "region"]

80 search_parameters = {k: v for k, v in kwargs.items() if k in search_parameters}

        json_bytes = json.dumps({"model": model, "query": prompt, **search_parameters}, sort_keys=True).encode(errors="ignore")

82 md5_hash = hashlib.md5(json_bytes).hexdigest()

83 cache_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "web_search" / f"{date.today()}"

84 cache_dir.mkdir(parents=True, exist_ok=True)

85 cache_file = cache_dir / f"{quote_plus(prompt[:20])}.{md5_hash}.cache"

87 search_results: Optional[SearchResults] = None

88 if cache_file.exists():

89 with cache_file.open("r") as f:

90 try:

91 search_results = SearchResults.from_dict(json.loads(f.read()))

92 except json.JSONDecodeError:

93 search_results = None

95 if search_results is None:

96 if model:

97 search_parameters["provider"] = model

98 search_results = await search(prompt, **search_parameters)

99 if search_results.results:

100 with cache_file.open("w") as f:

101 f.write(json.dumps(search_results.get_dict()))

102

103 yield search_results

Added g4f/Provider/search/DDGS.py +228 -0

@@ -0,0 +1,228 @@

1 from __future__ import annotations

3 import hashlib

4 import asyncio

5 from pathlib import Path

6 from typing import Iterator, List, Optional

7 from urllib.parse import urlparse, quote_plus

8 from aiohttp import ClientSession, ClientTimeout, ClientError

9 from datetime import date

10 import asyncio

12 # Optional dependencies using the new 'ddgs' package name

13 try:

14 from ddgs import DDGS as DDGSClient

15 from bs4 import BeautifulSoup

16 has_requirements = True

17 except ImportError:

18 has_requirements = False

20 from ...typing import Messages, AsyncResult

21 from ...cookies import get_cookies_dir

22 from ...providers.response import format_link, JsonMixin, Sources

23 from ...errors import MissingRequirementsError

24 from ...providers.base_provider import AsyncGeneratorProvider

25 from ..helper import format_media_prompt

def scrape_text(html: str, max_words: Optional[int] = None, add_source: bool = True, count_images: int = 2) -> Iterator[str]:

28 """

29 Parses the provided HTML and yields text fragments.

30 """

31 soup = BeautifulSoup(html, "html.parser")

32 for selector in [

33 "main", ".main-content-wrapper", ".main-content", ".emt-container-inner",

34 ".content-wrapper", "#content", "#mainContent",

35 ]:

36 selected = soup.select_one(selector)

37 if selected:

38 soup = selected

39 break

41 for remove_selector in [".c-globalDisclosure"]:

42 unwanted = soup.select_one(remove_selector)

43 if unwanted:

44 unwanted.extract()

46 image_selector = "img[alt][src^=http]:not([alt='']):not(.avatar):not([width])"

47 image_link_selector = f"a:has({image_selector})"

48 seen_texts = []

    for element in soup.select(f"h1, h2, h3, h4, h5, h6, p, pre, table:not(:has(p)), ul:not(:has(p)), {image_link_selector}"):

51 if count_images > 0:

52 image = element.select_one(image_selector)

53 if image:

54 title = str(element.get("title", element.text))

55 if title:

56 yield f"!{format_link(image['src'], title)}\n"

57 if max_words is not None:

58 max_words -= 10

59 count_images -= 1

60 continue

62 for line in element.get_text(" ").splitlines():

63 words = [word for word in line.split() if word]

64 if not words:

65 continue

66 joined_line = " ".join(words)

67 if joined_line in seen_texts:

68 continue

69 if max_words is not None:

70 max_words -= len(words)

71 if max_words <= 0:

72 break

73 yield joined_line + "\n"

74 seen_texts.append(joined_line)

76 if add_source:

77 canonical_link = soup.find("link", rel="canonical")

78 if canonical_link and "href" in canonical_link.attrs:

79 link = canonical_link["href"]

80 domain = urlparse(link).netloc

81 yield f"\nSource: [{domain}]({link})"

async def fetch_and_scrape(session: ClientSession, url: str, max_words: Optional[int] = None, add_source: bool = False, proxy: str = None) -> str:

84 """

85 Fetches a URL and returns the scraped text, using caching to avoid redundant downloads.

86 """

87 try:

88 cache_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "fetch_and_scrape"

89 cache_dir.mkdir(parents=True, exist_ok=True)

90 md5_hash = hashlib.md5(url.encode(errors="ignore")).hexdigest()

        cache_file = cache_dir / f"{quote_plus(url.split('?')[0].split('//')[1].replace('/', ' ')[:48])}.{date.today()}.{md5_hash[:16]}.cache"

92 if cache_file.exists():

93 return cache_file.read_text()

95 async with session.get(url, proxy=proxy) as response:

96 if response.status == 200:

97 html = await response.text(errors="replace")

98 scraped_text = "".join(scrape_text(html, max_words, add_source))

99 with open(cache_file, "wb") as f:

100 f.write(scraped_text.encode(errors="replace"))

101 return scraped_text

102 except (ClientError, asyncio.TimeoutError):

103 return ""

104 return ""

105

106 class SearchResults(JsonMixin):

107 """

108 Represents a collection of search result entries along with the count of used words.

109 """

110 def __init__(self, results: List[SearchResultEntry], used_words: int):

111 self.results = results

112 self.used_words = used_words

113

114 @classmethod

115 def from_dict(cls, data: dict) -> SearchResults:

116 return cls(

117 [SearchResultEntry(**item) for item in data["results"]],

118 data["used_words"]

119 )

120

121 def __iter__(self) -> Iterator[SearchResultEntry]:

122 yield from self.results

123

124 def __str__(self) -> str:

125 # Build a string representation of the search results with markdown formatting.

126 output = []

127 for idx, result in enumerate(self.results):

128 parts = [

129 f"### Title: {result.title}",

130 "",

131 result.text if result.text else result.snippet,

132 "",

133 f"> **Source:** [[{idx}]]({result.url})"

134 ]

135 output.append("\n".join(parts))

136 return "\n\n\n\n".join(output)

137

138 def __len__(self) -> int:

139 return len(self.results)

140

141 def get_sources(self) -> Sources:

142 return Sources([{"url": result.url, "title": result.title} for result in self.results])

143

144 def get_dict(self) -> dict:

145 return {

146 "results": [result.get_dict() for result in self.results],

147 "used_words": self.used_words

148 }

149

150 class SearchResultEntry(JsonMixin):

151 """

152 Represents a single search result entry.

153 """

154 def __init__(self, title: str, url: str, snippet: str, text: Optional[str] = None):

155 self.title = title

156 self.url = url

157 self.snippet = snippet

158 self.text = text

159

160 def set_text(self, text: str) -> None:

161 self.text = text

162

163 class DDGS(AsyncGeneratorProvider):

164 working = has_requirements

165

166 @classmethod

167 async def create_async_generator(

168 cls,

169 model: str,

170 messages: Messages,

171 prompt: str = None,

172 proxy: str = None,

173 timeout: int = 30,

174 region: str = None,

175 backend: str = None,

176 max_results: int = 5,

177 max_words: int = 2500,

178 add_text: bool = True,

179 **kwargs

180 ) -> AsyncResult:

181 if not has_requirements:

182 raise MissingRequirementsError('Install "ddgs" and "beautifulsoup4" | pip install -U g4f[search]')

183

184 prompt = format_media_prompt(messages, prompt)

185 results: List[SearchResultEntry] = []

186

187 # Use the new DDGS() context manager style

188 with DDGSClient() as ddgs:

189 for result in ddgs.text(

190 prompt,

191 region=region,

192 safesearch="moderate",

193 timelimit="y",

194 max_results=max_results,

195 backend=backend,

196 ):

197 if ".google." in result["href"]:

198 continue

199 results.append(SearchResultEntry(

200 title=result["title"],

201 url=result["href"],

202 snippet=result["body"]

203 ))

204

205 if add_text:

206 tasks = []

207 async with ClientSession(timeout=ClientTimeout(timeout)) as session:

208 for entry in results:

209

                    tasks.append(fetch_and_scrape(session, entry.url, int(max_words / (max_results - 1)), False, proxy=proxy))

210 texts = await asyncio.gather(*tasks)

211

212 formatted_results: List[SearchResultEntry] = []

213 used_words = 0

214 left_words = max_words

215 for i, entry in enumerate(results):

216 if add_text:

217 entry.text = texts[i]

218 left_words -= entry.title.count(" ") + 5

219 if entry.text:

220 left_words -= entry.text.count(" ")

221 else:

222 left_words -= entry.snippet.count(" ")

223 if left_words < 0:

224 break

225 used_words = max_words - left_words

226 formatted_results.append(entry)

227

228 yield SearchResults(formatted_results, used_words)

Modified g4f/Provider/search/SearXNG.py +5 -2

@@ -1,11 +1,14 @@

1 from __future__ import annotations

1 3 import os

2 4 import aiohttp

3 5 import asyncio

4 7 from ...typing import Messages, AsyncResult

5 8 from ...providers.base_provider import AsyncGeneratorProvider

6 9 from ...providers.response import FinishReason

7 from ...tools.web_search import fetch_and_scrape

8 10 from ..helper import format_media_prompt

11 from .DDGS import fetch_and_scrape

9 12 from ... import debug

10 13

11 14 class SearXNG(AsyncGeneratorProvider):

@@ -20,7 +23,7 @@ class SearXNG(AsyncGeneratorProvider):

20 23 prompt: str = None,

21 24 proxy: str = None,

22 25 timeout: int = 30,

23 language: str = "it",

26 language: str = None,

24 27 max_results: int = 5,

25 28 max_words: int = 2500,

26 29 add_text: bool = True,

Modified g4f/Provider/search/__init__.py +1 -0

@@ -1,3 +1,4 @@

1 from .CachedSearch import CachedSearch

1 2 from .GoogleSearch import GoogleSearch

2 3 from .SearXNG import SearXNG

3 4 from .YouTube import YouTube

Modified g4f/gui/server/api.py +2 -0

@@ -278,6 +278,8 @@ class Api:

278 278 yield self._format_json("request", chunk.get_dict())

279 279 elif isinstance(chunk, JsonResponse):

280 280 yield self._format_json("response", chunk.get_dict())

281 elif isinstance(chunk, PlainTextResponse):

282 yield self._format_json("response", chunk.text)

281 283 else:

282 284 yield self._format_json("content", str(chunk))

283 285 except MissingAuthError as e:

Modified g4f/image/copy_images.py +1 -9

@@ -16,7 +16,6 @@ from ..requests.aiohttp import get_connector

16 16 from ..image import MEDIA_TYPE_MAP, EXTENSIONS_MAP

17 17 from ..tools.files import secure_filename

18 18 from ..providers.response import ImageResponse, AudioResponse, VideoResponse, quote_url

19 from ..Provider.template import BackendApi

20 19 from . import is_accepted_format, extract_data_uri

21 20 from .. import debug

22 21

@@ -171,15 +170,8 @@ async def copy_media(

171 170 with open(target_path, "wb") as f:

172 171 f.write(extract_data_uri(image))

173 172 elif not os.path.exists(target_path) or os.lstat(target_path).st_size <= 0:

174 # Apply BackendApi settings if needed

175 if BackendApi.working and image.startswith(BackendApi.url):

176 request_headers = BackendApi.headers if headers is None else headers

177 request_ssl = BackendApi.ssl

178 else:

179 request_headers = headers

180 request_ssl = ssl

181 173 # Use aiohttp to fetch the image

182 async with session.get(image, ssl=request_ssl, headers=request_headers) as response:

174 async with session.get(image, ssl=ssl) as response:

183 175 response.raise_for_status()

184 176 if target is None:

185 177 filename = update_filename(response, filename)

Modified g4f/providers/base_provider.py +1 -1

@@ -21,7 +21,7 @@ from .response import BaseConversation, AuthResult

21 21 from .helper import concat_chunks

22 22 from ..cookies import get_cookies_dir

23 23

from ..errors import ModelNotFoundError, ResponseError, MissingAuthError, NoValidHarFileError, PaymentRequiredError, CloudflareError

24 from ..tools.run_tools import AuthManager

24 from ..tools.auth import AuthManager

25 25 from .. import debug

26 26

27 27 SAFE_PARAMETERS = [

Modified g4f/providers/response.py +6 -3

@@ -231,10 +231,13 @@ class DebugResponse(HiddenResponse):

231 231 """Initialize with a log message."""

232 232 self.log = log

233 233

234 class PlainTextResponse(HiddenResponse):

235 def __init__(self, text: str) -> None:

236 self.text = text

237

234 238 class ContinueResponse(HiddenResponse):

235 def __init__(self, log: str) -> None:

236 """Initialize with a log message."""

237 self.log = log

239 def __init__(self, text: str) -> None:

240 self.text = text

238 241

239 242 class Reasoning(ResponseType):

240 243 def __init__(

Added g4f/tools/auth.py +32 -0

@@ -0,0 +1,32 @@

1 from __future__ import annotations

3 import os

4 from typing import Optional

6 from ..providers.types import ProviderType

7 from .. import debug

9 class AuthManager:

10 """Handles API key management"""

11 aliases = {

12 "GeminiPro": "Gemini",

13 "PollinationsAI": "Pollinations",

14 "OpenaiAPI": "Openai",

15 "PuterJS": "Puter",

16 }

18 @classmethod

19 def load_api_key(cls, provider: ProviderType) -> Optional[str]:

20 """Load API key from config file"""

21 if not provider.needs_auth and not hasattr(provider, "login_url"):

22 return None

23 provider_name = provider.get_parent()

24 env_var = f"{provider_name.upper()}_API_KEY"

25 api_key = os.environ.get(env_var)

26 if not api_key and provider_name in cls.aliases:

27 env_var = f"{cls.aliases[provider_name].upper()}_API_KEY"

28 api_key = os.environ.get(env_var)

29 if api_key:

30 debug.log(f"Loading API key for {provider_name} from environment variable {env_var}")

31 return api_key

32 return None

Added g4f/tools/fetch_and_scrape.py +98 -0

@@ -0,0 +1,98 @@

1 from __future__ import annotations

3 import hashlib

4 import asyncio

5 from pathlib import Path

6 from typing import Iterator, Optional

7 from urllib.parse import urlparse, quote_plus

8 from aiohttp import ClientSession, ClientError

9 from datetime import date

10 import asyncio

12 try:

13 from bs4 import BeautifulSoup

14 has_requirements = True

15 except ImportError:

16 has_requirements = False

18 from ..cookies import get_cookies_dir

19 from ..providers.response import format_link

def scrape_text(html: str, max_words: Optional[int] = None, add_source: bool = True, count_images: int = 2) -> Iterator[str]:

22 """

23 Parses the provided HTML and yields text fragments.

24 """

25 soup = BeautifulSoup(html, "html.parser")

26 for selector in [

27 "main", ".main-content-wrapper", ".main-content", ".emt-container-inner",

28 ".content-wrapper", "#content", "#mainContent",

29 ]:

30 selected = soup.select_one(selector)

31 if selected:

32 soup = selected

33 break

35 for remove_selector in [".c-globalDisclosure"]:

36 unwanted = soup.select_one(remove_selector)

37 if unwanted:

38 unwanted.extract()

40 image_selector = "img[alt][src^=http]:not([alt='']):not(.avatar):not([width])"

41 image_link_selector = f"a:has({image_selector})"

42 seen_texts = []

    for element in soup.select(f"h1, h2, h3, h4, h5, h6, p, pre, table:not(:has(p)), ul:not(:has(p)), {image_link_selector}"):

45 if count_images > 0:

46 image = element.select_one(image_selector)

47 if image:

48 title = str(element.get("title", element.text))

49 if title:

50 yield f"!{format_link(image['src'], title)}\n"

51 if max_words is not None:

52 max_words -= 10

53 count_images -= 1

54 continue

56 for line in element.get_text(" ").splitlines():

57 words = [word for word in line.split() if word]

58 if not words:

59 continue

60 joined_line = " ".join(words)

61 if joined_line in seen_texts:

62 continue

63 if max_words is not None:

64 max_words -= len(words)

65 if max_words <= 0:

66 break

67 yield joined_line + "\n"

68 seen_texts.append(joined_line)

70 if add_source:

71 canonical_link = soup.find("link", rel="canonical")

72 if canonical_link and "href" in canonical_link.attrs:

73 link = canonical_link["href"]

74 domain = urlparse(link).netloc

75 yield f"\nSource: [{domain}]({link})"

async def fetch_and_scrape(session: ClientSession, url: str, max_words: Optional[int] = None, add_source: bool = False, proxy: str = None) -> str:

78 """

79 Fetches a URL and returns the scraped text, using caching to avoid redundant downloads.

80 """

81 try:

82 cache_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "fetch_and_scrape"

83 cache_dir.mkdir(parents=True, exist_ok=True)

84 md5_hash = hashlib.md5(url.encode(errors="ignore")).hexdigest()

        cache_file = cache_dir / f"{quote_plus(url.split('?')[0].split('//')[1].replace('/', ' ')[:48])}.{date.today()}.{md5_hash[:16]}.cache"

86 if cache_file.exists():

87 return cache_file.read_text()

89 async with session.get(url, proxy=proxy) as response:

90 if response.status == 200:

91 html = await response.text(errors="replace")

92 scraped_text = "".join(scrape_text(html, max_words, add_source))

93 with open(cache_file, "wb") as f:

94 f.write(scraped_text.encode(errors="replace"))

95 return scraped_text

96 except (ClientError, asyncio.TimeoutError):

97 return ""

98 return ""

Modified g4f/tools/files.py +1 -1

Modified g4f/tools/run_tools.py +1 -25

Modified g4f/tools/web_search.py +15 -267

Modified requirements.txt +1 -0

Modified setup.py +2 -1