gpt4free

公开

关注 0 Fork 0 Star 1

XFEstudio/gpt4free

fix: Update deprecated DuckDuckGo search backend from 'api' to 'auto' (#2516)

* fix: Update deprecated DuckDuckGo search backend from 'api' to 'auto' Fixes UserWarning: 'api' backend is deprecated, using backend='auto' - Updated default backend parameter from 'api' to 'auto' in search function - Aligns with latest duckduckgo-search library recommendations * Update g4f/Provider/Blackbox.py * Fix response_format=b64_json (g4f/client/__init__.py) * Update g4f/Provider/Blackbox2.py g4f/Provider/BlackboxCreateAgent.py --------- Co-authored-by: kqlio67 <>

90360ccf

kqlio67 <166700875+kqlio67@users.noreply.github.com>

提交于 2024-12-30 02:04

父提交 9a5c337

代码差异

5 个文件 +35 -217

Modified g4f/Provider/Blackbox.py +18 -8

@@ -8,6 +8,7 @@ import re

8 8 import aiohttp

9 9 import asyncio

10 10 from pathlib import Path

11 import concurrent.futures

11 12

12 13 from ..typing import AsyncResult, Messages, ImagesType

13 14 from .base_provider import AsyncGeneratorProvider, ProviderModelMixin

@@ -220,15 +221,24 @@ class Blackbox(AsyncGeneratorProvider, ProviderModelMixin):

220 221 use_internal_search = web_search and model in cls.web_search_models

221 222

222 223 if web_search and not use_internal_search:

223

224 def run_search():

225 return get_search_message(messages[-1]["content"])

224 try:

225 # Create a timeout for web search

226 async def run_search():

227 with concurrent.futures.ThreadPoolExecutor() as executor:

228 return await asyncio.get_event_loop().run_in_executor(

229 executor,

230 lambda: get_search_message(messages[-1]["content"])

231 )

232

233 # Set a timeout of 10 seconds for web search

234 search_result = await asyncio.wait_for(run_search(), timeout=10.0)

235 messages[-1]["content"] = search_result

226 236

227 import concurrent.futures

228 with concurrent.futures.ThreadPoolExecutor() as executor:

229 messages[-1]["content"] = await asyncio.get_event_loop().run_in_executor(

230 executor, run_search

231 )

237 except asyncio.TimeoutError:

238 debug.log("Web search timed out, proceeding with original message")

239 except Exception as e:

240 debug.log(f"Web search failed: {str(e)}, proceeding with original message")

241

232 242 web_search = False

233 243

234 244 async def process_request():

Deleted g4f/Provider/Blackbox2.py +0 -197

@@ -1,197 +0,0 @@

1 from __future__ import annotations

3 import random

4 import asyncio

5 import re

6 import json

7 from pathlib import Path

8 from aiohttp import ClientSession

9 from typing import AsyncIterator

11 from ..typing import AsyncResult, Messages

12 from ..image import ImageResponse

13 from .base_provider import AsyncGeneratorProvider, ProviderModelMixin

14 from ..cookies import get_cookies_dir

16 from .. import debug

18 class Blackbox2(AsyncGeneratorProvider, ProviderModelMixin):

19 url = "https://www.blackbox.ai"

20 api_endpoints = {

21 "llama-3.1-70b": "https://www.blackbox.ai/api/improve-prompt",

22 "flux": "https://www.blackbox.ai/api/image-generator"

23 }

25 working = True

26 supports_system_message = True

27 supports_message_history = True

28 supports_stream = False

30 default_model = 'llama-3.1-70b'

31 chat_models = ['llama-3.1-70b']

32 image_models = ['flux']

33 models = [*chat_models, *image_models]

35 @classmethod

36 def _get_cache_file(cls) -> Path:

37 """Returns the path to the cache file."""

38 dir = Path(get_cookies_dir())

39 dir.mkdir(exist_ok=True)

40 return dir / 'blackbox2.json'

42 @classmethod

43 def _load_cached_license(cls) -> str | None:

44 """Loads the license key from the cache."""

45 cache_file = cls._get_cache_file()

46 if cache_file.exists():

47 try:

48 with open(cache_file, 'r') as f:

49 data = json.load(f)

50 return data.get('license_key')

51 except Exception as e:

52 debug.log(f"Error reading cache file: {e}")

53 return None

55 @classmethod

56 def _save_cached_license(cls, license_key: str):

57 """Saves the license key to the cache."""

58 cache_file = cls._get_cache_file()

59 try:

60 with open(cache_file, 'w') as f:

61 json.dump({'license_key': license_key}, f)

62 except Exception as e:

63 debug.log(f"Error writing to cache file: {e}")

65 @classmethod

66 async def _get_license_key(cls, session: ClientSession) -> str:

67 cached_license = cls._load_cached_license()

68 if cached_license:

69 return cached_license

71 try:

72 async with session.get(cls.url) as response:

73 html = await response.text()

74 js_files = re.findall(r'static/chunks/\d{4}-[a-fA-F0-9]+\.js', html)

76 license_format = r'["\'](\d{6}-\d{6}-\d{6}-\d{6}-\d{6})["\']'

78 def is_valid_context(text_around):

79 return any(char + '=' in text_around for char in 'abcdefghijklmnopqrstuvwxyz')

81 for js_file in js_files:

82 js_url = f"{cls.url}/_next/{js_file}"

83 async with session.get(js_url) as js_response:

84 js_content = await js_response.text()

85 for match in re.finditer(license_format, js_content):

86 start = max(0, match.start() - 10)

87 end = min(len(js_content), match.end() + 10)

88 context = js_content[start:end]

90 if is_valid_context(context):

91 license_key = match.group(1)

92 cls._save_cached_license(license_key)

93 return license_key

95 raise ValueError("License key not found")

96 except Exception as e:

97 debug.log(f"Error getting license key: {str(e)}")

98 raise

100 @classmethod

101 async def create_async_generator(

102 cls,

103 model: str,

104 messages: Messages,

105 prompt: str = None,

106 proxy: str = None,

107 max_retries: int = 3,

108 delay: int = 1,

109 max_tokens: int = None,

110 **kwargs

111 ) -> AsyncResult:

112 if not model:

113 model = cls.default_model

114

115 if model in cls.chat_models:

116 async for result in cls._generate_text(model, messages, proxy, max_retries, delay, max_tokens):

117 yield result

118 elif model in cls.image_models:

119 prompt = messages[-1]["content"]

120 async for result in cls._generate_image(model, prompt, proxy):

121 yield result

122 else:

123 raise ValueError(f"Unsupported model: {model}")

124

125 @classmethod

126 async def _generate_text(

127 cls,

128 model: str,

129 messages: Messages,

130 proxy: str = None,

131 max_retries: int = 3,

132 delay: int = 1,

133 max_tokens: int = None,

134 ) -> AsyncIterator[str]:

135 headers = cls._get_headers()

136

137 async with ClientSession(headers=headers) as session:

138 license_key = await cls._get_license_key(session)

139 api_endpoint = cls.api_endpoints[model]

140

141 data = {

142 "messages": messages,

143 "max_tokens": max_tokens,

144 "validated": license_key

145 }

146

147 for attempt in range(max_retries):

148 try:

149 async with session.post(api_endpoint, json=data, proxy=proxy) as response:

150 response.raise_for_status()

151 response_data = await response.json()

152 if 'prompt' in response_data:

153 yield response_data['prompt']

154 return

155 else:

156 raise KeyError("'prompt' key not found in the response")

157 except Exception as e:

158 if attempt == max_retries - 1:

159 raise RuntimeError(f"Error after {max_retries} attempts: {str(e)}")

160 else:

161 wait_time = delay * (2 ** attempt) + random.uniform(0, 1)

162 debug.log(f"Attempt {attempt + 1} failed. Retrying in {wait_time:.2f} seconds...")

163 await asyncio.sleep(wait_time)

164

165 @classmethod

166 async def _generate_image(

167 cls,

168 model: str,

169 prompt: str,

170 proxy: str = None

171 ) -> AsyncIterator[ImageResponse]:

172 headers = cls._get_headers()

173 api_endpoint = cls.api_endpoints[model]

174

175 async with ClientSession(headers=headers) as session:

176 data = {

177 "query": prompt

178 }

179

180 async with session.post(api_endpoint, headers=headers, json=data, proxy=proxy) as response:

181 response.raise_for_status()

182 response_data = await response.json()

183

184 if 'markdown' in response_data:

185 image_url = response_data['markdown'].split('(')[1].split(')')[0]

186 yield ImageResponse(images=image_url, alt=prompt)

187

188 @staticmethod

189 def _get_headers() -> dict:

190 return {

191 'accept': '*/*',

192 'accept-language': 'en-US,en;q=0.9',

193 'content-type': 'text/plain;charset=UTF-8',

194 'origin': 'https://www.blackbox.ai',

195 'referer': 'https://www.blackbox.ai',

196

            'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'

197 }

Modified g4f/Provider/BlackboxCreateAgent.py +1 -1

@@ -37,7 +37,7 @@ class BlackboxCreateAgent(AsyncGeneratorProvider, ProviderModelMixin):

37 37 """Returns the path to the cache file."""

38 38 dir = Path(get_cookies_dir())

39 39 dir.mkdir(exist_ok=True)

40 return dir / 'blackbox2.json'

40 return dir / 'blackbox_create_agent.json'

41 41

42 42 @classmethod

43 43 def _load_cached_value(cls) -> str | None:

Modified g4f/client/__init__.py +13 -8

@@ -5,6 +5,7 @@ import time

5 5 import random

6 6 import string

7 7 import asyncio

8 import aiohttp

8 9 import base64

9 10 import json

10 11 from typing import Union, AsyncIterator, Iterator, Coroutine, Optional

@@ -486,17 +487,21 @@ class Images:

486 487 if response_format == "url":

487 488 # Return original URLs without saving locally

488 489 images = [Image.model_construct(url=image, revised_prompt=response.alt) for image in response.get_list()]

490 elif response_format == "b64_json":

491 # Convert URLs directly to base64 without saving

492 async def get_b64_from_url(url: str) -> Image:

493 async with aiohttp.ClientSession() as session:

494 async with session.get(url, proxy=proxy) as resp:

495 if resp.status == 200:

496 image_data = await resp.read()

497 b64_data = base64.b64encode(image_data).decode()

498 return Image.model_construct(b64_json=b64_data, revised_prompt=response.alt)

499 images = await asyncio.gather(*[get_b64_from_url(image) for image in response.get_list()])

489 500 else:

490 501 # Save locally for None (default) case

491 502 images = await copy_images(response.get_list(), response.get("cookies"), proxy)

492 if response_format == "b64_json":

493 async def process_image_item(image_file: str) -> Image:

494 with open(os.path.join(images_dir, os.path.basename(image_file)), "rb") as file:

495 image_data = base64.b64encode(file.read()).decode()

496 return Image.model_construct(b64_json=image_data, revised_prompt=response.alt)

497 images = await asyncio.gather(*[process_image_item(image) for image in images])

498 else:

499

                images = [Image.model_construct(url=f"/images/{os.path.basename(image)}", revised_prompt=response.alt) for image in images]

503

            images = [Image.model_construct(url=f"/images/{os.path.basename(image)}", revised_prompt=response.alt) for image in images]

504

500 505 return ImagesResponse.model_construct(

501 506 created=int(time.time()),

502 507 data=images,

Modified g4f/web_search.py +3 -3

@@ -102,7 +102,7 @@ async def fetch_and_scrape(session: ClientSession, url: str, max_words: int = No

102 102 except ClientError:

103 103 return

104 104

105

async def search(query: str, max_results: int = 5, max_words: int = 2500, backend: str = "api", add_text: bool = True, timeout: int = 5, region: str = "wt-wt") -> SearchResults:

105

async def search(query: str, max_results: int = 5, max_words: int = 2500, backend: str = "auto", add_text: bool = True, timeout: int = 5, region: str = "wt-wt") -> SearchResults:

106 106 if not has_requirements:

107 107

        raise MissingRequirementsError('Install "duckduckgo-search" and "beautifulsoup4" package | pip install -U g4f[search]')

108 108 with DDGS() as ddgs:

@@ -113,7 +113,7 @@ async def search(query: str, max_results: int = 5, max_words: int = 2500, backen

113 113 safesearch="moderate",

114 114 timelimit="y",

115 115 max_results=max_results,

116 backend=backend,

116 backend=backend, # Changed from 'api' to 'auto'

117 117 ):

118 118 results.append(SearchResultEntry(

119 119 result["title"],

@@ -169,4 +169,4 @@ def get_search_message(prompt: str, raise_search_exceptions=False, **kwargs) ->

169 169 if raise_search_exceptions:

170 170 raise e

171 171 debug.log(f"Couldn't do web search: {e.__class__.__name__}: {e}")

172 return prompt