gpt4free

公开

关注 0 Fork 0 Star 0

XFEstudio/gpt4free

Add speech synthesize from Gemini (#2404)

* Improve slim docker image example, clean up OpenaiChat provider * Enhance event loop management for asynchronous generators * Fix attribute " shutdown_default_executor" not found in old python versions * asyncio file created with all async helpers * Add speech synthesize from Gemini. You can use it without a account

e8bd24a2

H Lohaus <hlohaus@users.noreply.github.com>

提交于 2024-11-22 01:50

父提交 e4bfd9d

代码差异

9 个文件 +189 -49

Modified g4f/Provider/needs_auth/Gemini.py +65 -1

@@ -4,8 +4,10 @@ import os

4 4 import json

5 5 import random

6 6 import re

7 import base64

7 8

8 9 from aiohttp import ClientSession, BaseConnector

9 11 try:

10 12 import nodriver

11 13 has_nodriver = True

@@ -14,12 +16,13 @@ except ImportError:

14 16

15 17 from ... import debug

16 18 from ...typing import Messages, Cookies, ImageType, AsyncResult, AsyncIterator

17 from ..base_provider import AsyncGeneratorProvider, BaseConversation

19 from ..base_provider import AsyncGeneratorProvider, BaseConversation, SynthesizeData

18 20 from ..helper import format_prompt, get_cookies

19 21 from ...requests.raise_for_status import raise_for_status

20 22 from ...requests.aiohttp import get_connector

21 23 from ...errors import MissingAuthError

22 24 from ...image import ImageResponse, to_bytes

25 from ... import debug

23 26

24 27 REQUEST_HEADERS = {

25 28 "authority": "gemini.google.com",

@@ -54,6 +57,7 @@ class Gemini(AsyncGeneratorProvider):

54 57 image_models = ["gemini"]

55 58 default_vision_model = "gemini"

56 59 models = ["gemini", "gemini-1.5-flash", "gemini-1.5-pro"]

60 synthesize_content_type = "audio/vnd.wav"

57 61 _cookies: Cookies = None

58 62 _snlm0e: str = None

59 63 _sid: str = None

@@ -106,6 +110,7 @@ class Gemini(AsyncGeneratorProvider):

106 110 prompt = format_prompt(messages) if conversation is None else messages[-1]["content"]

107 111 cls._cookies = cookies or cls._cookies or get_cookies(".google.com", False, True)

108 112 base_connector = get_connector(connector, proxy)

113

109 114 async with ClientSession(

110 115 headers=REQUEST_HEADERS,

111 116 connector=base_connector

@@ -122,6 +127,7 @@ class Gemini(AsyncGeneratorProvider):

122 127 if not cls._snlm0e:

123 128 raise RuntimeError("Invalid cookies. SNlM0e not found")

124 129

130 yield SynthesizeData(cls.__name__, {"text": messages[-1]["content"]})

125 131 image_url = await cls.upload_image(base_connector, to_bytes(image), image_name) if image else None

126 132

127 133 async with ClientSession(

@@ -198,6 +204,40 @@ class Gemini(AsyncGeneratorProvider):

198 204 except TypeError:

199 205 pass

200 206

207 @classmethod

208 async def synthesize(cls, params: dict, proxy: str = None) -> AsyncIterator[bytes]:

209 async with ClientSession(

210 cookies=cls._cookies,

211 headers=REQUEST_HEADERS,

212 connector=get_connector(proxy=proxy),

213 ) as session:

214 if not cls._snlm0e:

215 await cls.fetch_snlm0e(session, cls._cookies) if cls._cookies else None

216 if not cls._snlm0e:

217 async for chunk in cls.nodriver_login(proxy):

218 debug.log(chunk)

219 inner_data = json.dumps([None, params["text"], "de-DE", None, 2])

220 async with session.post(

221 "https://gemini.google.com/_/BardChatUi/data/batchexecute",

222 data={

223 "f.req": json.dumps([[["XqA3Ic", inner_data, None, "generic"]]]),

224 "at": cls._snlm0e,

225 },

226 params={

227 "rpcids": "XqA3Ic",

228 "source-path": "/app/2704fb4aafcca926",

229 "bl": "boq_assistant-bard-web-server_20241119.00_p1",

230 "f.sid": "" if cls._sid is None else cls._sid,

231 "hl": "de",

232 "_reqid": random.randint(1111, 9999),

233 "rt": "c"

234 },

235 ) as response:

236 await raise_for_status(response)

237 iter_base64_response = iter_filter_base64(response.content.iter_chunked(1024))

238 async for chunk in iter_base64_decode(iter_base64_response):

239 yield chunk

240

201 241 def build_request(

202 242 prompt: str,

203 243 language: str,

@@ -280,3 +320,27 @@ class Conversation(BaseConversation):

280 320 self.conversation_id = conversation_id

281 321 self.response_id = response_id

282 322 self.choice_id = choice_id

323 async def iter_filter_base64(response_iter: AsyncIterator[bytes]) -> AsyncIterator[bytes]:

324 search_for = b'[["wrb.fr","XqA3Ic","[\\"'

325 end_with = b'\\'

326 is_started = False

327 async for chunk in response_iter:

328 if is_started:

329 if end_with in chunk:

330 yield chunk.split(end_with, 1).pop(0)

331 break

332 else:

333 yield chunk

334 elif search_for in chunk:

335 is_started = True

336 yield chunk.split(search_for, 1).pop()

337 else:

338 raise RuntimeError(f"Response: {chunk}")

339

340 async def iter_base64_decode(response_iter: AsyncIterator[bytes]) -> AsyncIterator[bytes]:

341 buffer = b""

342 async for chunk in response_iter:

343 chunk = buffer + chunk

344 rest = len(chunk) % 4

345 buffer = chunk[-rest:]

346 yield base64.b64decode(chunk[:-rest])

Modified g4f/Provider/needs_auth/OpenaiChat.py +1 -0

@@ -61,6 +61,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

61 61 fallback_models = [default_model, "gpt-4", "gpt-4o", "gpt-4o-mini", "gpt-4o-canmore", "o1-preview", "o1-mini"]

62 62 vision_models = fallback_models

63 63 image_models = fallback_models

64 synthesize_content_type = "audio/mpeg"

64 65

65 66 _api_key: str = None

66 67 _headers: dict = None

Modified g4f/api/__init__.py +38 -1

@@ -26,6 +26,7 @@ from g4f.client.helper import filter_none

26 26 from g4f.image import is_accepted_format, images_dir

27 27 from g4f.typing import Messages

28 28 from g4f.cookies import read_cookie_files

29 from g4f.Provider import ProviderType, ProviderUtils, __providers__

29 30

30 31 logger = logging.getLogger(__name__)

31 32

@@ -152,7 +153,9 @@ class Api:

152 153 return HTMLResponse('g4f API: Go to '

153 154 '<a href="/v1/models">models</a>, '

154 155 '<a href="/v1/chat/completions">chat/completions</a>, or '

155 '<a href="/v1/images/generate">images/generate</a>.')

156 '<a href="/v1/images/generate">images/generate</a> <br><br>'

157 'Open Swagger UI at: '

158 '<a href="/docs">/docs</a>')

156 159

157 160 @self.app.get("/v1/models")

158 161 async def models():

@@ -290,6 +293,40 @@ class Api:

290 293

291 294 return FileResponse(target, media_type=content_type)

292 295

296 @self.app.get("/providers")

297 async def providers():

298 model_list = [{

299 'id': provider.__name__,

300 'object': 'provider',

301 'created': 0,

302 'url': provider.url,

303 'label': getattr(provider, "label", None),

304 } for provider in __providers__ if provider.working]

305 return JSONResponse(model_list)

306

307 @self.app.get("/providers/{provider}")

308 async def providers_info(provider: str):

309 if provider not in ProviderUtils.convert:

310 return JSONResponse({"error": "The model does not exist."}, 404)

311 provider: ProviderType = ProviderUtils.convert[provider]

312 def safe_get_models(provider: ProviderType) -> list[str]:

313 try:

314 return provider.get_models() if hasattr(provider, "get_models") else []

315 except:

316 return []

317 provider_info = {

318 'id': provider.__name__,

319 'object': 'provider',

320 'created': 0,

321 'url': provider.url,

322 'label': getattr(provider, "label", None),

323 'models': safe_get_models(provider),

324 'image_models': getattr(provider, "image_models", []) or [],

325 'vision_models': [model for model in [getattr(provider, "default_vision_model", None)] if model],

326 'params': [*provider.get_parameters()] if hasattr(provider, "get_parameters") else []

327 }

328 return JSONResponse(provider_info)

329

293 330

def format_exception(e: Exception, config: Union[ChatCompletionsConfig, ImageGenerationConfig], image: bool = False) -> str:

294 331 last_provider = {} if not image else g4f.get_last_provider(True)

295 332

    provider = (AppConfig.image_provider if image else AppConfig.provider) if config.provider is None else config.provider

Modified g4f/gui/client/index.html +3 -0

@@ -191,6 +191,9 @@

191 191 <button class="slide-systemPrompt">

192 192 <i class="fa-solid fa-angles-up"></i>

193 193 </button>

194 <div class="media_player">

195 <i class="fa-regular fa-x"></i>

196 </div>

194 197 <div class="toolbar">

195 198 <div id="input-count" class="">

196 199 <button class="hide-input">

Modified g4f/gui/client/static/css/style.css +16 -3

@@ -434,15 +434,28 @@ body {

434 434 font-size: 12px;

435 435 }

436 436

437 .message audio {

437 .media_player {

438 438 display: none;

439 max-width: 400px;

440 439 }

441 440

442 .message audio.show {

441 .media_player audio {

442 right: 28px;

443 position: absolute;

444 top: -4px;

445 z-index: 900;

446 }

447

448 .media_player.show {

443 449 display: block;

444 450 }

445 451

452 .media_player .fa-x {

453 position: absolute;

454 right: 8px;

455 top: 8px;

456 z-index: 1000;

457 }

458

446 459 .count_total {

447 460 font-size: 12px;

448 461 padding-left: 25px;

Modified g4f/gui/client/static/js/chat.v1.js +42 -24

@@ -44,17 +44,18 @@ appStorage = window.localStorage || {

44 44 removeItem: (key) => delete self[key],

45 45 length: 0

46 46 }

48 const markdown = window.markdownit();

49 const markdown_render = (content) => {

50 return markdown.render(content

51 .replaceAll(/|/gm, "")

52 .replaceAll(/<img data-prompt="[^>]+">/gm, "")

53 )

54 .replaceAll("<a href=", '<a target="_blank" href=')

55 .replaceAll('<code>', '<code class="language-plaintext">')

47 let markdown_render = () => null;

48 if (window.markdownit) {

49 const markdown = window.markdownit();

50 markdown_render = (content) => {

51 return markdown.render(content

52 .replaceAll(/|/gm, "")

53 .replaceAll(/<img data-prompt="[^>]+">/gm, "")

54 )

55 .replaceAll("<a href=", '<a target="_blank" href=')

56 .replaceAll('<code>', '<code class="language-plaintext">')

57 }

56 58 }

58 59 function filter_message(text) {

59 60 return text.replaceAll(

60 61 /[\s\S]+/gm, ""

@@ -135,10 +136,21 @@ const register_message_buttons = async () => {

135 136 if (!("click" in el.dataset)) {

136 137 el.dataset.click = "true";

137 138 el.addEventListener("click", async () => {

138 const content_el = el.parentElement.parentElement;

139 const audio = content_el.querySelector("audio");

140 if (audio) {

141 audio.classList.add("show");

139 const message_el = el.parentElement.parentElement.parentElement;

140 let audio;

141 if (message_el.dataset.synthesize_url) {

142 el.classList.add("active");

143 setTimeout(()=>el.classList.remove("active"), 2000);

144 const media_player = document.querySelector(".media_player");

145 if (!media_player.classList.contains("show")) {

146 media_player.classList.add("show");

147 audio = new Audio(message_el.dataset.synthesize_url);

148 audio.controls = true;

149 media_player.appendChild(audio);

150 } else {

151 audio = media_player.querySelector("audio");

152 audio.src = message_el.dataset.synthesize_url;

153 }

142 154 audio.play();

143 155 return;

144 156 }

@@ -163,7 +175,7 @@ const register_message_buttons = async () => {

163 175 el.dataset.running = true;

164 176 el.classList.add("blink")

165 177 el.classList.add("active")

166 const message_el = content_el.parentElement;

178

167 179 let speechText = await get_message(window.conversation_id, message_el.dataset.index);

168 180

169 181 speechText = speechText.replaceAll(/([^0-9])\./gm, "$1.;");

@@ -351,6 +363,13 @@ stop_generating.addEventListener("click", async () => {

351 363 await load_conversation(window.conversation_id, false);

352 364 });

353 365

366 document.querySelector(".media_player .fa-x").addEventListener("click", ()=>{

367 const media_player = document.querySelector(".media_player");

368 media_player.classList.remove("show");

369 const audio = document.querySelector(".media_player audio");

370 media_player.removeChild(audio);

371 });

372

354 373 const prepare_messages = (messages, message_index = -1) => {

355 374 if (message_index >= 0) {

356 375 messages = messages.filter((_, index) => message_index >= index);

@@ -726,17 +745,17 @@ const load_conversation = async (conversation_id, scroll=true) => {

726 745 ${item.provider.model ? ' with ' + item.provider.model : ''}

727 746 </div>

728 747 ` : "";

729 let audio = "";

748 let synthesize_params = {text: item.content}

749 let synthesize_provider = "Gemini";

730 750 if (item.synthesize) {

731 const synthesize_params = (new URLSearchParams(item.synthesize.data)).toString();

732 audio = `

733 <audio controls preload="none">

734

                    <source src="/backend-api/v2/synthesize/${item.synthesize.provider}?${synthesize_params}" type="audio/mpeg">

735 </audio>

736 `;

751 synthesize_params = item.synthesize.data

752 synthesize_provider = item.synthesize.provider;

737 753 }

754 synthesize_params = (new URLSearchParams(synthesize_params)).toString();

755 let synthesize_url = `/backend-api/v2/synthesize/${synthesize_provider}?${synthesize_params}`;

756

738 757 elements += `

739 <div class="message${item.regenerate ? " regenerate": ""}" data-index="${i}">

758

            <div class="message${item.regenerate ? " regenerate": ""}" data-index="${i}" data-synthesize_url="${synthesize_url}">

740 759 <div class="${item.role}">

741 760 ${item.role == "assistant" ? gpt_image : user_image}

742 761 <i class="fa-solid fa-xmark"></i>

@@ -748,7 +767,6 @@ const load_conversation = async (conversation_id, scroll=true) => {

748 767 <div class="content">

749 768 ${provider}

750 769 <div class="content_inner">${markdown_render(item.content)}</div>

751 ${audio}

752 770 <div class="count">

753 771 ${count_words_and_tokens(item.content, next_provider?.model)}

754 772 <i class="fa-solid fa-volume-high"></i>

Modified g4f/gui/server/api.py +6 -7

@@ -140,13 +140,12 @@ class Api:

140 140 }

141 141

142 142

    def _create_response_stream(self, kwargs: dict, conversation_id: str, provider: str, download_images: bool = True) -> Iterator:

143 if debug.logging:

144 debug.logs = []

145 print_callback = debug.log_handler

146 def log_handler(text: str):

147 debug.logs.append(text)

148 print_callback(text)

149 debug.log_handler = log_handler

143 debug.logs = []

144 print_callback = debug.log_handler

145 def log_handler(text: str):

146 debug.logs.append(text)

147 print_callback(text)

148 debug.log_handler = log_handler

150 149 try:

151 150 result = ChatCompletion.create(**kwargs)

152 151 first = True

Modified g4f/gui/server/backend.py +15 -9

@@ -1,6 +1,8 @@

1 1 import json

2 2 import flask

3 3 import os

4 import logging

5 import asyncio

4 6 from flask import request, Flask

5 7 from typing import Generator

6 8 from werkzeug.utils import secure_filename

@@ -12,6 +14,8 @@ from g4f.errors import ProviderNotFoundError

12 14 from g4f.cookies import get_cookies_dir

13 15 from .api import Api

14 16

17 logger = logging.getLogger(__name__)

15 19 def safe_iter_generator(generator: Generator) -> Generator:

16 20 start = next(generator)

17 21 def iter_generator():

@@ -127,15 +131,17 @@ class Backend_Api(Api):

127 131 return "Provider not found", 404

128 132 if not hasattr(provider_handler, "synthesize"):

129 133 return "Provider doesn't support synthesize", 500

130 try:

131 response_generator = provider_handler.synthesize({**request.args})

132 if hasattr(response_generator, "__aiter__"):

133 response_generator = to_sync_generator(response_generator)

134 response = flask.Response(safe_iter_generator(response_generator), content_type="audio/mpeg")

135 response.headers['Cache-Control'] = "max-age=604800"

136 return response

137 except Exception as e:

138 return f"{e.__class__.__name__}: {e}", 500

134 response_data = provider_handler.synthesize({**request.args})

135 if asyncio.iscoroutinefunction(provider_handler.synthesize):

136 response_data = asyncio.run(response_data)

137 else:

138 if hasattr(response_data, "__aiter__"):

139 response_data = to_sync_generator(response_data)

140 response_data = safe_iter_generator(response_data)

141 content_type = getattr(provider_handler, "synthesize_content_type", "application/octet-stream")

142 response = flask.Response(response_data, content_type=content_type)

143 response.headers['Cache-Control'] = "max-age=604800"

144 return response

139 145

140 146 def get_provider_models(self, provider: str):

141 147 api_key = None if request.authorization is None else request.authorization.token

Modified g4f/providers/base_provider.py +3 -4

@@ -66,11 +66,12 @@ class AbstractProvider(BaseProvider):

66 66

67 67 @classmethod

68 68 def get_parameters(cls) -> dict[str, Parameter]:

69 return signature(

69 return {name: parameter for name, parameter in signature(

70 70 cls.create_async_generator if issubclass(cls, AsyncGeneratorProvider) else

71 71 cls.create_async if issubclass(cls, AsyncProvider) else

72 72 cls.create_completion

73 ).parameters

73 ).parameters.items() if name not in ["kwargs", "model", "messages"]

74 and (name != "stream" or cls.supports_stream)}

74 75

75 76 @classmethod

76 77 @property

@@ -90,8 +91,6 @@ class AbstractProvider(BaseProvider):

90 91

91 92 args = ""

92 93 for name, param in cls.get_parameters().items():

93 if name in ("self", "kwargs") or (name == "stream" and not cls.supports_stream):

94 continue

95 94 args += f"\n {name}"

96 95 args += f": {get_type_name(param.annotation)}" if param.annotation is not Parameter.empty else ""

97 96 default_value = f'"{param.default}"' if isinstance(param.default, str) else param.default