gpt4free

公开

关注 0 Fork 0 Star 1

XFEstudio/gpt4free

feat: add audio speech generation endpoint and media handling refactor

- Added new `/v1/audio/speech` and `/api/{path_provider}/audio/speech` endpoints in `g4f/api/__init__.py` for generating speech from text - Introduced `AudioSpeechConfig` model in `g4f/api/stubs.py` with fields for input, model, provider, voice, instructions, and response format - Updated `PollinationsAI.py` to support `modalities` in `kwargs` when checking for audio - Set default voice for audio models in `PollinationsAI.py` if not provided in `kwargs` - Added debug print in `PollinationsAI.py` to log request data to text API endpoint - Extended supported FastAPI response types in `g4f/api/__init__.py` to include `FileResponse` from `starlette.responses` - Added `BackgroundTask` to clean up generated audio files after serving in `g4f/api/__init__.py` - Modified `AnyProvider.py` to include `EdgeTTS`, `gTTS`, and `MarkItDown` as audio providers when `audio` is in `kwargs` or `modalities` - Created `resolve_media` helper in `g4f/client/__init__.py` to standardize media handling for audio/image input - Replaced manual media preprocessing in `Completions`, `AsyncCompletions`, and `Images` classes with `resolve_media` - Added `/docs/README.md` with a link to the documentation site

c3632984

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-04-26 12:21

父提交 b15a83a

代码差异

6 个文件 +90 -29

Added docs/README.md +1 -0

@@ -0,0 +1 @@

1 [Documentation](https://gpt4free.github.io/docs/main.html)

Modified g4f/Provider/PollinationsAI.py +4 -1

@@ -177,7 +177,7 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

177 177 # Load model list

178 178 cls.get_models()

179 179 if not model:

180 has_audio = "audio" in kwargs

180 has_audio = "audio" in kwargs or "audio" in kwargs.get("modalities", [])

181 181 if not has_audio and media is not None:

182 182 for media_data, filename in media:

183 183 if is_data_an_audio(media_data, filename):

@@ -311,6 +311,8 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

311 311

312 312 async with ClientSession(headers=DEFAULT_HEADERS, connector=get_connector(proxy=proxy)) as session:

313 313 if model in cls.audio_models:

314 if "audio" in kwargs and kwargs.get("audio", {}).get("voice") is None:

315 kwargs["audio"]["voice"] = cls.audio_models[model][0]

314 316 url = cls.text_api_endpoint

315 317 stream = False

316 318 else:

@@ -329,6 +331,7 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

329 331 "cache": cache,

330 332 **extra_parameters

331 333 })

334 print(f"Requesting {url} with data: {data}")

332 335 async with session.post(url, json=data) as response:

333 336 await raise_for_status(response)

334 337 if response.headers["content-type"].startswith("text/plain"):

Modified g4f/api/__init__.py +55 -4

@@ -13,7 +13,7 @@ import asyncio

13 13 from urllib.parse import quote_plus

14 14 from fastapi import FastAPI, Response, Request, UploadFile, Form, Depends

15 15 from fastapi.middleware.wsgi import WSGIMiddleware

16 from fastapi.responses import StreamingResponse, RedirectResponse, HTMLResponse, JSONResponse

16 from fastapi.responses import StreamingResponse, RedirectResponse, HTMLResponse, JSONResponse, FileResponse

17 17 from fastapi.exceptions import RequestValidationError

18 18 from fastapi.security import APIKeyHeader

19 19 from starlette.exceptions import HTTPException

@@ -30,6 +30,7 @@ from fastapi.encoders import jsonable_encoder

30 30 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials, HTTPBasic

31 31 from fastapi.middleware.cors import CORSMiddleware

32 32 from starlette.responses import FileResponse

33 from starlette.background import BackgroundTask

33 34 from types import SimpleNamespace

34 35 from typing import Union, Optional, List

35 36

@@ -49,6 +50,7 @@ from g4f.image.copy_images import get_media_dir, copy_media, get_source_url

49 50 from g4f.errors import ProviderNotFoundError, ModelNotFoundError, MissingAuthError, NoValidHarFileError

50 51 from g4f.cookies import read_cookie_files, get_cookies_dir

51 52 from g4f.providers.types import ProviderType

53 from g4f.providers.response import AudioResponse

52 54 from g4f.providers.any_provider import AnyProvider

53 55 from g4f import Provider

54 56 from g4f.gui import get_gui_app

@@ -58,7 +60,7 @@ from .stubs import (

58 60 ProviderResponseModel, ModelResponseModel,

59 61 ErrorResponseModel, ProviderResponseDetailModel,

60 62 FileResponseModel, UploadResponseModel,

61 TranscriptionResponseModel

63 TranscriptionResponseModel, AudioSpeechConfig

62 64 )

63 65 from g4f import debug

64 66

@@ -492,10 +494,11 @@ class Api:

492 494 }

493 495 @self.app.post("/v1/audio/transcriptions", responses=responses)

494 496 @self.app.post("/api/{path_provider}/audio/transcriptions", responses=responses)

495 async def generate_image(

497 @self.app.post("/api/markitdown", responses=responses)

498 async def convert(

496 499 file: UploadFile,

497 500 model: Annotated[Optional[str], Form()] = None,

498 provider: Annotated[Optional[str], Form()] = AppConfig.media_provider,

501 provider: Annotated[Optional[str], Form()] = "MarkItDown",

499 502 path_provider: str = None,

500 503 prompt: Annotated[Optional[str], Form()] = "Transcribe this audio",

501 504 api_key: Annotated[Optional[str], Form()] = None,

@@ -525,6 +528,54 @@ class Api:

525 528 logger.exception(e)

526 529 return ErrorResponse.from_exception(e, None, HTTP_500_INTERNAL_SERVER_ERROR)

527 530

531 responses = {

532 HTTP_200_OK: {"class": FileResponse},

533 HTTP_401_UNAUTHORIZED: {"model": ErrorResponseModel},

534 HTTP_404_NOT_FOUND: {"model": ErrorResponseModel},

535 HTTP_500_INTERNAL_SERVER_ERROR: {"model": ErrorResponseModel},

536 }

537 @self.app.post("/v1/audio/speech", responses=responses)

538 @self.app.post("/api/{path_provider}/audio/speech", responses=responses)

539 async def generate_speech(

540 config: AudioSpeechConfig,

541 provider: str = AppConfig.media_provider,

542 credentials: Annotated[HTTPAuthorizationCredentials, Depends(Api.security)] = None

543 ):

544 api_key = None

545 if credentials is not None and credentials.credentials != "secret":

546 api_key = credentials.credentials

547 try:

548 response = await self.client.chat.completions.create(

549 messages=[

550 {"role": "user", "content": f"{config.instrcutions} Text: {config.input}"}

551 ],

552 model=config.model,

553 provider=config.provider if provider is None else provider,

554 prompt=config.input,

555 audio=filter_none(voice=config.voice, format=config.response_format),

556 **filter_none(

557 api_key=api_key,

558 )

559 )

560 if isinstance(response.choices[0].message.content, AudioResponse):

561 response = response.choices[0].message.content.data

562 response = response.replace("/media", get_media_dir())

563 def delete_file():

564 try:

565 os.remove(response)

566 except Exception as e:

567 logger.exception(e)

568 return FileResponse(response, background=BackgroundTask(delete_file))

569 except (ModelNotFoundError, ProviderNotFoundError) as e:

570 logger.exception(e)

571 return ErrorResponse.from_exception(e, None, HTTP_404_NOT_FOUND)

572 except MissingAuthError as e:

573 logger.exception(e)

574 return ErrorResponse.from_exception(e, None, HTTP_401_UNAUTHORIZED)

575 except Exception as e:

576 logger.exception(e)

577 return ErrorResponse.from_exception(e, None, HTTP_500_INTERNAL_SERVER_ERROR)

578

528 579 @self.app.post("/v1/upload_cookies", responses={

529 580 HTTP_200_OK: {"model": List[FileResponseModel]},

530 581 })

Modified g4f/api/stubs.py +9 -1

@@ -118,4 +118,12 @@ class FileResponseModel(BaseModel):

118 118 class TranscriptionResponseModel(BaseModel):

119 119 text: str

120 120 model: str

121 provider: str

122

123 class AudioSpeechConfig(BaseModel):

124 input: str

125 model: Optional[str] = None

126 provider: Optional[str] = None

127 voice: Optional[str] = None

128 instrcutions: str = "Speech this text in a natural way."

129 response_format: Optional[str] = None

Modified g4f/client/__init__.py +15 -19

@@ -45,6 +45,17 @@ def add_chunk(content, chunk):

45 45 content = str(content) + str(chunk)

46 46 return content

47 47

48 def resolve_media(kwargs: dict, image = None, image_name: str = None) -> None:

49 if image is not None:

50 kwargs["media"] = [(image, image_name)]

51 elif "images" in kwargs:

52 kwargs["media"] = kwargs.pop("images")

53 if "media" in kwargs and not isinstance(kwargs["media"], list):

54 kwargs["media"] = [kwargs["media"]]

55 for idx, media in enumerate(kwargs.get("media", [])):

56 if not isinstance(media, (list, tuple)):

57 kwargs["media"][idx] = (media, getattr(media, "name", None))

48 59 # Synchronous iter_response function

49 60 def iter_response(

50 61 response: Union[Iterator[Union[str, ResponseType]]],

@@ -296,13 +307,7 @@ class Completions:

296 307 ) -> ChatCompletion:

297 308 if isinstance(messages, str):

298 309 messages = [{"role": "user", "content": messages}]

299 if image is not None:

300 kwargs["media"] = [(image, image_name)]

301 elif "images" in kwargs:

302 kwargs["media"] = kwargs.pop("images")

303 for idx, media in enumerate(kwargs.get("media", [])):

304 if not isinstance(media, (list, tuple)):

305 kwargs["media"][idx] = (media, getattr(media, "name", None))

310 resolve_media(kwargs, image, image_name)

306 311 if provider is None:

307 312 provider = self.provider

308 313 if provider is None:

@@ -483,6 +488,7 @@ class Images:

483 488 async def async_create_variation(

484 489 self,

485 490 image: ImageType,

491 image_name: str = None,

486 492 model: Optional[str] = None,

487 493 provider: Optional[ProviderType] = None,

488 494 response_format: Optional[str] = None,

@@ -494,11 +500,7 @@ class Images:

494 500 if proxy is None:

495 501 proxy = self.client.proxy

496 502 prompt = "create a variation of this image"

497 if image is not None:

498 kwargs["media"] = image

499 for idx, media in enumerate(kwargs.get("media", [])):

500 if not isinstance(media, (list, tuple)):

501 kwargs["media"][idx] = (media, getattr(media, "name", None))

503 resolve_media(kwargs, image, image_name)

502 504 error = None

503 505 response = None

504 506 if isinstance(provider_handler, IterListProvider):

@@ -600,13 +602,7 @@ class AsyncCompletions:

600 602 ) -> Awaitable[ChatCompletion]:

601 603 if isinstance(messages, str):

602 604 messages = [{"role": "user", "content": messages}]

603 if image is not None:

604 kwargs["media"] = [(image, image_name)]

605 elif "images" in kwargs:

606 kwargs["media"] = kwargs.pop("images")

607 for idx, media in enumerate(kwargs.get("media", [])):

608 if not isinstance(media, (list, tuple)):

609 kwargs["media"][idx] = (media, getattr(media, "name", None))

605 resolve_media(kwargs, image, image_name)

610 606 if provider is None:

611 607 provider = self.provider

612 608 if provider is None:

Modified g4f/providers/any_provider.py +6 -4

@@ -11,7 +11,7 @@ from ..Provider.hf_space import HuggingSpace

11 11 from .. import Provider

12 12 from .. import models

13 13

from ..Provider import Cloudflare, LMArenaProvider, Gemini, Grok, DeepSeekAPI, PerplexityLabs, LambdaChat, PollinationsAI, FreeRouter

14 from ..Provider import Microsoft_Phi_4, DeepInfraChat, Blackbox

14 from ..Provider import Microsoft_Phi_4, DeepInfraChat, Blackbox, EdgeTTS, gTTS, MarkItDown

15 15 from .base_provider import AsyncGeneratorProvider, ProviderModelMixin

16 16

17 17 class AnyProvider(AsyncGeneratorProvider, ProviderModelMixin):

@@ -124,15 +124,17 @@ class AnyProvider(AsyncGeneratorProvider, ProviderModelMixin):

124 124 elif not model or model == cls.default_model:

125 125 model = ""

126 126 has_image = False

127 has_audio = "audio" in kwargs

127 has_audio = False

128 128 if not has_audio and media is not None:

129 129 for media_data, filename in media:

130 130 if is_data_an_audio(media_data, filename):

131 131 has_audio = True

132 132 break

133 133 has_image = True

134 if has_audio:

135 providers = [PollinationsAI, Microsoft_Phi_4]

134 if "audio" in kwargs or "audio" in kwargs.get("modalities", []):

135 providers = [PollinationsAI, EdgeTTS, gTTS]

136 elif has_audio:

137 providers = [PollinationsAI, Microsoft_Phi_4, MarkItDown]

136 138 elif has_image:

137 139 providers = models.default_vision.best_provider.providers

138 140 else: