返回提交历史
Added
g4f/Provider/audio/PollinationsAudio.py
+205
-0
Modified
g4f/Provider/audio/__init__.py
+1
-0
Modified
g4f/gui/server/api.py
+1
-1
Modified
g4f/providers/any_model_map.py
+2
-2
Modified
g4f/providers/any_provider.py
+3
-2
XFEstudio/gpt4free
feat(PollinationsAudio): add new audio provider with transcription and speech capabilities
b4e6f291
代码差异
5 个文件
+212
-5
@@ -0,0 +1,205 @@
1
from __future__ import annotations
2
3
from aiohttp import ClientSession, FormData
4
from urllib.parse import quote
5
6
from curl_cffi import requests
7
8
from ...typing import AsyncResult, Messages, MediaListType
9
from ...image import is_data_an_audio
10
from ...image.copy_images import save_response_media
11
from ...providers.response import AudioResponse
12
from ...providers.base_provider import AsyncGeneratorProvider, ProviderModelMixin
13
from ...requests.aiohttp import get_connector
14
from ...requests.defaults import DEFAULT_HEADERS
15
from ...requests.raise_for_status import raise_for_status
16
from ...tools.run_tools import AuthManager
17
from ...image import to_bytes
18
from ...tools.media import merge_media
19
from ..helper import filter_none, format_media_prompt
20
from ..PollinationsAI import PollinationsAI
21
from ... import debug
22
23
class PollinationsAudio(AsyncGeneratorProvider, ProviderModelMixin):
24
label = "PollinationsAudio"
25
parent = PollinationsAI.__name__
26
active_by_default = False
27
working = True
28
supports_stream = False
29
default_model = "elevenlabs"
30
default_transcription_model = "openai-audio"
31
models_endpoint = "https://gen.pollinations.ai/audio/models"
32
speech_api_endpoint = "https://gen.pollinations.ai/v1/audio/speech"
33
transcription_api_endpoint = "https://gen.pollinations.ai/v1/audio/transcriptions"
34
simple_audio_endpoint = "https://gen.pollinations.ai/audio/{}"
35
public_key = "".join(["pk", "_", "pqjxQN7C", "oSJUShHF"])
36
available_voices = [
37
"alloy", "echo", "fable", "onyx", "nova", "shimmer", "ash", "ballad", "coral", "sage", "verse",
38
"rachel", "domi", "bella", "elli", "charlotte", "dorothy", "sarah", "emily", "lily", "matilda",
39
"adam", "antoni", "arnold", "josh", "sam", "daniel", "charlie", "james", "fin", "callum", "liam",
40
"george", "brian", "bill"
41
]
42
documented_audio_models = [
43
"openai-audio", "elevenlabs", "elevenmusic", "whisper", "whisper-large-v3", "whisper-1",
44
"scribe", "acestep", "qwen-tts", "qwen-tts-instruct"
45
]
46
47
@classmethod
48
def get_models(cls, **kwargs) -> list[str]:
49
if not cls.models:
50
try:
51
response = requests.get(cls.models_endpoint, timeout=kwargs.get("timeout", 15))
52
response.raise_for_status()
53
models = response.json()
54
cls.models = {data.get("name"): {"id": data.get("name"), **data} for data in models}
55
except Exception as e:
56
debug.error(e)
57
cls.models = {model: {"id": model} for model in cls.documented_audio_models}
58
return cls.models
59
60
@classmethod
61
def _get_audio_voices(cls) -> list[str]:
62
for model in cls.get_models().values():
63
if "voices" in model:
64
return model["voices"]
65
return cls.available_voices
66
67
@classmethod
68
def get_grouped_models(cls) -> list[dict[str, list[str]]]:
69
return [
70
{"group": model.get("id"), "models": model.get("voices")} if model.get("voices") else model
71
for model in
72
cls.get_models().values()
73
]
74
75
@classmethod
76
async def create_async_generator(
77
cls,
78
model: str,
79
messages: Messages,
80
media: MediaListType = None,
81
proxy: str = None,
82
api_key: str = None,
83
prompt: str = None,
84
audio: dict = None,
85
stream: bool = False,
86
**kwargs
87
) -> AsyncResult:
88
api_key = api_key or kwargs.get("api_key") or AuthManager.load_api_key(cls) or cls.public_key
89
audio = {} if audio is None else dict(audio)
90
if model in cls._get_audio_voices() and audio.get("voice") is None:
91
audio["voice"] = model
92
model = cls.default_model
93
94
# Any audio media input is treated as a transcription request.
95
media = list(merge_media(media, messages)) if model != cls.default_model else media
96
if media and any(is_data_an_audio(media_data, filename) for media_data, filename in media):
97
async for chunk in cls._create_transcription(
98
media=media,
99
api_key=api_key,
100
proxy=proxy,
101
model=model,
102
**kwargs,
103
):
104
yield chunk
105
return
106
if model == cls.default_transcription_model:
107
model = cls.default_model
108
async for chunk in cls._create_speech(
109
model=model if model else cls.default_model,
110
messages=messages,
111
prompt=prompt,
112
api_key=api_key,
113
proxy=proxy,
114
audio=audio,
115
**kwargs,
116
):
117
yield chunk
118
119
@classmethod
120
async def _create_speech(
121
cls,
122
model: str,
123
messages: Messages,
124
prompt: str,
125
api_key: str,
126
proxy: str,
127
audio: dict,
128
**kwargs,
129
) -> AsyncResult:
130
text = format_media_prompt(messages, prompt)
131
voice = audio.get("voice") or kwargs.get("voice")
132
response_format = audio.get("format") or kwargs.get("response_format")
133
payload = filter_none(
134
model=model,
135
input=text[:4096],
136
voice=voice,
137
response_format=response_format,
138
speed=audio.get("speed") if "speed" in audio else kwargs.get("speed"),
139
duration=audio.get("duration") if "duration" in audio else kwargs.get("duration"),
140
instrumental=audio.get("instrumental") if "instrumental" in audio else kwargs.get("instrumental"),
141
seed=audio.get("seed") if "seed" in audio else kwargs.get("seed"),
142
style=audio.get("style") if "style" in audio else kwargs.get("style"),
143
instruct=audio.get("instruct") if "instruct" in audio else kwargs.get("instruct"),
144
)
145
146
if not kwargs.get("download_media", True) and api_key.startswith("pk_"):
147
encoded_text = quote(text[:4096])
148
query = "&".join(f"{key}={quote(str(value))}" for key, value in payload.items() if key not in {"input"} and value is not None)
149
if query:
150
query = f"{query}&key={quote(api_key)}"
151
else:
152
query = f"key={quote(api_key)}"
153
yield AudioResponse(f"{cls.simple_audio_endpoint.format(encoded_text)}?{query}", text, headers=headers)
154
return
155
156
headers = {
157
**DEFAULT_HEADERS,
158
"authorization": f"Bearer {api_key}",
159
}
160
async with ClientSession(headers=headers, connector=get_connector(proxy=proxy)) as session:
161
async with session.post(cls.speech_api_endpoint, json=payload) as response:
162
await raise_for_status(response)
163
async for chunk in save_response_media(response, text, [model, voice]):
164
yield chunk
165
166
@classmethod
167
async def _create_transcription(
168
cls,
169
media: MediaListType,
170
api_key: str,
171
proxy: str,
172
model: str = None,
173
**kwargs,
174
) -> AsyncResult:
175
media_data, filename = media[0]
176
file_bytes = to_bytes(media_data)
177
if not file_bytes:
178
raise ValueError("No valid audio data found for transcription")
179
180
form = FormData()
181
form.add_field("file", file_bytes, filename=filename or "audio.wav", content_type="application/octet-stream")
182
183
transcription_model = model
184
if transcription_model in (None, "openai-audio"):
185
transcription_model = cls.default_transcription_model
186
form_fields = filter_none(
187
model=transcription_model,
188
language=kwargs.get("language"),
189
prompt=kwargs.get("prompt"),
190
response_format=kwargs.get("response_format"),
191
temperature=kwargs.get("temperature"),
192
)
193
for key, value in form_fields.items():
194
form.add_field(key, str(value))
195
196
headers = {"authorization": f"Bearer {api_key}"}
197
async with ClientSession(headers=headers, connector=get_connector(proxy=proxy)) as session:
198
async with session.post(cls.transcription_api_endpoint, data=form) as response:
199
await raise_for_status(response)
200
content_type = response.headers.get("content-type", "")
201
if "application/json" in content_type:
202
data = await response.json()
203
yield data.get("text", "")
204
else:
205
yield await response.text()
@@ -2,3 +2,4 @@ from .EdgeTTS import EdgeTTS
2
2
from .gTTS import gTTS
3
3
from .MarkItDown import MarkItDown
4
4
from .OpenAIFM import OpenAIFM
5
from .PollinationsAudio import PollinationsAudio
@@ -62,7 +62,7 @@ class Api:
62
62
return [{
63
63
"group": model.get("group"),
64
64
"models": [get_model_data(provider, name) for name in (model.get("models", {}).values() if isinstance(model.get("models"), dict) else model.get("models", []))]
65
} for model in models]
65
} if model.get("models") else model for model in models]
66
66
return [
67
67
get_model_data(provider, model)
68
68
for model in (models.values() if isinstance(models, dict) else models)
@@ -1140,7 +1140,7 @@ model_map = {
1140
1140
"ApiAirforce": "z-image"
1141
1141
},
1142
1142
"openai-audio": {
1143
"PollinationsAI": "openai-audio"
1143
"PollinationsAudio": "openai-audio"
1144
1144
},
1145
1145
"": {
1146
1146
"Gemini": "",
@@ -4343,7 +4343,7 @@ parents = {
4343
4343
model_aliases = {
4344
4344
"openai": "gpt-5-mini",
4345
4345
"gpt-4o-mini-2024-07-18": "gpt-4o-mini",
4346
"openai-audio": "PollinationsAI:openai-audio",
4346
"openai-audio": "PollinationsAudio:openai-audio",
4347
4347
"coral": "gpt-4o-mini-tts",
4348
4348
"Think Deeper": "o1",
4349
4349
"openrouter:openai/o3-mini-high": "o3-mini-high",
@@ -31,6 +31,7 @@ from ..Provider import (
31
31
gTTS,
32
32
MarkItDown,
33
33
OpenAIFM,
34
PollinationsAudio,
34
35
)
35
36
from ..Provider import (
36
37
HuggingFace,
@@ -461,9 +462,9 @@ class AnyProvider(AsyncGeneratorProvider, AnyModelProviderMixin):
461
462
# Tool calling is an API-level feature; routing should be based on model/media.
462
463
if "audio" in kwargs or "audio" in kwargs.get("modalities", []):
463
464
if kwargs.get("audio", {}).get("language") is None:
464
providers = [PollinationsAI, OpenAIFM, Gemini]
465
providers = [PollinationsAudio, OpenAIFM, Gemini]
465
466
else:
466
providers = [PollinationsAI, OpenAIFM, EdgeTTS, gTTS]
467
providers = [PollinationsAudio, OpenAIFM, EdgeTTS, gTTS]
467
468
elif has_audio:
468
469
providers = [PollinationsAI, Microsoft_Phi_4_Multimodal, MarkItDown]
469
470
elif has_image: