gpt4free

公开

关注 0 Fork 0 Star 0

XFEstudio/gpt4free

feat: add tool_call emulation for OpenAI API (#3352)

* feat: add tool_call emulation for OpenAI API Avoid forcing PollinationsAI when tools are present, and add an opt-in tool_emulation mode (or G4F_TOOL_EMULATION=1) to emit OpenAI-compatible tool_calls for providers that ignore tools. * fix: avoid duplicate stream kwarg in tool emulation Tool emulation calls the upstream provider with stream=False; remove stream/stream_timeout from forwarded kwargs to prevent conflicts. * fix: prefer non-auth providers when api_key missing When routing via AnyProvider without an api_key, try providers with needs_auth=false first to reduce MissingAuthError for tool-enabled clients like MarksCode. * test: cover tool call emulation Route tool_emulation through ToolSupportProvider (avoid circular imports) and add unittest coverage for multi-tool JSON plans and run_tools integration.

405868c5

Marcos Vinícius Claudiano <mvclaudiano@hotmail.com>

提交于 2026-02-09 08:01

父提交 0096673

代码差异

6 个文件 +656 -179

Modified etc/unittest/__main__.py +2 -1

@@ -16,5 +16,6 @@ from .thinking import *

16 16 from .web_search import *

17 17 from .models import *

18 18 from .mcp import *

19 from .tool_support_provider import *

19 20

20 unittest.main()

21 unittest.main()

Added etc/unittest/tool_support_provider.py +94 -0

@@ -0,0 +1,94 @@

1 import asyncio

2 import unittest

4 from g4f.providers.base_provider import AsyncGeneratorProvider

5 from g4f.providers.response import FinishReason, ToolCalls

6 from g4f.providers.tool_support import ToolSupportProvider

7 from g4f.tools.run_tools import async_iter_run_tools

10 class ToolPlanProviderMock(AsyncGeneratorProvider):

11 working = True

13 @staticmethod

14 async def create_async_generator(model, messages, stream=True, **kwargs):

15 # Always return a tool call plan.

16 yield (

17 '{"tool_calls":['

18 '{"name":"read","arguments":{"filePath":"README.md"}},'

19 '{"name":"glob","arguments":{"pattern":"**/*.py"}}'

20 "]}"

21 )

22 yield FinishReason("stop")

25 TOOLS = [

26 {

27 "type": "function",

28 "function": {

29 "name": "read",

30 "description": "Read a file",

31 "parameters": {

32 "type": "object",

33 "properties": {"filePath": {"type": "string"}},

34 "required": ["filePath"],

35 },

36 },

37 },

38 {

39 "type": "function",

40 "function": {

41 "name": "glob",

42 "description": "Glob files",

43 "parameters": {

44 "type": "object",

45 "properties": {"pattern": {"type": "string"}},

46 "required": ["pattern"],

47 },

48 },

49 },

50 ]

53 class TestToolSupportProvider(unittest.TestCase):

54 def test_emits_tool_calls_from_json_plan(self):

55 async def run():

56 out = []

57 async for chunk in ToolSupportProvider.create_async_generator(

58 model="test-model",

59 messages=[{"role": "user", "content": "list files"}],

60 stream=True,

61 tools=TOOLS,

62 provider=ToolPlanProviderMock,

63 ):

64 out.append(chunk)

65 return out

67 out = asyncio.run(run())

68 tool_chunks = [x for x in out if isinstance(x, ToolCalls)]

69 self.assertEqual(len(tool_chunks), 1)

70 calls = tool_chunks[0].get_list()

71 self.assertEqual(len(calls), 2)

72 self.assertEqual(calls[0]["function"]["name"], "read")

73 self.assertEqual(calls[1]["function"]["name"], "glob")

75 def test_run_tools_routes_to_tool_support_provider(self):

76 async def run():

77 out = []

78 async for chunk in async_iter_run_tools(

79 ToolPlanProviderMock,

80 model="test-model",

81 messages=[{"role": "user", "content": "list files"}],

82 stream=True,

83 tools=TOOLS,

84 tool_emulation=True,

85 ):

86 out.append(chunk)

87 return out

89 out = asyncio.run(run())

90 self.assertTrue(any(isinstance(x, ToolCalls) for x in out))

93 if __name__ == "__main__":

94 unittest.main()

Modified g4f/api/stubs.py +52 -22

@@ -5,6 +5,7 @@ from typing import Union, Optional

5 5

6 6 from ..typing import Messages

7 7

8 9 class RequestConfig(BaseModel):

9 10 model: str = Field(default="")

10 11 provider: Optional[str] = None

@@ -17,21 +18,33 @@ class RequestConfig(BaseModel):

17 18 max_tokens: Optional[int] = None

18 19 stop: Union[list[str], str, None] = None

19 20 api_key: Optional[Union[str, dict[str, str]]] = None

20 base_url: str = None

21 base_url: Optional[str] = None

21 22 web_search: Optional[bool] = None

22 23 proxy: Optional[str] = None

23 24 conversation: Optional[dict] = None

24 25 timeout: Optional[int] = None

25 26 stream_timeout: Optional[int] = None

26 tool_calls: list = Field(default=[], examples=[[

27 {

28 "function": {

				"arguments": {"query":"search query", "max_results":5, "max_words": 2500, "backend": "auto", "add_text": True, "timeout": 5},

30 "name": "search_tool"

31 },

32 "type": "function"

33 }

34 ]])

27 tool_calls: list = Field(

28 default=[],

29 examples=[

30 [

31 {

32 "function": {

33 "arguments": {

34 "query": "search query",

35 "max_results": 5,

36 "max_words": 2500,

37 "backend": "auto",

38 "add_text": True,

39 "timeout": 5,

40 },

41 "name": "search_tool",

42 },

43 "type": "function",

44 }

45 ]

46 ],

47 )

35 48 reasoning_effort: Optional[str] = None

36 49 logit_bias: Optional[dict] = None

37 50 modalities: Optional[list[str]] = None

@@ -40,21 +53,29 @@ class RequestConfig(BaseModel):

40 53 download_media: bool = False

41 54 raw: bool = False

42 55 extra_body: Optional[dict] = None

56 # When set (or when env G4F_TOOL_EMULATION=1), the server will attempt to

57 # emulate OpenAI tool_calls for providers that don't support tools natively.

58 tool_emulation: Optional[bool] = None

43 60

44 61 class ChatCompletionsConfig(RequestConfig):

45 messages: Messages = Field(examples=[[{"role": "system", "content": ""}, {"role": "user", "content": ""}]])

62 messages: Messages = Field(

63 examples=[[{"role": "system", "content": ""}, {"role": "user", "content": ""}]]

64 )

46 65 stream: bool = False

47 66 image: Optional[str] = None

48 67 image_name: Optional[str] = None

49 68 images: Optional[list[tuple[str, str]]] = None

50 tools: list = None

51 parallel_tool_calls: bool = None

69 tools: Optional[list] = None

70 parallel_tool_calls: Optional[bool] = None

52 71 tool_choice: Optional[str] = None

53 72 conversation_id: Optional[str] = None

54 73

55 75 class ResponsesConfig(RequestConfig):

56 76 input: Union[Messages, str]

57 77

58 79 class ImageGenerationConfig(BaseModel):

59 80 prompt: str

60 81 model: Optional[str] = None

@@ -74,21 +95,22 @@ class ImageGenerationConfig(BaseModel):

74 95 audio: Optional[dict] = None

75 96 download_media: bool = True

76 97

78 @model_validator(mode='before')

98 @model_validator(mode="before")

79 99 def parse_size(cls, values):

80 if values.get('width') is not None and values.get('height') is not None:

100 if values.get("width") is not None and values.get("height") is not None:

81 101 return values

82 102

83 size = values.get('size')

103 size = values.get("size")

84 104 if size:

85 105 try:

86 width, height = map(int, size.split('x'))

87 values['width'] = width

88 values['height'] = height

89 except (ValueError, AttributeError): pass # If the format is incorrect, we simply ignore it.

106 width, height = map(int, size.split("x"))

107 values["width"] = width

108 values["height"] = height

109 except (ValueError, AttributeError):

110 pass # If the format is incorrect, we simply ignore it.

90 111 return values

91 112

113

92 114 class ProviderResponseModel(BaseModel):

93 115 id: str

94 116 object: str = "provider"

@@ -96,38 +118,46 @@ class ProviderResponseModel(BaseModel):

96 118 url: Optional[str]

97 119 label: Optional[str]

98 120

121

99 122 class ProviderResponseDetailModel(ProviderResponseModel):

100 123 models: list[str]

101 124 image_models: list[str]

102 125 vision_models: list[str]

103 126 params: list[str]

104 127

128

105 129 class ModelResponseModel(BaseModel):

106 130 id: str

107 131 object: str = "model"

108 132 created: int

109 133 owned_by: Optional[str]

110 134

135

111 136 class UploadResponseModel(BaseModel):

112 137 bucket_id: str

113 138 url: str

114 139

140

115 141 class ErrorResponseModel(BaseModel):

116 142 error: ErrorResponseMessageModel

117 143 model: Optional[str] = None

118 144 provider: Optional[str] = None

119 145

146

120 147 class ErrorResponseMessageModel(BaseModel):

121 148 message: str

122 149

150

123 151 class FileResponseModel(BaseModel):

124 152 filename: str

125 153

154

126 155 class TranscriptionResponseModel(BaseModel):

127 156 text: str

128 157 model: str

129 158 provider: str

130 159

160

131 161 class AudioSpeechConfig(BaseModel):

132 162 input: str

133 163 model: Optional[str] = None

@@ -136,4 +166,4 @@ class AudioSpeechConfig(BaseModel):

136 166 instrcutions: str = "Speech this text in a natural way."

137 167 response_format: Optional[str] = None

138 168 language: Optional[str] = None

139 download_media: bool = True

169 download_media: bool = True

Modified g4f/providers/any_provider.py +187 -54

@@ -9,24 +9,80 @@ from ..image import is_data_an_audio

9 9 from ..providers.retry_provider import RotatedProvider

10 10 from ..Provider.needs_auth import OpenaiChat, CopilotAccount

11 11 from ..Provider.hf_space import HuggingSpace

from ..Provider import Custom, PollinationsImage, OpenaiAccount, Copilot, Cloudflare, Gemini, Grok, Perplexity, LambdaChat, PollinationsAI, PuterJS

13 from ..Provider import Microsoft_Phi_4_Multimodal, DeepInfra, LMArena, EdgeTTS, gTTS, MarkItDown, OpenAIFM

14 from ..Provider import HuggingFace, HuggingFaceMedia, Azure, Qwen, EasyChat, GLM, OpenRouterFree, GeminiPro, Perplexity

12 from ..Provider import (

13 Custom,

14 PollinationsImage,

15 OpenaiAccount,

16 Copilot,

17 Cloudflare,

18 Gemini,

19 Grok,

20 Perplexity,

21 LambdaChat,

22 PollinationsAI,

23 PuterJS,

24 )

25 from ..Provider import (

26 Microsoft_Phi_4_Multimodal,

27 DeepInfra,

28 LMArena,

29 EdgeTTS,

30 gTTS,

31 MarkItDown,

32 OpenAIFM,

33 )

34 from ..Provider import (

35 HuggingFace,

36 HuggingFaceMedia,

37 Azure,

38 Qwen,

39 EasyChat,

40 GLM,

41 OpenRouterFree,

42 GeminiPro,

43 Perplexity,

44 )

15 45 from .base_provider import AsyncGeneratorProvider, ProviderModelMixin

16 46 from .. import Provider

17 47 from .. import models

18 48 from .. import debug

from .any_model_map import audio_models, image_models, vision_models, video_models, model_map, models_count, parents, model_aliases

49 from .any_model_map import (

50 audio_models,

51 image_models,

52 vision_models,

53 video_models,

54 model_map,

55 models_count,

56 parents,

57 model_aliases,

58 )

20 59

21 60 # Add providers to existing models on map

22 61 PROVIDERS_LIST_2 = [

    OpenaiChat, Copilot, CopilotAccount, PollinationsAI, Perplexity, Gemini, Grok, Azure, Qwen, EasyChat, GLM, OpenRouterFree

62 OpenaiChat,

63 Copilot,

64 CopilotAccount,

65 PollinationsAI,

66 Perplexity,

67 Gemini,

68 Grok,

69 Azure,

70 Qwen,

71 EasyChat,

72 GLM,

73 OpenRouterFree,

24 74 ]

25 75

26 76 # Add all models to the model map

27 77 PROVIDERS_LIST_3 = [

28 LambdaChat, DeepInfra, HuggingFace, HuggingFaceMedia, LMArena,

29 PuterJS, Cloudflare, HuggingSpace

78 LambdaChat,

79 DeepInfra,

80 HuggingFace,

81 HuggingFaceMedia,

82 LMArena,

83 PuterJS,

84 Cloudflare,

85 HuggingSpace,

30 86 ]

31 87

32 88 LABELS = {

@@ -54,6 +110,7 @@ LABELS = {

54 110 "other": "Other Models",

55 111 }

56 112

113

57 114 class AnyModelProviderMixin(ProviderModelMixin):

58 115 """Mixin to provide model-related methods for providers."""

59 116

@@ -95,9 +152,20 @@ class AnyModelProviderMixin(ProviderModelMixin):

95 152 cls.create_model_map()

96 153 file = os.path.join(os.path.dirname(__file__), "any_model_map.py")

97 154 with open(file, "w", encoding="utf-8") as f:

            for key in ["audio_models", "image_models", "vision_models", "video_models", "model_map", "models_count", "parents", "model_aliases"]:

155 for key in [

156 "audio_models",

157 "image_models",

158 "vision_models",

159 "video_models",

160 "model_map",

161 "models_count",

162 "parents",

163 "model_aliases",

164 ]:

99 165 value = getattr(cls, key)

100 f.write(f"{key} = {json.dumps(value, indent=2) if isinstance(value, dict) else repr(value)}\n")

166 f.write(

167 f"{key} = {json.dumps(value, indent=2) if isinstance(value, dict) else repr(value)}\n"

168 )

101 169

102 170 @classmethod

103 171 def create_model_map(cls):

@@ -108,14 +176,21 @@ class AnyModelProviderMixin(ProviderModelMixin):

108 176

109 177 # Get models from the models registry

110 178 cls.model_map = {

111 "default": {provider.__name__: "" for provider in models.default.best_provider.providers},

179 "default": {

180 provider.__name__: ""

181 for provider in models.default.best_provider.providers

182 },

112 183 }

113 cls.model_map.update({

114 name: {

115 provider.__name__: model.get_long_name() for provider in providers

116 if provider.working

117 } for name, (model, providers) in models.__models__.items()

118 })

184 cls.model_map.update(

185 {

186 name: {

187 provider.__name__: model.get_long_name()

188 for provider in providers

189 if provider.working

190 }

191 for name, (model, providers) in models.__models__.items()

192 }

193 )

119 194 for name, (model, providers) in models.__models__.items():

120 195 if isinstance(model, models.ImageModel):

121 196 cls.image_models.append(name)

@@ -137,15 +212,17 @@ class AnyModelProviderMixin(ProviderModelMixin):

137 212 cls.model_map[cleaned] = {}

138 213 cls.model_map[cleaned].update({provider.__name__: model})

139 214 except Exception as e:

140 debug.error(f"Error getting models for provider {provider.__name__}:", e)

215 debug.error(

216 f"Error getting models for provider {provider.__name__}:", e

217 )

141 218 continue

142 219

143 220 # Update special model lists

144 if hasattr(provider, 'image_models'):

221 if hasattr(provider, "image_models"):

145 222 cls.image_models.extend(provider.image_models)

146 if hasattr(provider, 'vision_models'):

223 if hasattr(provider, "vision_models"):

147 224 cls.vision_models.extend(provider.vision_models)

148 if hasattr(provider, 'video_models'):

225 if hasattr(provider, "video_models"):

149 226 cls.video_models.extend(provider.video_models)

150 227

151 228 for provider in PROVIDERS_LIST_3:

@@ -154,7 +231,9 @@ class AnyModelProviderMixin(ProviderModelMixin):

154 231 try:

155 232 new_models = provider.get_models()

156 233 except Exception as e:

157 debug.error(f"Error getting models for provider {provider.__name__}:", e)

234 debug.error(

235 f"Error getting models for provider {provider.__name__}:", e

236 )

158 237 continue

159 238 if provider == HuggingFaceMedia:

160 239 new_models = provider.video_models

@@ -171,15 +250,21 @@ class AnyModelProviderMixin(ProviderModelMixin):

171 250 cls.model_map[alias].update({provider.__name__: model})

172 251

173 252 # Update special model lists with both original and cleaned names

174 if hasattr(provider, 'image_models'):

253 if hasattr(provider, "image_models"):

175 254 cls.image_models.extend(provider.image_models)

176 cls.image_models.extend([clean_name(model) for model in provider.image_models])

177 if hasattr(provider, 'vision_models'):

255 cls.image_models.extend(

256 [clean_name(model) for model in provider.image_models]

257 )

258 if hasattr(provider, "vision_models"):

178 259 cls.vision_models.extend(provider.vision_models)

179 cls.vision_models.extend([clean_name(model) for model in provider.vision_models])

180 if hasattr(provider, 'video_models'):

260 cls.vision_models.extend(

261 [clean_name(model) for model in provider.vision_models]

262 )

263 if hasattr(provider, "video_models"):

181 264 cls.video_models.extend(provider.video_models)

182 cls.video_models.extend([clean_name(model) for model in provider.video_models])

265 cls.video_models.extend(

266 [clean_name(model) for model in provider.video_models]

267 )

183 268

184 269 for provider in Provider.__providers__:

185 270 try:

@@ -188,7 +273,12 @@ class AnyModelProviderMixin(ProviderModelMixin):

188 273 if model not in cls.model_map:

189 274 cls.model_map[model] = {}

190 275 cls.model_map[model].update({provider.__name__: model})

191

                elif provider.working and hasattr(provider, "get_models") and provider not in [AnyProvider, Custom, PollinationsImage, OpenaiAccount]:

276 elif (

277 provider.working

278 and hasattr(provider, "get_models")

279 and provider

280 not in [AnyProvider, Custom, PollinationsImage, OpenaiAccount]

281 ):

192 282 for model in provider.get_models():

193 283 clean = clean_name(model)

194 284 if clean in cls.model_map:

@@ -201,13 +291,21 @@ class AnyModelProviderMixin(ProviderModelMixin):

201 291 if "gemini" in model or "gemma" in model:

202 292 cls.model_map[alias].update({provider.__name__: model})

203 293 except Exception as e:

204 debug.error(f"Error getting models for provider {provider.__name__}:", e)

294 debug.error(

295 f"Error getting models for provider {provider.__name__}:", e

296 )

205 297 continue

206 298

207 299 # Process audio providers

208 300 for provider in [Microsoft_Phi_4_Multimodal, PollinationsAI]:

209 301 if provider.working:

210 cls.audio_models.extend([model for model in provider.audio_models if model not in cls.audio_models])

302 cls.audio_models.extend(

303 [

304 model

305 for model in provider.audio_models

306 if model not in cls.audio_models

307 ]

308 )

211 309

212 310 # Update model counts

213 311 for model, providers in cls.model_map.items():

@@ -229,7 +327,11 @@ class AnyModelProviderMixin(ProviderModelMixin):

229 327

230 328 for model, providers in cls.model_map.items():

231 329 for provider, alias in providers.items():

232 if alias != model and isinstance(alias, str) and alias not in cls.model_map:

330 if (

331 alias != model

332 and isinstance(alias, str)

333 and alias not in cls.model_map

334 ):

233 335 cls.model_aliases[alias] = model

234 336

235 337 @classmethod

@@ -250,10 +352,18 @@ class AnyModelProviderMixin(ProviderModelMixin):

250 352 if start in ("PollinationsAI", "openrouter"):

251 353 added = True

252 354 # Check for Mistral company models specifically

253 elif model.startswith("mistral") and not any(x in model for x in ["dolphin", "nous", "openhermes"]):

355 elif model.startswith("mistral") and not any(

356 x in model for x in ["dolphin", "nous", "openhermes"]

357 ):

254 358 groups["mistral"].append(model)

255 359 added = True

256

            elif model.startswith(("pixtral-", "ministral-", "codestral", "devstral", "magistral")) or "mistral" in model or "mixtral" in model:

360 elif (

361 model.startswith(

362 ("pixtral-", "ministral-", "codestral", "devstral", "magistral")

363 )

364 or "mistral" in model

365 or "mixtral" in model

366 ):

257 367 groups["mistral"].append(model)

258 368 added = True

259 369 # Check for Qwen models

@@ -261,7 +371,9 @@ class AnyModelProviderMixin(ProviderModelMixin):

261 371 groups["qwen"].append(model)

262 372 added = True

263 373 # Check for Microsoft Phi models

264 elif model.startswith(("phi-", "microsoft/")) or "wizardlm" in model.lower():

374 elif (

375 model.startswith(("phi-", "microsoft/")) or "wizardlm" in model.lower()

376 ):

265 377 groups["phi"].append(model)

266 378 added = True

267 379 # Check for Meta LLaMA models

@@ -292,7 +404,9 @@ class AnyModelProviderMixin(ProviderModelMixin):

292 404 groups["image"].append(model)

293 405 added = True

294 406 # Check for OpenAI models

295 elif model.startswith(("gpt-", "chatgpt-", "o1", "o1", "o3", "o4")) or model in ("auto", "searchgpt"):

407 elif model.startswith(

408 ("gpt-", "chatgpt-", "o1", "o1", "o3", "o4")

409 ) or model in ("auto", "searchgpt"):

296 410 groups["openai"].append(model)

297 411 added = True

298 412 # Check for video models

@@ -312,6 +426,7 @@ class AnyModelProviderMixin(ProviderModelMixin):

312 426 {"group": LABELS[group], "models": names} for group, names in groups.items()

313 427 ]

314 428

429

315 430 class AnyProvider(AsyncGeneratorProvider, AnyModelProviderMixin):

316 431 working = True

317 432 active_by_default = True

@@ -325,7 +440,7 @@ class AnyProvider(AsyncGeneratorProvider, AnyModelProviderMixin):

325 440 media: MediaListType = None,

326 441 ignored: list[str] = [],

327 442 api_key: Union[str, dict[str, str]] = None,

328 **kwargs

443 **kwargs,

329 444 ) -> AsyncResult:

330 445 providers = []

331 446 if not model or model == cls.default_model:

@@ -338,9 +453,9 @@ class AnyProvider(AsyncGeneratorProvider, AnyModelProviderMixin):

338 453 has_audio = True

339 454 break

340 455 has_image = True

341 if kwargs.get("tools", None):

342 providers = [PollinationsAI]

343 elif "audio" in kwargs or "audio" in kwargs.get("modalities", []):

456 # Do not override provider selection just because tools are present.

457 # Tool calling is an API-level feature; routing should be based on model/media.

458 if "audio" in kwargs or "audio" in kwargs.get("modalities", []):

344 459 if kwargs.get("audio", {}).get("language") is None:

345 460 providers = [PollinationsAI, OpenAIFM, Gemini]

346 461 else:

@@ -381,37 +496,54 @@ class AnyProvider(AsyncGeneratorProvider, AnyModelProviderMixin):

381 496 elif model in provider.model_aliases:

382 497 providers.append(provider)

383 498 except Exception as e:

384 debug.error(f"Error checking provider {provider.__name__} for model {model}:", e)

385 providers = [provider for provider in providers if provider.working and provider.get_parent() not in ignored]

386 providers = list({provider.__name__: provider for provider in providers}.values())

499 debug.error(

500 f"Error checking provider {provider.__name__} for model {model}:",

501 e,

502 )

503 providers = [

504 provider

505 for provider in providers

506 if provider.working and provider.get_parent() not in ignored

507 ]

508 providers = list(

509 {provider.__name__: provider for provider in providers}.values()

510 )

511

512 # Free-first routing: if no api_key is provided, prioritize providers that

513 # don't require auth before trying auth-gated providers.

514 has_api_key = bool(api_key) or bool(kwargs.get("api_key"))

515 if not has_api_key:

516 providers.sort(key=lambda p: bool(getattr(p, "needs_auth", False)))

387 517

388 518 if len(providers) == 0:

389 raise ModelNotFoundError(f"AnyProvider: Model {model} not found in any provider.")

519 raise ModelNotFoundError(

520 f"AnyProvider: Model {model} not found in any provider."

521 )

390 522

391 debug.log(f"AnyProvider: Using providers: {[provider.__name__ for provider in providers]} for model '{model}'")

523 debug.log(

524 f"AnyProvider: Using providers: {[provider.__name__ for provider in providers]} for model '{model}'"

525 )

392 526

393 527 async for chunk in RotatedProvider(providers).create_async_generator(

394 model,

395 messages,

396 stream=stream,

397 media=media,

398 api_key=api_key,

399 **kwargs

528 model, messages, stream=stream, media=media, api_key=api_key, **kwargs

400 529 ):

401 530 yield chunk

402 531

403 532 async_create_function = create_async_generator

404 533

534

405 535 # Clean model names function

406 536 def clean_name(name: str) -> str:

407 537 name = name.split("/")[-1].split(":")[0].lower()

408 538 # Date patterns

409 name = re.sub(r'-\d{4}-\d{2}-\d{2}', '', name)

539 name = re.sub(r"-\d{4}-\d{2}-\d{2}", "", name)

410 540 # name = re.sub(r'-\d{3,8}', '', name)

411 name = re.sub(r'-\d{2}-\d{2}', '', name)

412 name = re.sub(r'-[0-9a-f]{8}$', '', name)

541 name = re.sub(r"-\d{2}-\d{2}", "", name)

542 name = re.sub(r"-[0-9a-f]{8}$", "", name)

413 543 # Version patterns

414 name = re.sub(r'-(instruct|preview|experimental|v\d+|fp8|bf16|hf|free|tput)$', '', name)

544 name = re.sub(

545 r"-(instruct|preview|experimental|v\d+|fp8|bf16|hf|free|tput)$", "", name

546 )

415 547 # Other replacements

416 548 name = name.replace("_", ".")

417 549 name = name.replace("c4ai-", "")

@@ -430,6 +562,7 @@ def clean_name(name: str) -> str:

430 562 name = name.replace("claude-sonnet-4.5", "claude-sonnet-4-5")

431 563 return name

432 564

565

433 566 setattr(Provider, "AnyProvider", AnyProvider)

434 567 Provider.__map__["AnyProvider"] = AnyProvider

435 568 Provider.__providers__.append(AnyProvider)

Modified g4f/providers/tool_support.py +116 -30

@@ -1,46 +1,75 @@

1 1 from __future__ import annotations

2 2

3 3 import json

4 import re

5 from typing import Optional, Union

4 6

5 7 from ..typing import AsyncResult, Messages, MediaListType

6 8 from ..client.service import get_model_and_provider

7 9 from ..client.helper import filter_json

10 from ..providers.types import ProviderType

8 11 from .base_provider import AsyncGeneratorProvider

9 12 from .response import ToolCalls, FinishReason, Usage

10 13

11 15 class ToolSupportProvider(AsyncGeneratorProvider):

12 16 working = True

13 17

14 @classmethod

18 @staticmethod

15 19 async def create_async_generator(

16 cls,

17 20 model: str,

18 21 messages: Messages,

19 22 stream: bool = True,

20 23 media: MediaListType = None,

21 tools: list[str] = None,

24 tools: list = None,

25 tool_choice: Optional[Union[str, dict]] = None,

22 26 response_format: dict = None,

23 **kwargs

27 provider: Optional[Union[ProviderType, str]] = None,

28 **kwargs,

24 29 ) -> AsyncResult:

25 provider = None

26 if ":" in model:

30 if provider is None and ":" in model:

27 31 provider, model = model.split(":", 1)

28 32 model, provider = get_model_and_provider(

29 model, provider,

30 stream, logging=False,

31 has_images=media is not None

33 model, provider, stream, logging=False, has_images=media is not None

32 34 )

33 if tools is not None:

34 if len(tools) > 1:

35 raise ValueError("Only one tool is supported.")

35 tool_names: list[str] = []

36 tool_schemas: dict[str, dict] = {}

37 if tools:

38 # Tool emulation: ask for a tool call plan in strict JSON.

36 39 if response_format is None:

37 40 response_format = {"type": "json"}

38 tools = tools.pop()

39 lines = ["Respone in JSON format."]

40 properties = tools["function"]["parameters"]["properties"]

41 properties = {key: value["type"] for key, value in properties.items()}

42 lines.append(f"Response format: {json.dumps(properties, indent=2)}")

43 messages = [{"role": "user", "content": "\n".join(lines)}] + messages

42 tool_defs = tools if isinstance(tools, list) else []

43 for t in tool_defs:

44 if not isinstance(t, dict) or t.get("type") != "function":

45 continue

46 fn = t.get("function")

47 if not isinstance(fn, dict):

48 continue

49 name = fn.get("name")

50 if not isinstance(name, str) or not name:

51 continue

52 tool_names.append(name)

53 params = fn.get("parameters")

54 if isinstance(params, dict):

55 tool_schemas[name] = params

57 if tool_names:

58 lines = [

59 "If you need to use tools, respond with ONLY valid JSON (no markdown).",

60 "Format:",

61 '{"tool_calls": [{"name": "TOOL_NAME", "arguments": {}}]}',

62 "You may include multiple tool calls in the array.",

63 "If no tool is needed, respond normally with plain text.",

64 f"Available tools: {', '.join(tool_names)}",

65 ]

66 if tool_choice is not None:

67 lines.append(f"Tool choice: {tool_choice}")

68 if tool_schemas:

69 lines.append(

70 f"Tool schemas: {json.dumps(tool_schemas, ensure_ascii=True)}"

71 )

72 messages = [{"role": "system", "content": "\n".join(lines)}] + messages

44 73

45 74 finish = None

46 75 chunks = []

@@ -51,7 +80,7 @@ class ToolSupportProvider(AsyncGeneratorProvider):

51 80 stream=stream,

52 81 media=media,

53 82 response_format=response_format,

54 **kwargs

83 **kwargs,

55 84 ):

56 85 if isinstance(chunk, str):

57 86 chunks.append(chunk)

@@ -68,16 +97,73 @@ class ToolSupportProvider(AsyncGeneratorProvider):

68 97 yield Usage(completion_tokens=len(chunks), total_tokens=len(chunks))

69 98

70 99 chunks = "".join(chunks)

71 if tools is not None:

72 yield ToolCalls([{

73 "id": "",

74 "type": "function",

75 "function": {

76 "name": tools["function"]["name"],

77 "arguments": filter_json(chunks)

78 }

79 }])

80 yield chunks

81 100

101 if tool_names:

102 payload = filter_json(chunks)

103

104 def parse_json_maybe(s: str):

105 if not s:

106 return None

107 try:

108 return json.loads(s)

109 except Exception:

110 pass

111 m = None

112 if "{" in s and "}" in s:

113 m = re.search(r"\{[\s\S]*\}", s)

114 if m is None and "[" in s and "]" in s:

115 m = re.search(r"\[[\s\S]*\]", s)

116 if not m:

117 return None

118 try:

119 return json.loads(m.group(0))

120 except Exception:

121 return None

122

123 obj = parse_json_maybe(payload)

124 calls = None

125 if isinstance(obj, dict) and isinstance(obj.get("tool_calls"), list):

126 calls = obj.get("tool_calls")

127 elif isinstance(obj, dict) and ("name" in obj or "tool" in obj):

128 calls = [obj]

129 elif isinstance(obj, list):

130 calls = obj

131

132 openai_calls = []

133 if isinstance(calls, list):

134 idx = 0

135 for c in calls:

136 if not isinstance(c, dict):

137 continue

138 name = c.get("name") or c.get("tool")

139 if not isinstance(name, str) or not name or name not in tool_names:

140 continue

141 args = c.get("arguments")

142 if isinstance(args, str):

143 arguments_str = args

144 else:

145 try:

146 arguments_str = json.dumps(

147 args if isinstance(args, dict) else {},

148 ensure_ascii=True,

149 )

150 except Exception:

151 arguments_str = "{}"

152 idx += 1

153 openai_calls.append(

154 {

155 "id": f"call_{idx}",

156 "type": "function",

157 "function": {"name": name, "arguments": arguments_str},

158 }

159 )

160

161 if openai_calls:

162 yield ToolCalls(openai_calls)

163 yield FinishReason("tool_calls")

164 return

165

166 if chunks:

167 yield chunks

82 168 if finish is not None:

83 yield finish

169 yield finish

Modified g4f/tools/run_tools.py +205 -72

@@ -2,6 +2,7 @@ from __future__ import annotations

2 2

3 3 import re

4 4 import json

5 import os

5 6 import math

6 7 import asyncio

7 8 import time

@@ -11,13 +12,14 @@ from typing import Optional, AsyncIterator, Iterator, Dict, Any, Tuple, List, Un

11 12

12 13 try:

13 14 from aiofile import async_open

14 16 has_aiofile = True

15 17 except ImportError:

16 18 has_aiofile = False

17 19

18 20 from ..typing import Messages

19 21 from ..providers.helper import filter_none

20 from ..providers.asyncio import to_async_iterator

22 from ..providers.asyncio import to_async_iterator, to_sync_generator

21 23 from ..providers.response import Reasoning, FinishReason, Sources, Usage, ProviderInfo

22 24 from ..providers.types import ProviderType

23 25 from ..cookies import get_cookies_dir

@@ -34,12 +36,13 @@ Instruction: Make sure to add the sources of cites using [[domain]](Url) notatio

34 36 TOOL_NAMES = {

35 37 "SEARCH": "search_tool",

36 38 "CONTINUE": "continue_tool",

37 "BUCKET": "bucket_tool"

39 "BUCKET": "bucket_tool",

38 40 }

39 41

40 43 class ToolHandler:

41 44 """Handles processing of different tool types"""

43 46 @staticmethod

44 47 def validate_arguments(data: dict) -> dict:

45 48 """Validate and parse tool arguments"""

@@ -47,25 +50,28 @@ class ToolHandler:

47 50 if isinstance(data["arguments"], str):

48 51 data["arguments"] = json.loads(data["arguments"])

49 52 if not isinstance(data["arguments"], dict):

50 raise ValueError("Tool function arguments must be a dictionary or a json string")

53 raise ValueError(

54 "Tool function arguments must be a dictionary or a json string"

55 )

51 56 else:

52 57 return filter_none(**data["arguments"])

53 58 else:

54 59 return {}

56 61 @staticmethod

57 62 async def process_search_tool(messages: Messages, tool: dict) -> Messages:

58 63 """Process search tool requests"""

59 64 messages = messages.copy()

60 65 args = ToolHandler.validate_arguments(tool["function"])

61 66 messages[-1]["content"], sources = await do_search(

62 messages[-1]["content"],

63 **args

67 messages[-1]["content"], **args

64 68 )

65 69 return messages, sources

67 71 @staticmethod

68 def process_continue_tool(messages: Messages, tool: dict, provider: Any) -> Tuple[Messages, Dict[str, Any]]:

72 def process_continue_tool(

73 messages: Messages, tool: dict, provider: Any

74 ) -> Tuple[Messages, Dict[str, Any]]:

69 75 """Process continue tool requests"""

70 76 kwargs = {}

71 77 if provider not in ("OpenaiAccount", "HuggingFaceAPI"):

@@ -77,32 +83,36 @@ class ToolHandler:

77 83 # Enable provider native continue

78 84 kwargs["action"] = "continue"

79 85 return messages, kwargs

81 87 @staticmethod

82 88 def process_bucket_tool(messages: Messages, tool: dict) -> Messages:

83 89 """Process bucket tool requests"""

84 90 messages = messages.copy()

86 92 def on_bucket(match):

87 93 return "".join(read_bucket(get_bucket_dir(match.group(1))))

89 95 has_bucket = False

90 96 for message in messages:

91 97 if "content" in message and isinstance(message["content"], str):

92 new_message_content = re.sub(r'{"bucket_id":\s*"([^"]*)"}', on_bucket, message["content"])

98 new_message_content = re.sub(

99 r'{"bucket_id":\s*"([^"]*)"}', on_bucket, message["content"]

100 )

93 101 if new_message_content != message["content"]:

94 102 has_bucket = True

95 103 message["content"] = new_message_content

96 104

97 last_message_content = messages[-1]["content"]

105 last_message_content = messages[-1]["content"]

98 106 if has_bucket and isinstance(last_message_content, str):

99 107 if "\nSource: " in last_message_content:

100 108 messages[-1]["content"] = last_message_content + BUCKET_INSTRUCTIONS

101

109

102 110 return messages

103 111

104 112 @staticmethod

105

    async def process_tools(messages: Messages, tool_calls: List[dict], provider: Any) -> Tuple[Messages, Dict[str, Any]]:

113 async def process_tools(

114 messages: Messages, tool_calls: List[dict], provider: Any

115 ) -> Tuple[Messages, Dict[str, Any]]:

106 116 """Process all tool calls and return updated messages and kwargs"""

107 117 if not tool_calls:

108 118 return messages, {}

@@ -119,10 +129,14 @@ class ToolHandler:

119 129

120 130 debug.log(f"Processing tool call: {function_name}")

121 131 if function_name == TOOL_NAMES["SEARCH"]:

122 messages, sources = await ToolHandler.process_search_tool(messages, tool)

132 messages, sources = await ToolHandler.process_search_tool(

133 messages, tool

134 )

123 135

124 136 elif function_name == TOOL_NAMES["CONTINUE"]:

125 messages, kwargs = ToolHandler.process_continue_tool(messages, tool, provider)

137 messages, kwargs = ToolHandler.process_continue_tool(

138 messages, tool, provider

139 )

126 140 extra_kwargs.update(kwargs)

127 141

128 142 elif function_name == TOOL_NAMES["BUCKET"]:

@@ -130,27 +144,30 @@ class ToolHandler:

130 144

131 145 return messages, sources, extra_kwargs

132 146

147

133 148 class ThinkingProcessor:

134 149 """Processes thinking chunks"""

135

150

136 151 @staticmethod

137 def process_thinking_chunk(chunk: str, start_time: float = 0) -> Tuple[float, List[Union[str, Reasoning]]]:

152 def process_thinking_chunk(

153 chunk: str, start_time: float = 0

154 ) -> Tuple[float, List[Union[str, Reasoning]]]:

138 155 """Process a thinking chunk and return timing and results."""

139 156 results = []

140

157

141 158 # Handle non-thinking chunk

142 159 if not start_time and "<think>" not in chunk and "</think>" not in chunk:

143 160 return 0, [chunk]

144

161

145 162 # Handle thinking start

146 163 if "<think>" in chunk and "`<think>`" not in chunk:

147 164 before_think, *after = chunk.split("<think>", 1)

148

165

149 166 if before_think:

150 167 results.append(before_think)

151

168

152 169 results.append(Reasoning(status="🤔 Is thinking...", is_thinking="<think>"))

153

170

154 171 if after:

155 172 if "</think>" in after[0]:

156 173 after, *after_end = after[0].split("</think>", 1)

@@ -161,45 +178,55 @@ class ThinkingProcessor:

161 178 return 0, results

162 179 else:

163 180 results.append(Reasoning(after[0]))

164

181

165 182 return time.time(), results

166

183

167 184 # Handle thinking end

168 185 if "</think>" in chunk:

169 186 before_end, *after = chunk.split("</think>", 1)

170

187

171 188 if before_end:

172 189 results.append(Reasoning(before_end))

173

190

174 191 thinking_duration = time.time() - start_time if start_time > 0 else 0

175 192

176 status = f"Thought for {thinking_duration:.2f}s" if thinking_duration > 1 else ""

193 status = (

194 f"Thought for {thinking_duration:.2f}s" if thinking_duration > 1 else ""

195 )

177 196 results.append(Reasoning(status=status, is_thinking="</think>"))

178 197

179 198 # Make sure to handle text after the closing tag

180 199 if after and after[0].strip():

181 200 results.append(after[0])

182

201

183 202 return 0, results

184

203

185 204 # Handle ongoing thinking

186 205 if start_time:

187 206 return start_time, [Reasoning(chunk)]

188

207

189 208 return start_time, [chunk]

190 209

191 210

192 async def perform_web_search(messages: Messages, web_search_param: Any) -> Tuple[Messages, Optional[Sources]]:

211 async def perform_web_search(

212 messages: Messages, web_search_param: Any

213 ) -> Tuple[Messages, Optional[Sources]]:

193 214 """Perform web search and return updated messages and sources"""

194 215 messages = messages.copy()

195 216 sources = None

196

217

197 218 if not web_search_param:

198 219 return messages, sources

199

220

200 221 try:

201 search_query = web_search_param if isinstance(web_search_param, str) and web_search_param != "true" else None

202 messages[-1]["content"], sources = await do_search(messages[-1]["content"], search_query)

222 search_query = (

223 web_search_param

224 if isinstance(web_search_param, str) and web_search_param != "true"

225 else None

226 )

227 messages[-1]["content"], sources = await do_search(

228 messages[-1]["content"], search_query

229 )

203 230 except Exception as e:

204 231 debug.error(f"Couldn't do web search:", e)

205 232

@@ -207,16 +234,49 @@ async def perform_web_search(messages: Messages, web_search_param: Any) -> Tuple

207 234

208 235

209 236 async def async_iter_run_tools(

210 provider: ProviderType,

211 model: str,

212 messages: Messages,

213 tool_calls: Optional[List[dict]] = None,

214 **kwargs

237 provider: ProviderType,

238 model: str,

239 messages: Messages,

240 tool_calls: Optional[List[dict]] = None,

241 **kwargs,

215 242 ) -> AsyncIterator:

216 243 """Asynchronously run tools and yield results"""

244

245 tool_emulation = kwargs.pop("tool_emulation", None)

246 if tool_emulation is None:

247 tool_emulation = os.environ.get("G4F_TOOL_EMULATION", "").strip().lower() in (

248 "1",

249 "true",

250 "yes",

251 )

252

253 stream = bool(kwargs.get("stream"))

254 tools = kwargs.get("tools")

255 if tool_emulation and tools and not tool_calls:

256 from ..providers.tool_support import ToolSupportProvider

257

258 emu_kwargs = dict(kwargs)

259 emu_kwargs.pop("tools", None)

260 tool_choice = emu_kwargs.pop("tool_choice", None)

261 emu_kwargs.pop("parallel_tool_calls", None)

262 emu_kwargs.pop("stream", None)

263 emu_kwargs.pop("stream_timeout", None)

264 async for chunk in ToolSupportProvider.create_async_generator(

265 model=model,

266 messages=messages,

267 stream=stream,

268 media=kwargs.get("media"),

269 tools=tools,

270 tool_choice=tool_choice,

271 provider=provider,

272 **emu_kwargs,

273 ):

274 yield chunk

275 return

276

217 277 # Process web search

218 278 sources = None

219 web_search = kwargs.get('web_search')

279 web_search = kwargs.get("web_search")

220 280 if web_search:

221 281 debug.log(f"Performing web search with value: {web_search}")

222 282 messages, sources = await perform_web_search(messages, web_search)

@@ -226,15 +286,19 @@ async def async_iter_run_tools(

226 286 api_key = AuthManager.load_api_key(provider)

227 287 if api_key:

228 288 kwargs["api_key"] = api_key

229

289

230 290 # Process tool calls

231 291 if tool_calls:

232 messages, sources, extra_kwargs = await ToolHandler.process_tools(messages, tool_calls, provider)

292 messages, sources, extra_kwargs = await ToolHandler.process_tools(

293 messages, tool_calls, provider

294 )

233 295 kwargs.update(extra_kwargs)

234

296

235 297 # Generate response

236 response = to_async_iterator(provider.async_create_function(model=model, messages=messages, **kwargs))

237

298 response = to_async_iterator(

299 provider.async_create_function(model=model, messages=messages, **kwargs)

300 )

301

238 302 try:

239 303 usage_model = model

240 304 usage_provider = provider.__name__

@@ -250,7 +314,7 @@ async def async_iter_run_tools(

250 314 elif isinstance(chunk, Sources):

251 315 sources = None

252 316 elif isinstance(chunk, str):

253 completion_tokens += round(len(chunk.encode("utf-8"))/4)

317 completion_tokens += round(len(chunk.encode("utf-8")) / 4)

254 318 elif isinstance(chunk, ProviderInfo):

255 319 usage_model = getattr(chunk, "model", usage_model)

256 320 usage_provider = getattr(chunk, "name", usage_provider)

@@ -260,7 +324,12 @@ async def async_iter_run_tools(

260 324 if usage is None:

261 325 usage = get_usage(messages, completion_tokens)

262 326 yield usage

263 usage = {"user": kwargs.get("user"), "model": usage_model, "provider": usage_provider, **usage.get_dict()}

327 usage = {

328 "user": kwargs.get("user"),

329 "model": usage_model,

330 "provider": usage_provider,

331 **usage.get_dict(),

332 }

264 333 usage_dir = Path(get_cookies_dir()) / ".usage"

265 334 usage_file = usage_dir / f"{datetime.date.today()}.jsonl"

266 335 usage_dir.mkdir(parents=True, exist_ok=True)

@@ -280,28 +349,69 @@ async def async_iter_run_tools(

280 349 if sources is not None:

281 350 yield sources

282 351

352

283 353 def iter_run_tools(

284 354 provider: ProviderType,

285 355 model: str,

286 356 messages: Messages,

287 357 tool_calls: Optional[List[dict]] = None,

288 **kwargs

358 **kwargs,

289 359 ) -> Iterator:

290 360 """Run tools synchronously and yield results"""

361

362 tool_emulation = kwargs.pop("tool_emulation", None)

363 if tool_emulation is None:

364 tool_emulation = os.environ.get("G4F_TOOL_EMULATION", "").strip().lower() in (

365 "1",

366 "true",

367 "yes",

368 )

369

370 stream = bool(kwargs.get("stream"))

371 tools = kwargs.get("tools")

372 if tool_emulation and tools and not tool_calls:

373 from ..providers.tool_support import ToolSupportProvider

374

375 emu_kwargs = dict(kwargs)

376 emu_kwargs.pop("tools", None)

377 tool_choice = emu_kwargs.pop("tool_choice", None)

378 emu_kwargs.pop("parallel_tool_calls", None)

379 emu_kwargs.pop("stream", None)

380 emu_kwargs.pop("stream_timeout", None)

381 yield from to_sync_generator(

382 ToolSupportProvider.create_async_generator(

383 model=model,

384 messages=messages,

385 stream=stream,

386 media=kwargs.get("media"),

387 tools=tools,

388 tool_choice=tool_choice,

389 provider=provider,

390 **emu_kwargs,

391 ),

392 stream=stream,

393 )

394 return

291 395 # Process web search

292 web_search = kwargs.get('web_search')

396 web_search = kwargs.get("web_search")

293 397 sources = None

294

398

295 399 if web_search:

296 400 debug.log(f"Performing web search with value: {web_search}")

297 401 try:

298 402 messages = messages.copy()

299 search_query = web_search if isinstance(web_search, str) and web_search != "true" else None

403 search_query = (

404 web_search

405 if isinstance(web_search, str) and web_search != "true"

406 else None

407 )

300 408 # Note: Using asyncio.run inside sync function is not ideal, but maintaining original pattern

301 messages[-1]["content"], sources = asyncio.run(do_search(messages[-1]["content"], search_query))

409 messages[-1]["content"], sources = asyncio.run(

410 do_search(messages[-1]["content"], search_query)

411 )

302 412 except Exception as e:

303 413 debug.error(f"Couldn't do web search:", e)

304

414

305 415 # Get API key if needed

306 416 if not kwargs.get("api_key"):

307 417 api_key = AuthManager.load_api_key(provider)

@@ -315,11 +425,13 @@ def iter_run_tools(

315 425 function_name = tool.get("function", {}).get("name")

316 426 debug.log(f"Processing tool call: {function_name}")

317 427 if function_name == TOOL_NAMES["SEARCH"]:

318 tool["function"]["arguments"] = ToolHandler.validate_arguments(tool["function"])

428 tool["function"]["arguments"] = ToolHandler.validate_arguments(

429 tool["function"]

430 )

319 431 messages[-1]["content"] = get_search_message(

320 432 messages[-1]["content"],

321 433 raise_search_exceptions=True,

322 **tool["function"]["arguments"]

434 **tool["function"]["arguments"],

323 435 )

324 436 elif function_name == TOOL_NAMES["CONTINUE"]:

325 437 if provider.__name__ not in ("OpenaiAccount", "HuggingFace"):

@@ -330,12 +442,18 @@ def iter_run_tools(

330 442 # Enable provider native continue

331 443 kwargs["action"] = "continue"

332 444 elif function_name == TOOL_NAMES["BUCKET"]:

445

333 446 def on_bucket(match):

334 447 return "".join(read_bucket(get_bucket_dir(match.group(1))))

448

335 449 has_bucket = False

336 450 for message in messages:

337 451 if "content" in message and isinstance(message["content"], str):

338 new_message_content = re.sub(r'{"bucket_id":"([^"]*)"}', on_bucket, message["content"])

452 new_message_content = re.sub(

453 r'{"bucket_id":"([^"]*)"}',

454 on_bucket,

455 message["content"],

456 )

339 457 if new_message_content != message["content"]:

340 458 has_bucket = True

341 459 message["content"] = new_message_content

@@ -343,7 +461,7 @@ def iter_run_tools(

343 461 if has_bucket and isinstance(last_message, str):

344 462 if "\nSource: " in last_message:

345 463 messages[-1]["content"] = last_message + BUCKET_INSTRUCTIONS

346

464

347 465 # Process response chunks

348 466 try:

349 467 thinking_start_time = 0

@@ -352,7 +470,9 @@ def iter_run_tools(

352 470 usage_provider = provider.__name__

353 471 completion_tokens = 0

354 472 usage = None

355 for chunk in provider.create_function(model=model, messages=messages, provider=provider, **kwargs):

473 for chunk in provider.create_function(

474 model=model, messages=messages, provider=provider, **kwargs

475 ):

356 476 if isinstance(chunk, FinishReason):

357 477 if sources is not None:

358 478 yield sources

@@ -362,7 +482,7 @@ def iter_run_tools(

362 482 elif isinstance(chunk, Sources):

363 483 sources = None

364 484 elif isinstance(chunk, str):

365 completion_tokens += round(len(chunk.encode("utf-8"))/4)

485 completion_tokens += round(len(chunk.encode("utf-8")) / 4)

366 486 elif isinstance(chunk, ProviderInfo):

367 487 usage_model = getattr(chunk, "model", usage_model)

368 488 usage_provider = getattr(chunk, "name", usage_provider)

@@ -371,14 +491,21 @@ def iter_run_tools(

371 491 if not isinstance(chunk, str):

372 492 yield chunk

373 493 continue

374

375 thinking_start_time, results = processor.process_thinking_chunk(chunk, thinking_start_time)

494

495 thinking_start_time, results = processor.process_thinking_chunk(

496 chunk, thinking_start_time

497 )

376 498 for result in results:

377 499 yield result

378 500 if usage is None:

379 501 usage = get_usage(messages, completion_tokens)

380 502 yield usage

381 usage = {"user": kwargs.get("user"), "model": usage_model, "provider": usage_provider, **usage.get_dict()}

503 usage = {

504 "user": kwargs.get("user"),

505 "model": usage_model,

506 "provider": usage_provider,

507 **usage.get_dict(),

508 }

382 509 usage_dir = Path(get_cookies_dir()) / ".usage"

383 510 usage_file = usage_dir / f"{datetime.date.today()}.jsonl"

384 511 usage_dir.mkdir(parents=True, exist_ok=True)

@@ -393,26 +520,32 @@ def iter_run_tools(

393 520 if sources is not None:

394 521 yield sources

395 522

523

396 524 def caculate_prompt_tokens(messages: Messages) -> int:

397 525 """Calculate the total number of tokens in messages"""

398 token_count = 1 # Bos Token

526 token_count = 1 # Bos Token

399 527 for message in messages:

400 528 if isinstance(message.get("content"), str):

401 529 token_count += math.floor(len(message["content"].encode("utf-8")) / 4)

402 token_count += 4 # Role and start/end message token

530 token_count += 4 # Role and start/end message token

403 531 elif isinstance(message.get("content"), list):

404 532 for item in message["content"]:

405 533 if isinstance(item, str):

406 534 token_count += math.floor(len(item.encode("utf-8")) / 4)

407 elif isinstance(item, dict) and "text" in item and isinstance(item["text"], str):

535 elif (

536 isinstance(item, dict)

537 and "text" in item

538 and isinstance(item["text"], str)

539 ):

408 540 token_count += math.floor(len(item["text"].encode("utf-8")) / 4)

409 token_count += 4 # Role and start/end message token

541 token_count += 4 # Role and start/end message token

410 542 return token_count

411 543

544

412 545 def get_usage(messages: Messages, completion_tokens: int) -> Usage:

413 546 prompt_tokens = caculate_prompt_tokens(messages)

414 547 return Usage(

415 548 completion_tokens=completion_tokens,

416 549 prompt_tokens=prompt_tokens,

417 total_tokens=prompt_tokens + completion_tokens

418 )

550 total_tokens=prompt_tokens + completion_tokens,

551 )