gpt4free

公开

关注 0 Fork 0 Star 0

返回提交历史

XFEstudio/gpt4free

Add token_optimizer, add /v1/messages and /v1/responses API

da241e2c

hlohaus <hlohaus@users.noreply.github.com>

提交于 2026-07-13 23:10

父提交 5f2bbd8

代码差异

7 个文件 +690 -4

Modified g4f/api/__init__.py +235 -2

@@ -81,6 +81,7 @@ from g4f.Provider import ProviderUtils

81 81 from g4f.gui import get_gui_app

82 82 from .stubs import (

83 83 ChatCompletionsConfig, ImageGenerationConfig,

84 ResponsesConfig, MessagesConfig,

84 85 ProviderResponseModel, ModelResponseModel,

85 86 ErrorResponseModel, ProviderResponseDetailModel,

86 87 FileResponseModel,

@@ -540,7 +541,9 @@ class Api:

540 541 return ErrorResponse.from_message("Invalid G4F API key", HTTP_403_FORBIDDEN)

541 542

                    elif path.startswith("/backend-api/") or path.startswith("/chat/") or path.startswith("/playground/") or path in ["/logs"]:

542 543 try:

543 user = await self.get_username(request)

544 new_user = await self.get_username(request)

545 if user is None:

546 user = new_user

544 547 except HTTPException as e:

545 548 return ErrorResponse.from_message(e.detail, e.status_code, e.headers)

546 549 if user_g4f_api_key and update_authorization:

@@ -580,7 +583,9 @@ class Api:

580 583 async def read_root_v1():

581 584 return HTMLResponse('g4f API: Go to '

582 585 '<a href="/v1/models">models</a>, '

583 '<a href="/v1/chat/completions">chat/completions</a>, or '

586 '<a href="/v1/chat/completions">chat/completions</a>, '

587 '<a href="/v1/responses">responses</a> (OpenAI), '

588 '<a href="/v1/messages">messages</a> (Anthropic), or '

584 589 '<a href="/v1/media/generate">media/generate</a> <br><br>'

585 590 'Open Swagger UI at: '

586 591 '<a href="/docs">/docs</a>')

@@ -809,6 +814,234 @@ class Api:

809 814 logger.exception(e)

810 815 return ErrorResponse.from_exception(e, config, HTTP_500_INTERNAL_SERVER_ERROR)

811 816

817 # ------------------------------------------------------------------ #

818 # OpenAI Responses API (/v1/responses) #

819 # https://platform.openai.com/docs/api-reference/responses #

820 # ------------------------------------------------------------------ #

821 @self.app.post("/v1/responses", responses=responses)

822 @self.app.post("/api/{provider:path}/responses", responses=responses)

823 async def create_response(

824 config: ResponsesConfig,

825 credentials: Annotated[HTTPAuthorizationCredentials, Depends(Api.security)] = None,

826 provider: str = None,

827 x_user: Annotated[str | None, Header()] = None,

828 ):

829 if provider is not None:

830 config.provider = provider

831 if config.provider is None:

832 config.provider = AppConfig.provider

833 try:

834 provider = AbstractClientFactory.create_provider(None, config.provider)

835 except ProviderNotFoundError as e:

836 return ErrorResponse.from_message(str(e), 404)

837 try:

838 if config.timeout is None:

839 config.timeout = AppConfig.timeout

840 if config.stream_timeout is None and config.stream:

841 config.stream_timeout = AppConfig.stream_timeout

842 if credentials is not None and credentials.credentials != "secret":

843 config.api_key = credentials.credentials

844

845 # Normalize `input` into a messages list.

846 messages = config.input

847 if isinstance(messages, str):

848 messages = [{"role": "user", "content": messages}]

849 if config.instructions:

850 messages = [{"role": "system", "content": config.instructions}, *messages]

851

852 response = self.client.chat.completions.create(

853 **filter_none(

854 **{

855 "model": AppConfig.model,

856 "provider": AppConfig.provider,

857 "proxy": AppConfig.proxy,

858

                            **(config.model_dump(exclude_none=True) if hasattr(config, "model_dump") else config.dict(exclude_none=True)),

859 **{

860 "provider": provider,

861 "messages": messages,

862 "user": x_user,

863 }

864 },

865 ignored=AppConfig.ignored_providers

866 ),

867 )

868

869 if not config.stream:

870 result = await response

871 text = result.choices[0].message.content if result.choices else ""

872 usage = getattr(result, "usage", None)

873 if usage is not None and hasattr(usage, "model_dump"):

874 usage = usage.model_dump()

875 elif usage is not None and hasattr(usage, "dict"):

876 usage = usage.dict()

877 return JSONResponse({

878 "id": getattr(result, "id", f"resp_{secrets.token_hex(12)}"),

879 "object": "response",

880 "created_at": getattr(result, "created", int(time.time())),

881 "model": getattr(result, "model", config.model),

882 "provider": getattr(provider, "__name__", config.provider),

883 "output": [

884 {

885 "type": "message",

886 "role": "assistant",

887 "content": [{"type": "output_text", "text": text}],

888 }

889 ],

890 "output_text": text,

891 "usage": usage,

892 })

893

894 first_chunk = await response.__anext__()

895 async def responses_streaming():

896

                    yield f"data: {first_chunk.model_dump_json() if hasattr(first_chunk, 'model_dump_json') else first_chunk.json()}\n\n"

897 try:

898 async for chunk in response:

899 if isinstance(chunk, BaseConversation):

900 pass

901 else:

902

                                yield f"data: {chunk.model_dump_json() if hasattr(chunk, 'model_dump_json') else chunk.json()}\n\n"

903 except GeneratorExit:

904 pass

905 except RateLimitError as e:

906 debug.error(e)

907 yield f'data: {format_exception(e, config)}\n\n'

908 except Exception as e:

909 logger.exception(e)

910 yield f'data: {format_exception(e, config)}\n\n'

911 yield "data: [DONE]\n\n"

912 headers = getattr(first_chunk, "_headers").get_dict() if hasattr(first_chunk, "_headers") else {}

913

                headers = {k.encode("latin-1","ignore").decode("latin-1"): v.encode("latin-1","ignore").decode("latin-1") for k, v in headers.items()}

914 return StreamingResponse(

915 responses_streaming(),

916 media_type="text/event-stream",

917 headers=headers

918 )

919 except (ModelNotFoundError, ProviderNotFoundError) as e:

920 logger.exception(e)

921 return ErrorResponse.from_exception(e, config, HTTP_404_NOT_FOUND)

922 except (MissingAuthError, NoValidHarFileError) as e:

923 logger.exception(e)

924 return ErrorResponse.from_exception(e, config, HTTP_401_UNAUTHORIZED)

925 except RateLimitError as e:

926 return ErrorResponse.from_exception(e, config, HTTP_429_TOO_MANY_REQUESTS)

927 except Exception as e:

928 logger.exception(e)

929 return ErrorResponse.from_exception(e, config, HTTP_500_INTERNAL_SERVER_ERROR)

930

931 # ------------------------------------------------------------------ #

932 # Anthropic Messages API (/v1/messages) #

933 # https://docs.anthropic.com/en/api/messages #

934 # ------------------------------------------------------------------ #

935 @self.app.post("/v1/messages", responses=responses)

936 @self.app.post("/api/{provider:path}/messages", responses=responses)

937 async def create_message(

938 config: MessagesConfig,

939 credentials: Annotated[HTTPAuthorizationCredentials, Depends(Api.security)] = None,

940 provider: str = None,

941 x_user: Annotated[str | None, Header()] = None,

942 ):

943 if provider is not None:

944 config.provider = provider

945 if config.provider is None:

946 config.provider = AppConfig.provider

947 try:

948 provider = AbstractClientFactory.create_provider(None, config.provider)

949 except ProviderNotFoundError as e:

950 return ErrorResponse.from_message(str(e), 404)

951 try:

952 if config.timeout is None:

953 config.timeout = AppConfig.timeout

954 if config.stream_timeout is None and config.stream:

955 config.stream_timeout = AppConfig.stream_timeout

956 if credentials is not None and credentials.credentials != "secret":

957 config.api_key = credentials.credentials

958

959 # Anthropic uses a top-level `system` field; fold it into messages.

960 messages = config.messages

961 if config.system:

962 system_content = config.system

963 if isinstance(system_content, list):

964 system_content = " ".join(

965 b.get("text", "") if isinstance(b, dict) else str(b)

966 for b in system_content

967 )

968 messages = [{"role": "system", "content": system_content}, *messages]

969

970 response = self.client.chat.completions.create(

971 **filter_none(

972 **{

973 "model": AppConfig.model,

974 "provider": AppConfig.provider,

975 "proxy": AppConfig.proxy,

976

                            **(config.model_dump(exclude_none=True) if hasattr(config, "model_dump") else config.dict(exclude_none=True)),

977 **{

978 "provider": provider,

979 "messages": messages,

980 "user": x_user,

981 }

982 },

983 ignored=AppConfig.ignored_providers

984 ),

985 )

986

987 if not config.stream:

988 result = await response

989 text = result.choices[0].message.content if result.choices else ""

990 usage = getattr(result, "usage", None)

991 input_tokens = getattr(usage, "prompt_tokens", 0) or 0

992 output_tokens = getattr(usage, "completion_tokens", 0) or 0

993 return JSONResponse({

994 "id": getattr(result, "id", f"msg_{secrets.token_hex(12)}"),

995 "type": "message",

996 "role": "assistant",

997 "model": getattr(result, "model", config.model),

998 "provider": getattr(provider, "__name__", config.provider),

999 "content": [{"type": "text", "text": text}],

1000 "stop_reason": getattr(result.choices[0], "finish_reason", None) if result.choices else None,

1001 "stop_sequence": None,

1002 "usage": {

1003 "input_tokens": input_tokens,

1004 "output_tokens": output_tokens,

1005 },

1006 })

1007

1008 first_chunk = await response.__anext__()

1009 async def messages_streaming():

1010

                    yield f"data: {first_chunk.model_dump_json() if hasattr(first_chunk, 'model_dump_json') else first_chunk.json()}\n\n"

1011 try:

1012 async for chunk in response:

1013 if isinstance(chunk, BaseConversation):

1014 pass

1015 else:

1016

                                yield f"data: {chunk.model_dump_json() if hasattr(chunk, 'model_dump_json') else chunk.json()}\n\n"

1017 except GeneratorExit:

1018 pass

1019 except RateLimitError as e:

1020 debug.error(e)

1021 yield f'data: {format_exception(e, config)}\n\n'

1022 except Exception as e:

1023 logger.exception(e)

1024 yield f'data: {format_exception(e, config)}\n\n'

1025 yield "data: [DONE]\n\n"

1026 headers = getattr(first_chunk, "_headers").get_dict() if hasattr(first_chunk, "_headers") else {}

1027

                headers = {k.encode("latin-1","ignore").decode("latin-1"): v.encode("latin-1","ignore").decode("latin-1") for k, v in headers.items()}

1028 return StreamingResponse(

1029 messages_streaming(),

1030 media_type="text/event-stream",

1031 headers=headers

1032 )

1033 except (ModelNotFoundError, ProviderNotFoundError) as e:

1034 logger.exception(e)

1035 return ErrorResponse.from_exception(e, config, HTTP_404_NOT_FOUND)

1036 except (MissingAuthError, NoValidHarFileError) as e:

1037 logger.exception(e)

1038 return ErrorResponse.from_exception(e, config, HTTP_401_UNAUTHORIZED)

1039 except RateLimitError as e:

1040 return ErrorResponse.from_exception(e, config, HTTP_429_TOO_MANY_REQUESTS)

1041 except Exception as e:

1042 logger.exception(e)

1043 return ErrorResponse.from_exception(e, config, HTTP_500_INTERNAL_SERVER_ERROR)

1044

812 1045 responses = {

813 1046 HTTP_200_OK: {"model": ImagesResponse},

814 1047 HTTP_401_UNAUTHORIZED: {"model": ErrorResponseModel},

Modified g4f/api/stubs.py +37 -0

@@ -71,6 +71,43 @@ class ChatCompletionsConfig(RequestConfig):

71 71

72 72 class ResponsesConfig(RequestConfig):

73 73 input: Union[Messages, str]

74 stream: bool = False

75 instructions: Optional[str] = None

76 previous_response_id: Optional[str] = None

79 class MessagesConfig(BaseModel):

80 """Anthropic Messages API request body."""

81 model: str = Field(default="")

82 messages: Messages = Field(

83 examples=[[{"role": "user", "content": "Hello"}]]

84 )

85 system: Optional[Union[str, list]] = None

86 max_tokens: int = 4096

87 stream: bool = False

88 temperature: Optional[float] = None

89 top_p: Optional[float] = None

90 top_k: Optional[int] = None

91 stop_sequences: Optional[list[str]] = None

92 metadata: Optional[dict] = None

93 tools: Optional[list] = None

94 tool_choice: Optional[Union[str, dict]] = None

95 provider: Optional[str] = None

96 api_key: Optional[Union[str, dict[str, str]]] = None

97 conversation: Optional[dict] = None

98 conversation_id: Optional[str] = None

99 timeout: Optional[int] = None

100 stream_timeout: Optional[int] = None

101 image: Optional[str] = None

102 image_name: Optional[str] = None

103 images: Optional[list[tuple[str, str]]] = None

104 media: Optional[list[tuple[str, str]]] = None

105 web_search: Optional[Union[str, bool]] = None

106 response_format: Optional[dict] = None

107 reasoning_effort: Optional[Literal["none", "low", "medium", "high", "x-high"]] = None

108 raw: bool = False

109 extra_body: Optional[dict] = None

110 tool_emulation: Optional[bool] = None

74 111

75 112

76 113 class ImageGenerationConfig(BaseModel):

Modified g4f/mcp/__init__.py +2 -0

@@ -27,6 +27,7 @@ from .tools import (

27 27 GrepSearchTool,

28 28 GithubRepoTool,

29 29 GithubTextSearchTool,

30 TokenOptimizerTool,

30 31 )

31 32 from .pa_provider import (

32 33 execute_safe_code,

@@ -58,6 +59,7 @@ __all__ = [

58 59 'GrepSearchTool',

59 60 'GithubRepoTool',

60 61 'GithubTextSearchTool',

62 'TokenOptimizerTool',

61 63 # PA provider system

62 64 'execute_safe_code',

63 65 'load_pa_provider',

Modified g4f/mcp/server.py +4 -1

@@ -30,8 +30,9 @@ from .tools import (

30 30 MarkItDownTool, TextToAudioTool, WebSearchTool, ImageGenerationTool,

31 31 PythonExecuteTool, FileReadTool,

32 32 FileListTool, FileDeleteTool, ApplyPatchTool,

33 CreateDirectoryTool, CreateFileTool, FetchWebpageTool,

33 CreateDirectoryTool, CreateFileTool, FileWriteTool, FetchWebpageTool,

34 34 FileSearchGlobTool, GrepSearchTool, GithubRepoTool, GithubTextSearchTool,

35 TokenOptimizerTool,

35 36 )

36 37

37 38

@@ -82,11 +83,13 @@ class MCPServer:

82 83 'file_delete': FileDeleteTool(),

83 84 'create_directory': CreateDirectoryTool(),

84 85 'create_file': CreateFileTool(),

86 'file_write': FileWriteTool(),

85 87 'fetch_webpage': FetchWebpageTool(),

86 88 'file_search_glob': FileSearchGlobTool(),

87 89 'grep_search': GrepSearchTool(),

88 90 'github_repo': GithubRepoTool(),

89 91 'github_text_search': GithubTextSearchTool(),

92 'token_optimizer': TokenOptimizerTool(),

90 93 }

91 94 self.server_info = {

92 95 "name": "gpt4free-mcp-server",

Modified g4f/mcp/tools.py +147 -1

@@ -827,6 +827,75 @@ class CreateFileTool(MCPTool):

827 827 return {"error": f"Create file failed: {exc}"}

828 828

829 829

830 class FileWriteTool(MCPTool):

831 """Write (or create) a file inside the ``~/.g4f/workspace`` directory."""

832

833 @property

834 def description(self) -> str:

835 return (

836 "Write text content to a file inside the ~/.g4f/workspace directory. "

837 "Creates parent directories as needed. Overwrites the file if it already exists. "

838 "Provide a relative path from the workspace root and the content to write."

839 )

840

841 @property

842 def input_schema(self) -> Dict[str, Any]:

843 return {

844 "type": "object",

845 "properties": {

846 "path": {

847 "type": "string",

848 "description": "Relative path to the file inside the workspace",

849 },

850 "content": {

851 "type": "string",

852 "description": "Text content to write to the file",

853 },

854 "append": {

855 "type": "boolean",

856 "description": "If true, append to existing file instead of overwriting (default: false)",

857 "default": False,

858 },

859 },

860 "required": ["path", "content"],

861 }

862

863 async def execute(self, arguments: Dict[str, Any]) -> Dict[str, Any]:

864 from .pa_provider import get_workspace_dir

865

866 rel_path = arguments.get("path", "")

867 content = arguments.get("content")

868 append = bool(arguments.get("append", False))

869

870 if not rel_path:

871 return {"error": "path parameter is required"}

872 if content is None:

873 return {"error": "content parameter is required"}

874

875 workspace = get_workspace_dir().resolve()

876 try:

877 target = (workspace / rel_path).resolve()

878 if not str(target).startswith(str(workspace)):

879 return {"error": "Access outside the workspace is not allowed"}

880 target.parent.mkdir(parents=True, exist_ok=True)

881 if append:

882 with open(target, "a", encoding="utf-8") as f:

883 f.write(content)

884 else:

885 target.write_text(content, encoding="utf-8")

886 result: Dict[str, Any] = {

887 "path": rel_path,

888 "size": len(content),

889 "appended": append,

890 }

891 origin = arguments.get("origin")

892 if origin:

893 result["url"] = f"{origin}/pa/files/{rel_path}"

894 return result

895 except Exception as exc:

896 return {"error": f"Write failed: {exc}"}

897

898

830 899 class FetchWebpageTool(MCPTool):

831 900 """Fetch and return the main content from one or more web pages."""

832 901

@@ -1250,4 +1319,81 @@ class ApplyPatchTool(MCPTool):

1250 1319 except Exception as exc:

1251 1320 return {"error": f"Invalid target path: {exc}"}

1252 1321

1253 return apply_patch_with_fallback(patch_content, str(target), backup, dry_run)

1322 return apply_patch_with_fallback(patch_content, str(target), backup, dry_run)

1323

1324

1325 class TokenOptimizerTool(MCPTool):

1326 """Optimize a prompt (messages list) to reduce token waste before sending to providers.

1327

1328 Wraps the optional `token_optimizer` plugin

1329 (https://github.com/alexgreensh/token-optimizer). When the package is not

1330 installed the tool reports that it is unavailable so callers can fall back

1331 to the built-in request optimizer.

1332 """

1333

1334 @property

1335 def description(self) -> str:

1336 return (

1337 "Optimize a prompt (OpenAI-style messages list) to cut wasted input tokens "

1338 "before it reaches AI providers. Compresses verbose system prompts, repeated "

1339 "tool output, and stale context. Requires the optional `token_optimizer` package. "

1340 "Returns the optimized messages and the number of tokens saved."

1341 )

1342

1343 @property

1344 def input_schema(self) -> Dict[str, Any]:

1345 return {

1346 "type": "object",

1347 "properties": {

1348 "messages": {

1349 "type": "array",

1350 "description": "OpenAI-style messages list to optimize.",

1351 "items": {"type": "object"},

1352 },

1353 "tools": {

1354 "type": "array",

1355 "description": "Optional list of tool definitions to compress alongside the messages.",

1356 "items": {"type": "object"},

1357 },

1358 },

1359 "required": ["messages"],

1360 }

1361

1362 async def execute(self, arguments: Dict[str, Any]) -> Dict[str, Any]:

1363 from ..tools.token_optimizer import (

1364 is_available,

1365 get_install_path,

1366 optimize_messages,

1367 )

1368

1369 if not is_available():

1370 return {

1371 "available": False,

1372 "error": (

1373 "token_optimizer package is not installed. "

1374 "Install it from https://github.com/alexgreensh/token-optimizer "

1375 "to enable prompt optimization."

1376 ),

1377 }

1378

1379 messages = arguments.get("messages")

1380 if not isinstance(messages, list) or not messages:

1381 return {"available": True, "error": "messages must be a non-empty list"}

1382

1383 tools = arguments.get("tools")

1384 # Copy so we don't mutate the caller's input across the MCP boundary.

1385 messages_copy = [dict(m) if isinstance(m, dict) else m for m in messages]

1386 tools_copy = [dict(t) if isinstance(t, dict) else t for t in tools] if isinstance(tools, list) else None

1387

1388 try:

1389 saved_tokens, logs = optimize_messages(messages_copy, tools_copy)

1390 except Exception as exc:

1391 return {"available": True, "error": f"Optimization failed: {exc}"}

1392

1393 return {

1394 "available": True,

1395 "install_path": get_install_path(),

1396 "saved_tokens": saved_tokens,

1397 "logs": logs,

1398 "optimized_messages": messages_copy,

1399 }

Modified g4f/tools/run_tools.py +19 -0

@@ -23,6 +23,7 @@ from ..providers.helper import filter_none

23 23 from ..providers.asyncio import to_sync_generator

24 24

from ..providers.response import Reasoning, FinishReason, Sources, Usage, ProviderInfo, HeadersResponse, JsonConversation

25 25 from .optimize_request import optimize_request

from .token_optimizer import optimize_messages as token_optimizer_optimize_messages, is_available as token_optimizer_available

26 27 from ..providers.types import ProviderType

27 28 from ..providers.base_provider import get_async_provider_method, get_provider_method, wait_for

28 29 from ..cookies import get_cookies_dir

@@ -342,6 +343,15 @@ async def async_iter_run_tools(

342 343 if saved_tokens:

343 344 debug.log(f"Optimized request: saved ~{saved_tokens} tokens")

344 345

346 # Optional token-optimizer plugin: compress the prompt messages before

347 # they reach the provider. Only active when the `token_optimizer` package

348 # is installed in the environment.

349 if token_optimizer_available():

350 to_saved, _to_logs = token_optimizer_optimize_messages(messages, tools_ref)

351 if to_saved:

352 saved_tokens += to_saved

353 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

354

345 355 tool_emulation = kwargs.pop("tool_emulation", None)

346 356 if tool_emulation is None:

347 357 tool_emulation = os.environ.get("G4F_TOOL_EMULATION", "").strip().lower() in (

@@ -491,6 +501,15 @@ def iter_run_tools(

491 501 if saved_tokens:

492 502 debug.log(f"Optimized request: saved ~{saved_tokens} tokens")

493 503

504 # Optional token-optimizer plugin: compress the prompt messages before

505 # they reach the provider. Only active when the `token_optimizer` package

506 # is installed in the environment.

507 if token_optimizer_available():

508 to_saved, _to_logs = token_optimizer_optimize_messages(messages, tools_ref)

509 if to_saved:

510 saved_tokens += to_saved

511 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

512

494 513 tool_emulation = kwargs.pop("tool_emulation", None)

495 514 if tool_emulation is None:

496 515 tool_emulation = os.environ.get("G4F_TOOL_EMULATION", "").strip().lower() in (

Added g4f/tools/token_optimizer.py +246 -0

@@ -0,0 +1,246 @@

1 """Token Optimizer plugin integration.

3 This module integrates the external `token-optimizer` project

4 (https://github.com/alexgreensh/token-optimizer) as an optional plugin.

6 The plugin is loaded only when the `token_optimizer` Python package is

7 installed in the environment. When available, it compresses the prompt

8 messages *before* they reach the providers, cutting wasted tokens on the

9 input side (verbose system prompts, repeated tool output, stale context).

11 Design

12 ------

13 * **Optional**: every public function fails open. If the package is not

14 installed, ``is_available()`` returns ``False`` and ``optimize_messages``

15 returns the messages unchanged with zero saved tokens.

16 * **Cache-safe**: the optimizer never modifies the existing context prefix

17 in a way that would invalidate provider prompt caches — it only trims

18 redundant content within individual messages.

19 * **Measured**: the number of tokens saved is returned so callers (e.g.

20 ``run_tools``) can accumulate it into the per-request usage log alongside

21 the built-in ``optimize_request`` savings.

23 The integration mirrors the upstream project's ``bash_compress.compress``

24 entry point but adapts it to the OpenAI-style ``Messages`` list format used

25 throughout gpt4free. When the upstream package exposes a

26 ``token_optimizer.optimize_messages`` callable, it is used directly;

27 otherwise we fall back to a vendored lightweight compressor that applies

28 the same pattern-based trimming to message ``content`` strings.

29 """

31 from __future__ import annotations

33 import os

34 import re

35 from typing import Any, Dict, List, Optional, Tuple

37 from ..typing import Messages

38 from .. import debug

41 # ---------------------------------------------------------------------------

42 # Availability detection

43 # ---------------------------------------------------------------------------

45 _AVAILABLE: Optional[bool] = None

46 _OPTIMIZE_FUNC = None # type: Optional[Any]

49 def _detect() -> Tuple[bool, Any]:

50 """Probe the environment for the token-optimizer package.

52 Returns ``(available, optimize_func)`` where ``optimize_func`` is the

53 callable to use (or ``None`` when unavailable). The result is cached on

54 the module so repeated calls are cheap.

55 """

56 global _AVAILABLE, _OPTIMIZE_FUNC

57 if _AVAILABLE is not None:

58 return _AVAILABLE, _OPTIMIZE_FUNC

60 # Allow explicit opt-out via env var.

61 if os.environ.get("G4F_TOKEN_OPTIMIZER", "").strip().lower() in ("0", "false", "no", "off"):

62 _AVAILABLE = False

63 _OPTIMIZE_FUNC = None

64 return _AVAILABLE, _OPTIMIZE_FUNC

66 optimize_func = None

67 try:

68 import token_optimizer # type: ignore # noqa: F401

70 # Prefer the documented entry point if present.

71 optimize_func = getattr(token_optimizer, "optimize_messages", None)

72 if not callable(optimize_func):

73 # Some builds expose it under a different name.

74 optimize_func = getattr(token_optimizer, "compress_messages", None)

75 if not callable(optimize_func):

76 optimize_func = None

77 _AVAILABLE = True

78 except ImportError:

79 _AVAILABLE = False

80 optimize_func = None

82 _OPTIMIZE_FUNC = optimize_func

83 return _AVAILABLE, _OPTIMIZE_FUNC

86 def is_available() -> bool:

87 """Return ``True`` when the token-optimizer package is importable."""

88 available, _ = _detect()

89 return bool(available)

92 def get_install_path() -> Optional[str]:

93 """Return the filesystem path of the installed package, if any."""

94 try:

95 import token_optimizer # type: ignore

97 return getattr(token_optimizer, "__file__", None) or getattr(

98 token_optimizer, "__path__", [None])[0]

99 except Exception:

100 return None

101

102

103 # ---------------------------------------------------------------------------

104 # Vendored fallback compressor

105 # ---------------------------------------------------------------------------

106

107 # These patterns mirror the upstream bash_compress handlers but operate on

108 # arbitrary message content. They trim the most common sources of input-side

109 # waste: repeated blank lines, verbose boilerplate, and oversized tool

110 # outputs embedded in assistant messages.

111

112 _REPEATED_BLANK = re.compile(r"\n{4,}")

113 _TRAILING_WHITESPACE = re.compile(r"[ \t]+\n")

114 _LONG_LINE_CAP = 2000 # lines longer than this get head/tail truncated

115

116

117 def _compress_content(text: str) -> Tuple[str, int]:

118 """Apply lightweight, loss-tolerant compression to a single string.

119

120 Returns ``(new_text, bytes_saved)``. Always fails open: on any error the

121 original text is returned with zero savings.

122 """

123 if not isinstance(text, str) or len(text) < 512:

124 return text, 0

125 try:

126 original_len = len(text.encode("utf-8", errors="replace"))

127 out = _TRAILING_WHITESPACE.sub("\n", text)

128 out = _REPEATED_BLANK.sub("\n\n\n", out)

129

130 # Truncate very long lines (e.g. minified bundles pasted into context)

131 # keeping the head and tail with a marker in the middle.

132 new_lines: List[str] = []

133 for line in out.splitlines():

134 if len(line) > _LONG_LINE_CAP:

135 head = line[:800]

136 tail = line[-800:]

137 omitted = len(line) - 1600

138 new_lines.append(

139 f"{head}\n... [{omitted} chars omitted] ...\n{tail}"

140 )

141 else:

142 new_lines.append(line)

143 out = "\n".join(new_lines)

144

145 new_len = len(out.encode("utf-8", errors="replace"))

146 saved = max(0, original_len - new_len)

147 # Only accept the result if it actually saved something meaningful.

148 if saved < 64:

149 return text, 0

150 return out, saved

151 except Exception:

152 return text, 0

153

154

155 # ---------------------------------------------------------------------------

156 # Public API

157 # ---------------------------------------------------------------------------

158

159 def _bytes_to_tokens(num_bytes: int) -> int:

160 """Approximate token count from byte count (~4 bytes/token)."""

161 return round(num_bytes / 4)

162

163

164 def optimize_messages(

165 messages: Messages,

166 tools: Any = None,

167 ) -> Tuple[int, Dict[str, str]]:

168 """Optimize the prompt messages in place before they reach providers.

169

170 Mutates ``messages`` (and, when supported, ``tools``) and returns

171 ``(saved_tokens, logs)``. When the token-optimizer package is not

172 installed this is a no-op returning ``(0, {})``.

173

174 Args:

175 messages: OpenAI-style messages list. Mutated in place.

176 tools: Optional list of tool definitions. When the upstream

177 optimizer supports tool compression it is applied here.

178

179 Returns:

180 Tuple of (saved_tokens, logs) where logs maps event keys to

181 human-readable descriptions of what was trimmed.

182 """

183 if not messages:

184 return 0, {}

185

186 available, optimize_func = _detect()

187 if not available:

188 return 0, {}

189

190 logs: Dict[str, str] = {}

191 saved_bytes = 0

192

193 if optimize_func is not None:

194 # Delegate to the upstream package. It is expected to return either

195 # a tuple ``(new_messages, saved_tokens)`` or just ``new_messages``.

196 try:

197 result = optimize_func(messages, tools) if tools is not None else optimize_func(messages)

198 if isinstance(result, tuple) and len(result) == 2:

199 new_messages, saved_tokens = result

200 if isinstance(new_messages, list):

201 messages[:] = new_messages

202 if isinstance(saved_tokens, int) and saved_tokens > 0:

203 logs["token_optimizer"] = f"upstream optimizer saved ~{saved_tokens} tokens"

204 return saved_tokens, logs

205 elif isinstance(result, list):

206 messages[:] = result

207 # Upstream did not report savings; estimate from byte delta.

208 return 0, logs

209 except Exception as exc:

210 debug.error(f"token_optimizer.optimize_messages failed:", exc)

211 # Fall through to the vendored fallback.

212

213 # Vendored fallback: compress each message's string content.

214 for i, msg in enumerate(messages):

215 if not isinstance(msg, dict):

216 continue

217 content = msg.get("content")

218 if isinstance(content, str):

219 new_content, saved = _compress_content(content)

220 if saved:

221 msg["content"] = new_content

222 saved_bytes += saved

223 logs[f"msg-{i:02d}"] = f"trimmed ~{saved} bytes"

224 elif isinstance(content, list):

225 # Multi-part content (e.g. tool results embedded as parts).

226 for part in content:

227 if isinstance(part, dict):

228 text = part.get("text")

229 if isinstance(text, str):

230 new_text, saved = _compress_content(text)

231 if saved:

232 part["text"] = new_text

233 saved_bytes += saved

234 logs[f"msg-{i:02d}-part"] = f"trimmed ~{saved} bytes"

235

236 if saved_bytes:

237 logs["token_optimizer"] = f"vendored compressor saved ~{_bytes_to_tokens(saved_bytes)} tokens"

238

239 return _bytes_to_tokens(saved_bytes), logs

240

241

242 __all__ = [

243 "is_available",

244 "get_install_path",

245 "optimize_messages",

246 ]