gpt4free

公开

关注 0 Fork 0 Star 1

返回提交历史

XFEstudio/gpt4free

fix(security): resolve polynomial ReDoS vulnerabilities across client, tools, and markitdown

697a9268

Anand Mall <anand@example.com>

提交于 2026-09-07 12:52

父提交 a2259e8

代码差异

4 个文件 +66 -28

Modified g4f/client/helper.py +4 -1

@@ -6,6 +6,9 @@ import logging

6 6 from typing import AsyncIterator, Iterator, AsyncGenerator, Optional

7 7

8 8

9 _CODE_BLOCK_RE = re.compile(r"```([^\r\n\s]+)?\r?\n(?P<code>[\s\S]*?)(?:\r?\n```|$)")

9 12 def filter_markdown(text: str, allowed_types=None, default=None) -> str:

10 13 """

11 14 Parses code block from a string.

@@ -16,7 +19,7 @@ def filter_markdown(text: str, allowed_types=None, default=None) -> str:

16 19 Returns:

17 20 dict: A dictionary parsed from the code block.

18 21 """

19 match = re.search(r"```(.+)\n(?P<code>[\S\s]+?)(\n```|$)", text)

22 match = _CODE_BLOCK_RE.search(text)

20 23 if match:

21 24 if allowed_types is None or match.group(1) in allowed_types:

22 25 return match.group("code")

Modified g4f/integration/markitdown/__init__.py +12 -16

@@ -1,6 +1,7 @@

1 1 import re

2 2 import sys

3 3 import io

4 from urllib.parse import urlsplit

4 5 from typing import List, Union, BinaryIO, Optional, Any

5 6 from markitdown import MarkItDown as BaseMarkItDown

6 7 from markitdown._stream_info import StreamInfo

@@ -356,24 +357,19 @@ class MarkItDown(BaseMarkItDown):

356 357 return url

357 358

358 359 # Gist URLs

359 m = re.match(

360 r"^https?://gist\.github\.com/([^/]+)/([0-9a-fA-F]+)(?:/.*)?$",

361 url,

362 )

363 if m:

364 user, gist_id = m.group(1), m.group(2)

365 return f"https://gist.githubusercontent.com/{user}/{gist_id}/raw"

360 parsed = urlsplit(url)

361 if parsed.netloc == "gist.github.com":

362 parts = [p for p in parsed.path.split("/") if p]

363 if len(parts) >= 2 and all(c in "0123456789abcdefABCDEF" for c in parts[1]):

364 return f"https://gist.githubusercontent.com/{parts[0]}/{parts[1]}/raw"

366 365

367 366 # github.com/{owner}/{repo}/blob/{ref}/{path}

368 m = re.match(

369 r"^https?://github\.com/([^/]+)/([^/]+)/(?:blob|raw)/([^/]+)/(.+?)(?:[?#].*)?$",

370 url,

371 )

372 if m:

373 owner, repo, ref, path = (m.group(1), m.group(2), m.group(3), m.group(4))

374 # Strip a trailing slash if any

375 path = path.rstrip("/")

376 return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}"

367 if parsed.netloc == "github.com":

368 parts = [p for p in parsed.path.split("/") if p]

369 if len(parts) >= 5 and parts[2] in ("blob", "raw"):

370 owner, repo, _, ref = parts[:4]

371 path = "/".join(parts[4:]).rstrip("/")

372 return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}"

377 373

378 374 # Tree (directory) URLs and repo roots: cannot map to a single raw file

379 375 return url

Modified g4f/tools/optimize_request.py +49 -9

@@ -22,7 +22,6 @@ _MAX_TOOL_REPEATS = 3 # max times the same tool call may appear before breaking

22 22 _TOOL_RESULT_CAP = 4096 # bytes per tool result

23 23 _OLD_TOOL_RESULT_CAP = 1200 # stricter cap for results older than 2 turns

24 24

25 _WS_RE = re.compile(r"[ \t]+\n")

26 25 _BLANK_RUN_RE = re.compile(r"\n{3,}")

27 26 _MAX_TURNS = 40 # keep at most this many non-system messages

28 27

@@ -911,8 +910,48 @@ def strip_reasoning_echo(messages: Messages) -> int:

911 910 if not messages:

912 911 return 0

913 912

914 _THINK = re.compile(r"<think>[\s\S]*?</think>", re.IGNORECASE)

915 _REASONING_TAG = re.compile(r"<reasoning[\s\S]*?</reasoning>", re.IGNORECASE)

913 def _strip_think(text: str) -> str:

914 lower = text.lower()

915 start = lower.find("<think>")

916 if start == -1:

917 return text

918 result = []

919 last_idx = 0

920 while start != -1:

921 end = lower.find("</think>", start + 7)

922 if end == -1:

923 result.append(text[start:])

924 last_idx = len(text)

925 break

926 result.append(text[last_idx:start])

927 last_idx = end + 8

928 start = lower.find("<think>", last_idx)

929 result.append(text[last_idx:])

930 return "".join(result)

931

932 def _strip_reasoning(text: str) -> str:

933 lower = text.lower()

934 start = lower.find("<reasoning")

935 if start == -1:

936 return text

937 result = []

938 last_idx = 0

939 while start != -1:

940 tag_end = lower.find(">", start)

941 if tag_end == -1:

942 result.append(text[start:])

943 last_idx = len(text)

944 break

945 end = lower.find("</reasoning>", tag_end + 1)

946 if end == -1:

947 result.append(text[start:])

948 last_idx = len(text)

949 break

950 result.append(text[last_idx:start])

951 last_idx = end + 12

952 start = lower.find("<reasoning", last_idx)

953 result.append(text[last_idx:])

954 return "".join(result)

916 955

917 956 seen_think = False

918 957 seen_reasoning = False

@@ -926,18 +965,19 @@ def strip_reasoning_echo(messages: Messages) -> int:

926 965 continue

927 966

928 967 new_content = content

929 if _THINK.search(new_content):

968 lower_content = new_content.lower()

969 if "<think>" in lower_content and "</think>" in lower_content:

930 970 if seen_think:

931 971 before = len(new_content.encode("utf-8", errors="replace"))

932 new_content = _THINK.sub("", new_content)

972 new_content = _strip_think(new_content)

933 973 after = len(new_content.encode("utf-8", errors="replace"))

934 974 saved_bytes += before - after

935 975 else:

936 976 seen_think = True

937 if _REASONING_TAG.search(new_content):

977 if "<reasoning" in lower_content and "</reasoning>" in lower_content:

938 978 if seen_reasoning:

939 979 before = len(new_content.encode("utf-8", errors="replace"))

940 new_content = _REASONING_TAG.sub("", new_content)

980 new_content = _strip_reasoning(new_content)

941 981 after = len(new_content.encode("utf-8", errors="replace"))

942 982 saved_bytes += before - after

943 983 else:

@@ -1099,7 +1139,7 @@ def _collapse_message_whitespace(messages: Messages) -> int:

1099 1139 content = msg.get("content")

1100 1140 if isinstance(content, str) and len(content) > 64:

1101 1141 original = len(content.encode("utf-8", errors="replace"))

1102 new = _WS_RE.sub("\n", content)

1142 new = "\n".join(line.rstrip(" \t") for line in content.split("\n"))

1103 1143 new = _BLANK_RUN_RE.sub("\n\n", new)

1104 1144 if new != content:

1105 1145 saved_bytes += original - len(new.encode("utf-8", errors="replace"))

@@ -1110,7 +1150,7 @@ def _collapse_message_whitespace(messages: Messages) -> int:

1110 1150 text = part.get("text")

1111 1151 if isinstance(text, str) and len(text) > 64:

1112 1152 original = len(text.encode("utf-8", errors="replace"))

1113 new = _WS_RE.sub("\n", text)

1153 new = "\n".join(line.rstrip(" \t") for line in text.split("\n"))

1114 1154 new = _BLANK_RUN_RE.sub("\n\n", new)

1115 1155 if new != text:

1116 1156 saved_bytes += original - len(

Modified g4f/tools/token_optimizer.py +1 -2

@@ -117,7 +117,6 @@ def get_install_path() -> Optional[str]:

117 117 # outputs embedded in assistant messages.

118 118

119 119 _REPEATED_BLANK = re.compile(r"\n{4,}")

120 _TRAILING_WHITESPACE = re.compile(r"[ \t]+\n")

121 120 _LONG_LINE_CAP = 2000 # lines longer than this get head/tail truncated

122 121

123 122

@@ -131,7 +130,7 @@ def _compress_content(text: str) -> Tuple[str, int]:

131 130 return text, 0

132 131 try:

133 132 original_len = len(text.encode("utf-8", errors="replace"))

134 out = _TRAILING_WHITESPACE.sub("\n", text)

133 out = "\n".join(line.rstrip(" \t") for line in text.split("\n"))

135 134 out = _REPEATED_BLANK.sub("\n\n\n", out)

136 135

137 136 # Truncate very long lines (e.g. minified bundles pasted into context)