gpt4free

公开

关注 0 Fork 0 Star 1

返回提交历史

XFEstudio/gpt4free

Add markitdown from http

ea471ae1

hlohaus <hlohaus@users.noreply.github.com>

提交于 2026-07-14 02:47

父提交 da241e2

代码差异

6 个文件 +206 -29

Modified .github/workflows/build-packages.yml +0 -1

@@ -154,7 +154,6 @@ jobs:

154 154 run: |

155 155 python -m pip install --upgrade pip

156 156 pip install -r requirements.txt

157 pip uninstall -y wasmtime

158 157 pip install nuitka

159 158 pip install -e .

160 159 - name: Write g4f_cli.py

Modified g4f/api/__init__.py +60 -5

@@ -2,6 +2,7 @@ from __future__ import annotations

2 2

3 3 import logging

4 4 import json

5 import asyncio

5 6 import uvicorn

6 7 import secrets

7 8 import os

@@ -22,13 +23,16 @@ from fastapi.security import APIKeyHeader

22 23 from starlette.exceptions import HTTPException

23 24 from starlette.status import (

24 25 HTTP_200_OK,

25 HTTP_422_UNPROCESSABLE_ENTITY,

26 26 HTTP_404_NOT_FOUND,

27 27 HTTP_401_UNAUTHORIZED,

28 28 HTTP_403_FORBIDDEN,

29 29 HTTP_429_TOO_MANY_REQUESTS,

30 30 HTTP_500_INTERNAL_SERVER_ERROR,

31 31 )

32 try:

33 from starlette.status import HTTP_422_UNPROCESSABLE_CONTENT

34 except ImportError:

35 HTTP_422_UNPROCESSABLE_CONTENT = 422

32 36 from starlette.staticfiles import NotModifiedResponse

33 37 from fastapi.encoders import jsonable_encoder

34 38 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials, HTTPBasic

@@ -569,7 +573,7 @@ class Api:

569 573 "type": error["type"],

570 574 })

571 575 return JSONResponse(

572 status_code=HTTP_422_UNPROCESSABLE_ENTITY,

576 status_code=HTTP_422_UNPROCESSABLE_CONTENT,

573 577 content=jsonable_encoder({"detail": modified_details}),

574 578 )

575 579

@@ -692,7 +696,7 @@ class Api:

692 696 HTTP_200_OK: {"model": ChatCompletion},

693 697 HTTP_401_UNAUTHORIZED: {"model": ErrorResponseModel},

694 698 HTTP_404_NOT_FOUND: {"model": ErrorResponseModel},

695 HTTP_422_UNPROCESSABLE_ENTITY: {"model": ErrorResponseModel},

699 HTTP_422_UNPROCESSABLE_CONTENT: {"model": ErrorResponseModel},

696 700 HTTP_429_TOO_MANY_REQUESTS: {"model": ErrorResponseModel},

697 701 HTTP_500_INTERNAL_SERVER_ERROR: {"model": ErrorResponseModel},

698 702 }

@@ -736,7 +740,7 @@ class Api:

736 740 try:

737 741 is_data_an_media(config.image)

738 742 except ValueError as e:

739

                        return ErrorResponse.from_message(f"The image you send must be a data URI. Example: data:image/jpeg;base64,...", status_code=HTTP_422_UNPROCESSABLE_ENTITY)

743

                        return ErrorResponse.from_message(f"The image you send must be a data URI. Example: data:image/jpeg;base64,...", status_code=HTTP_422_UNPROCESSABLE_CONTENT)

740 744 if config.media is None:

741 745 config.media = config.images

742 746 if config.media is not None:

@@ -745,7 +749,7 @@ class Api:

745 749 is_data_an_media(image[0], image[1])

746 750 except ValueError as e:

747 751 example = json.dumps({"media": [["data:image/jpeg;base64,...", "filename.jpg"]]})

748

                            return ErrorResponse.from_message(f'The media you send must be a data URIs. Example: {example}', status_code=HTTP_422_UNPROCESSABLE_ENTITY)

752

                            return ErrorResponse.from_message(f'The media you send must be a data URIs. Example: {example}', status_code=HTTP_422_UNPROCESSABLE_CONTENT)

749 753

750 754 # Create the completion response

751 755 response = self.client.chat.completions.create(

@@ -1349,6 +1353,57 @@ class Api:

1349 1353 logger.exception(e)

1350 1354 return ErrorResponse.from_exception(e, None, HTTP_500_INTERNAL_SERVER_ERROR)

1351 1355

1356 responses = {

1357 HTTP_200_OK: {"model": TranscriptionResponseModel},

1358 HTTP_401_UNAUTHORIZED: {"model": ErrorResponseModel},

1359 HTTP_404_NOT_FOUND: {"model": ErrorResponseModel},

1360 HTTP_500_INTERNAL_SERVER_ERROR: {"model": ErrorResponseModel},

1361 }

1362 @self.app.get("/markitdown/{url:path}", responses=responses)

1363 async def convert_url(

1364 request: Request,

1365 url: str,

1366 credentials: Annotated[HTTPAuthorizationCredentials, Depends(Api.security)] = None

1367 ):

1368 """Convert a URL to Markdown using MarkItDown.

1369

1370 The full URL (including scheme) is passed in the path, e.g.:

1371 GET /markitdown/https://example.com/page

1372

1373 Query strings are preserved by reading them from the incoming

1374 request and re-appending them to the target URL, e.g.:

1375 GET /markitdown/https://example.com/page?foo=bar

1376 """

1377 # FastAPI strips the query string from the {url:path} parameter,

1378 # so re-attach it from the incoming request when present.

1379 query_string = request.url.query

1380 if query_string and "?" not in url:

1381 url = f"{url}?{query_string}"

1382 elif query_string:

1383 # url already contains a '?', append remaining params with '&'

1384 url = f"{url}&{query_string}"

1385 if not url.startswith(("http://", "https://")):

1386 return ErrorResponse.from_message(

1387 f"Invalid URL: {url}. URL must start with http:// or https://",

1388 HTTP_422_UNPROCESSABLE_CONTENT,

1389 )

1390 try:

1391 from g4f.integration.markitdown import MarkItDown

1392 md = MarkItDown()

1393 result = md.convert_url(url)

1394 text = result.text_content

1395 if asyncio.iscoroutine(text):

1396 text = await text

1397 return JSONResponse(

1398 {"text": text, "title": result.title, "url": url},

1399 )

1400 except ImportError as e:

1401 logger.exception(e)

1402 return ErrorResponse.from_exception(e, None, HTTP_500_INTERNAL_SERVER_ERROR)

1403 except Exception as e:

1404 logger.exception(e)

1405 return ErrorResponse.from_exception(e, None, HTTP_500_INTERNAL_SERVER_ERROR)

1406

1352 1407 responses = {

1353 1408 HTTP_200_OK: {"content": {"audio/*": {}}},

1354 1409 HTTP_401_UNAUTHORIZED: {"model": ErrorResponseModel},

Modified g4f/integration/markitdown/__init__.py +66 -0

@@ -168,3 +168,69 @@ class MarkItDown(BaseMarkItDown):

168 168 file_stream=stream, base_guess=base_guess or StreamInfo()

169 169 )

170 170 return self._convert(file_stream=stream, stream_info_guesses=guesses, **kwargs)

171

172 @staticmethod

173 def _convert_github_url_to_raw(url: str) -> str:

174 """Convert a github.com URL to a raw.githubusercontent.com content URL.

175

176 Handles the following patterns:

177 - https://github.com/{owner}/{repo}/blob/{ref}/{path}

178 -> https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}

179 - https://github.com/{owner}/{repo}/raw/{ref}/{path}

180 -> https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}

181 - https://gist.github.com/{user}/{gist_id}

182 -> https://gist.githubusercontent.com/{user}/{gist_id}/raw

183 - URLs already pointing to raw.githubusercontent.com or

184 gist.githubusercontent.com are returned unchanged.

185

186 Tree URLs (directories) and repository root URLs cannot be converted

187 to a single raw file and are returned unchanged so the caller can

188 decide how to handle them.

189 """

190 if url is None:

191 raise ValueError("url must not be None")

192

193 # Already raw -- nothing to do

194 if url.startswith(("https://raw.githubusercontent.com/",

195 "https://gist.githubusercontent.com/")):

196 return url

197

198 # Gist URLs

199 m = re.match(

200 r"^https?://gist\.github\.com/([^/]+)/([0-9a-fA-F]+)(?:/.*)?$",

201 url,

202 )

203 if m:

204 user, gist_id = m.group(1), m.group(2)

205 return f"https://gist.githubusercontent.com/{user}/{gist_id}/raw"

206

207 # github.com/{owner}/{repo}/blob/{ref}/{path}

208 m = re.match(

209 r"^https?://github\.com/([^/]+)/([^/]+)/(?:blob|raw)/([^/]+)/(.+?)(?:[?#].*)?$",

210 url,

211 )

212 if m:

213 owner, repo, ref, path = (m.group(1), m.group(2),

214 m.group(3), m.group(4))

215 # Strip a trailing slash if any

216 path = path.rstrip("/")

217 return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}"

218

219 # Tree (directory) URLs and repo roots: cannot map to a single raw file

220 return url

221

222 def convert_url(

223 self,

224 url: str,

225 *,

226 stream_info: Optional[StreamInfo] = None,

227 **kwargs: Any,

228 ) -> DocumentConverterResult:

229 if url is None or not isinstance(url, str) or url.strip() == "":

230 raise ValueError("url must be a non-empty string")

231 if not url.startswith(("http://", "https://")):

232 raise ValueError("url must start with http:// or https://")

233 if url.startswith("https://github.com/"):

234 # Special case for GitHub URLs -- convert to raw content URL

235 url = self._convert_github_url_to_raw(url)

236 return super().convert_url(url, stream_info=stream_info, **kwargs)

Modified g4f/integration/markitdown/_youtube_converter.py +68 -9

@@ -2,12 +2,75 @@ import json

2 2 import time

3 3 import re

4 4 import bs4

5 from typing import Any, BinaryIO, Dict, List, Union

5 from typing import Any, BinaryIO, Dict, List, Optional, Union

6 6 from urllib.parse import parse_qs, urlparse, unquote

7 7

8 8 from markitdown._base_converter import DocumentConverter, DocumentConverterResult

9 9 from markitdown._stream_info import StreamInfo

10 10

11 # Hosts that serve YouTube content we can transcribe

12 _YOUTUBE_HOSTS = (

13 "www.youtube.com",

14 "youtube.com",

15 "m.youtube.com",

16 "music.youtube.com",

17 "youtu.be",

18 )

20 # Regex patterns for the various YouTube URL formats

21 _YOUTUBE_PATTERNS = [

22 # youtu.be/{video_id}

23 re.compile(r"^https?://(?:www\.)?youtu\.be/(?P<id>[A-Za-z0-9_-]{6,})(?:[?&#].*)?$"),

24 # youtube.com/watch?v={video_id}

    re.compile(r"^https?://(?:www\.|m\.|music\.)?youtube\.com/watch(?:[?&#].*)?(?:[?&]v=)(?P<id>[A-Za-z0-9_-]{6,})(?:[&#].*)?$"),

26 # youtube.com/embed/{video_id}

27 re.compile(r"^https?://(?:www\.|m\.|music\.)?youtube\.com/embed/(?P<id>[A-Za-z0-9_-]{6,})(?:[?&#].*)?$"),

28 # youtube.com/shorts/{video_id}

29 re.compile(r"^https?://(?:www\.|m\.|music\.)?youtube\.com/shorts/(?P<id>[A-Za-z0-9_-]{6,})(?:[?&#].*)?$"),

30 # youtube.com/live/{video_id}

31 re.compile(r"^https?://(?:www\.|m\.|music\.)?youtube\.com/live/(?P<id>[A-Za-z0-9_-]{6,})(?:[?&#].*)?$"),

32 # youtube.com/v/{video_id}

33 re.compile(r"^https?://(?:www\.|m\.|music\.)?youtube\.com/v/(?P<id>[A-Za-z0-9_-]{6,})(?:[?&#].*)?$"),

34 ]

37 def _extract_youtube_video_id(url: str) -> Optional[str]:

38 """Extract the YouTube video ID from any supported URL format.

40 Supports:

41 - https://www.youtube.com/watch?v=ID

42 - https://youtu.be/ID

43 - https://www.youtube.com/embed/ID

44 - https://www.youtube.com/shorts/ID

45 - https://www.youtube.com/live/ID

46 - https://www.youtube.com/v/ID

47 - m.youtube.com / music.youtube.com variants

48 """

49 if not url:

50 return None

51 url = unquote(url).strip()

52 # Normalize escaped characters that some sources emit

53 url = url.replace(r"\?", "?").replace(r"\=", "=")

54 for pattern in _YOUTUBE_PATTERNS:

55 m = pattern.match(url)

56 if m:

57 return m.group("id")

58 # Fallback: parse query string for a `v` parameter on a youtube host

59 try:

60 parsed = urlparse(url)

61 if parsed.hostname and parsed.hostname.endswith("youtube.com"):

62 params = parse_qs(parsed.query)

63 if "v" in params and params["v"][0]:

64 return str(params["v"][0])

65 except Exception:

66 pass

67 return None

70 def _is_youtube_url(url: str) -> bool:

71 """Return True if the URL points to a YouTube video page we can transcribe."""

72 return _extract_youtube_video_id(url) is not None

11 74 # Optional YouTube transcription support

12 75 try:

13 76 # Suppress some warnings on library import

@@ -50,13 +113,11 @@ class YouTubeConverter(DocumentConverter):

50 113 mimetype = (stream_info.mimetype or "").lower()

51 114 extension = (stream_info.extension or "").lower()

52 115

53 url = unquote(url)

54 url = url.replace(r"\?", "?").replace(r"\=", "=")

55 116

56 if not url.startswith("https://www.youtube.com/watch?"):

117 if not _is_youtube_url(url):

57 118 # Not a YouTube URL

58 119 return False

120

60 121 if extension in ACCEPTED_FILE_EXTENSIONS:

61 122 return True

62 123

@@ -148,10 +209,8 @@ class YouTubeConverter(DocumentConverter):

148 209 try:

149 210 ytt_api = YouTubeTranscriptApi()

150 211 transcript_text = ""

151 parsed_url = urlparse(stream_info.url) # type: ignore

152 params = parse_qs(parsed_url.query) # type: ignore

153 if "v" in params and params["v"][0]:

154 video_id = str(params["v"][0])

212 video_id = _extract_youtube_video_id(stream_info.url or "")

213 if video_id:

155 214 transcript_list = ytt_api.list(video_id)

156 215 languages = ["en"]

157 216 for transcript in transcript_list:

Modified g4f/tools/run_tools.py +10 -12

@@ -23,7 +23,7 @@ from ..providers.helper import filter_none

23 23 from ..providers.asyncio import to_sync_generator

24 24

from ..providers.response import Reasoning, FinishReason, Sources, Usage, ProviderInfo, HeadersResponse, JsonConversation

25 25 from .optimize_request import optimize_request

from .token_optimizer import optimize_messages as token_optimizer_optimize_messages, is_available as token_optimizer_available

26 from .token_optimizer import optimize_messages

27 27 from ..providers.types import ProviderType

28 28 from ..providers.base_provider import get_async_provider_method, get_provider_method, wait_for

29 29 from ..cookies import get_cookies_dir

@@ -346,11 +346,10 @@ async def async_iter_run_tools(

346 346 # Optional token-optimizer plugin: compress the prompt messages before

347 347 # they reach the provider. Only active when the `token_optimizer` package

348 348 # is installed in the environment.

349 if token_optimizer_available():

350 to_saved, _to_logs = token_optimizer_optimize_messages(messages, tools_ref)

351 if to_saved:

352 saved_tokens += to_saved

353 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

349 to_saved, _to_logs = optimize_messages(messages, tools_ref)

350 if to_saved:

351 saved_tokens += to_saved

352 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

354 353

355 354 tool_emulation = kwargs.pop("tool_emulation", None)

356 355 if tool_emulation is None:

@@ -499,16 +498,15 @@ def iter_run_tools(

499 498 tools_ref = kwargs.get("tools")

500 499 saved_tokens, _optimize_logs = optimize_request(messages, tools_ref)

501 500 if saved_tokens:

502 debug.log(f"Optimized request: saved ~{saved_tokens} tokens")

501 debug.log(f"VCS - Optimized request: saved ~{saved_tokens} tokens")

503 502

504 503 # Optional token-optimizer plugin: compress the prompt messages before

505 504 # they reach the provider. Only active when the `token_optimizer` package

506 505 # is installed in the environment.

507 if token_optimizer_available():

508 to_saved, _to_logs = token_optimizer_optimize_messages(messages, tools_ref)

509 if to_saved:

510 saved_tokens += to_saved

511 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

506 to_saved, _to_logs = optimize_messages(messages, tools_ref)

507 if to_saved:

508 saved_tokens += to_saved

509 debug.log(f"Token Optimizer plugin: saved ~{to_saved} tokens")

512 510

513 511 tool_emulation = kwargs.pop("tool_emulation", None)

514 512 if tool_emulation is None:

Modified requirements.txt +2 -2

@@ -19,6 +19,6 @@ a2wsgi

19 19 python-dotenv

20 20 ddgs

21 21 cloudscraper

22 wasmtime

23 22 numpy

24 PyYAML

23 PyYAML

24 prompt_optimizer