gpt4free

公开

关注 0 Fork 0 Star 0

XFEstudio/gpt4free

refactor: improve media rendering and response formatting with precise changes

- Modified g4f/providers/response.py to ensure format_images_markdown returns the result directly without additional flags in the 'format_images_markdown' function. - Updated g4f/gui/server/api.py to add 'tempfiles' parameter with default empty list to '_create_response_stream' method. - Changed or added code in API response handling to iterate over 'tempfiles' and attempt to remove each file after response completion, with exception handling (try-except block with logger.exception). - Adjusted g4f/Tools/files.py to fix tempfile creation: corrected the 'suffix' parameter in 'get_tempfile' to use 'suffix' directly instead of splitting. - In g4f/tools/media.py, changed 'render_part' function to handle 'text' key properly, checking 'part.get("text")' and returning a dictionary with 'type': 'text' and 'text': value, if present.

c0d31c2a

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-05-19 08:15

父提交 ff0618e

代码差异

15 个文件 +516 -61

Modified g4f/Provider/ImageLabs.py +14 -6

@@ -6,6 +6,7 @@ import asyncio

6 6

7 7 from ..typing import AsyncResult, Messages

8 8 from ..providers.response import ImageResponse

9 from ..image import use_aspect_ratio

9 10 from .base_provider import AsyncGeneratorProvider, ProviderModelMixin

10 11

11 12

@@ -32,10 +33,18 @@ class ImageLabs(AsyncGeneratorProvider, ProviderModelMixin):

32 33 # Image

33 34 prompt: str = None,

34 35 negative_prompt: str = "",

35 width: int = 1152,

36 height: int = 896,

36 aspect_ratio: str = "1:1",

37 width: int = None,

38 height: int = None,

39 extra_body: dict = {},

37 40 **kwargs

38 ) -> AsyncResult:

41 ) -> AsyncResult:

42 extra_body = use_aspect_ratio({

43 "width": width,

44 "height": height,

45 **extra_body

46 }, aspect_ratio)

39 48 headers = {

40 49 'accept': '*/*',

41 50 'accept-language': 'en-US,en;q=0.9',

@@ -56,13 +65,12 @@ class ImageLabs(AsyncGeneratorProvider, ProviderModelMixin):

56 65 "seed": str(int(time.time())),

57 66 "subseed": str(int(time.time() * 1000)),

58 67 "attention": 0,

59 "width": width,

60 "height": height,

61 68 "tiling": False,

62 69 "negative_prompt": negative_prompt,

63 70 "reference_image": "",

64 71 "reference_image_type": None,

65 "reference_strength": 30

72 "reference_strength": 30,

73 **extra_body

66 74 }

67 75

68 76 async with session.post(f'{cls.url}/txt2img', json=payload, proxy=proxy) as generate_response:

Modified g4f/Provider/PollinationsAI.py +6 -3

@@ -359,18 +359,19 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

359 359 return f"{url}&seed={seed}" if seed else url

360 360 async with ClientSession(headers=DEFAULT_HEADERS, connector=get_connector(proxy=proxy)) as session:

361 361 responses = set()

362 responses.add(Reasoning(status=f"Generating {n} {'image' if n == 1 else 'images'}"))

362 363 finished = 0

364 start = time.time()

363 365 async def get_image(responses: set, i: int, seed: Optional[int] = None):

364 366 nonlocal finished

365 start = time.time()

366 367

                async with session.get(get_image_url(i, seed), allow_redirects=False, headers={"referer": referrer}) as response:

367 368 try:

368 369 await raise_for_status(response)

369 370 except Exception as e:

370 371 debug.error(f"Error fetching image: {e}")

371 responses.add(Reasoning(status=f"Image #{i+1} generated in {time.time() - start:.2f}s"))

372 372 responses.add(ImageResponse(str(response.url), prompt))

373 373 finished += 1

374 responses.add(Reasoning(status=f"Image {finished}/{n} generated in {time.time() - start:.2f}s"))

374 375 tasks = []

375 376 for i in range(int(n)):

376 377 tasks.append(asyncio.create_task(get_image(responses, i, seed)))

@@ -426,6 +427,8 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

426 427 **extra_body

427 428 )

428 429 async with session.post(url, json=data, headers={"referer": referrer}) as response:

430 if response.status == 400:

431 debug.error(f"Error: 400 - Bad Request: {data}")

429 432 await raise_for_status(response)

430 433 if response.headers["content-type"].startswith("text/plain"):

431 434 yield await response.text()

@@ -492,6 +495,6 @@ class PollinationsAI(AsyncGeneratorProvider, ProviderModelMixin):

492 495 if finish_reason:

493 496 yield FinishReason(finish_reason)

494 497 else:

495

                    async for chunk in save_response_media(response, format_image_prompt(messages), [model, extra_parameters.get("audio", {}).get("voice")]):

498

                    async for chunk in save_response_media(response, format_image_prompt(messages), [model, extra_body.get("audio", {}).get("voice")]):

496 499 yield chunk

497 500 return

Modified g4f/Provider/audio/MarkItDown.py +1 -1

@@ -5,7 +5,7 @@ import asyncio

5 5 from typing import Any

6 6

7 7 try:

8 from markitdown import MarkItDown as MaItDo, StreamInfo

8 from ...integration.markitdown import MarkItDown as MaItDo, StreamInfo

9 9 has_markitdown = True

10 10 except ImportError:

11 11 has_markitdown = False

Modified g4f/gui/server/api.py +8 -5

@@ -146,7 +146,7 @@ class Api:

146 146 **kwargs

147 147 }

148 148

149 def _create_response_stream(self, kwargs: dict, provider: str, download_media: bool = True) -> Iterator:

149

    def _create_response_stream(self, kwargs: dict, provider: str, download_media: bool = True, tempfiles: list[str] = []) -> Iterator:

150 150 def decorated_log(text: str, file = None):

151 151 debug.logs.append(text)

152 152 if debug.logging:

@@ -163,7 +163,7 @@ class Api:

163 163 has_images="media" in kwargs,

164 164 )

165 165 except Exception as e:

166 debug.error(e)

166 logger.exception(e)

167 167 yield self._format_json('error', type(e).__name__, message=get_error_message(e))

168 168 return

169 169 if not isinstance(provider_handler, BaseRetryProvider):

@@ -198,7 +198,7 @@ class Api:

198 198

                        tags = [model, kwargs.get("aspect_ratio"), kwargs.get("resolution"), kwargs.get("width"), kwargs.get("height")]

199 199

                        media = asyncio.run(copy_media(chunk.get_list(), chunk.get("cookies"), chunk.get("headers"), proxy=proxy, alt=chunk.alt, tags=tags))

200 200

                        media = ImageResponse(media, chunk.alt) if isinstance(chunk, ImageResponse) else VideoResponse(media, chunk.alt)

201 yield self._format_json("content", str(media), urls=chunk.urls, alt=chunk.alt)

201 yield self._format_json("content", str(media), urls=media.urls, alt=media.alt)

202 202 elif isinstance(chunk, SynthesizeData):

203 203 yield self._format_json("synthesize", chunk.get_dict())

204 204 elif isinstance(chunk, TitleGeneration):

@@ -232,6 +232,11 @@ class Api:

232 232 yield self._format_json('error', type(e).__name__, message=get_error_message(e))

233 233 finally:

234 234 yield from self._yield_logs()

235 for tempfile in tempfiles:

236 try:

237 os.remove(tempfile)

238 except Exception as e:

239 logger.exception(e)

235 240

236 241 def _yield_logs(self):

237 242 if debug.logs:

@@ -252,8 +257,6 @@ class Api:

252 257 }

253 258

254 259 def handle_provider(self, provider_handler, model):

255 if isinstance(provider_handler, BaseRetryProvider) and provider_handler.last_provider is not None:

256 provider_handler = provider_handler.last_provider

257 260 if model:

258 261 return self._format_json("provider", {**provider_handler.get_dict(), "model": model})

259 262 return self._format_json("provider", provider_handler.get_dict())

Modified g4f/gui/server/backend_api.py +45 -40

@@ -8,8 +8,7 @@ import asyncio

8 8 import shutil

9 9 import random

10 10 import datetime

11 import tempfile

12 from flask import Flask, Response, redirect, request, jsonify, render_template, send_from_directory

11 from flask import Flask, Response, redirect, request, jsonify, send_from_directory

13 12 from werkzeug.exceptions import NotFound

14 13 from typing import Generator

15 14 from pathlib import Path

@@ -17,19 +16,20 @@ from urllib.parse import quote_plus

17 16 from hashlib import sha256

18 17

19 18 try:

20 from markitdown import MarkItDown

19 from ...integration.markitdown import MarkItDown, StreamInfo

21 20 has_markitdown = True

22 except ImportError:

21 except ImportError as e:

22 print(e)

23 23 has_markitdown = False

24 24

25 25 from ...client.service import convert_to_provider

26 26 from ...providers.asyncio import to_sync_generator

27 27 from ...providers.response import FinishReason

28 28 from ...client.helper import filter_markdown

29 from ...tools.files import supports_filename, get_streaming, get_bucket_dir, get_buckets

29 from ...tools.files import supports_filename, get_streaming, get_bucket_dir, get_tempfile

30 30 from ...tools.run_tools import iter_run_tools

31 31 from ...errors import ProviderNotFoundError

32 from ...image import is_allowed_extension

32 from ...image import is_allowed_extension, MEDIA_TYPE_MAP

33 33 from ...cookies import get_cookies_dir

34 34 from ...image.copy_images import secure_filename, get_source_url, get_media_dir

35 35 from ... import ChatCompletion

@@ -79,9 +79,7 @@ class Backend_Api(Api):

79 79 @app.route('/backend-api/v2/providers', methods=['GET'])

80 80 def jsonify_providers(**kwargs):

81 81 response = self.get_providers(**kwargs)

82 if isinstance(response, list):

83 return jsonify(response)

84 return response

82 return jsonify(response)

85 83

86 84 def get_demo_models():

87 85 return [{

@@ -91,7 +89,7 @@ class Backend_Api(Api):

91 89 "audio": isinstance(model, models.AudioModel),

92 90 "video": isinstance(model, models.VideoModel),

93 91 "providers": [

94 getattr(provider, "parent", provider.__name__)

92 provider.get_parent()

95 93 for provider in providers

96 94 ],

97 95 "demo": True

@@ -109,13 +107,14 @@ class Backend_Api(Api):

109 107 json_data = json.loads(request.form['json'])

110 108 else:

111 109 json_data = request.json

110 tempfiles = []

112 111 if "files" in request.files:

113 112 media = []

114 113 for file in request.files.getlist('files'):

115 114 if file.filename != '' and is_allowed_extension(file.filename):

116 newfile = tempfile.TemporaryFile()

117 shutil.copyfileobj(file.stream, newfile)

118 media.append((newfile, file.filename))

115 newfile = get_tempfile(file)

116 tempfiles.append(newfile)

117 media.append((Path(newfile), file.filename))

119 118 json_data['media'] = media

120 119

121 120 if app.demo and not json_data.get("provider"):

@@ -130,6 +129,7 @@ class Backend_Api(Api):

130 129 kwargs,

131 130 json_data.get("provider"),

132 131 json_data.get("download_media", True),

132 tempfiles

133 133 ),

134 134 mimetype='text/event-stream'

135 135 )

@@ -306,41 +306,46 @@ class Backend_Api(Api):

306 306 filenames = []

307 307 media = []

308 308 for file in request.files.getlist('files'):

309 # Copy the file to a temporary location

310 309 filename = secure_filename(file.filename)

311 copyfile = tempfile.NamedTemporaryFile(suffix=filename, delete=False)

312 shutil.copyfileobj(file.stream, copyfile)

313 copyfile.close()

314 file.stream.close()

315

310 mimetype = file.mimetype.split(";")[0]

311 if (not filename or filename == "blob") and mimetype in MEDIA_TYPE_MAP:

312 filename = f"file.{MEDIA_TYPE_MAP[mimetype]}"

313 suffix = os.path.splitext(filename)[1].lower()

314 copyfile = get_tempfile(file, suffix)

316 315 result = None

317 316 if has_markitdown:

318 317 try:

318 language = request.headers.get("x-recognition-language")

319 319 md = MarkItDown()

320 result = md.convert(copyfile.name).text_content

321 with open(os.path.join(bucket_dir, f"{filename}.md"), 'w') as f:

322 f.write(f"{result}\n")

323 filenames.append(f"{filename}.md")

320 result = md.convert(copyfile, stream_info=StreamInfo(

321 extension=suffix,

322 mimetype=file.mimetype,

323 ), language=language).text_content

324 324 except Exception as e:

325 325 logger.exception(e)

326 if not result:

327 if is_allowed_extension(filename):

328 os.makedirs(media_dir, exist_ok=True)

329 newfile = os.path.join(media_dir, filename)

330 media.append(filename)

331 elif supports_filename(filename):

332 newfile = os.path.join(bucket_dir, filename)

333 filenames.append(filename)

334 else:

335 os.remove(copyfile.name)

336 continue

337 try:

338 os.rename(copyfile.name, newfile)

339 except OSError:

340 shutil.copyfile(copyfile.name, newfile)

341 os.remove(copyfile.name)

326 is_media = is_allowed_extension(filename)

327 is_supported = supports_filename(filename)

328 if not is_media and not is_supported:

329 os.remove(copyfile)

330 continue

331 if not is_media and result:

332 with open(os.path.join(bucket_dir, f"{filename}.md"), 'w') as f:

333 f.write(f"{result}\n")

334 filenames.append(f"{filename}.md")

335 if is_media:

336 os.makedirs(media_dir, exist_ok=True)

337 newfile = os.path.join(media_dir, filename)

338 media.append({"name": filename, "text": result})

339 elif not result and supports_filename(filename):

340 newfile = os.path.join(bucket_dir, filename)

341 filenames.append(filename)

342 try:

343 os.rename(copyfile, newfile)

344 except OSError:

345 shutil.copyfile(copyfile, newfile)

346 os.remove(copyfile)

342 347 with open(os.path.join(bucket_dir, "files.txt"), 'w') as f:

343 [f.write(f"{filename}\n") for filename in filenames]

348 f.write("".join([f"{filename}\n" for filename in filenames]))

344 349 return {"bucket_id": bucket_id, "files": filenames, "media": media}

345 350

346 351 @app.route('/files/<bucket_id>/media/<filename>', methods=['GET'])

Modified g4f/image/__init__.py +2 -1

@@ -38,6 +38,7 @@ EXTENSIONS_MAP: dict[str, str] = {

38 38 }

39 39

40 40 MEDIA_TYPE_MAP: dict[str, str] = {value: key for key, value in EXTENSIONS_MAP.items()}

41 MEDIA_TYPE_MAP["audio/webm"] = "webm"

41 42

42 43 def to_image(image: ImageType, is_svg: bool = False) -> Image:

43 44 """

@@ -111,7 +112,7 @@ def is_data_an_audio(data_uri: str = None, filename: str = None) -> str:

111 112 extension = get_extension(filename)

112 113 if extension is not None:

113 114 media_type = EXTENSIONS_MAP[extension]

114 if media_type.startswith("audio/"):

115 if media_type.startswith("audio/") or media_type == "video/webm":

115 116 return media_type

116 117 if isinstance(data_uri, str):

117 118 audio_format = re.match(r'^data:(audio/\w+);base64,', data_uri)

Added g4f/integration/markitdown/__init__.py +120 -0

@@ -0,0 +1,120 @@

1 import re

2 import sys

3 from typing import List, Union, BinaryIO

4 from markitdown import MarkItDown as BaseMarkItDown

5 from markitdown._stream_info import StreamInfo

6 from markitdown._base_converter import DocumentConverterResult

8 from markitdown._exceptions import (

9 FileConversionException,

10 UnsupportedFormatException,

11 FailedConversionAttempt,

12 )

14 from ._audio_converter import AudioConverter

15 from ._image_converter import ImageConverter

17 class MarkItDown(BaseMarkItDown):

18 """(In preview) An extremely simple text-based document reader, suitable for LLM use.

19 This reader will convert common file-types or webpages to Markdown."""

21 def __init__(

22 self,

23 **kwargs,

24 ):

25 super().__init__(**kwargs)

26 self.register_converter(AudioConverter())

27 self.register_converter(ImageConverter())

29 def _convert(

30 self, *, file_stream: BinaryIO, stream_info_guesses: List[StreamInfo], **kwargs

31 ) -> DocumentConverterResult:

32 res: Union[None, DocumentConverterResult] = None

34 # Keep track of which converters throw exceptions

35 failed_attempts: List[FailedConversionAttempt] = []

37 # Create a copy of the page_converters list, sorted by priority.

38 # We do this with each call to _convert because the priority of converters may change between calls.

39 # The sort is guaranteed to be stable, so converters with the same priority will remain in the same order.

40 sorted_registrations = sorted(self._converters, key=lambda x: x.priority)

42 # Remember the initial stream position so that we can return to it

43 cur_pos = file_stream.tell()

45 for stream_info in stream_info_guesses + [StreamInfo()]:

46 for converter_registration in sorted_registrations:

47 converter = converter_registration.converter

48 # Sanity check -- make sure the cur_pos is still the same

49 assert (

50 cur_pos == file_stream.tell()

51 ), f"File stream position should NOT change between guess iterations"

53 _kwargs = {k: v for k, v in kwargs.items()}

55 # Copy any additional global options

56 if "llm_client" not in _kwargs and self._llm_client is not None:

57 _kwargs["llm_client"] = self._llm_client

59 if "llm_model" not in _kwargs and self._llm_model is not None:

60 _kwargs["llm_model"] = self._llm_model

62 if "style_map" not in _kwargs and self._style_map is not None:

63 _kwargs["style_map"] = self._style_map

65 if "exiftool_path" not in _kwargs and self._exiftool_path is not None:

66 _kwargs["exiftool_path"] = self._exiftool_path

68 # Add the list of converters for nested processing

69 _kwargs["_parent_converters"] = self._converters

71 # Add legaxy kwargs

72 if stream_info is not None:

73 if stream_info.extension is not None:

74 _kwargs["file_extension"] = stream_info.extension

76 if stream_info.url is not None:

77 _kwargs["url"] = stream_info.url

79 # Check if the converter will accept the file, and if so, try to convert it

80 _accepts = False

81 try:

82 _accepts = converter.accepts(file_stream, stream_info, **_kwargs)

83 except NotImplementedError:

84 pass

86 # accept() should not have changed the file stream position

87 assert (

88 cur_pos == file_stream.tell()

89 ), f"{type(converter).__name__}.accept() should NOT change the file_stream position"

91 # Attempt the conversion

92 if _accepts:

93 try:

94 res = converter.convert(file_stream, stream_info, **_kwargs)

95 except Exception:

96 failed_attempts.append(

97 FailedConversionAttempt(

98 converter=converter, exc_info=sys.exc_info()

99 )

100 )

101 finally:

102 file_stream.seek(cur_pos)

103

104 if res is not None:

105 if isinstance(res.text_content, str):

106 # Normalize the content

107 res.text_content = "\n".join(

108 [line.rstrip() for line in re.split(r"\r?\n", res.text_content)]

109 )

110 res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)

111 return res

112

113 # If we got this far without success, report any exceptions

114 if len(failed_attempts) > 0:

115 raise FileConversionException(attempts=failed_attempts)

116

117 # Nothing can handle it!

118 raise UnsupportedFormatException(

119

            f"Could not convert stream to Markdown. No converter attempted a conversion, suggesting that the filetype is simply not supported."

120 )

Added g4f/integration/markitdown/_audio_converter.py +105 -0

@@ -0,0 +1,105 @@

1 from typing import Any, BinaryIO

3 from markitdown.converters._exiftool import exiftool_metadata

4 from markitdown._base_converter import DocumentConverter, DocumentConverterResult

5 from markitdown._stream_info import StreamInfo

6 from markitdown._exceptions import MissingDependencyException

8 from ._transcribe_audio import transcribe_audio

10 ACCEPTED_MIME_TYPE_PREFIXES = [

11 "audio/x-wav",

12 "audio/mpeg",

13 "video/mp4",

14 "video/webm",

15 "audio/webm",

16 ]

18 ACCEPTED_FILE_EXTENSIONS = [

19 ".wav",

20 ".mp3",

21 ".m4a",

22 ".mp4",

23 ".webm",

24 ]

26 class AudioConverter(DocumentConverter):

27 """

    Converts audio files to markdown via extraction of metadata (if `exiftool` is installed), and speech transcription (if `speech_recognition` is installed).

29 """

31 def accepts(

32 self,

33 file_stream: BinaryIO,

34 stream_info: StreamInfo,

35 **kwargs: Any, # Options to pass to the converter

36 ) -> bool:

37 mimetype = (stream_info.mimetype or "").lower()

38 extension = (stream_info.extension or "").lower()

40 if extension in ACCEPTED_FILE_EXTENSIONS:

41 return True

43 for prefix in ACCEPTED_MIME_TYPE_PREFIXES:

44 if mimetype.startswith(prefix):

45 return True

47 return False

49 def convert(

50 self,

51 file_stream: BinaryIO,

52 stream_info: StreamInfo,

53 language: str = "en-US",

54 **kwargs: Any, # Options to pass to the converter

55 ) -> DocumentConverterResult:

56 md_content = ""

58 # Add metadata

59 metadata = exiftool_metadata(

60 file_stream, exiftool_path=kwargs.get("exiftool_path")

61 )

62 if metadata:

63 for f in [

64 "Title",

65 "Artist",

66 "Author",

67 "Band",

68 "Album",

69 "Genre",

70 "Track",

71 "DateTimeOriginal",

72 "CreateDate",

73 # "Duration", -- Wrong values when read from memory

74 "NumChannels",

75 "SampleRate",

76 "AvgBytesPerSec",

77 "BitsPerSample",

78 ]:

79 if f in metadata:

80 md_content += f"{f}: {metadata[f]}\n"

82 # Figure out the audio format for transcription

83 if stream_info.extension == ".wav" or stream_info.mimetype == "audio/x-wav":

84 audio_format = "wav"

85 elif stream_info.extension == ".mp3" or stream_info.mimetype == "audio/mpeg":

86 audio_format = "mp3"

87 elif (

88 stream_info.extension in [".mp4", ".m4a"]

89 or stream_info.mimetype == "video/mp4"

90 ):

91 audio_format = "mp4"

92 elif stream_info.extension == ".webm" or stream_info.mimetype in ("audio/webm", "video/webm"):

93 audio_format = "webm"

94 else:

95 audio_format = None

97 # Transcribe

98 if audio_format:

99 try:

100 md_content = transcribe_audio(file_stream, audio_format=audio_format, language=language)

101 except MissingDependencyException:

102 pass

103

104 # Return the result

105 return DocumentConverterResult(markdown=md_content.strip())

Added g4f/integration/markitdown/_base_converter.py +10 -0

@@ -0,0 +1,10 @@

1 from typing import Awaitable

3 class AsyncDocumentConverterResult:

4 """The result of converting a document to Markdown."""

6 def __init__(

7 self,

8 text_content: Awaitable[str],

9 ):

10 self.text_content = text_content

Added g4f/integration/markitdown/_image_converter.py +92 -0

@@ -0,0 +1,92 @@

1 from typing import BinaryIO, Any

2 import asyncio

3 from markitdown._base_converter import DocumentConverter, DocumentConverterResult

4 from markitdown._stream_info import StreamInfo

5 from markitdown.converters._llm_caption import llm_caption

6 from markitdown.converters._exiftool import exiftool_metadata

8 from ._base_converter import AsyncDocumentConverterResult

10 ACCEPTED_MIME_TYPE_PREFIXES = [

11 "image/jpeg",

12 "image/png",

13 ]

15 ACCEPTED_FILE_EXTENSIONS = [".jpg", ".jpeg", ".png"]

18 class ImageConverter(DocumentConverter):

19 """

    Converts images to markdown via extraction of metadata (if `exiftool` is installed), and description via a multimodal LLM (if an llm_client is configured).

21 """

23 def accepts(

24 self,

25 file_stream: BinaryIO,

26 stream_info: StreamInfo,

27 **kwargs: Any,

28 ) -> bool:

29 mimetype = (stream_info.mimetype or "").lower()

30 extension = (stream_info.extension or "").lower()

32 if extension in ACCEPTED_FILE_EXTENSIONS:

33 return True

35 for prefix in ACCEPTED_MIME_TYPE_PREFIXES:

36 if mimetype.startswith(prefix):

37 return True

39 return False

41 def convert(

42 self,

43 file_stream: BinaryIO,

44 stream_info: StreamInfo,

45 **kwargs: Any, # Options to pass to the converter

46 ) -> DocumentConverterResult:

47 md_content = ""

49 # Add metadata

50 metadata = exiftool_metadata(

51 file_stream, exiftool_path=kwargs.get("exiftool_path")

52 )

54 if metadata:

55 for f in [

56 "ImageSize",

57 "Title",

58 "Caption",

59 "Description",

60 "Keywords",

61 "Artist",

62 "Author",

63 "DateTimeOriginal",

64 "CreateDate",

65 "GPSPosition",

66 ]:

67 if f in metadata:

68 md_content += f"{f}: {metadata[f]}\n"

70 # Try describing the image with GPT

71 llm_client = kwargs.get("llm_client")

72 llm_model = kwargs.get("llm_model")

73 if llm_client is not None and llm_model is not None:

74 llm_description = llm_caption(

75 file_stream,

76 stream_info,

77 client=llm_client,

78 model=llm_model,

79 prompt=kwargs.get("llm_prompt"),

80 )

82 if asyncio.iscoroutine(llm_description):

83 return AsyncDocumentConverterResult(

84 llm_description,

85 )

87 if llm_description is not None:

88 md_content += "\n# Description:\n" + llm_description.strip() + "\n"

90 return DocumentConverterResult(

91 markdown=md_content,

92 )

Added g4f/integration/markitdown/_llm_caption.py +56 -0

@@ -0,0 +1,56 @@

1 from typing import BinaryIO, Union, Awaitable

2 import base64

3 import mimetypes

4 import asyncio

5 from markitdown._stream_info import StreamInfo

8 def llm_caption(

9 file_stream: BinaryIO, stream_info: StreamInfo, *, client, model, prompt=None

10 ) -> Union[None, str, Awaitable[str]]:

11 if prompt is None or prompt.strip() == "":

12 prompt = "Write a detailed caption for this image."

14 # Get the content type

15 content_type = stream_info.mimetype

16 if not content_type:

17 content_type, _ = mimetypes.guess_type("_dummy" + (stream_info.extension or ""))

18 if not content_type:

19 content_type = "application/octet-stream"

21 # Convert to base64

22 cur_pos = file_stream.tell()

23 try:

24 base64_image = base64.b64encode(file_stream.read()).decode("utf-8")

25 except Exception as e:

26 return None

27 finally:

28 file_stream.seek(cur_pos)

30 # Prepare the data-uri

31 data_uri = f"data:{content_type};base64,{base64_image}"

33 # Prepare the OpenAI API request

34 messages = [

35 {

36 "role": "user",

37 "content": [

38 {"type": "text", "text": prompt},

39 {

40 "type": "image_url",

41 "image_url": {

42 "url": data_uri,

43 },

44 },

45 ],

46 }

47 ]

49 # Call the OpenAI API

50 response = client.chat.completions.create(model=model, messages=messages)

51 if asyncio.iscoroutine(response):

52 async def read_content(response):

53 response = await response

54 return response.choices[0].message.content

55 return read_content(response)

56 return response.choices[0].message.content

Added g4f/integration/markitdown/_transcribe_audio.py +49 -0

@@ -0,0 +1,49 @@

1 import io

2 import sys

3 from typing import BinaryIO

4 from markitdown._exceptions import MissingDependencyException

6 # Try loading optional (but in this case, required) dependencies

7 # Save reporting of any exceptions for later

8 _dependency_exc_info = None

9 try:

10 # Suppress some warnings on library import

11 import warnings

13 with warnings.catch_warnings():

14 warnings.filterwarnings("ignore", category=DeprecationWarning)

15 warnings.filterwarnings("ignore", category=SyntaxWarning)

16 import speech_recognition as sr

17 import pydub

18 except ImportError:

19 # Preserve the error and stack trace for later

20 _dependency_exc_info = sys.exc_info()

23 def transcribe_audio(file_stream: BinaryIO, *, audio_format: str = "wav", language: str = "en-US") -> str:

24 # Check for installed dependencies

25 if _dependency_exc_info is not None:

26 raise MissingDependencyException(

            "Speech transcription requires installing MarkItdown with the [audio-transcription] optional dependencies. E.g., `pip install markitdown[audio-transcription]` or `pip install markitdown[all]`"

28 ) from _dependency_exc_info[

29 1

30 ].with_traceback( # type: ignore[union-attr]

31 _dependency_exc_info[2]

32 )

34 if audio_format in ["wav", "aiff", "flac"]:

35 audio_source = file_stream

36 elif audio_format in ["mp3", "mp4", "webm"]:

37 audio_segment = pydub.AudioSegment.from_file(file_stream, format=audio_format)

39 audio_source = io.BytesIO()

40 audio_segment.export(audio_source, format="wav")

41 audio_source.seek(0)

42 else:

43 raise ValueError(f"Unsupported audio format: {audio_format}")

45 recognizer = sr.Recognizer()

46 with sr.AudioFile(audio_source) as source:

47 audio = recognizer.record(source)

48 transcript = recognizer.recognize_google(audio, language=language).strip()

49 return "[No speech detected]" if transcript == "" else transcript

Modified g4f/providers/response.py +1 -4

Modified g4f/tools/files.py +1 -1

Modified g4f/tools/media.py +6 -0