gpt4free

公开

关注 0 Fork 0 Star 1

XFEstudio/gpt4free

feat: enhance HAR provider, image handling, markdown upload & cache

- **g4f/Provider/har/__init__.py** - `get_models`/`create_async`: iterate over `(domain, harFile)` and filter with `domain in request_url` - `read_har_files` now yields `(domain, har_data)`; fixes file variable shadowing and uses `json.load` - remove stray `print`, add type hint for `find_str`, replace manual loops with `yield from` - small whitespace clean-up - **g4f/Provider/needs_auth/Grok.py** - `ImagePreview` now passes `auth_result.cookies` and `auth_result.headers` - **g4f/Provider/needs_auth/OpenaiChat.py** - add `Union` import; rename/refactor `get_generated_images` → `get_generated_image` - support `file-service://` and `sediment://` pointers; choose correct download URL - return `ImagePreview` or `ImageResponse` accordingly and stream each image part - propagate 422 errors, update prompt assignment and image handling paths - **g4f/client/__init__.py** - drop unused `ignore_working` parameter in sync/async `Completions` - normalise `media` argument: accept single tuple, infer filename when missing, fix index loop - `Images.create_variation` updated to use the new media logic - **g4f/gui/server/api.py** - expose `latest_version_cached` via `?cache=` query parameter - **g4f/gui/server/backend_api.py** - optional Markdown extraction via `MarkItDown`; save rendered text as `<file>.md` - upload flow rewrites: copy to temp file, move to bucket/media dir, clean temp, store filenames - introduce `has_markitdown` guard and improved logging/exception handling - **g4f/tools/files.py** - remove trailing spaces in HAR code-block header string - **g4f/version.py** - add `latest_version_cached` `@cached_property` for memoised version lookup

8f63f656

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-04-24 15:18

父提交 db47d71

代码差异

8 个文件 +95 -36

Modified g4f/Provider/har/__init__.py +8 -12

@@ -17,10 +17,10 @@ class HarProvider(AsyncGeneratorProvider, ProviderModelMixin):

17 17

18 18 @classmethod

19 19 def get_models(cls):

20 for harFile in read_har_files():

20 for domain, harFile in read_har_files():

21 21 for v in harFile['log']['entries']:

22 22 request_url = v['request']['url']

                if not request_url.startswith(cls.url) or "." in urlparse(request_url).path or "heartbeat" in request_url:

23 if domain not in request_url or "." in urlparse(request_url).path or "heartbeat" in request_url:

24 24 continue

25 25 if "\n\ndata: " not in v['response']['content']['text']:

26 26 continue

@@ -41,11 +41,11 @@ class HarProvider(AsyncGeneratorProvider, ProviderModelMixin):

41 41 session_hash = str(uuid.uuid4()).replace("-", "")

42 42 prompt = get_last_user_message(messages)

43 43

44 for harFile in read_har_files():

44 for domain, harFile in read_har_files():

45 45 async with StreamSession(impersonate="chrome") as session:

46 46 for v in harFile['log']['entries']:

47 47 request_url = v['request']['url']

                    if not request_url.startswith(cls.url) or "." in urlparse(request_url).path or "heartbeat" in request_url:

48 if domain not in request_url or "." in urlparse(request_url).path or "heartbeat" in request_url:

49 49 continue

50 50 postData = None

51 51 if "postData" in v['request']:

@@ -59,8 +59,6 @@ class HarProvider(AsyncGeneratorProvider, ProviderModelMixin):

59 59

60 60

                    async with getattr(session, method)(request_url, data=postData, headers=get_headers(v), proxy=proxy) as response:

61 61 await raise_for_status(response)

62 if "heartbeat" in request_url:

63 continue

64 62 returned_data = ""

65 63 async for line in response.iter_lines():

66 64 if not line.startswith(b"data: "):

@@ -83,9 +81,9 @@ def read_har_files():

83 81 for file in files:

84 82 if not file.endswith(".har"):

85 83 continue

86 with open(os.path.join(root, file), 'rb') as file:

84 with open(os.path.join(root, file), 'rb') as f:

87 85 try:

88 yield json.loads(file.read())

86 yield os.path.splitext(file)[0], json.load(f)

89 87 except json.JSONDecodeError:

90 88 raise RuntimeError(f"Failed to read HAR file: {file}")

91 89

@@ -98,7 +96,7 @@ def read_str_recusive(data):

98 96 elif isinstance(item, str):

99 97 yield item

100 98

101 def find_str(data, skip=0):

99 def find_str(data, skip: int = 0):

102 100 for item in read_str_recusive(data):

103 101 if skip > 0:

104 102 skip -= 1

@@ -110,7 +108,6 @@ def read_list_recusive(data, key):

110 108 if isinstance(data, dict):

111 109 for k, v in data.items():

112 110 if k == key:

113 print(k, v)

114 111 yield v

115 112 else:

116 113 yield from read_list_recusive(v, key)

@@ -123,8 +120,7 @@ def find_list(data, key):

123 120 if isinstance(item, str):

124 121 yield item

125 122 elif isinstance(item, list):

126 for sub_item in item:

127 yield sub_item

123 yield from item

128 124

129 125 def get_str_list(data):

130 126 for item in data:

Modified g4f/Provider/needs_auth/Grok.py +1 -1

@@ -111,7 +111,7 @@ class Grok(AsyncAuthedProvider, ProviderModelMixin):

111 111 response_data = result.get("response", {})

112 112 image = response_data.get("streamingImageGenerationResponse", None)

113 113 if image is not None:

114

                                yield ImagePreview(f'{cls.assets_url}/{image["imageUrl"]}', "", {"cookies": cookies, "headers": headers})

114

                                yield ImagePreview(f'{cls.assets_url}/{image["imageUrl"]}', "", {"cookies": auth_result.cookies, "headers": auth_result.headers})

115 115 token = response_data.get("token", result.get("token"))

116 116 is_thinking = response_data.get("isThinking", result.get("isThinking"))

117 117 if token:

Modified g4f/Provider/needs_auth/OpenaiChat.py +37 -10

@@ -8,7 +8,7 @@ import json

8 8 import base64

9 9 import time

10 10 import random

11 from typing import AsyncIterator, Iterator, Optional, Generator, Dict

11 from typing import AsyncIterator, Iterator, Optional, Generator, Dict, Union

12 12 from copy import copy

13 13

14 14 try:

@@ -254,22 +254,43 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

254 254 return messages

255 255

256 256 @classmethod

257

    async def get_generated_images(cls, session: StreamSession, auth_result: AuthResult, parts: list, prompt: str, conversation_id: str) -> AsyncIterator:

257

    async def get_generated_image(cls, session: StreamSession, auth_result: AuthResult, element: Union[dict, str], prompt: str = None, conversation_id: str = None) -> AsyncIterator:

258 258 download_urls = []

259 element = element.split("sediment://")[-1]

260 url = f"{cls.url}/backend-api/conversation/{conversation_id}/attachment/{element}/download"

261 debug.log(f"OpenaiChat: Downloading image: {url}")

259 is_sediment = False

260 if prompt is None:

261 try:

262 prompt = element["metadata"]["dalle"]["prompt"]

263 except KeyError:

264 pass

265 if "asset_pointer" in element:

266 element = element["asset_pointer"]

267 if isinstance(element, str) and element.startswith("file-service://"):

268 element = element.split("file-service://", 1)[-1]

269 if isinstance(element, str) and element.startswith("sediment://"):

270 is_sediment = True

271 element = element.split("sediment://")[-1]

272 else:

273 raise RuntimeError(f"Invalid image element: {element}")

274 if is_sediment:

275 url = f"{cls.url}/backend-api/conversation/{conversation_id}/attachment/{element}/download"

276 else:

277 url =f"{cls.url}/backend-api/files/{element}/download"

262 278 try:

263 279 async with session.get(url, headers=auth_result.headers) as response:

264 280 cls._update_request_args(auth_result, session)

265 281 await raise_for_status(response)

266 282 data = await response.json()

267 283 download_url = data.get("download_url")

268 download_urls.append(download_url)

284 if download_url is not None:

285 download_urls.append(download_url)

286 debug.log(f"OpenaiChat: Found image: {download_url}")

287 else:

288 debug.log("OpenaiChat: No download URL found in response: ", data)

269 289 except Exception as e:

270 290 debug.error("OpenaiChat: Download image failed")

271 291 debug.error(e)

272 return ImagePreview(download_urls, prompt)

292 if download_urls:

293 return ImagePreview(download_urls, prompt) if is_sediment else ImageResponse(download_urls, prompt)

273 294

274 295 @classmethod

275 296 async def create_authed(

@@ -402,7 +423,7 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

402 423 if conversation.conversation_id is not None:

403 424 data["conversation_id"] = conversation.conversation_id

404 425 debug.log(f"OpenaiChat: Use conversation: {conversation.conversation_id}")

405 conversation.prompt = format_image_prompt(messages, prompt)

426 prompt = conversation.prompt = format_image_prompt(messages, prompt)

406 427 if action != "continue":

407 428 data["parent_message_id"] = getattr(conversation, "parent_message_id", conversation.message_id)

408 429 conversation.parent_message_id = None

@@ -430,6 +451,8 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

430 451 cls._update_request_args(auth_result, session)

431 452 if response.status in (401, 403, 429):

432 453 raise MissingAuthError("Access token is not valid")

454 elif response.status == 422:

455 raise RuntimeError((await response.json()), data)

433 456 await raise_for_status(response)

434 457 buffer = u""

435 458 async for line in response.iter_lines():

@@ -515,7 +538,9 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

515 538 elif m.get("p") == "/message/metadata/image_gen_title":

516 539 fields.prompt = m.get("v")

517 540 elif m.get("p") == "/message/content/parts/0/asset_pointer":

518

                        fields.generated_images = await cls.get_generated_images(session, auth_result, m.get("v"), fields.prompt, fields.conversation_id)

541

                        generated_images = fields.generated_images = await cls.get_generated_image(session, auth_result, m.get("v"), fields.prompt, fields.conversation_id)

542 if generated_images is not None:

543 yield generated_images

519 544 elif m.get("p") == "/message/metadata/search_result_groups":

520 545 for entry in [p.get("entries") for p in m.get("v")]:

521 546 for link in entry:

@@ -544,7 +569,9 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

544 569 fields.is_thinking = True

545 570 yield Reasoning(status=m.get("metadata", {}).get("initial_text"))

546 571 if c.get("content_type") == "multimodal_text":

547

                        yield await cls.get_generated_images(session, auth_result, c.get("parts"), fields.prompt, fields.conversation_id)

572 for part in c.get("parts"):

573 if isinstance(part, dict) and part.get("content_type") == "image_asset_pointer":

574

                                yield await cls.get_generated_image(session, auth_result, part, fields.prompt, fields.conversation_id)

548 575 if m.get("author", {}).get("role") == "assistant":

549 576 if fields.parent_message_id is None:

550 577 fields.parent_message_id = v.get("message", {}).get("id")

Modified g4f/client/__init__.py +12 -4

@@ -291,16 +291,18 @@ class Completions:

291 291 max_tokens: Optional[int] = None,

292 292 stop: Optional[Union[list[str], str]] = None,

293 293 api_key: Optional[str] = None,

294 ignore_working: Optional[bool] = False,

295 294 ignore_stream: Optional[bool] = False,

296 295 **kwargs

297 296 ) -> ChatCompletion:

298 297 if isinstance(messages, str):

299 298 messages = [{"role": "user", "content": messages}]

300 299 if image is not None:

301 kwargs["media"] = [(image, image_name)]

300 kwargs["media"] = (image, image_name)

302 301 elif "images" in kwargs:

303 302 kwargs["media"] = kwargs.pop("images")

303 for idx, media in kwargs.get("media", []):

304 if not isinstance(media, (list, tuple)):

305 kwargs["media"][idx] = (media[0], media[1] if media[1] is not None else getattr(image, "name", None))

304 306 if provider is None:

305 307 provider = self.provider

306 308 if provider is None:

@@ -493,7 +495,10 @@ class Images:

493 495 proxy = self.client.proxy

494 496 prompt = "create a variation of this image"

495 497 if image is not None:

496 kwargs["media"] = [(image, None)]

498 kwargs["media"] = image

499 for idx, media in kwargs.get("media", []):

500 if not isinstance(media, (list, tuple)):

501 kwargs["media"][idx] = (media[0], media[1] if media[1] is not None else getattr(image, "name", None))

497 502

498 503 error = None

499 504 response = None

@@ -591,7 +596,6 @@ class AsyncCompletions:

591 596 max_tokens: Optional[int] = None,

592 597 stop: Optional[Union[list[str], str]] = None,

593 598 api_key: Optional[str] = None,

594 ignore_working: Optional[bool] = False,

595 599 ignore_stream: Optional[bool] = False,

596 600 **kwargs

597 601 ) -> Awaitable[ChatCompletion]:

@@ -601,6 +605,10 @@ class AsyncCompletions:

601 605 kwargs["media"] = [(image, image_name)]

602 606 elif "images" in kwargs:

603 607 kwargs["media"] = kwargs.pop("images")

608 for idx, media in kwargs.get("media", []):

609 if not isinstance(media, (list, tuple)):

610 kwargs["media"][idx] = (media[0], media[1] if media[1] is not None else getattr(image, "name", None))

611

604 612 if provider is None:

605 613 provider = self.provider

606 614 if provider is None:

Modified g4f/gui/server/api.py +5 -2

@@ -4,7 +4,7 @@ import logging

4 4 import os

5 5 import asyncio

6 6 from typing import Iterator

7 from flask import send_from_directory

7 from flask import send_from_directory, request

8 8 from inspect import signature

9 9

10 10 from ...errors import VersionNotFoundError, MissingAuthError

@@ -87,7 +87,10 @@ class Api:

87 87 latest_version = None

88 88 try:

89 89 current_version = version.utils.current_version

90 latest_version = version.utils.latest_version

90 if request.args.get("cache"):

91 latest_version = version.utils.latest_version_cached

92 else:

93 latest_version = version.utils.latest_version

91 94 except VersionNotFoundError:

92 95 pass

93 96 return {

Modified g4f/gui/server/backend_api.py +27 -6

@@ -16,6 +16,12 @@ from pathlib import Path

16 16 from urllib.parse import quote_plus

17 17 from hashlib import sha256

18 18

19 try:

20 from markitdown import MarkItDown

21 has_markitdown = True

22 except ImportError:

23 has_markitdown = False

19 25 from ...client.service import convert_to_provider

20 26 from ...providers.asyncio import to_sync_generator

21 27 from ...providers.response import FinishReason

@@ -299,8 +305,24 @@ class Backend_Api(Api):

299 305 filenames = []

300 306 media = []

301 307 for file in request.files.getlist('files'):

302 try:

303 filename = secure_filename(file.filename)

308 # Copy the file to a temporary location

309 filename = secure_filename(file.filename)

310 copyfile = tempfile.NamedTemporaryFile(suffix=filename, delete=False)

311 shutil.copyfileobj(file.stream, copyfile)

312 copyfile.close()

313 file.stream.close()

314

315 result = None

316 if has_markitdown:

317 try:

318 md = MarkItDown()

319 result = md.convert(copyfile.name).text_content

320 with open(os.path.join(bucket_dir, f"{filename}.md"), 'w') as f:

321 f.write(f"{result.text_content}\n")

322 filenames.append(f"{filename}.md")

323 except Exception as e:

324 logger.exception(e)

325 if not result:

304 326 if is_allowed_extension(filename):

305 327 os.makedirs(media_dir, exist_ok=True)

306 328 newfile = os.path.join(media_dir, filename)

@@ -309,11 +331,10 @@ class Backend_Api(Api):

309 331 newfile = os.path.join(bucket_dir, filename)

310 332 filenames.append(filename)

311 333 else:

334 os.remove(copyfile.name)

312 335 continue

313 with open(newfile, 'wb') as f:

314 shutil.copyfileobj(file.stream, f)

315 finally:

316 file.stream.close()

336 shutil.copyfile(copyfile.name, newfile)

337 os.remove(copyfile.name)

317 338 with open(os.path.join(bucket_dir, "files.txt"), 'w') as f:

318 339 [f.write(f"{filename}\n") for filename in filenames]

319 340 return {"bucket_id": bucket_id, "files": filenames, "media": media}

Modified g4f/tools/files.py +1 -1

@@ -187,7 +187,7 @@ def stream_read_files(bucket_dir: Path, filenames: list, delete_files: bool = Fa

187 187 else:

188 188 os.unlink(filepath)

189 189 continue

190 yield f"```{filename}\n"

191 191 if has_pypdf2 and filename.endswith(".pdf"):

192 192 try:

193 193 reader = PyPDF2.PdfReader(file_path)

Modified g4f/version.py +4 -0

@@ -104,6 +104,10 @@ class VersionUtils:

104 104 return get_github_version(GITHUB_REPOSITORY)

105 105 return get_pypi_version(PACKAGE_NAME)

106 106

107 @cached_property

108 def latest_version_cached(self) -> str:

109 return self.latest_version

110

107 111 def check_version(self) -> None:

108 112 """

109 113 Checks if the current version of 'g4f' is up to date with the latest version.