gpt4free

公开

关注 0 Fork 0 Star 0

XFEstudio/gpt4free

Improve LMArena auth, image handling & cookies (#3413)

* Improve LMArena auth, image handling & cookies Qwen: avoid calling auth endpoint when no token is provided. LMArena: add URL expiry check for cached images and use cls.url as referer/endpoint for image requests; extract and validate next-action IDs from JS via a new __extract_actions helper; update several next-action fingerprints; add parsing for AWS-style signed URLs using parse_qs and UTC-aware datetime; change prepare_images to skip expired cached images. Introduce multi-model support (modelA/modelB) with mode selection (direct/side-by-side/battle), additional message IDs, and include model IDs in the evaluation payload. Improve stream parsing to handle b2 image chunks and tag ImageResponse with model metadata; adjust heartbeat handling and rate-limit behavior to trigger cookie clearing. Requests/nodriver: add clear_cookies_for_url to remove cookies via CDP and wire a clear_cookies_except argument through get_args_from_nodriver so callers can clear cookies selectively; add typing for Callable in get_nodriver return. Misc: minor imports and logging tweaks. * Update LMArena.py * datetime.timezone.utc)

079a5fd2

Ammar <ammar.alkotb@gmail.com>

提交于 2026-04-04 02:21

父提交 e5bdc90

代码差异

3 个文件 +147 -49

Modified g4f/Provider/Qwen.py +7 -5

@@ -197,7 +197,7 @@ class Qwen(AsyncGeneratorProvider, ProviderModelMixin):

197 197 file_id = data.get("file_id")

198 198

199 199 # Put File into Url

200 str_date = datetime.datetime.now(datetime.UTC).strftime('%Y%m%dT%H%M%SZ')

200 str_date = datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ')

201 201 headers = get_oss_headers('PUT', str_date, data, file_type)

202 202 async with session.put(

203 203 file_url.split("?")[0],

@@ -397,11 +397,13 @@ class Qwen(AsyncGeneratorProvider, ProviderModelMixin):

397 397 model_name = cls.get_model(model)

398 398 prompt = get_last_user_message(messages)

399 399 timeout = kwargs.get("timeout") or 5 * 60

400 async with StreamSession(headers=cls._get_headers(kwargs.get("token"))) as session:

400 token = kwargs.get("token")

401 async with StreamSession(headers=cls._get_headers(token)) as session:

401 402 try:

402 async with session.get('https://chat.qwen.ai/api/v1/auths/', proxy=proxy) as user_info_res:

403 await cls.raise_for_status(user_info_res)

404 debug.log(await user_info_res.json())

403 if token:

404 async with session.get('https://chat.qwen.ai/api/v1/auths/', proxy=proxy) as user_info_res:

405 await cls.raise_for_status(user_info_res)

406 debug.log(await user_info_res.json())

405 407 except Exception as e:

406 408 debug.error(e)

407 409 for attempt in range(5):

Modified g4f/Provider/needs_auth/LMArena.py +107 -42

@@ -7,12 +7,12 @@ import os

7 7 import re

8 8 import secrets

9 9 import time

10 from datetime import datetime

10 from datetime import datetime, timezone

11 11 from pathlib import Path

12 12 from typing import Dict

13 from urllib.parse import urlparse

13 from urllib.parse import urlparse, parse_qs

14 15

15 import requests

16 16

17 17 from g4f.image import to_bytes, detect_file_type

18 18

@@ -62,6 +62,22 @@ def uuid7():

62 62

63 63 # Global variables to manage Image Cache

64 64 ImagesCache: Dict[str, dict[str, str]] = {}

67 def check_link_expiry(url):

68 # Parse the URL and its query parameters

69 parsed_url = urlparse(url)

70 params = parse_qs(parsed_url.query)

71 amz_date_str = params.get("X-Amz-Date", [None])[0]

72 expires_delta = params.get("X-Amz-Expires", [None])[0]

73 if not amz_date_str or not expires_delta:

74 return False

75 creation_time = datetime.strptime(amz_date_str, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)

76 expiry_time = creation_time.timestamp() + int(expires_delta)

77 current_time = datetime.now(timezone.utc).timestamp()

78 return current_time <= expiry_time

65 81 if has_nodriver:

66 82 async def click_trunstile(page: nodriver.Tab, element='document.getElementById("cf-turnstile")'):

67 83 for _ in range(3):

@@ -93,12 +109,15 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

93 109 _models_loaded = False

94 110 image_cache = True

95 111 _next_actions = {

96 "generateUploadUrl":"7020462b741e358317f3b5a1929766d8b9c241c7c6",

97 "getSignedUrl":"60ff7bb683b22dd00024c9aee7664bbd39749e25c9",

112 "generateUploadUrl": "7012303914af71fce235a732cde90253f7e2986f2b",

113 "getSignedUrl": "605373b76a30947cc26be49fc7b00c885910e21559",

98 114 "updateTouConsent": "40efff1040868c07750a939a0d8120025f246dfe28",

99 115 "createPointwiseFeedback": "605a0e3881424854b913fe1d76d222e50731b6037b",

100 "createPairwiseFeedback":"600777eb84863d7e79d85d214130d3214fc744c80f",

101 "getProxyImage": "60049198d4936e6b7acc63719b63b89284c58683e6"

116 "createPairwiseFeedback": "600777eb84863d7e79d85d214130d3214fc744c80f",

117 "getProxyImage": "60049198d4936e6b7acc63719b63b89284c58683e6",

118 "deleteEvaluationSession": "6009c985d7e84eae2ec94547453ba388005b22e2a5",

119 "getEmailProvider": "607c2dd3d84af5a00b322b577498d1b2a739c5dfe0",

120 "deleteAccount": "40a57e8c369eaf8a82483fae2f8106489ce041dffd",

102 121 }

103 122

104 123 @classmethod

@@ -178,7 +197,7 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

178 197 return cls.models

179 198

180 199 @classmethod

181 async def get_args_from_nodriver(cls, proxy):

200 async def get_args_from_nodriver(cls, proxy, clear_cookies=False):

182 201 cache_file = cls.get_cache_file()

183 202 grecaptcha = []

184 203

@@ -234,7 +253,8 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

234 253 html = await page.get_content()

235 254 await cls.__load_actions(html)

236 255

237 args = await get_args_from_nodriver(cls.url, proxy=proxy, callback=callback)

256 args = await get_args_from_nodriver(cls.url, proxy=proxy, callback=callback,

257

                                            clear_cookies_except=["cf_clearance", "app_banner_state"] if clear_cookies else None)

238 258

239 259 with cache_file.open("w") as f:

240 260 json.dump(args, f)

@@ -338,12 +358,9 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

338 358 js_url = f"{cls.url}/_next/{js}"

339 359 async with session.get(js_url) as js_response:

340 360 js_text = await js_response.text()

341 if "generateUploadUrl" in js_text:

342

                                        # updateTouConsent, createPointwiseFeedback, createPairwiseFeedback, generateUploadUrl, getSignedUrl, getProxyImage

343 start_id = re.findall(r'\("([a-f0-9]{40,})".*?"(\w+)"\)', js_text)

344 for v, k in start_id:

345 cls._next_actions[k] = v

346 break

361 if "createServerReference" in js_text:

362 cls.__extract_actions(js_text)

363

347 364 elif chunk_data.startswith(("[", "{")):

348 365 try:

349 366 data = json.loads(chunk_data)

@@ -351,12 +368,22 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

351 368 except json.decoder.JSONDecodeError:

352 369 ...

353 370

371 @classmethod

372 def __extract_actions(cls, js_text):

373

        # updateTouConsent, createPointwiseFeedback, createPairwiseFeedback, generateUploadUrl, getSignedUrl, getProxyImage

374 start_id = re.findall(r'\("([a-f0-9]{40,})".*?"(\w+)"\)', js_text)

375 for v, k in start_id:

376 if len(v) == 42:

377 cls._next_actions[k] = v

378 debug.log(f"{k}: {v}")

379 else:

380 debug.error(f"wrong {k} value: {v}")

381

354 382 @classmethod

355 383 async def prepare_images(cls, args, media: list[tuple]) -> list[dict[str, str]]:

356 384 files = []

357 385 if not media:

358 386 return files

359 url = "https://arena.ai/?chat-modality=image"

360 387 async with StreamSession(**args, ) as session:

361 388 for index, (_file, file_name) in enumerate(media):

362 389 data_bytes = to_bytes(_file)

@@ -366,20 +393,22 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

366 393 image_hash = hasher.hexdigest()

367 394 file = ImagesCache.get(image_hash)

368 395 if cls.image_cache and file:

369 debug.log("Using cached image")

370 files.append(file)

371 continue

396 if check_link_expiry(file.get("url")):

397 debug.log("Using cached image")

398 files.append(file)

399 continue

400 debug.log("Expiry cached image")

372 401

373 402 extension, file_type = detect_file_type(data_bytes)

374 403 file_name = file_name or f"file-{len(data_bytes)}{extension}"

375 404 async with session.post(

376 url="https://arena.ai/?chat-modality=image",

405 url=cls.url,

377 406 json=[file_name, file_type],

378 407 headers={

379 408 "accept": "text/x-component",

380 409 "content-type": "text/plain;charset=UTF-8",

381 410 "next-action": cls._next_actions["generateUploadUrl"],

382 "referer": url

411 "referer": cls.url

383 412 }

384 413 ) as response:

385 414 await raise_for_status(response)

@@ -404,13 +433,13 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

404 433 ) as response:

405 434 await raise_for_status(response)

406 435 async with session.post(

407 url=url,

436 url=cls.url,

408 437 json=[key],

409 438 headers={

410 439 "accept": "text/x-component",

411 440 "content-type": "text/plain;charset=UTF-8",

412 441 "next-action": cls._next_actions["getSignedUrl"],

413 "referer": url

442 "referer": cls.url

414 443 }

415 444 ) as response:

416 445 await raise_for_status(response)

@@ -468,47 +497,62 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

468 497 cache_file = cls.get_cache_file()

469 498 args = cls.read_args(kwargs.get("lmarena_args", {}))

470 499 grecaptcha = kwargs.pop("grecaptcha", "")

500 _need_clear_cookies = False

471 501 for _ in range(2):

472 502 if args:

473 503 pass

474 504 elif has_nodriver:

475 args, grecaptcha = await cls.get_args_from_nodriver(proxy)

505 args, grecaptcha = await cls.get_args_from_nodriver(proxy, _need_clear_cookies)

476 506 else:

477 507 raise MissingRequirementsError("No auth file found and nodriver is not available.")

478 508

479 509 if not cls._models_loaded:

480 510 # change to async

481 511 await cls.get_models_async()

482 is_image_model = model in cls.image_models

483 if not model:

484 model = cls.default_model

485 if model in cls.model_aliases:

486 model = cls.model_aliases[model]

487 if model in cls.text_models:

488 model_id = cls.text_models[model]

489 elif model in cls.image_models:

490 model_id = cls.image_models[model]

491 elif model in cls.video_models:

492 model_id = cls.video_models[model]

493 else:

494 raise ModelNotFoundError(f"Model '{model}' is not supported by LMArena provider.")

495 512

513 def get_mode_id(_model):

514 model_id = None

515 # if not model:

516 # model = cls.default_model

517 if _model in cls.model_aliases:

518 _model = cls.model_aliases[_model]

519 if _model in cls.text_models:

520 model_id = cls.text_models[_model]

521 elif _model in cls.image_models:

522 model_id = cls.image_models[_model]

523 elif _model in cls.video_models:

524 model_id = cls.video_models[_model]

525 elif _model:

526 raise ModelNotFoundError(f"Model '{_model}' is not supported by LMArena provider.")

527 return model_id

528

529 modelA:str = model

530 modelB:str = kwargs.get("modelB", "")

531 modelAId = get_mode_id(modelA)

532 modelBId = get_mode_id(modelB) if modelB else None

533 if modelAId and modelBId:

534 mode = "side-by-side"

535 elif modelAId:

536 mode = "direct"

537 else:

538 mode = "battle"

496 539 if conversation and getattr(conversation, "evaluationSessionId", None):

497 540 url = cls.post_to_evaluation.format(id=conversation.evaluationSessionId)

498 541 evaluationSessionId = conversation.evaluationSessionId

499 542 else:

500 543 url = cls.create_evaluation

501 544 evaluationSessionId = str(uuid7())

545 is_image_model = modelA in cls.image_models

502 546 userMessageId = str(uuid7())

503 547 modelAMessageId = str(uuid7())

548 modelBMessageId = str(uuid7())

504 549 if not grecaptcha and has_nodriver:

505 550 debug.log("No grecaptcha token found, obtaining new one...")

506 551 args, grecaptcha = await cls.get_grecaptcha(args, proxy)

507 552 files = await cls.prepare_images(args, media)

508 553 data = {

509 554 "id": evaluationSessionId,

510 "mode": "direct",

511 "modelAId": model_id,

555 "mode": mode,

512 556 "userMessageId": userMessageId,

513 557 "modelAMessageId": modelAMessageId,

514 558 "userMessage": {

@@ -519,6 +563,13 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

519 563 "modality": "image" if is_image_model else "chat",

520 564 "recaptchaV3Token": grecaptcha

521 565 }

566 if modelAId:

567 data["modelAId"] = modelAId

568 if modelBId:

569 data["modelBId"] = modelBId

570 if mode in ["side-by-side", "battle"]:

571 data["modelBMessageId"] = modelBMessageId

572

522 573 yield JsonRequest.from_dict(data)

523 574 try:

524 575 async with StreamSession(**args, timeout=timeout or 5 * 60) as session:

@@ -537,17 +588,26 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

537 588 if chunk == "hasArenaError":

538 589 raise ModelNotFoundError("LMArena Beta encountered an error: hasArenaError")

539 590 yield chunk

591 elif line.startswith("b0:"):

592 ...

540 593 elif line.startswith("ag:"):

541 594 chunk = json.loads(line[3:])

542 595 yield Reasoning(chunk)

543 elif line.startswith("a2:") and line == 'a2:[{"type":"heartbeat"}]':

596

                            elif (line.startswith("a2:") or line.startswith("b2:")) and line == 'a2:[{"type":"heartbeat"}]':

544 597 # 'a2:[{"type":"heartbeat"}]'

545 598 continue

546 599 elif line.startswith("a2:"):

547 600 chunk = json.loads(line[3:])

548 601 __images = [image.get("image") for image in chunk if image.get("image")]

549 602 if __images:

550 yield ImageResponse(__images, prompt)

603 yield ImageResponse(__images, prompt, {"model": modelA})

604

605 elif line.startswith("b2:"):

606 chunk = json.loads(line[3:])

607 __images = [image.get("image") for image in chunk if image.get("image")]

608 if __images:

609 yield ImageResponse(__images, prompt, {"model": modelB})

610

551 611 elif line.startswith("ad:"):

552 612 yield JsonConversation(evaluationSessionId=evaluationSessionId)

553 613 finish = json.loads(line[3:])

@@ -555,8 +615,12 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

555 615 yield FinishReason(finish["finishReason"])

556 616 if "usage" in finish:

557 617 yield Usage(**finish["usage"])

618 elif line.startswith("bd:"):

619 ...

558 620 elif line.startswith("a3:"):

559 621 raise RuntimeError(f"LMArena: {json.loads(line[3:])}")

622 elif line.startswith("b3:"):

623 ...

560 624 else:

561 625 debug.log(f"LMArena: Unknown line prefix: {line[:2]}")

562 626 break

@@ -565,8 +629,9 @@ class LMArena(AsyncGeneratorProvider, ProviderModelMixin, AuthFileMixin):

565 629 debug.error(error)

566 630 debug.log(f"{cls.__name__}: Cloudflare error")

567 631 continue

568 except (RateLimitError) as error:

632 except RateLimitError as error:

569 633 args = None

634 _need_clear_cookies = True

570 635 debug.error(error)

571 636 continue

572 637 except:

Modified g4f/requests/__init__.py +33 -2

@@ -5,6 +5,7 @@ import json

5 5 import os

6 6 import random

7 7 import time

8 from collections.abc import Callable

8 9 from contextlib import asynccontextmanager

9 10 from http.cookies import Morsel

10 11 from pathlib import Path

@@ -90,6 +91,29 @@ def get_cookie_params_from_dict(cookies: Cookies, url: str = None, domain: str =

90 91 }) for key, value in cookies.items()]

91 92

92 93

94 async def clear_cookies_for_url(browser: Browser, url: str, ignore_cookies: list[str] = None):

95 host = urlparse(url).hostname

96 if not host:

97 raise ValueError(f"Bad url: {url}")

99 if ignore_cookies is None:

100 ignore_cookies = []

101 tab = browser.main_tab # any open tab is fine

102 cookies = await browser.cookies.get_all() # returns CDP cookies :contentReference[oaicite:2]{index=2}

103 for c in cookies:

104 dom = (c.domain or "").lstrip(".")

105 if dom and (host == dom or host.endswith("." + dom)):

106 if c.name in ignore_cookies:

107 continue

108 await tab.send(

109 nodriver.cdp.network.delete_cookies(

110 name=c.name,

111 domain=dom, # exact domain :contentReference[oaicite:3]{index=3}

112 path=c.path, # exact path :contentReference[oaicite:4]{index=4}

113 # partition_key=c.partition_key, # if you use partitioned cookies

114 )

115 )

116

93 117 async def get_args_from_nodriver(

94 118 url: str,

95 119 proxy: str = None,

@@ -99,14 +123,21 @@ async def get_args_from_nodriver(

99 123 cookies: Cookies = None,

100 124 browser: Browser = None,

101 125 user_data_dir: str = "nodriver",

102 browser_args: list = None

126 browser_args: list = None,

127 clear_cookies_except:list[str]=None,

103 128 ) -> dict:

129 if clear_cookies_except is None:

130 clear_cookies_except = []

104 131 if browser is None:

105 132

        browser, stop_browser = await get_nodriver(proxy=proxy, timeout=timeout, user_data_dir=user_data_dir, browser_args=browser_args)

106 133 else:

107 134 async def stop_browser():

108 135 pass

109 136 try:

137 if clear_cookies_except:

138 debug.log(f"Clear Cookies for url: {url}")

139 await clear_cookies_for_url(browser, url)

140

110 141 debug.log(f"Open nodriver with url: {url}")

111 142 if cookies is None:

112 143 cookies = {}

@@ -161,7 +192,7 @@ async def get_nodriver(

161 192 timeout: int = 300,

162 193 browser_executable_path: str = None,

163 194 **kwargs

164 ) -> tuple[Browser, callable]:

195 ) -> tuple[Browser, Callable]:

165 196 if not has_nodriver:

166 197 raise MissingRequirementsError(

167 198 'Install "zendriver" and "platformdirs" package | pip install -U zendriver platformdirs')