gpt4free

公开

关注 0 Fork 0 Star 0

返回提交历史

XFEstudio/gpt4free

Update docker tags in workfloe for slim images, Update read har file in OpenaiChat provider Remove webdriver in OpenaiChat provider Add supported_encodings and supported_encodings in OpenaiChat

b7a8e032

Heiner Lohaus <hlohaus@users.noreply.github.com>

提交于 2024-11-17 19:51

父提交 ea14480

代码差异

4 个文件 +104 -191

Modified .github/workflows/publish-workflow.yaml +2 -2

@@ -55,8 +55,8 @@ jobs:

55 55 file: docker/Dockerfile-slim

56 56 push: true

57 57 tags: |

58 hlohaus789/g4f=slim

59 hlohaus789/g4f=${{ github.ref_name }}-slim

58 hlohaus789/g4f:slim-latest

59 hlohaus789/g4f:${{ github.ref_name }}-slim

60 60 labels: ${{ steps.metadata.outputs.labels }}

61 61 build-args: |

62 62 G4F_VERSION=${{ github.ref_name }}

Modified g4f/Provider/needs_auth/OpenaiChat.py +73 -162

@@ -9,22 +9,19 @@ from aiohttp import ClientWebSocketResponse

9 9 from copy import copy

10 10

11 11 try:

12 import webview

13 has_webview = True

12 import nodriver

13 has_nodriver = True

14 14 except ImportError:

15 has_webview = False

15 has_nodriver = False

17 16 try:

18 from selenium.webdriver.common.by import By

19 from selenium.webdriver.support.ui import WebDriverWait

20 from selenium.webdriver.support import expected_conditions as EC

17 from platformdirs import user_config_dir

18 has_platformdirs = True

21 19 except ImportError:

22 pass

20 has_platformdirs = False

23 21

24 22 from ..base_provider import AsyncGeneratorProvider, ProviderModelMixin

25 from ...webdriver import get_browser

26 23 from ...typing import AsyncResult, Messages, Cookies, ImageType, AsyncIterator

27 from ...requests import get_args_from_browser, raise_for_status

24 from ...requests.raise_for_status import raise_for_status

28 25 from ...requests.aiohttp import StreamSession

29 26 from ...image import ImageResponse, ImageRequest, to_image, to_bytes, is_accepted_format

30 27 from ...errors import MissingAuthError, ResponseError

@@ -62,12 +59,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

62 59 default_model = None

63 60 default_vision_model = "gpt-4o"

64 61 models = [ "auto", "gpt-4o-mini", "gpt-4o", "gpt-4", "gpt-4-gizmo"]

66 model_aliases = {

67 #"gpt-4-turbo": "gpt-4",

68 #"gpt-4": "gpt-4-gizmo",

69 #"dalle": "gpt-4",

70 }

71 63 _api_key: str = None

72 64 _headers: dict = None

73 65 _cookies: Cookies = None

@@ -219,9 +211,12 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

219 211 """

220 212 # Create a message object with the user role and the content

221 213 messages = [{

222 "id": str(uuid.uuid4()),

223 214 "author": {"role": message["role"]},

224 215 "content": {"content_type": "text", "parts": [message["content"]]},

216 "id": str(uuid.uuid4()),

217 "create_time": int(time.time()),

218 "id": str(uuid.uuid4()),

219 "metadata": {"serialization_metadata": {"custom_symbol_offsets": []}}

225 220 } for message in messages]

226 221

227 222 # Check if there is an image response

@@ -250,7 +245,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

250 245 return messages

251 246

252 247 @classmethod

253 async def get_generated_image(cls, session: StreamSession, headers: dict, line: dict) -> ImageResponse:

248 async def get_generated_image(cls, session: StreamSession, headers: dict, element: dict) -> ImageResponse:

254 249 """

255 250 Retrieves the image response based on the message content.

256 251

@@ -269,15 +264,8 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

269 264 Raises:

270 265

            RuntimeError: If there'san error in downloading the image, including issues with the HTTP request or response.

271 266 """

272 if "parts" not in line["message"]["content"]:

273 return

274 first_part = line["message"]["content"]["parts"][0]

275 if "asset_pointer" not in first_part or "metadata" not in first_part:

276 return

277 if first_part["metadata"] is None or first_part["metadata"]["dalle"] is None:

278 return

279 prompt = first_part["metadata"]["dalle"]["prompt"]

280 file_id = first_part["asset_pointer"].split("file-service://", 1)[1]

267 prompt = element["metadata"]["dalle"]["prompt"]

268 file_id = element["asset_pointer"].split("file-service://", 1)[1]

281 269 try:

282 270 async with session.get(f"{cls.url}/backend-api/files/{file_id}/download", headers=headers) as response:

283 271 cls._update_request_args(session)

@@ -365,9 +353,10 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

365 353 if cls._expires is not None and cls._expires < time.time():

366 354 cls._headers = cls._api_key = None

367 355 arkose_token = None

368 proofTokens = None

356 proofToken = None

357 turnstileToken = None

369 358 try:

370 arkose_token, api_key, cookies, headers, proofTokens = await getArkoseAndAccessToken(proxy)

359

                arkose_token, api_key, cookies, headers, proofToken, turnstileToken = await getArkoseAndAccessToken(proxy)

371 360 cls._create_request_args(cookies, headers)

372 361 cls._set_api_key(api_key)

373 362 except NoValidHarFileError as e:

@@ -400,32 +389,28 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

400 389 f"{cls.url}/backend-anon/sentinel/chat-requirements"

401 390 if cls._api_key is None else

402 391 f"{cls.url}/backend-api/sentinel/chat-requirements",

403

                    json={"p": generate_proof_token(True, user_agent=cls._headers["user-agent"], proofTokens=proofTokens)},

392

                    json={"p": generate_proof_token(True, user_agent=cls._headers["user-agent"], proofToken=proofToken)},

404 393 headers=cls._headers

405 394 ) as response:

406 395 cls._update_request_args(session)

407 396 await raise_for_status(response)

408 requirements = await response.json()

409 text_data = json.loads(requirements.get("text", "{}"))

410 need_arkose = text_data.get("turnstile", {}).get("required", False)

411 if need_arkose:

412 arkose_token = text_data.get("turnstile", {}).get("dx")

413 else:

414 need_arkose = requirements.get("arkose", {}).get("required", False)

415 chat_token = requirements["token"]

397 chat_requirements = await response.json()

398 need_turnstile = chat_requirements.get("turnstile", {}).get("required", False)

399 need_arkose = chat_requirements.get("arkose", {}).get("required", False)

400 chat_token = chat_requirements.get("token")

416 401

417 402 if need_arkose and arkose_token is None:

418 arkose_token, api_key, cookies, headers, proofTokens = await getArkoseAndAccessToken(proxy)

403

                    arkose_token, api_key, cookies, headers, proofToken, turnstileToken = await getArkoseAndAccessToken(proxy)

419 404 cls._create_request_args(cookies, headers)

420 405 cls._set_api_key(api_key)

421 406 if arkose_token is None:

422 407 raise MissingAuthError("No arkose token found in .har file")

423 408

424 if "proofofwork" in requirements:

409 if "proofofwork" in chat_requirements:

425 410 proofofwork = generate_proof_token(

426 **requirements["proofofwork"],

411 **chat_requirements["proofofwork"],

427 412 user_agent=cls._headers["user-agent"],

428 proofTokens=proofTokens

413 proofToken=proofToken

429 414 )

430 415 if debug.logging:

431 416 print(

@@ -441,15 +426,18 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

441 426 websocket_request_id = str(uuid.uuid4())

442 427 data = {

443 428 "action": action,

444 "conversation_mode": {"kind": "primary_assistant"},

445 "force_paragen": False,

446 "force_rate_limit": False,

447 "conversation_id": conversation.conversation_id,

429 "messages": None,

448 430 "parent_message_id": conversation.message_id,

449 431 "model": model,

432 "paragen_cot_summary_display_override": "allow",

450 433 "history_and_training_disabled": history_disabled and not auto_continue and not return_conversation,

451 "websocket_request_id": websocket_request_id

434 "conversation_mode": {"kind":"primary_assistant"},

435 "websocket_request_id": websocket_request_id,

436 "supported_encodings": ["v1"],

437 "supports_buffering": True

452 438 }

439 if conversation.conversation_id is not None:

440 data["conversation_id"] = conversation.conversation_id

453 441 if action != "continue":

454 442 messages = messages if conversation_id is None else [messages[-1]]

455 443 data["messages"] = cls.create_messages(messages, image_request)

@@ -458,10 +446,12 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

458 446 "Openai-Sentinel-Chat-Requirements-Token": chat_token,

459 447 **cls._headers

460 448 }

461 if need_arkose:

449 if arkose_token:

462 450 headers["Openai-Sentinel-Arkose-Token"] = arkose_token

463 451 if proofofwork is not None:

464 452 headers["Openai-Sentinel-Proof-Token"] = proofofwork

453 if need_turnstile and turnstileToken is not None:

454 headers['openai-sentinel-turnstile-token'] = turnstileToken

465 455 async with session.post(

466 456 f"{cls.url}/backend-anon/conversation"

467 457 if cls._api_key is None else

@@ -510,7 +500,6 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

510 500 fields: Conversation,

511 501 ws = None

512 502 ) -> AsyncIterator:

513 last_message: int = 0

514 503 async for message in messages:

515 504 if message.startswith(b'{"wss_url":'):

516 505 message = json.loads(message)

@@ -527,10 +516,6 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

527 516 async for chunk in cls.iter_messages_line(session, message, fields):

528 517 if fields.finish_reason is not None:

529 518 break

530 elif isinstance(chunk, str):

531 if len(chunk) > last_message:

532 yield chunk[last_message:]

533 last_message = len(chunk)

534 519 else:

535 520 yield chunk

536 521 if fields.finish_reason is not None:

@@ -548,82 +533,50 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):

548 533 line = json.loads(line[6:])

549 534 except:

550 535 return

551 if "message" not in line:

536 if isinstance(line, dict) and "v" in line:

537 v = line.get("v")

538 r = ""

539 if isinstance(v, str):

540 yield v

541 elif isinstance(v, list):

542 for m in v:

543 if m.get("p") == "/message/content/parts/0":

544 yield m.get("v")

545 elif m.get("p") == "/message/metadata":

546 fields.finish_reason = m.get("v", {}).get("finish_details", {}).get("type")

547 break

548 elif isinstance(v, dict):

549 if fields.conversation_id is None:

550 fields.conversation_id = v.get("conversation_id")

551 fields.message_id = v.get("message", {}).get("id")

552 c = v.get("message", {}).get("content", {})

553 if c.get("content_type") == "multimodal_text":

554 generated_images = []

555 for element in c.get("parts"):

556 if element.get("content_type") == "image_asset_pointer":

557 generated_images.append(

558 cls.get_generated_image(session, cls._headers, element)

559 )

560 elif element.get("content_type") == "text":

561 for part in element.get("parts", []):

562 yield part

563 for image_response in await asyncio.gather(*generated_images):

564 yield image_response

552 565 return

553 if "error" in line and line["error"]:

554 raise RuntimeError(line["error"])

555 if "message_type" not in line["message"]["metadata"]:

556 return

557 image_response = await cls.get_generated_image(session, cls._headers, line)

558 if image_response is not None:

559 yield image_response

560 if line["message"]["author"]["role"] != "assistant":

561 return

562 if line["message"]["content"]["content_type"] != "text":

563 return

564 if line["message"]["metadata"]["message_type"] not in ("next", "continue", "variant"):

565 return

566 if line["message"]["recipient"] != "all":

567 return

568 if fields.conversation_id is None:

569 fields.conversation_id = line["conversation_id"]

570 fields.message_id = line["message"]["id"]

571 if "parts" in line["message"]["content"]:

572 yield line["message"]["content"]["parts"][0]

573 if "finish_details" in line["message"]["metadata"]:

574 fields.finish_reason = line["message"]["metadata"]["finish_details"]["type"]

575

576 @classmethod

577 async def webview_access_token(cls) -> str:

578 window = webview.create_window("OpenAI Chat", cls.url)

579 await asyncio.sleep(3)

580 prompt_input = None

581 while not prompt_input:

582 try:

583 await asyncio.sleep(1)

584 prompt_input = window.dom.get_element("#prompt-textarea")

585 except:

586 ...

587 window.evaluate_js("""

588 this._fetch = this.fetch;

589 this.fetch = async (url, options) => {

590 const response = await this._fetch(url, options);

591 if (url == "https://chatgpt.com/backend-api/conversation") {

592 this._headers = options.headers;

593 return response;

594 }

595 return response;

596 };

597 """)

598 window.evaluate_js("""

599 document.querySelector('.from-token-main-surface-secondary').click();

600 """)

601 headers = None

602 while headers is None:

603 headers = window.evaluate_js("this._headers")

604 await asyncio.sleep(1)

605 headers["User-Agent"] = window.evaluate_js("this.navigator.userAgent")

606 cookies = [list(*cookie.items()) for cookie in window.get_cookies()]

607 window.destroy()

608 cls._cookies = dict([(name, cookie.value) for name, cookie in cookies])

609 cls._headers = headers

610 cls._expires = int(time.time()) + 60 * 60 * 4

611 cls._update_cookie_header()

566 if "error" in line and line.get("error"):

567 raise RuntimeError(line.get("error"))

612 568

613 569 @classmethod

614 570 async def nodriver_access_token(cls, proxy: str = None):

615 try:

616 import nodriver as uc

617 except ImportError:

571 if not has_nodriver:

618 572 return

619 try:

620 from platformdirs import user_config_dir

573 if has_platformdirs:

621 574 user_data_dir = user_config_dir("g4f-nodriver")

622 except:

575 else:

623 576 user_data_dir = None

624 577 if debug.logging:

625 578 print(f"Open nodriver with user_dir: {user_data_dir}")

626 browser = await uc.start(

579 browser = await nodriver.start(

627 580 user_data_dir=user_data_dir,

628 581 browser_args=None if proxy is None else [f"--proxy-server={proxy}"],

629 582 )

@@ -649,48 +602,6 @@ this.fetch = async (url, options) => {

649 602 cls._create_request_args(cookies, user_agent=user_agent)

650 603 cls._set_api_key(api_key)

651 604

652 @classmethod

653 def browse_access_token(cls, proxy: str = None, timeout: int = 1200) -> None:

654 """

655 Browse to obtain an access token.

656

657 Args:

658 proxy (str): Proxy to use for browsing.

659

660 Returns:

661 tuple[str, dict]: A tuple containing the access token and cookies.

662 """

663 driver = get_browser(proxy=proxy)

664 try:

665 driver.get(f"{cls.url}/")

666 WebDriverWait(driver, timeout).until(EC.presence_of_element_located((By.ID, "prompt-textarea")))

667 access_token = driver.execute_script(

668 "let session = await fetch('/api/auth/session');"

669 "let data = await session.json();"

670 "let accessToken = data['accessToken'];"

671 "let expires = new Date(); expires.setTime(expires.getTime() + 60 * 60 * 4 * 1000);"

672 "document.cookie = 'access_token=' + accessToken + ';expires=' + expires.toUTCString() + ';path=/';"

673 "return accessToken;"

674 )

675 args = get_args_from_browser(f"{cls.url}/", driver, do_bypass_cloudflare=False)

676 cls._headers = args["headers"]

677 cls._cookies = args["cookies"]

678 cls._update_cookie_header()

679 cls._set_api_key(access_token)

680 finally:

681 driver.close()

682

683 @classmethod

684 async def fetch_access_token(cls, session: StreamSession, headers: dict):

685 async with session.get(

686 f"{cls.url}/api/auth/session",

687 headers=headers

688 ) as response:

689 if response.ok:

690 data = await response.json()

691 if "accessToken" in data:

692 return data["accessToken"]

693

694 605 @staticmethod

695 606 def get_default_headers() -> dict:

696 607 return {

Modified g4f/Provider/openai/har_file.py +24 -20

@@ -27,16 +27,18 @@ class arkReq:

27 27 self.arkCookies = arkCookies

28 28 self.userAgent = userAgent

29 29

30 arkPreURL = "https://tcr9i.chat.openai.com/fc/gt2/public_key/35536E1E-65B4-4D96-9D97-6ADB7EFF8147"

31 sessionUrl = "https://chatgpt.com/"

32 chatArk: arkReq = None

30 arkoseURL = "https://tcr9i.chat.openai.com/fc/gt2/public_key/35536E1E-65B4-4D96-9D97-6ADB7EFF8147"

31 startUrl = "https://chatgpt.com/"

32 conversationUrl = "https://chatgpt.com/c/"

33 arkoseRequest: arkReq = None

33 34 accessToken: str = None

34 35 cookies: dict = None

35 36 headers: dict = None

36 proofTokens: list = []

37 proofToken: list = []

38 turnstileToken: str = None

37 39

38 40 def readHAR():

39 global proofTokens

41 global arkoseRequest, accessToken, proofToken, turnstileToken

40 42 harPath = []

41 43 chatArks = []

42 44 accessToken = None

@@ -58,15 +60,17 @@ def readHAR():

58 60 v_headers = get_headers(v)

59 61 try:

60 62 if "openai-sentinel-proof-token" in v_headers:

61 proofTokens.append(json.loads(base64.b64decode(

63 proofToken = json.loads(base64.b64decode(

62 64 v_headers["openai-sentinel-proof-token"].split("gAAAAAB", 1)[-1].encode()

63 ).decode()))

65 ).decode())

66 if "openai-sentinel-turnstile-token" in v_headers:

67 turnstileToken = v_headers["openai-sentinel-turnstile-token"]

64 68 except Exception as e:

65 69 if debug.logging:

66 70 print(f"Read proof token: {e}")

67 if arkPreURL in v['request']['url']:

68 chatArks.append(parseHAREntry(v))

69 elif v['request']['url'] == sessionUrl:

71 if arkoseURL in v['request']['url']:

72 arkoseRequest = parseHAREntry(v)

73 elif v['request']['url'] == startUrl or v['request']['url'].startswith(conversationUrl):

70 74 try:

71 75 match = re.search(r'"accessToken":"(.*?)"', v["response"]["content"]["text"])

72 76 if match:

@@ -78,8 +82,8 @@ def readHAR():

78 82 if not accessToken:

79 83 raise NoValidHarFileError("No accessToken found in .har files")

80 84 if not chatArks:

81 return None, accessToken, cookies, headers

82 return chatArks.pop(), accessToken, cookies, headers

85 return cookies, headers

86 return cookies, headers

83 87

84 88 def get_headers(entry) -> dict:

85 89

    return {h['name'].lower(): h['value'] for h in entry['request']['headers'] if h['name'].lower() not in ['content-length', 'cookie'] and not h['name'].startswith(':')}

@@ -110,7 +114,7 @@ def genArkReq(chatArk: arkReq) -> arkReq:

110 114 tmpArk.arkHeader['x-ark-esync-value'] = bw

111 115 return tmpArk

112 116

113 async def sendRequest(tmpArk: arkReq, proxy: str = None):

117 async def sendRequest(tmpArk: arkReq, proxy: str = None) -> str:

114 118 async with StreamSession(headers=tmpArk.arkHeader, cookies=tmpArk.arkCookies, proxies={"https": proxy}) as session:

115 119 async with session.post(tmpArk.arkURL, data=tmpArk.arkBody) as response:

116 120 data = await response.json()

@@ -144,10 +148,10 @@ def getN() -> str:

144 148 return base64.b64encode(timestamp.encode()).decode()

145 149

146 150 async def getArkoseAndAccessToken(proxy: str) -> tuple[str, str, dict, dict]:

147 global chatArk, accessToken, cookies, headers, proofTokens

148 if chatArk is None or accessToken is None:

149 chatArk, accessToken, cookies, headers = readHAR()

150 if chatArk is None:

151 return None, accessToken, cookies, headers, proofTokens

152 newReq = genArkReq(chatArk)

153 return await sendRequest(newReq, proxy), accessToken, cookies, headers, proofTokens

151 global arkoseRequest, accessToken, cookies, headers, proofToken, turnstileToken

152 if arkoseRequest is None or accessToken is None:

153 cookies, headers = readHAR()

154 if arkoseRequest is None:

155 return None, accessToken, cookies, headers, proofToken, turnstileToken

156 newReq = genArkReq(arkoseRequest)

157 return await sendRequest(newReq, proxy), accessToken, cookies, headers, proofToken, turnstileToken

Modified g4f/Provider/openai/proofofwork.py +5 -7

@@ -4,18 +4,16 @@ import json

4 4 import base64

5 5 from datetime import datetime, timezone

6 6

def generate_proof_token(required: bool, seed: str = "", difficulty: str = "", user_agent: str = None, proofTokens: list = None):

def generate_proof_token(required: bool, seed: str = "", difficulty: str = "", user_agent: str = None, proofToken: str = None):

8 8 if not required:

9 9 return

10 10

11 if proofTokens:

12 config = proofTokens[-1]

13 else:

11 if proofToken is None:

14 12 screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4])

15 13 # Get current UTC time

16 14 now_utc = datetime.now(timezone.utc)

17 15 parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT')

18 config = [

16 proofToken = [

19 17 screen, parse_time,

20 18 None, 0, user_agent,

21 19 "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js",

@@ -28,8 +26,8 @@ def generate_proof_token(required: bool, seed: str = "", difficulty: str = "", u

28 26

29 27 diff_len = len(difficulty)

30 28 for i in range(100000):

31 config[3] = i

32 json_data = json.dumps(config)

29 proofToken[3] = i

30 json_data = json.dumps(proofToken)

33 31 base = base64.b64encode(json_data.encode()).decode()

34 32 hash_value = hashlib.sha3_512((seed + base).encode()).digest()

35 33