gpt4free

公开

关注 0 Fork 0 Star 1

返回提交历史

XFEstudio/gpt4free

Add lock file for nodriver, add finally stop browser to all calls

3e1e5056

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-01-25 12:02

父提交 cc6f9d1

代码差异

8 个文件 +191 -143

Modified g4f/Provider/Copilot.py +27 -24

@@ -207,30 +207,33 @@ class Copilot(AbstractProvider, ProviderModelMixin):

207 207

208 208 async def get_access_token_and_cookies(url: str, proxy: str = None, target: str = "ChatAI",):

209 209 browser = await get_nodriver(proxy=proxy, user_data_dir="copilot")

210 page = await browser.get(url)

211 access_token = None

212 while access_token is None:

213 access_token = await page.evaluate("""

214 (() => {

215 for (var i = 0; i < localStorage.length; i++) {

216 try {

217 item = JSON.parse(localStorage.getItem(localStorage.key(i)));

218 if (item.credentialType == "AccessToken"

219 && item.expiresOn > Math.floor(Date.now() / 1000)

220 && item.target.includes("target")) {

221 return item.secret;

222 }

223 } catch(e) {}

224 }

225 })()

226 """.replace('"target"', json.dumps(target)))

227 if access_token is None:

228 await asyncio.sleep(1)

229 cookies = {}

230 for c in await page.send(nodriver.cdp.network.get_cookies([url])):

231 cookies[c.name] = c.value

232 await page.close()

233 return access_token, cookies

210 try:

211 page = await browser.get(url)

212 access_token = None

213 while access_token is None:

214 access_token = await page.evaluate("""

215 (() => {

216 for (var i = 0; i < localStorage.length; i++) {

217 try {

218 item = JSON.parse(localStorage.getItem(localStorage.key(i)));

219 if (item.credentialType == "AccessToken"

220 && item.expiresOn > Math.floor(Date.now() / 1000)

221 && item.target.includes("target")) {

222 return item.secret;

223 }

224 } catch(e) {}

225 }

226 })()

227 """.replace('"target"', json.dumps(target)))

228 if access_token is None:

229 await asyncio.sleep(1)

230 cookies = {}

231 for c in await page.send(nodriver.cdp.network.get_cookies([url])):

232 cookies[c.name] = c.value

233 await page.close()

234 return access_token, cookies

235 finally:

236 browser.stop()

234 237

235 238 def readHAR(url: str):

236 239 api_key = None

Modified g4f/Provider/You.py +9 -6

@@ -76,12 +76,15 @@ class You(AsyncGeneratorProvider, ProviderModelMixin):

76 76 cookies = get_cookies(".you.com")

77 77 except MissingRequirementsError:

78 78 browser = await get_nodriver(proxy=proxy)

79 page = await browser.get(cls.url)

80 await page.wait_for('[data-testid="user-profile-button"]', timeout=900)

81 cookies = {}

82 for c in await page.send(nodriver.cdp.network.get_cookies([cls.url])):

83 cookies[c.name] = c.value

84 await page.close()

79 try:

80 page = await browser.get(cls.url)

81 await page.wait_for('[data-testid="user-profile-button"]', timeout=900)

82 cookies = {}

83 for c in await page.send(nodriver.cdp.network.get_cookies([cls.url])):

84 cookies[c.name] = c.value

85 await page.close()

86 finally:

87 browser.stop()

85 88 async with StreamSession(

86 89 proxy=proxy,

87 90 impersonate="chrome",

Modified g4f/Provider/needs_auth/Gemini.py +13 -10

@@ -78,16 +78,19 @@ class Gemini(AsyncGeneratorProvider, ProviderModelMixin):

78 78 print("Skip nodriver login in Gemini provider")

79 79 return

80 80 browser = await get_nodriver(proxy=proxy, user_data_dir="gemini")

81 login_url = os.environ.get("G4F_LOGIN_URL")

82 if login_url:

83 yield RequestLogin(cls.label, login_url)

84 page = await browser.get(f"{cls.url}/app")

85 await page.select("div.ql-editor.textarea", 240)

86 cookies = {}

87 for c in await page.send(nodriver.cdp.network.get_cookies([cls.url])):

88 cookies[c.name] = c.value

89 await page.close()

90 cls._cookies = cookies

81 try:

82 login_url = os.environ.get("G4F_LOGIN_URL")

83 if login_url:

84 yield RequestLogin(cls.label, login_url)

85 page = await browser.get(f"{cls.url}/app")

86 await page.select("div.ql-editor.textarea", 240)

87 cookies = {}

88 for c in await page.send(nodriver.cdp.network.get_cookies([cls.url])):

89 cookies[c.name] = c.value

90 await page.close()

91 cls._cookies = cookies

92 finally:

93 browser.stop()

91 94

92 95 @classmethod

93 96 async def create_async_generator(

Modified g4f/Provider/needs_auth/MicrosoftDesigner.py +25 -22

@@ -143,25 +143,28 @@ def readHAR(url: str) -> tuple[str, str]:

143 143

144 144 async def get_access_token_and_user_agent(url: str, proxy: str = None):

145 145 browser = await get_nodriver(proxy=proxy, user_data_dir="designer")

146 page = await browser.get(url)

147 user_agent = await page.evaluate("navigator.userAgent")

148 access_token = None

149 while access_token is None:

150 access_token = await page.evaluate("""

151 (() => {

152 for (var i = 0; i < localStorage.length; i++) {

153 try {

154 item = JSON.parse(localStorage.getItem(localStorage.key(i)));

155 if (item.credentialType == "AccessToken"

156 && item.expiresOn > Math.floor(Date.now() / 1000)

157 && item.target.includes("designerappservice")) {

158 return item.secret;

159 }

160 } catch(e) {}

161 }

162 })()

163 """)

164 if access_token is None:

165 await asyncio.sleep(1)

166 await page.close()

167 return access_token, user_agent

146 try:

147 page = await browser.get(url)

148 user_agent = await page.evaluate("navigator.userAgent")

149 access_token = None

150 while access_token is None:

151 access_token = await page.evaluate("""

152 (() => {

153 for (var i = 0; i < localStorage.length; i++) {

154 try {

155 item = JSON.parse(localStorage.getItem(localStorage.key(i)));

156 if (item.credentialType == "AccessToken"

157 && item.expiresOn > Math.floor(Date.now() / 1000)

158 && item.target.includes("designerappservice")) {

159 return item.secret;

160 }

161 } catch(e) {}

162 }

163 })()

164 """)

165 if access_token is None:

166 await asyncio.sleep(1)

167 await page.close()

168 return access_token, user_agent

169 finally:

170 browser.stop()

Modified g4f/Provider/needs_auth/OpenaiChat.py +49 -46

@@ -624,53 +624,56 @@ class OpenaiChat(AsyncAuthedProvider, ProviderModelMixin):

624 624 @classmethod

625 625 async def nodriver_auth(cls, proxy: str = None):

626 626 browser = await get_nodriver(proxy=proxy)

627 page = browser.main_tab

628 def on_request(event: nodriver.cdp.network.RequestWillBeSent):

629 if event.request.url == start_url or event.request.url.startswith(conversation_url):

630 RequestConfig.headers = event.request.headers

631 elif event.request.url in (backend_url, backend_anon_url):

632 if "OpenAI-Sentinel-Proof-Token" in event.request.headers:

633 RequestConfig.proof_token = json.loads(base64.b64decode(

634 event.request.headers["OpenAI-Sentinel-Proof-Token"].split("gAAAAAB", 1)[-1].encode()

635 ).decode())

636 if "OpenAI-Sentinel-Turnstile-Token" in event.request.headers:

637 RequestConfig.turnstile_token = event.request.headers["OpenAI-Sentinel-Turnstile-Token"]

638 if "Authorization" in event.request.headers:

639 cls._api_key = event.request.headers["Authorization"].split()[-1]

640 elif event.request.url == arkose_url:

641 RequestConfig.arkose_request = arkReq(

642 arkURL=event.request.url,

643 arkBx=None,

644 arkHeader=event.request.headers,

645 arkBody=event.request.post_data,

646 userAgent=event.request.headers.get("User-Agent")

647 )

648 await page.send(nodriver.cdp.network.enable())

649 page.add_handler(nodriver.cdp.network.RequestWillBeSent, on_request)

650 page = await browser.get(cls.url)

651 user_agent = await page.evaluate("window.navigator.userAgent")

652 await page.select("#prompt-textarea", 240)

653 await page.evaluate("document.getElementById('prompt-textarea').innerText = 'Hello'")

654 await page.evaluate("document.querySelector('[data-testid=\"send-button\"]').click()")

655 while True:

656 if cls._api_key is not None or not cls.needs_auth:

657 break

658 body = await page.evaluate("JSON.stringify(window.__remixContext)")

659 if body:

660 match = re.search(r'"accessToken":"(.*?)"', body)

661 if match:

662 cls._api_key = match.group(1)

627 try:

628 page = browser.main_tab

629 def on_request(event: nodriver.cdp.network.RequestWillBeSent):

630 if event.request.url == start_url or event.request.url.startswith(conversation_url):

631 RequestConfig.headers = event.request.headers

632 elif event.request.url in (backend_url, backend_anon_url):

633 if "OpenAI-Sentinel-Proof-Token" in event.request.headers:

634 RequestConfig.proof_token = json.loads(base64.b64decode(

635 event.request.headers["OpenAI-Sentinel-Proof-Token"].split("gAAAAAB", 1)[-1].encode()

636 ).decode())

637 if "OpenAI-Sentinel-Turnstile-Token" in event.request.headers:

638 RequestConfig.turnstile_token = event.request.headers["OpenAI-Sentinel-Turnstile-Token"]

639 if "Authorization" in event.request.headers:

640 cls._api_key = event.request.headers["Authorization"].split()[-1]

641 elif event.request.url == arkose_url:

642 RequestConfig.arkose_request = arkReq(

643 arkURL=event.request.url,

644 arkBx=None,

645 arkHeader=event.request.headers,

646 arkBody=event.request.post_data,

647 userAgent=event.request.headers.get("User-Agent")

648 )

649 await page.send(nodriver.cdp.network.enable())

650 page.add_handler(nodriver.cdp.network.RequestWillBeSent, on_request)

651 page = await browser.get(cls.url)

652 user_agent = await page.evaluate("window.navigator.userAgent")

653 await page.select("#prompt-textarea", 240)

654 await page.evaluate("document.getElementById('prompt-textarea').innerText = 'Hello'")

655 await page.evaluate("document.querySelector('[data-testid=\"send-button\"]').click()")

656 while True:

657 if cls._api_key is not None or not cls.needs_auth:

663 658 break

664 await asyncio.sleep(1)

665 while True:

666 if RequestConfig.proof_token:

667 break

668 await asyncio.sleep(1)

669 RequestConfig.data_build = await page.evaluate("document.documentElement.getAttribute('data-build')")

670 RequestConfig.cookies = await page.send(get_cookies([cls.url]))

671 await page.close()

672 cls._create_request_args(RequestConfig.cookies, RequestConfig.headers, user_agent=user_agent)

673 cls._set_api_key(cls._api_key)

659 body = await page.evaluate("JSON.stringify(window.__remixContext)")

660 if body:

661 match = re.search(r'"accessToken":"(.*?)"', body)

662 if match:

663 cls._api_key = match.group(1)

664 break

665 await asyncio.sleep(1)

666 while True:

667 if RequestConfig.proof_token:

668 break

669 await asyncio.sleep(1)

670 RequestConfig.data_build = await page.evaluate("document.documentElement.getAttribute('data-build')")

671 RequestConfig.cookies = await page.send(get_cookies([cls.url]))

672 await page.close()

673 cls._create_request_args(RequestConfig.cookies, RequestConfig.headers, user_agent=user_agent)

674 cls._set_api_key(cls._api_key)

675 finally:

676 browser.stop()

674 677

675 678 @staticmethod

676 679 def get_default_headers() -> Dict[str, str]:

Modified g4f/client/__init__.py +2 -2

@@ -534,8 +534,8 @@ class AsyncCompletions:

534 534 self.provider if provider is None else provider,

535 535 stream,

536 536 ignore_working,

537 ignore_stream,,

538 has_images="images" in kwargs

537 ignore_stream,

538 has_images="images" in kwargs,

539 539 )

540 540 stop = [stop] if isinstance(stop, str) else stop

541 541 if ignore_stream:

Modified g4f/providers/base_provider.py +0 -1

@@ -425,7 +425,6 @@ class AsyncAuthedProvider(AsyncGeneratorProvider):

425 425 else:

426 426 auth_result = cls.on_auth(**kwargs)

427 427 for chunk in auth_result:

428 print(hasattr(chunk, "get_dict"))

429 428 if hasattr(chunk, "get_dict"):

430 429 auth_result = chunk

431 430 else:

Modified g4f/requests/__init__.py +66 -32

@@ -1,9 +1,12 @@

1 1 from __future__ import annotations

2 2

3 3 import os

4 import time

5 import random

4 6 from urllib.parse import urlparse

5 7 from typing import Iterator

6 8 from http.cookies import Morsel

9 from pathlib import Path

7 10 try:

8 11 from curl_cffi.requests import Session, Response

9 12 from .curl_cffi import StreamResponse, StreamSession, FormData

@@ -37,6 +40,7 @@ from .. import debug

37 40 from .raise_for_status import raise_for_status

38 41 from ..errors import MissingRequirementsError

39 42 from ..typing import Cookies

43 from ..cookies import get_cookies_dir

40 44 from .defaults import DEFAULT_HEADERS, WEBVIEW_HAEDERS

41 45

42 46 if not has_curl_cffi:

@@ -83,35 +87,37 @@ async def get_args_from_nodriver(

83 87 callback: callable = None,

84 88 cookies: Cookies = None

85 89 ) -> dict:

86 browser = await get_nodriver(proxy=proxy)

87 if debug.logging:

88 print(f"Open nodriver with url: {url}")

89 domain = urlparse(url).netloc

90 if cookies is None:

91 cookies = {}

92 else:

93 await browser.cookies.set_all(get_cookie_params_from_dict(cookies, url=url, domain=domain))

94 page = await browser.get(url)

95 user_agent = await page.evaluate("window.navigator.userAgent")

96 await page.wait_for("body:not(.no-js)", timeout=timeout)

97 if wait_for is not None:

98 await page.wait_for(wait_for, timeout=timeout)

99 if callback is not None:

100 await callback(page)

101 for c in await page.send(nodriver.cdp.network.get_cookies([url])):

102 cookies[c.name] = c.value

103 await page.close()

104 browser.stop()

105 return {

106 "impersonate": "chrome",

107 "cookies": cookies,

108 "headers": {

109 **DEFAULT_HEADERS,

110 "user-agent": user_agent,

111 "referer": url,

112 },

113 "proxy": proxy,

114 }

90 browser = await get_nodriver(proxy=proxy, timeout=timeout)

91 try:

92 if debug.logging:

93 print(f"Open nodriver with url: {url}")

94 domain = urlparse(url).netloc

95 if cookies is None:

96 cookies = {}

97 else:

98 await browser.cookies.set_all(get_cookie_params_from_dict(cookies, url=url, domain=domain))

99 page = await browser.get(url)

100 user_agent = await page.evaluate("window.navigator.userAgent")

101 await page.wait_for("body:not(.no-js)", timeout=timeout)

102 if wait_for is not None:

103 await page.wait_for(wait_for, timeout=timeout)

104 if callback is not None:

105 await callback(page)

106 for c in await page.send(nodriver.cdp.network.get_cookies([url])):

107 cookies[c.name] = c.value

108 await page.close()

109 return {

110 "impersonate": "chrome",

111 "cookies": cookies,

112 "headers": {

113 **DEFAULT_HEADERS,

114 "user-agent": user_agent,

115 "referer": url,

116 },

117 "proxy": proxy,

118 }

119 finally:

120 browser.stop()

115 121

116 122 def merge_cookies(cookies: Iterator[Morsel], response: Response) -> Cookies:

117 123 if cookies is None:

@@ -119,7 +125,13 @@ def merge_cookies(cookies: Iterator[Morsel], response: Response) -> Cookies:

119 125 for cookie in response.cookies.jar:

120 126 cookies[cookie.name] = cookie.value

121 127

122 async def get_nodriver(proxy: str = None, user_data_dir = "nodriver", browser_executable_path=None, **kwargs)-> Browser:

128 async def get_nodriver(

129 proxy: str = None,

130 user_data_dir = "nodriver",

131 timeout: int = 120,

132 browser_executable_path=None,

133 **kwargs

134 ) -> Browser:

123 135 if not has_nodriver:

124 136

        raise MissingRequirementsError('Install "nodriver" and "platformdirs" package | pip install -U nodriver platformdirs')

125 137 user_data_dir = user_config_dir(f"g4f-{user_data_dir}") if has_platformdirs else None

@@ -131,10 +143,32 @@ async def get_nodriver(proxy: str = None, user_data_dir = "nodriver", browser_ex

131 143 browser_executable_path = "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"

132 144 if not os.path.exists(browser_executable_path):

133 145 browser_executable_path = None

146 lock_file = Path(get_cookies_dir()) / ".nodriver_is_open"

147 # Implement a short delay (milliseconds) to prevent race conditions.

148 await asyncio.sleep(0.1 * random.randint(0, 50))

149 if lock_file.exists():

150 opend_at = float(lock_file.read_text())

151 time_open = time.time() - opend_at

152 if timeout * 2 > time_open:

153 debug.log(f"Nodriver: Browser is already in use since {time_open} secs.")

154 for _ in range(timeout):

155 if lock_file.exists():

156 await asyncio.sleep(1)

157 else:

158 break

159 lock_file.write_text(str(time.time()))

134 160 debug.log(f"Open nodriver with user_dir: {user_data_dir}")

135 return await nodriver.start(

161 browser = await nodriver.start(

136 162 user_data_dir=user_data_dir,

137 163 browser_args=None if proxy is None else [f"--proxy-server={proxy}"],

138 164 browser_executable_path=browser_executable_path,

139 165 **kwargs

140 )

166 )

167 stop = browser.stop

168 def on_stop():

169 try:

170 stop()

171 finally:

172 lock_file.unlink(missing_ok=True)

173 browser.stop = on_stop

174 return browser