XFE Git
XFE Studio Git
Git 首页 全局搜索
XFE 主站 文档 NuGet
公开
关注 0 Fork 0 Star 1
返回提交历史

XFEstudio/gpt4free

Fix PerplexityLabs Provider, Improve bypass_cloudflare helper

47b50b48
Heiner Lohaus <hlohaus@users.noreply.github.com>
提交于
父提交 1eb7dc0

代码差异

11 个文件 +186 -164
Modified g4f/Provider/Aura.py +4 -1
@@ -1,5 +1,7 @@
1 1 from __future__ import annotations
2
2 3 from aiohttp import ClientSession
4
3 5 from ..typing import AsyncResult, Messages
4 6 from .base_provider import AsyncGeneratorProvider
5 7
@@ -23,7 +25,7 @@ class Aura(AsyncGeneratorProvider):
23 25 "Content-Type": "application/json",
24 26 "Origin": f"{cls.url}",
25 27 "Referer": f"{cls.url}/",
26 "Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
28 "Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
27 29 "Sec-Ch-Ua-Mobile": "?0",
28 30 "Sec-Ch-Ua-Platform": '"Linux"',
29 31 "Sec-Fetch-Dest": "empty",
@@ -52,5 +54,6 @@ class Aura(AsyncGeneratorProvider):
52 54 "temperature": 0.5
53 55 }
54 56 async with session.post(f"{cls.url}/api/chat", json=data, proxy=proxy) as response:
57 response.raise_for_status()
55 58 async for chunk in response.content.iter_any():
56 59 yield chunk.decode()
Modified g4f/Provider/Bestim.py +1 -14
@@ -4,7 +4,6 @@ from ..typing import Messages
4 4 from .base_provider import BaseProvider, CreateResult
5 5 from ..requests import get_session_from_browser
6 6 from uuid import uuid4
7 import requests
8 7
9 8 class Bestim(BaseProvider):
10 9 url = "https://chatgpt.bestim.org"
@@ -24,18 +23,7 @@ class Bestim(BaseProvider):
24 23 ) -> CreateResult:
25 24 session = get_session_from_browser(cls.url, proxy=proxy)
26 25 headers = {
27 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
28 26 'Accept': 'application/json, text/event-stream',
29 'Accept-Language': 'en-US,en;q=0.5',
30 'Accept-Encoding': 'gzip, deflate, br',
31 'Referer': 'https://chatgpt.bestim.org/chat/',
32 'Origin': 'https://chatgpt.bestim.org',
33 'Alt-Used': 'chatgpt.bestim.org',
34 'Connection': 'keep-alive',
35 'Sec-Fetch-Dest': 'empty',
36 'Sec-Fetch-Mode': 'cors',
37 'Sec-Fetch-Site': 'same-origin',
38 'TE': 'trailers'
39 27 }
40 28 data = {
41 29 "messagesHistory": [{
@@ -47,9 +35,8 @@ class Bestim(BaseProvider):
47 35 }
48 36 response = session.post(
49 37 url="https://chatgpt.bestim.org/chat/send2/",
50 headers=headers,
51 38 json=data,
52 proxies={"https": proxy},
39 headers=headers,
53 40 stream=True
54 41 )
55 42 response.raise_for_status()
Modified g4f/Provider/Chatgpt4Online.py +12 -18
@@ -5,6 +5,7 @@ import json
5 5 from aiohttp import ClientSession
6 6
7 7 from ..typing import Messages, AsyncResult
8 from ..requests import get_args_from_browser
8 9 from .base_provider import AsyncGeneratorProvider
9 10 from .helper import get_random_string
10 11
@@ -12,8 +13,9 @@ class Chatgpt4Online(AsyncGeneratorProvider):
12 13 url = "https://chatgpt4online.org"
13 14 supports_message_history = True
14 15 supports_gpt_35_turbo = True
15 working = False # cloudfare block !
16 working = False
16 17 _wpnonce = None
18 _context_id = None
17 19
18 20 @classmethod
19 21 async def create_async_generator(
@@ -23,23 +25,10 @@ class Chatgpt4Online(AsyncGeneratorProvider):
23 25 proxy: str = None,
24 26 **kwargs
25 27 ) -> AsyncResult:
26 headers = {
27 "accept": "*/*",
28 "accept-language": "en-US",
29 "content-type": "application/json",
30 "sec-ch-ua": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"",
31 "sec-ch-ua-mobile": "?0",
32 "sec-ch-ua-platform": "\"Windows\"",
33 "sec-fetch-dest": "empty",
34 "sec-fetch-mode": "cors",
35 "sec-fetch-site": "same-origin",
36 "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
37 "referer": "https://chatgpt4online.org/",
38 "referrer-policy": "strict-origin-when-cross-origin"
39 }
40 async with ClientSession(headers=headers) as session:
28 args = get_args_from_browser(f"{cls.url}/chat/", proxy=proxy)
29 async with ClientSession(**args) as session:
41 30 if not cls._wpnonce:
42 async with session.get(f"{cls.url}/", proxy=proxy) as response:
31 async with session.get(f"{cls.url}/chat/", proxy=proxy) as response:
43 32 response.raise_for_status()
44 33 response = await response.text()
45 34 result = re.search(r'restNonce&quot;:&quot;(.*?)&quot;', response)
@@ -47,12 +36,17 @@ class Chatgpt4Online(AsyncGeneratorProvider):
47 36 cls._wpnonce = result.group(1)
48 37 else:
49 38 raise RuntimeError("No nonce found")
39 result = re.search(r'contextId&quot;:(.*?),', response)
40 if result:
41 cls._context_id = result.group(1)
42 else:
43 raise RuntimeError("No contextId found")
50 44 data = {
51 45 "botId":"default",
52 46 "customId":None,
53 47 "session":"N/A",
54 48 "chatId":get_random_string(11),
55 "contextId":58,
49 "contextId":cls._context_id,
56 50 "messages":messages[:-1],
57 51 "newMessage":messages[-1]["content"],
58 52 "newImageId":None,
Modified g4f/Provider/PerplexityLabs.py +1 -1
@@ -76,7 +76,7 @@ class PerplexityLabs(AsyncGeneratorProvider, ProviderModelMixin):
76 76 'model': cls.get_model(model),
77 77 'messages': messages
78 78 }
79 await ws.send_str('42' + json.dumps(['perplexity_playground', message_data]))
79 await ws.send_str('42' + json.dumps(['perplexity_labs', message_data]))
80 80 last_message = 0
81 81 while True:
82 82 message = await ws.receive_str()
Modified g4f/Provider/helper.py +4 -4
@@ -22,11 +22,11 @@ except ImportError:
22 22 has_browser_cookie3 = False
23 23
24 24 from ..typing import Dict, Messages, Cookies, Optional
25 from ..errors import AiohttpSocksError, MissingRequirementsError
25 from ..errors import MissingAiohttpSocksError, MissingRequirementsError
26 26 from .. import debug
27 27
28 28 # Global variable to store cookies
29 _cookies: Dict[str, Dict[str, str]] = {}
29 _cookies: Dict[str, Cookies] = {}
30 30
31 31 if has_browser_cookie3 and os.environ.get('DBUS_SESSION_BUS_ADDRESS') == "/dev/null":
32 32 _LinuxPasswordManager.get_password = lambda a, b: b"secret"
@@ -54,7 +54,7 @@ def set_cookies(domain_name: str, cookies: Cookies = None) -> None:
54 54 else:
55 55 _cookies.pop(domain_name)
56 56
57 def load_cookies_from_browsers(domain_name: str, raise_requirements_error: bool = True) -> Dict[str, str]:
57 def load_cookies_from_browsers(domain_name: str, raise_requirements_error: bool = True) -> Cookies:
58 58 """
59 59 Helper function to load cookies from various browsers.
60 60
@@ -149,5 +149,5 @@ def get_connector(connector: BaseConnector = None, proxy: str = None) -> Optiona
149 149 from aiohttp_socks import ProxyConnector
150 150 connector = ProxyConnector.from_url(proxy)
151 151 except ImportError:
152 raise AiohttpSocksError('Install "aiohttp_socks" package for proxy support')
152 raise MissingAiohttpSocksError('Install "aiohttp_socks" package for proxy support')
153 153 return connector
Modified g4f/Provider/selenium/AItianhuSpace.py +2 -2
@@ -52,9 +52,9 @@ class AItianhuSpace(AbstractProvider):
52 52 wait.until(EC.visibility_of_element_located((By.ID, "sheet")))
53 53 driver.execute_script(f"""
54 54 document.getElementById('sheet').addEventListener('click', () => {{
55 window.open('{url}', '_blank');
55 window.open(arguments[0]);
56 56 }});
57 """)
57 """, url)
58 58 driver.find_element(By.ID, "sheet").click()
59 59 time.sleep(10)
60 60
Modified g4f/errors.py +2 -2
@@ -31,10 +31,10 @@ class NestAsyncioError(Exception):
31 31 class ModelNotSupportedError(Exception):
32 32 pass
33 33
34 class MissingRequirementsError(ImportError):
34 class MissingRequirementsError(Exception):
35 35 pass
36 36
37 class AiohttpSocksError(MissingRequirementsError):
37 class MissingAiohttpSocksError(MissingRequirementsError):
38 38 pass
39 39
40 40 class MissingAccessToken(Exception):
Modified g4f/requests.py +22 -121
@@ -1,130 +1,21 @@
1 1 from __future__ import annotations
2 2
3 import json
4 from functools import partialmethod
5 from typing import AsyncGenerator
6 3 from urllib.parse import urlparse
7 4
8 5 try:
9 from curl_cffi.requests import AsyncSession, Session, Response
6 from curl_cffi.requests import Session
7 from .requests_curl_cffi import StreamResponse, StreamSession
10 8 has_curl_cffi = True
11 9 except ImportError:
12 10 Session = type
11 from .requests_aiohttp import StreamResponse, StreamSession
13 12 has_curl_cffi = False
14 13
15 14 from .webdriver import WebDriver, WebDriverSession, bypass_cloudflare, get_driver_cookies
16 15 from .errors import MissingRequirementsError
17 16
18 if not has_curl_cffi:
19 from aiohttp import ClientSession, ClientResponse, ClientTimeout
20 from .Provider.helper import get_connector
21
22 class StreamResponse(ClientResponse):
23 async def iter_lines(self) -> iter[bytes, None]:
24 async for line in self.content:
25 yield line.rstrip(b"\r\n")
26
27 async def json(self):
28 return await super().json(content_type=None)
29
30 class StreamSession(ClientSession):
31 def __init__(self, headers: dict = {}, timeout: int = None, proxies: dict = {}, impersonate = None, **kwargs):
32 if impersonate:
33 headers = {
34 'Accept-Encoding': 'gzip, deflate, br',
35 'Accept-Language': 'en-US',
36 'Connection': 'keep-alive',
37 'Sec-Fetch-Dest': 'empty',
38 'Sec-Fetch-Mode': 'cors',
39 'Sec-Fetch-Site': 'same-site',
40 "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
41 'Accept': '*/*',
42 'sec-ch-ua': '"Google Chrome";v="107", "Chromium";v="107", "Not?A_Brand";v="24"',
43 'sec-ch-ua-mobile': '?0',
44 'sec-ch-ua-platform': '"Windows"',
45 **headers
46 }
47 super().__init__(
48 **kwargs,
49 timeout=ClientTimeout(timeout) if timeout else None,
50 response_class=StreamResponse,
51 connector=get_connector(kwargs.get("connector"), proxies.get("https")),
52 headers=headers
53 )
54 else:
55 class StreamResponse:
56 """
57 A wrapper class for handling asynchronous streaming responses.
58
59 Attributes:
60 inner (Response): The original Response object.
61 """
62
63 def __init__(self, inner: Response) -> None:
64 """Initialize the StreamResponse with the provided Response object."""
65 self.inner: Response = inner
66
67 async def text(self) -> str:
68 """Asynchronously get the response text."""
69 return await self.inner.atext()
70
71 def raise_for_status(self) -> None:
72 """Raise an HTTPError if one occurred."""
73 self.inner.raise_for_status()
74
75 async def json(self, **kwargs) -> dict:
76 """Asynchronously parse the JSON response content."""
77 return json.loads(await self.inner.acontent(), **kwargs)
78
79 async def iter_lines(self) -> AsyncGenerator[bytes, None]:
80 """Asynchronously iterate over the lines of the response."""
81 async for line in self.inner.aiter_lines():
82 yield line
83
84 async def iter_content(self) -> AsyncGenerator[bytes, None]:
85 """Asynchronously iterate over the response content."""
86 async for chunk in self.inner.aiter_content():
87 yield chunk
88
89 async def __aenter__(self):
90 """Asynchronously enter the runtime context for the response object."""
91 inner: Response = await self.inner
92 self.inner = inner
93 self.request = inner.request
94 self.status_code: int = inner.status_code
95 self.reason: str = inner.reason
96 self.ok: bool = inner.ok
97 self.headers = inner.headers
98 self.cookies = inner.cookies
99 return self
100
101 async def __aexit__(self, *args):
102 """Asynchronously exit the runtime context for the response object."""
103 await self.inner.aclose()
104
105 class StreamSession(AsyncSession):
106 """
107 An asynchronous session class for handling HTTP requests with streaming.
108
109 Inherits from AsyncSession.
110 """
111 17
112 def request(
113 self, method: str, url: str, **kwargs
114 ) -> StreamResponse:
115 """Create and return a StreamResponse object for the given HTTP request."""
116 return StreamResponse(super().request(method, url, stream=True, **kwargs))
117
118 # Defining HTTP methods as partial methods of the request method.
119 head = partialmethod(request, "HEAD")
120 get = partialmethod(request, "GET")
121 post = partialmethod(request, "POST")
122 put = partialmethod(request, "PUT")
123 patch = partialmethod(request, "PATCH")
124 delete = partialmethod(request, "DELETE")
125
126
127 def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = None, timeout: int = 120) -> Session:
18 def get_args_from_browser(url: str, webdriver: WebDriver = None, proxy: str = None, timeout: int = 120) -> dict:
128 19 """
129 20 Create a Session object using a WebDriver to handle cookies and headers.
130 21
@@ -137,26 +28,36 @@ def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str =
137 28 Returns:
138 29 Session: A Session object configured with cookies and headers from the WebDriver.
139 30 """
140 if not has_curl_cffi:
141 raise MissingRequirementsError('Install "curl_cffi" package')
142 with WebDriverSession(webdriver, "", proxy=proxy, virtual_display=True) as driver:
31 with WebDriverSession(webdriver, "", proxy=proxy, virtual_display=False) as driver:
143 32 bypass_cloudflare(driver, url, timeout)
144 33 cookies = get_driver_cookies(driver)
145 34 user_agent = driver.execute_script("return navigator.userAgent")
146
147 35 parse = urlparse(url)
148 return Session(
149 cookies=cookies,
150 headers={
36 return {
37 'cookies': cookies,
38 'headers': {
151 39 'accept': '*/*',
40 "accept-language": "en-US",
41 "accept-encoding": "gzip, deflate, br",
152 42 'authority': parse.netloc,
153 43 'origin': f'{parse.scheme}://{parse.netloc}',
154 44 'referer': url,
45 "sec-ch-ua": "\"Google Chrome\";v=\"121\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"121\"",
46 "sec-ch-ua-mobile": "?0",
47 "sec-ch-ua-platform": "Windows",
155 48 'sec-fetch-dest': 'empty',
156 49 'sec-fetch-mode': 'cors',
157 50 'sec-fetch-site': 'same-origin',
158 'user-agent': user_agent
51 'user-agent': user_agent,
159 52 },
53 }
54
55 def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = None, timeout: int = 120) -> Session:
56 if not has_curl_cffi:
57 raise MissingRequirementsError('Install "curl_cffi" package')
58 args = get_args_from_browser(url, webdriver, proxy, timeout)
59 return Session(
60 **args,
160 61 proxies={"https": proxy, "http": proxy},
161 62 timeout=timeout,
162 63 impersonate="chrome110"
Added g4f/requests_aiohttp.py +39 -0
@@ -0,0 +1,39 @@
1 from __future__ import annotations
2
3 from aiohttp import ClientSession, ClientResponse, ClientTimeout
4 from typing import AsyncGenerator, Any
5
6 from .Provider.helper import get_connector
7
8 class StreamResponse(ClientResponse):
9 async def iter_lines(self) -> AsyncGenerator[bytes, None]:
10 async for line in self.content:
11 yield line.rstrip(b"\r\n")
12
13 async def json(self) -> Any:
14 return await super().json(content_type=None)
15
16 class StreamSession(ClientSession):
17 def __init__(self, headers: dict = {}, timeout: int = None, proxies: dict = {}, impersonate = None, **kwargs):
18 if impersonate:
19 headers = {
20 'Accept-Encoding': 'gzip, deflate, br',
21 'Accept-Language': 'en-US',
22 'Connection': 'keep-alive',
23 'Sec-Fetch-Dest': 'empty',
24 'Sec-Fetch-Mode': 'cors',
25 'Sec-Fetch-Site': 'same-site',
26 "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
27 'Accept': '*/*',
28 'sec-ch-ua': '"Google Chrome";v="107", "Chromium";v="107", "Not?A_Brand";v="24"',
29 'sec-ch-ua-mobile': '?0',
30 'sec-ch-ua-platform': '"Windows"',
31 **headers
32 }
33 super().__init__(
34 **kwargs,
35 timeout=ClientTimeout(timeout) if timeout else None,
36 response_class=StreamResponse,
37 connector=get_connector(kwargs.get("connector"), proxies.get("https")),
38 headers=headers
39 )
Added g4f/requests_curl_cffi.py +77 -0
@@ -0,0 +1,77 @@
1 from __future__ import annotations
2
3 from curl_cffi.requests import AsyncSession, Response
4 from typing import AsyncGenerator, Any
5 from functools import partialmethod
6 import json
7
8 class StreamResponse:
9 """
10 A wrapper class for handling asynchronous streaming responses.
11
12 Attributes:
13 inner (Response): The original Response object.
14 """
15
16 def __init__(self, inner: Response) -> None:
17 """Initialize the StreamResponse with the provided Response object."""
18 self.inner: Response = inner
19
20 async def text(self) -> str:
21 """Asynchronously get the response text."""
22 return await self.inner.atext()
23
24 def raise_for_status(self) -> None:
25 """Raise an HTTPError if one occurred."""
26 self.inner.raise_for_status()
27
28 async def json(self, **kwargs) -> Any:
29 """Asynchronously parse the JSON response content."""
30 return json.loads(await self.inner.acontent(), **kwargs)
31
32 async def iter_lines(self) -> AsyncGenerator[bytes, None]:
33 """Asynchronously iterate over the lines of the response."""
34 async for line in self.inner.aiter_lines():
35 yield line
36
37 async def iter_content(self) -> AsyncGenerator[bytes, None]:
38 """Asynchronously iterate over the response content."""
39 async for chunk in self.inner.aiter_content():
40 yield chunk
41
42 async def __aenter__(self):
43 """Asynchronously enter the runtime context for the response object."""
44 inner: Response = await self.inner
45 self.inner = inner
46 self.request = inner.request
47 self.status_code: int = inner.status_code
48 self.reason: str = inner.reason
49 self.ok: bool = inner.ok
50 self.headers = inner.headers
51 self.cookies = inner.cookies
52 return self
53
54 async def __aexit__(self, *args):
55 """Asynchronously exit the runtime context for the response object."""
56 await self.inner.aclose()
57
58 class StreamSession(AsyncSession):
59 """
60 An asynchronous session class for handling HTTP requests with streaming.
61
62 Inherits from AsyncSession.
63 """
64
65 def request(
66 self, method: str, url: str, **kwargs
67 ) -> StreamResponse:
68 """Create and return a StreamResponse object for the given HTTP request."""
69 return StreamResponse(super().request(method, url, stream=True, **kwargs))
70
71 # Defining HTTP methods as partial methods of the request method.
72 head = partialmethod(request, "HEAD")
73 get = partialmethod(request, "GET")
74 post = partialmethod(request, "POST")
75 put = partialmethod(request, "PUT")
76 patch = partialmethod(request, "PATCH")
77 delete = partialmethod(request, "DELETE")
Modified g4f/webdriver.py +22 -1
@@ -13,7 +13,8 @@ try:
13 13 except ImportError:
14 14 from typing import Type as WebDriver
15 15 has_requirements = False
16
16
17 import time
17 18 from os import path
18 19 from os import access, R_OK
19 20 from .errors import MissingRequirementsError
@@ -92,7 +93,27 @@ def bypass_cloudflare(driver: WebDriver, url: str, timeout: int) -> None:
92 93 if driver.find_element(By.TAG_NAME, "body").get_attribute("class") == "no-js":
93 94 if debug.logging:
94 95 print("Cloudflare protection detected:", url)
96
97 # Open website in a new tab
98 element = driver.find_element(By.ID, "challenge-body-text")
99 driver.execute_script(f"""
100 arguments[0].addEventListener('click', () => {{
101 window.open(arguments[1]);
102 }});
103 """, element, url)
104 element.click()
105 time.sleep(3)
106
107 # Switch to the new tab and close the old tab
108 original_window = driver.current_window_handle
109 for window_handle in driver.window_handles:
110 if window_handle != original_window:
111 driver.close()
112 driver.switch_to.window(window_handle)
113 break
114
95 115 try:
116 # Click on the challenge button in the iframe
96 117 driver.switch_to.frame(driver.find_element(By.CSS_SELECTOR, "#turnstile-wrapper iframe"))
97 118 WebDriverWait(driver, 5).until(
98 119 EC.presence_of_element_located((By.CSS_SELECTOR, "#challenge-stage input"))