返回提交历史
Modified
g4f/Provider/AItianhuSpace.py
+26
-32
Modified
g4f/Provider/MyShell.py
+14
-20
Modified
g4f/Provider/PerplexityAi.py
+28
-34
Modified
g4f/Provider/Phind.py
+4
-11
Modified
g4f/Provider/TalkAi.py
+7
-14
Modified
g4f/Provider/helper.py
+52
-0
Modified
g4f/Provider/needs_auth/Bard.py
+19
-26
Modified
g4f/Provider/needs_auth/OpenaiChat.py
+200
-74
Modified
g4f/Provider/needs_auth/Poe.py
+43
-50
Modified
g4f/Provider/needs_auth/Theb.py
+49
-43
Modified
g4f/requests.py
+19
-144
Modified
requirements.txt
+4
-2
XFEstudio/gpt4free
Add Response Handler to OpenaiChat Update Providers with WebDriver Add WebDriverSession helper Use native streaming in curl_cffi
9239cadd
代码差异
12 个文件
+465
-450
@@ -5,7 +5,7 @@ import random
5
5
6
6
from ..typing import CreateResult, Messages
7
7
from .base_provider import BaseProvider
8
from .helper import WebDriver, format_prompt, get_browser, get_random_string
8
from .helper import WebDriver, WebDriverSession, format_prompt, get_random_string
9
9
from .. import debug
10
10
11
11
class AItianhuSpace(BaseProvider):
@@ -24,7 +24,7 @@ class AItianhuSpace(BaseProvider):
24
24
domain: str = None,
25
25
proxy: str = None,
26
26
timeout: int = 120,
27
browser: WebDriver = None,
27
web_driver: WebDriver = None,
28
28
headless: bool = True,
29
29
**kwargs
30
30
) -> CreateResult:
@@ -38,36 +38,35 @@ class AItianhuSpace(BaseProvider):
38
38
print(f"AItianhuSpace | using domain: {domain}")
39
39
url = f"https://{domain}"
40
40
prompt = format_prompt(messages)
41
driver = browser if browser else get_browser("", headless, proxy)
42
41
43
from selenium.webdriver.common.by import By
44
from selenium.webdriver.support.ui import WebDriverWait
45
from selenium.webdriver.support import expected_conditions as EC
42
with WebDriverSession(web_driver, "", headless=headless, proxy=proxy) as driver:
43
from selenium.webdriver.common.by import By
44
from selenium.webdriver.support.ui import WebDriverWait
45
from selenium.webdriver.support import expected_conditions as EC
46
46
47
wait = WebDriverWait(driver, timeout)
47
wait = WebDriverWait(driver, timeout)
48
48
49
# Bypass devtools detection
50
driver.get("https://blank.page/")
51
wait.until(EC.visibility_of_element_located((By.ID, "sheet")))
52
driver.execute_script(f"""
53
document.getElementById('sheet').addEventListener('click', () => {{
54
window.open('{url}', '_blank');
55
}});
56
""")
57
driver.find_element(By.ID, "sheet").click()
58
time.sleep(10)
49
# Bypass devtools detection
50
driver.get("https://blank.page/")
51
wait.until(EC.visibility_of_element_located((By.ID, "sheet")))
52
driver.execute_script(f"""
53
document.getElementById('sheet').addEventListener('click', () => {{
54
window.open('{url}', '_blank');
55
}});
56
""")
57
driver.find_element(By.ID, "sheet").click()
58
time.sleep(10)
59
59
60
original_window = driver.current_window_handle
61
for window_handle in driver.window_handles:
62
if window_handle != original_window:
63
driver.close()
64
driver.switch_to.window(window_handle)
65
break
60
original_window = driver.current_window_handle
61
for window_handle in driver.window_handles:
62
if window_handle != original_window:
63
driver.close()
64
driver.switch_to.window(window_handle)
65
break
66
66
67
# Wait for page load
68
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea.n-input__textarea-el")))
67
# Wait for page load
68
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea.n-input__textarea-el")))
69
69
70
try:
71
70
# Register hook in XMLHttpRequest
72
71
script = """
73
72
const _http_request_open = XMLHttpRequest.prototype.open;
@@ -114,9 +113,4 @@ return "";
114
113
elif chunk != "":
115
114
break
116
115
else:
117
time.sleep(0.1)
118
finally:
119
if not browser:
120
driver.close()
121
time.sleep(0.1)
122
driver.quit()
116
time.sleep(0.1)
@@ -4,7 +4,7 @@ import time, json
4
4
5
5
from ..typing import CreateResult, Messages
6
6
from .base_provider import BaseProvider
7
from .helper import WebDriver, format_prompt, get_browser
7
from .helper import WebDriver, WebDriverSession, format_prompt
8
8
9
9
class MyShell(BaseProvider):
10
10
url = "https://app.myshell.ai/chat"
@@ -20,22 +20,27 @@ class MyShell(BaseProvider):
20
20
stream: bool,
21
21
proxy: str = None,
22
22
timeout: int = 120,
23
browser: WebDriver = None,
23
web_driver: WebDriver = None,
24
24
**kwargs
25
25
) -> CreateResult:
26
driver = browser if browser else get_browser("", False, proxy)
26
with WebDriverSession(web_driver, "", proxy=proxy) as driver:
27
from selenium.webdriver.common.by import By
28
from selenium.webdriver.support.ui import WebDriverWait
29
from selenium.webdriver.support import expected_conditions as EC
27
30
28
from selenium.webdriver.common.by import By
29
from selenium.webdriver.support.ui import WebDriverWait
30
from selenium.webdriver.support import expected_conditions as EC
31
driver.get(cls.url)
31
32
32
driver.get(cls.url)
33
try:
34
33
# Wait for page load and cloudflare validation
35
34
WebDriverWait(driver, timeout).until(
36
35
EC.presence_of_element_located((By.CSS_SELECTOR, "body:not(.no-js)"))
37
36
)
38
37
# Send request with message
38
data = {
39
"botId": "4738",
40
"conversation_scenario": 3,
41
"message": format_prompt(messages),
42
"messageType": 1
43
}
39
44
script = """
40
45
response = await fetch("https://api.myshell.ai/v1/bot/chat/send_message", {
41
46
"headers": {
@@ -49,12 +54,6 @@ response = await fetch("https://api.myshell.ai/v1/bot/chat/send_message", {
49
54
})
50
55
window.reader = response.body.getReader();
51
56
"""
52
data = {
53
"botId": "4738",
54
"conversation_scenario": 3,
55
"message": format_prompt(messages),
56
"messageType": 1
57
}
58
57
driver.execute_script(script.replace("{body}", json.dumps(data)))
59
58
script = """
60
59
chunk = await window.reader.read();
@@ -80,9 +79,4 @@ return content;
80
79
elif chunk != "":
81
80
break
82
81
else:
83
time.sleep(0.1)
84
finally:
85
if not browser:
86
driver.close()
87
time.sleep(0.1)
88
driver.quit()
82
time.sleep(0.1)
@@ -4,7 +4,7 @@ import time
4
4
5
5
from ..typing import CreateResult, Messages
6
6
from .base_provider import BaseProvider
7
from .helper import WebDriver, format_prompt, get_browser
7
from .helper import WebDriver, WebDriverSession, format_prompt
8
8
9
9
class PerplexityAi(BaseProvider):
10
10
url = "https://www.perplexity.ai"
@@ -20,27 +20,27 @@ class PerplexityAi(BaseProvider):
20
20
stream: bool,
21
21
proxy: str = None,
22
22
timeout: int = 120,
23
browser: WebDriver = None,
23
web_driver: WebDriver = None,
24
virtual_display: bool = True,
24
25
copilot: bool = False,
25
26
**kwargs
26
27
) -> CreateResult:
27
driver = browser if browser else get_browser("", False, proxy)
28
with WebDriverSession(web_driver, "", virtual_display=virtual_display, proxy=proxy) as driver:
29
from selenium.webdriver.common.by import By
30
from selenium.webdriver.support.ui import WebDriverWait
31
from selenium.webdriver.support import expected_conditions as EC
32
from selenium.webdriver.common.keys import Keys
28
33
29
from selenium.webdriver.common.by import By
30
from selenium.webdriver.support.ui import WebDriverWait
31
from selenium.webdriver.support import expected_conditions as EC
32
from selenium.webdriver.common.keys import Keys
34
prompt = format_prompt(messages)
33
35
34
prompt = format_prompt(messages)
36
driver.get(f"{cls.url}/")
37
wait = WebDriverWait(driver, timeout)
35
38
36
driver.get(f"{cls.url}/")
37
wait = WebDriverWait(driver, timeout)
39
# Is page loaded?
40
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']")))
38
41
39
# Is page loaded?
40
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']")))
41
42
# Register WebSocket hook
43
script = """
42
# Register WebSocket hook
43
script = """
44
44
window._message = window._last_message = "";
45
45
window._message_finished = false;
46
46
const _socket_send = WebSocket.prototype.send;
@@ -67,22 +67,21 @@ WebSocket.prototype.send = function(...args) {
67
67
return _socket_send.call(this, ...args);
68
68
};
69
69
"""
70
driver.execute_script(script)
70
driver.execute_script(script)
71
71
72
if copilot:
73
try:
74
# Check for account
75
driver.find_element(By.CSS_SELECTOR, "img[alt='User avatar']")
76
# Enable copilot
77
driver.find_element(By.CSS_SELECTOR, "button[data-testid='copilot-toggle']").click()
78
except:
79
raise RuntimeError("You need a account for copilot")
72
if copilot:
73
try:
74
# Check for account
75
driver.find_element(By.CSS_SELECTOR, "img[alt='User avatar']")
76
# Enable copilot
77
driver.find_element(By.CSS_SELECTOR, "button[data-testid='copilot-toggle']").click()
78
except:
79
raise RuntimeError("You need a account for copilot")
80
80
81
# Submit prompt
82
driver.find_element(By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']").send_keys(prompt)
83
driver.find_element(By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']").send_keys(Keys.ENTER)
81
# Submit prompt
82
driver.find_element(By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']").send_keys(prompt)
83
driver.find_element(By.CSS_SELECTOR, "textarea[placeholder='Ask anything...']").send_keys(Keys.ENTER)
84
84
85
try:
86
85
# Stream response
87
86
script = """
88
87
if(window._message && window._message != window._last_message) {
@@ -104,9 +103,4 @@ if(window._message && window._message != window._last_message) {
104
103
elif chunk != "":
105
104
break
106
105
else:
107
time.sleep(0.1)
108
finally:
109
if not browser:
110
driver.close()
111
time.sleep(0.1)
112
driver.quit()
106
time.sleep(0.1)
@@ -5,7 +5,7 @@ from urllib.parse import quote
5
5
6
6
from ..typing import CreateResult, Messages
7
7
from .base_provider import BaseProvider
8
from .helper import WebDriver, format_prompt, get_browser
8
from .helper import WebDriver, WebDriverSession, format_prompt
9
9
10
10
class Phind(BaseProvider):
11
11
url = "https://www.phind.com"
@@ -21,13 +21,11 @@ class Phind(BaseProvider):
21
21
stream: bool,
22
22
proxy: str = None,
23
23
timeout: int = 120,
24
browser: WebDriver = None,
24
web_driver: WebDriver = None,
25
25
creative_mode: bool = None,
26
26
**kwargs
27
27
) -> CreateResult:
28
try:
29
driver = browser if browser else get_browser("", False, proxy)
30
28
with WebDriverSession(web_driver, "", proxy=proxy) as driver:
31
29
from selenium.webdriver.common.by import By
32
30
from selenium.webdriver.support.ui import WebDriverWait
33
31
from selenium.webdriver.support import expected_conditions as EC
@@ -102,9 +100,4 @@ if(window._reader) {
102
100
elif chunk != "":
103
101
break
104
102
else:
105
time.sleep(0.1)
106
finally:
107
if not browser:
108
driver.close()
109
time.sleep(0.1)
110
driver.quit()
103
time.sleep(0.1)
@@ -4,7 +4,7 @@ import time, json, time
4
4
5
5
from ..typing import CreateResult, Messages
6
6
from .base_provider import BaseProvider
7
from .helper import WebDriver, get_browser
7
from .helper import WebDriver, WebDriverSession
8
8
9
9
class TalkAi(BaseProvider):
10
10
url = "https://talkai.info"
@@ -19,16 +19,14 @@ class TalkAi(BaseProvider):
19
19
messages: Messages,
20
20
stream: bool,
21
21
proxy: str = None,
22
browser: WebDriver = None,
22
web_driver: WebDriver = None,
23
23
**kwargs
24
24
) -> CreateResult:
25
driver = browser if browser else get_browser("", False, proxy)
25
with WebDriverSession(web_driver, "", virtual_display=True, proxy=proxy) as driver:
26
from selenium.webdriver.common.by import By
27
from selenium.webdriver.support.ui import WebDriverWait
28
from selenium.webdriver.support import expected_conditions as EC
26
29
27
from selenium.webdriver.common.by import By
28
from selenium.webdriver.support.ui import WebDriverWait
29
from selenium.webdriver.support import expected_conditions as EC
30
31
try:
32
30
driver.get(f"{cls.url}/chat/")
33
31
34
32
# Wait for page load
@@ -86,9 +84,4 @@ return content;
86
84
elif chunk != "":
87
85
break
88
86
else:
89
time.sleep(0.1)
90
finally:
91
if not browser:
92
driver.close()
93
time.sleep(0.1)
94
driver.quit()
87
time.sleep(0.1)
@@ -6,6 +6,7 @@ import webbrowser
6
6
import random
7
7
import string
8
8
import secrets
9
import time
9
10
from os import path
10
11
from asyncio import AbstractEventLoop
11
12
from platformdirs import user_config_dir
@@ -34,6 +35,10 @@ except ImportError:
34
35
class ChromeOptions():
35
36
def add_argument():
36
37
pass
38
try:
39
from pyvirtualdisplay import Display
40
except ImportError:
41
pass
37
42
38
43
from ..typing import Dict, Messages, Union, Tuple
39
44
from .. import debug
@@ -144,6 +149,53 @@ def get_browser(
144
149
options.add_argument(f'--proxy-server={proxy}')
145
150
return Chrome(options=options, user_data_dir=user_data_dir, headless=headless)
146
151
152
class WebDriverSession():
153
def __init__(
154
self,
155
web_driver: WebDriver = None,
156
user_data_dir: str = None,
157
headless: bool = False,
158
virtual_display: bool = False,
159
proxy: str = None,
160
options: ChromeOptions = None
161
):
162
self.web_driver = web_driver
163
self.user_data_dir = user_data_dir
164
self.headless = headless
165
self.virtual_display = virtual_display
166
self.proxy = proxy
167
self.options = options
168
169
def reopen(
170
self,
171
user_data_dir: str = None,
172
headless: bool = False,
173
virtual_display: bool = False
174
) -> WebDriver:
175
if user_data_dir == None:
176
user_data_dir = self.user_data_dir
177
self.default_driver.quit()
178
if not virtual_display and self.virtual_display:
179
self.virtual_display.stop()
180
self.default_driver = get_browser(user_data_dir, headless, self.proxy)
181
return self.default_driver
182
183
def __enter__(self) -> WebDriver:
184
if self.web_driver:
185
return self.web_driver
186
if self.virtual_display == True:
187
self.virtual_display = Display(size=(1920,1080))
188
self.virtual_display.start()
189
self.default_driver = get_browser(self.user_data_dir, self.headless, self.proxy, self.options)
190
return self.default_driver
191
192
def __exit__(self, exc_type, exc_val, exc_tb):
193
if self.default_driver:
194
self.default_driver.close()
195
time.sleep(0.1)
196
self.default_driver.quit()
197
if self.virtual_display:
198
self.virtual_display.stop()
147
199
148
200
def get_random_string(length: int = 10) -> str:
149
201
return ''.join(
@@ -4,7 +4,7 @@ import time
4
4
5
5
from ...typing import CreateResult, Messages
6
6
from ..base_provider import BaseProvider
7
from ..helper import WebDriver, format_prompt, get_browser
7
from ..helper import WebDriver, WebDriverSession, format_prompt
8
8
9
9
class Bard(BaseProvider):
10
10
url = "https://bard.google.com"
@@ -18,34 +18,32 @@ class Bard(BaseProvider):
18
18
messages: Messages,
19
19
stream: bool,
20
20
proxy: str = None,
21
browser: WebDriver = None,
21
web_driver: WebDriver = None,
22
22
user_data_dir: str = None,
23
23
headless: bool = True,
24
24
**kwargs
25
25
) -> CreateResult:
26
26
prompt = format_prompt(messages)
27
driver = browser if browser else get_browser(user_data_dir, headless, proxy)
27
session = WebDriverSession(web_driver, user_data_dir, headless, proxy=proxy)
28
with session as driver:
29
from selenium.webdriver.common.by import By
30
from selenium.webdriver.support.ui import WebDriverWait
31
from selenium.webdriver.support import expected_conditions as EC
28
32
29
from selenium.webdriver.common.by import By
30
from selenium.webdriver.support.ui import WebDriverWait
31
from selenium.webdriver.support import expected_conditions as EC
32
33
try:
34
driver.get(f"{cls.url}/chat")
35
wait = WebDriverWait(driver, 10 if headless else 240)
36
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.ql-editor.textarea")))
37
except:
38
# Reopen browser for login
39
if not browser:
40
driver.quit()
41
driver = get_browser(None, False, proxy)
33
try:
42
34
driver.get(f"{cls.url}/chat")
43
wait = WebDriverWait(driver, 240)
35
wait = WebDriverWait(driver, 10 if headless else 240)
44
36
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.ql-editor.textarea")))
45
else:
46
raise RuntimeError("Prompt textarea not found. You may not be logged in.")
37
except:
38
# Reopen browser for login
39
if not web_driver:
40
driver = session.reopen(headless=False)
41
driver.get(f"{cls.url}/chat")
42
wait = WebDriverWait(driver, 240)
43
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.ql-editor.textarea")))
44
else:
45
raise RuntimeError("Prompt textarea not found. You may not be logged in.")
47
46
48
try:
49
47
# Add hook in XMLHttpRequest
50
48
script = """
51
49
const _http_request_open = XMLHttpRequest.prototype.open;
@@ -72,9 +70,4 @@ XMLHttpRequest.prototype.open = function(method, url) {
72
70
yield chunk
73
71
return
74
72
else:
75
time.sleep(0.1)
76
finally:
77
if not browser:
78
driver.close()
79
time.sleep(0.1)
80
driver.quit()
73
time.sleep(0.1)
@@ -1,20 +1,64 @@
1
1
from __future__ import annotations
2
2
3
import uuid, json, time, asyncio
3
import uuid, json, asyncio
4
4
from py_arkose_generator.arkose import get_values_for_request
5
from asyncstdlib.itertools import tee
6
from async_property import async_cached_property
5
7
6
8
from ..base_provider import AsyncGeneratorProvider
7
from ..helper import get_browser, get_cookies, format_prompt, get_event_loop
9
from ..helper import get_browser, get_event_loop
8
10
from ...typing import AsyncResult, Messages
9
11
from ...requests import StreamSession
10
from ... import debug
12
13
models = {
14
"gpt-3.5": "text-davinci-002-render-sha",
15
"gpt-3.5-turbo": "text-davinci-002-render-sha",
16
"gpt-4": "gpt-4",
17
"gpt-4-gizmo": "gpt-4-gizmo"
18
}
11
19
12
20
class OpenaiChat(AsyncGeneratorProvider):
13
21
url = "https://chat.openai.com"
14
needs_auth = True
15
22
working = True
23
needs_auth = True
16
24
supports_gpt_35_turbo = True
17
_access_token = None
25
supports_gpt_4 = True
26
_access_token: str = None
27
28
@classmethod
29
async def create(
30
cls,
31
prompt: str = None,
32
model: str = "",
33
messages: Messages = [],
34
history_disabled: bool = False,
35
action: str = "next",
36
conversation_id: str = None,
37
parent_id: str = None,
38
**kwargs
39
) -> Response:
40
if prompt:
41
messages.append({"role": "user", "content": prompt})
42
generator = cls.create_async_generator(
43
model,
44
messages,
45
history_disabled=history_disabled,
46
action=action,
47
conversation_id=conversation_id,
48
parent_id=parent_id,
49
response_fields=True,
50
**kwargs
51
)
52
fields: ResponseFields = await anext(generator)
53
if "access_token" not in kwargs:
54
kwargs["access_token"] = cls._access_token
55
return Response(
56
generator,
57
fields,
58
action,
59
messages,
60
kwargs
61
)
18
62
19
63
@classmethod
20
64
async def create_async_generator(
@@ -25,50 +69,56 @@ class OpenaiChat(AsyncGeneratorProvider):
25
69
timeout: int = 120,
26
70
access_token: str = None,
27
71
auto_continue: bool = False,
28
cookies: dict = None,
72
history_disabled: bool = True,
73
action: str = "next",
74
conversation_id: str = None,
75
parent_id: str = None,
76
response_fields: bool = False,
29
77
**kwargs
30
78
) -> AsyncResult:
31
proxies = {"https": proxy}
79
if not model:
80
model = "gpt-3.5"
81
elif model not in models:
82
raise ValueError(f"Model are not supported: {model}")
83
if not parent_id:
84
parent_id = str(uuid.uuid4())
32
85
if not access_token:
33
access_token = await cls.get_access_token(cookies, proxies)
86
access_token = await cls.get_access_token(proxy)
34
87
headers = {
35
88
"Accept": "text/event-stream",
36
89
"Authorization": f"Bearer {access_token}",
90
"Cookie": 'intercom-device-id-dgkjq2bp=0f047573-a750-46c8-be62-6d54b56e7bf0; ajs_user_id=user-iv3vxisaoNodwWpxmNpMfekH; ajs_anonymous_id=fd91be0b-0251-4222-ac1e-84b1071e9ec1; __Host-next-auth.csrf-token=d2b5f67d56f7dd6a0a42ae4becf2d1a6577b820a5edc88ab2018a59b9b506886%7Ce5c33eecc460988a137cbc72d90ee18f1b4e2f672104f368046df58e364376ac; _cfuvid=gt_mA.q6rue1.7d2.AR0KHpbVBS98i_ppfi.amj2._o-1700353424353-0-604800000; cf_clearance=GkHCfPSFU.NXGcHROoe4FantnqmnNcluhTNHz13Tk.M-1700353425-0-1-dfe77f81.816e9bc2.714615da-0.2.1700353425; __Secure-next-auth.callback-url=https%3A%2F%2Fchat.openai.com; intercom-session-dgkjq2bp=UWdrS1hHazk5VXN1c0V5Q1F0VXdCQmsyTU9pVjJMUkNpWnFnU3dKWmtIdGwxTC9wbjZuMk5hcEc0NWZDOGdndS0tSDNiaDNmMEdIL1RHU1dFWDBwOHFJUT09--f754361b91fddcd23a13b288dcb2bf8c7f509e91; _uasid="Z0FBQUFBQmxXVnV0a3dmVno4czRhcDc2ZVcwaUpSNUdZejlDR25YSk5NYTJQQkpyNmRvOGxjTHMyTlAxWmJhaURrMVhjLXZxQXdZeVpBbU1aczA5WUpHT2dwaS1MOWc4MnhyNWFnbGRzeGdJcGFKT0ZRdnBTMVJHcGV2MGNTSnVQY193c0hqUWIycHhQRVF4dENlZ3phcDdZeHgxdVhoalhrZmtZME9NbWhMQjdVR3Vzc3FRRk0ybjJjNWMwTWtIRjdPb19lUkFtRmV2MDVqd1kwWU11QTYtQkdZenEzVHhLMGplY1hZM3FlYUt1cVZaNWFTRldleEJETzJKQjk1VTJScy1GUnMxUVZWMnVxYklxMjdockVZbkZyd1R4U1RtMnA1ZzlSeXphdmVOVk9xeEdrRkVOSjhwTVd1QzFtQjhBcWdDaE92Q1VlM2pwcjFQTXRuLVJNRVlZSGpIdlZ0aGV3PT0="; _dd_s=rum=0&expire=1700356244884; __Secure-next-auth.session-token=eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..3aK6Fbdy2_8f07bf.8eT2xgonrCnz7ySY6qXFsg3kzL6UQfXKAYaw3tyn-6_X9657zy47k9qGvmi9mF0QKozj5jau3_Ca62AQQ7FmeC6Y2F1urtzqrXqwTTsQ2LuzFPIQkx6KKb2DXc8zW2-oyEzJ_EY5yxfLB2RlRkSh3M7bYNZh4_ltEcfkj38s_kIPGMxv34udtPWGWET99MCjkdwQWXylJag4s0fETA0orsBAKnGCyqAUNJbb_D7BYtGSV-MQ925kZMG6Di_QmfO0HQWURDYjmdRNcuy1PT_xJ1DJko8sjL42i4j3RhkNDkhqCIqyYImz2eHFWHW7rYKxTkrBhlCPMS5hRdcCswD7JYPcSBiwnVRYgyOocFGXoFvQgIZ2FX9NiZ3SMEVM1VwIGSE-qH0H2nMa8_iBvsOgOWJgKjVAvzzyzZvRVDUUHzJrikSFPNONVDU3h-04c1kVL4qIu9DfeTPN7n8AvNmYwMbro0L9-IUAeXNo4-pwF0Kt-AtTsamqWvMqnK4O_YOyLnDDlvkmnOvDC2d5uinwlQIxr6APO6qFfGLlHiLZemKoekxEE1Fx70dl-Ouhk1VIzbF3OC6XNNxeBm9BUYUiHdL0wj2H9rHgX4cz6ZmS_3VTgpD6UJh-evu5KJ2gIvjYmVbyzEN0aPNDxfvBaOm-Ezpy4bUJ2bUrOwNn-0knWkDiTvjYmNhCyefPCtCF6rpKNay8PCw_yh79C4SdEP6Q4V7LI0Tvdi5uz7kLCiBC4AT9L0ao1WDX03mkUOpjvzHDvPLmj8chW3lTVm_kA0eYGQY4wT0jzleWlfV0Q8rB2oYECNLWksA3F1zlGfcl4lQjprvTXRePkvAbMpoJEsZD3Ylq7-foLDLk4-M2LYAFZDs282AY04sFjAjQBxTELFCCuDgTIgTXSIskY_XCxpVXDbdLlbCJY7XVK45ybwtfqwlKRp8Mo0B131uQAFc-migHaUaoGujxJJk21bP8F0OmhNYHBo4FQqE1rQm2JH5bNM7txKeh5KXdJgVUVbRSr7OIp_OF5-Bx_v9eRBGAIDkue26E2-O8Rnrp5zQ5TnvecQLDaUzWavCLPwsZ0_gsOLBxNOmauNYZtF8IElCsQSFDdhoiMxXsYUm4ZYKEAy3GWq8HGTAvBhNkh1hvnI7y-d8-DOaZf_D_D98-olZfm-LUkeosLNpPB9rxYMqViCiW3KrXE9Yx0wlFm5ePKaVvR7Ym_EPhSOhJBKFPCvdTdMZSNPUcW0ZJBVByq0A9sxD51lYq3gaFyqh94S4s_ox182AQ3szGzHkdgLcnQmJG9OYvKxAVcd43eg6_gODAYhx02GjbMw-7JTAhyXSeCrlMteHyOXl8hai-3LilC3PmMzi7Vbu49dhF1s4LcVlUowen5ira44rQQaB26mdaOUoQfodgt66M3RTWGPXyK1Nb72AzSXsCKyaQPbzeb6cN0fdGSdG4ktwvR04eFNEkquo_3aKu2GmUKTD0XcRx9dYrfXjgY-X1DDTVs1YND2gRhdx7FFEeBVjtbj2UqmG3Rvd4IcHGe7OnYWw2MHDcol68SsR1KckXWwWREz7YTGUnDB2M1kx_H4W2mjclytnlHOnYU3RflegRPeSTbdzUZJvGKXCCz45luHkQWN_4DExE76D-9YqbFIz-RY5yL4h-Zs-i2xjm2K-4xCMM9nQIOqhLMqixIZQ2ldDAidKoYtbs5ppzbcBLyrZM96bq9DwRBY3aacqWdlRd-TfX0wv5KO4fo0sSh5FsuhuN0zcEV_NNXgqIEM_p14EcPqgbrAvCBQ8os70TRBQLXiF0EniSofGjxwF8kQvUk3C6Wfc8cTTeN-E6GxCVTn91HBwA1iSEZlRLMVb8_BcRJNqwbgnb_07jR6-eo42u88CR3KQdAWwbQRdMxsURFwZ0ujHXVGG0Ll6qCFBcHXWyDO1x1yHdHnw8_8yF26pnA2iPzrFR-8glMgIA-639sLuGAxjO1_ZuvJ9CAB41Az9S_jaZwaWy215Hk4-BRYD-MKmHtonwo3rrxhE67WJgbbu14efsw5nT6ow961pffgwXov5VA1Rg7nv1E8RvQOx7umWW6o8R4W6L8f2COsmPTXfgwIjoJKkjhUqAQ8ceG7cM0ET-38yaC0ObU8EkXfdGGgxI28qTEZWczG66_iM4hw7QEGCY5Cz2kbO6LETAiw9OsSigtBvDS7f0Ou0bZ41pdK7G3FmvdZAnjWPjObnDF4k4uWfn7mzt0fgj3FyqK20JezRDyGuAbUUhOvtZpc9sJpzxR34eXEZTouuALrHcGuNij4z6rx51FrQsaMtiup8QVrhtZbXtKLMYnWYSbkhuTeN2wY-xV1ZUsQlakIZszzGF7kuIG87KKWMpuPMvbXjz6Pp_gWJiIC6aQuk8xl5g0iBPycf_6Q-MtpuYxzNE2TpI1RyR9mHeXmteoRzrFiWp7yEC-QGNFyAJgxTqxM3CjHh1Jt6IddOsmn89rUo1dZM2Smijv_fbIv3avXLkIPX1KZjILeJCtpU0wAdsihDaRiRgDdx8fG__F8zuP0n7ziHas73cwrfg-Ujr6DhC0gTNxyd9dDA_oho9N7CQcy6EFmfNF2te7zpLony0859jtRv2t1TnpzAa1VvMK4u6mXuJ2XDo04_6GzLO3aPHinMdl1BcIAWnqAqWAu3euGFLTHOhXlfijut9N1OCifd_zWjhVtzlR39uFeCQBU5DyQArzQurdoMx8U1ETsnWgElxGSStRW-YQoPsAJ87eg9trqKspFpTVlAVN3t1GtoEAEhcwhe81SDssLmKGLc.7PqS6jRGTIfgTPlO7Ognvg; __cf_bm=VMWoAKEB45hQSwxXtnYXcurPaGZDJS4dMi6dIMFLwdw-1700355394-0-ATVsbq97iCaTaJbtYr8vtg1Zlbs3nLrJLKVBHYa2Jn7hhkGclqAy8Gbyn5ePEhDRqj93MsQmtayfYLqY5n4WiLY=; __cflb=0H28vVfF4aAyg2hkHFH9CkdHRXPsfCUf6VpYf2kz3RX'
37
91
}
38
messages = [
39
{
40
"id": str(uuid.uuid4()),
41
"author": {"role": "user"},
42
"content": {"content_type": "text", "parts": [format_prompt(messages)]},
43
},
44
]
45
message_id = str(uuid.uuid4())
46
data = {
47
"action": "next",
48
"arkose_token": await get_arkose_token(proxy),
49
"messages": messages,
50
"conversation_id": None,
51
"parent_message_id": message_id,
52
"model": "text-davinci-002-render-sha",
53
"history_and_training_disabled": not auto_continue,
54
}
55
conversation_id = None
56
end_turn = False
57
while not end_turn:
58
if not auto_continue:
59
end_turn = True
60
async with StreamSession(
61
proxies=proxies,
62
headers=headers,
63
impersonate="chrome107",
64
timeout=timeout
65
) as session:
92
async with StreamSession(
93
proxies={"https": proxy},
94
impersonate="chrome110",
95
headers=headers,
96
timeout=timeout
97
) as session:
98
data = {
99
"action": action,
100
"arkose_token": await get_arkose_token(proxy, timeout),
101
"conversation_id": conversation_id,
102
"parent_message_id": parent_id,
103
"model": models[model],
104
"history_and_training_disabled": history_disabled and not auto_continue,
105
}
106
if action != "continue":
107
data["messages"] = [{
108
"id": str(uuid.uuid4()),
109
"author": {"role": "user"},
110
"content": {"content_type": "text", "parts": [messages[-1]["content"]]},
111
}]
112
first = True
113
end_turn = EndTurn()
114
while first or auto_continue and not end_turn.is_end:
115
first = False
66
116
async with session.post(f"{cls.url}/backend-api/conversation", json=data) as response:
67
117
try:
68
118
response.raise_for_status()
69
119
except:
70
raise RuntimeError(f"Response: {await response.text()}")
71
last_message = ""
120
raise RuntimeError(f"Error {response.status_code}: {await response.text()}")
121
last_message = 0
72
122
async for line in response.iter_lines():
73
123
if line.startswith(b"data: "):
74
124
line = line[6:]
@@ -82,50 +132,52 @@ class OpenaiChat(AsyncGeneratorProvider):
82
132
continue
83
133
if "error" in line and line["error"]:
84
134
raise RuntimeError(line["error"])
85
end_turn = line["message"]["end_turn"]
86
message_id = line["message"]["id"]
87
if line["conversation_id"]:
88
conversation_id = line["conversation_id"]
89
135
if "message_type" not in line["message"]["metadata"]:
90
136
continue
91
if line["message"]["metadata"]["message_type"] in ("next", "continue"):
137
if line["message"]["author"]["role"] != "assistant":
138
continue
139
if line["message"]["metadata"]["message_type"] in ("next", "continue", "variant"):
140
conversation_id = line["conversation_id"]
141
parent_id = line["message"]["id"]
142
if response_fields:
143
response_fields = False
144
yield ResponseFields(conversation_id, parent_id, end_turn)
92
145
new_message = line["message"]["content"]["parts"][0]
93
yield new_message[len(last_message):]
94
last_message = new_message
95
if end_turn:
96
return
146
yield new_message[last_message:]
147
last_message = len(new_message)
148
if "finish_details" in line["message"]["metadata"]:
149
if line["message"]["metadata"]["finish_details"]["type"] == "max_tokens":
150
end_turn.end()
151
97
152
data = {
98
153
"action": "continue",
99
"arkose_token": await get_arkose_token(proxy),
154
"arkose_token": await get_arkose_token(proxy, timeout),
100
155
"conversation_id": conversation_id,
101
"parent_message_id": message_id,
102
"model": "text-davinci-002-render-sha",
156
"parent_message_id": parent_id,
157
"model": models[model],
103
158
"history_and_training_disabled": False,
104
159
}
105
160
await asyncio.sleep(5)
106
161
107
162
@classmethod
108
async def browse_access_token(cls) -> str:
163
async def browse_access_token(cls, proxy: str = None) -> str:
109
164
def browse() -> str:
110
165
try:
111
166
from selenium.webdriver.common.by import By
112
167
from selenium.webdriver.support.ui import WebDriverWait
113
168
from selenium.webdriver.support import expected_conditions as EC
114
169
115
driver = get_browser()
170
driver = get_browser("~/openai", proxy=proxy)
116
171
except ImportError:
117
172
return
118
119
driver.get(f"{cls.url}/")
120
173
try:
174
driver.get(f"{cls.url}/")
121
175
WebDriverWait(driver, 1200).until(
122
176
EC.presence_of_element_located((By.ID, "prompt-textarea"))
123
177
)
124
178
javascript = "return (await (await fetch('/api/auth/session')).json())['accessToken']"
125
179
return driver.execute_script(javascript)
126
180
finally:
127
driver.close()
128
time.sleep(0.1)
129
181
driver.quit()
130
182
loop = get_event_loop()
131
183
return await loop.run_in_executor(
@@ -134,22 +186,9 @@ class OpenaiChat(AsyncGeneratorProvider):
134
186
)
135
187
136
188
@classmethod
137
async def fetch_access_token(cls, cookies: dict, proxies: dict = None) -> str:
138
async with StreamSession(proxies=proxies, cookies=cookies, impersonate="chrome107") as session:
139
async with session.get(f"{cls.url}/api/auth/session") as response:
140
response.raise_for_status()
141
auth = await response.json()
142
if "accessToken" in auth:
143
return auth["accessToken"]
144
145
@classmethod
146
async def get_access_token(cls, cookies: dict = None, proxies: dict = None) -> str:
147
if not cls._access_token:
148
cookies = cookies if cookies else get_cookies("chat.openai.com")
149
if cookies:
150
cls._access_token = await cls.fetch_access_token(cookies, proxies)
189
async def get_access_token(cls, proxy: str = None) -> str:
151
190
if not cls._access_token:
152
cls._access_token = await cls.browse_access_token()
191
cls._access_token = await cls.browse_access_token(proxy)
153
192
if not cls._access_token:
154
193
raise RuntimeError("Read access token failed")
155
194
return cls._access_token
@@ -163,12 +202,11 @@ class OpenaiChat(AsyncGeneratorProvider):
163
202
("stream", "bool"),
164
203
("proxy", "str"),
165
204
("access_token", "str"),
166
("cookies", "dict[str, str]")
167
205
]
168
206
param = ", ".join([": ".join(p) for p in params])
169
207
return f"g4f.provider.{cls.__name__} supports: ({param})"
170
208
171
async def get_arkose_token(proxy: str = None) -> str:
209
async def get_arkose_token(proxy: str = None, timeout: int = None) -> str:
172
210
config = {
173
211
"pkey": "3D86FBBA-9D22-402A-B512-3420086BA6CC",
174
212
"surl": "https://tcr9i.chat.openai.com",
@@ -181,10 +219,98 @@ async def get_arkose_token(proxy: str = None) -> str:
181
219
async with StreamSession(
182
220
proxies={"https": proxy},
183
221
impersonate="chrome107",
222
timeout=timeout
184
223
) as session:
185
224
async with session.post(**args_for_request) as response:
186
225
response.raise_for_status()
187
226
decoded_json = await response.json()
188
227
if "token" in decoded_json:
189
228
return decoded_json["token"]
190
raise RuntimeError(f"Response: {decoded_json}")
229
raise RuntimeError(f"Response: {decoded_json}")
230
231
class EndTurn():
232
def __init__(self):
233
self.is_end = False
234
235
def end(self):
236
self.is_end = True
237
238
class ResponseFields():
239
def __init__(
240
self,
241
conversation_id: str,
242
message_id: str,
243
end_turn: EndTurn
244
):
245
self.conversation_id = conversation_id
246
self.message_id = message_id
247
self._end_turn = end_turn
248
249
class Response():
250
def __init__(
251
self,
252
generator: AsyncResult,
253
fields: ResponseFields,
254
action: str,
255
messages: Messages,
256
options: dict
257
):
258
self.aiter, self.copy = tee(generator)
259
self.fields = fields
260
self.action = action
261
self._messages = messages
262
self._options = options
263
264
def __aiter__(self):
265
return self.aiter
266
267
@async_cached_property
268
async def message(self) -> str:
269
return "".join([chunk async for chunk in self.copy])
270
271
async def next(self, prompt: str, **kwargs) -> Response:
272
return await OpenaiChat.create(
273
**self._options,
274
prompt=prompt,
275
messages=await self.messages,
276
action="next",
277
conversation_id=self.fields.conversation_id,
278
parent_id=self.fields.message_id,
279
**kwargs
280
)
281
282
async def do_continue(self, **kwargs) -> Response:
283
if self.end_turn:
284
raise RuntimeError("Can't continue message. Message already finished.")
285
return await OpenaiChat.create(
286
**self._options,
287
messages=await self.messages,
288
action="continue",
289
conversation_id=self.fields.conversation_id,
290
parent_id=self.fields.message_id,
291
**kwargs
292
)
293
294
async def variant(self, **kwargs) -> Response:
295
if self.action != "next":
296
raise RuntimeError("Can't create variant with continue or variant request.")
297
return await OpenaiChat.create(
298
**self._options,
299
messages=self._messages,
300
action="variant",
301
conversation_id=self.fields.conversation_id,
302
parent_id=self.fields.message_id,
303
**kwargs
304
)
305
306
@async_cached_property
307
async def messages(self):
308
messages = self._messages
309
messages.append({
310
"role": "assistant", "content": await self.message
311
})
312
return messages
313
314
@property
315
def end_turn(self):
316
return self.fields._end_turn.is_end
@@ -4,7 +4,7 @@ import time
4
4
5
5
from ...typing import CreateResult, Messages
6
6
from ..base_provider import BaseProvider
7
from ..helper import WebDriver, format_prompt, get_browser
7
from ..helper import WebDriver, WebDriverSession, format_prompt
8
8
9
9
models = {
10
10
"meta-llama/Llama-2-7b-chat-hf": {"name": "Llama-2-7b"},
@@ -33,7 +33,7 @@ class Poe(BaseProvider):
33
33
messages: Messages,
34
34
stream: bool,
35
35
proxy: str = None,
36
browser: WebDriver = None,
36
web_driver: WebDriver = None,
37
37
user_data_dir: str = None,
38
38
headless: bool = True,
39
39
**kwargs
@@ -43,56 +43,54 @@ class Poe(BaseProvider):
43
43
elif model not in models:
44
44
raise ValueError(f"Model are not supported: {model}")
45
45
prompt = format_prompt(messages)
46
driver = browser if browser else get_browser(user_data_dir, headless, proxy)
47
46
48
script = """
49
window._message = window._last_message = "";
50
window._message_finished = false;
51
class ProxiedWebSocket extends WebSocket {
52
constructor(url, options) {
53
super(url, options);
54
this.addEventListener("message", (e) => {
55
const data = JSON.parse(JSON.parse(e.data)["messages"][0])["payload"]["data"];
56
if ("messageAdded" in data) {
57
if (data["messageAdded"]["author"] != "human") {
58
window._message = data["messageAdded"]["text"];
59
if (data["messageAdded"]["state"] == "complete") {
60
window._message_finished = true;
47
session = WebDriverSession(web_driver, user_data_dir, headless, proxy=proxy)
48
with session as driver:
49
from selenium.webdriver.common.by import By
50
from selenium.webdriver.support.ui import WebDriverWait
51
from selenium.webdriver.support import expected_conditions as EC
52
53
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
54
"source": """
55
window._message = window._last_message = "";
56
window._message_finished = false;
57
class ProxiedWebSocket extends WebSocket {
58
constructor(url, options) {
59
super(url, options);
60
this.addEventListener("message", (e) => {
61
const data = JSON.parse(JSON.parse(e.data)["messages"][0])["payload"]["data"];
62
if ("messageAdded" in data) {
63
if (data["messageAdded"]["author"] != "human") {
64
window._message = data["messageAdded"]["text"];
65
if (data["messageAdded"]["state"] == "complete") {
66
window._message_finished = true;
67
}
61
68
}
62
69
}
63
}
64
});
65
}
66
}
67
window.WebSocket = ProxiedWebSocket;
68
"""
69
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
70
"source": script
71
})
72
73
from selenium.webdriver.common.by import By
74
from selenium.webdriver.support.ui import WebDriverWait
75
from selenium.webdriver.support import expected_conditions as EC
70
});
71
}
72
}
73
window.WebSocket = ProxiedWebSocket;
74
"""
75
})
76
76
77
try:
78
driver.get(f"{cls.url}/{models[model]['name']}")
79
wait = WebDriverWait(driver, 10 if headless else 240)
80
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea[class^='GrowingTextArea']")))
81
except:
82
# Reopen browser for login
83
if not browser:
84
driver.quit()
85
driver = get_browser(None, False, proxy)
77
try:
86
78
driver.get(f"{cls.url}/{models[model]['name']}")
87
wait = WebDriverWait(driver, 240)
79
wait = WebDriverWait(driver, 10 if headless else 240)
88
80
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea[class^='GrowingTextArea']")))
89
else:
90
raise RuntimeError("Prompt textarea not found. You may not be logged in.")
81
except:
82
# Reopen browser for login
83
if not web_driver:
84
driver = session.reopen(headless=False)
85
driver.get(f"{cls.url}/{models[model]['name']}")
86
wait = WebDriverWait(driver, 240)
87
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "textarea[class^='GrowingTextArea']")))
88
else:
89
raise RuntimeError("Prompt textarea not found. You may not be logged in.")
91
90
92
driver.find_element(By.CSS_SELECTOR, "footer textarea[class^='GrowingTextArea']").send_keys(prompt)
93
driver.find_element(By.CSS_SELECTOR, "footer button[class*='ChatMessageSendButton']").click()
91
driver.find_element(By.CSS_SELECTOR, "footer textarea[class^='GrowingTextArea']").send_keys(prompt)
92
driver.find_element(By.CSS_SELECTOR, "footer button[class*='ChatMessageSendButton']").click()
94
93
95
try:
96
94
script = """
97
95
if(window._message && window._message != window._last_message) {
98
96
try {
@@ -113,9 +111,4 @@ if(window._message && window._message != window._last_message) {
113
111
elif chunk != "":
114
112
break
115
113
else:
116
time.sleep(0.1)
117
finally:
118
if not browser:
119
driver.close()
120
time.sleep(0.1)
121
driver.quit()
114
time.sleep(0.1)
@@ -4,7 +4,7 @@ import time
4
4
5
5
from ...typing import CreateResult, Messages
6
6
from ..base_provider import BaseProvider
7
from ..helper import WebDriver, format_prompt, get_browser
7
from ..helper import WebDriver, WebDriverSession, format_prompt
8
8
9
9
models = {
10
10
"theb-ai": "TheB.AI",
@@ -44,26 +44,60 @@ class Theb(BaseProvider):
44
44
messages: Messages,
45
45
stream: bool,
46
46
proxy: str = None,
47
browser: WebDriver = None,
48
headless: bool = True,
47
web_driver: WebDriver = None,
48
virtual_display: bool = True,
49
49
**kwargs
50
50
) -> CreateResult:
51
51
if model in models:
52
52
model = models[model]
53
53
prompt = format_prompt(messages)
54
driver = browser if browser else get_browser(None, headless, proxy)
54
web_session = WebDriverSession(web_driver, virtual_display=virtual_display, proxy=proxy)
55
with web_session as driver:
56
from selenium.webdriver.common.by import By
57
from selenium.webdriver.support.ui import WebDriverWait
58
from selenium.webdriver.support import expected_conditions as EC
59
from selenium.webdriver.common.keys import Keys
55
60
56
from selenium.webdriver.common.by import By
57
from selenium.webdriver.support.ui import WebDriverWait
58
from selenium.webdriver.support import expected_conditions as EC
59
from selenium.webdriver.common.keys import Keys
61
# Register fetch hook
62
script = """
63
window._fetch = window.fetch;
64
window.fetch = (url, options) => {
65
// Call parent fetch method
66
const result = window._fetch(url, options);
67
if (!url.startsWith("/api/conversation")) {
68
return result;
69
}
70
// Load response reader
71
result.then((response) => {
72
if (!response.body.locked) {
73
window._reader = response.body.getReader();
74
}
75
});
76
// Return dummy response
77
return new Promise((resolve, reject) => {
78
resolve(new Response(new ReadableStream()))
79
});
80
}
81
window._last_message = "";
82
"""
83
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
84
"source": script
85
})
86
87
try:
88
driver.get(f"{cls.url}/home")
89
wait = WebDriverWait(driver, 5)
90
wait.until(EC.visibility_of_element_located((By.ID, "textareaAutosize")))
91
except:
92
driver = web_session.reopen()
93
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
94
"source": script
95
})
96
driver.get(f"{cls.url}/home")
97
wait = WebDriverWait(driver, 240)
98
wait.until(EC.visibility_of_element_located((By.ID, "textareaAutosize")))
60
99
61
62
try:
63
driver.get(f"{cls.url}/home")
64
wait = WebDriverWait(driver, 10 if headless else 240)
65
wait.until(EC.visibility_of_element_located((By.TAG_NAME, "body")))
66
time.sleep(0.1)
100
time.sleep(200)
67
101
try:
68
102
driver.find_element(By.CSS_SELECTOR, ".driver-overlay").click()
69
103
driver.find_element(By.CSS_SELECTOR, ".driver-overlay").click()
@@ -87,29 +121,6 @@ class Theb(BaseProvider):
87
121
button = container.find_element(By.CSS_SELECTOR, "button.btn-blue.btn-small.border")
88
122
button.click()
89
123
90
# Register fetch hook
91
script = """
92
window._fetch = window.fetch;
93
window.fetch = (url, options) => {
94
// Call parent fetch method
95
const result = window._fetch(url, options);
96
if (!url.startsWith("/api/conversation")) {
97
return result;
98
}
99
// Load response reader
100
result.then((response) => {
101
if (!response.body.locked) {
102
window._reader = response.body.getReader();
103
}
104
});
105
// Return dummy response
106
return new Promise((resolve, reject) => {
107
resolve(new Response(new ReadableStream()))
108
});
109
}
110
window._last_message = "";
111
"""
112
driver.execute_script(script)
113
124
114
125
# Submit prompt
115
126
wait.until(EC.visibility_of_element_located((By.ID, "textareaAutosize")))
@@ -150,9 +161,4 @@ return '';
150
161
elif chunk != "":
151
162
break
152
163
else:
153
time.sleep(0.1)
154
finally:
155
if not browser:
156
driver.close()
157
time.sleep(0.1)
158
driver.quit()
164
time.sleep(0.1)
@@ -1,24 +1,15 @@
1
1
from __future__ import annotations
2
2
3
import warnings
4
3
import json
5
import asyncio
4
from contextlib import asynccontextmanager
6
5
from functools import partialmethod
7
from asyncio import Future, Queue
8
from typing import AsyncGenerator, Union, Optional
6
from typing import AsyncGenerator
9
7
10
8
from curl_cffi.requests import AsyncSession, Response
11
import curl_cffi
12
13
is_newer_0_5_8: bool = hasattr(AsyncSession, "_set_cookies") or hasattr(curl_cffi.requests.Cookies, "get_cookies_for_curl")
14
is_newer_0_5_9: bool = hasattr(curl_cffi.AsyncCurl, "remove_handle")
15
is_newer_0_5_10: bool = hasattr(AsyncSession, "release_curl")
16
17
9
18
10
class StreamResponse:
19
def __init__(self, inner: Response, queue: Queue[bytes]) -> None:
11
def __init__(self, inner: Response) -> None:
20
12
self.inner: Response = inner
21
self.queue: Queue[bytes] = queue
22
13
self.request = inner.request
23
14
self.status_code: int = inner.status_code
24
15
self.reason: str = inner.reason
@@ -27,148 +18,32 @@ class StreamResponse:
27
18
self.cookies = inner.cookies
28
19
29
20
async def text(self) -> str:
30
content: bytes = await self.read()
31
return content.decode()
21
return await self.inner.atext()
32
22
33
23
def raise_for_status(self) -> None:
34
if not self.ok:
35
raise RuntimeError(f"HTTP Error {self.status_code}: {self.reason}")
24
self.inner.raise_for_status()
36
25
37
26
async def json(self, **kwargs) -> dict:
38
return json.loads(await self.read(), **kwargs)
39
40
async def iter_lines(
41
self, chunk_size: Optional[int] = None, decode_unicode: bool = False, delimiter: Optional[str] = None
42
) -> AsyncGenerator[bytes, None]:
43
"""
44
Copied from: https://requests.readthedocs.io/en/latest/_modules/requests/models/
45
which is under the License: Apache 2.0
46
"""
47
48
pending: bytes = None
49
50
async for chunk in self.iter_content(
51
chunk_size=chunk_size, decode_unicode=decode_unicode
52
):
53
if pending is not None:
54
chunk = pending + chunk
55
lines = chunk.split(delimiter) if delimiter else chunk.splitlines()
56
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
57
pending = lines.pop()
58
else:
59
pending = None
27
return json.loads(await self.inner.acontent(), **kwargs)
60
28
61
for line in lines:
62
yield line
29
async def iter_lines(self) -> AsyncGenerator[bytes, None]:
30
async for line in self.inner.aiter_lines():
31
yield line
63
32
64
if pending is not None:
65
yield pending
66
67
async def iter_content(
68
self, chunk_size: Optional[int] = None, decode_unicode: bool = False
69
) -> AsyncGenerator[bytes, None]:
70
if chunk_size:
71
warnings.warn("chunk_size is ignored, there is no way to tell curl that.")
72
if decode_unicode:
73
raise NotImplementedError()
74
while True:
75
chunk = await self.queue.get()
76
if chunk is None:
77
return
33
async def iter_content(self) -> AsyncGenerator[bytes, None]:
34
async for chunk in self.inner.aiter_content():
78
35
yield chunk
79
36
80
async def read(self) -> bytes:
81
return b"".join([chunk async for chunk in self.iter_content()])
82
83
84
class StreamRequest:
85
def __init__(self, session: AsyncSession, method: str, url: str, **kwargs: Union[bool, int, str]) -> None:
86
self.session: AsyncSession = session
87
self.loop: asyncio.AbstractEventLoop = session.loop if session.loop else asyncio.get_running_loop()
88
self.queue: Queue[bytes] = Queue()
89
self.method: str = method
90
self.url: str = url
91
self.options: dict = kwargs
92
self.handle: Optional[curl_cffi.AsyncCurl] = None
93
94
def _on_content(self, data: bytes) -> None:
95
if not self.enter.done():
96
self.enter.set_result(None)
97
self.queue.put_nowait(data)
98
99
def _on_done(self, task: Future) -> None:
100
if not self.enter.done():
101
self.enter.set_result(None)
102
self.queue.put_nowait(None)
103
104
self.loop.call_soon(self.release_curl)
105
106
async def fetch(self) -> StreamResponse:
107
if self.handle:
108
raise RuntimeError("Request already started")
109
self.curl: curl_cffi.AsyncCurl = await self.session.pop_curl()
110
self.enter: asyncio.Future = self.loop.create_future()
111
if is_newer_0_5_10:
112
request, _, header_buffer, _, _ = self.session._set_curl_options(
113
self.curl,
114
self.method,
115
self.url,
116
content_callback=self._on_content,
117
**self.options
118
)
119
else:
120
request, _, header_buffer = self.session._set_curl_options(
121
self.curl,
122
self.method,
123
self.url,
124
content_callback=self._on_content,
125
**self.options
126
)
127
if is_newer_0_5_9:
128
self.handle = self.session.acurl.add_handle(self.curl)
129
else:
130
await self.session.acurl.add_handle(self.curl, False)
131
self.handle = self.session.acurl._curl2future[self.curl]
132
self.handle.add_done_callback(self._on_done)
133
# Wait for headers
134
await self.enter
135
# Raise exceptions
136
if self.handle.done():
137
self.handle.result()
138
if is_newer_0_5_8:
139
response = self.session._parse_response(self.curl, _, header_buffer)
140
response.request = request
141
else:
142
response = self.session._parse_response(self.curl, request, _, header_buffer)
143
return StreamResponse(response, self.queue)
144
145
async def __aenter__(self) -> StreamResponse:
146
return await self.fetch()
147
148
async def __aexit__(self, *args) -> None:
149
self.release_curl()
150
151
def release_curl(self) -> None:
152
if is_newer_0_5_10:
153
self.session.release_curl(self.curl)
154
return
155
if not self.curl:
156
return
157
self.curl.clean_after_perform()
158
if is_newer_0_5_9:
159
self.session.acurl.remove_handle(self.curl)
160
elif not self.handle.done() and not self.handle.cancelled():
161
self.session.acurl.set_result(self.curl)
162
self.curl.reset()
163
self.session.push_curl(self.curl)
164
self.curl = None
165
166
167
37
class StreamSession(AsyncSession):
168
def request(
38
@asynccontextmanager
39
async def request(
169
40
self, method: str, url: str, **kwargs
170
) -> StreamRequest:
171
return StreamRequest(self, method, url, **kwargs)
41
) -> AsyncGenerator[StreamResponse]:
42
response = await super().request(method, url, stream=True, **kwargs)
43
try:
44
yield StreamResponse(response)
45
finally:
46
await response.aclose()
172
47
173
48
head = partialmethod(request, "HEAD")
174
49
get = partialmethod(request, "GET")
@@ -1,6 +1,6 @@
1
1
requests
2
2
pycryptodome
3
curl_cffi
3
curl_cffi>=0.5.10b4
4
4
aiohttp
5
5
certifi
6
6
browser_cookie3
@@ -21,4 +21,6 @@ asgiref
21
21
fastapi
22
22
uvicorn
23
23
flask
24
py-arkose-generator
24
py-arkose-generator
25
asyncstdlib
26
async-property