返回提交历史
Modified
g4f/local/__init__.py
+5
-68
XFEstudio/gpt4free
~ | support local llm inference
479ef89f
代码差异
1 个文件
+5
-68
@@ -1,75 +1,13 @@
1
import random, string, time, re
2
3
1
from ..typing import Union, Iterator, Messages
4
2
from ..stubs import ChatCompletion, ChatCompletionChunk
5
3
from .core.engine import LocalProvider
6
4
from .core.models import models
7
8
IterResponse = Iterator[Union[ChatCompletion, ChatCompletionChunk]]
9
10
def read_json(text: str) -> dict:
11
match = re.search(r"```(json|)\n(?P<code>[\S\s]+?)\n```", text)
12
if match:
13
return match.group("code")
14
return text
15
16
def iter_response(
17
response: Iterator[str],
18
stream: bool,
19
response_format: dict = None,
20
max_tokens: int = None,
21
stop: list = None
22
) -> IterResponse:
23
24
content = ""
25
finish_reason = None
26
completion_id = ''.join(random.choices(string.ascii_letters + string.digits, k=28))
27
for idx, chunk in enumerate(response):
28
content += str(chunk)
29
if max_tokens is not None and idx + 1 >= max_tokens:
30
finish_reason = "length"
31
first = -1
32
word = None
33
if stop is not None:
34
for word in list(stop):
35
first = content.find(word)
36
if first != -1:
37
content = content[:first]
38
break
39
if stream and first != -1:
40
first = chunk.find(word)
41
if first != -1:
42
chunk = chunk[:first]
43
else:
44
first = 0
45
if first != -1:
46
finish_reason = "stop"
47
if stream:
48
yield ChatCompletionChunk(chunk, None, completion_id, int(time.time()))
49
if finish_reason is not None:
50
break
51
finish_reason = "stop" if finish_reason is None else finish_reason
52
if stream:
53
yield ChatCompletionChunk(None, finish_reason, completion_id, int(time.time()))
54
else:
55
if response_format is not None and "type" in response_format:
56
if response_format["type"] == "json_object":
57
content = read_json(content)
58
yield ChatCompletion(content, finish_reason, completion_id, int(time.time()))
59
60
def filter_none(**kwargs):
61
for key in list(kwargs.keys()):
62
if kwargs[key] is None:
63
del kwargs[key]
64
return kwargs
5
from ..client import iter_response, filter_none, IterResponse
65
6
66
7
class LocalClient():
67
def __init__(
68
self,
69
**kwargs
70
) -> None:
8
def __init__(self, **kwargs) -> None:
71
9
self.chat: Chat = Chat(self)
72
10
73
11
@staticmethod
74
12
def list_models():
75
13
return list(models.keys())
@@ -100,10 +38,9 @@ class Completions():
100
38
)
101
39
response = iter_response(response, stream, response_format, max_tokens, stop)
102
40
return response if stream else next(response)
103
41
104
42
class Chat():
105
43
completions: Completions
106
44
107
45
def __init__(self, client: LocalClient):
108
self.completions = Completions(client)
109
46
self.completions = Completions(client)