返回提交历史
Modified
README.md
+1
-1
Modified
g4f/api/__init__.py
+195
-151
Added
g4f/api/_logging.py
+32
-0
Added
g4f/api/_tokenizer.py
+9
-0
Modified
g4f/api/run.py
+3
-2
Modified
g4f/models.py
+1
-1
Modified
requirements.txt
+4
-1
XFEstudio/gpt4free
~ | updated g4f.api
new api and requirements
8e7e694d
代码差异
7 个文件
+245
-156
@@ -369,7 +369,7 @@ python -m g4f.api
369
369
import openai
370
370
371
371
openai.api_key = "Empty if you don't use embeddings, otherwise your hugginface token"
372
openai.api_base = "http://localhost:1337"
372
openai.api_base = "http://localhost:1337/v1"
373
373
374
374
375
375
def main():
@@ -1,162 +1,206 @@
1
import g4f
2
import time
1
3
import json
2
4
import random
3
5
import string
4
import time
5
6
# import requests
7
from flask import Flask, request
8
from flask_cors import CORS
9
# from transformers import AutoTokenizer
10
11
from g4f import ChatCompletion
12
13
app = Flask(__name__)
14
CORS(app)
15
16
17
@app.route("/")
18
def index():
19
return "interference api, url: http://127.0.0.1:1337"
20
21
22
@app.route("/chat/completions", methods=["POST"])
23
def chat_completions():
24
model = request.get_json().get("model", "gpt-3.5-turbo")
25
stream = request.get_json().get("stream", False)
26
messages = request.get_json().get("messages")
27
28
response = ChatCompletion.create(model=model, stream=stream, messages=messages)
29
30
completion_id = "".join(random.choices(string.ascii_letters + string.digits, k=28))
31
completion_timestamp = int(time.time())
32
33
if not stream:
34
return {
35
"id": f"chatcmpl-{completion_id}",
36
"object": "chat.completion",
37
"created": completion_timestamp,
38
"model": model,
39
"choices": [
40
{
41
"index": 0,
42
"message": {
43
"role": "assistant",
44
"content": response,
45
},
46
"finish_reason": "stop",
47
}
48
],
49
"usage": {
50
"prompt_tokens": None,
51
"completion_tokens": None,
52
"total_tokens": None,
53
},
54
}
55
56
def streaming():
57
for chunk in response:
58
completion_data = {
59
"id": f"chatcmpl-{completion_id}",
60
"object": "chat.completion.chunk",
61
"created": completion_timestamp,
62
"model": model,
63
"choices": [
6
import logging
7
8
from typing import Union
9
from loguru import logger
10
from waitress import serve
11
from ._logging import hook_logging
12
from ._tokenizer import tokenize
13
from flask_cors import CORS
14
from werkzeug.serving import WSGIRequestHandler
15
from werkzeug.exceptions import default_exceptions
16
from werkzeug.middleware.proxy_fix import ProxyFix
17
18
from flask import (
19
Flask,
20
jsonify,
21
make_response,
22
request,
23
)
24
25
class Api:
26
__default_ip = '127.0.0.1'
27
__default_port = 1337
28
29
def __init__(self, engine: g4f, debug: bool = True, sentry: bool = False) -> None:
30
self.engine = engine
31
self.debug = debug
32
self.sentry = sentry
33
self.log_level = logging.DEBUG if debug else logging.WARN
34
35
hook_logging(level=self.log_level, format='[%(asctime)s] %(levelname)s in %(module)s: %(message)s')
36
self.logger = logging.getLogger('waitress')
37
38
self.app = Flask(__name__)
39
self.app.wsgi_app = ProxyFix(self.app.wsgi_app, x_port=1)
40
self.app.after_request(self.__after_request)
41
42
def run(self, bind_str, threads=8):
43
host, port = self.__parse_bind(bind_str)
44
45
CORS(self.app, resources={r'/v1/*': {'supports_credentials': True, 'expose_headers': [
46
'Content-Type',
47
'Authorization',
48
'X-Requested-With',
49
'Accept',
50
'Origin',
51
'Access-Control-Request-Method',
52
'Access-Control-Request-Headers',
53
'Content-Disposition'], 'max_age': 600}})
54
55
self.app.route('/v1/models', methods=['GET'])(self.models)
56
self.app.route('v1/models/<model_id>', methods=['GET'])(self.model_info)
57
58
self.app.route('/v1/chat/completions', methods=['POST'])(self.chat_completions)
59
self.app.route('/v1/completions', methods=['POST'])(self.completions)
60
61
for ex in default_exceptions:
62
self.app.register_error_handler(ex, self.__handle_error)
63
64
if not self.debug:
65
self.logger.warning('Serving on http://{}:{}'.format(host, port))
66
67
WSGIRequestHandler.protocol_version = 'HTTP/1.1'
68
serve(self.app, host=host, port=port, ident=None, threads=threads)
69
70
def __handle_error(self, e: Exception):
71
self.logger.error(e)
72
73
return make_response(jsonify({
74
'code': e.code,
75
'message': str(e.original_exception if self.debug and hasattr(e, 'original_exception') else e.name)}), 500)
76
77
@staticmethod
78
def __after_request(resp):
79
resp.headers['X-Server'] = 'g4f/%s' % g4f.version
80
81
return resp
82
83
def __parse_bind(self, bind_str):
84
sections = bind_str.split(':', 2)
85
if len(sections) < 2:
86
try:
87
port = int(sections[0])
88
return self.__default_ip, port
89
except ValueError:
90
return sections[0], self.__default_port
91
92
return sections[0], int(sections[1])
93
94
async def home(self):
95
return 'Hello world | https://127.0.0.1:1337/v1'
96
97
async def chat_completions(self):
98
model = request.json.get('model', 'gpt-3.5-turbo')
99
stream = request.json.get('stream', False)
100
messages = request.json.get('messages')
101
102
logger.info(f'model: {model}, stream: {stream}, request: {messages[-1]["content"]}')
103
104
response = self.engine.ChatCompletion.create(model=model,
105
stream=stream, messages=messages)
106
107
completion_id = ''.join(random.choices(string.ascii_letters + string.digits, k=28))
108
completion_timestamp = int(time.time())
109
110
if not stream:
111
prompt_tokens, _ = tokenize(''.join([message['content'] for message in messages]))
112
completion_tokens, _ = tokenize(response)
113
114
return {
115
'id': f'chatcmpl-{completion_id}',
116
'object': 'chat.completion',
117
'created': completion_timestamp,
118
'model': model,
119
'choices': [
64
120
{
65
"index": 0,
66
"delta": {
67
"content": chunk,
121
'index': 0,
122
'message': {
123
'role': 'assistant',
124
'content': response,
68
125
},
69
"finish_reason": None,
126
'finish_reason': 'stop',
70
127
}
71
128
],
129
'usage': {
130
'prompt_tokens': prompt_tokens,
131
'completion_tokens': completion_tokens,
132
'total_tokens': prompt_tokens + completion_tokens,
133
},
72
134
}
73
135
74
content = json.dumps(completion_data, separators=(",", ":"))
75
yield f"data: {content}\n\n"
76
time.sleep(0.1)
136
def streaming():
137
try:
138
for chunk in response:
139
completion_data = {
140
'id': f'chatcmpl-{completion_id}',
141
'object': 'chat.completion.chunk',
142
'created': completion_timestamp,
143
'model': model,
144
'choices': [
145
{
146
'index': 0,
147
'delta': {
148
'content': chunk,
149
},
150
'finish_reason': None,
151
}
152
],
153
}
77
154
78
end_completion_data = {
79
"id": f"chatcmpl-{completion_id}",
80
"object": "chat.completion.chunk",
81
"created": completion_timestamp,
82
"model": model,
83
"choices": [
84
{
85
"index": 0,
86
"delta": {},
87
"finish_reason": "stop",
155
content = json.dumps(completion_data, separators=(',', ':'))
156
yield f'data: {content}\n\n'
157
time.sleep(0.03)
158
159
end_completion_data = {
160
'id': f'chatcmpl-{completion_id}',
161
'object': 'chat.completion.chunk',
162
'created': completion_timestamp,
163
'model': model,
164
'choices': [
165
{
166
'index': 0,
167
'delta': {},
168
'finish_reason': 'stop',
169
}
170
],
88
171
}
89
],
90
}
91
content = json.dumps(end_completion_data, separators=(",", ":"))
92
yield f"data: {content}\n\n"
93
94
return app.response_class(streaming(), mimetype="text/event-stream")
95
96
97
# Get the embedding from huggingface
98
# def get_embedding(input_text, token):
99
# huggingface_token = token
100
# embedding_model = "sentence-transformers/all-mpnet-base-v2"
101
# max_token_length = 500
102
103
# # Load the tokenizer for the 'all-mpnet-base-v2' model
104
# tokenizer = AutoTokenizer.from_pretrained(embedding_model)
105
# # Tokenize the text and split the tokens into chunks of 500 tokens each
106
# tokens = tokenizer.tokenize(input_text)
107
# token_chunks = [
108
# tokens[i : i + max_token_length]
109
# for i in range(0, len(tokens), max_token_length)
110
# ]
111
112
# # Initialize an empty list
113
# embeddings = []
114
115
# # Create embeddings for each chunk
116
# for chunk in token_chunks:
117
# # Convert the chunk tokens back to text
118
# chunk_text = tokenizer.convert_tokens_to_string(chunk)
119
120
# # Use the Hugging Face API to get embeddings for the chunk
121
# api_url = f"https://api-inference.huggingface.co/pipeline/feature-extraction/{embedding_model}"
122
# headers = {"Authorization": f"Bearer {huggingface_token}"}
123
# chunk_text = chunk_text.replace("\n", " ")
124
125
# # Make a POST request to get the chunk's embedding
126
# response = requests.post(
127
# api_url,
128
# headers=headers,
129
# json={"inputs": chunk_text, "options": {"wait_for_model": True}},
130
# )
131
132
# # Parse the response and extract the embedding
133
# chunk_embedding = response.json()
134
# # Append the embedding to the list
135
# embeddings.append(chunk_embedding)
136
137
# # averaging all the embeddings
138
# # this isn't very effective
139
# # someone a better idea?
140
# num_embeddings = len(embeddings)
141
# average_embedding = [sum(x) / num_embeddings for x in zip(*embeddings)]
142
# embedding = average_embedding
143
# return embedding
144
145
146
# @app.route("/embeddings", methods=["POST"])
147
# def embeddings():
148
# input_text_list = request.get_json().get("input")
149
# input_text = " ".join(map(str, input_text_list))
150
# token = request.headers.get("Authorization").replace("Bearer ", "")
151
# embedding = get_embedding(input_text, token)
152
153
# return {
154
# "data": [{"embedding": embedding, "index": 0, "object": "embedding"}],
155
# "model": "text-embedding-ada-002",
156
# "object": "list",
157
# "usage": {"prompt_tokens": None, "total_tokens": None},
158
# }
159
160
161
def run_api():
162
app.run(host="0.0.0.0", port=1337)
172
173
content = json.dumps(end_completion_data, separators=(',', ':'))
174
yield f'data: {content}\n\n'
175
176
logger.success(f'model: {model}, stream: {stream}')
177
178
except GeneratorExit:
179
pass
180
181
return self.app.response_class(streaming(), mimetype='text/event-stream')
182
183
async def completions(self):
184
return 'not working yet', 500
185
186
async def model_info(self, model_name):
187
model_info = (g4f.ModelUtils.convert[model_name])
188
189
return jsonify({
190
'id' : model_name,
191
'object' : 'model',
192
'created' : 0,
193
'owned_by' : model_info.base_provider
194
})
195
196
async def models(self):
197
model_list = [{
198
'id' : model,
199
'object' : 'model',
200
'created' : 0,
201
'owned_by' : 'g4f'} for model in g4f.Model.__all__()]
202
203
return jsonify({
204
'object': 'list',
205
'data': model_list})
206
@@ -0,0 +1,32 @@
1
import sys,logging
2
3
from loguru import logger
4
5
def __exception_handle(e_type, e_value, e_traceback):
6
if issubclass(e_type, KeyboardInterrupt):
7
print('\nBye...')
8
sys.exit(0)
9
10
sys.__excepthook__(e_type, e_value, e_traceback)
11
12
class __InterceptHandler(logging.Handler):
13
def emit(self, record):
14
try:
15
level = logger.level(record.levelname).name
16
except ValueError:
17
level = record.levelno
18
19
frame, depth = logging.currentframe(), 2
20
while frame.f_code.co_filename == logging.__file__:
21
frame = frame.f_back
22
depth += 1
23
24
logger.opt(depth=depth, exception=record.exc_info).log(
25
level, record.getMessage()
26
)
27
28
def hook_except_handle():
29
sys.excepthook = __exception_handle
30
31
def hook_logging(**kwargs):
32
logging.basicConfig(handlers=[__InterceptHandler()], **kwargs)
@@ -0,0 +1,9 @@
1
import tiktoken
2
from typing import Union
3
4
def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
5
encoding = tiktoken.encoding_for_model(model)
6
encoded = encoding.encode(text)
7
num_tokens = len(encoded)
8
9
return num_tokens, encoded
@@ -1,4 +1,5 @@
1
from g4f.api import run_api
1
import g4f
2
import g4f.api
2
3
3
4
if __name__ == "__main__":
4
run_api()
5
g4f.api.Api(g4f).run('localhost:1337', 8)
@@ -71,7 +71,7 @@ gpt_35_turbo = Model(
71
71
base_provider = 'openai',
72
72
best_provider = RetryProvider([
73
73
Aichat, ChatgptDemo, AiAsk, ChatForAi, GPTalk,
74
GptGo, You, Vercel, GptForLove, ChatBase, Bing
74
GptGo, You, GptForLove, ChatBase
75
75
])
76
76
)
77
77
@@ -10,4 +10,7 @@ flask
10
10
flask-cors
11
11
typing-extensions
12
12
PyExecJS
13
duckduckgo-search
13
duckduckgo-search
14
nest_asyncio
15
waitress
16
werkzeug