返回提交历史
Modified
g4f/api/__init__.py
+6
-6
Modified
g4f/api/_tokenizer.py
+7
-7
XFEstudio/gpt4free
~ | improve compatibility with lower python versions
remove tiktoken
2345588d
代码差异
2 个文件
+13
-13
@@ -4,7 +4,7 @@ import logging
4
4
from fastapi import FastAPI, Response, Request
5
5
from fastapi.responses import StreamingResponse
6
6
from typing import List, Union, Any, Dict, AnyStr
7
from ._tokenizer import tokenize
7
#from ._tokenizer import tokenize
8
8
from .. import BaseProvider
9
9
10
10
import time
@@ -95,8 +95,8 @@ class Api:
95
95
completion_timestamp = int(time.time())
96
96
97
97
if not stream:
98
prompt_tokens, _ = tokenize(''.join([message['content'] for message in messages]))
99
completion_tokens, _ = tokenize(response)
98
#prompt_tokens, _ = tokenize(''.join([message['content'] for message in messages]))
99
#completion_tokens, _ = tokenize(response)
100
100
101
101
json_data = {
102
102
'id': f'chatcmpl-{completion_id}',
@@ -114,9 +114,9 @@ class Api:
114
114
}
115
115
],
116
116
'usage': {
117
'prompt_tokens': prompt_tokens,
118
'completion_tokens': completion_tokens,
119
'total_tokens': prompt_tokens + completion_tokens,
117
'prompt_tokens': 0, #prompt_tokens,
118
'completion_tokens': 0, #completion_tokens,
119
'total_tokens': 0, #prompt_tokens + completion_tokens,
120
120
},
121
121
}
122
122
@@ -1,9 +1,9 @@
1
import tiktoken
2
from typing import Union
1
# import tiktoken
2
# from typing import Union
3
3
4
def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
5
encoding = tiktoken.encoding_for_model(model)
6
encoded = encoding.encode(text)
7
num_tokens = len(encoded)
4
# def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
5
# encoding = tiktoken.encoding_for_model(model)
6
# encoded = encoding.encode(text)
7
# num_tokens = len(encoded)
8
8
9
return num_tokens, encoded
9
# return num_tokens, encoded