返回提交历史
Added
g4f/files.py
+26
-0
Modified
g4f/image/__init__.py
+1
-1
Modified
g4f/tools/files.py
+1
-28
XFEstudio/gpt4free
Move file functions
5b4f98c0
代码差异
3 个文件
+28
-29
@@ -0,0 +1,26 @@
1
from __future__ import annotations
2
3
import re
4
from urllib.parse import unquote
5
import os
6
7
from .cookies import get_cookies_dir
8
9
def secure_filename(filename: str) -> str:
10
if filename is None:
11
return None
12
# Keep letters, numbers, basic punctuation and all Unicode chars
13
filename = re.sub(
14
r'[^\w.,_+-]+',
15
'_',
16
unquote(filename).strip(),
17
flags=re.UNICODE
18
)
19
encoding = 'utf-8'
20
max_length = 100
21
encoded = filename.encode(encoding)[:max_length]
22
decoded = encoded.decode(encoding, 'ignore')
23
return decoded.strip(".,_+-")
24
25
def get_bucket_dir(*parts):
26
return os.path.join(get_cookies_dir(), "buckets", *[secure_filename(part) for part in parts if part])
@@ -17,7 +17,7 @@ except ImportError:
17
17
18
18
from ..typing import ImageType
19
19
from ..errors import MissingRequirementsError
20
from ..tools.files import get_bucket_dir
20
from ..files import get_bucket_dir
21
21
22
22
EXTENSIONS_MAP: dict[str, str] = {
23
23
# Image
@@ -7,7 +7,6 @@ from pathlib import Path
7
7
from typing import Iterator, Optional, AsyncIterator
8
8
from aiohttp import ClientSession, ClientError, ClientResponse, ClientTimeout
9
9
import urllib.parse
10
from urllib.parse import unquote
11
10
import time
12
11
import zipfile
13
12
import asyncio
@@ -76,7 +75,7 @@ except ImportError:
76
75
has_markitdown = False
77
76
78
77
from .web_search import scrape_text
79
from ..cookies import get_cookies_dir
78
from ..files import secure_filename, get_bucket_dir
80
79
from ..image import is_allowed_extension
81
80
from ..requests.aiohttp import get_connector
82
81
from ..providers.asyncio import to_sync_generator
@@ -88,22 +87,6 @@ PLAIN_CACHE = "plain.cache"
88
87
DOWNLOADS_FILE = "downloads.json"
89
88
FILE_LIST = "files.txt"
90
89
91
def secure_filename(filename: str) -> str:
92
if filename is None:
93
return None
94
# Keep letters, numbers, basic punctuation and all Unicode chars
95
filename = re.sub(
96
r'[^\w.,_+-]+',
97
'_',
98
unquote(filename).strip(),
99
flags=re.UNICODE
100
)
101
encoding = 'utf-8'
102
max_length = 100
103
encoded = filename.encode(encoding)[:max_length]
104
decoded = encoded.decode(encoding, 'ignore')
105
return decoded.strip(".,_+-")
106
107
90
def supports_filename(filename: str):
108
91
if filename.endswith(".pdf"):
109
92
if has_pypdf2:
@@ -139,16 +122,6 @@ def supports_filename(filename: str):
139
122
return True
140
123
return False
141
124
142
def get_bucket_dir(*parts):
143
return os.path.join(get_cookies_dir(), "buckets", *[secure_filename(part) for part in parts if part])
144
145
def get_buckets():
146
buckets_dir = os.path.join(get_cookies_dir(), "buckets")
147
try:
148
return [d for d in os.listdir(buckets_dir) if os.path.isdir(os.path.join(buckets_dir, d))]
149
except OSError:
150
return None
151
152
125
def spacy_refine_chunks(source_iterator):
153
126
if not has_spacy:
154
127
raise MissingRequirementsError(f'Install "spacy" requirements | pip install -U g4f[files]')