gpt4free

公开

关注 0 Fork 0 Star 1

返回提交历史

XFEstudio/gpt4free

Fix errors on read buckets Update requirements for docx

796d6d6e

hlohaus <983577+hlohaus@users.noreply.github.com>

提交于 2025-02-19 17:10

父提交 292d5b6

代码差异

4 个文件 +19 -17

Modified g4f/tools/files.py +13 -13

@@ -126,7 +126,7 @@ def get_buckets():

126 126 buckets_dir = os.path.join(get_cookies_dir(), "buckets")

127 127 try:

128 128 return [d for d in os.listdir(buckets_dir) if os.path.isdir(os.path.join(buckets_dir, d))]

129 except OSError as e:

129 except OSError:

130 130 return None

131 131

132 132 def spacy_refine_chunks(source_iterator):

@@ -254,14 +254,14 @@ def read_bucket(bucket_dir: Path):

254 254 cache_file = bucket_dir / PLAIN_CACHE

255 255 spacy_file = bucket_dir / f"spacy_0001.cache"

256 256 if not spacy_file.exists():

257 yield cache_file.read_text()

257 yield cache_file.read_text(errors="replace")

258 258 for idx in range(1, 1000):

259 259 spacy_file = bucket_dir / f"spacy_{idx:04d}.cache"

260 260 plain_file = bucket_dir / f"plain_{idx:04d}.cache"

261 261 if spacy_file.exists():

262 yield spacy_file.read_text()

262 yield spacy_file.read_text(errors="replace")

263 263 elif plain_file.exists():

264 yield plain_file.read_text()

264 yield plain_file.read_text(errors="replace")

265 265 else:

266 266 break

267 267

@@ -277,7 +277,7 @@ def stream_read_parts_and_refine(bucket_dir: Path, delete_files: bool = False) -

277 277 cache_file = Path(bucket_dir) / f"spacy_{idx:04d}.cache"

278 278 if cache_file.exists():

279 279 with open(cache_file, "r") as f:

280 yield f.read()

280 yield f.read(errors="replace")

281 281 continue

282 282 if not part.exists():

283 283 break

@@ -485,8 +485,8 @@ def get_downloads_urls(bucket_dir: Path, delete_files: bool = False) -> Iterator

485 485 elif "urls" in item:

486 486 yield item

487 487

488 def read_and_download_urls(bucket_dir: Path, event_stream: bool = False) -> Iterator[str]:

489 urls = get_downloads_urls(bucket_dir)

488 def read_and_download_urls(bucket_dir: Path, delete_files: bool = False, event_stream: bool = False) -> Iterator[str]:

489 urls = get_downloads_urls(bucket_dir, delete_files)

490 490 if urls:

491 491 count = 0

492 492 with open(os.path.join(bucket_dir, FILE_LIST), 'a') as f:

@@ -497,8 +497,8 @@ def read_and_download_urls(bucket_dir: Path, event_stream: bool = False) -> Iter

497 497 count += 1

498 498 yield f'data: {json.dumps({"action": "download", "count": count})}\n\n'

499 499

500 async def async_read_and_download_urls(bucket_dir: Path, event_stream: bool = False) -> AsyncIterator[str]:

501 urls = get_downloads_urls(bucket_dir)

500

async def async_read_and_download_urls(bucket_dir: Path, delete_files: bool = False, event_stream: bool = False) -> AsyncIterator[str]:

501 urls = get_downloads_urls(bucket_dir, delete_files)

502 502 if urls:

503 503 count = 0

504 504 with open(os.path.join(bucket_dir, FILE_LIST), 'a') as f:

@@ -513,7 +513,7 @@ def stream_chunks(bucket_dir: Path, delete_files: bool = False, refine_chunks_wi

513 513 if refine_chunks_with_spacy:

514 514 for chunk in stream_read_parts_and_refine(bucket_dir, delete_files):

515 515 if event_stream:

516 size += len(chunk)

516 size += len(chunk.decode('utf-8'))

517 517 yield f'data: {json.dumps({"action": "refine", "size": size})}\n\n'

518 518 else:

519 519 yield chunk

@@ -522,7 +522,7 @@ def stream_chunks(bucket_dir: Path, delete_files: bool = False, refine_chunks_wi

522 522 streaming = cache_stream(streaming, bucket_dir)

523 523 for chunk in streaming:

524 524 if event_stream:

525 size += len(chunk)

525 size += len(chunk.decode('utf-8'))

526 526 yield f'data: {json.dumps({"action": "load", "size": size})}\n\n'

527 527 else:

528 528 yield chunk

@@ -541,7 +541,7 @@ def get_streaming(bucket_dir: str, delete_files = False, refine_chunks_with_spac

541 541 bucket_dir = Path(bucket_dir)

542 542 bucket_dir.mkdir(parents=True, exist_ok=True)

543 543 try:

544 yield from read_and_download_urls(bucket_dir, event_stream)

544 yield from read_and_download_urls(bucket_dir, delete_files, event_stream)

545 545 yield from stream_chunks(bucket_dir, delete_files, refine_chunks_with_spacy, event_stream)

546 546 except Exception as e:

547 547 if event_stream:

@@ -552,7 +552,7 @@ async def get_async_streaming(bucket_dir: str, delete_files = False, refine_chun

552 552 bucket_dir = Path(bucket_dir)

553 553 bucket_dir.mkdir(parents=True, exist_ok=True)

554 554 try:

555 async for chunk in async_read_and_download_urls(bucket_dir, event_stream):

555 async for chunk in async_read_and_download_urls(bucket_dir, delete_files, event_stream):

556 556 yield chunk

557 557 for chunk in stream_chunks(bucket_dir, delete_files, refine_chunks_with_spacy, event_stream):

558 558 yield chunk

Modified requirements-slim.txt +2 -1

@@ -15,4 +15,5 @@ beautifulsoup4

15 15 aiohttp_socks

16 16 cryptography

17 17 python-multipart

18 pypdf2

19 python-docx

Modified requirements.txt +1 -1

@@ -19,4 +19,4 @@ cryptography

19 19 nodriver

20 20 python-multipart

21 21 pypdf2

22 docx

22 python-docx

Modified setup.py +3 -2

@@ -40,7 +40,7 @@ EXTRA_REQUIRE = {

40 40 "plyer",

41 41 "setuptools",

42 42 "pypdf2", # files

43 "docx",

43 "python-docx",

44 44 "odfpy",

45 45 "ebooklib",

46 46 "openpyxl",

@@ -58,6 +58,7 @@ EXTRA_REQUIRE = {

58 58 "uvicorn", # api

59 59 "python-multipart",

60 60 "pypdf2", # files

61 "python-docx",

61 62 ],

62 63 "image": [

63 64 "pillow",

@@ -92,7 +93,7 @@ EXTRA_REQUIRE = {

92 93 "spacy",

93 94 "beautifulsoup4",

94 95 "pypdf2",

95 "docx",

96 "python-docx",

96 97 "odfpy",

97 98 "ebooklib",

98 99 "openpyxl",