返回提交历史
Modified
README.md
+2
-1
Added
docs/file.md
+182
-0
Modified
g4f/gui/client/static/js/chat.v1.js
+12
-2
Modified
g4f/tools/files.py
+60
-43
Modified
g4f/tools/run_tools.py
+15
-3
Modified
g4f/tools/web_search.py
+14
-5
XFEstudio/gpt4free
Add File API Documentation for Python and JS Format Bucket Placeholder in GUI
0d59789e
代码差异
6 个文件
+285
-54
@@ -243,7 +243,8 @@ print(f"Generated image URL: {image_url}")
243
243
- **Requests API from G4F:** [/docs/requests](docs/requests.md)
244
244
- **Client API from G4F:** [/docs/client](docs/client.md)
245
245
- **AsyncClient API from G4F:** [/docs/async_client](docs/async_client.md)
246
246
- **File API from G4F:** [/docs/file](docs/file.md)
247
247
248
- **Legacy:**
248
249
- **Legacy API with python modules:** [/docs/legacy](docs/legacy.md)
249
250
@@ -0,0 +1,182 @@
1
## G4F - File API Documentation with Web Download and Enhanced File Support
2
3
This document details the enhanced G4F File API, allowing users to upload files, download files from web URLs, and process a wider range of file types for integration with language models.
4
5
**Key Improvements:**
6
7
* **Web URL Downloads:** Upload a `downloads.json` file to your bucket containing a list of URLs. The API will download and process these files. Example: `[{"url": "https://example.com/document.pdf"}]`
8
9
* **Expanded File Support:** Added support for additional plain text file extensions: `.txt`, `.xml`, `.json`, `.js`, `.har`, `.sh`, `.py`, `.php`, `.css`, `.yaml`, `.sql`, `.log`, `.csv`, `.twig`, `.md`. Binary file support remains for `.pdf`, `.html`, `.docx`, `.odt`, `.epub`, `.xlsx`, and `.zip`.
10
11
* **Server-Sent Events (SSE):** SSE are now used to provide asynchronous updates on file download and processing progress. This improves the user experience, particularly for large files and multiple downloads.
12
13
14
**API Endpoints:**
15
16
* **Upload:** `/v1/files/{bucket_id}` (POST)
17
18
* **Method:** POST
19
* **Path Parameters:** `bucket_id` (Generated by your own. For example a UUID)
20
* **Body:** Multipart/form-data with files OR a `downloads.json` file containing URLs.
21
* **Response:** JSON object with `bucket_id`, `url`, and a list of uploaded/downloaded filenames.
22
23
24
* **Retrieve:** `/v1/files/{bucket_id}` (GET)
25
26
* **Method:** GET
27
* **Path Parameters:** `bucket_id`
28
* **Query Parameters:**
29
* `delete_files`: (Optional, boolean, default `true`) Delete files after retrieval.
30
* `refine_chunks_with_spacy`: (Optional, boolean, default `false`) Apply spaCy-based refinement.
31
* **Response:** Streaming response with extracted text, separated by ``` markers. SSE updates are sent if the `Accept` header includes `text/event-stream`.
32
33
34
**Example Usage (Python):**
35
36
```python
37
import requests
38
import uuid
39
import json
40
41
def upload_and_process(files_or_urls, bucket_id=None):
42
if bucket_id is None:
43
bucket_id = str(uuid.uuid4())
44
45
if isinstance(files_or_urls, list): #URLs
46
files = {'files': ('downloads.json', json.dumps(files_or_urls), 'application/json')}
47
elif isinstance(files_or_urls, dict): #Files
48
files = files_or_urls
49
else:
50
raise ValueError("files_or_urls must be a list of URLs or a dictionary of files")
51
52
upload_response = requests.post(f'http://localhost:1337/v1/files/{bucket_id}', files=files)
53
54
if upload_response.status_code == 200:
55
upload_data = upload_response.json()
56
print(f"Upload successful. Bucket ID: {upload_data['bucket_id']}")
57
else:
58
print(f"Upload failed: {upload_response.status_code} - {upload_response.text}")
59
60
response = requests.get(f'http://localhost:1337/v1/files/{bucket_id}', stream=True, headers={'Accept': 'text/event-stream'})
61
for line in response.iter_lines():
62
if line:
63
line = line.decode('utf-8')
64
if line.startswith('data:'):
65
try:
66
data = json.loads(line[5:]) #remove data: prefix
67
if "action" in data:
68
print(f"SSE Event: {data}")
69
elif "error" in data:
70
print(f"Error: {data['error']['message']}")
71
else:
72
print(f"File data received: {data}") #Assuming it's file content
73
except json.JSONDecodeError as e:
74
print(f"Error decoding JSON: {e}")
75
else:
76
print(f"Unhandled SSE event: {line}")
77
response.close()
78
79
# Example with URLs
80
urls = [{"url": "https://github.com/xtekky/gpt4free/issues"}]
81
bucket_id = upload_and_process(urls)
82
83
#Example with files
84
files = {'files': open('document.pdf', 'rb'), 'files': open('data.json', 'rb')}
85
bucket_id = upload_and_process(files)
86
```
87
88
89
**Example Usage (JavaScript):**
90
91
```javascript
92
function uuid() {
93
return ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
94
(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
95
);
96
}
97
98
async function upload_files_or_urls(data) {
99
let bucket_id = uuid(); // Use a random generated key for your bucket
100
101
let formData = new FormData();
102
if (typeof data === "object" && data.constructor === Array) { //URLs
103
const blob = new Blob([JSON.stringify(data)], { type: 'application/json' });
104
const file = new File([blob], 'downloads.json', { type: 'application/json' }); // Create File object
105
formData.append('files', file); // Append as a file
106
} else { //Files
107
Array.from(data).forEach(file => {
108
formData.append('files', file);
109
});
110
}
111
112
await fetch("/v1/files/" + bucket_id, {
113
method: 'POST',
114
body: formData
115
});
116
117
function connectToSSE(url) {
118
const eventSource = new EventSource(url);
119
eventSource.onmessage = (event) => {
120
const data = JSON.parse(event.data);
121
if (data.error) {
122
console.error("Error:", data.error.message);
123
} else if (data.action === "done") {
124
console.log("Files loaded successfully. Bucket ID:", bucket_id);
125
// Use bucket_id in your LLM prompt.
126
const prompt = `Use files from bucket. ${JSON.stringify({"bucket_id": bucket_id})} to answer this: ...your question...`;
127
// ... Send prompt to your language model ...
128
} else {
129
console.log("SSE Event:", data); // Update UI with progress as needed
130
}
131
};
132
eventSource.onerror = (event) => {
133
console.error("SSE Error:", event);
134
eventSource.close();
135
};
136
}
137
138
connectToSSE(`/v1/files/${bucket_id}`); //Retrieve and refine
139
}
140
141
// Example with URLs
142
const urls = [{"url": "https://github.com/xtekky/gpt4free/issues"}];
143
upload_files_or_urls(urls)
144
145
// Example with files (using a file input element)
146
const fileInput = document.getElementById('fileInput');
147
fileInput.addEventListener('change', () => {
148
upload_files_or_urls(fileInput.files);
149
});
150
```
151
152
**Integrating with `ChatCompletion`:**
153
154
To incorporate file uploads into your client applications, include the `tool_calls` parameter in your chat completion requests, using the `bucket_tool` function. The `bucket_id` is passed as a JSON object within your prompt.
155
156
157
```json
158
{
159
"messages": [
160
{
161
"role": "user",
162
"content": "Answer this question using the files in the specified bucket: ...your question...\n{\"bucket_id\": \"your_actual_bucket_id\"}"
163
}
164
],
165
"tool_calls": [
166
{
167
"function": {
168
"name": "bucket_tool"
169
},
170
"type": "function"
171
}
172
]
173
}
174
```
175
176
**Important Considerations:**
177
178
* **Error Handling:** Implement robust error handling in both Python and JavaScript to gracefully manage potential issues during file uploads, downloads, and API interactions.
179
* **Dependencies:** Ensure all required packages are installed (`pip install -U g4f[files]` for Python).
180
181
---
182
[Return to Home](/)
@@ -59,6 +59,10 @@ if (window.markdownit) {
59
59
return markdown.render(content
60
60
.replaceAll(/<!-- generated images start -->|<!-- generated images end -->/gm, "")
61
61
.replaceAll(/<img data-prompt="[^>]+">/gm, "")
62
.replaceAll(/{"bucket_id":"([^"]+)"}/gm, (match, p1) => {
63
size = appStorage.getItem(`bucket:${p1}`);
64
return `**Bucket:** [[${p1}]](/backend-api/v2/files/${p1})${size ? ` (${formatFileSize(size)})` : ""}`;
65
})
62
66
)
63
67
.replaceAll("<a href=", '<a target="_blank" href=')
64
68
.replaceAll('<code>', '<code class="language-plaintext">')
@@ -1802,16 +1806,18 @@ function formatFileSize(bytes) {
1802
1806
async function upload_files(fileInput) {
1803
1807
const paperclip = document.querySelector(".user-input .fa-paperclip");
1804
1808
const bucket_id = uuid();
1809
delete fileInput.dataset.text;
1810
paperclip.classList.add("blink");
1805
1811
1806
1812
const formData = new FormData();
1807
1813
Array.from(fileInput.files).forEach(file => {
1808
1814
formData.append('files[]', file);
1809
1815
});
1810
paperclip.classList.add("blink");
1811
1816
await fetch("/backend-api/v2/files/" + bucket_id, {
1812
1817
method: 'POST',
1813
1818
body: formData
1814
1819
});
1820
1815
1821
let do_refine = document.getElementById("refine").checked;
1816
1822
function connectToSSE(url) {
1817
1823
const eventSource = new EventSource(url);
@@ -1819,21 +1825,25 @@ async function upload_files(fileInput) {
1819
1825
const data = JSON.parse(event.data);
1820
1826
if (data.error) {
1821
1827
inputCount.innerText = `Error: ${data.error.message}`;
1828
paperclip.classList.remove("blink");
1829
fileInput.value = "";
1822
1830
} else if (data.action == "load") {
1823
1831
inputCount.innerText = `Read data: ${formatFileSize(data.size)}`;
1824
1832
} else if (data.action == "refine") {
1825
1833
inputCount.innerText = `Refine data: ${formatFileSize(data.size)}`;
1834
} else if (data.action == "download") {
1835
inputCount.innerText = `Download: ${data.count} files`;
1826
1836
} else if (data.action == "done") {
1827
1837
if (do_refine) {
1828
1838
do_refine = false;
1829
1839
connectToSSE(`/backend-api/v2/files/${bucket_id}?refine_chunks_with_spacy=true`);
1830
1840
return;
1831
1841
}
1842
appStorage.setItem(`bucket:${bucket_id}`, data.size);
1832
1843
inputCount.innerText = "Files are loaded successfully";
1833
1844
messageInput.value += (messageInput.value ? "\n" : "") + JSON.stringify({bucket_id: bucket_id}) + "\n";
1834
1845
paperclip.classList.remove("blink");
1835
1846
fileInput.value = "";
1836
delete fileInput.dataset.text;
1837
1847
}
1838
1848
};
1839
1849
eventSource.onerror = (event) => {
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
import os
4
4
import json
5
5
from pathlib import Path
6
from typing import Iterator, Optional
6
from typing import Iterator, Optional, AsyncIterator
7
7
from aiohttp import ClientSession, ClientError, ClientResponse, ClientTimeout
8
8
import urllib.parse
9
9
import time
@@ -74,6 +74,7 @@ except ImportError:
74
74
from .web_search import scrape_text
75
75
from ..cookies import get_cookies_dir
76
76
from ..requests.aiohttp import get_connector
77
from ..providers.asyncio import to_sync_generator
77
78
from ..errors import MissingRequirementsError
78
79
from .. import debug
79
80
@@ -148,10 +149,12 @@ def spacy_refine_chunks(source_iterator):
148
149
149
150
def get_filenames(bucket_dir: Path):
150
151
files = bucket_dir / FILE_LIST
151
with files.open('r') as f:
152
return [filename.strip() for filename in f.readlines()]
152
if files.exists():
153
with files.open('r') as f:
154
return [filename.strip() for filename in f.readlines()]
155
return []
153
156
154
def stream_read_files(bucket_dir: Path, filenames: list) -> Iterator[str]:
157
def stream_read_files(bucket_dir: Path, filenames: list, delete_files: bool = False) -> Iterator[str]:
155
158
for filename in filenames:
156
159
file_path: Path = bucket_dir / filename
157
160
if not file_path.exists() and 0 > file_path.lstat().st_size:
@@ -161,17 +164,18 @@ def stream_read_files(bucket_dir: Path, filenames: list) -> Iterator[str]:
161
164
with zipfile.ZipFile(file_path, 'r') as zip_ref:
162
165
zip_ref.extractall(bucket_dir)
163
166
try:
164
yield from stream_read_files(bucket_dir, [f for f in zip_ref.namelist() if supports_filename(f)])
167
yield from stream_read_files(bucket_dir, [f for f in zip_ref.namelist() if supports_filename(f)], delete_files)
165
168
except zipfile.BadZipFile:
166
169
pass
167
170
finally:
168
for unlink in zip_ref.namelist()[::-1]:
169
filepath = os.path.join(bucket_dir, unlink)
170
if os.path.exists(filepath):
171
if os.path.isdir(filepath):
172
os.rmdir(filepath)
173
else:
174
os.unlink(filepath)
171
if delete_files:
172
for unlink in zip_ref.namelist()[::-1]:
173
filepath = os.path.join(bucket_dir, unlink)
174
if os.path.exists(filepath):
175
if os.path.isdir(filepath):
176
os.rmdir(filepath)
177
else:
178
os.unlink(filepath)
175
179
continue
176
180
yield f"```{filename}\n"
177
181
if has_pypdf2 and filename.endswith(".pdf"):
@@ -320,7 +324,7 @@ def split_file_by_size_and_newline(input_filename, output_dir, chunk_size_bytes=
320
324
with open(output_filename, 'w', encoding='utf-8') as outfile:
321
325
outfile.write(current_chunk)
322
326
323
async def get_filename(response: ClientResponse):
327
async def get_filename(response: ClientResponse) -> str:
324
328
"""
325
329
Attempts to extract a filename from an aiohttp response. Prioritizes Content-Disposition, then URL.
326
330
@@ -347,8 +351,9 @@ async def get_filename(response: ClientResponse):
347
351
if extension:
348
352
parsed_url = urllib.parse.urlparse(url)
349
353
sha256_hash = hashlib.sha256(url.encode()).digest()
350
base64_encoded = base64.b32encode(sha256_hash).decode().lower()
351
return f"{parsed_url.netloc} {parsed_url.path[1:].replace('/', '_')} {base64_encoded[:6]}{extension}"
354
base32_encoded = base64.b32encode(sha256_hash).decode()
355
url_hash = base32_encoded[:24].lower()
356
return f"{parsed_url.netloc} {parsed_url.path[1:].replace('/', '_')} {url_hash}{extension}"
352
357
353
358
return None
354
359
@@ -404,21 +409,22 @@ def read_links(html: str, base: str) -> set[str]:
404
409
for link in soup.select("a"):
405
410
if "rel" not in link.attrs or "nofollow" not in link.attrs["rel"]:
406
411
url = link.attrs.get("href")
407
if url and url.startswith("https://"):
412
if url and url.startswith("https://") or url.startswith("/"):
408
413
urls.append(url.split("#")[0])
409
414
return set([urllib.parse.urljoin(base, link) for link in urls])
410
415
411
416
async def download_urls(
412
417
bucket_dir: Path,
413
418
urls: list[str],
414
max_depth: int = 2,
415
loaded_urls: set[str] = set(),
419
max_depth: int = 1,
420
loading_urls: set[str] = set(),
416
421
lock: asyncio.Lock = None,
417
422
delay: int = 3,
423
new_urls: list[str] = list(),
418
424
group_size: int = 5,
419
425
timeout: int = 10,
420
426
proxy: Optional[str] = None
421
) -> list[str]:
427
) -> AsyncIterator[str]:
422
428
if lock is None:
423
429
lock = asyncio.Lock()
424
430
async with ClientSession(
@@ -433,30 +439,37 @@ async def download_urls(
433
439
if not filename:
434
440
print(f"Failed to get filename for {url}")
435
441
return None
436
newfiles = [filename]
442
if not supports_filename(filename) or filename == DOWNLOADS_FILE:
443
return None
437
444
if filename.endswith(".html") and max_depth > 0:
438
new_urls = read_links(await response.text(), str(response.url))
439
async with lock:
440
new_urls = [new_url for new_url in new_urls if new_url not in loaded_urls]
441
[loaded_urls.add(url) for url in new_urls]
442
if new_urls:
443
for i in range(0, len(new_urls), group_size):
444
newfiles += await download_urls(bucket_dir, new_urls[i:i + group_size], max_depth - 1, loaded_urls, lock, delay + 1)
445
await asyncio.sleep(delay)
446
if supports_filename(filename) and filename != DOWNLOADS_FILE:
447
target = bucket_dir / filename
448
with target.open("wb") as f:
449
async for chunk in response.content.iter_chunked(4096):
450
f.write(chunk)
451
return newfiles
445
add_urls = read_links(await response.text(), str(response.url))
446
if add_urls:
447
async with lock:
448
add_urls = [add_url for add_url in add_urls if add_url not in loading_urls]
449
[loading_urls.add(add_url) for add_url in add_urls]
450
[new_urls.append(add_url) for add_url in add_urls if add_url not in new_urls]
451
target = bucket_dir / filename
452
with target.open("wb") as f:
453
async for chunk in response.content.iter_chunked(4096):
454
if b'<link rel="canonical"' not in chunk:
455
f.write(chunk.replace(b'</head>', f'<link rel="canonical" href="{response.url}">\n</head>'.encode()))
456
return filename
452
457
except (ClientError, asyncio.TimeoutError) as e:
453
458
debug.log(f"Download failed: {e.__class__.__name__}: {e}")
454
459
return None
455
files = set()
456
for results in await asyncio.gather(*[download_url(url) for url in urls]):
457
if results:
458
[files.add(url) for url in results]
459
return files
460
for filename in await asyncio.gather(*[download_url(url) for url in urls]):
461
if filename:
462
yield filename
463
else:
464
await asyncio.sleep(delay)
465
while new_urls:
466
next_urls = list()
467
for i in range(0, len(new_urls), group_size):
468
chunked_urls = new_urls[i:i + group_size]
469
async for filename in download_urls(bucket_dir, chunked_urls, max_depth - 1, loading_urls, lock, delay + 1, next_urls):
470
yield filename
471
await asyncio.sleep(delay)
472
new_urls = next_urls
460
473
461
474
def get_streaming(bucket_dir: str, delete_files = False, refine_chunks_with_spacy = False, event_stream: bool = False) -> Iterator[str]:
462
475
bucket_dir = Path(bucket_dir)
@@ -473,9 +486,13 @@ def get_streaming(bucket_dir: str, delete_files = False, refine_chunks_with_spac
473
486
if "url" in item:
474
487
urls.append(item["url"])
475
488
if urls:
476
filenames = asyncio.run(download_urls(bucket_dir, urls))
489
count = 0
477
490
with open(os.path.join(bucket_dir, FILE_LIST), 'w') as f:
478
[f.write(f"{filename}\n") for filename in filenames if filename]
491
for filename in to_sync_generator(download_urls(bucket_dir, urls)):
492
f.write(f"{filename}\n")
493
if event_stream:
494
count += 1
495
yield f'data: {json.dumps({"action": "download", "count": count})}\n\n'
479
496
480
497
if refine_chunks_with_spacy:
481
498
size = 0
@@ -486,7 +503,7 @@ def get_streaming(bucket_dir: str, delete_files = False, refine_chunks_with_spac
486
503
else:
487
504
yield chunk
488
505
else:
489
streaming = stream_read_files(bucket_dir, get_filenames(bucket_dir))
506
streaming = stream_read_files(bucket_dir, get_filenames(bucket_dir), delete_files)
490
507
streaming = cache_stream(streaming, bucket_dir)
491
508
size = 0
492
509
for chunk in streaming:
@@ -504,7 +521,7 @@ def get_streaming(bucket_dir: str, delete_files = False, refine_chunks_with_spac
504
521
if event_stream:
505
522
yield f'data: {json.dumps({"action": "delete_files"})}\n\n'
506
523
if event_stream:
507
yield f'data: {json.dumps({"action": "done"})}\n\n'
524
yield f'data: {json.dumps({"action": "done", "size": size})}\n\n'
508
525
except Exception as e:
509
526
if event_stream:
510
527
yield f'data: {json.dumps({"error": {"message": str(e)}})}\n\n'
@@ -12,6 +12,10 @@ from .web_search import do_search, get_search_message
12
12
from .files import read_bucket, get_bucket_dir
13
13
from .. import debug
14
14
15
BUCKET_INSTRUCTIONS = """
16
Instruction: Make sure to add the sources of cites using [[domain]](Url) notation after the reference. Example: [[a-z0-9.]](http://example.com)
17
"""
18
15
19
def validate_arguments(data: dict) -> dict:
16
20
if "arguments" in data:
17
21
if isinstance(data["arguments"], str):
@@ -36,7 +40,7 @@ async def async_iter_run_tools(async_iter_callback, model, messages, tool_calls:
36
40
)
37
41
elif tool.get("function", {}).get("name") == "continue":
38
42
last_line = messages[-1]["content"].strip().splitlines()[-1]
39
content = f"Continue writing the story after this line start with a plus sign if you begin a new word.\n{last_line}"
43
content = f"Continue after this line.\n{last_line}"
40
44
messages.append({"role": "user", "content": content})
41
45
response = async_iter_callback(model=model, messages=messages, **kwargs)
42
46
if not hasattr(response, "__aiter__"):
@@ -73,7 +77,7 @@ def iter_run_tools(
73
77
elif tool.get("function", {}).get("name") == "continue_tool":
74
78
if provider not in ("OpenaiAccount", "HuggingFace"):
75
79
last_line = messages[-1]["content"].strip().splitlines()[-1]
76
content = f"continue after this line:\n{last_line}"
80
content = f"Continue after this line:\n{last_line}"
77
81
messages.append({"role": "user", "content": content})
78
82
else:
79
83
# Enable provider native continue
@@ -82,6 +86,14 @@ def iter_run_tools(
82
86
elif tool.get("function", {}).get("name") == "bucket_tool":
83
87
def on_bucket(match):
84
88
return "".join(read_bucket(get_bucket_dir(match.group(1))))
85
messages[-1]["content"] = re.sub(r'{"bucket_id":"([^"]*)"}', on_bucket, messages[-1]["content"])
89
has_bucket = False
90
for message in messages:
91
if "content" in message and isinstance(message["content"], str):
92
new_message_content = re.sub(r'{"bucket_id":"([^"]*)"}', on_bucket, message["content"])
93
if new_message_content != message["content"]:
94
has_bucket = True
95
message["content"] = new_message_content
96
if has_bucket and isinstance(messages[-1]["content"], str):
97
messages[-1]["content"] += BUCKET_INSTRUCTIONS
86
98
print(messages[-1])
87
99
return iter_callback(model=model, messages=messages, provider=provider, **kwargs)
@@ -4,7 +4,10 @@ from aiohttp import ClientSession, ClientTimeout, ClientError
4
4
import json
5
5
import hashlib
6
6
from pathlib import Path
7
from collections import Counter
7
from urllib.parse import urlparse
8
import datetime
9
import asyncio
10
8
11
try:
9
12
from duckduckgo_search import DDGS
10
13
from duckduckgo_search.exceptions import DuckDuckGoSearchException
@@ -17,13 +20,12 @@ try:
17
20
has_spacy = True
18
21
except:
19
22
has_spacy = False
23
20
24
from typing import Iterator
21
25
from ..cookies import get_cookies_dir
22
26
from ..errors import MissingRequirementsError
23
27
from .. import debug
24
28
25
import asyncio
26
27
29
DEFAULT_INSTRUCTIONS = """
28
30
Using the provided web search results, to write a comprehensive reply to the user request.
29
31
Make sure to add the sources of cites using [[Number]](Url) notation after the reference. Example: [[0]](http://google.com)
@@ -64,7 +66,8 @@ class SearchResultEntry():
64
66
self.text = text
65
67
66
68
def scrape_text(html: str, max_words: int = None) -> Iterator[str]:
67
soup = BeautifulSoup(html, "html.parser")
69
source = BeautifulSoup(html, "html.parser")
70
soup = source
68
71
for selector in [
69
72
"main",
70
73
".main-content-wrapper",
@@ -96,12 +99,18 @@ def scrape_text(html: str, max_words: int = None) -> Iterator[str]:
96
99
break
97
100
yield " ".join(words) + "\n"
98
101
102
canonical_link = source.find("link", rel="canonical")
103
if canonical_link and "href" in canonical_link.attrs:
104
link = canonical_link["href"]
105
domain = urlparse(link).netloc
106
yield f"\nSource: [{domain}]({link})"
107
99
108
async def fetch_and_scrape(session: ClientSession, url: str, max_words: int = None) -> str:
100
109
try:
101
110
bucket_dir: Path = Path(get_cookies_dir()) / ".scrape_cache" / "fetch_and_scrape"
102
111
bucket_dir.mkdir(parents=True, exist_ok=True)
103
112
md5_hash = hashlib.md5(url.encode()).hexdigest()
104
cache_file = bucket_dir / f"{url.split('/')[3]}.{md5_hash}.txt"
113
cache_file = bucket_dir / f"{url.split('/')[3]}.{datetime.date.today()}.{md5_hash}.txt"
105
114
if cache_file.exists():
106
115
return cache_file.read_text()
107
116
async with session.get(url) as response: