返回提交历史
Modified
.github/workflows/build-packages.yml
+2
-1
Modified
g4f/integration/markitdown/__init__.py
+2
-0
Added
g4f/integration/markitdown/_youtube_converter.py
+245
-0
Modified
g4f/tools/files.py
+2
-2
XFEstudio/gpt4free
Add YouTubeConverter for enhanced document conversion and update import paths
f571237e
代码差异
4 个文件
+251
-3
@@ -430,6 +430,7 @@ jobs:
430
430
with:
431
431
tag_name: ${{ needs.prepare.outputs.version }}
432
432
name: Release ${{ needs.prepare.outputs.version }}
433
append_body: true
433
434
body: |
434
435
## g4f ${{ needs.prepare.outputs.version }}
435
436
@@ -446,7 +447,7 @@ jobs:
446
447
- macOS ARM64: `g4f-macos-${{ needs.prepare.outputs.version }}-arm64`
447
448
448
449
**System Packages:**
449
- WinGet: `winget install g4f` (after manifest approval)
450
- WinGet: `winget install gpt4free`
450
451
451
452
**Docker:**
452
453
- `docker pull hlohaus789/g4f:${{ needs.prepare.outputs.version }}`
@@ -14,6 +14,7 @@ from markitdown._exceptions import (
14
14
15
15
from ._audio_converter import AudioConverter
16
16
from ._image_converter import ImageConverter
17
from ._youtube_converter import YouTubeConverter
17
18
18
19
class MarkItDown(BaseMarkItDown):
19
20
"""(In preview) An extremely simple text-based document reader, suitable for LLM use.
@@ -26,6 +27,7 @@ class MarkItDown(BaseMarkItDown):
26
27
super().__init__(**kwargs)
27
28
self.register_converter(AudioConverter())
28
29
self.register_converter(ImageConverter())
30
self.register_converter(YouTubeConverter())
29
31
30
32
def _convert(
31
33
self, *, file_stream: BinaryIO, stream_info_guesses: List[StreamInfo], **kwargs
@@ -0,0 +1,245 @@
1
import json
2
import time
3
import re
4
import bs4
5
from typing import Any, BinaryIO, Dict, List, Union
6
from urllib.parse import parse_qs, urlparse, unquote
7
8
from markitdown._base_converter import DocumentConverter, DocumentConverterResult
9
from markitdown._stream_info import StreamInfo
10
11
# Optional YouTube transcription support
12
try:
13
# Suppress some warnings on library import
14
import warnings
15
16
with warnings.catch_warnings():
17
warnings.filterwarnings("ignore", category=SyntaxWarning)
18
# Patch submitted upstream to fix the SyntaxWarning
19
from youtube_transcript_api import YouTubeTranscriptApi
20
21
IS_YOUTUBE_TRANSCRIPT_CAPABLE = True
22
except ModuleNotFoundError:
23
IS_YOUTUBE_TRANSCRIPT_CAPABLE = False
24
25
26
ACCEPTED_MIME_TYPE_PREFIXES = [
27
"text/html",
28
"application/xhtml",
29
]
30
31
ACCEPTED_FILE_EXTENSIONS = [
32
".html",
33
".htm",
34
]
35
36
37
class YouTubeConverter(DocumentConverter):
38
"""Handle YouTube specially, focusing on the video title, description, and transcript."""
39
40
def accepts(
41
self,
42
file_stream: BinaryIO,
43
stream_info: StreamInfo,
44
**kwargs: Any, # Options to pass to the converter
45
) -> bool:
46
"""
47
Make sure we're dealing with HTML content *from* YouTube.
48
"""
49
url = stream_info.url or ""
50
mimetype = (stream_info.mimetype or "").lower()
51
extension = (stream_info.extension or "").lower()
52
53
url = unquote(url)
54
url = url.replace(r"\?", "?").replace(r"\=", "=")
55
56
if not url.startswith("https://www.youtube.com/watch?"):
57
# Not a YouTube URL
58
return False
59
60
if extension in ACCEPTED_FILE_EXTENSIONS:
61
return True
62
63
for prefix in ACCEPTED_MIME_TYPE_PREFIXES:
64
if mimetype.startswith(prefix):
65
return True
66
67
# Not HTML content
68
return False
69
70
def convert(
71
self,
72
file_stream: BinaryIO,
73
stream_info: StreamInfo,
74
**kwargs: Any, # Options to pass to the converter
75
) -> DocumentConverterResult:
76
# Parse the stream
77
encoding = "utf-8" if stream_info.charset is None else stream_info.charset
78
print(file_stream)
79
soup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)
80
81
# Read the meta tags
82
metadata: Dict[str, str] = {}
83
84
if soup.title and soup.title.string:
85
metadata["title"] = soup.title.string
86
87
for meta in soup(["meta"]):
88
if not isinstance(meta, bs4.Tag):
89
continue
90
91
for a in meta.attrs:
92
if a in ["itemprop", "property", "name"]:
93
key = str(meta.get(a, ""))
94
content = str(meta.get("content", ""))
95
if key and content: # Only add non-empty content
96
metadata[key] = content
97
break
98
99
print(f"Extracted metadata keys: {list(metadata.keys())}")
100
101
# Try reading the description
102
try:
103
for script in soup(["script"]):
104
if not isinstance(script, bs4.Tag):
105
continue
106
if not script.string: # Skip empty scripts
107
continue
108
content = script.string
109
if "ytInitialData" in content:
110
match = re.search(r"var ytInitialData = ({.*?});", content)
111
if match:
112
data = json.loads(match.group(1))
113
attrdesc = self._findKey(data, "attributedDescriptionBodyText")
114
if attrdesc and isinstance(attrdesc, dict):
115
metadata["description"] = str(attrdesc.get("content", ""))
116
break
117
except Exception as e:
118
print(f"Error extracting description: {e}")
119
pass
120
121
# Start preparing the page
122
webpage_text = "# YouTube\n"
123
124
title = self._get(metadata, ["title", "og:title", "name"]) # type: ignore
125
assert isinstance(title, str)
126
127
if title:
128
webpage_text += f"\n## {title}\n"
129
130
stats = ""
131
views = self._get(metadata, ["interactionCount"]) # type: ignore
132
if views:
133
stats += f"- **Views:** {views}\n"
134
135
keywords = self._get(metadata, ["keywords"]) # type: ignore
136
if keywords:
137
stats += f"- **Keywords:** {keywords}\n"
138
139
runtime = self._get(metadata, ["duration"]) # type: ignore
140
if runtime:
141
stats += f"- **Runtime:** {runtime}\n"
142
143
if len(stats) > 0:
144
webpage_text += f"\n### Video Metadata\n{stats}\n"
145
146
description = self._get(metadata, ["description", "og:description"]) # type: ignore
147
if description:
148
webpage_text += f"\n### Description\n{description}\n"
149
150
if IS_YOUTUBE_TRANSCRIPT_CAPABLE:
151
try:
152
ytt_api = YouTubeTranscriptApi()
153
transcript_text = ""
154
parsed_url = urlparse(stream_info.url) # type: ignore
155
params = parse_qs(parsed_url.query) # type: ignore
156
if "v" in params and params["v"][0]:
157
video_id = str(params["v"][0])
158
transcript_list = ytt_api.list(video_id)
159
languages = ["en"]
160
for transcript in transcript_list:
161
languages.append(transcript.language_code)
162
break
163
try:
164
youtube_transcript_languages = kwargs.get(
165
"youtube_transcript_languages", languages
166
)
167
# Retry the transcript fetching operation
168
transcript = self._retry_operation(
169
lambda: ytt_api.fetch(
170
video_id, languages=youtube_transcript_languages
171
),
172
retries=3, # Retry 3 times
173
delay=2, # 2 seconds delay between retries
174
)
175
176
if transcript:
177
transcript_text = " ".join(
178
[part.text for part in transcript]
179
) # type: ignore
180
except Exception as e:
181
# No transcript available
182
if len(languages) == 1:
183
print(f"Error fetching transcript: {e}")
184
else:
185
# Translate transcript into first kwarg
186
transcript = (
187
transcript_list.find_transcript(languages)
188
.translate(youtube_transcript_languages[0])
189
.fetch()
190
)
191
transcript_text = " ".join([part.text for part in transcript])
192
if transcript_text:
193
webpage_text += f"\n### Transcript\n{transcript_text}\n"
194
except Exception as e:
195
print(f"Error processing transcript: {e}")
196
pass
197
198
title = title if title else (soup.title.string if soup.title else "")
199
assert isinstance(title, str)
200
201
return DocumentConverterResult(
202
markdown=webpage_text,
203
title=title,
204
)
205
206
def _get(
207
self,
208
metadata: Dict[str, str],
209
keys: List[str],
210
default: Union[str, None] = None,
211
) -> Union[str, None]:
212
"""Get first non-empty value from metadata matching given keys."""
213
for k in keys:
214
if k in metadata:
215
return metadata[k]
216
return default
217
218
def _findKey(self, json: Any, key: str) -> Union[str, None]: # TODO: Fix json type
219
"""Recursively search for a key in nested dictionary/list structures."""
220
if isinstance(json, list):
221
for elm in json:
222
ret = self._findKey(elm, key)
223
if ret is not None:
224
return ret
225
elif isinstance(json, dict):
226
for k, v in json.items():
227
if k == key:
228
return json[k]
229
if result := self._findKey(v, key):
230
return result
231
return None
232
233
def _retry_operation(self, operation, retries=3, delay=2):
234
"""Retries the operation if it fails."""
235
attempt = 0
236
while attempt < retries:
237
try:
238
return operation() # Attempt the operation
239
except Exception as e:
240
print(f"Attempt {attempt + 1} failed: {e}")
241
if attempt < retries - 1:
242
time.sleep(delay) # Wait before retrying
243
attempt += 1
244
# If all attempts fail, raise the last exception
245
raise Exception(f"Operation failed after {retries} attempts.")
@@ -69,7 +69,7 @@ try:
69
69
except ImportError:
70
70
has_beautifulsoup4 = False
71
71
try:
72
from markitdown import MarkItDown
72
from g4f.integration.markitdown import MarkItDown
73
73
has_markitdown = True
74
74
except ImportError:
75
75
has_markitdown = False
@@ -434,7 +434,7 @@ async def download_urls(
434
434
text_content = None
435
435
if has_markitdown:
436
436
try:
437
text_content = md.convert(url).text_content
437
text_content = md.convert_url(url).text_content
438
438
if text_content:
439
439
filename = get_filename_from_url(url)
440
440
target = bucket_dir / filename