返回提交历史
Modified
g4f/integration/markitdown/__init__.py
+170
-1
XFEstudio/gpt4free
Fix using default provider
0d350985
代码差异
1 个文件
+170
-1
@@ -16,6 +16,158 @@ from ._audio_converter import AudioConverter
16
16
from ._image_converter import ImageConverter
17
17
from ._youtube_converter import YouTubeConverter
18
18
19
GITHUB_SKIP_LINES = [
20
"[Skip to content](#start-of-content)",
21
"## Navigation Menu",
22
"Appearance settings",
23
"* Platform",
24
"+ AI CODE CREATION",
25
"- [GitHub CopilotWrite better code with AI](https://github.com/features/copilot)",
26
"- [GitHub Copilot appDirect agents from issue to merge](https://github.com/features/ai/github-app)",
27
"- [MCP RegistryIntegrate external tools](https://github.com/mcp)",
28
"+ DEVELOPER WORKFLOWS",
29
"- [ActionsAutomate any workflow](https://github.com/features/actions)",
30
"- [CodespacesInstant dev environments](https://github.com/features/codespaces)",
31
"- [IssuesPlan and track work](https://github.com/features/issues)",
32
"- [Code ReviewManage code changes](https://github.com/features/code-review)",
33
"- [Code QualityEnforce quality at merge](https://github.com/features/code-quality)",
34
"+ APPLICATION SECURITY",
35
"- [GitHub Advanced SecurityFind and fix vulnerabilities](https://github.com/security/advanced-security)",
36
"- [Code securitySecure your code as you build](https://github.com/security/advanced-security/code-security)",
37
"- [Secret protectionStop leaks before they start](https://github.com/security/advanced-security/secret-protection)",
38
"+ EXPLORE",
39
"- [Why GitHub](https://github.com/why-github)",
40
"- [Documentation](https://docs.github.com)",
41
"- [Blog](https://github.blog)",
42
"- [Changelog](https://github.blog/changelog)",
43
"- [Marketplace](https://github.com/marketplace)",
44
"[View all features](https://github.com/features)",
45
"* Solutions",
46
"+ BY COMPANY SIZE",
47
"- [Enterprises](https://github.com/enterprise)",
48
"- [Small and medium teams](https://github.com/team)",
49
"- [Startups](https://github.com/enterprise/startups)",
50
"- [Nonprofits](https://github.com/solutions/industry/nonprofits)",
51
"+ BY USE CASE",
52
"- [App Modernization](https://github.com/solutions/use-case/app-modernization)",
53
"- [DevSecOps](https://github.com/solutions/use-case/devsecops)",
54
"- [DevOps](https://github.com/solutions/use-case/devops)",
55
"- [CI/CD](https://github.com/solutions/use-case/ci-cd)",
56
"- [View all use cases](https://github.com/solutions/use-case)",
57
"+ BY INDUSTRY",
58
"- [Healthcare](https://github.com/solutions/industry/healthcare)",
59
"- [Financial services](https://github.com/solutions/industry/financial-services)",
60
"- [Manufacturing](https://github.com/solutions/industry/manufacturing)",
61
"- [Government](https://github.com/solutions/industry/government)",
62
"- [View all industries](https://github.com/solutions/industry)",
63
"[View all solutions](https://github.com/solutions)",
64
"* Resources",
65
"+ EXPLORE BY TOPIC",
66
"- [AI](https://github.com/resources/articles?topic=ai)",
67
"- [Software Development](https://github.com/resources/articles?topic=software-development)",
68
"- [DevOps](https://github.com/resources/articles?topic=devops)",
69
"- [Security](https://github.com/resources/articles?topic=security)",
70
"- [View all topics](https://github.com/resources/articles)",
71
"+ EXPLORE BY TYPE",
72
"- [Customer stories](https://github.com/customer-stories)",
73
"- [Events & webinars](https://github.com/resources/events)",
74
"- [Ebooks & reports](https://github.com/resources/whitepapers)",
75
"- [Business insights](https://github.com/solutions/executive-insights)",
76
"- [GitHub Skills](https://skills.github.com)",
77
"+ SUPPORT & SERVICES",
78
"- [Documentation](https://docs.github.com)",
79
"- [Customer support](https://support.github.com)",
80
"- [Community forum](https://github.com/orgs/community/discussions)",
81
"- [Trust center](https://github.com/trust-center)",
82
"- [Partners](https://github.com/partners)",
83
"[View all resources](https://github.com/resources)",
84
"* Open Source",
85
"+ COMMUNITY",
86
"- [GitHub SponsorsFund open source developers](https://github.com/open-source/sponsors)",
87
"+ PROGRAMS",
88
"- [Security Lab](https://securitylab.github.com)",
89
"- [Maintainer Community](https://maintainers.github.com)",
90
"- [GitHub Stars](https://stars.github.com)",
91
"- [Archive Program](https://archiveprogram.github.com)",
92
"+ REPOSITORIES",
93
"- [Topics](https://github.com/topics)",
94
"- [Trending](https://github.com/trending)",
95
"- [Collections](https://github.com/collections)",
96
"* Enterprise",
97
"+ ENTERPRISE SOLUTIONS",
98
"- [Enterprise platformAI-powered developer platform](https://github.com/enterprise)",
99
"+ AVAILABLE ADD-ONS",
100
"- [GitHub Advanced SecurityEnterprise-grade security features](https://github.com/security/advanced-security)",
101
"- [Copilot for BusinessEnterprise-grade AI features](https://github.com/features/copilot/copilot-business)",
102
"- [Premium SupportEnterprise-grade 24/7 support](https://github.com/enterprise/premium-support)",
103
"* [Pricing](https://github.com/pricing)",
104
"Search`/`",
105
"Appearance settings",
106
"You signed in with another tab or window. Reload to refresh your session.",
107
"You signed out in another tab or window. Reload to refresh your session.",
108
"You switched accounts on another tab or window. Reload to refresh your session.",
109
"Dismiss alert",
110
"{{ message }}",
111
"Public",
112
"* ### Uh oh!"
113
"There was an error while loading. Please reload this page.",
114
"* [Notifications]",
115
"Choose two branches to see what’s changed or to start a new pull request.",
116
"If you need to, you can also compare across forks",
117
"or",
118
"[learn more about diff comparisons](https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons).",
119
"# Open a pull request",
120
"Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks",
121
"[Learn more about diff comparisons here](https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons).",
122
"Failed to load repositories. Confirm that selected base ref is valid, then try again.",
123
"Loading",
124
"### Uh oh!",
125
"Could not load branches",
126
"Could not load tags",
127
"Nothing to show",
128
"Loading",
129
"[{{ refName }}",
130
"...",
131
"Failed to load repositories. Confirm that selected head ref is valid, then try again.",
132
"### Uh oh!",
133
"There was an error while loading. Please reload this page.",
134
"### This comparison is taking too long to generate.",
135
"Unfortunately it looks like we can’t render this comparison for you right now. It might be too big, or there might be something weird with your repository.",
136
"You can try running this command locally to see the comparison on your machine:",
137
"There was an error while loading. Please reload this page.",
138
"### Footer navigation",
139
"* [Terms](https://docs.github.com/site-policy/github-terms/github-terms-of-service)",
140
"* [Privacy](https://docs.github.com/site-policy/privacy-policies/github-privacy-statement)",
141
"* [Security](https://github.com/security)",
142
"* [Status](https://www.githubstatus.com/)",
143
"* [Community](https://github.community/)",
144
"* [Docs](https://docs.github.com/)",
145
"* [Contact](https://support.github.com?tags=dotcom-footer)",
146
"* Manage cookies",
147
"* Do not share my personal information",
148
"You can’t perform that action at this time.",
149
"* ### Uh oh!",
150
"/",
151
".",
152
"Retry",
153
"* [FeedbackPreview](https://gh.io/issues-sidebar-feedback)",
154
"* Collapse sidebar",
155
"Search Issues",
156
"Search"
157
]
158
159
GITHUB_STARTS_WITH = [
160
"[Sign in]",
161
"[Sign up]",
162
"default](",
163
"* [Notifications]",
164
"* [Views]",
165
"* [Projects]",
166
"* [Milestones]",
167
"* [Labels]",
168
"* [Insights]",
169
"[Report repository]"
170
]
19
171
20
172
class MarkItDown(BaseMarkItDown):
21
173
"""(In preview) An extremely simple text-based document reader, suitable for LLM use.
@@ -240,4 +392,21 @@ class MarkItDown(BaseMarkItDown):
240
392
if url.startswith("https://github.com/"):
241
393
# Special case for GitHub URLs -- convert to raw content URL
242
394
url = self._convert_github_url_to_raw(url)
243
return super().convert_url(url, stream_info=stream_info, **kwargs)
395
result = super().convert_url(url, stream_info=stream_info, **kwargs)
396
if url.startswith("https://github.com/"):
397
result.text_content = re.sub(r"Additional navigation options[\s\S]+---", "", result.text_content)
398
result.text_content = re.sub(r"\* \[Security and quality[\s\S]+\* Collapse sidebar", "", result.text_content)
399
lines = []
400
empty = 0
401
for l in [l for l in result.text_content.splitlines() if l.strip() not in GITHUB_SKIP_LINES and not l.strip().startswith(tuple(GITHUB_STARTS_WITH))]:
402
striped = l.strip()
403
if striped == "":
404
empty += 1
405
else:
406
empty = 0
407
if empty > 2:
408
continue
409
else:
410
lines.append(l)
411
result.text_content = "\n".join(lines).strip()
412
return result