|
import urllib.parse |
|
from bs4 import BeautifulSoup |
|
from crawl4ai import AsyncWebCrawler |
|
import re |
|
import asyncio |
|
|
|
def process_url(url, sub_url): |
|
return urllib.parse.urljoin(url, sub_url) |
|
|
|
|
|
def clean_markdown(res): |
|
pattern = r'\[.*?\]\(.*?\)' |
|
try: |
|
|
|
result = re.sub(pattern, '', res) |
|
url_pattern = pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' |
|
result = re.sub(url_pattern, '', result) |
|
result = result.replace("* \n","") |
|
result = re.sub(r"\n\n+", "\n", result) |
|
return result |
|
except Exception: |
|
return res |
|
|
|
async def get_info(url, screentshot = True) -> str: |
|
async with AsyncWebCrawler() as crawler: |
|
if screentshot: |
|
result = await crawler.arun(url, screenshot=screentshot) |
|
|
|
return result.html, clean_markdown(result.markdown), result.screenshot |
|
else: |
|
result = await crawler.arun(url, screenshot=screentshot) |
|
return result.html, clean_markdown(result.markdown) |
|
|
|
if __name__ == "__main__": |
|
asyncio.run(get_info("https://2024.aclweb.org/")) |
