tian-ye1214/redlotus/web-scraping

import requests
from bs4 import BeautifulSoup
def scrape_static_page(url: str) -> dict:
    """抓取静态网页内容"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    
    response = requests.get(url, headers=headers, timeout=10)
    response.raise_for_status()
    response.encoding = response.apparent_encoding
    
    soup = BeautifulSoup(response.text, 'html.parser')
    
    return {
        'title': soup.title.string if soup.title else None,
        'text': soup.get_text(separator='\n', strip=True),
        'links': [a.get('href') for a in soup.find_all('a', href=True)],
        'images': [img.get('src') for img in soup.find_all('img', src=True)]
    }

def extract_elements(html: str, selector: str) -> list:
    """使用 CSS 选择器提取元素"""
    soup = BeautifulSoup(html, 'html.parser')
    elements = soup.select(selector)
    return [elem.get_text(strip=True) for elem in elements]