johnnycrypto289/masonclawv4/web-scraper

pip3 install requests beautifulsoup4 lxml
# For JavaScript-rendered pages:
pip3 install playwright && playwright install chromium

python3 -c "
import requests
from bs4 import BeautifulSoup
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
# Get all text
print(soup.get_text(separator='\n', strip=True)[:3000])
"

python3 -c "
import requests
from bs4 import BeautifulSoup
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
for a in soup.find_all('a', href=True):
    print(f\"{a.get_text(strip=True):50s} {a['href']}\")
" | head -30

python3 -c "
import requests, csv, sys
from bs4 import BeautifulSoup
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
table = soup.find('table')
if table:
    writer = csv.writer(sys.stdout)
    for row in table.find_all('tr'):
        cells = [c.get_text(strip=True) for c in row.find_all(['td','th'])]
        writer.writerow(cells)
else:
    print('No table found')
"

python3 -c "
import requests
from bs4 import BeautifulSoup
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
for el in soup.select('CSS_SELECTOR_HERE'):
    print(el.get_text(strip=True))
"

python3 -c "
import requests, json, os
from bs4 import BeautifulSoup
url = 'PRODUCT_URL'
r = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
# Adjust selector for the site
price_el = soup.select_one('.price, .product-price, [data-price]')
price = price_el.get_text(strip=True) if price_el else 'Not found'
print(f'Current price: {price}')
 
# Save to history
history_file = '/tmp/price_history.json'
history = json.load(open(history_file)) if os.path.exists(history_file) else []
from datetime import datetime
history.append({'date': datetime.now().isoformat(), 'price': price, 'url': url})
json.dump(history, open(history_file, 'w'), indent=2)
print(f'History: {len(history)} entries saved')
"

python3 -c "
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    page.goto('URL_HERE')
    page.wait_for_load_state('networkidle')
    content = page.content()
    
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(content, 'lxml')
    print(soup.get_text(separator='\n', strip=True)[:3000])
    browser.close()
"

python3 -c "
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
for img in soup.find_all('img', src=True):
    url = urljoin('URL_HERE', img['src'])
    alt = img.get('alt','')
    print(f\"{alt:40s} {url}\")
"

python3 -c "
import requests, json
from bs4 import BeautifulSoup
r = requests.get('URL_HERE', headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(r.text, 'lxml')
items = []
for el in soup.select('CSS_SELECTOR'):
    items.append({
        'title': el.select_one('.title').get_text(strip=True) if el.select_one('.title') else '',
        'link': el.select_one('a')['href'] if el.select_one('a') else '',
    })
print(json.dumps(items, indent=2))
"