diff --git a/diff-web.py b/diff-web.py index 22cdf07..b7b280b 100644 --- a/diff-web.py +++ b/diff-web.py @@ -9,19 +9,51 @@ import hashlib import os import time import argparse -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode from datetime import datetime +from bs4 import BeautifulSoup, Comment + +# List of known tracking parameters to strip +TRACKING_PARAMS = { + 'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', + 'fbclid', 'gclid', 'mc_cid', 'mc_eid', 'ref', 'ref_src' +} def normalize_url(url): if not urlparse(url).scheme: url = 'https://' + url - return url + return strip_tracking_params(url) + +def strip_tracking_params(url): + parsed = urlparse(url) + clean_query = [(k, v) for k, v in parse_qsl(parsed.query) if k not in TRACKING_PARAMS] + new_query = urlencode(clean_query) + return urlunparse(parsed._replace(query=new_query)) def fetch_content(url): response = requests.get(url, timeout=10) response.raise_for_status() return response.text +def clean_content(html): + soup = BeautifulSoup(html, 'html.parser') + + # Remove dynamic or non-content elements + for tag in soup(['script', 'style', 'noscript', 'meta']): + tag.decompose() + + # Remove HTML comments + for comment in soup.find_all(string=lambda text: isinstance(text, Comment)): + comment.extract() + + # Remove tracking params from all anchor links + for a in soup.find_all('a', href=True): + a['href'] = strip_tracking_params(a['href']) + + # Get visible text, normalize whitespace + text = soup.get_text(separator=' ', strip=True) + return ' '.join(text.split()) + def get_hash(content): return hashlib.sha256(content.encode('utf-8')).hexdigest() @@ -35,41 +67,53 @@ def get_cache_filename(url): domain = urlparse(url).netloc return f".{domain}.cache" -def log_message(message, log_file): - print(message) +def timestamp(): + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +def log_message(message, log_file, quiet=False): + if not quiet: + print(message) with open(log_file, 'a') as f: f.write(message + '\n') -def monitor_website(url, interval, log_file_override=None): +def monitor_website(url, interval, log_file_override=None, max_checks=None, quiet=False): url = normalize_url(url) domain = urlparse(url).netloc log_file = get_log_filename(url, log_file_override) cache_file = get_cache_filename(url) - log_message(f"[{datetime.now()}] Monitoring {domain}", log_file) + log_message(f"[{timestamp()}] Monitoring {domain}", log_file, quiet) + + checks_done = 0 while True: try: - current_content = fetch_content(url) - current_hash = get_hash(current_content) + raw_html = fetch_content(url) + cleaned_text = clean_content(raw_html) + current_hash = get_hash(cleaned_text) except Exception as e: - log_message(f"[{datetime.now()}] [ERROR] Failed to fetch {url}: {e}", log_file) + log_message(f"[{timestamp()}] [ERROR] Failed to fetch {url}: {e}", log_file, quiet) time.sleep(interval) continue + is_changed = True if os.path.exists(cache_file): with open(cache_file, 'r') as f: old_hash = f.read().strip() - if current_hash != old_hash: - msg = f"[{datetime.now()}] Change detected on {url}" - log_message(msg, log_file) - with open(cache_file, 'w') as f: - f.write(current_hash) + is_changed = current_hash != old_hash + + if is_changed: + log_message(f"[{timestamp()}] Change detected on {url}", log_file, quiet) else: - msg = f"[{datetime.now()}] First-time check for {url}: storing baseline." - log_message(msg, log_file) - with open(cache_file, 'w') as f: - f.write(current_hash) + log_message(f"[{timestamp()}] First-time check for {url}: storing baseline.", log_file, quiet) + + with open(cache_file, 'w') as f: + f.write(current_hash) + + checks_done += 1 + if max_checks is not None and checks_done >= max_checks: + log_message(f"[{timestamp()}] Reached max checks ({max_checks}). Stopping.", log_file, quiet) + break time.sleep(interval) @@ -80,7 +124,15 @@ if __name__ == "__main__": help="Time interval between checks in seconds (default: 60).") parser.add_argument("-l", "--log", type=str, help="Optional log file name (default: domain.tld.log)") + parser.add_argument("-c", "--count", type=int, + help="Optional number of times to check before stopping.") + parser.add_argument("-q", "--quiet", action="store_true", + help="Quiet mode: suppress console output") args = parser.parse_args() - monitor_website(args.url, args.time, args.log) + try: + monitor_website(args.url, args.time, args.log, args.count, args.quiet) + except KeyboardInterrupt: + log_file = get_log_filename(args.url, args.log) + log_message(f"[{timestamp()}] Monitoring stopped by user (KeyboardInterrupt).", log_file, args.quiet)