Last active
September 7, 2026 20:25
-
-
Save AnythingLinux/acb49fcf12a993e8e06148d807d32ac6 to your computer and use it in GitHub Desktop.
Sitemap Generate RFC 3986 RFC 3987 XML safety checks
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """ | |
| generate_sitemap.py — Canonical-first, RFC 3986/3987 XML Sitemap Generator & Validator | |
| Usage: | |
| python3 /root/script/generate_sitemap.py --apply | |
| """ | |
| import argparse | |
| import datetime | |
| import fnmatch | |
| from html.parser import HTMLParser | |
| import ipaddress | |
| import os | |
| import re | |
| import sys | |
| import unicodedata | |
| import urllib.parse | |
| import xml.etree.ElementTree as ET | |
| # --------------------------------------------------------------------------- | |
| # Defaults & XML Namespace | |
| # --------------------------------------------------------------------------- | |
| NS = "http://www.sitemaps.org/schemas/sitemap/0.9" | |
| ET.register_namespace("", NS) | |
| INDEX_FILE = "index.html" | |
| DEFAULT_ROOT_DIR = "/var/www/html" | |
| DEFAULT_DOMAIN = "https://schooloffreelancing.com" | |
| DEFAULT_CHANGEFREQ = "always" | |
| # Hierarchy priority tiers | |
| DEFAULT_ROOT_PRIORITY = 1.0 # Root page (/) | |
| DEFAULT_PAGE_PRIORITY = 0.8 # First-level pages (/about-us) | |
| DEFAULT_DEEP_PRIORITY = 0.6 # Deep pages (/category/subcategory/page) | |
| DEFAULT_EXCLUDES = [ | |
| "assets", "assets/*", | |
| "api", "api/*", | |
| "mcp", "mcp/*", | |
| "gsc-repair-reports", "gsc-repair-reports/*", | |
| ".git", ".git/*", | |
| ".well-known", ".well-known/*", | |
| "node_modules", "node_modules/*", | |
| ] | |
| VALID_CHANGEFREQ = {"always", "hourly", "daily", "weekly", "monthly", "yearly", "never"} | |
| # --------------------------------------------------------------------------- | |
| # HTML Parser to Extract <link rel="canonical"> | |
| # --------------------------------------------------------------------------- | |
| class CanonicalParser(HTMLParser): | |
| """Parses HTML to extract the <link rel="canonical" href="..."> value.""" | |
| def __init__(self): | |
| super().__init__() | |
| self.canonical_url = None | |
| def handle_starttag(self, tag, attrs): | |
| if self.canonical_url: | |
| return | |
| if tag.lower() == "link": | |
| attr_dict = {k.lower(): v for k, v in attrs if k and v} | |
| rel_tokens = attr_dict.get("rel", "").lower().split() | |
| if "canonical" in rel_tokens: | |
| self.canonical_url = attr_dict.get("href") | |
| def extract_canonical_from_file(filepath, base_domain): | |
| """ | |
| Reads index.html, extracts the canonical tag, and normalizes it. | |
| If no canonical tag is found, raises a SystemExit error to ensure | |
| all sitemap URLs strictly originate from declared canonical tags. | |
| """ | |
| parser = CanonicalParser() | |
| try: | |
| with open(filepath, "r", encoding="utf-8", errors="ignore") as f: | |
| for line in f: | |
| parser.feed(line) | |
| if parser.canonical_url: | |
| break | |
| if "</head>" in line.lower(): | |
| break | |
| except OSError as exc: | |
| raise SystemExit(f"ERROR: Failed to read file {filepath}: {exc}") | |
| raw_canonical = parser.canonical_url | |
| if not raw_canonical or not raw_canonical.strip(): | |
| raise SystemExit(f"ERROR: Missing <link rel=\"canonical\"> tag in {filepath}") | |
| raw_canonical = raw_canonical.strip() | |
| # Convert relative canonical paths into full URLs | |
| if not urllib.parse.urlsplit(raw_canonical).netloc: | |
| raw_canonical = urllib.parse.urljoin(base_domain + "/", raw_canonical.lstrip("/")) | |
| return raw_canonical | |
| # --------------------------------------------------------------------------- | |
| # RFC 3986 / RFC 3987 / Hostname Validation Engine | |
| # --------------------------------------------------------------------------- | |
| def validate_hostname(hostname): | |
| """Ensures host is a valid RFC 3986 non-www hostname or valid IP.""" | |
| if any(ch.isspace() for ch in hostname): | |
| raise ValueError("hostname contains whitespace") | |
| # Lowercase and check non-www | |
| if hostname.lower().startswith("www."): | |
| raise ValueError(f"must be non-www hostname (got {hostname!r})") | |
| try: | |
| ipaddress.ip_address(hostname) | |
| return | |
| except ValueError: | |
| pass | |
| # RFC 3987 Unicode / IDNA normalization | |
| try: | |
| ascii_hostname = hostname.encode("idna").decode("ascii") | |
| except UnicodeError as exc: | |
| raise ValueError(f"invalid internationalized hostname: {hostname}") from exc | |
| if len(ascii_hostname) > 253: | |
| raise ValueError("hostname exceeds 253 characters") | |
| labels = ascii_hostname.split(".") | |
| if any(not label for label in labels): | |
| raise ValueError("hostname contains empty labels") | |
| label_pattern = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$") | |
| for label in labels: | |
| if not label_pattern.fullmatch(label): | |
| raise ValueError(f"invalid hostname label: {label}") | |
| def validate_and_normalize_url(url, expected_domain): | |
| """ | |
| Validates URL strictly against: | |
| - RFC 3986 / RFC 3987 (Unicode IRI, percent-encoding) | |
| - HTTPS scheme only | |
| - Non-www domain match | |
| - No query parameters, fragments (#), or whitespace | |
| - XML safety | |
| """ | |
| errors = [] | |
| if not isinstance(url, str) or not url.strip(): | |
| return None, ["URL is empty."] | |
| # Check for illegal XML control characters | |
| for char in url: | |
| code = ord(char) | |
| if code in (0, 0x7F) or (1 <= code <= 8) or (11 <= code <= 12) or (14 <= code <= 31): | |
| errors.append(f"Contains illegal XML control character U+{code:04X}.") | |
| break | |
| if any(ch.isspace() for ch in url): | |
| errors.append("URL contains whitespace.") | |
| # Parse scheme, netloc, path, query, fragment | |
| parsed = urllib.parse.urlsplit(url) | |
| # 1. Force HTTPS only | |
| if parsed.scheme.lower() != "https": | |
| errors.append(f"Scheme must be 'https' (got '{parsed.scheme}').") | |
| if not parsed.netloc: | |
| errors.append("Missing hostname.") | |
| # 2. Disallow credentials, query params, and fragments | |
| if parsed.username or parsed.password: | |
| errors.append("Must not contain credentials.") | |
| if parsed.query: | |
| errors.append("Must not contain query parameters (?...).") | |
| if parsed.fragment: | |
| errors.append("Must not contain fragments (#...).") | |
| # 3. Hostname validation | |
| try: | |
| if parsed.hostname: | |
| validate_hostname(parsed.hostname) | |
| except ValueError as exc: | |
| errors.append(f"Hostname error: {exc}") | |
| # 4. Domain matching | |
| expected_split = urllib.parse.urlsplit(expected_domain) | |
| if parsed.netloc.lower() != expected_split.netloc.lower(): | |
| errors.append(f"Domain mismatch (expected '{expected_split.netloc}', got '{parsed.netloc}').") | |
| # 5. Percent-encoding check | |
| if re.search(r"%(?![0-9A-Fa-f]{2})", url): | |
| errors.append("Invalid percent-encoding.") | |
| # 6. RFC 3987 Unicode Normalization (NFC) | |
| normalized_path = unicodedata.normalize("NFC", parsed.path) if parsed.path else "/" | |
| # Reconstruct clean, normalized URL | |
| clean_url = urllib.parse.urlunsplit(( | |
| parsed.scheme.lower(), | |
| parsed.netloc.lower(), | |
| normalized_path, | |
| "", # No query | |
| "" # No fragment | |
| )) | |
| return clean_url, errors | |
| # --------------------------------------------------------------------------- | |
| # Helper Functions | |
| # --------------------------------------------------------------------------- | |
| def is_excluded(reldir, excludes): | |
| if reldir == "": | |
| return False | |
| candidates = {reldir, reldir + "/"} | |
| return any(fnmatch.fnmatch(c, pattern) for c in candidates for pattern in excludes) | |
| def calculate_priority(url_path, root_p, page_p, deep_p): | |
| """Calculates priority tier based on URL path depth.""" | |
| path = url_path.strip("/") | |
| if not path: | |
| return root_p | |
| depth = len([part for part in path.split("/") if part]) | |
| return page_p if depth <= 1 else deep_p | |
| def scan_website_index_files(root, excludes, domain, root_p, page_p, deep_p): | |
| """Scans /var/www/html, parses index.html canonical tags, and returns page metadata.""" | |
| pages = {} | |
| abs_root = os.path.abspath(root) | |
| if not os.path.isdir(abs_root): | |
| raise SystemExit(f"ERROR: Target directory does not exist: {abs_root}") | |
| for dirpath, dirnames, filenames in os.walk(abs_root): | |
| dirnames[:] = sorted(d for d in dirnames if not d.startswith(".")) | |
| rel_dir = os.path.relpath(dirpath, abs_root).replace(os.sep, "/") | |
| if rel_dir == ".": | |
| rel_dir = "" | |
| if is_excluded(rel_dir, excludes): | |
| dirnames[:] = [] | |
| continue | |
| if INDEX_FILE not in filenames: | |
| continue | |
| full_filepath = os.path.join(dirpath, INDEX_FILE) | |
| # Extract Canonical URL | |
| raw_canonical = extract_canonical_from_file(full_filepath, domain) | |
| # RFC / Scheme / Syntax Validation | |
| clean_url, errors = validate_and_normalize_url(raw_canonical, domain) | |
| if errors: | |
| raise SystemExit( | |
| f"ERROR in {full_filepath}:\n" | |
| f" Canonical URL: {raw_canonical}\n" | |
| + "\n".join(f" - {e}" for e in errors) | |
| ) | |
| # File modification time | |
| try: | |
| mtime = os.path.getmtime(full_filepath) | |
| lastmod = datetime.datetime.fromtimestamp(mtime, datetime.timezone.utc).date().isoformat() | |
| except OSError as exc: | |
| raise SystemExit(f"ERROR reading file timestamp for {full_filepath}: {exc}") | |
| # Determine Priority based on canonical path | |
| parsed_path = urllib.parse.urlsplit(clean_url).path | |
| priority_val = calculate_priority(parsed_path, root_p, page_p, deep_p) | |
| pages[clean_url] = { | |
| "filepath": full_filepath, | |
| "lastmod": lastmod, | |
| "priority": f"{priority_val:.1f}", | |
| } | |
| return pages | |
| def build_sitemap_xml(pages, changefreq=DEFAULT_CHANGEFREQ): | |
| """Generates an XML tree with built-in XML-safe escaping.""" | |
| root = ET.Element(f"{{{NS}}}urlset") | |
| for url in sorted(pages): | |
| data = pages[url] | |
| url_el = ET.SubElement(root, f"{{{NS}}}url") | |
| # loc element (ET.SubElement handles XML escaping automatically) | |
| ET.SubElement(url_el, f"{{{NS}}}loc").text = url | |
| ET.SubElement(url_el, f"{{{NS}}}lastmod").text = data["lastmod"] | |
| ET.SubElement(url_el, f"{{{NS}}}changefreq").text = changefreq | |
| ET.SubElement(url_el, f"{{{NS}}}priority").text = data["priority"] | |
| ET.indent(root, space=" ") | |
| xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=False) | |
| return b'<?xml version="1.0" encoding="UTF-8"?>\n' + xml_bytes + b"\n" | |
| def atomic_write(path, data): | |
| """Safely writes the file using a temporary buffer before replacing.""" | |
| directory = os.path.dirname(os.path.abspath(path)) | |
| os.makedirs(directory, exist_ok=True) | |
| temp_path = path + ".tmp" | |
| try: | |
| with open(temp_path, "wb") as handle: | |
| handle.write(data) | |
| handle.flush() | |
| os.fsync(handle.fileno()) | |
| os.replace(temp_path, path) | |
| except Exception: | |
| if os.path.exists(temp_path): | |
| os.remove(temp_path) | |
| raise | |
| # --------------------------------------------------------------------------- | |
| # CLI Argument Parsing & Entry Point | |
| # --------------------------------------------------------------------------- | |
| def parse_args(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--root", default=DEFAULT_ROOT_DIR, help="Web root directory") | |
| parser.add_argument("--domain", default=DEFAULT_DOMAIN, help="Base domain (https://schooloffreelancing.com)") | |
| parser.add_argument("--out", default=None, help="Output destination for sitemap.xml") | |
| parser.add_argument("--apply", action="store_true", help="Write sitemap.xml to disk") | |
| args = parser.parse_args() | |
| # Force HTTPS non-www domain validation | |
| if not args.domain.startswith("https://"): | |
| args.domain = "https://" + args.domain.lstrip("http://").lstrip("https://") | |
| args.domain = args.domain.rstrip("/") | |
| if "www." in urllib.parse.urlsplit(args.domain).hostname.lower(): | |
| raise SystemExit(f"ERROR: Domain must be non-www (got {args.domain}).") | |
| args.out = args.out or os.path.join(args.root, "sitemap.xml") | |
| args.excludes = DEFAULT_EXCLUDES | |
| return args | |
| def main(): | |
| args = parse_args() | |
| print("=" * 72) | |
| print("SITEMAP GENERATOR — CANONICAL-BASED (RFC 3986 / RFC 3987)") | |
| print("=" * 72) | |
| print(f"Mode: {'APPLY' if args.apply else 'DRY RUN'}") | |
| print(f"Target Root: {args.root}") | |
| print(f"Output File: {args.out}") | |
| print(f"Target Domain:{args.domain}\n") | |
| print("Scanning directory structure and extracting canonical tags...") | |
| pages = scan_website_index_files( | |
| root=args.root, | |
| excludes=args.excludes, | |
| domain=args.domain, | |
| root_p=DEFAULT_ROOT_PRIORITY, | |
| page_p=DEFAULT_PAGE_PRIORITY, | |
| deep_p=DEFAULT_DEEP_PRIORITY, | |
| ) | |
| print(f"Found {len(pages)} unique canonical URL entries.\n") | |
| xml_data = build_sitemap_xml(pages, changefreq=DEFAULT_CHANGEFREQ) | |
| if args.apply: | |
| print(f"Writing sitemap to {args.out}...") | |
| atomic_write(args.out, xml_data) | |
| print("SUCCESS: sitemap.xml created successfully!") | |
| else: | |
| print("[DRY RUN] Generated sitemap preview:") | |
| print("-" * 72) | |
| preview = xml_data.decode("utf-8") | |
| print("\n".join(preview.splitlines()[:20])) | |
| print("-" * 72) | |
| print("\nRun with --apply to write the sitemap file.") | |
| if __name__ == "__main__": | |
| main() |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
python3 /root/script/generate_sitemap.py --root /var/www/html --domain https://schooloffreelancing.com
python3 /root/script/generate_sitemap.py --root /var/www/html --domain https://schooloffreelancing.com --apply