Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save AnythingLinux/acb49fcf12a993e8e06148d807d32ac6 to your computer and use it in GitHub Desktop.

Select an option

Save AnythingLinux/acb49fcf12a993e8e06148d807d32ac6 to your computer and use it in GitHub Desktop.
Sitemap Generate RFC 3986 RFC 3987 XML safety checks
#!/usr/bin/env python3
"""
generate_sitemap.py — Canonical-first, RFC 3986/3987 XML Sitemap Generator & Validator
Usage:
python3 /root/script/generate_sitemap.py --apply
"""
import argparse
import datetime
import fnmatch
from html.parser import HTMLParser
import ipaddress
import os
import re
import sys
import unicodedata
import urllib.parse
import xml.etree.ElementTree as ET
# ---------------------------------------------------------------------------
# Defaults & XML Namespace
# ---------------------------------------------------------------------------
NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
ET.register_namespace("", NS)
INDEX_FILE = "index.html"
DEFAULT_ROOT_DIR = "/var/www/html"
DEFAULT_DOMAIN = "https://schooloffreelancing.com"
DEFAULT_CHANGEFREQ = "always"
# Hierarchy priority tiers
DEFAULT_ROOT_PRIORITY = 1.0 # Root page (/)
DEFAULT_PAGE_PRIORITY = 0.8 # First-level pages (/about-us)
DEFAULT_DEEP_PRIORITY = 0.6 # Deep pages (/category/subcategory/page)
DEFAULT_EXCLUDES = [
"assets", "assets/*",
"api", "api/*",
"mcp", "mcp/*",
"gsc-repair-reports", "gsc-repair-reports/*",
".git", ".git/*",
".well-known", ".well-known/*",
"node_modules", "node_modules/*",
]
VALID_CHANGEFREQ = {"always", "hourly", "daily", "weekly", "monthly", "yearly", "never"}
# ---------------------------------------------------------------------------
# HTML Parser to Extract <link rel="canonical">
# ---------------------------------------------------------------------------
class CanonicalParser(HTMLParser):
"""Parses HTML to extract the <link rel="canonical" href="..."> value."""
def __init__(self):
super().__init__()
self.canonical_url = None
def handle_starttag(self, tag, attrs):
if self.canonical_url:
return
if tag.lower() == "link":
attr_dict = {k.lower(): v for k, v in attrs if k and v}
rel_tokens = attr_dict.get("rel", "").lower().split()
if "canonical" in rel_tokens:
self.canonical_url = attr_dict.get("href")
def extract_canonical_from_file(filepath, base_domain):
"""
Reads index.html, extracts the canonical tag, and normalizes it.
If no canonical tag is found, raises a SystemExit error to ensure
all sitemap URLs strictly originate from declared canonical tags.
"""
parser = CanonicalParser()
try:
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
for line in f:
parser.feed(line)
if parser.canonical_url:
break
if "</head>" in line.lower():
break
except OSError as exc:
raise SystemExit(f"ERROR: Failed to read file {filepath}: {exc}")
raw_canonical = parser.canonical_url
if not raw_canonical or not raw_canonical.strip():
raise SystemExit(f"ERROR: Missing <link rel=\"canonical\"> tag in {filepath}")
raw_canonical = raw_canonical.strip()
# Convert relative canonical paths into full URLs
if not urllib.parse.urlsplit(raw_canonical).netloc:
raw_canonical = urllib.parse.urljoin(base_domain + "/", raw_canonical.lstrip("/"))
return raw_canonical
# ---------------------------------------------------------------------------
# RFC 3986 / RFC 3987 / Hostname Validation Engine
# ---------------------------------------------------------------------------
def validate_hostname(hostname):
"""Ensures host is a valid RFC 3986 non-www hostname or valid IP."""
if any(ch.isspace() for ch in hostname):
raise ValueError("hostname contains whitespace")
# Lowercase and check non-www
if hostname.lower().startswith("www."):
raise ValueError(f"must be non-www hostname (got {hostname!r})")
try:
ipaddress.ip_address(hostname)
return
except ValueError:
pass
# RFC 3987 Unicode / IDNA normalization
try:
ascii_hostname = hostname.encode("idna").decode("ascii")
except UnicodeError as exc:
raise ValueError(f"invalid internationalized hostname: {hostname}") from exc
if len(ascii_hostname) > 253:
raise ValueError("hostname exceeds 253 characters")
labels = ascii_hostname.split(".")
if any(not label for label in labels):
raise ValueError("hostname contains empty labels")
label_pattern = re.compile(r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$")
for label in labels:
if not label_pattern.fullmatch(label):
raise ValueError(f"invalid hostname label: {label}")
def validate_and_normalize_url(url, expected_domain):
"""
Validates URL strictly against:
- RFC 3986 / RFC 3987 (Unicode IRI, percent-encoding)
- HTTPS scheme only
- Non-www domain match
- No query parameters, fragments (#), or whitespace
- XML safety
"""
errors = []
if not isinstance(url, str) or not url.strip():
return None, ["URL is empty."]
# Check for illegal XML control characters
for char in url:
code = ord(char)
if code in (0, 0x7F) or (1 <= code <= 8) or (11 <= code <= 12) or (14 <= code <= 31):
errors.append(f"Contains illegal XML control character U+{code:04X}.")
break
if any(ch.isspace() for ch in url):
errors.append("URL contains whitespace.")
# Parse scheme, netloc, path, query, fragment
parsed = urllib.parse.urlsplit(url)
# 1. Force HTTPS only
if parsed.scheme.lower() != "https":
errors.append(f"Scheme must be 'https' (got '{parsed.scheme}').")
if not parsed.netloc:
errors.append("Missing hostname.")
# 2. Disallow credentials, query params, and fragments
if parsed.username or parsed.password:
errors.append("Must not contain credentials.")
if parsed.query:
errors.append("Must not contain query parameters (?...).")
if parsed.fragment:
errors.append("Must not contain fragments (#...).")
# 3. Hostname validation
try:
if parsed.hostname:
validate_hostname(parsed.hostname)
except ValueError as exc:
errors.append(f"Hostname error: {exc}")
# 4. Domain matching
expected_split = urllib.parse.urlsplit(expected_domain)
if parsed.netloc.lower() != expected_split.netloc.lower():
errors.append(f"Domain mismatch (expected '{expected_split.netloc}', got '{parsed.netloc}').")
# 5. Percent-encoding check
if re.search(r"%(?![0-9A-Fa-f]{2})", url):
errors.append("Invalid percent-encoding.")
# 6. RFC 3987 Unicode Normalization (NFC)
normalized_path = unicodedata.normalize("NFC", parsed.path) if parsed.path else "/"
# Reconstruct clean, normalized URL
clean_url = urllib.parse.urlunsplit((
parsed.scheme.lower(),
parsed.netloc.lower(),
normalized_path,
"", # No query
"" # No fragment
))
return clean_url, errors
# ---------------------------------------------------------------------------
# Helper Functions
# ---------------------------------------------------------------------------
def is_excluded(reldir, excludes):
if reldir == "":
return False
candidates = {reldir, reldir + "/"}
return any(fnmatch.fnmatch(c, pattern) for c in candidates for pattern in excludes)
def calculate_priority(url_path, root_p, page_p, deep_p):
"""Calculates priority tier based on URL path depth."""
path = url_path.strip("/")
if not path:
return root_p
depth = len([part for part in path.split("/") if part])
return page_p if depth <= 1 else deep_p
def scan_website_index_files(root, excludes, domain, root_p, page_p, deep_p):
"""Scans /var/www/html, parses index.html canonical tags, and returns page metadata."""
pages = {}
abs_root = os.path.abspath(root)
if not os.path.isdir(abs_root):
raise SystemExit(f"ERROR: Target directory does not exist: {abs_root}")
for dirpath, dirnames, filenames in os.walk(abs_root):
dirnames[:] = sorted(d for d in dirnames if not d.startswith("."))
rel_dir = os.path.relpath(dirpath, abs_root).replace(os.sep, "/")
if rel_dir == ".":
rel_dir = ""
if is_excluded(rel_dir, excludes):
dirnames[:] = []
continue
if INDEX_FILE not in filenames:
continue
full_filepath = os.path.join(dirpath, INDEX_FILE)
# Extract Canonical URL
raw_canonical = extract_canonical_from_file(full_filepath, domain)
# RFC / Scheme / Syntax Validation
clean_url, errors = validate_and_normalize_url(raw_canonical, domain)
if errors:
raise SystemExit(
f"ERROR in {full_filepath}:\n"
f" Canonical URL: {raw_canonical}\n"
+ "\n".join(f" - {e}" for e in errors)
)
# File modification time
try:
mtime = os.path.getmtime(full_filepath)
lastmod = datetime.datetime.fromtimestamp(mtime, datetime.timezone.utc).date().isoformat()
except OSError as exc:
raise SystemExit(f"ERROR reading file timestamp for {full_filepath}: {exc}")
# Determine Priority based on canonical path
parsed_path = urllib.parse.urlsplit(clean_url).path
priority_val = calculate_priority(parsed_path, root_p, page_p, deep_p)
pages[clean_url] = {
"filepath": full_filepath,
"lastmod": lastmod,
"priority": f"{priority_val:.1f}",
}
return pages
def build_sitemap_xml(pages, changefreq=DEFAULT_CHANGEFREQ):
"""Generates an XML tree with built-in XML-safe escaping."""
root = ET.Element(f"{{{NS}}}urlset")
for url in sorted(pages):
data = pages[url]
url_el = ET.SubElement(root, f"{{{NS}}}url")
# loc element (ET.SubElement handles XML escaping automatically)
ET.SubElement(url_el, f"{{{NS}}}loc").text = url
ET.SubElement(url_el, f"{{{NS}}}lastmod").text = data["lastmod"]
ET.SubElement(url_el, f"{{{NS}}}changefreq").text = changefreq
ET.SubElement(url_el, f"{{{NS}}}priority").text = data["priority"]
ET.indent(root, space=" ")
xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=False)
return b'<?xml version="1.0" encoding="UTF-8"?>\n' + xml_bytes + b"\n"
def atomic_write(path, data):
"""Safely writes the file using a temporary buffer before replacing."""
directory = os.path.dirname(os.path.abspath(path))
os.makedirs(directory, exist_ok=True)
temp_path = path + ".tmp"
try:
with open(temp_path, "wb") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
os.replace(temp_path, path)
except Exception:
if os.path.exists(temp_path):
os.remove(temp_path)
raise
# ---------------------------------------------------------------------------
# CLI Argument Parsing & Entry Point
# ---------------------------------------------------------------------------
def parse_args():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", default=DEFAULT_ROOT_DIR, help="Web root directory")
parser.add_argument("--domain", default=DEFAULT_DOMAIN, help="Base domain (https://schooloffreelancing.com)")
parser.add_argument("--out", default=None, help="Output destination for sitemap.xml")
parser.add_argument("--apply", action="store_true", help="Write sitemap.xml to disk")
args = parser.parse_args()
# Force HTTPS non-www domain validation
if not args.domain.startswith("https://"):
args.domain = "https://" + args.domain.lstrip("http://").lstrip("https://")
args.domain = args.domain.rstrip("/")
if "www." in urllib.parse.urlsplit(args.domain).hostname.lower():
raise SystemExit(f"ERROR: Domain must be non-www (got {args.domain}).")
args.out = args.out or os.path.join(args.root, "sitemap.xml")
args.excludes = DEFAULT_EXCLUDES
return args
def main():
args = parse_args()
print("=" * 72)
print("SITEMAP GENERATOR — CANONICAL-BASED (RFC 3986 / RFC 3987)")
print("=" * 72)
print(f"Mode: {'APPLY' if args.apply else 'DRY RUN'}")
print(f"Target Root: {args.root}")
print(f"Output File: {args.out}")
print(f"Target Domain:{args.domain}\n")
print("Scanning directory structure and extracting canonical tags...")
pages = scan_website_index_files(
root=args.root,
excludes=args.excludes,
domain=args.domain,
root_p=DEFAULT_ROOT_PRIORITY,
page_p=DEFAULT_PAGE_PRIORITY,
deep_p=DEFAULT_DEEP_PRIORITY,
)
print(f"Found {len(pages)} unique canonical URL entries.\n")
xml_data = build_sitemap_xml(pages, changefreq=DEFAULT_CHANGEFREQ)
if args.apply:
print(f"Writing sitemap to {args.out}...")
atomic_write(args.out, xml_data)
print("SUCCESS: sitemap.xml created successfully!")
else:
print("[DRY RUN] Generated sitemap preview:")
print("-" * 72)
preview = xml_data.decode("utf-8")
print("\n".join(preview.splitlines()[:20]))
print("-" * 72)
print("\nRun with --apply to write the sitemap file.")
if __name__ == "__main__":
main()
@AnythingLinux

AnythingLinux commented Sep 5, 2026 •

Copy link
Copy Markdown
Author

python3 /root/script/generate_sitemap.py --root /var/www/html --domain https://schooloffreelancing.com

python3 /root/script/generate_sitemap.py --root /var/www/html --domain https://schooloffreelancing.com --apply

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment