Last active
September 12, 2026 16:16
-
-
Save AnythingLinux/e480a57f4b94e878fabc8f44d43ee527 to your computer and use it in GitHub Desktop.
Google Search Console Diagnostic script
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """ | |
| Production Google Search Console Diagnostic | |
| ============================================ | |
| Target: | |
| https://schooloffreelancing.com | |
| GSC property: | |
| sc-domain:schooloffreelancing.com | |
| Features: | |
| - Google OAuth 2.0 authorization-code setup | |
| - Protected local .env credential/configuration file | |
| - Automatic .env loading without python-dotenv | |
| - .env permissions enforced as 0600 | |
| - GSC property validation | |
| - HTTPS / HTTP diagnostics | |
| - robots.txt diagnostics | |
| - sitemap diagnostics | |
| - XML sitemap URL extraction | |
| - GSC URL Inspection | |
| - canonical/indexability checks | |
| - Search Analytics performance report | |
| - sitemap submission | |
| - Action list: pages that need changes to align with GSC, with | |
| specific per-page issues and recommended fixes | |
| - JSON and HTML reports | |
| - retries for transient failures | |
| - structured logging | |
| - configurable URL limits | |
| Recommended directory: | |
| /home/YOUR_USER/gsc-diagnostic/ | |
| gsc_diagnostic.py | |
| .env | |
| reports/ | |
| Environment file: | |
| .env | |
| Example: | |
| GSC_CLIENT_ID=YOUR_CLIENT_ID.apps.googleusercontent.com | |
| GSC_CLIENT_SECRET=YOUR_CLIENT_SECRET | |
| GSC_REFRESH_TOKEN=YOUR_REFRESH_TOKEN | |
| GSC_DAYS=28 | |
| GSC_MAX_URLS=250 | |
| GSC_TIMEOUT=20 | |
| GSC_OUTPUT_DIR=/home/YOUR_USER/gsc-diagnostic/reports | |
| GSC_SUBMIT_SITEMAP=true | |
| GSC_INSPECT_SITEMAP_URLS=true | |
| Secure permissions: | |
| chmod 600 .env | |
| Recommended shell workflow: | |
| set -a | |
| source .env | |
| set +a | |
| python3 gsc_diagnostic.py | |
| Create a secure .env template: | |
| python3 gsc_diagnostic.py --init-env | |
| OAuth setup: | |
| python3 gsc_diagnostic.py --oauth-setup | |
| For a remote Linux server, use an SSH tunnel: | |
| ssh -L 8080:127.0.0.1:8080 root@YOUR_SERVER | |
| Then, on the server: | |
| python3 gsc_diagnostic.py --oauth-setup --port 8080 | |
| Open the displayed Google authorization URL in your local browser. | |
| The OAuth refresh token is written into: | |
| .env | |
| with permissions: | |
| 0600 | |
| Recommended Google Cloud OAuth client: | |
| Application type: | |
| Desktop app | |
| Required OAuth scope: | |
| https://www.googleapis.com/auth/webmasters | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import ast | |
| import csv | |
| import html | |
| import json | |
| import logging | |
| import os | |
| import re | |
| import shlex | |
| import stat | |
| import sys | |
| import time | |
| from dataclasses import asdict, dataclass, field | |
| from datetime import date, timedelta | |
| from pathlib import Path | |
| from typing import Any, Dict, Iterable, List, Optional | |
| from urllib.parse import urljoin, urlparse | |
| import requests | |
| from google.auth.transport.requests import Request | |
| from google.oauth2.credentials import Credentials | |
| from google_auth_oauthlib.flow import InstalledAppFlow | |
| from googleapiclient.discovery import build | |
| from googleapiclient.errors import HttpError | |
| # ============================================================ | |
| # PATHS | |
| # ============================================================ | |
| SCRIPT_DIR = Path( | |
| __file__ | |
| ).resolve().parent | |
| ENV_FILE = SCRIPT_DIR / ".env" | |
| DEFAULT_OUTPUT_DIR = ( | |
| SCRIPT_DIR / "reports" | |
| ) | |
| # ============================================================ | |
| # CONFIGURATION | |
| # ============================================================ | |
| SITE_URL = "sc-domain:schooloffreelancing.com" | |
| BASE_URL = "https://schooloffreelancing.com" | |
| SITEMAP_URL = f"{BASE_URL}/sitemap.xml" | |
| ROBOTS_URL = f"{BASE_URL}/robots.txt" | |
| SCOPES = [ | |
| "https://www.googleapis.com/auth/webmasters", | |
| ] | |
| USER_AGENT = ( | |
| "SchoolOfFreelancing-GSC-Diagnostic/5.0 " | |
| "(Google Search Console diagnostic)" | |
| ) | |
| HTTP_HEADERS = { | |
| "User-Agent": USER_AGENT, | |
| "Accept": "*/*", | |
| } | |
| # ============================================================ | |
| # LOGGING | |
| # ============================================================ | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format="%(asctime)s | %(levelname)s | %(message)s", | |
| ) | |
| logger = logging.getLogger( | |
| "gsc-diagnostic" | |
| ) | |
| # ============================================================ | |
| # .ENV SECURITY | |
| # ============================================================ | |
| ENV_ALLOWED_KEYS = { | |
| "GSC_CLIENT_ID", | |
| "GSC_CLIENT_SECRET", | |
| "GSC_REFRESH_TOKEN", | |
| "GSC_DAYS", | |
| "GSC_MAX_URLS", | |
| "GSC_TIMEOUT", | |
| "GSC_OUTPUT_DIR", | |
| "GSC_SUBMIT_SITEMAP", | |
| "GSC_INSPECT_SITEMAP_URLS", | |
| } | |
| def secure_file_owner_only( | |
| path: Path, | |
| ) -> None: | |
| """ | |
| Set a file to owner read/write only. | |
| """ | |
| os.chmod( | |
| path, | |
| stat.S_IRUSR | stat.S_IWUSR, | |
| ) | |
| def secure_env_file( | |
| path: Path, | |
| ) -> None: | |
| """ | |
| Validate and enforce secure .env permissions. | |
| Requirements: | |
| - must exist | |
| - must not be a symlink | |
| - must be owned by current user | |
| - must have 0600 permissions | |
| """ | |
| if path.is_symlink(): | |
| raise RuntimeError( | |
| f"Refusing to use symlinked environment file: {path}" | |
| ) | |
| if not path.exists(): | |
| raise RuntimeError( | |
| f"Environment file does not exist: {path}" | |
| ) | |
| if not path.is_file(): | |
| raise RuntimeError( | |
| f"Environment path is not a regular file: {path}" | |
| ) | |
| stat_result = path.stat() | |
| if stat_result.st_uid != os.geteuid(): | |
| raise RuntimeError( | |
| f"Environment file is not owned by the current user: {path}" | |
| ) | |
| mode = stat_result.st_mode & 0o777 | |
| if mode != 0o600: | |
| logger.warning( | |
| ".env permissions are %04o; changing to 0600.", | |
| mode, | |
| ) | |
| secure_file_owner_only( | |
| path | |
| ) | |
| final_mode = ( | |
| path.stat().st_mode | |
| & 0o777 | |
| ) | |
| if final_mode != 0o600: | |
| raise RuntimeError( | |
| f"Unable to enforce 0600 permissions on {path}" | |
| ) | |
| def parse_env_value( | |
| raw_value: str, | |
| ) -> str: | |
| """ | |
| Parse a simple .env value without executing shell code. | |
| Supports: | |
| KEY=value | |
| KEY="value" | |
| KEY='value' | |
| Comments are supported for unquoted values. | |
| """ | |
| value = raw_value.strip() | |
| if not value: | |
| return "" | |
| try: | |
| # Handle single/double quoted values. | |
| if ( | |
| len(value) >= 2 | |
| and value[0] == value[-1] | |
| and value[0] in {"'", '"'} | |
| ): | |
| try: | |
| parsed = ast.literal_eval( | |
| value | |
| ) | |
| if isinstance( | |
| parsed, | |
| str, | |
| ): | |
| return parsed | |
| except Exception: | |
| pass | |
| # shlex handles escaped characters and comments | |
| # without executing anything. | |
| parts = shlex.split( | |
| value, | |
| comments=True, | |
| posix=True, | |
| ) | |
| if parts: | |
| return parts[0] | |
| except ValueError: | |
| pass | |
| return value | |
| def read_env_file( | |
| path: Path, | |
| ) -> Dict[str, str]: | |
| """ | |
| Safely read a simple .env file. | |
| This function NEVER executes the file. | |
| """ | |
| secure_env_file( | |
| path | |
| ) | |
| values: Dict[str, str] = {} | |
| with path.open( | |
| "r", | |
| encoding="utf-8", | |
| ) as file: | |
| for line_number, line in enumerate( | |
| file, | |
| start=1, | |
| ): | |
| stripped = line.strip() | |
| if not stripped: | |
| continue | |
| if stripped.startswith("#"): | |
| continue | |
| # Accept optional "export KEY=value" | |
| if stripped.startswith( | |
| "export " | |
| ): | |
| stripped = stripped[ | |
| len("export "): | |
| ].lstrip() | |
| match = re.match( | |
| r"^([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)$", | |
| stripped, | |
| ) | |
| if not match: | |
| raise RuntimeError( | |
| f"Invalid .env syntax at " | |
| f"{path}:{line_number}" | |
| ) | |
| key = match.group(1) | |
| raw_value = match.group(2) | |
| if key not in ENV_ALLOWED_KEYS: | |
| logger.warning( | |
| "Ignoring unsupported .env variable: %s", | |
| key, | |
| ) | |
| continue | |
| values[key] = parse_env_value( | |
| raw_value | |
| ) | |
| return values | |
| def load_dotenv( | |
| path: Path = ENV_FILE, | |
| ) -> None: | |
| """ | |
| Load .env values into the current Python process. | |
| Existing process environment variables take precedence. | |
| Therefore: | |
| export GSC_TIMEOUT=30 | |
| overrides: | |
| GSC_TIMEOUT=20 | |
| in .env. | |
| """ | |
| if not path.exists(): | |
| logger.warning( | |
| "Environment file does not exist: %s", | |
| path, | |
| ) | |
| logger.warning( | |
| "Create it with:" | |
| ) | |
| logger.warning( | |
| " python3 %s --init-env", | |
| Path(__file__).name, | |
| ) | |
| return | |
| try: | |
| values = read_env_file( | |
| path | |
| ) | |
| except Exception as exc: | |
| raise RuntimeError( | |
| f"Unable to load protected .env: {exc}" | |
| ) from exc | |
| for key, value in values.items(): | |
| if key not in os.environ: | |
| os.environ[ | |
| key | |
| ] = value | |
| def shell_quote_single( | |
| value: str, | |
| ) -> str: | |
| """ | |
| Safely quote a value for a POSIX shell. | |
| """ | |
| return "'" + value.replace( | |
| "'", | |
| "'\"'\"'", | |
| ) + "'" | |
| def write_env_file( | |
| values: Dict[str, str], | |
| ) -> None: | |
| """ | |
| Write the complete .env securely. | |
| File permissions are set to 0600 before the file | |
| becomes visible at the final path. | |
| """ | |
| if ENV_FILE.is_symlink(): | |
| raise RuntimeError( | |
| f"Refusing to overwrite symlinked .env: {ENV_FILE}" | |
| ) | |
| ENV_FILE.parent.mkdir( | |
| parents=True, | |
| exist_ok=True, | |
| ) | |
| temporary = ENV_FILE.with_name( | |
| ".env.tmp" | |
| ) | |
| content_lines = [ | |
| "# School of Freelancing Google Search Console Diagnostic", | |
| "# Protected configuration and OAuth credentials", | |
| "# File permissions must remain 0600", | |
| "", | |
| ] | |
| for key in [ | |
| "GSC_CLIENT_ID", | |
| "GSC_CLIENT_SECRET", | |
| "GSC_REFRESH_TOKEN", | |
| "GSC_DAYS", | |
| "GSC_MAX_URLS", | |
| "GSC_TIMEOUT", | |
| "GSC_OUTPUT_DIR", | |
| "GSC_SUBMIT_SITEMAP", | |
| "GSC_INSPECT_SITEMAP_URLS", | |
| ]: | |
| value = values.get( | |
| key, | |
| "", | |
| ) | |
| content_lines.append( | |
| f"{key}={shell_quote_single(value)}" | |
| ) | |
| content = ( | |
| "\n".join( | |
| content_lines | |
| ) | |
| + "\n" | |
| ) | |
| flags = ( | |
| os.O_WRONLY | |
| | os.O_CREAT | |
| | os.O_TRUNC | |
| ) | |
| fd = os.open( | |
| temporary, | |
| flags, | |
| 0o600, | |
| ) | |
| try: | |
| with os.fdopen( | |
| fd, | |
| "w", | |
| encoding="utf-8", | |
| ) as file: | |
| file.write( | |
| content | |
| ) | |
| except Exception: | |
| try: | |
| temporary.unlink() | |
| except OSError: | |
| pass | |
| raise | |
| secure_file_owner_only( | |
| temporary | |
| ) | |
| temporary.replace( | |
| ENV_FILE | |
| ) | |
| secure_file_owner_only( | |
| ENV_FILE | |
| ) | |
| secure_env_file( | |
| ENV_FILE | |
| ) | |
| def initialize_env_file() -> bool: | |
| """ | |
| Create a protected .env template. | |
| Existing .env is never overwritten. | |
| """ | |
| if ENV_FILE.exists(): | |
| logger.info( | |
| ".env already exists: %s", | |
| ENV_FILE, | |
| ) | |
| try: | |
| secure_env_file( | |
| ENV_FILE | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "Existing .env security check failed: %s", | |
| exc, | |
| ) | |
| return False | |
| logger.info( | |
| ".env permissions verified as 0600." | |
| ) | |
| return True | |
| values = { | |
| "GSC_CLIENT_ID": ( | |
| "YOUR_CLIENT_ID.apps.googleusercontent.com" | |
| ), | |
| "GSC_CLIENT_SECRET": ( | |
| "YOUR_CLIENT_SECRET" | |
| ), | |
| "GSC_REFRESH_TOKEN": ( | |
| "YOUR_REFRESH_TOKEN" | |
| ), | |
| "GSC_DAYS": "28", | |
| "GSC_MAX_URLS": "250", | |
| "GSC_TIMEOUT": "20", | |
| "GSC_OUTPUT_DIR": str( | |
| DEFAULT_OUTPUT_DIR | |
| ), | |
| "GSC_SUBMIT_SITEMAP": "true", | |
| "GSC_INSPECT_SITEMAP_URLS": "true", | |
| } | |
| try: | |
| write_env_file( | |
| values | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "Unable to create protected .env: %s", | |
| exc, | |
| ) | |
| return False | |
| logger.info( | |
| "Protected .env created:" | |
| ) | |
| logger.info( | |
| " %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| "Permissions: 0600" | |
| ) | |
| logger.info( | |
| "" | |
| ) | |
| logger.info( | |
| "Edit it with:" | |
| ) | |
| logger.info( | |
| " nano %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| "" | |
| ) | |
| logger.info( | |
| "Then load it:" | |
| ) | |
| logger.info( | |
| " set -a" | |
| ) | |
| logger.info( | |
| " source %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| " set +a" | |
| ) | |
| return True | |
| def update_refresh_token_in_env( | |
| refresh_token: str, | |
| ) -> None: | |
| """ | |
| Update only GSC_REFRESH_TOKEN while preserving all | |
| other .env configuration. | |
| """ | |
| if not refresh_token.strip(): | |
| raise RuntimeError( | |
| "Cannot save an empty OAuth refresh token." | |
| ) | |
| if ENV_FILE.exists(): | |
| existing = read_env_file( | |
| ENV_FILE | |
| ) | |
| else: | |
| existing = { | |
| "GSC_CLIENT_ID": os.getenv( | |
| "GSC_CLIENT_ID", | |
| "", | |
| ), | |
| "GSC_CLIENT_SECRET": os.getenv( | |
| "GSC_CLIENT_SECRET", | |
| "", | |
| ), | |
| "GSC_DAYS": os.getenv( | |
| "GSC_DAYS", | |
| "28", | |
| ), | |
| "GSC_MAX_URLS": os.getenv( | |
| "GSC_MAX_URLS", | |
| "250", | |
| ), | |
| "GSC_TIMEOUT": os.getenv( | |
| "GSC_TIMEOUT", | |
| "20", | |
| ), | |
| "GSC_OUTPUT_DIR": os.getenv( | |
| "GSC_OUTPUT_DIR", | |
| str(DEFAULT_OUTPUT_DIR), | |
| ), | |
| "GSC_SUBMIT_SITEMAP": os.getenv( | |
| "GSC_SUBMIT_SITEMAP", | |
| "true", | |
| ), | |
| "GSC_INSPECT_SITEMAP_URLS": os.getenv( | |
| "GSC_INSPECT_SITEMAP_URLS", | |
| "true", | |
| ), | |
| } | |
| existing[ | |
| "GSC_REFRESH_TOKEN" | |
| ] = refresh_token.strip() | |
| # Ensure client credentials from the process are | |
| # preserved if they were not already in .env. | |
| for key in [ | |
| "GSC_CLIENT_ID", | |
| "GSC_CLIENT_SECRET", | |
| ]: | |
| if not existing.get(key): | |
| environment_value = os.getenv( | |
| key | |
| ) | |
| if environment_value: | |
| existing[ | |
| key | |
| ] = environment_value | |
| write_env_file( | |
| existing | |
| ) | |
| def print_env_instructions() -> None: | |
| """ | |
| Print safe instructions without exposing credentials. | |
| """ | |
| logger.info("") | |
| logger.info( | |
| "================================================" | |
| ) | |
| logger.info( | |
| "Protected environment file:" | |
| ) | |
| logger.info( | |
| " %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| "Permissions: 0600" | |
| ) | |
| logger.info("") | |
| logger.info( | |
| "Load it into the current shell:" | |
| ) | |
| logger.info( | |
| " set -a" | |
| ) | |
| logger.info( | |
| " source %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| " set +a" | |
| ) | |
| logger.info("") | |
| logger.info( | |
| "Verify without displaying credentials:" | |
| ) | |
| logger.info( | |
| " test -n \"$GSC_CLIENT_ID\" " | |
| "&& echo 'GSC_CLIENT_ID: SET' " | |
| "|| echo 'GSC_CLIENT_ID: MISSING'" | |
| ) | |
| logger.info( | |
| " test -n \"$GSC_CLIENT_SECRET\" " | |
| "&& echo 'GSC_CLIENT_SECRET: SET' " | |
| "|| echo 'GSC_CLIENT_SECRET: MISSING'" | |
| ) | |
| logger.info( | |
| " test -n \"$GSC_REFRESH_TOKEN\" " | |
| "&& echo 'GSC_REFRESH_TOKEN: SET' " | |
| "|| echo 'GSC_REFRESH_TOKEN: MISSING'" | |
| ) | |
| logger.info( | |
| "================================================" | |
| ) | |
| # ============================================================ | |
| # LOAD .ENV BEFORE CONFIGURATION | |
| # ============================================================ | |
| load_dotenv( | |
| ENV_FILE | |
| ) | |
| # ============================================================ | |
| # RUNTIME CONFIGURATION | |
| # ============================================================ | |
| TIMEOUT = int( | |
| os.getenv( | |
| "GSC_TIMEOUT", | |
| "20", | |
| ) | |
| ) | |
| MAX_URLS = int( | |
| os.getenv( | |
| "GSC_MAX_URLS", | |
| "250", | |
| ) | |
| ) | |
| DAYS = int( | |
| os.getenv( | |
| "GSC_DAYS", | |
| "28", | |
| ) | |
| ) | |
| OUTPUT_DIR = Path( | |
| os.getenv( | |
| "GSC_OUTPUT_DIR", | |
| str(DEFAULT_OUTPUT_DIR), | |
| ) | |
| ).expanduser() | |
| SUBMIT_SITEMAP = ( | |
| os.getenv( | |
| "GSC_SUBMIT_SITEMAP", | |
| "true", | |
| ).strip().lower() | |
| in { | |
| "1", | |
| "true", | |
| "yes", | |
| "on", | |
| } | |
| ) | |
| INSPECT_SITEMAP_URLS = ( | |
| os.getenv( | |
| "GSC_INSPECT_SITEMAP_URLS", | |
| "true", | |
| ).strip().lower() | |
| in { | |
| "1", | |
| "true", | |
| "yes", | |
| "on", | |
| } | |
| ) | |
| # ============================================================ | |
| # VALIDATE CONFIGURATION | |
| # ============================================================ | |
| def validate_runtime_configuration() -> None: | |
| if TIMEOUT <= 0: | |
| raise RuntimeError( | |
| "GSC_TIMEOUT must be greater than 0." | |
| ) | |
| if MAX_URLS <= 0: | |
| raise RuntimeError( | |
| "GSC_MAX_URLS must be greater than 0." | |
| ) | |
| if DAYS <= 0: | |
| raise RuntimeError( | |
| "GSC_DAYS must be greater than 0." | |
| ) | |
| # ============================================================ | |
| # DATA STRUCTURES | |
| # ============================================================ | |
| @dataclass | |
| class HTTPDiagnostic: | |
| url: str | |
| status_code: Optional[int] | |
| final_url: Optional[str] | |
| response_time_ms: Optional[float] | |
| content_type: Optional[str] | |
| robots_meta: Optional[str] | |
| canonical: Optional[str] | |
| error: Optional[str] | |
| ok: bool | |
| @dataclass | |
| class GSCInspection: | |
| url: str | |
| verdict: Optional[str] | |
| coverage_state: Optional[str] | |
| indexing_state: Optional[str] | |
| robots_txt_state: Optional[str] | |
| page_fetch_state: Optional[str] | |
| google_canonical: Optional[str] | |
| user_canonical: Optional[str] | |
| mobile_usability: Optional[str] | |
| referring_urls: Optional[List[str]] | |
| last_crawl_time: Optional[str] | |
| error: Optional[str] | |
| @dataclass | |
| class PageAction: | |
| """ | |
| A single URL that needs attention, with the specific | |
| issues detected and the recommended fix for each. | |
| """ | |
| url: str | |
| severity: str # "critical" | "warning" | "info" | |
| issues: List[str] = field(default_factory=list) | |
| recommendations: List[str] = field(default_factory=list) | |
| # ============================================================ | |
| # COMMAND-LINE ARGUMENTS | |
| # ============================================================ | |
| def parse_args(): | |
| parser = argparse.ArgumentParser( | |
| description=( | |
| "School of Freelancing Google Search Console Diagnostic" | |
| ) | |
| ) | |
| parser.add_argument( | |
| "--init-env", | |
| action="store_true", | |
| help=( | |
| "Create a protected .env template with 0600 permissions" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--oauth-setup", | |
| action="store_true", | |
| help=( | |
| "Create/replace the Google OAuth refresh token" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--port", | |
| type=int, | |
| default=8080, | |
| help=( | |
| "Local OAuth callback port " | |
| "(default: 8080)" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--validate-only", | |
| action="store_true", | |
| help=( | |
| "Authenticate and validate the GSC property only" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--no-submit-sitemap", | |
| action="store_true", | |
| help=( | |
| "Do not submit sitemap to Google" | |
| ), | |
| ) | |
| parser.add_argument( | |
| "--no-inspect", | |
| action="store_true", | |
| help=( | |
| "Do not perform GSC URL inspections" | |
| ), | |
| ) | |
| return parser.parse_args() | |
| # ============================================================ | |
| # ENVIRONMENT CREDENTIALS | |
| # ============================================================ | |
| def get_client_credentials() -> Dict[str, str]: | |
| """ | |
| Read Google OAuth client credentials from environment. | |
| Never logs the actual credentials. | |
| """ | |
| client_id = os.getenv( | |
| "GSC_CLIENT_ID" | |
| ) | |
| client_secret = os.getenv( | |
| "GSC_CLIENT_SECRET" | |
| ) | |
| if not client_id: | |
| raise RuntimeError( | |
| "GSC_CLIENT_ID is not set.\n\n" | |
| "Create/edit .env:\n" | |
| f" nano {ENV_FILE}\n\n" | |
| "Then load it with:\n" | |
| " set -a\n" | |
| f" source {ENV_FILE}\n" | |
| " set +a" | |
| ) | |
| if not client_secret: | |
| raise RuntimeError( | |
| "GSC_CLIENT_SECRET is not set.\n\n" | |
| "Create/edit .env:\n" | |
| f" nano {ENV_FILE}\n\n" | |
| "Then load it with:\n" | |
| " set -a\n" | |
| f" source {ENV_FILE}\n" | |
| " set +a" | |
| ) | |
| return { | |
| "client_id": client_id.strip(), | |
| "client_secret": client_secret.strip(), | |
| } | |
| def get_refresh_token() -> str: | |
| """ | |
| Read the Google OAuth refresh token exclusively from | |
| GSC_REFRESH_TOKEN. | |
| No token.json or legacy token-file fallback is used. | |
| """ | |
| refresh_token = os.getenv( | |
| "GSC_REFRESH_TOKEN" | |
| ) | |
| if not refresh_token: | |
| raise RuntimeError( | |
| "GSC_REFRESH_TOKEN is not set.\n\n" | |
| "If OAuth has already been completed, load .env:\n" | |
| " set -a\n" | |
| f" source {ENV_FILE}\n" | |
| " set +a\n\n" | |
| "Otherwise run:\n" | |
| " python3 gsc_diagnostic.py --oauth-setup" | |
| ) | |
| refresh_token = refresh_token.strip() | |
| if not refresh_token: | |
| raise RuntimeError( | |
| "GSC_REFRESH_TOKEN is empty." | |
| ) | |
| return refresh_token | |
| # ============================================================ | |
| # OAUTH SETUP | |
| # ============================================================ | |
| def oauth_setup( | |
| port: int, | |
| ) -> bool: | |
| """ | |
| Perform Google OAuth authorization. | |
| Uses a Desktop OAuth client and localhost loopback | |
| redirect. | |
| IMPORTANT: | |
| host and redirect URI intentionally both use localhost. | |
| """ | |
| logger.info( | |
| "Starting Google OAuth setup..." | |
| ) | |
| if port < 1024 or port > 65535: | |
| logger.error( | |
| "Invalid OAuth port: %d. " | |
| "Use a port between 1024 and 65535.", | |
| port, | |
| ) | |
| return False | |
| try: | |
| client = get_client_credentials() | |
| except RuntimeError as exc: | |
| logger.error( | |
| "%s", | |
| exc, | |
| ) | |
| return False | |
| redirect_uri = ( | |
| f"http://localhost:{port}/" | |
| ) | |
| client_config = { | |
| "installed": { | |
| "client_id": client["client_id"], | |
| "client_secret": client["client_secret"], | |
| "auth_uri": ( | |
| "https://accounts.google.com/o/oauth2/auth" | |
| ), | |
| "token_uri": ( | |
| "https://oauth2.googleapis.com/token" | |
| ), | |
| "redirect_uris": [ | |
| redirect_uri, | |
| ], | |
| } | |
| } | |
| logger.info( | |
| "OAuth callback: %s", | |
| redirect_uri, | |
| ) | |
| logger.info( | |
| "OAuth scope: %s", | |
| SCOPES[0], | |
| ) | |
| try: | |
| flow = InstalledAppFlow.from_client_config( | |
| client_config, | |
| scopes=SCOPES, | |
| ) | |
| flow.redirect_uri = redirect_uri | |
| except Exception as exc: | |
| logger.error( | |
| "Unable to initialize Google OAuth flow: %s", | |
| exc, | |
| ) | |
| return False | |
| logger.info( | |
| "Starting localhost OAuth callback server..." | |
| ) | |
| logger.info( | |
| "The browser will NOT be opened automatically." | |
| ) | |
| logger.info( | |
| "Open the authorization URL displayed below " | |
| "in your local browser." | |
| ) | |
| try: | |
| credentials = flow.run_local_server( | |
| host="localhost", | |
| port=port, | |
| authorization_prompt_message=( | |
| "\n" | |
| "================================================\n" | |
| "OPEN THIS GOOGLE AUTHORIZATION URL\n" | |
| "================================================\n" | |
| "\n" | |
| "{url}\n" | |
| "\n" | |
| "================================================\n" | |
| ), | |
| success_message=( | |
| "Google authorization completed. " | |
| "You can close this browser tab." | |
| ), | |
| open_browser=False, | |
| access_type="offline", | |
| prompt="consent", | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "OAuth authorization failed: %s", | |
| exc, | |
| ) | |
| logger.error( | |
| "If this is a remote server, verify that " | |
| "the SSH tunnel is active:" | |
| ) | |
| logger.error( | |
| " ssh -L %d:127.0.0.1:%d root@YOUR_SERVER", | |
| port, | |
| port, | |
| ) | |
| logger.error( | |
| "Then run the OAuth setup again." | |
| ) | |
| return False | |
| if not credentials: | |
| logger.error( | |
| "Google OAuth returned no credentials." | |
| ) | |
| return False | |
| if not credentials.refresh_token: | |
| logger.error( | |
| "Google did not return a refresh token." | |
| ) | |
| logger.error( | |
| "Run OAuth setup again with consent." | |
| ) | |
| return False | |
| refresh_token = ( | |
| credentials.refresh_token.strip() | |
| ) | |
| if not refresh_token: | |
| logger.error( | |
| "Google returned an empty refresh token." | |
| ) | |
| return False | |
| # -------------------------------------------------------- | |
| # Save refresh token securely into .env | |
| # -------------------------------------------------------- | |
| try: | |
| update_refresh_token_in_env( | |
| refresh_token | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "Unable to save OAuth refresh token securely: %s", | |
| exc, | |
| ) | |
| return False | |
| # -------------------------------------------------------- | |
| # Update current process environment. | |
| # | |
| # This allows --oauth-setup to immediately authenticate | |
| # without requiring a second shell command. | |
| # -------------------------------------------------------- | |
| os.environ[ | |
| "GSC_REFRESH_TOKEN" | |
| ] = refresh_token | |
| logger.info( | |
| "OAuth refresh token received successfully." | |
| ) | |
| logger.info( | |
| "Refresh token was not printed or logged." | |
| ) | |
| logger.info( | |
| "Protected .env updated successfully." | |
| ) | |
| logger.info( | |
| "OAuth setup completed successfully." | |
| ) | |
| print_env_instructions() | |
| return True | |
| # ============================================================ | |
| # GOOGLE AUTHENTICATION | |
| # ============================================================ | |
| def get_gsc_credentials() -> Credentials: | |
| """ | |
| Build Google OAuth credentials exclusively from: | |
| GSC_CLIENT_ID | |
| GSC_CLIENT_SECRET | |
| GSC_REFRESH_TOKEN | |
| """ | |
| client = get_client_credentials() | |
| refresh_token = get_refresh_token() | |
| credentials = Credentials( | |
| token=None, | |
| refresh_token=refresh_token, | |
| token_uri=( | |
| "https://oauth2.googleapis.com/token" | |
| ), | |
| client_id=client["client_id"], | |
| client_secret=client["client_secret"], | |
| scopes=SCOPES, | |
| ) | |
| try: | |
| credentials.refresh( | |
| Request() | |
| ) | |
| except Exception as exc: | |
| message = str(exc).lower() | |
| if ( | |
| "invalid_grant" in message | |
| or "expired or revoked" in message | |
| or "token has been expired" in message | |
| or "token has been revoked" in message | |
| ): | |
| logger.error( | |
| "The Google OAuth refresh token is " | |
| "expired or revoked." | |
| ) | |
| logger.error( | |
| "Run:" | |
| ) | |
| logger.error( | |
| " python3 gsc_diagnostic.py --oauth-setup" | |
| ) | |
| raise | |
| return credentials | |
| def get_gsc_service(): | |
| credentials = ( | |
| get_gsc_credentials() | |
| ) | |
| return build( | |
| "searchconsole", | |
| "v1", | |
| credentials=credentials, | |
| cache_discovery=False, | |
| ) | |
| # ============================================================ | |
| # GSC PROPERTY VALIDATION | |
| # ============================================================ | |
| def validate_gsc_property( | |
| service, | |
| ) -> Dict[str, Any]: | |
| logger.info( | |
| "Validating GSC property: %s", | |
| SITE_URL, | |
| ) | |
| try: | |
| response = ( | |
| service | |
| .sites() | |
| .get( | |
| siteUrl=SITE_URL | |
| ) | |
| .execute() | |
| ) | |
| permission = response.get( | |
| "permissionLevel" | |
| ) | |
| logger.info( | |
| "GSC property found." | |
| ) | |
| logger.info( | |
| "Property: %s", | |
| SITE_URL, | |
| ) | |
| logger.info( | |
| "Permission: %s", | |
| permission, | |
| ) | |
| return { | |
| "valid": True, | |
| "site_url": SITE_URL, | |
| "permission_level": permission, | |
| "response": response, | |
| "error": None, | |
| } | |
| except HttpError as exc: | |
| status = getattr( | |
| exc.resp, | |
| "status", | |
| None, | |
| ) | |
| logger.error( | |
| "GSC property validation failed." | |
| ) | |
| logger.error( | |
| "HTTP status: %s", | |
| status, | |
| ) | |
| logger.error( | |
| "Google response: %s", | |
| exc, | |
| ) | |
| return { | |
| "valid": False, | |
| "site_url": SITE_URL, | |
| "permission_level": None, | |
| "response": None, | |
| "error": str(exc), | |
| } | |
| except Exception as exc: | |
| logger.error( | |
| "GSC property validation failed: %s", | |
| exc, | |
| ) | |
| return { | |
| "valid": False, | |
| "site_url": SITE_URL, | |
| "permission_level": None, | |
| "response": None, | |
| "error": str(exc), | |
| } | |
| # ============================================================ | |
| # HTTP SESSION | |
| # ============================================================ | |
| def create_http_session() -> requests.Session: | |
| session = requests.Session() | |
| session.headers.update( | |
| HTTP_HEADERS | |
| ) | |
| return session | |
| HTTP = create_http_session() | |
| # ============================================================ | |
| # GENERIC RETRY | |
| # ============================================================ | |
| def retry_call( | |
| func, | |
| retries: int = 4, | |
| base_delay: float = 2.0, | |
| ): | |
| last_error: Optional[Exception] = None | |
| for attempt in range( | |
| retries | |
| ): | |
| try: | |
| return func() | |
| except HttpError as exc: | |
| last_error = exc | |
| status = getattr( | |
| exc.resp, | |
| "status", | |
| None, | |
| ) | |
| if status not in { | |
| 429, | |
| 500, | |
| 502, | |
| 503, | |
| 504, | |
| }: | |
| raise | |
| except ( | |
| requests.exceptions.Timeout, | |
| requests.exceptions.ConnectionError, | |
| ) as exc: | |
| last_error = exc | |
| if attempt < retries - 1: | |
| delay = ( | |
| base_delay | |
| * (2 ** attempt) | |
| ) | |
| logger.warning( | |
| "Transient failure. " | |
| "Retry in %.1fs: %s", | |
| delay, | |
| last_error, | |
| ) | |
| time.sleep( | |
| delay | |
| ) | |
| if last_error is not None: | |
| raise last_error | |
| raise RuntimeError( | |
| "Retry operation failed without an exception." | |
| ) | |
| # ============================================================ | |
| # URL VALIDATION | |
| # ============================================================ | |
| def is_same_site( | |
| url: str, | |
| ) -> bool: | |
| parsed = urlparse( | |
| url | |
| ) | |
| hostname = ( | |
| parsed.hostname or "" | |
| ).lower() | |
| return hostname in { | |
| "schooloffreelancing.com", | |
| "www.schooloffreelancing.com", | |
| } | |
| def normalize_url( | |
| url: str, | |
| ) -> str: | |
| url = url.strip() | |
| if not url: | |
| return url | |
| if url.startswith("/"): | |
| url = urljoin( | |
| BASE_URL, | |
| url, | |
| ) | |
| return url | |
| # ============================================================ | |
| # ROBOTS.TXT | |
| # ============================================================ | |
| def diagnose_robots() -> Dict[str, Any]: | |
| logger.info( | |
| "Checking robots.txt: %s", | |
| ROBOTS_URL, | |
| ) | |
| try: | |
| response = HTTP.get( | |
| ROBOTS_URL, | |
| timeout=TIMEOUT, | |
| allow_redirects=True, | |
| ) | |
| text = response.text | |
| sitemap_lines = [ | |
| line.strip() | |
| for line in text.splitlines() | |
| if line.lower().startswith( | |
| "sitemap:" | |
| ) | |
| ] | |
| return { | |
| "url": ROBOTS_URL, | |
| "status_code": response.status_code, | |
| "final_url": response.url, | |
| "sitemap_directives": sitemap_lines, | |
| "has_disallow_all": bool( | |
| re.search( | |
| r"Disallow:\s*/\s*$", | |
| text, | |
| re.IGNORECASE | |
| | re.MULTILINE, | |
| ) | |
| ), | |
| "ok": ( | |
| response.status_code == 200 | |
| ), | |
| } | |
| except requests.RequestException as exc: | |
| return { | |
| "url": ROBOTS_URL, | |
| "status_code": None, | |
| "final_url": None, | |
| "sitemap_directives": [], | |
| "has_disallow_all": False, | |
| "ok": False, | |
| "error": str(exc), | |
| } | |
| # ============================================================ | |
| # XML SITEMAP | |
| # ============================================================ | |
| def extract_sitemap_urls( | |
| xml: str, | |
| ) -> List[str]: | |
| locations = re.findall( | |
| r"<loc>\s*(.*?)\s*</loc>", | |
| xml, | |
| flags=( | |
| re.IGNORECASE | |
| | re.DOTALL | |
| ), | |
| ) | |
| result = [] | |
| for location in locations: | |
| location = ( | |
| location.strip() | |
| ) | |
| location = ( | |
| location | |
| .replace( | |
| "&", | |
| "&", | |
| ) | |
| ) | |
| if ( | |
| location | |
| and is_same_site( | |
| location | |
| ) | |
| ): | |
| result.append( | |
| location | |
| ) | |
| return list( | |
| dict.fromkeys( | |
| result | |
| ) | |
| ) | |
| def diagnose_sitemap() -> Dict[str, Any]: | |
| logger.info( | |
| "Checking sitemap: %s", | |
| SITEMAP_URL, | |
| ) | |
| try: | |
| response = HTTP.get( | |
| SITEMAP_URL, | |
| timeout=TIMEOUT, | |
| allow_redirects=True, | |
| ) | |
| content = response.text | |
| urls = extract_sitemap_urls( | |
| content | |
| ) | |
| return { | |
| "url": SITEMAP_URL, | |
| "status_code": response.status_code, | |
| "final_url": response.url, | |
| "content_type": response.headers.get( | |
| "Content-Type" | |
| ), | |
| "url_count": len(urls), | |
| "urls": urls[:MAX_URLS], | |
| "ok": ( | |
| response.status_code == 200 | |
| and len(urls) > 0 | |
| ), | |
| } | |
| except requests.RequestException as exc: | |
| return { | |
| "url": SITEMAP_URL, | |
| "status_code": None, | |
| "final_url": None, | |
| "content_type": None, | |
| "url_count": 0, | |
| "urls": [], | |
| "ok": False, | |
| "error": str(exc), | |
| } | |
| # ============================================================ | |
| # HTTP URL DIAGNOSTICS | |
| # ============================================================ | |
| def extract_html_signals( | |
| response: requests.Response, | |
| ): | |
| text = response.text[ | |
| :2_000_000 | |
| ] | |
| canonical = None | |
| robots_meta = None | |
| canonical_match = re.search( | |
| r"<link\b(?=[^>]*\brel\s*=\s*[\"']canonical[\"'])" | |
| r"(?=[^>]*\bhref\s*=\s*[\"']([^\"']+)[\"'])" | |
| r"[^>]*>", | |
| text, | |
| re.IGNORECASE, | |
| ) | |
| if canonical_match: | |
| canonical = ( | |
| canonical_match.group(1) | |
| ) | |
| robots_match = re.search( | |
| r"<meta\b(?=[^>]*\bname\s*=\s*[\"']robots[\"'])" | |
| r"(?=[^>]*\bcontent\s*=\s*[\"']([^\"']+)[\"'])" | |
| r"[^>]*>", | |
| text, | |
| re.IGNORECASE, | |
| ) | |
| if robots_match: | |
| robots_meta = ( | |
| robots_match.group(1) | |
| ) | |
| return ( | |
| canonical, | |
| robots_meta, | |
| ) | |
| def diagnose_url( | |
| url: str, | |
| ) -> HTTPDiagnostic: | |
| start = time.perf_counter() | |
| try: | |
| response = HTTP.get( | |
| url, | |
| timeout=TIMEOUT, | |
| allow_redirects=True, | |
| ) | |
| elapsed = ( | |
| time.perf_counter() | |
| - start | |
| ) * 1000 | |
| canonical = None | |
| robots_meta = None | |
| content_type = ( | |
| response.headers.get( | |
| "Content-Type", | |
| "", | |
| ) | |
| ) | |
| if ( | |
| "text/html" | |
| in content_type.lower() | |
| ): | |
| ( | |
| canonical, | |
| robots_meta, | |
| ) = extract_html_signals( | |
| response | |
| ) | |
| ok = ( | |
| response.status_code | |
| == 200 | |
| ) | |
| return HTTPDiagnostic( | |
| url=url, | |
| status_code=( | |
| response.status_code | |
| ), | |
| final_url=response.url, | |
| response_time_ms=round( | |
| elapsed, | |
| 2, | |
| ), | |
| content_type=content_type, | |
| robots_meta=robots_meta, | |
| canonical=canonical, | |
| error=None, | |
| ok=ok, | |
| ) | |
| except requests.RequestException as exc: | |
| return HTTPDiagnostic( | |
| url=url, | |
| status_code=None, | |
| final_url=None, | |
| response_time_ms=None, | |
| content_type=None, | |
| robots_meta=None, | |
| canonical=None, | |
| error=str(exc), | |
| ok=False, | |
| ) | |
| # ============================================================ | |
| # GSC URL INSPECTION | |
| # ============================================================ | |
| def inspect_url( | |
| service, | |
| url: str, | |
| ) -> GSCInspection: | |
| body = { | |
| "inspectionUrl": url, | |
| "siteUrl": SITE_URL, | |
| } | |
| try: | |
| response = retry_call( | |
| lambda: ( | |
| service | |
| .urlInspection() | |
| .index() | |
| .inspect( | |
| body=body | |
| ) | |
| .execute() | |
| ) | |
| ) | |
| result = response.get( | |
| "inspectionResult", | |
| {}, | |
| ) | |
| index_status = result.get( | |
| "indexStatusResult", | |
| {}, | |
| ) | |
| mobile = result.get( | |
| "mobileUsabilityResult", | |
| {}, | |
| ) | |
| return GSCInspection( | |
| url=url, | |
| verdict=index_status.get( | |
| "verdict" | |
| ), | |
| coverage_state=index_status.get( | |
| "coverageState" | |
| ), | |
| indexing_state=index_status.get( | |
| "indexingState" | |
| ), | |
| robots_txt_state=index_status.get( | |
| "robotsTxtState" | |
| ), | |
| page_fetch_state=index_status.get( | |
| "pageFetchState" | |
| ), | |
| google_canonical=index_status.get( | |
| "googleCanonical" | |
| ), | |
| user_canonical=index_status.get( | |
| "userCanonical" | |
| ), | |
| mobile_usability=mobile.get( | |
| "verdict" | |
| ), | |
| referring_urls=index_status.get( | |
| "referringUrls" | |
| ), | |
| last_crawl_time=index_status.get( | |
| "lastCrawlTime" | |
| ), | |
| error=None, | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "GSC inspection failed for %s: %s", | |
| url, | |
| exc, | |
| ) | |
| return GSCInspection( | |
| url=url, | |
| verdict=None, | |
| coverage_state=None, | |
| indexing_state=None, | |
| robots_txt_state=None, | |
| page_fetch_state=None, | |
| google_canonical=None, | |
| user_canonical=None, | |
| mobile_usability=None, | |
| referring_urls=None, | |
| last_crawl_time=None, | |
| error=str(exc), | |
| ) | |
| # ============================================================ | |
| # SITEMAP SUBMISSION | |
| # ============================================================ | |
| def submit_sitemap( | |
| service, | |
| ) -> Dict[str, Any]: | |
| logger.info( | |
| "Submitting sitemap to Google Search Console..." | |
| ) | |
| try: | |
| response = retry_call( | |
| lambda: ( | |
| service | |
| .sitemaps() | |
| .submit( | |
| siteUrl=SITE_URL, | |
| feedpath=SITEMAP_URL, | |
| ) | |
| .execute() | |
| ) | |
| ) | |
| logger.info( | |
| "Sitemap submission completed." | |
| ) | |
| return { | |
| "submitted": True, | |
| "skipped": False, | |
| "sitemap": SITEMAP_URL, | |
| "response": response, | |
| "error": None, | |
| } | |
| except Exception as exc: | |
| logger.error( | |
| "Sitemap submission failed: %s", | |
| exc, | |
| ) | |
| return { | |
| "submitted": False, | |
| "skipped": False, | |
| "sitemap": SITEMAP_URL, | |
| "response": None, | |
| "error": str(exc), | |
| } | |
| # ============================================================ | |
| # SEARCH ANALYTICS | |
| # ============================================================ | |
| def query_search_analytics( | |
| service, | |
| dimensions: List[str], | |
| start_date: str, | |
| end_date: str, | |
| row_limit: int = 25000, | |
| ) -> List[Dict[str, Any]]: | |
| body = { | |
| "startDate": start_date, | |
| "endDate": end_date, | |
| "dimensions": dimensions, | |
| "type": "web", | |
| "aggregationType": "auto", | |
| "rowLimit": min( | |
| row_limit, | |
| 25000, | |
| ), | |
| "dataState": "final", | |
| } | |
| try: | |
| response = retry_call( | |
| lambda: ( | |
| service | |
| .searchanalytics() | |
| .query( | |
| siteUrl=SITE_URL, | |
| body=body, | |
| ) | |
| .execute() | |
| ) | |
| ) | |
| rows = response.get( | |
| "rows", | |
| [], | |
| ) | |
| normalized = [] | |
| for row in rows: | |
| normalized.append( | |
| { | |
| "keys": row.get( | |
| "keys", | |
| [], | |
| ), | |
| "clicks": row.get( | |
| "clicks", | |
| 0, | |
| ), | |
| "impressions": row.get( | |
| "impressions", | |
| 0, | |
| ), | |
| "ctr": row.get( | |
| "ctr", | |
| 0, | |
| ), | |
| "position": row.get( | |
| "position", | |
| 0, | |
| ), | |
| } | |
| ) | |
| return normalized | |
| except Exception as exc: | |
| logger.error( | |
| "Search Analytics query failed: %s", | |
| exc, | |
| ) | |
| return [] | |
| def build_search_analytics_report( | |
| service, | |
| ) -> Dict[str, Any]: | |
| end_date = ( | |
| date.today() | |
| - timedelta(days=2) | |
| ) | |
| start_date = ( | |
| end_date | |
| - timedelta( | |
| days=DAYS - 1 | |
| ) | |
| ) | |
| start = ( | |
| start_date.isoformat() | |
| ) | |
| end = ( | |
| end_date.isoformat() | |
| ) | |
| logger.info( | |
| "Search Analytics period: %s -> %s", | |
| start, | |
| end, | |
| ) | |
| return { | |
| "period": { | |
| "start": start, | |
| "end": end, | |
| }, | |
| "pages": query_search_analytics( | |
| service, | |
| ["page"], | |
| start, | |
| end, | |
| ), | |
| "queries": query_search_analytics( | |
| service, | |
| ["query"], | |
| start, | |
| end, | |
| ), | |
| "countries": query_search_analytics( | |
| service, | |
| ["country"], | |
| start, | |
| end, | |
| ), | |
| "devices": query_search_analytics( | |
| service, | |
| ["device"], | |
| start, | |
| end, | |
| ), | |
| "search_appearances": query_search_analytics( | |
| service, | |
| ["searchAppearance"], | |
| start, | |
| end, | |
| ), | |
| } | |
| # ============================================================ | |
| # PAGE ACTION LIST | |
| # | |
| # Cross-references HTTP diagnostics, GSC inspection results, | |
| # and on-page signals (canonical / robots meta) to build a | |
| # concrete list of pages that need changes, with the specific | |
| # issue and the recommended fix for each. | |
| # ============================================================ | |
| def _add_issue( | |
| issues: List[str], | |
| recommendations: List[str], | |
| issue: str, | |
| recommendation: str, | |
| ): | |
| issues.append(issue) | |
| recommendations.append(recommendation) | |
| def analyze_page( | |
| http_result: Optional[HTTPDiagnostic], | |
| gsc_result: Optional[GSCInspection], | |
| ) -> Optional[PageAction]: | |
| """ | |
| Inspect a single URL's HTTP + GSC signals and return a | |
| PageAction describing what to fix, or None if the page | |
| looks healthy. | |
| """ | |
| url = ( | |
| http_result.url | |
| if http_result | |
| else ( | |
| gsc_result.url | |
| if gsc_result | |
| else None | |
| ) | |
| ) | |
| if not url: | |
| return None | |
| issues: List[str] = [] | |
| recommendations: List[str] = [] | |
| severity = "info" | |
| # -------------------------------------------------- | |
| # HTTP-level problems | |
| # -------------------------------------------------- | |
| if http_result: | |
| if http_result.error: | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"URL could not be fetched: {http_result.error}", | |
| "Confirm the page is reachable, DNS resolves " | |
| "correctly, and no firewall/WAF is blocking " | |
| "Googlebot or automated requests.", | |
| ) | |
| severity = "critical" | |
| elif http_result.status_code and http_result.status_code >= 500: | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Server error (HTTP {http_result.status_code}).", | |
| "Investigate server logs; fix the underlying " | |
| "server error so the page returns HTTP 200.", | |
| ) | |
| severity = "critical" | |
| elif http_result.status_code and http_result.status_code == 404: | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| "Page returns HTTP 404 (Not Found).", | |
| "Restore the page, 301-redirect it to the correct " | |
| "replacement URL, or remove the URL from the " | |
| "sitemap if it should no longer exist.", | |
| ) | |
| severity = "critical" | |
| elif http_result.status_code and http_result.status_code in (301, 302, 307, 308): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Sitemap URL redirects (HTTP {http_result.status_code}) " | |
| f"to {http_result.final_url}.", | |
| "Update the sitemap to list the final destination " | |
| "URL directly instead of a redirecting URL.", | |
| ) | |
| severity = "warning" | |
| elif http_result.status_code and http_result.status_code not in (200,): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Unexpected HTTP status: {http_result.status_code}.", | |
| "Review the response and ensure the page returns " | |
| "HTTP 200 for a normal, indexable page.", | |
| ) | |
| severity = "warning" | |
| if ( | |
| http_result.robots_meta | |
| and re.search( | |
| r"noindex", | |
| http_result.robots_meta, | |
| re.IGNORECASE, | |
| ) | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Page has a robots meta noindex directive " | |
| f"({http_result.robots_meta!r}).", | |
| "Remove the 'noindex' directive from the " | |
| "<meta name=\"robots\"> tag if this page should " | |
| "be indexed, or remove it from the sitemap if " | |
| "it should stay out of the index.", | |
| ) | |
| severity = "critical" | |
| if ( | |
| http_result.canonical | |
| and http_result.final_url | |
| and normalize_url( | |
| http_result.canonical | |
| ) | |
| != normalize_url( | |
| http_result.final_url | |
| ) | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Canonical tag points to a different URL " | |
| f"({http_result.canonical}) than the page itself.", | |
| "Either update the canonical tag to self-reference " | |
| "this URL, or if the canonical target is correct, " | |
| "list that canonical URL in the sitemap instead " | |
| "and 301-redirect this URL to it.", | |
| ) | |
| if severity == "info": | |
| severity = "warning" | |
| # -------------------------------------------------- | |
| # GSC-level problems | |
| # -------------------------------------------------- | |
| if gsc_result: | |
| if gsc_result.error: | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"GSC URL Inspection failed: {gsc_result.error}", | |
| "Retry the inspection later; if it persists, " | |
| "verify GSC property permissions and API quota.", | |
| ) | |
| if severity == "info": | |
| severity = "warning" | |
| else: | |
| if gsc_result.verdict and gsc_result.verdict != "PASS": | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Google indexing verdict is '{gsc_result.verdict}' " | |
| f"(coverage: {gsc_result.coverage_state}).", | |
| "Review the coverage state in Search Console, " | |
| "resolve the blocking issue, then request " | |
| "indexing again once fixed.", | |
| ) | |
| severity = "critical" | |
| if ( | |
| gsc_result.robots_txt_state | |
| and gsc_result.robots_txt_state.upper() | |
| not in ("ALLOWED",) | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"robots.txt state is " | |
| f"'{gsc_result.robots_txt_state}'.", | |
| "Update robots.txt to allow crawling of this " | |
| "URL path if it should be indexed.", | |
| ) | |
| severity = "critical" | |
| if ( | |
| gsc_result.page_fetch_state | |
| and gsc_result.page_fetch_state.upper() | |
| not in ("SUCCESSFUL",) | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Google could not fetch the page " | |
| f"(state: {gsc_result.page_fetch_state}).", | |
| "Check server availability and response time " | |
| "for Googlebot; ensure no server errors or " | |
| "timeouts occur on this URL.", | |
| ) | |
| severity = "critical" | |
| if ( | |
| gsc_result.google_canonical | |
| and gsc_result.user_canonical | |
| and normalize_url( | |
| gsc_result.google_canonical | |
| ) | |
| != normalize_url( | |
| gsc_result.user_canonical | |
| ) | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| "Google selected a different canonical " | |
| f"({gsc_result.google_canonical}) than the " | |
| f"one declared on the page " | |
| f"({gsc_result.user_canonical}).", | |
| "Consolidate duplicate/similar content, add " | |
| "internal links pointing to the preferred " | |
| "URL, and ensure the declared canonical " | |
| "matches the URL Google should treat as " | |
| "canonical.", | |
| ) | |
| if severity == "info": | |
| severity = "warning" | |
| if ( | |
| gsc_result.mobile_usability | |
| and gsc_result.mobile_usability != "PASS" | |
| ): | |
| _add_issue( | |
| issues, | |
| recommendations, | |
| f"Mobile usability verdict: " | |
| f"{gsc_result.mobile_usability}.", | |
| "Fix mobile usability issues (viewport " | |
| "configuration, text size, tap-target " | |
| "spacing) reported in Search Console.", | |
| ) | |
| if severity == "info": | |
| severity = "warning" | |
| if not issues: | |
| return None | |
| return PageAction( | |
| url=url, | |
| severity=severity, | |
| issues=issues, | |
| recommendations=recommendations, | |
| ) | |
| def build_page_action_list( | |
| http_results: List[HTTPDiagnostic], | |
| gsc_results: List[GSCInspection], | |
| ) -> List[Dict[str, Any]]: | |
| """ | |
| Build a de-duplicated, per-URL list of pages that need | |
| changes to align with Google Search Console, each with | |
| the specific issues found and recommended fixes. | |
| """ | |
| gsc_by_url: Dict[str, GSCInspection] = { | |
| item.url: item | |
| for item in gsc_results | |
| } | |
| http_by_url: Dict[str, HTTPDiagnostic] = { | |
| item.url: item | |
| for item in http_results | |
| } | |
| all_urls = list( | |
| dict.fromkeys( | |
| list(http_by_url.keys()) | |
| + list(gsc_by_url.keys()) | |
| ) | |
| ) | |
| actions: List[PageAction] = [] | |
| for url in all_urls: | |
| page_action = analyze_page( | |
| http_by_url.get(url), | |
| gsc_by_url.get(url), | |
| ) | |
| if page_action: | |
| actions.append(page_action) | |
| severity_rank = { | |
| "critical": 0, | |
| "warning": 1, | |
| "info": 2, | |
| } | |
| actions.sort( | |
| key=lambda item: ( | |
| severity_rank.get( | |
| item.severity, | |
| 3, | |
| ), | |
| item.url, | |
| ) | |
| ) | |
| logger.info( | |
| "Pages needing changes: %d " | |
| "(critical=%d, warning=%d, info=%d)", | |
| len(actions), | |
| sum( | |
| 1 | |
| for a in actions | |
| if a.severity == "critical" | |
| ), | |
| sum( | |
| 1 | |
| for a in actions | |
| if a.severity == "warning" | |
| ), | |
| sum( | |
| 1 | |
| for a in actions | |
| if a.severity == "info" | |
| ), | |
| ) | |
| return [ | |
| asdict(item) | |
| for item in actions | |
| ] | |
| # ============================================================ | |
| # DIAGNOSTIC SCORING | |
| # ============================================================ | |
| def calculate_summary( | |
| http_results: List[HTTPDiagnostic], | |
| gsc_results: List[GSCInspection], | |
| robots: Dict[str, Any], | |
| sitemap: Dict[str, Any], | |
| page_actions: List[Dict[str, Any]], | |
| ) -> Dict[str, Any]: | |
| summary = { | |
| "urls_checked": len( | |
| http_results | |
| ), | |
| "http_ok": sum( | |
| 1 | |
| for x in http_results | |
| if x.ok | |
| ), | |
| "http_failed": sum( | |
| 1 | |
| for x in http_results | |
| if not x.ok | |
| ), | |
| "gsc_indexed": 0, | |
| "gsc_not_indexed": 0, | |
| "gsc_errors": 0, | |
| "robots_ok": robots.get( | |
| "ok", | |
| False, | |
| ), | |
| "sitemap_ok": sitemap.get( | |
| "ok", | |
| False, | |
| ), | |
| "sitemap_url_count": sitemap.get( | |
| "url_count", | |
| 0, | |
| ), | |
| "pages_needing_action": len( | |
| page_actions | |
| ), | |
| "pages_critical": sum( | |
| 1 | |
| for a in page_actions | |
| if a.get("severity") == "critical" | |
| ), | |
| "pages_warning": sum( | |
| 1 | |
| for a in page_actions | |
| if a.get("severity") == "warning" | |
| ), | |
| } | |
| for result in gsc_results: | |
| if result.error: | |
| summary[ | |
| "gsc_errors" | |
| ] += 1 | |
| elif result.verdict == "PASS": | |
| summary[ | |
| "gsc_indexed" | |
| ] += 1 | |
| else: | |
| summary[ | |
| "gsc_not_indexed" | |
| ] += 1 | |
| return summary | |
| # ============================================================ | |
| # CSV EXPORT | |
| # ============================================================ | |
| def write_csv( | |
| path: Path, | |
| rows: Iterable[Dict[str, Any]], | |
| ): | |
| rows = list(rows) | |
| if not rows: | |
| return | |
| fieldnames = sorted( | |
| { | |
| key | |
| for row in rows | |
| for key in row.keys() | |
| } | |
| ) | |
| with path.open( | |
| "w", | |
| newline="", | |
| encoding="utf-8", | |
| ) as file: | |
| writer = csv.DictWriter( | |
| file, | |
| fieldnames=fieldnames, | |
| extrasaction="ignore", | |
| ) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow( | |
| row | |
| ) | |
| # ============================================================ | |
| # HTML REPORT | |
| # ============================================================ | |
| def _escape( | |
| value: Any, | |
| ) -> str: | |
| return html.escape( | |
| str( | |
| value | |
| if value is not None | |
| else "" | |
| ) | |
| ) | |
| def _severity_badge( | |
| severity: str, | |
| ) -> str: | |
| colors = { | |
| "critical": "#b91c1c", | |
| "warning": "#b45309", | |
| "info": "#374151", | |
| } | |
| color = colors.get( | |
| severity, | |
| "#374151", | |
| ) | |
| return ( | |
| f'<span style="display:inline-block;padding:2px 8px;' | |
| f'border-radius:10px;color:#fff;background:{color};' | |
| f'font-size:12px;font-weight:600;text-transform:uppercase;">' | |
| f"{_escape(severity)}</span>" | |
| ) | |
| def _bool_badge( | |
| value: bool, | |
| ) -> str: | |
| if value: | |
| return ( | |
| '<span style="color:#15803d;font-weight:600;">OK</span>' | |
| ) | |
| return ( | |
| '<span style="color:#b91c1c;font-weight:600;">FAIL</span>' | |
| ) | |
| def render_html_report( | |
| report: Dict[str, Any], | |
| ) -> str: | |
| """ | |
| Render a single self-contained HTML report summarizing | |
| the diagnostic run, including the page action list. | |
| """ | |
| summary = report.get( | |
| "summary", | |
| {}, | |
| ) | |
| property_validation = report.get( | |
| "gsc_property_validation", | |
| {}, | |
| ) | |
| robots = report.get( | |
| "robots", | |
| {}, | |
| ) | |
| sitemap = report.get( | |
| "sitemap_diagnostic", | |
| {}, | |
| ) | |
| sitemap_submission = report.get( | |
| "sitemap_submission", | |
| {}, | |
| ) | |
| page_actions = report.get( | |
| "page_actions", | |
| [], | |
| ) | |
| analytics = report.get( | |
| "search_analytics", | |
| {}, | |
| ) | |
| top_pages = analytics.get( | |
| "pages", | |
| [], | |
| )[:20] | |
| top_queries = analytics.get( | |
| "queries", | |
| [], | |
| )[:20] | |
| # --- summary cards ------------------------------------------------- | |
| summary_cards = "".join( | |
| f'<div class="card"><div class="card-label">{_escape(label)}</div>' | |
| f'<div class="card-value">{_escape(value)}</div></div>' | |
| for label, value in [ | |
| ("URLs checked", summary.get("urls_checked", 0)), | |
| ("HTTP OK", summary.get("http_ok", 0)), | |
| ("HTTP failed", summary.get("http_failed", 0)), | |
| ("GSC indexed (PASS)", summary.get("gsc_indexed", 0)), | |
| ("GSC not indexed", summary.get("gsc_not_indexed", 0)), | |
| ("Pages needing action", summary.get("pages_needing_action", 0)), | |
| ("Critical issues", summary.get("pages_critical", 0)), | |
| ("Warnings", summary.get("pages_warning", 0)), | |
| ] | |
| ) | |
| # --- page action rows ------------------------------------------------ | |
| if page_actions: | |
| action_rows = [] | |
| for item in page_actions: | |
| issues_html = "".join( | |
| f"<li>{_escape(issue)}</li>" | |
| for issue in item.get("issues", []) | |
| ) | |
| recs_html = "".join( | |
| f"<li>{_escape(rec)}</li>" | |
| for rec in item.get("recommendations", []) | |
| ) | |
| action_rows.append( | |
| "<tr>" | |
| f'<td>{_severity_badge(item.get("severity", "info"))}</td>' | |
| f'<td class="url-cell"><a href="{_escape(item.get("url"))}" ' | |
| f'target="_blank" rel="noopener">{_escape(item.get("url"))}</a></td>' | |
| f'<td><ul class="issue-list">{issues_html}</ul></td>' | |
| f'<td><ul class="issue-list">{recs_html}</ul></td>' | |
| "</tr>" | |
| ) | |
| action_table = ( | |
| '<table class="report-table">' | |
| "<thead><tr>" | |
| "<th>Severity</th><th>Page</th>" | |
| "<th>Issues found</th><th>Recommended fix</th>" | |
| "</tr></thead><tbody>" | |
| + "".join(action_rows) | |
| + "</tbody></table>" | |
| ) | |
| else: | |
| action_table = ( | |
| '<p class="ok-message">No pages currently require ' | |
| "changes to align with Google Search Console.</p>" | |
| ) | |
| # --- top pages table ------------------------------------------------- | |
| def analytics_table(rows, key_label): | |
| if not rows: | |
| return "<p>No data available for this period.</p>" | |
| body_rows = "".join( | |
| "<tr>" | |
| f'<td>{_escape(", ".join(row.get("keys", [])))}</td>' | |
| f'<td>{_escape(row.get("clicks", 0))}</td>' | |
| f'<td>{_escape(row.get("impressions", 0))}</td>' | |
| f'<td>{_escape(round(row.get("ctr", 0) * 100, 2))}%</td>' | |
| f'<td>{_escape(round(row.get("position", 0), 1))}</td>' | |
| "</tr>" | |
| for row in rows | |
| ) | |
| return ( | |
| '<table class="report-table">' | |
| f"<thead><tr><th>{key_label}</th><th>Clicks</th>" | |
| "<th>Impressions</th><th>CTR</th><th>Avg. position</th>" | |
| "</tr></thead><tbody>" | |
| + body_rows | |
| + "</tbody></table>" | |
| ) | |
| top_pages_table = analytics_table( | |
| top_pages, | |
| "Page", | |
| ) | |
| top_queries_table = analytics_table( | |
| top_queries, | |
| "Query", | |
| ) | |
| period = analytics.get( | |
| "period", | |
| {}, | |
| ) | |
| html_doc = f"""<!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <title>GSC Diagnostic Report - {_escape(report.get('site'))}</title> | |
| <style> | |
| body {{ | |
| font-family: -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif; | |
| background: #f7f7f8; | |
| color: #111827; | |
| margin: 0; | |
| padding: 32px; | |
| }} | |
| h1 {{ font-size: 22px; margin-bottom: 4px; }} | |
| h2 {{ font-size: 17px; margin-top: 40px; border-bottom: 1px solid #e5e7eb; padding-bottom: 6px; }} | |
| .meta {{ color: #6b7280; font-size: 13px; margin-bottom: 24px; }} | |
| .cards {{ display: flex; flex-wrap: wrap; gap: 12px; margin-top: 16px; }} | |
| .card {{ | |
| background: #fff; border: 1px solid #e5e7eb; border-radius: 8px; | |
| padding: 14px 18px; min-width: 140px; flex: 1 1 140px; | |
| }} | |
| .card-label {{ font-size: 12px; color: #6b7280; text-transform: uppercase; letter-spacing: 0.03em; }} | |
| .card-value {{ font-size: 24px; font-weight: 700; margin-top: 4px; }} | |
| table.report-table {{ width: 100%; border-collapse: collapse; margin-top: 12px; background: #fff; }} | |
| table.report-table th, table.report-table td {{ | |
| border: 1px solid #e5e7eb; padding: 8px 10px; font-size: 13px; text-align: left; vertical-align: top; | |
| }} | |
| table.report-table th {{ background: #f3f4f6; }} | |
| .url-cell {{ max-width: 320px; word-break: break-all; }} | |
| ul.issue-list {{ margin: 0; padding-left: 18px; }} | |
| ul.issue-list li {{ margin-bottom: 4px; }} | |
| .ok-message {{ color: #15803d; font-weight: 600; }} | |
| .status-line {{ margin: 4px 0; }} | |
| </style> | |
| </head> | |
| <body> | |
| <h1>Google Search Console Diagnostic Report</h1> | |
| <div class="meta"> | |
| Site: {_escape(report.get('site'))} · | |
| Property: {_escape(report.get('gsc_property'))} · | |
| Generated: {_escape(report.get('generated_at'))} | |
| </div> | |
| <div class="status-line">Property valid: {_bool_badge(property_validation.get('valid', False))} | |
| | Permission: {_escape(property_validation.get('permission_level'))}</div> | |
| <div class="status-line">robots.txt: {_bool_badge(robots.get('ok', False))} | |
| ({_escape(robots.get('status_code'))}) | | |
| sitemap: {_bool_badge(sitemap.get('ok', False))} | |
| ({_escape(sitemap.get('url_count', 0))} URLs)</div> | |
| <div class="status-line">Sitemap submission: {_bool_badge(sitemap_submission.get('submitted', False))} | |
| {"(" + _escape(sitemap_submission.get('reason')) + ")" if sitemap_submission.get('reason') else ""}</div> | |
| <div class="cards">{summary_cards}</div> | |
| <h2>Pages that need changes</h2> | |
| {action_table} | |
| <h2>Top pages (Search Analytics{" · " + _escape(period.get('start')) + " to " + _escape(period.get('end')) if period else ""})</h2> | |
| {top_pages_table} | |
| <h2>Top queries</h2> | |
| {top_queries_table} | |
| </body> | |
| </html> | |
| """ | |
| return html_doc | |
| # ============================================================ | |
| # REPORT GENERATION | |
| # ============================================================ | |
| def write_reports( | |
| report: Dict[str, Any], | |
| ): | |
| OUTPUT_DIR.mkdir( | |
| parents=True, | |
| exist_ok=True, | |
| ) | |
| timestamp = time.strftime( | |
| "%Y%m%d-%H%M%S" | |
| ) | |
| json_path = ( | |
| OUTPUT_DIR | |
| / f"gsc-diagnostic-{timestamp}.json" | |
| ) | |
| with json_path.open( | |
| "w", | |
| encoding="utf-8", | |
| ) as file: | |
| json.dump( | |
| report, | |
| file, | |
| indent=2, | |
| ensure_ascii=False, | |
| ) | |
| html_path = ( | |
| OUTPUT_DIR | |
| / f"gsc-diagnostic-{timestamp}.html" | |
| ) | |
| try: | |
| html_content = render_html_report( | |
| report | |
| ) | |
| with html_path.open( | |
| "w", | |
| encoding="utf-8", | |
| ) as file: | |
| file.write( | |
| html_content | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "Unable to write HTML report: %s", | |
| exc, | |
| ) | |
| html_path = None | |
| http_rows = report.get( | |
| "http_diagnostics", | |
| [], | |
| ) | |
| gsc_rows = report.get( | |
| "gsc_inspections", | |
| [], | |
| ) | |
| page_action_rows = report.get( | |
| "page_actions", | |
| [], | |
| ) | |
| analytics = report.get( | |
| "search_analytics", | |
| {}, | |
| ) | |
| write_csv( | |
| OUTPUT_DIR | |
| / f"http-diagnostics-{timestamp}.csv", | |
| http_rows, | |
| ) | |
| write_csv( | |
| OUTPUT_DIR | |
| / f"gsc-inspections-{timestamp}.csv", | |
| gsc_rows, | |
| ) | |
| write_csv( | |
| OUTPUT_DIR | |
| / f"page-actions-{timestamp}.csv", | |
| page_action_rows, | |
| ) | |
| for name, rows in analytics.items(): | |
| if name == "period": | |
| continue | |
| write_csv( | |
| OUTPUT_DIR | |
| / f"search-{name}-{timestamp}.csv", | |
| rows, | |
| ) | |
| logger.info( | |
| "Reports written to: %s", | |
| OUTPUT_DIR, | |
| ) | |
| logger.info( | |
| "JSON report: %s", | |
| json_path, | |
| ) | |
| if html_path: | |
| logger.info( | |
| "HTML report: %s", | |
| html_path, | |
| ) | |
| return json_path, html_path | |
| # ============================================================ | |
| # MAIN | |
| # ============================================================ | |
| def main() -> int: | |
| args = parse_args() | |
| logger.info( | |
| "================================================" | |
| ) | |
| logger.info( | |
| "School of Freelancing GSC Diagnostic" | |
| ) | |
| logger.info( | |
| "Property: %s", | |
| SITE_URL, | |
| ) | |
| logger.info( | |
| "Environment: %s", | |
| ENV_FILE, | |
| ) | |
| logger.info( | |
| "================================================" | |
| ) | |
| # -------------------------------------------------------- | |
| # Initialize protected .env | |
| # -------------------------------------------------------- | |
| if args.init_env: | |
| success = initialize_env_file() | |
| if not success: | |
| return 2 | |
| # If --init-env was supplied by itself, stop here. | |
| if not args.oauth_setup: | |
| return 0 | |
| # -------------------------------------------------------- | |
| # Verify .env when present | |
| # -------------------------------------------------------- | |
| if ENV_FILE.exists(): | |
| try: | |
| secure_env_file( | |
| ENV_FILE | |
| ) | |
| except Exception as exc: | |
| logger.error( | |
| "Protected .env validation failed: %s", | |
| exc, | |
| ) | |
| return 2 | |
| # -------------------------------------------------------- | |
| # Validate configuration | |
| # -------------------------------------------------------- | |
| try: | |
| validate_runtime_configuration() | |
| except Exception as exc: | |
| logger.error( | |
| "Configuration error: %s", | |
| exc, | |
| ) | |
| return 2 | |
| # -------------------------------------------------------- | |
| # OAuth setup | |
| # -------------------------------------------------------- | |
| if args.oauth_setup: | |
| success = oauth_setup( | |
| args.port | |
| ) | |
| if not success: | |
| return 2 | |
| logger.info( | |
| "Testing the newly created refresh token..." | |
| ) | |
| # -------------------------------------------------------- | |
| # Authentication | |
| # -------------------------------------------------------- | |
| try: | |
| service = get_gsc_service() | |
| except Exception as exc: | |
| logger.error( | |
| "Google authentication failed: %s", | |
| exc, | |
| ) | |
| return 2 | |
| # -------------------------------------------------------- | |
| # Validate GSC property | |
| # -------------------------------------------------------- | |
| property_result = ( | |
| validate_gsc_property( | |
| service | |
| ) | |
| ) | |
| if not property_result[ | |
| "valid" | |
| ]: | |
| logger.error( | |
| "The authenticated Google account " | |
| "does not have access to:" | |
| ) | |
| logger.error( | |
| " %s", | |
| SITE_URL, | |
| ) | |
| logger.error( | |
| "Grant the Google account access to " | |
| "the domain property in Search Console." | |
| ) | |
| return 2 | |
| # -------------------------------------------------------- | |
| # Validate-only | |
| # -------------------------------------------------------- | |
| if args.validate_only: | |
| logger.info( | |
| "GSC authentication and property validation succeeded." | |
| ) | |
| logger.info( | |
| "Permission level: %s", | |
| property_result[ | |
| "permission_level" | |
| ], | |
| ) | |
| return 0 | |
| # -------------------------------------------------------- | |
| # robots.txt | |
| # -------------------------------------------------------- | |
| robots = diagnose_robots() | |
| logger.info( | |
| "robots.txt status: %s", | |
| robots.get( | |
| "status_code" | |
| ), | |
| ) | |
| if robots.get( | |
| "has_disallow_all" | |
| ): | |
| logger.warning( | |
| "robots.txt appears to contain Disallow: /" | |
| ) | |
| # -------------------------------------------------------- | |
| # sitemap | |
| # -------------------------------------------------------- | |
| sitemap = diagnose_sitemap() | |
| logger.info( | |
| "Sitemap status: %s | URLs discovered: %s", | |
| sitemap.get( | |
| "status_code" | |
| ), | |
| sitemap.get( | |
| "url_count" | |
| ), | |
| ) | |
| urls = sitemap.get( | |
| "urls", | |
| [], | |
| ) | |
| # -------------------------------------------------------- | |
| # Sitemap submission | |
| # -------------------------------------------------------- | |
| sitemap_submission = { | |
| "submitted": False, | |
| "skipped": True, | |
| "reason": None, | |
| } | |
| submit_sitemap_enabled = ( | |
| SUBMIT_SITEMAP | |
| and not args.no_submit_sitemap | |
| ) | |
| if ( | |
| submit_sitemap_enabled | |
| and sitemap.get("ok") | |
| ): | |
| sitemap_submission = ( | |
| submit_sitemap( | |
| service | |
| ) | |
| ) | |
| elif args.no_submit_sitemap: | |
| sitemap_submission[ | |
| "reason" | |
| ] = "disabled by --no-submit-sitemap" | |
| elif not SUBMIT_SITEMAP: | |
| sitemap_submission[ | |
| "reason" | |
| ] = "GSC_SUBMIT_SITEMAP=false" | |
| elif not sitemap.get("ok"): | |
| sitemap_submission[ | |
| "reason" | |
| ] = "sitemap diagnostic failed" | |
| # -------------------------------------------------------- | |
| # URL diagnostics | |
| # -------------------------------------------------------- | |
| http_results: List[ | |
| HTTPDiagnostic | |
| ] = [] | |
| gsc_results: List[ | |
| GSCInspection | |
| ] = [] | |
| inspect_enabled = ( | |
| INSPECT_SITEMAP_URLS | |
| and not args.no_inspect | |
| ) | |
| if inspect_enabled: | |
| urls = urls[ | |
| :MAX_URLS | |
| ] | |
| logger.info( | |
| "Inspecting %d sitemap URLs...", | |
| len(urls), | |
| ) | |
| for index, url in enumerate( | |
| urls, | |
| start=1, | |
| ): | |
| url = normalize_url( | |
| url | |
| ) | |
| logger.info( | |
| "[%d/%d] %s", | |
| index, | |
| len(urls), | |
| url, | |
| ) | |
| if not is_same_site( | |
| url | |
| ): | |
| logger.warning( | |
| "Skipping external URL: %s", | |
| url, | |
| ) | |
| continue | |
| http_result = ( | |
| diagnose_url( | |
| url | |
| ) | |
| ) | |
| http_results.append( | |
| http_result | |
| ) | |
| if http_result.ok: | |
| gsc_result = ( | |
| inspect_url( | |
| service, | |
| url, | |
| ) | |
| ) | |
| gsc_results.append( | |
| gsc_result | |
| ) | |
| else: | |
| logger.warning( | |
| "Skipping GSC inspection " | |
| "because HTTP check failed: %s", | |
| url, | |
| ) | |
| else: | |
| logger.info( | |
| "GSC URL inspection disabled." | |
| ) | |
| # -------------------------------------------------------- | |
| # Search Analytics | |
| # -------------------------------------------------------- | |
| logger.info( | |
| "Collecting Search Analytics..." | |
| ) | |
| analytics = ( | |
| build_search_analytics_report( | |
| service | |
| ) | |
| ) | |
| # -------------------------------------------------------- | |
| # Page action list (what needs to change, and why) | |
| # -------------------------------------------------------- | |
| page_actions = build_page_action_list( | |
| http_results, | |
| gsc_results, | |
| ) | |
| # -------------------------------------------------------- | |
| # Summary | |
| # -------------------------------------------------------- | |
| summary = calculate_summary( | |
| http_results, | |
| gsc_results, | |
| robots, | |
| sitemap, | |
| page_actions, | |
| ) | |
| # -------------------------------------------------------- | |
| # Final report | |
| # -------------------------------------------------------- | |
| report = { | |
| "generated_at": time.strftime( | |
| "%Y-%m-%dT%H:%M:%S%z" | |
| ), | |
| "site": BASE_URL, | |
| "gsc_property": SITE_URL, | |
| "gsc_property_validation": ( | |
| property_result | |
| ), | |
| "sitemap": SITEMAP_URL, | |
| "robots": robots, | |
| "sitemap_diagnostic": sitemap, | |
| "sitemap_submission": ( | |
| sitemap_submission | |
| ), | |
| "summary": summary, | |
| "http_diagnostics": [ | |
| asdict(item) | |
| for item in http_results | |
| ], | |
| "gsc_inspections": [ | |
| asdict(item) | |
| for item in gsc_results | |
| ], | |
| "page_actions": page_actions, | |
| "search_analytics": analytics, | |
| } | |
| write_reports( | |
| report | |
| ) | |
| # -------------------------------------------------------- | |
| # Final console summary | |
| # -------------------------------------------------------- | |
| logger.info("") | |
| logger.info( | |
| "================ FINAL SUMMARY ================" | |
| ) | |
| logger.info( | |
| "GSC property valid: %s", | |
| property_result[ | |
| "valid" | |
| ], | |
| ) | |
| logger.info( | |
| "GSC permission: %s", | |
| property_result.get( | |
| "permission_level" | |
| ), | |
| ) | |
| logger.info( | |
| "URLs checked: %d", | |
| summary[ | |
| "urls_checked" | |
| ], | |
| ) | |
| logger.info( | |
| "HTTP OK: %d", | |
| summary[ | |
| "http_ok" | |
| ], | |
| ) | |
| logger.info( | |
| "HTTP failed: %d", | |
| summary[ | |
| "http_failed" | |
| ], | |
| ) | |
| logger.info( | |
| "GSC PASS: %d", | |
| summary[ | |
| "gsc_indexed" | |
| ], | |
| ) | |
| logger.info( | |
| "GSC not PASS: %d", | |
| summary[ | |
| "gsc_not_indexed" | |
| ], | |
| ) | |
| logger.info( | |
| "GSC errors: %d", | |
| summary[ | |
| "gsc_errors" | |
| ], | |
| ) | |
| logger.info( | |
| "robots.txt OK: %s", | |
| summary[ | |
| "robots_ok" | |
| ], | |
| ) | |
| logger.info( | |
| "sitemap OK: %s", | |
| summary[ | |
| "sitemap_ok" | |
| ], | |
| ) | |
| logger.info( | |
| "sitemap URLs: %d", | |
| summary[ | |
| "sitemap_url_count" | |
| ], | |
| ) | |
| logger.info( | |
| "Pages needing fix: %d (critical=%d, warning=%d)", | |
| summary[ | |
| "pages_needing_action" | |
| ], | |
| summary[ | |
| "pages_critical" | |
| ], | |
| summary[ | |
| "pages_warning" | |
| ], | |
| ) | |
| logger.info( | |
| "================================================" | |
| ) | |
| # -------------------------------------------------------- | |
| # Exit status | |
| # -------------------------------------------------------- | |
| if not summary[ | |
| "robots_ok" | |
| ]: | |
| return 1 | |
| if not summary[ | |
| "sitemap_ok" | |
| ]: | |
| return 1 | |
| if summary[ | |
| "http_failed" | |
| ] > 0: | |
| return 1 | |
| if summary[ | |
| "pages_critical" | |
| ] > 0: | |
| return 1 | |
| return 0 | |
| # ============================================================ | |
| # ENTRY POINT | |
| # ============================================================ | |
| if __name__ == "__main__": | |
| try: | |
| sys.exit( | |
| main() | |
| ) | |
| except KeyboardInterrupt: | |
| logger.warning( | |
| "Interrupted by user." | |
| ) | |
| sys.exit(130) | |
| except Exception as exc: | |
| logger.exception( | |
| "Unexpected fatal error: %s", | |
| exc, | |
| ) | |
| sys.exit(3) |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
cd script
python3 -m venv .venv
source .venv/bin/activate
gedit gsc_diagnostic.py
chmod +x gsc_diagnostic.py
pip install requests google-api-python-client google-auth-httplib2 google-auth-oauthlib
pip install --upgrade pip
pip install requests google-auth google-api-python-client
sed -i '1{/^```python$/d;}' gsc_diagnostic.py
sed -i '${/^```$/d;}' gsc_diagnostic.py
python3 -m py_compile gsc_diagnostic.py
head -n 3 gsc_diagnostic.py
tail -n 3 gsc_diagnostic.py
nano .env
GSC_CLIENT_ID=YOUR_CLIENT_ID.apps.googleusercontent.com
GSC_CLIENT_SECRET=YOUR_CLIENT_SECRET
GSC_REFRESH_TOKEN=YOUR_REFRESH_TOKEN
GSC_DAYS=28
GSC_MAX_URLS=250
GSC_TIMEOUT=20
GSC_OUTPUT_DIR='/home/allah/Downloads/gsc-diagnostic/reports'
GSC_SUBMIT_SITEMAP=true
GSC_INSPECT_SITEMAP_URLS=true
ctrl+s and ctrl+x
chmod 600 .env
set -a
source .env
set +a
test -n "$GSC_CLIENT_ID" && echo "GSC_CLIENT_ID: SET" || echo "GSC_CLIENT_ID: MISSING"
test -n "$GSC_CLIENT_SECRET" && echo "GSC_CLIENT_SECRET: SET" || echo "GSC_CLIENT_SECRET: MISSING"
test -n "$GSC_REFRESH_TOKEN" && echo "GSC_REFRESH_TOKEN: SET" || echo "GSC_REFRESH_TOKEN: MISSING"
python3 gsc_diagnostic.py --init-env
python3 gsc_diagnostic.py --validate-only
python3 gsc_diagnostic.py --oauth-setup --port 8080