Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save AnythingLinux/e480a57f4b94e878fabc8f44d43ee527 to your computer and use it in GitHub Desktop.

Select an option

Save AnythingLinux/e480a57f4b94e878fabc8f44d43ee527 to your computer and use it in GitHub Desktop.
Google Search Console Diagnostic script
#!/usr/bin/env python3
"""
Production Google Search Console Diagnostic
============================================
Target:
https://schooloffreelancing.com
GSC property:
sc-domain:schooloffreelancing.com
Features:
- Google OAuth 2.0 authorization-code setup
- Protected local .env credential/configuration file
- Automatic .env loading without python-dotenv
- .env permissions enforced as 0600
- GSC property validation
- HTTPS / HTTP diagnostics
- robots.txt diagnostics
- sitemap diagnostics
- XML sitemap URL extraction
- GSC URL Inspection
- canonical/indexability checks
- Search Analytics performance report
- sitemap submission
- Action list: pages that need changes to align with GSC, with
specific per-page issues and recommended fixes
- JSON and HTML reports
- retries for transient failures
- structured logging
- configurable URL limits
Recommended directory:
/home/YOUR_USER/gsc-diagnostic/
gsc_diagnostic.py
.env
reports/
Environment file:
.env
Example:
GSC_CLIENT_ID=YOUR_CLIENT_ID.apps.googleusercontent.com
GSC_CLIENT_SECRET=YOUR_CLIENT_SECRET
GSC_REFRESH_TOKEN=YOUR_REFRESH_TOKEN
GSC_DAYS=28
GSC_MAX_URLS=250
GSC_TIMEOUT=20
GSC_OUTPUT_DIR=/home/YOUR_USER/gsc-diagnostic/reports
GSC_SUBMIT_SITEMAP=true
GSC_INSPECT_SITEMAP_URLS=true
Secure permissions:
chmod 600 .env
Recommended shell workflow:
set -a
source .env
set +a
python3 gsc_diagnostic.py
Create a secure .env template:
python3 gsc_diagnostic.py --init-env
OAuth setup:
python3 gsc_diagnostic.py --oauth-setup
For a remote Linux server, use an SSH tunnel:
ssh -L 8080:127.0.0.1:8080 root@YOUR_SERVER
Then, on the server:
python3 gsc_diagnostic.py --oauth-setup --port 8080
Open the displayed Google authorization URL in your local browser.
The OAuth refresh token is written into:
.env
with permissions:
0600
Recommended Google Cloud OAuth client:
Application type:
Desktop app
Required OAuth scope:
https://www.googleapis.com/auth/webmasters
"""
from __future__ import annotations
import argparse
import ast
import csv
import html
import json
import logging
import os
import re
import shlex
import stat
import sys
import time
from dataclasses import asdict, dataclass, field
from datetime import date, timedelta
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
from urllib.parse import urljoin, urlparse
import requests
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
# ============================================================
# PATHS
# ============================================================
SCRIPT_DIR = Path(
__file__
).resolve().parent
ENV_FILE = SCRIPT_DIR / ".env"
DEFAULT_OUTPUT_DIR = (
SCRIPT_DIR / "reports"
)
# ============================================================
# CONFIGURATION
# ============================================================
SITE_URL = "sc-domain:schooloffreelancing.com"
BASE_URL = "https://schooloffreelancing.com"
SITEMAP_URL = f"{BASE_URL}/sitemap.xml"
ROBOTS_URL = f"{BASE_URL}/robots.txt"
SCOPES = [
"https://www.googleapis.com/auth/webmasters",
]
USER_AGENT = (
"SchoolOfFreelancing-GSC-Diagnostic/5.0 "
"(Google Search Console diagnostic)"
)
HTTP_HEADERS = {
"User-Agent": USER_AGENT,
"Accept": "*/*",
}
# ============================================================
# LOGGING
# ============================================================
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s | %(levelname)s | %(message)s",
)
logger = logging.getLogger(
"gsc-diagnostic"
)
# ============================================================
# .ENV SECURITY
# ============================================================
ENV_ALLOWED_KEYS = {
"GSC_CLIENT_ID",
"GSC_CLIENT_SECRET",
"GSC_REFRESH_TOKEN",
"GSC_DAYS",
"GSC_MAX_URLS",
"GSC_TIMEOUT",
"GSC_OUTPUT_DIR",
"GSC_SUBMIT_SITEMAP",
"GSC_INSPECT_SITEMAP_URLS",
}
def secure_file_owner_only(
path: Path,
) -> None:
"""
Set a file to owner read/write only.
"""
os.chmod(
path,
stat.S_IRUSR | stat.S_IWUSR,
)
def secure_env_file(
path: Path,
) -> None:
"""
Validate and enforce secure .env permissions.
Requirements:
- must exist
- must not be a symlink
- must be owned by current user
- must have 0600 permissions
"""
if path.is_symlink():
raise RuntimeError(
f"Refusing to use symlinked environment file: {path}"
)
if not path.exists():
raise RuntimeError(
f"Environment file does not exist: {path}"
)
if not path.is_file():
raise RuntimeError(
f"Environment path is not a regular file: {path}"
)
stat_result = path.stat()
if stat_result.st_uid != os.geteuid():
raise RuntimeError(
f"Environment file is not owned by the current user: {path}"
)
mode = stat_result.st_mode & 0o777
if mode != 0o600:
logger.warning(
".env permissions are %04o; changing to 0600.",
mode,
)
secure_file_owner_only(
path
)
final_mode = (
path.stat().st_mode
& 0o777
)
if final_mode != 0o600:
raise RuntimeError(
f"Unable to enforce 0600 permissions on {path}"
)
def parse_env_value(
raw_value: str,
) -> str:
"""
Parse a simple .env value without executing shell code.
Supports:
KEY=value
KEY="value"
KEY='value'
Comments are supported for unquoted values.
"""
value = raw_value.strip()
if not value:
return ""
try:
# Handle single/double quoted values.
if (
len(value) >= 2
and value[0] == value[-1]
and value[0] in {"'", '"'}
):
try:
parsed = ast.literal_eval(
value
)
if isinstance(
parsed,
str,
):
return parsed
except Exception:
pass
# shlex handles escaped characters and comments
# without executing anything.
parts = shlex.split(
value,
comments=True,
posix=True,
)
if parts:
return parts[0]
except ValueError:
pass
return value
def read_env_file(
path: Path,
) -> Dict[str, str]:
"""
Safely read a simple .env file.
This function NEVER executes the file.
"""
secure_env_file(
path
)
values: Dict[str, str] = {}
with path.open(
"r",
encoding="utf-8",
) as file:
for line_number, line in enumerate(
file,
start=1,
):
stripped = line.strip()
if not stripped:
continue
if stripped.startswith("#"):
continue
# Accept optional "export KEY=value"
if stripped.startswith(
"export "
):
stripped = stripped[
len("export "):
].lstrip()
match = re.match(
r"^([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)$",
stripped,
)
if not match:
raise RuntimeError(
f"Invalid .env syntax at "
f"{path}:{line_number}"
)
key = match.group(1)
raw_value = match.group(2)
if key not in ENV_ALLOWED_KEYS:
logger.warning(
"Ignoring unsupported .env variable: %s",
key,
)
continue
values[key] = parse_env_value(
raw_value
)
return values
def load_dotenv(
path: Path = ENV_FILE,
) -> None:
"""
Load .env values into the current Python process.
Existing process environment variables take precedence.
Therefore:
export GSC_TIMEOUT=30
overrides:
GSC_TIMEOUT=20
in .env.
"""
if not path.exists():
logger.warning(
"Environment file does not exist: %s",
path,
)
logger.warning(
"Create it with:"
)
logger.warning(
" python3 %s --init-env",
Path(__file__).name,
)
return
try:
values = read_env_file(
path
)
except Exception as exc:
raise RuntimeError(
f"Unable to load protected .env: {exc}"
) from exc
for key, value in values.items():
if key not in os.environ:
os.environ[
key
] = value
def shell_quote_single(
value: str,
) -> str:
"""
Safely quote a value for a POSIX shell.
"""
return "'" + value.replace(
"'",
"'\"'\"'",
) + "'"
def write_env_file(
values: Dict[str, str],
) -> None:
"""
Write the complete .env securely.
File permissions are set to 0600 before the file
becomes visible at the final path.
"""
if ENV_FILE.is_symlink():
raise RuntimeError(
f"Refusing to overwrite symlinked .env: {ENV_FILE}"
)
ENV_FILE.parent.mkdir(
parents=True,
exist_ok=True,
)
temporary = ENV_FILE.with_name(
".env.tmp"
)
content_lines = [
"# School of Freelancing Google Search Console Diagnostic",
"# Protected configuration and OAuth credentials",
"# File permissions must remain 0600",
"",
]
for key in [
"GSC_CLIENT_ID",
"GSC_CLIENT_SECRET",
"GSC_REFRESH_TOKEN",
"GSC_DAYS",
"GSC_MAX_URLS",
"GSC_TIMEOUT",
"GSC_OUTPUT_DIR",
"GSC_SUBMIT_SITEMAP",
"GSC_INSPECT_SITEMAP_URLS",
]:
value = values.get(
key,
"",
)
content_lines.append(
f"{key}={shell_quote_single(value)}"
)
content = (
"\n".join(
content_lines
)
+ "\n"
)
flags = (
os.O_WRONLY
| os.O_CREAT
| os.O_TRUNC
)
fd = os.open(
temporary,
flags,
0o600,
)
try:
with os.fdopen(
fd,
"w",
encoding="utf-8",
) as file:
file.write(
content
)
except Exception:
try:
temporary.unlink()
except OSError:
pass
raise
secure_file_owner_only(
temporary
)
temporary.replace(
ENV_FILE
)
secure_file_owner_only(
ENV_FILE
)
secure_env_file(
ENV_FILE
)
def initialize_env_file() -> bool:
"""
Create a protected .env template.
Existing .env is never overwritten.
"""
if ENV_FILE.exists():
logger.info(
".env already exists: %s",
ENV_FILE,
)
try:
secure_env_file(
ENV_FILE
)
except Exception as exc:
logger.error(
"Existing .env security check failed: %s",
exc,
)
return False
logger.info(
".env permissions verified as 0600."
)
return True
values = {
"GSC_CLIENT_ID": (
"YOUR_CLIENT_ID.apps.googleusercontent.com"
),
"GSC_CLIENT_SECRET": (
"YOUR_CLIENT_SECRET"
),
"GSC_REFRESH_TOKEN": (
"YOUR_REFRESH_TOKEN"
),
"GSC_DAYS": "28",
"GSC_MAX_URLS": "250",
"GSC_TIMEOUT": "20",
"GSC_OUTPUT_DIR": str(
DEFAULT_OUTPUT_DIR
),
"GSC_SUBMIT_SITEMAP": "true",
"GSC_INSPECT_SITEMAP_URLS": "true",
}
try:
write_env_file(
values
)
except Exception as exc:
logger.error(
"Unable to create protected .env: %s",
exc,
)
return False
logger.info(
"Protected .env created:"
)
logger.info(
" %s",
ENV_FILE,
)
logger.info(
"Permissions: 0600"
)
logger.info(
""
)
logger.info(
"Edit it with:"
)
logger.info(
" nano %s",
ENV_FILE,
)
logger.info(
""
)
logger.info(
"Then load it:"
)
logger.info(
" set -a"
)
logger.info(
" source %s",
ENV_FILE,
)
logger.info(
" set +a"
)
return True
def update_refresh_token_in_env(
refresh_token: str,
) -> None:
"""
Update only GSC_REFRESH_TOKEN while preserving all
other .env configuration.
"""
if not refresh_token.strip():
raise RuntimeError(
"Cannot save an empty OAuth refresh token."
)
if ENV_FILE.exists():
existing = read_env_file(
ENV_FILE
)
else:
existing = {
"GSC_CLIENT_ID": os.getenv(
"GSC_CLIENT_ID",
"",
),
"GSC_CLIENT_SECRET": os.getenv(
"GSC_CLIENT_SECRET",
"",
),
"GSC_DAYS": os.getenv(
"GSC_DAYS",
"28",
),
"GSC_MAX_URLS": os.getenv(
"GSC_MAX_URLS",
"250",
),
"GSC_TIMEOUT": os.getenv(
"GSC_TIMEOUT",
"20",
),
"GSC_OUTPUT_DIR": os.getenv(
"GSC_OUTPUT_DIR",
str(DEFAULT_OUTPUT_DIR),
),
"GSC_SUBMIT_SITEMAP": os.getenv(
"GSC_SUBMIT_SITEMAP",
"true",
),
"GSC_INSPECT_SITEMAP_URLS": os.getenv(
"GSC_INSPECT_SITEMAP_URLS",
"true",
),
}
existing[
"GSC_REFRESH_TOKEN"
] = refresh_token.strip()
# Ensure client credentials from the process are
# preserved if they were not already in .env.
for key in [
"GSC_CLIENT_ID",
"GSC_CLIENT_SECRET",
]:
if not existing.get(key):
environment_value = os.getenv(
key
)
if environment_value:
existing[
key
] = environment_value
write_env_file(
existing
)
def print_env_instructions() -> None:
"""
Print safe instructions without exposing credentials.
"""
logger.info("")
logger.info(
"================================================"
)
logger.info(
"Protected environment file:"
)
logger.info(
" %s",
ENV_FILE,
)
logger.info(
"Permissions: 0600"
)
logger.info("")
logger.info(
"Load it into the current shell:"
)
logger.info(
" set -a"
)
logger.info(
" source %s",
ENV_FILE,
)
logger.info(
" set +a"
)
logger.info("")
logger.info(
"Verify without displaying credentials:"
)
logger.info(
" test -n \"$GSC_CLIENT_ID\" "
"&& echo 'GSC_CLIENT_ID: SET' "
"|| echo 'GSC_CLIENT_ID: MISSING'"
)
logger.info(
" test -n \"$GSC_CLIENT_SECRET\" "
"&& echo 'GSC_CLIENT_SECRET: SET' "
"|| echo 'GSC_CLIENT_SECRET: MISSING'"
)
logger.info(
" test -n \"$GSC_REFRESH_TOKEN\" "
"&& echo 'GSC_REFRESH_TOKEN: SET' "
"|| echo 'GSC_REFRESH_TOKEN: MISSING'"
)
logger.info(
"================================================"
)
# ============================================================
# LOAD .ENV BEFORE CONFIGURATION
# ============================================================
load_dotenv(
ENV_FILE
)
# ============================================================
# RUNTIME CONFIGURATION
# ============================================================
TIMEOUT = int(
os.getenv(
"GSC_TIMEOUT",
"20",
)
)
MAX_URLS = int(
os.getenv(
"GSC_MAX_URLS",
"250",
)
)
DAYS = int(
os.getenv(
"GSC_DAYS",
"28",
)
)
OUTPUT_DIR = Path(
os.getenv(
"GSC_OUTPUT_DIR",
str(DEFAULT_OUTPUT_DIR),
)
).expanduser()
SUBMIT_SITEMAP = (
os.getenv(
"GSC_SUBMIT_SITEMAP",
"true",
).strip().lower()
in {
"1",
"true",
"yes",
"on",
}
)
INSPECT_SITEMAP_URLS = (
os.getenv(
"GSC_INSPECT_SITEMAP_URLS",
"true",
).strip().lower()
in {
"1",
"true",
"yes",
"on",
}
)
# ============================================================
# VALIDATE CONFIGURATION
# ============================================================
def validate_runtime_configuration() -> None:
if TIMEOUT <= 0:
raise RuntimeError(
"GSC_TIMEOUT must be greater than 0."
)
if MAX_URLS <= 0:
raise RuntimeError(
"GSC_MAX_URLS must be greater than 0."
)
if DAYS <= 0:
raise RuntimeError(
"GSC_DAYS must be greater than 0."
)
# ============================================================
# DATA STRUCTURES
# ============================================================
@dataclass
class HTTPDiagnostic:
url: str
status_code: Optional[int]
final_url: Optional[str]
response_time_ms: Optional[float]
content_type: Optional[str]
robots_meta: Optional[str]
canonical: Optional[str]
error: Optional[str]
ok: bool
@dataclass
class GSCInspection:
url: str
verdict: Optional[str]
coverage_state: Optional[str]
indexing_state: Optional[str]
robots_txt_state: Optional[str]
page_fetch_state: Optional[str]
google_canonical: Optional[str]
user_canonical: Optional[str]
mobile_usability: Optional[str]
referring_urls: Optional[List[str]]
last_crawl_time: Optional[str]
error: Optional[str]
@dataclass
class PageAction:
"""
A single URL that needs attention, with the specific
issues detected and the recommended fix for each.
"""
url: str
severity: str # "critical" | "warning" | "info"
issues: List[str] = field(default_factory=list)
recommendations: List[str] = field(default_factory=list)
# ============================================================
# COMMAND-LINE ARGUMENTS
# ============================================================
def parse_args():
parser = argparse.ArgumentParser(
description=(
"School of Freelancing Google Search Console Diagnostic"
)
)
parser.add_argument(
"--init-env",
action="store_true",
help=(
"Create a protected .env template with 0600 permissions"
),
)
parser.add_argument(
"--oauth-setup",
action="store_true",
help=(
"Create/replace the Google OAuth refresh token"
),
)
parser.add_argument(
"--port",
type=int,
default=8080,
help=(
"Local OAuth callback port "
"(default: 8080)"
),
)
parser.add_argument(
"--validate-only",
action="store_true",
help=(
"Authenticate and validate the GSC property only"
),
)
parser.add_argument(
"--no-submit-sitemap",
action="store_true",
help=(
"Do not submit sitemap to Google"
),
)
parser.add_argument(
"--no-inspect",
action="store_true",
help=(
"Do not perform GSC URL inspections"
),
)
return parser.parse_args()
# ============================================================
# ENVIRONMENT CREDENTIALS
# ============================================================
def get_client_credentials() -> Dict[str, str]:
"""
Read Google OAuth client credentials from environment.
Never logs the actual credentials.
"""
client_id = os.getenv(
"GSC_CLIENT_ID"
)
client_secret = os.getenv(
"GSC_CLIENT_SECRET"
)
if not client_id:
raise RuntimeError(
"GSC_CLIENT_ID is not set.\n\n"
"Create/edit .env:\n"
f" nano {ENV_FILE}\n\n"
"Then load it with:\n"
" set -a\n"
f" source {ENV_FILE}\n"
" set +a"
)
if not client_secret:
raise RuntimeError(
"GSC_CLIENT_SECRET is not set.\n\n"
"Create/edit .env:\n"
f" nano {ENV_FILE}\n\n"
"Then load it with:\n"
" set -a\n"
f" source {ENV_FILE}\n"
" set +a"
)
return {
"client_id": client_id.strip(),
"client_secret": client_secret.strip(),
}
def get_refresh_token() -> str:
"""
Read the Google OAuth refresh token exclusively from
GSC_REFRESH_TOKEN.
No token.json or legacy token-file fallback is used.
"""
refresh_token = os.getenv(
"GSC_REFRESH_TOKEN"
)
if not refresh_token:
raise RuntimeError(
"GSC_REFRESH_TOKEN is not set.\n\n"
"If OAuth has already been completed, load .env:\n"
" set -a\n"
f" source {ENV_FILE}\n"
" set +a\n\n"
"Otherwise run:\n"
" python3 gsc_diagnostic.py --oauth-setup"
)
refresh_token = refresh_token.strip()
if not refresh_token:
raise RuntimeError(
"GSC_REFRESH_TOKEN is empty."
)
return refresh_token
# ============================================================
# OAUTH SETUP
# ============================================================
def oauth_setup(
port: int,
) -> bool:
"""
Perform Google OAuth authorization.
Uses a Desktop OAuth client and localhost loopback
redirect.
IMPORTANT:
host and redirect URI intentionally both use localhost.
"""
logger.info(
"Starting Google OAuth setup..."
)
if port < 1024 or port > 65535:
logger.error(
"Invalid OAuth port: %d. "
"Use a port between 1024 and 65535.",
port,
)
return False
try:
client = get_client_credentials()
except RuntimeError as exc:
logger.error(
"%s",
exc,
)
return False
redirect_uri = (
f"http://localhost:{port}/"
)
client_config = {
"installed": {
"client_id": client["client_id"],
"client_secret": client["client_secret"],
"auth_uri": (
"https://accounts.google.com/o/oauth2/auth"
),
"token_uri": (
"https://oauth2.googleapis.com/token"
),
"redirect_uris": [
redirect_uri,
],
}
}
logger.info(
"OAuth callback: %s",
redirect_uri,
)
logger.info(
"OAuth scope: %s",
SCOPES[0],
)
try:
flow = InstalledAppFlow.from_client_config(
client_config,
scopes=SCOPES,
)
flow.redirect_uri = redirect_uri
except Exception as exc:
logger.error(
"Unable to initialize Google OAuth flow: %s",
exc,
)
return False
logger.info(
"Starting localhost OAuth callback server..."
)
logger.info(
"The browser will NOT be opened automatically."
)
logger.info(
"Open the authorization URL displayed below "
"in your local browser."
)
try:
credentials = flow.run_local_server(
host="localhost",
port=port,
authorization_prompt_message=(
"\n"
"================================================\n"
"OPEN THIS GOOGLE AUTHORIZATION URL\n"
"================================================\n"
"\n"
"{url}\n"
"\n"
"================================================\n"
),
success_message=(
"Google authorization completed. "
"You can close this browser tab."
),
open_browser=False,
access_type="offline",
prompt="consent",
)
except Exception as exc:
logger.error(
"OAuth authorization failed: %s",
exc,
)
logger.error(
"If this is a remote server, verify that "
"the SSH tunnel is active:"
)
logger.error(
" ssh -L %d:127.0.0.1:%d root@YOUR_SERVER",
port,
port,
)
logger.error(
"Then run the OAuth setup again."
)
return False
if not credentials:
logger.error(
"Google OAuth returned no credentials."
)
return False
if not credentials.refresh_token:
logger.error(
"Google did not return a refresh token."
)
logger.error(
"Run OAuth setup again with consent."
)
return False
refresh_token = (
credentials.refresh_token.strip()
)
if not refresh_token:
logger.error(
"Google returned an empty refresh token."
)
return False
# --------------------------------------------------------
# Save refresh token securely into .env
# --------------------------------------------------------
try:
update_refresh_token_in_env(
refresh_token
)
except Exception as exc:
logger.error(
"Unable to save OAuth refresh token securely: %s",
exc,
)
return False
# --------------------------------------------------------
# Update current process environment.
#
# This allows --oauth-setup to immediately authenticate
# without requiring a second shell command.
# --------------------------------------------------------
os.environ[
"GSC_REFRESH_TOKEN"
] = refresh_token
logger.info(
"OAuth refresh token received successfully."
)
logger.info(
"Refresh token was not printed or logged."
)
logger.info(
"Protected .env updated successfully."
)
logger.info(
"OAuth setup completed successfully."
)
print_env_instructions()
return True
# ============================================================
# GOOGLE AUTHENTICATION
# ============================================================
def get_gsc_credentials() -> Credentials:
"""
Build Google OAuth credentials exclusively from:
GSC_CLIENT_ID
GSC_CLIENT_SECRET
GSC_REFRESH_TOKEN
"""
client = get_client_credentials()
refresh_token = get_refresh_token()
credentials = Credentials(
token=None,
refresh_token=refresh_token,
token_uri=(
"https://oauth2.googleapis.com/token"
),
client_id=client["client_id"],
client_secret=client["client_secret"],
scopes=SCOPES,
)
try:
credentials.refresh(
Request()
)
except Exception as exc:
message = str(exc).lower()
if (
"invalid_grant" in message
or "expired or revoked" in message
or "token has been expired" in message
or "token has been revoked" in message
):
logger.error(
"The Google OAuth refresh token is "
"expired or revoked."
)
logger.error(
"Run:"
)
logger.error(
" python3 gsc_diagnostic.py --oauth-setup"
)
raise
return credentials
def get_gsc_service():
credentials = (
get_gsc_credentials()
)
return build(
"searchconsole",
"v1",
credentials=credentials,
cache_discovery=False,
)
# ============================================================
# GSC PROPERTY VALIDATION
# ============================================================
def validate_gsc_property(
service,
) -> Dict[str, Any]:
logger.info(
"Validating GSC property: %s",
SITE_URL,
)
try:
response = (
service
.sites()
.get(
siteUrl=SITE_URL
)
.execute()
)
permission = response.get(
"permissionLevel"
)
logger.info(
"GSC property found."
)
logger.info(
"Property: %s",
SITE_URL,
)
logger.info(
"Permission: %s",
permission,
)
return {
"valid": True,
"site_url": SITE_URL,
"permission_level": permission,
"response": response,
"error": None,
}
except HttpError as exc:
status = getattr(
exc.resp,
"status",
None,
)
logger.error(
"GSC property validation failed."
)
logger.error(
"HTTP status: %s",
status,
)
logger.error(
"Google response: %s",
exc,
)
return {
"valid": False,
"site_url": SITE_URL,
"permission_level": None,
"response": None,
"error": str(exc),
}
except Exception as exc:
logger.error(
"GSC property validation failed: %s",
exc,
)
return {
"valid": False,
"site_url": SITE_URL,
"permission_level": None,
"response": None,
"error": str(exc),
}
# ============================================================
# HTTP SESSION
# ============================================================
def create_http_session() -> requests.Session:
session = requests.Session()
session.headers.update(
HTTP_HEADERS
)
return session
HTTP = create_http_session()
# ============================================================
# GENERIC RETRY
# ============================================================
def retry_call(
func,
retries: int = 4,
base_delay: float = 2.0,
):
last_error: Optional[Exception] = None
for attempt in range(
retries
):
try:
return func()
except HttpError as exc:
last_error = exc
status = getattr(
exc.resp,
"status",
None,
)
if status not in {
429,
500,
502,
503,
504,
}:
raise
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
) as exc:
last_error = exc
if attempt < retries - 1:
delay = (
base_delay
* (2 ** attempt)
)
logger.warning(
"Transient failure. "
"Retry in %.1fs: %s",
delay,
last_error,
)
time.sleep(
delay
)
if last_error is not None:
raise last_error
raise RuntimeError(
"Retry operation failed without an exception."
)
# ============================================================
# URL VALIDATION
# ============================================================
def is_same_site(
url: str,
) -> bool:
parsed = urlparse(
url
)
hostname = (
parsed.hostname or ""
).lower()
return hostname in {
"schooloffreelancing.com",
"www.schooloffreelancing.com",
}
def normalize_url(
url: str,
) -> str:
url = url.strip()
if not url:
return url
if url.startswith("/"):
url = urljoin(
BASE_URL,
url,
)
return url
# ============================================================
# ROBOTS.TXT
# ============================================================
def diagnose_robots() -> Dict[str, Any]:
logger.info(
"Checking robots.txt: %s",
ROBOTS_URL,
)
try:
response = HTTP.get(
ROBOTS_URL,
timeout=TIMEOUT,
allow_redirects=True,
)
text = response.text
sitemap_lines = [
line.strip()
for line in text.splitlines()
if line.lower().startswith(
"sitemap:"
)
]
return {
"url": ROBOTS_URL,
"status_code": response.status_code,
"final_url": response.url,
"sitemap_directives": sitemap_lines,
"has_disallow_all": bool(
re.search(
r"Disallow:\s*/\s*$",
text,
re.IGNORECASE
| re.MULTILINE,
)
),
"ok": (
response.status_code == 200
),
}
except requests.RequestException as exc:
return {
"url": ROBOTS_URL,
"status_code": None,
"final_url": None,
"sitemap_directives": [],
"has_disallow_all": False,
"ok": False,
"error": str(exc),
}
# ============================================================
# XML SITEMAP
# ============================================================
def extract_sitemap_urls(
xml: str,
) -> List[str]:
locations = re.findall(
r"<loc>\s*(.*?)\s*</loc>",
xml,
flags=(
re.IGNORECASE
| re.DOTALL
),
)
result = []
for location in locations:
location = (
location.strip()
)
location = (
location
.replace(
"&amp;",
"&",
)
)
if (
location
and is_same_site(
location
)
):
result.append(
location
)
return list(
dict.fromkeys(
result
)
)
def diagnose_sitemap() -> Dict[str, Any]:
logger.info(
"Checking sitemap: %s",
SITEMAP_URL,
)
try:
response = HTTP.get(
SITEMAP_URL,
timeout=TIMEOUT,
allow_redirects=True,
)
content = response.text
urls = extract_sitemap_urls(
content
)
return {
"url": SITEMAP_URL,
"status_code": response.status_code,
"final_url": response.url,
"content_type": response.headers.get(
"Content-Type"
),
"url_count": len(urls),
"urls": urls[:MAX_URLS],
"ok": (
response.status_code == 200
and len(urls) > 0
),
}
except requests.RequestException as exc:
return {
"url": SITEMAP_URL,
"status_code": None,
"final_url": None,
"content_type": None,
"url_count": 0,
"urls": [],
"ok": False,
"error": str(exc),
}
# ============================================================
# HTTP URL DIAGNOSTICS
# ============================================================
def extract_html_signals(
response: requests.Response,
):
text = response.text[
:2_000_000
]
canonical = None
robots_meta = None
canonical_match = re.search(
r"<link\b(?=[^>]*\brel\s*=\s*[\"']canonical[\"'])"
r"(?=[^>]*\bhref\s*=\s*[\"']([^\"']+)[\"'])"
r"[^>]*>",
text,
re.IGNORECASE,
)
if canonical_match:
canonical = (
canonical_match.group(1)
)
robots_match = re.search(
r"<meta\b(?=[^>]*\bname\s*=\s*[\"']robots[\"'])"
r"(?=[^>]*\bcontent\s*=\s*[\"']([^\"']+)[\"'])"
r"[^>]*>",
text,
re.IGNORECASE,
)
if robots_match:
robots_meta = (
robots_match.group(1)
)
return (
canonical,
robots_meta,
)
def diagnose_url(
url: str,
) -> HTTPDiagnostic:
start = time.perf_counter()
try:
response = HTTP.get(
url,
timeout=TIMEOUT,
allow_redirects=True,
)
elapsed = (
time.perf_counter()
- start
) * 1000
canonical = None
robots_meta = None
content_type = (
response.headers.get(
"Content-Type",
"",
)
)
if (
"text/html"
in content_type.lower()
):
(
canonical,
robots_meta,
) = extract_html_signals(
response
)
ok = (
response.status_code
== 200
)
return HTTPDiagnostic(
url=url,
status_code=(
response.status_code
),
final_url=response.url,
response_time_ms=round(
elapsed,
2,
),
content_type=content_type,
robots_meta=robots_meta,
canonical=canonical,
error=None,
ok=ok,
)
except requests.RequestException as exc:
return HTTPDiagnostic(
url=url,
status_code=None,
final_url=None,
response_time_ms=None,
content_type=None,
robots_meta=None,
canonical=None,
error=str(exc),
ok=False,
)
# ============================================================
# GSC URL INSPECTION
# ============================================================
def inspect_url(
service,
url: str,
) -> GSCInspection:
body = {
"inspectionUrl": url,
"siteUrl": SITE_URL,
}
try:
response = retry_call(
lambda: (
service
.urlInspection()
.index()
.inspect(
body=body
)
.execute()
)
)
result = response.get(
"inspectionResult",
{},
)
index_status = result.get(
"indexStatusResult",
{},
)
mobile = result.get(
"mobileUsabilityResult",
{},
)
return GSCInspection(
url=url,
verdict=index_status.get(
"verdict"
),
coverage_state=index_status.get(
"coverageState"
),
indexing_state=index_status.get(
"indexingState"
),
robots_txt_state=index_status.get(
"robotsTxtState"
),
page_fetch_state=index_status.get(
"pageFetchState"
),
google_canonical=index_status.get(
"googleCanonical"
),
user_canonical=index_status.get(
"userCanonical"
),
mobile_usability=mobile.get(
"verdict"
),
referring_urls=index_status.get(
"referringUrls"
),
last_crawl_time=index_status.get(
"lastCrawlTime"
),
error=None,
)
except Exception as exc:
logger.error(
"GSC inspection failed for %s: %s",
url,
exc,
)
return GSCInspection(
url=url,
verdict=None,
coverage_state=None,
indexing_state=None,
robots_txt_state=None,
page_fetch_state=None,
google_canonical=None,
user_canonical=None,
mobile_usability=None,
referring_urls=None,
last_crawl_time=None,
error=str(exc),
)
# ============================================================
# SITEMAP SUBMISSION
# ============================================================
def submit_sitemap(
service,
) -> Dict[str, Any]:
logger.info(
"Submitting sitemap to Google Search Console..."
)
try:
response = retry_call(
lambda: (
service
.sitemaps()
.submit(
siteUrl=SITE_URL,
feedpath=SITEMAP_URL,
)
.execute()
)
)
logger.info(
"Sitemap submission completed."
)
return {
"submitted": True,
"skipped": False,
"sitemap": SITEMAP_URL,
"response": response,
"error": None,
}
except Exception as exc:
logger.error(
"Sitemap submission failed: %s",
exc,
)
return {
"submitted": False,
"skipped": False,
"sitemap": SITEMAP_URL,
"response": None,
"error": str(exc),
}
# ============================================================
# SEARCH ANALYTICS
# ============================================================
def query_search_analytics(
service,
dimensions: List[str],
start_date: str,
end_date: str,
row_limit: int = 25000,
) -> List[Dict[str, Any]]:
body = {
"startDate": start_date,
"endDate": end_date,
"dimensions": dimensions,
"type": "web",
"aggregationType": "auto",
"rowLimit": min(
row_limit,
25000,
),
"dataState": "final",
}
try:
response = retry_call(
lambda: (
service
.searchanalytics()
.query(
siteUrl=SITE_URL,
body=body,
)
.execute()
)
)
rows = response.get(
"rows",
[],
)
normalized = []
for row in rows:
normalized.append(
{
"keys": row.get(
"keys",
[],
),
"clicks": row.get(
"clicks",
0,
),
"impressions": row.get(
"impressions",
0,
),
"ctr": row.get(
"ctr",
0,
),
"position": row.get(
"position",
0,
),
}
)
return normalized
except Exception as exc:
logger.error(
"Search Analytics query failed: %s",
exc,
)
return []
def build_search_analytics_report(
service,
) -> Dict[str, Any]:
end_date = (
date.today()
- timedelta(days=2)
)
start_date = (
end_date
- timedelta(
days=DAYS - 1
)
)
start = (
start_date.isoformat()
)
end = (
end_date.isoformat()
)
logger.info(
"Search Analytics period: %s -> %s",
start,
end,
)
return {
"period": {
"start": start,
"end": end,
},
"pages": query_search_analytics(
service,
["page"],
start,
end,
),
"queries": query_search_analytics(
service,
["query"],
start,
end,
),
"countries": query_search_analytics(
service,
["country"],
start,
end,
),
"devices": query_search_analytics(
service,
["device"],
start,
end,
),
"search_appearances": query_search_analytics(
service,
["searchAppearance"],
start,
end,
),
}
# ============================================================
# PAGE ACTION LIST
#
# Cross-references HTTP diagnostics, GSC inspection results,
# and on-page signals (canonical / robots meta) to build a
# concrete list of pages that need changes, with the specific
# issue and the recommended fix for each.
# ============================================================
def _add_issue(
issues: List[str],
recommendations: List[str],
issue: str,
recommendation: str,
):
issues.append(issue)
recommendations.append(recommendation)
def analyze_page(
http_result: Optional[HTTPDiagnostic],
gsc_result: Optional[GSCInspection],
) -> Optional[PageAction]:
"""
Inspect a single URL's HTTP + GSC signals and return a
PageAction describing what to fix, or None if the page
looks healthy.
"""
url = (
http_result.url
if http_result
else (
gsc_result.url
if gsc_result
else None
)
)
if not url:
return None
issues: List[str] = []
recommendations: List[str] = []
severity = "info"
# --------------------------------------------------
# HTTP-level problems
# --------------------------------------------------
if http_result:
if http_result.error:
_add_issue(
issues,
recommendations,
f"URL could not be fetched: {http_result.error}",
"Confirm the page is reachable, DNS resolves "
"correctly, and no firewall/WAF is blocking "
"Googlebot or automated requests.",
)
severity = "critical"
elif http_result.status_code and http_result.status_code >= 500:
_add_issue(
issues,
recommendations,
f"Server error (HTTP {http_result.status_code}).",
"Investigate server logs; fix the underlying "
"server error so the page returns HTTP 200.",
)
severity = "critical"
elif http_result.status_code and http_result.status_code == 404:
_add_issue(
issues,
recommendations,
"Page returns HTTP 404 (Not Found).",
"Restore the page, 301-redirect it to the correct "
"replacement URL, or remove the URL from the "
"sitemap if it should no longer exist.",
)
severity = "critical"
elif http_result.status_code and http_result.status_code in (301, 302, 307, 308):
_add_issue(
issues,
recommendations,
f"Sitemap URL redirects (HTTP {http_result.status_code}) "
f"to {http_result.final_url}.",
"Update the sitemap to list the final destination "
"URL directly instead of a redirecting URL.",
)
severity = "warning"
elif http_result.status_code and http_result.status_code not in (200,):
_add_issue(
issues,
recommendations,
f"Unexpected HTTP status: {http_result.status_code}.",
"Review the response and ensure the page returns "
"HTTP 200 for a normal, indexable page.",
)
severity = "warning"
if (
http_result.robots_meta
and re.search(
r"noindex",
http_result.robots_meta,
re.IGNORECASE,
)
):
_add_issue(
issues,
recommendations,
f"Page has a robots meta noindex directive "
f"({http_result.robots_meta!r}).",
"Remove the 'noindex' directive from the "
"<meta name=\"robots\"> tag if this page should "
"be indexed, or remove it from the sitemap if "
"it should stay out of the index.",
)
severity = "critical"
if (
http_result.canonical
and http_result.final_url
and normalize_url(
http_result.canonical
)
!= normalize_url(
http_result.final_url
)
):
_add_issue(
issues,
recommendations,
f"Canonical tag points to a different URL "
f"({http_result.canonical}) than the page itself.",
"Either update the canonical tag to self-reference "
"this URL, or if the canonical target is correct, "
"list that canonical URL in the sitemap instead "
"and 301-redirect this URL to it.",
)
if severity == "info":
severity = "warning"
# --------------------------------------------------
# GSC-level problems
# --------------------------------------------------
if gsc_result:
if gsc_result.error:
_add_issue(
issues,
recommendations,
f"GSC URL Inspection failed: {gsc_result.error}",
"Retry the inspection later; if it persists, "
"verify GSC property permissions and API quota.",
)
if severity == "info":
severity = "warning"
else:
if gsc_result.verdict and gsc_result.verdict != "PASS":
_add_issue(
issues,
recommendations,
f"Google indexing verdict is '{gsc_result.verdict}' "
f"(coverage: {gsc_result.coverage_state}).",
"Review the coverage state in Search Console, "
"resolve the blocking issue, then request "
"indexing again once fixed.",
)
severity = "critical"
if (
gsc_result.robots_txt_state
and gsc_result.robots_txt_state.upper()
not in ("ALLOWED",)
):
_add_issue(
issues,
recommendations,
f"robots.txt state is "
f"'{gsc_result.robots_txt_state}'.",
"Update robots.txt to allow crawling of this "
"URL path if it should be indexed.",
)
severity = "critical"
if (
gsc_result.page_fetch_state
and gsc_result.page_fetch_state.upper()
not in ("SUCCESSFUL",)
):
_add_issue(
issues,
recommendations,
f"Google could not fetch the page "
f"(state: {gsc_result.page_fetch_state}).",
"Check server availability and response time "
"for Googlebot; ensure no server errors or "
"timeouts occur on this URL.",
)
severity = "critical"
if (
gsc_result.google_canonical
and gsc_result.user_canonical
and normalize_url(
gsc_result.google_canonical
)
!= normalize_url(
gsc_result.user_canonical
)
):
_add_issue(
issues,
recommendations,
"Google selected a different canonical "
f"({gsc_result.google_canonical}) than the "
f"one declared on the page "
f"({gsc_result.user_canonical}).",
"Consolidate duplicate/similar content, add "
"internal links pointing to the preferred "
"URL, and ensure the declared canonical "
"matches the URL Google should treat as "
"canonical.",
)
if severity == "info":
severity = "warning"
if (
gsc_result.mobile_usability
and gsc_result.mobile_usability != "PASS"
):
_add_issue(
issues,
recommendations,
f"Mobile usability verdict: "
f"{gsc_result.mobile_usability}.",
"Fix mobile usability issues (viewport "
"configuration, text size, tap-target "
"spacing) reported in Search Console.",
)
if severity == "info":
severity = "warning"
if not issues:
return None
return PageAction(
url=url,
severity=severity,
issues=issues,
recommendations=recommendations,
)
def build_page_action_list(
http_results: List[HTTPDiagnostic],
gsc_results: List[GSCInspection],
) -> List[Dict[str, Any]]:
"""
Build a de-duplicated, per-URL list of pages that need
changes to align with Google Search Console, each with
the specific issues found and recommended fixes.
"""
gsc_by_url: Dict[str, GSCInspection] = {
item.url: item
for item in gsc_results
}
http_by_url: Dict[str, HTTPDiagnostic] = {
item.url: item
for item in http_results
}
all_urls = list(
dict.fromkeys(
list(http_by_url.keys())
+ list(gsc_by_url.keys())
)
)
actions: List[PageAction] = []
for url in all_urls:
page_action = analyze_page(
http_by_url.get(url),
gsc_by_url.get(url),
)
if page_action:
actions.append(page_action)
severity_rank = {
"critical": 0,
"warning": 1,
"info": 2,
}
actions.sort(
key=lambda item: (
severity_rank.get(
item.severity,
3,
),
item.url,
)
)
logger.info(
"Pages needing changes: %d "
"(critical=%d, warning=%d, info=%d)",
len(actions),
sum(
1
for a in actions
if a.severity == "critical"
),
sum(
1
for a in actions
if a.severity == "warning"
),
sum(
1
for a in actions
if a.severity == "info"
),
)
return [
asdict(item)
for item in actions
]
# ============================================================
# DIAGNOSTIC SCORING
# ============================================================
def calculate_summary(
http_results: List[HTTPDiagnostic],
gsc_results: List[GSCInspection],
robots: Dict[str, Any],
sitemap: Dict[str, Any],
page_actions: List[Dict[str, Any]],
) -> Dict[str, Any]:
summary = {
"urls_checked": len(
http_results
),
"http_ok": sum(
1
for x in http_results
if x.ok
),
"http_failed": sum(
1
for x in http_results
if not x.ok
),
"gsc_indexed": 0,
"gsc_not_indexed": 0,
"gsc_errors": 0,
"robots_ok": robots.get(
"ok",
False,
),
"sitemap_ok": sitemap.get(
"ok",
False,
),
"sitemap_url_count": sitemap.get(
"url_count",
0,
),
"pages_needing_action": len(
page_actions
),
"pages_critical": sum(
1
for a in page_actions
if a.get("severity") == "critical"
),
"pages_warning": sum(
1
for a in page_actions
if a.get("severity") == "warning"
),
}
for result in gsc_results:
if result.error:
summary[
"gsc_errors"
] += 1
elif result.verdict == "PASS":
summary[
"gsc_indexed"
] += 1
else:
summary[
"gsc_not_indexed"
] += 1
return summary
# ============================================================
# CSV EXPORT
# ============================================================
def write_csv(
path: Path,
rows: Iterable[Dict[str, Any]],
):
rows = list(rows)
if not rows:
return
fieldnames = sorted(
{
key
for row in rows
for key in row.keys()
}
)
with path.open(
"w",
newline="",
encoding="utf-8",
) as file:
writer = csv.DictWriter(
file,
fieldnames=fieldnames,
extrasaction="ignore",
)
writer.writeheader()
for row in rows:
writer.writerow(
row
)
# ============================================================
# HTML REPORT
# ============================================================
def _escape(
value: Any,
) -> str:
return html.escape(
str(
value
if value is not None
else ""
)
)
def _severity_badge(
severity: str,
) -> str:
colors = {
"critical": "#b91c1c",
"warning": "#b45309",
"info": "#374151",
}
color = colors.get(
severity,
"#374151",
)
return (
f'<span style="display:inline-block;padding:2px 8px;'
f'border-radius:10px;color:#fff;background:{color};'
f'font-size:12px;font-weight:600;text-transform:uppercase;">'
f"{_escape(severity)}</span>"
)
def _bool_badge(
value: bool,
) -> str:
if value:
return (
'<span style="color:#15803d;font-weight:600;">OK</span>'
)
return (
'<span style="color:#b91c1c;font-weight:600;">FAIL</span>'
)
def render_html_report(
report: Dict[str, Any],
) -> str:
"""
Render a single self-contained HTML report summarizing
the diagnostic run, including the page action list.
"""
summary = report.get(
"summary",
{},
)
property_validation = report.get(
"gsc_property_validation",
{},
)
robots = report.get(
"robots",
{},
)
sitemap = report.get(
"sitemap_diagnostic",
{},
)
sitemap_submission = report.get(
"sitemap_submission",
{},
)
page_actions = report.get(
"page_actions",
[],
)
analytics = report.get(
"search_analytics",
{},
)
top_pages = analytics.get(
"pages",
[],
)[:20]
top_queries = analytics.get(
"queries",
[],
)[:20]
# --- summary cards -------------------------------------------------
summary_cards = "".join(
f'<div class="card"><div class="card-label">{_escape(label)}</div>'
f'<div class="card-value">{_escape(value)}</div></div>'
for label, value in [
("URLs checked", summary.get("urls_checked", 0)),
("HTTP OK", summary.get("http_ok", 0)),
("HTTP failed", summary.get("http_failed", 0)),
("GSC indexed (PASS)", summary.get("gsc_indexed", 0)),
("GSC not indexed", summary.get("gsc_not_indexed", 0)),
("Pages needing action", summary.get("pages_needing_action", 0)),
("Critical issues", summary.get("pages_critical", 0)),
("Warnings", summary.get("pages_warning", 0)),
]
)
# --- page action rows ------------------------------------------------
if page_actions:
action_rows = []
for item in page_actions:
issues_html = "".join(
f"<li>{_escape(issue)}</li>"
for issue in item.get("issues", [])
)
recs_html = "".join(
f"<li>{_escape(rec)}</li>"
for rec in item.get("recommendations", [])
)
action_rows.append(
"<tr>"
f'<td>{_severity_badge(item.get("severity", "info"))}</td>'
f'<td class="url-cell"><a href="{_escape(item.get("url"))}" '
f'target="_blank" rel="noopener">{_escape(item.get("url"))}</a></td>'
f'<td><ul class="issue-list">{issues_html}</ul></td>'
f'<td><ul class="issue-list">{recs_html}</ul></td>'
"</tr>"
)
action_table = (
'<table class="report-table">'
"<thead><tr>"
"<th>Severity</th><th>Page</th>"
"<th>Issues found</th><th>Recommended fix</th>"
"</tr></thead><tbody>"
+ "".join(action_rows)
+ "</tbody></table>"
)
else:
action_table = (
'<p class="ok-message">No pages currently require '
"changes to align with Google Search Console.</p>"
)
# --- top pages table -------------------------------------------------
def analytics_table(rows, key_label):
if not rows:
return "<p>No data available for this period.</p>"
body_rows = "".join(
"<tr>"
f'<td>{_escape(", ".join(row.get("keys", [])))}</td>'
f'<td>{_escape(row.get("clicks", 0))}</td>'
f'<td>{_escape(row.get("impressions", 0))}</td>'
f'<td>{_escape(round(row.get("ctr", 0) * 100, 2))}%</td>'
f'<td>{_escape(round(row.get("position", 0), 1))}</td>'
"</tr>"
for row in rows
)
return (
'<table class="report-table">'
f"<thead><tr><th>{key_label}</th><th>Clicks</th>"
"<th>Impressions</th><th>CTR</th><th>Avg. position</th>"
"</tr></thead><tbody>"
+ body_rows
+ "</tbody></table>"
)
top_pages_table = analytics_table(
top_pages,
"Page",
)
top_queries_table = analytics_table(
top_queries,
"Query",
)
period = analytics.get(
"period",
{},
)
html_doc = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>GSC Diagnostic Report - {_escape(report.get('site'))}</title>
<style>
body {{
font-family: -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif;
background: #f7f7f8;
color: #111827;
margin: 0;
padding: 32px;
}}
h1 {{ font-size: 22px; margin-bottom: 4px; }}
h2 {{ font-size: 17px; margin-top: 40px; border-bottom: 1px solid #e5e7eb; padding-bottom: 6px; }}
.meta {{ color: #6b7280; font-size: 13px; margin-bottom: 24px; }}
.cards {{ display: flex; flex-wrap: wrap; gap: 12px; margin-top: 16px; }}
.card {{
background: #fff; border: 1px solid #e5e7eb; border-radius: 8px;
padding: 14px 18px; min-width: 140px; flex: 1 1 140px;
}}
.card-label {{ font-size: 12px; color: #6b7280; text-transform: uppercase; letter-spacing: 0.03em; }}
.card-value {{ font-size: 24px; font-weight: 700; margin-top: 4px; }}
table.report-table {{ width: 100%; border-collapse: collapse; margin-top: 12px; background: #fff; }}
table.report-table th, table.report-table td {{
border: 1px solid #e5e7eb; padding: 8px 10px; font-size: 13px; text-align: left; vertical-align: top;
}}
table.report-table th {{ background: #f3f4f6; }}
.url-cell {{ max-width: 320px; word-break: break-all; }}
ul.issue-list {{ margin: 0; padding-left: 18px; }}
ul.issue-list li {{ margin-bottom: 4px; }}
.ok-message {{ color: #15803d; font-weight: 600; }}
.status-line {{ margin: 4px 0; }}
</style>
</head>
<body>
<h1>Google Search Console Diagnostic Report</h1>
<div class="meta">
Site: {_escape(report.get('site'))} &middot;
Property: {_escape(report.get('gsc_property'))} &middot;
Generated: {_escape(report.get('generated_at'))}
</div>
<div class="status-line">Property valid: {_bool_badge(property_validation.get('valid', False))}
&nbsp;|&nbsp; Permission: {_escape(property_validation.get('permission_level'))}</div>
<div class="status-line">robots.txt: {_bool_badge(robots.get('ok', False))}
({_escape(robots.get('status_code'))}) &nbsp;|&nbsp;
sitemap: {_bool_badge(sitemap.get('ok', False))}
({_escape(sitemap.get('url_count', 0))} URLs)</div>
<div class="status-line">Sitemap submission: {_bool_badge(sitemap_submission.get('submitted', False))}
{"(" + _escape(sitemap_submission.get('reason')) + ")" if sitemap_submission.get('reason') else ""}</div>
<div class="cards">{summary_cards}</div>
<h2>Pages that need changes</h2>
{action_table}
<h2>Top pages (Search Analytics{" &middot; " + _escape(period.get('start')) + " to " + _escape(period.get('end')) if period else ""})</h2>
{top_pages_table}
<h2>Top queries</h2>
{top_queries_table}
</body>
</html>
"""
return html_doc
# ============================================================
# REPORT GENERATION
# ============================================================
def write_reports(
report: Dict[str, Any],
):
OUTPUT_DIR.mkdir(
parents=True,
exist_ok=True,
)
timestamp = time.strftime(
"%Y%m%d-%H%M%S"
)
json_path = (
OUTPUT_DIR
/ f"gsc-diagnostic-{timestamp}.json"
)
with json_path.open(
"w",
encoding="utf-8",
) as file:
json.dump(
report,
file,
indent=2,
ensure_ascii=False,
)
html_path = (
OUTPUT_DIR
/ f"gsc-diagnostic-{timestamp}.html"
)
try:
html_content = render_html_report(
report
)
with html_path.open(
"w",
encoding="utf-8",
) as file:
file.write(
html_content
)
except Exception as exc:
logger.error(
"Unable to write HTML report: %s",
exc,
)
html_path = None
http_rows = report.get(
"http_diagnostics",
[],
)
gsc_rows = report.get(
"gsc_inspections",
[],
)
page_action_rows = report.get(
"page_actions",
[],
)
analytics = report.get(
"search_analytics",
{},
)
write_csv(
OUTPUT_DIR
/ f"http-diagnostics-{timestamp}.csv",
http_rows,
)
write_csv(
OUTPUT_DIR
/ f"gsc-inspections-{timestamp}.csv",
gsc_rows,
)
write_csv(
OUTPUT_DIR
/ f"page-actions-{timestamp}.csv",
page_action_rows,
)
for name, rows in analytics.items():
if name == "period":
continue
write_csv(
OUTPUT_DIR
/ f"search-{name}-{timestamp}.csv",
rows,
)
logger.info(
"Reports written to: %s",
OUTPUT_DIR,
)
logger.info(
"JSON report: %s",
json_path,
)
if html_path:
logger.info(
"HTML report: %s",
html_path,
)
return json_path, html_path
# ============================================================
# MAIN
# ============================================================
def main() -> int:
args = parse_args()
logger.info(
"================================================"
)
logger.info(
"School of Freelancing GSC Diagnostic"
)
logger.info(
"Property: %s",
SITE_URL,
)
logger.info(
"Environment: %s",
ENV_FILE,
)
logger.info(
"================================================"
)
# --------------------------------------------------------
# Initialize protected .env
# --------------------------------------------------------
if args.init_env:
success = initialize_env_file()
if not success:
return 2
# If --init-env was supplied by itself, stop here.
if not args.oauth_setup:
return 0
# --------------------------------------------------------
# Verify .env when present
# --------------------------------------------------------
if ENV_FILE.exists():
try:
secure_env_file(
ENV_FILE
)
except Exception as exc:
logger.error(
"Protected .env validation failed: %s",
exc,
)
return 2
# --------------------------------------------------------
# Validate configuration
# --------------------------------------------------------
try:
validate_runtime_configuration()
except Exception as exc:
logger.error(
"Configuration error: %s",
exc,
)
return 2
# --------------------------------------------------------
# OAuth setup
# --------------------------------------------------------
if args.oauth_setup:
success = oauth_setup(
args.port
)
if not success:
return 2
logger.info(
"Testing the newly created refresh token..."
)
# --------------------------------------------------------
# Authentication
# --------------------------------------------------------
try:
service = get_gsc_service()
except Exception as exc:
logger.error(
"Google authentication failed: %s",
exc,
)
return 2
# --------------------------------------------------------
# Validate GSC property
# --------------------------------------------------------
property_result = (
validate_gsc_property(
service
)
)
if not property_result[
"valid"
]:
logger.error(
"The authenticated Google account "
"does not have access to:"
)
logger.error(
" %s",
SITE_URL,
)
logger.error(
"Grant the Google account access to "
"the domain property in Search Console."
)
return 2
# --------------------------------------------------------
# Validate-only
# --------------------------------------------------------
if args.validate_only:
logger.info(
"GSC authentication and property validation succeeded."
)
logger.info(
"Permission level: %s",
property_result[
"permission_level"
],
)
return 0
# --------------------------------------------------------
# robots.txt
# --------------------------------------------------------
robots = diagnose_robots()
logger.info(
"robots.txt status: %s",
robots.get(
"status_code"
),
)
if robots.get(
"has_disallow_all"
):
logger.warning(
"robots.txt appears to contain Disallow: /"
)
# --------------------------------------------------------
# sitemap
# --------------------------------------------------------
sitemap = diagnose_sitemap()
logger.info(
"Sitemap status: %s | URLs discovered: %s",
sitemap.get(
"status_code"
),
sitemap.get(
"url_count"
),
)
urls = sitemap.get(
"urls",
[],
)
# --------------------------------------------------------
# Sitemap submission
# --------------------------------------------------------
sitemap_submission = {
"submitted": False,
"skipped": True,
"reason": None,
}
submit_sitemap_enabled = (
SUBMIT_SITEMAP
and not args.no_submit_sitemap
)
if (
submit_sitemap_enabled
and sitemap.get("ok")
):
sitemap_submission = (
submit_sitemap(
service
)
)
elif args.no_submit_sitemap:
sitemap_submission[
"reason"
] = "disabled by --no-submit-sitemap"
elif not SUBMIT_SITEMAP:
sitemap_submission[
"reason"
] = "GSC_SUBMIT_SITEMAP=false"
elif not sitemap.get("ok"):
sitemap_submission[
"reason"
] = "sitemap diagnostic failed"
# --------------------------------------------------------
# URL diagnostics
# --------------------------------------------------------
http_results: List[
HTTPDiagnostic
] = []
gsc_results: List[
GSCInspection
] = []
inspect_enabled = (
INSPECT_SITEMAP_URLS
and not args.no_inspect
)
if inspect_enabled:
urls = urls[
:MAX_URLS
]
logger.info(
"Inspecting %d sitemap URLs...",
len(urls),
)
for index, url in enumerate(
urls,
start=1,
):
url = normalize_url(
url
)
logger.info(
"[%d/%d] %s",
index,
len(urls),
url,
)
if not is_same_site(
url
):
logger.warning(
"Skipping external URL: %s",
url,
)
continue
http_result = (
diagnose_url(
url
)
)
http_results.append(
http_result
)
if http_result.ok:
gsc_result = (
inspect_url(
service,
url,
)
)
gsc_results.append(
gsc_result
)
else:
logger.warning(
"Skipping GSC inspection "
"because HTTP check failed: %s",
url,
)
else:
logger.info(
"GSC URL inspection disabled."
)
# --------------------------------------------------------
# Search Analytics
# --------------------------------------------------------
logger.info(
"Collecting Search Analytics..."
)
analytics = (
build_search_analytics_report(
service
)
)
# --------------------------------------------------------
# Page action list (what needs to change, and why)
# --------------------------------------------------------
page_actions = build_page_action_list(
http_results,
gsc_results,
)
# --------------------------------------------------------
# Summary
# --------------------------------------------------------
summary = calculate_summary(
http_results,
gsc_results,
robots,
sitemap,
page_actions,
)
# --------------------------------------------------------
# Final report
# --------------------------------------------------------
report = {
"generated_at": time.strftime(
"%Y-%m-%dT%H:%M:%S%z"
),
"site": BASE_URL,
"gsc_property": SITE_URL,
"gsc_property_validation": (
property_result
),
"sitemap": SITEMAP_URL,
"robots": robots,
"sitemap_diagnostic": sitemap,
"sitemap_submission": (
sitemap_submission
),
"summary": summary,
"http_diagnostics": [
asdict(item)
for item in http_results
],
"gsc_inspections": [
asdict(item)
for item in gsc_results
],
"page_actions": page_actions,
"search_analytics": analytics,
}
write_reports(
report
)
# --------------------------------------------------------
# Final console summary
# --------------------------------------------------------
logger.info("")
logger.info(
"================ FINAL SUMMARY ================"
)
logger.info(
"GSC property valid: %s",
property_result[
"valid"
],
)
logger.info(
"GSC permission: %s",
property_result.get(
"permission_level"
),
)
logger.info(
"URLs checked: %d",
summary[
"urls_checked"
],
)
logger.info(
"HTTP OK: %d",
summary[
"http_ok"
],
)
logger.info(
"HTTP failed: %d",
summary[
"http_failed"
],
)
logger.info(
"GSC PASS: %d",
summary[
"gsc_indexed"
],
)
logger.info(
"GSC not PASS: %d",
summary[
"gsc_not_indexed"
],
)
logger.info(
"GSC errors: %d",
summary[
"gsc_errors"
],
)
logger.info(
"robots.txt OK: %s",
summary[
"robots_ok"
],
)
logger.info(
"sitemap OK: %s",
summary[
"sitemap_ok"
],
)
logger.info(
"sitemap URLs: %d",
summary[
"sitemap_url_count"
],
)
logger.info(
"Pages needing fix: %d (critical=%d, warning=%d)",
summary[
"pages_needing_action"
],
summary[
"pages_critical"
],
summary[
"pages_warning"
],
)
logger.info(
"================================================"
)
# --------------------------------------------------------
# Exit status
# --------------------------------------------------------
if not summary[
"robots_ok"
]:
return 1
if not summary[
"sitemap_ok"
]:
return 1
if summary[
"http_failed"
] > 0:
return 1
if summary[
"pages_critical"
] > 0:
return 1
return 0
# ============================================================
# ENTRY POINT
# ============================================================
if __name__ == "__main__":
try:
sys.exit(
main()
)
except KeyboardInterrupt:
logger.warning(
"Interrupted by user."
)
sys.exit(130)
except Exception as exc:
logger.exception(
"Unexpected fatal error: %s",
exc,
)
sys.exit(3)
@AnythingLinux

AnythingLinux commented Sep 1, 2026 •

Copy link
Copy Markdown
Author

cd script
python3 -m venv .venv
source .venv/bin/activate
gedit gsc_diagnostic.py
chmod +x gsc_diagnostic.py
pip install requests google-api-python-client google-auth-httplib2 google-auth-oauthlib
pip install --upgrade pip
pip install requests google-auth google-api-python-client
sed -i '1{/^```python$/d;}' gsc_diagnostic.py

sed -i '${/^```$/d;}' gsc_diagnostic.py

python3 -m py_compile gsc_diagnostic.py

head -n 3 gsc_diagnostic.py

tail -n 3 gsc_diagnostic.py

nano .env

GSC_CLIENT_ID=YOUR_CLIENT_ID.apps.googleusercontent.com
GSC_CLIENT_SECRET=YOUR_CLIENT_SECRET
GSC_REFRESH_TOKEN=YOUR_REFRESH_TOKEN
GSC_DAYS=28
GSC_MAX_URLS=250
GSC_TIMEOUT=20
GSC_OUTPUT_DIR='/home/allah/Downloads/gsc-diagnostic/reports'
GSC_SUBMIT_SITEMAP=true
GSC_INSPECT_SITEMAP_URLS=true

ctrl+s and ctrl+x

chmod 600 .env
set -a
source .env
set +a
test -n "$GSC_CLIENT_ID" && echo "GSC_CLIENT_ID: SET" || echo "GSC_CLIENT_ID: MISSING"
test -n "$GSC_CLIENT_SECRET" && echo "GSC_CLIENT_SECRET: SET" || echo "GSC_CLIENT_SECRET: MISSING"
test -n "$GSC_REFRESH_TOKEN" && echo "GSC_REFRESH_TOKEN: SET" || echo "GSC_REFRESH_TOKEN: MISSING"
python3 gsc_diagnostic.py --init-env
python3 gsc_diagnostic.py --validate-only
python3 gsc_diagnostic.py --oauth-setup --port 8080

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment