benoit74

#!/usr/bin/env bash
BASE="https://api.farm.openzim.org/v2"
LIMIT=100

for EP in recipes schedules; do
  SKIP=0; out=""
  while :; do
    page=$(curl -s "$BASE/$EP?name=gutenberg&limit=$LIMIT&skip=$SKIP")
    if ! jq -e '.items' >/dev/null 2>&1 <<<"$page"; then
      echo "[$EP] unexpected response:" >&2

benoit74

#!/usr/bin/env python3
"""List media entries in a single local ZIM file.

Prints a summary of media counts grouped by mimetype on the console, 
and writes the full list of media entries (path, mimetype, size) to 
a CSV file.

Usage:
    ./zim_media_list_local.py <path-to-zim> [--output FILE]
"""

benoit74

#!/usr/bin/env python3
"""
Script to import users into Ory from a CSV file with password hash transformation.
"""

import csv
import sys
import requests
from typing import Tuple
from pathlib import Path

benoit74

# !/usr/bin/env python3

import argparse
import logging
import sys
from typing import NamedTuple

from pydantic import BaseModel
import humanfriendly

benoit74

CONTENTS = [
    (
        "content1",
        "<html><head><title>Test</title></head><body><h1>Parse me!</h1></body></html>",
    ),
    (
        "content2",
        """<img width="200" src="image.png?param1=value1&aparam2=value2" />""",
    ),
    (

benoit74

import subprocess
import re
import json


def run_pyright():
    # Run Pyright and capture the output
    result = subprocess.run(["pyright", "--outputjson"], capture_output=True, text=True)
    return result.stdout

benoit74

import glob
from warcio import ArchiveIterator

for warc in glob.glob("output/.tmph919m5n3/collections/crawl-*/archive/*.warc.gz"):
    with open(warc, "rb") as fh:
        for record in ArchiveIterator(fh):
            if record.rec_type == "request":
                print(record.rec_headers.get_header('WARC-Target-URI'))

print("DONE")

benoit74

One might need to get a better grasp at what Browsertrix is performing in terms of trafic and where it is being blocked.

You might use the following docker compose stack to run:

• a mitmweb web proxy to intercept all Browsertrix traffic and dump them on file
• a screencasting Browsertrix crawler to get a grasp on crawler behavior

If you open http://localhost:9037, you will see Browsertrix screencasting of the browser.

mitmdump are placed in the output/mitmdump folder, with one file per minute.

benoit74

import json
import subprocess

nodes_str = subprocess.check_output(
    ["kubectl", "get", "nodes",  "-o", "json"]
)
nodes = json.loads(nodes_str)

node_data = {}
for node in nodes.get("items"):

benoit74

{  
    "[python]": {
        "editor.defaultFormatter": "ms-python.black-formatter",
        "editor.formatOnSave": true,
        "editor.codeActionsOnSave": {
            "source.organizeImports": true
        },
      },
    "isort.args": [
        "--profile",