mirror of
https://github.com/pluja/awesome-privacy.git
synced 2026-07-21 10:55:39 +02:00
ci: fold open-source verification into the health report
This commit is contained in:
pluja
@@ -1,8 +1,9 @@
|
||||
name: Health report
|
||||
|
||||
# Monthly maintenance scan of README.md. Finds dead links and repos that look
|
||||
# abandoned (archived, or no push in over 18 months), then opens one issue with
|
||||
# the findings. Opens nothing when everything is healthy.
|
||||
# Monthly maintenance + open-source audit of README.md. Finds dead links and
|
||||
# repos that are gone, archived, stale (no push in 18+ months), unlicensed, or
|
||||
# have no real source code. Opens one issue with the findings, nothing when clean.
|
||||
# Logic lives in scripts/health_report.py so it can be run and tested locally.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -25,7 +26,7 @@ jobs:
|
||||
uses: lycheeverse/lychee-action@v2
|
||||
with:
|
||||
lycheeVersion: v0.23.0
|
||||
# Whole file, JSON output for the parser below.
|
||||
# Whole file, JSON output for the parser in health_report.py.
|
||||
# fail:false keeps the job green so we can build the report either way.
|
||||
args: >-
|
||||
--no-progress
|
||||
@@ -42,173 +43,11 @@ jobs:
|
||||
fail: false
|
||||
failIfEmpty: false
|
||||
|
||||
- name: Build report (dead links + stale repos)
|
||||
- name: Build report (dead links + repository issues)
|
||||
id: scan
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import datetime as dt
|
||||
import json, os, re, urllib.error, urllib.request
|
||||
|
||||
NOW = dt.datetime.now(dt.timezone.utc)
|
||||
STALE_DAYS = 18 * 30 # ~18 months
|
||||
TOKEN = os.environ.get("GITHUB_TOKEN", "")
|
||||
|
||||
# ---- 1. Dead links, from the lychee JSON report ------------------------
|
||||
dead = []
|
||||
dead_ok = True
|
||||
try:
|
||||
with open("lychee/out.json") as fh:
|
||||
data = json.load(fh)
|
||||
emap = data.get("error_map") or data.get("fail_map") or {}
|
||||
for entries in emap.values():
|
||||
for item in entries:
|
||||
url = item.get("url", "")
|
||||
status = item.get("status") or {}
|
||||
text = status.get("text") or status.get("details") or "error"
|
||||
dead.append((url, text))
|
||||
except FileNotFoundError:
|
||||
dead_ok = False
|
||||
print("::warning::lychee/out.json not found; skipping dead-link section")
|
||||
except Exception as exc:
|
||||
dead_ok = False
|
||||
print(f"::warning::Could not parse lychee output ({exc})")
|
||||
# De-duplicate while keeping order.
|
||||
seen = set()
|
||||
dead = [d for d in dead if not (d[0] in seen or seen.add(d[0]))]
|
||||
|
||||
# ---- 2. Stale / archived repositories ---------------------------------
|
||||
with open("README.md", encoding="utf-8") as fh:
|
||||
readme = fh.read()
|
||||
|
||||
# Capture host/owner/repo; any subpath (/tree/..., /blob/...) is ignored.
|
||||
repo_re = re.compile(
|
||||
r"https?://(?:www\.)?(github\.com|codeberg\.org)/"
|
||||
r"([A-Za-z0-9._-]+)/([A-Za-z0-9._-]+)"
|
||||
)
|
||||
# First path segments that are never a real owner.
|
||||
RESERVED = {
|
||||
"sponsors", "orgs", "topics", "about", "features", "marketplace",
|
||||
"settings", "explore", "collections", "apps", "notifications",
|
||||
"login", "join", "pricing", "security", "site", "contact",
|
||||
"readme", "search", "new", "watching", "stars",
|
||||
}
|
||||
|
||||
repos = {} # (host, owner_lower, repo_lower) -> (host, owner, repo)
|
||||
for host, owner, repo in repo_re.findall(readme):
|
||||
if owner.lower() in RESERVED:
|
||||
continue
|
||||
repo = repo.rstrip(".")
|
||||
if repo.endswith(".git"):
|
||||
repo = repo[:-4]
|
||||
if not repo:
|
||||
continue
|
||||
key = (host, owner.lower(), repo.lower())
|
||||
repos.setdefault(key, (host, owner, repo))
|
||||
|
||||
def fetch(url, token=None):
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Accept", "application/json")
|
||||
req.add_header("User-Agent", "awesome-privacy-health-report")
|
||||
if token:
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
return json.load(resp)
|
||||
|
||||
def parse_ts(value):
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return dt.datetime.strptime(
|
||||
value.replace("Z", "+0000"), "%Y-%m-%dT%H:%M:%S%z"
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
stale = [] # (slug, reason)
|
||||
for host, owner, repo in sorted(repos.values()):
|
||||
slug = f"{host}/{owner}/{repo}"
|
||||
try:
|
||||
if host == "github.com":
|
||||
info = fetch(
|
||||
f"https://api.github.com/repos/{owner}/{repo}", TOKEN
|
||||
)
|
||||
last = parse_ts(info.get("pushed_at"))
|
||||
else: # codeberg.org (Forgejo/Gitea API, public, no token)
|
||||
info = fetch(
|
||||
f"https://codeberg.org/api/v1/repos/{owner}/{repo}"
|
||||
)
|
||||
last = parse_ts(info.get("updated_at"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (404, 410):
|
||||
stale.append((slug, f"repository not found (HTTP {exc.code})"))
|
||||
else:
|
||||
print(f"::warning::{slug}: HTTP {exc.code}, skipped")
|
||||
continue
|
||||
except Exception as exc:
|
||||
print(f"::warning::{slug}: {exc}, skipped")
|
||||
continue
|
||||
|
||||
reasons = []
|
||||
if info.get("archived"):
|
||||
reasons.append("archived")
|
||||
if last is not None:
|
||||
age = (NOW - last).days
|
||||
if age > STALE_DAYS:
|
||||
months = age // 30
|
||||
reasons.append(
|
||||
f"no push since {last.date()} (~{months} months)"
|
||||
)
|
||||
if reasons:
|
||||
stale.append((slug, "; ".join(reasons)))
|
||||
|
||||
# ---- 3. Assemble the report -------------------------------------------
|
||||
today = NOW.date().isoformat()
|
||||
lines = [f"# Health report {today}", ""]
|
||||
lines.append("Automated monthly scan of `README.md`.")
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"## Dead links ({len(dead)})")
|
||||
lines.append("")
|
||||
if dead:
|
||||
for url, text in dead:
|
||||
lines.append(f"- {url} - {text}")
|
||||
elif dead_ok:
|
||||
lines.append("None found.")
|
||||
else:
|
||||
lines.append("_Link scan produced no output this run; dead links were not checked._")
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"## Stale or archived repositories ({len(stale)})")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Archived, or no push in over 18 months. "
|
||||
"Consider marking with the skull icon or delisting."
|
||||
)
|
||||
if stale:
|
||||
lines.append("")
|
||||
for slug, reason in stale:
|
||||
lines.append(f"- {slug} - {reason}")
|
||||
else:
|
||||
lines.append("")
|
||||
lines.append("None found.")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"_Some entries may be transient. Verify before acting._"
|
||||
)
|
||||
|
||||
with open("health-report.md", "w", encoding="utf-8") as fh:
|
||||
fh.write("\n".join(lines) + "\n")
|
||||
|
||||
has_findings = bool(dead or stale or not dead_ok)
|
||||
out = os.environ.get("GITHUB_OUTPUT")
|
||||
if out:
|
||||
with open(out, "a") as fh:
|
||||
fh.write(f"has_findings={'true' if has_findings else 'false'}\n")
|
||||
fh.write(f"date={today}\n")
|
||||
print(f"Dead links: {len(dead)} | Stale repos: {len(stale)}")
|
||||
PY
|
||||
run: python3 scripts/health_report.py --lychee lychee/out.json --out health-report.md
|
||||
|
||||
- name: Ensure maintenance label exists
|
||||
if: steps.scan.outputs.has_findings == 'true'
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Monthly health + open-source audit of README.md.
|
||||
|
||||
One pass over every repository linked in the list, reporting:
|
||||
|
||||
- Dead links, parsed from a lychee JSON report (produced by the workflow step).
|
||||
- Repository issues: gone, archived, stale (no push in 18+ months), unlicensed,
|
||||
or no real source code (only binaries, images, or documentation, the
|
||||
"blob posing as open source" case).
|
||||
|
||||
Repos already marked unmaintained (💀) in the list are shown in a separate
|
||||
section and do not, on their own, trigger a new issue. Every finding is a signal
|
||||
for human review, not an automatic verdict.
|
||||
|
||||
Usage:
|
||||
python3 scripts/health_report.py [--readme README.md]
|
||||
[--lychee lychee/out.json] [--limit N] [--out FILE]
|
||||
|
||||
Environment:
|
||||
GITHUB_TOKEN / GH_TOKEN raises the GitHub rate limit from 60 to 5000 req/hr
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
STALE_DAYS = 18 * 30 # ~18 months
|
||||
REPO_RE = re.compile(
|
||||
r"https?://(?:www\.)?(github\.com|codeberg\.org)/"
|
||||
r"([A-Za-z0-9._-]+)/([A-Za-z0-9._-]+)"
|
||||
)
|
||||
RESERVED = {
|
||||
"sponsors", "orgs", "topics", "about", "features", "marketplace", "settings",
|
||||
"explore", "collections", "apps", "notifications", "login", "join", "pricing",
|
||||
"security", "site", "contact", "readme", "search", "new", "watching", "stars",
|
||||
}
|
||||
# The list's own repo (badges, mirror, edit and issue links) is not a listed tool.
|
||||
SELF_REPOS = {
|
||||
("github.com", "pluja", "awesome-privacy"),
|
||||
("codeberg.org", "pluja", "awesome-privacy"),
|
||||
}
|
||||
# Linguist "languages" that are documentation or markup, not real source code.
|
||||
DOC_ONLY = {
|
||||
"Markdown", "Text", "reStructuredText", "AsciiDoc", "Org", "TeX",
|
||||
"Roff", "Rich Text Format",
|
||||
}
|
||||
|
||||
|
||||
def get_json(url, token=None, timeout=20):
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Accept", "application/json")
|
||||
req.add_header("User-Agent", "awesome-privacy-health-report")
|
||||
if token:
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.load(resp), None
|
||||
except urllib.error.HTTPError as exc:
|
||||
return None, exc.code
|
||||
except Exception:
|
||||
return None, "error"
|
||||
|
||||
|
||||
def parse_ts(value):
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return dt.datetime.strptime(value.replace("Z", "+0000"), "%Y-%m-%dT%H:%M:%S%z")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _clean(host, owner, repo):
|
||||
if owner.lower() in RESERVED:
|
||||
return None
|
||||
repo = repo[:-4] if repo.rstrip(".").endswith(".git") else repo.rstrip(".")
|
||||
if not repo:
|
||||
return None
|
||||
key = (host, owner.lower(), repo.lower())
|
||||
return None if key in SELF_REPOS else (key, (host, owner, repo))
|
||||
|
||||
|
||||
def extract_repos(readme):
|
||||
repos = {}
|
||||
for host, owner, repo in REPO_RE.findall(readme):
|
||||
cleaned = _clean(host, owner, repo)
|
||||
if cleaned:
|
||||
key, value = cleaned
|
||||
repos.setdefault(key, value)
|
||||
return sorted(repos.values())
|
||||
|
||||
|
||||
def extract_skull_repos(readme):
|
||||
skull = set()
|
||||
for line in readme.splitlines():
|
||||
if "💀" not in line:
|
||||
continue
|
||||
for host, owner, repo in REPO_RE.findall(line):
|
||||
cleaned = _clean(host, owner, repo)
|
||||
if cleaned:
|
||||
skull.add(cleaned[0])
|
||||
return skull
|
||||
|
||||
|
||||
def source_concern(langs):
|
||||
"""Flag a repo whose 'source' is only binaries, images, or documentation."""
|
||||
if langs is None:
|
||||
return []
|
||||
if not langs:
|
||||
return ["no source code detected (only binaries, images, or empty)"]
|
||||
if not [l for l in langs if l not in DOC_ONLY]:
|
||||
return ["only documentation or markup (" + ", ".join(sorted(langs)) + ")"]
|
||||
return []
|
||||
|
||||
|
||||
def check_repo(host, owner, repo, now, token):
|
||||
"""Combined staleness + openness reasons for one repo (empty = healthy)."""
|
||||
if host == "github.com":
|
||||
info, err = get_json(f"https://api.github.com/repos/{owner}/{repo}", token)
|
||||
else:
|
||||
info, err = get_json(f"https://codeberg.org/api/v1/repos/{owner}/{repo}")
|
||||
if err == 403:
|
||||
return ["__ratelimit__"]
|
||||
if err in (404, 410):
|
||||
return ["repository not found"]
|
||||
if err or info is None:
|
||||
return [] # transient; do not flag on a network blip
|
||||
|
||||
reasons = []
|
||||
if info.get("archived"):
|
||||
reasons.append("archived")
|
||||
last = parse_ts(info.get("pushed_at") if host == "github.com" else info.get("updated_at"))
|
||||
if last is not None and (now - last).days > STALE_DAYS:
|
||||
reasons.append(f"no push since {last.date()} (~{(now - last).days // 30} months)")
|
||||
if host == "github.com" and not info.get("license"):
|
||||
reasons.append("no license") # Codeberg/Gitea does not expose this reliably
|
||||
|
||||
if host == "github.com":
|
||||
langs, lerr = get_json(f"https://api.github.com/repos/{owner}/{repo}/languages", token)
|
||||
if lerr == 403:
|
||||
return ["__ratelimit__"]
|
||||
else:
|
||||
langs, _ = get_json(f"https://codeberg.org/api/v1/repos/{owner}/{repo}/languages")
|
||||
reasons += source_concern(langs)
|
||||
return reasons
|
||||
|
||||
|
||||
def scan_repos(repos, skull_set, now, token, limit=0):
|
||||
findings, checked, rate_limited = [], 0, False
|
||||
for host, owner, repo in repos:
|
||||
if limit and checked >= limit:
|
||||
break
|
||||
checked += 1
|
||||
reasons = check_repo(host, owner, repo, now, token)
|
||||
if reasons == ["__ratelimit__"]:
|
||||
rate_limited, checked = True, checked - 1
|
||||
break
|
||||
if reasons:
|
||||
key = (host, owner.lower(), repo.lower())
|
||||
findings.append((f"{host}/{owner}/{repo}", "; ".join(reasons), key in skull_set))
|
||||
time.sleep(0.05)
|
||||
return findings, checked, rate_limited
|
||||
|
||||
|
||||
def parse_dead_links(path):
|
||||
"""Return (dead_links, scan_ran). dead_links = [(url, reason), ...]."""
|
||||
try:
|
||||
with open(path) as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return [], False
|
||||
except Exception as exc:
|
||||
print(f"::warning::could not parse lychee output ({exc})", file=sys.stderr)
|
||||
return [], False
|
||||
dead, seen = [], set()
|
||||
emap = data.get("error_map") or data.get("fail_map") or {}
|
||||
for entries in emap.values():
|
||||
for item in entries:
|
||||
url = item.get("url", "")
|
||||
status = item.get("status") or {}
|
||||
text = status.get("text") or status.get("details") or "error"
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
dead.append((url, text))
|
||||
return dead, True
|
||||
|
||||
|
||||
def build_report(today, dead, dead_ok, findings, checked, rate_limited):
|
||||
new = [(s, w) for s, w, sk in findings if not sk]
|
||||
known = [(s, w) for s, w, sk in findings if sk]
|
||||
L = [f"# Health report {today}", "", "Automated monthly scan of `README.md`. Every",
|
||||
"finding is a signal for human review, not an automatic verdict.", ""]
|
||||
|
||||
L.append(f"## Dead links ({len(dead)})")
|
||||
L.append("")
|
||||
if dead:
|
||||
L += [f"- {url} - {text}" for url, text in dead]
|
||||
elif dead_ok:
|
||||
L.append("None found.")
|
||||
else:
|
||||
L.append("_Link scan produced no output this run; dead links were not checked._")
|
||||
L.append("")
|
||||
|
||||
L.append(f"## Repository issues ({len(new)} new, {len(known)} already 💀, {checked} checked)")
|
||||
L.append("")
|
||||
L.append("Repos that are gone, archived, stale (no push in 18+ months), unlicensed,")
|
||||
L.append("or have no real source code (only binaries, images, or documentation).")
|
||||
L.append("Review before delisting, marking 💀, or trusting an \"open source\" claim.")
|
||||
L.append("")
|
||||
if new:
|
||||
L += [f"- {slug} - {why}" for slug, why in new]
|
||||
else:
|
||||
L.append("No new findings.")
|
||||
if rate_limited:
|
||||
L.append("")
|
||||
L.append("_Stopped early: GitHub rate limit hit. CI runs with a token for the full pass._")
|
||||
if known:
|
||||
L += ["", "### Already marked unmaintained (💀)", "",
|
||||
"These already carry the skull in the list. Shown for completeness.", ""]
|
||||
L += [f"- {slug} - {why}" for slug, why in known]
|
||||
L += ["", "_Some entries may be transient. Verify before acting._", ""]
|
||||
return "\n".join(L) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--readme", default="README.md")
|
||||
ap.add_argument("--lychee", default="lychee/out.json")
|
||||
ap.add_argument("--limit", type=int, default=0, help="cap repos checked (0 = all)")
|
||||
ap.add_argument("--out", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
today = now.date().isoformat()
|
||||
token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
|
||||
|
||||
with open(args.readme, encoding="utf-8") as fh:
|
||||
readme = fh.read()
|
||||
repos = extract_repos(readme)
|
||||
skull_set = extract_skull_repos(readme)
|
||||
print(f"[health] {len(repos)} repos, {len(skull_set)} already 💀", file=sys.stderr)
|
||||
|
||||
dead, dead_ok = parse_dead_links(args.lychee)
|
||||
findings, checked, rate_limited = scan_repos(repos, skull_set, now, token, args.limit)
|
||||
|
||||
report = build_report(today, dead, dead_ok, findings, checked, rate_limited)
|
||||
if args.out:
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
fh.write(report)
|
||||
print(f"[health] wrote {args.out}", file=sys.stderr)
|
||||
else:
|
||||
sys.stdout.write(report)
|
||||
|
||||
# New (non-skull) repo problems or any dead links open an issue.
|
||||
new_findings = [f for f in findings if not f[2]]
|
||||
has_findings = bool(dead) or bool(new_findings) or not dead_ok
|
||||
gh_out = os.environ.get("GITHUB_OUTPUT")
|
||||
if gh_out:
|
||||
with open(gh_out, "a") as fh:
|
||||
fh.write(f"has_findings={'true' if has_findings else 'false'}\n")
|
||||
fh.write(f"date={today}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user