diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 258c39d..0ec8d22 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -8,7 +8,10 @@ jobs: check: runs-on: ubuntu-latest steps: + # gitleaks reads the commit history, so a shallow clone would hide it. - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/.gitleaks-baseline.json b/.gitleaks-baseline.json new file mode 100644 index 0000000..67f0f4c --- /dev/null +++ b/.gitleaks-baseline.json @@ -0,0 +1,282 @@ +[ + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 82, + "EndLine": 82, + "StartColumn": 2, + "EndColumn": 52, + "Match": "yamtrack_secret: \"REDACTED\"", + "Secret": "REDACTED", + "File": "yamtrack/vars/default.yml", + "SymlinkFile": "", + "Commit": "af2e7394b8b2882b0a1f3cf74976dc1fafd610d0", + "Entropy": 3.8042293, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:53Z", + "Message": "Add the Yamtrack playbook", + "Tags": [], + "Fingerprint": "af2e7394b8b2882b0a1f3cf74976dc1fafd610d0:yamtrack/vars/default.yml:generic-api-key:82" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 86, + "EndLine": 86, + "StartColumn": 2, + "EndColumn": 40, + "Match": "auth_secret: \"REDACTED\"", + "Secret": "REDACTED", + "File": "sms-wa-bridge/vars/default.yml", + "SymlinkFile": "", + "Commit": "26a0654fea2b85cfcc1587418834f2358f384ae9", + "Entropy": 3.6572683, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:53Z", + "Message": "Add the sms-wa-bridge playbook", + "Tags": [], + "Fingerprint": "26a0654fea2b85cfcc1587418834f2358f384ae9:sms-wa-bridge/vars/default.yml:generic-api-key:86" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 17, + "EndLine": 17, + "StartColumn": 2, + "EndColumn": 77, + "Match": "secret_key: REDACTED", + "Secret": "REDACTED", + "File": "outline/vars/default.yml", + "SymlinkFile": "", + "Commit": "250456e3fe1ca58fa836fe182c0519327bba898c", + "Entropy": 3.7609768, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:52Z", + "Message": "Add the Outline playbook", + "Tags": [], + "Fingerprint": "250456e3fe1ca58fa836fe182c0519327bba898c:outline/vars/default.yml:generic-api-key:17" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 18, + "EndLine": 18, + "StartColumn": 2, + "EndColumn": 79, + "Match": "utils_secret: REDACTED", + "Secret": "REDACTED", + "File": "outline/vars/default.yml", + "SymlinkFile": "", + "Commit": "250456e3fe1ca58fa836fe182c0519327bba898c", + "Entropy": 3.9114883, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:52Z", + "Message": "Add the Outline playbook", + "Tags": [], + "Fingerprint": "250456e3fe1ca58fa836fe182c0519327bba898c:outline/vars/default.yml:generic-api-key:18" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 9, + "EndLine": 9, + "StartColumn": 2, + "EndColumn": 75, + "Match": "password: REDACTED", + "Secret": "REDACTED", + "File": "registry/vars/default.yml", + "SymlinkFile": "", + "Commit": "f04a29b5c7050fa4404e9fc94e3489211c9dff50", + "Entropy": 3.8824525, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2024-12-17T12:23:30Z", + "Message": "Docker registry", + "Tags": [], + "Fingerprint": "f04a29b5c7050fa4404e9fc94e3489211c9dff50:registry/vars/default.yml:generic-api-key:9" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 394, + "EndLine": 394, + "StartColumn": 8, + "EndColumn": 40, + "Match": "X-Auth: REDACTED'", + "Secret": "REDACTED", + "File": "sms-wa-bridge/readme.md", + "SymlinkFile": "", + "Commit": "26a0654fea2b85cfcc1587418834f2358f384ae9", + "Entropy": 3.6572683, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:53Z", + "Message": "Add the sms-wa-bridge playbook", + "Tags": [], + "Fingerprint": "26a0654fea2b85cfcc1587418834f2358f384ae9:sms-wa-bridge/readme.md:generic-api-key:394" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 408, + "EndLine": 408, + "StartColumn": 48, + "EndColumn": 80, + "Match": "X-Auth: REDACTED`", + "Secret": "REDACTED", + "File": "sms-wa-bridge/readme.md", + "SymlinkFile": "", + "Commit": "26a0654fea2b85cfcc1587418834f2358f384ae9", + "Entropy": 3.6572683, + "Author": "Amritanshu", + "Email": "git@tanshu.com", + "Date": "2026-10-18T16:05:53Z", + "Message": "Add the sms-wa-bridge playbook", + "Tags": [], + "Fingerprint": "26a0654fea2b85cfcc1587418834f2358f384ae9:sms-wa-bridge/readme.md:generic-api-key:408" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 2, + "EndLine": 2, + "StartColumn": 2, + "EndColumn": 75, + "Match": "API_TOKEN=REDACTED", + "Secret": "REDACTED", + "File": "bifrost/app/.env", + "SymlinkFile": "", + "Commit": "02cc1dafe599f7ceeba7ad79669568aa83970ccc", + "Entropy": 3.589582, + "Author": "tanshu", + "Email": "tanshu@gmail.com", + "Date": "2020-05-04T11:34:09Z", + "Message": "Bifrost deployed using ansible", + "Tags": [], + "Fingerprint": "02cc1dafe599f7ceeba7ad79669568aa83970ccc:bifrost/app/.env:generic-api-key:2" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 13, + "EndLine": 13, + "StartColumn": 2, + "EndColumn": 76, + "Match": "SECRET_KEY=REDACTED", + "Secret": "REDACTED", + "File": "brewman/files/.env-exp", + "SymlinkFile": "", + "Commit": "ed8feff302e70deb575295a15b81b74bb89b8e6e", + "Entropy": 3.8843737, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:57:59Z", + "Message": "New: Fastapi Brewman Fastapi Barker", + "Tags": [], + "Fingerprint": "ed8feff302e70deb575295a15b81b74bb89b8e6e:brewman/files/.env-exp:generic-api-key:13" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 13, + "EndLine": 13, + "StartColumn": 2, + "EndColumn": 76, + "Match": "SECRET_KEY=REDACTED", + "Secret": "REDACTED", + "File": "brewman/files/.env-acc", + "SymlinkFile": "", + "Commit": "ed8feff302e70deb575295a15b81b74bb89b8e6e", + "Entropy": 3.887898, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:57:59Z", + "Message": "New: Fastapi Brewman Fastapi Barker", + "Tags": [], + "Fingerprint": "ed8feff302e70deb575295a15b81b74bb89b8e6e:brewman/files/.env-acc:generic-api-key:13" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 13, + "EndLine": 13, + "StartColumn": 2, + "EndColumn": 76, + "Match": "SECRET_KEY=REDACTED", + "Secret": "REDACTED", + "File": "brewman/files/.env-mhl", + "SymlinkFile": "", + "Commit": "ed8feff302e70deb575295a15b81b74bb89b8e6e", + "Entropy": 3.8439462, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:57:59Z", + "Message": "New: Fastapi Brewman Fastapi Barker", + "Tags": [], + "Fingerprint": "ed8feff302e70deb575295a15b81b74bb89b8e6e:brewman/files/.env-mhl:generic-api-key:13" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 8, + "EndLine": 8, + "StartColumn": 2, + "EndColumn": 76, + "Match": "SECRET_KEY=REDACTED", + "Secret": "REDACTED", + "File": "barker/files/.env", + "SymlinkFile": "", + "Commit": "ed8feff302e70deb575295a15b81b74bb89b8e6e", + "Entropy": 3.8627045, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:57:59Z", + "Message": "New: Fastapi Brewman Fastapi Barker", + "Tags": [], + "Fingerprint": "ed8feff302e70deb575295a15b81b74bb89b8e6e:barker/files/.env:generic-api-key:8" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 13, + "EndLine": 13, + "StartColumn": 2, + "EndColumn": 76, + "Match": "SECRET_KEY=REDACTED", + "Secret": "REDACTED", + "File": "brewman/files/.env-hops", + "SymlinkFile": "", + "Commit": "ed8feff302e70deb575295a15b81b74bb89b8e6e", + "Entropy": 3.8454638, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:57:59Z", + "Message": "New: Fastapi Brewman Fastapi Barker", + "Tags": [], + "Fingerprint": "ed8feff302e70deb575295a15b81b74bb89b8e6e:brewman/files/.env-hops:generic-api-key:13" + }, + { + "RuleID": "generic-api-key", + "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", + "StartLine": 2, + "EndLine": 2, + "StartColumn": 2, + "EndColumn": 81, + "Match": "SECRET_KEY_BASE=REDACTED", + "Secret": "REDACTED", + "File": "snibox/files/.env", + "SymlinkFile": "", + "Commit": "5922227712b37dfd0e67df2a2b5334998d1f2702", + "Entropy": 3.8725784, + "Author": "tanshu", + "Email": "git@tanshu.com", + "Date": "2020-10-17T06:58:52Z", + "Message": "New: Snibox", + "Tags": [], + "Fingerprint": "5922227712b37dfd0e67df2a2b5334998d1f2702:snibox/files/.env:generic-api-key:2" + } +] diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..883da45 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,10 @@ +[extend] +useDefault = true + +[allowlist] +description = "Caches and vendored collections are not repo content" +paths = [ + '''\.ansible/''', + '''\.scratch/''', + '''\.gitleaks-baseline\.json''', +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 76fa5ce..a5fac49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ --- -# Local hook, same checks as CI. Enable with: +# Local hooks, same checks as CI. Enable with: # pre-commit install +# pre-commit install --hook-type commit-msg repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -18,3 +19,15 @@ repos: rev: v24.7.0 hooks: - id: ansible-lint + - repo: local + hooks: + - id: gitleaks + name: gitleaks + entry: scripts/check-secrets.sh --staged + language: script + pass_filenames: false + - id: commit-message + name: commit message + entry: scripts/check-commit-msg.sh + language: script + stages: [commit-msg] diff --git a/AGENTS.md b/AGENTS.md index 48f0691..62b1c07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,14 +3,18 @@ This repository holds one Ansible playbook per service. Read [CODING_STANDARDS.md](CODING_STANDARDS.md) before adding or changing a playbook. -- Layout and naming: [CODING_STANDARDS.md](CODING_STANDARDS.md). -- Checks: run `./scripts/check.sh`. It runs yamllint, ansible-lint, and a syntax - check on every `*/playbook.yml`. CI runs the same script. +- Checks: run `./scripts/check.sh`. Add `--changed` while iterating to lint only + the YAML files that differ from HEAD. CI runs the full script. - Review baseline: `origin/main`. The review diff is `git diff origin/main...HEAD`. - Migration spec: [docs/podman-migration.md](docs/podman-migration.md). Section 8 lists the repo changes the migration expects. - Issue tracker: [docs/agents/issue-tracker.md](docs/agents/issue-tracker.md). - Scratch files: write them under `.scratch/`. It is git-ignored, and `/tmp` is wiped between tool calls in this environment. -- Secrets: `vars/default.yml` holds deploy values, including passwords. Do not - copy a production value into a file you push to a public remote. +- Read-only `$HOME`: if `$HOME` is read-only, point `XDG_CACHE_HOME`, + `UV_CACHE_DIR`, `PIP_CACHE_DIR`, and `ANSIBLE_HOME` under `.scratch/`. The check + script already does this, and a vault password file has to live there too. +- Secrets: `vars/default.yml` holds deploy values, including passwords. gitleaks + runs before each commit and in CI. Known values are in + [.gitleaks-baseline.json](.gitleaks-baseline.json); rotate or vault a value + instead of adding a new entry. diff --git a/scripts/check-commit-msg.sh b/scripts/check-commit-msg.sh new file mode 100755 index 0000000..299227f --- /dev/null +++ b/scripts/check-commit-msg.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Reject placeholder commit subjects. pre-commit passes the message file. +set -euo pipefail + +message_file="$1" +subject=$(head -n 1 "$message_file") + +if [ -z "${subject//[[:space:]]/}" ]; then + echo "commit subject is empty" >&2 + exit 1 +fi + +case "${subject,,}" in + misc|misc.|wip|wip.|update|updates|fix|fixes|change|changes|stuff|temp|tmp) + echo "commit subject is a placeholder: '$subject'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/scripts/check-secrets.sh b/scripts/check-secrets.sh new file mode 100755 index 0000000..30745fc --- /dev/null +++ b/scripts/check-secrets.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Scan for secrets. Known values are listed in the redacted baseline. +# Usage: scripts/check-secrets.sh [--staged] +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +version="8.30.1" +bin="$PWD/.scratch/bin/gitleaks" + +if ! command -v gitleaks >/dev/null 2>&1; then + if [ ! -x "$bin" ]; then + case "$(uname -m)" in + x86_64|amd64) arch=x64 ;; + aarch64|arm64) arch=arm64 ;; + *) + echo "unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + echo "installing gitleaks $version into .scratch/bin" >&2 + mkdir -p .scratch/bin + curl -fsSL -o .scratch/bin/gitleaks.tar.gz \ + "https://github.com/gitleaks/gitleaks/releases/download/v${version}/gitleaks_${version}_linux_${arch}.tar.gz" + tar -xzf .scratch/bin/gitleaks.tar.gz -C .scratch/bin gitleaks + fi + PATH="$PWD/.scratch/bin:$PATH" +fi + +if [ "${1:-}" = "--staged" ]; then + gitleaks git --pre-commit --staged --redact --verbose \ + --baseline-path .gitleaks-baseline.json +else + gitleaks git --baseline-path .gitleaks-baseline.json --redact --no-banner +fi diff --git a/scripts/check.sh b/scripts/check.sh index 1749d3d..4947eab 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -1,18 +1,35 @@ #!/usr/bin/env bash # Run every check this repo has. CI and pre-commit call the same script. +# Usage: scripts/check.sh [--changed] set -euo pipefail +shopt -s nullglob cd "$(dirname "${BASH_SOURCE[0]}")/.." -# Ansible and uv both want a writable cache. Keep them in the checkout so the -# script works when $HOME is read-only, which is the case in the agent sandbox. +changed=0 +for arg in "$@"; do + case "$arg" in + --changed) changed=1 ;; + *) + echo "unknown argument: $arg" >&2 + echo "usage: scripts/check.sh [--changed]" >&2 + exit 2 + ;; + esac +done + +# Ansible and uv want writable caches. Keep them in the checkout so the script +# works when $HOME is read-only, which is the case in the agent sandbox. export ANSIBLE_CONFIG="${ANSIBLE_CONFIG:-$PWD/ansible.cfg}" export ANSIBLE_HOME="${ANSIBLE_HOME:-$PWD/.ansible/home}" export ANSIBLE_LOCAL_TEMP="${ANSIBLE_LOCAL_TEMP:-$PWD/.ansible/tmp}" +export ANSIBLE_GALAXY_CACHE_DIR="${ANSIBLE_GALAXY_CACHE_DIR:-$PWD/.ansible/galaxy_cache}" export UV_CACHE_DIR="${UV_CACHE_DIR:-$PWD/.ansible/uv-cache}" export UV_TOOL_DIR="${UV_TOOL_DIR:-$PWD/.ansible/uv-tools}" -export ANSIBLE_GALAXY_CACHE_DIR="${ANSIBLE_GALAXY_CACHE_DIR:-$PWD/.ansible/galaxy_cache}" -mkdir -p "$ANSIBLE_LOCAL_TEMP" "$UV_CACHE_DIR" "$UV_TOOL_DIR" "$ANSIBLE_GALAXY_CACHE_DIR" "$ANSIBLE_HOME" +export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$PWD/.scratch/cache}" +export PIP_CACHE_DIR="${PIP_CACHE_DIR:-$PWD/.scratch/cache/pip}" +mkdir -p "$ANSIBLE_LOCAL_TEMP" "$UV_CACHE_DIR" "$UV_TOOL_DIR" \ + "$ANSIBLE_GALAXY_CACHE_DIR" "$ANSIBLE_HOME" "$XDG_CACHE_HOME" "$PIP_CACHE_DIR" if [ -f requirements.yml ]; then echo "== ansible-galaxy collection install ==" @@ -29,15 +46,90 @@ run() { fi } -echo "== yamllint ==" -run yamllint . +# YAML files that differ from HEAD, including new files that are not staged yet. +yaml_files=() +if [ "$changed" = 1 ]; then + while IFS= read -r file; do + [ -n "$file" ] && yaml_files+=("$file") + done < <( + git diff --name-only --diff-filter=ACMR HEAD -- '*.yml' '*.yaml' + git ls-files --others --exclude-standard -- '*.yml' '*.yaml' + ) +fi -echo "== ansible-lint ==" -run ansible-lint +# ansible-lint only understands Ansible files, so keep the changed files that +# live under a service with a playbook. +ansible_files=() +if [ "$changed" = 1 ]; then + for file in "${yaml_files[@]}"; do + service="${file%%/*}" + if [ -f "$service/playbook.yml" ]; then + ansible_files+=("$file") + fi + done +fi + +if [ "$changed" = 1 ] && [ "${#yaml_files[@]}" -eq 0 ]; then + echo "== changed mode: no YAML files changed ==" +else + echo "== yamllint ==" + if [ "$changed" = 1 ]; then + run yamllint "${yaml_files[@]}" + else + run yamllint . + fi + + echo "== ansible-lint ==" + if [ "$changed" = 1 ] && [ "${#ansible_files[@]}" -eq 0 ]; then + echo "no changed Ansible files" + elif [ "$changed" = 1 ]; then + run ansible-lint "${ansible_files[@]}" + else + run ansible-lint + fi +fi + +echo "== layout ==" +layout_ok=1 +for playbook in */playbook.yml; do + service="${playbook%/playbook.yml}" + if [ ! -s "$service/readme.md" ]; then + echo "missing or empty $service/readme.md" >&2 + layout_ok=0 + fi +done +for entrypoint in */roles/*/tasks/main.yml */roles/*/handlers/main.yml; do + echo "role entrypoint must use .yaml: $entrypoint" >&2 + layout_ok=0 +done +if [ "$layout_ok" -ne 1 ]; then + exit 1 +fi + +echo "== gitleaks ==" +./scripts/check-secrets.sh echo "== ansible-playbook --syntax-check ==" -shopt -s nullglob -for playbook in */playbook.yml; do - echo "-- $playbook" - ansible-playbook --syntax-check "$playbook" -done +playbooks=() +if [ "$changed" = 1 ]; then + for file in "${yaml_files[@]}"; do + service="${file%%/*}" + if [ -f "$service/playbook.yml" ]; then + playbooks+=("$service/playbook.yml") + fi + done + if [ "${#playbooks[@]}" -gt 0 ]; then + mapfile -t playbooks < <(printf '%s\n' "${playbooks[@]}" | sort -u) + fi +else + playbooks=(*/playbook.yml) +fi + +if [ "${#playbooks[@]}" -eq 0 ]; then + echo "no playbooks to check" +else + for playbook in "${playbooks[@]}"; do + echo "-- $playbook" + ansible-playbook --syntax-check "$playbook" + done +fi