diff --git a/scripts/lib/push-protection.sh b/scripts/lib/push-protection.sh index df3ed364..8cc0c150 100644 --- a/scripts/lib/push-protection.sh +++ b/scripts/lib/push-protection.sh @@ -42,11 +42,19 @@ PP_REQUIRED_SA_SETTINGS=( "secret_scanning:enabled:error:Secret scanning must be enabled" "secret_scanning_push_protection:enabled:error:Secret scanning push protection must be enabled" - "secret_scanning_ai_detection:enabled:warning:Secret scanning AI detection should be enabled" - "secret_scanning_non_provider_patterns:enabled:warning:Secret scanning non-provider patterns should be enabled" + "secret_scanning_ai_detection:enabled:warning:Secret scanning AI detection should be enabled (requires GitHub Copilot subscription; absent means the feature is unavailable for the current org plan)" + "secret_scanning_non_provider_patterns:enabled:warning:Secret scanning non-provider patterns should be enabled (requires GitHub Advanced Security)" "dependabot_security_updates:enabled:warning:Dependabot security updates should be enabled" ) +# Plan-gated keys that cannot be configured without a plan upgrade. A null/absent +# status for these keys is not actionable and should not generate a finding. +PP_PLAN_GATED_KEYS=( + "secret_scanning_ai_detection" + "secret_scanning_non_provider_patterns" + "dependabot_security_updates" +) + # Minimum entries that every repo's .gitignore MUST contain. Every repo # starting from the org baseline at /.gitignore satisfies these by default. PP_REQUIRED_GITIGNORE_PATTERNS=( @@ -115,7 +123,34 @@ pp_apply_security_and_analysis() { full_payload=$(echo "$payload" | jq '{security_and_analysis: .}') if echo "$full_payload" | gh api -X PATCH "repos/$ORG/$repo" --input - > /dev/null 2>&1; then - ok "$ORG/$repo security_and_analysis updated successfully" + ok "$ORG/$repo security_and_analysis PATCH accepted" + + # Verify each patched key was actually applied. Some settings (e.g. + # secret_scanning_ai_detection, secret_scanning_non_provider_patterns) + # require GitHub Advanced Security or a Copilot subscription; the API + # accepts the PATCH (HTTP 200) but silently ignores keys that the org + # plan does not support. Re-fetch and warn for any key still not applied. + local post_patch post_actuals + post_patch=$(gh api "repos/$ORG/$repo" --jq '.security_and_analysis // {}' 2>/dev/null || echo "{}") + post_actuals=$(echo "$post_patch" | jq -r 'to_entries | .[] | "\(.key):\(.value.status? // "null")"' 2>/dev/null || echo "") + + local -A actuals + while IFS=':' read -r k v; do + if [ -n "$k" ]; then + actuals["$k"]="$v" + fi + done <<< "$post_actuals" + + local post_entry post_key post_expected post_actual + for post_entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do + IFS=':' read -r post_key post_expected _ _ <<< "$post_entry" + post_actual="${actuals[$post_key]:-null}" + if [ "$post_actual" != "$post_expected" ]; then + info " $post_key still not $post_expected after PATCH — the org plan may not support this feature (current: $post_actual)" + else + ok " $post_key: $post_actual (verified)" + fi + done else err "Failed to PATCH security_and_analysis for $ORG/$repo — the authenticated token must have repository admin permissions (or the org plan may not support these features)" return 1 @@ -167,15 +202,29 @@ pp_check_security_and_analysis() { return fi - local entry key expected severity detail actual + local entry key expected severity detail actual is_plan_gated for entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do IFS=':' read -r key expected severity detail <<< "$entry" actual=$(echo "$sa" | jq -r ".\"$key\".status // \"null\"") - if [ "$actual" != "$expected" ]; then - add_finding "$repo" "push-protection" "$key" "$severity" \ - "$detail (current: \`$actual\`, expected: \`$expected\`)" \ - "$PP_STANDARD_REF#required-repo-level-settings" + if [ "$actual" = "$expected" ]; then + continue + fi + # A null/absent status for a plan-gated key means the feature is unavailable + # for the current org plan — skip rather than creating a non-actionable + # compliance finding that cannot be remediated without a plan upgrade. + is_plan_gated=false + for plan_key in "${PP_PLAN_GATED_KEYS[@]}"; do + if [ "$key" = "$plan_key" ]; then + is_plan_gated=true + break + fi + done + if [ "$is_plan_gated" = true ] && [ "$actual" = "null" ]; then + continue fi + add_finding "$repo" "push-protection" "$key" "$severity" \ + "$detail (current: \`$actual\`, expected: \`$expected\`)" \ + "$PP_STANDARD_REF#required-repo-level-settings" done } diff --git a/test/scripts/lib/helpers/setup.bash b/test/scripts/lib/helpers/setup.bash new file mode 100644 index 00000000..e0696f47 --- /dev/null +++ b/test/scripts/lib/helpers/setup.bash @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Common test helpers for scripts/lib bats suites. + +TT_REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../../.." && pwd)" +export TT_REPO_ROOT + +TT_SCRIPTS_LIB_DIR="${TT_REPO_ROOT}/scripts/lib" +export TT_SCRIPTS_LIB_DIR + +tt_make_tmpdir() { + TT_TMP="$(mktemp -d)" + export TT_TMP +} + +tt_cleanup_tmpdir() { + if [ -n "${TT_TMP:-}" ] && [ -d "${TT_TMP}" ]; then + rm -rf "${TT_TMP}" + fi +} diff --git a/test/scripts/lib/push-protection.bats b/test/scripts/lib/push-protection.bats new file mode 100644 index 00000000..3ca26231 --- /dev/null +++ b/test/scripts/lib/push-protection.bats @@ -0,0 +1,208 @@ +#!/usr/bin/env bats +# Tests for scripts/lib/push-protection.sh — pp_check_security_and_analysis() +# +# Covers the plan-gated feature handling: warning-severity settings whose API +# status is null (absent from the response) must NOT generate a finding, since +# a null status indicates the org plan does not support the feature and there +# is nothing the operator can do to remediate it. + +load 'helpers/setup' + +# --------------------------------------------------------------------------- +# Helpers: build repo-API JSON payloads +# --------------------------------------------------------------------------- + +# Build the full repo JSON object that gh api "repos/ORG/REPO" returns. +# Accepts an inline security_and_analysis JSON fragment as $1. +# The mock gh_api then applies ".security_and_analysis // {}" to this object. +make_repo_json() { + local sa_json="$1" + printf '{"security_and_analysis":%s}' "$sa_json" +} + +# --------------------------------------------------------------------------- +# Test setup / teardown +# --------------------------------------------------------------------------- + +setup() { + tt_make_tmpdir + + # Log file for add_finding calls — each call appends one line: + # "||||" + FINDINGS_LOG="${TT_TMP}/findings.log" + export FINDINGS_LOG + + # Default: gh_api returns a fully-compliant security_and_analysis object + # wrapped in the full repo JSON so the jq filter ".security_and_analysis // {}" + # resolves correctly. + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + export GH_API_RESPONSE + + # Second-call response for the secret-scanning alerts proxy check + # (used only when the primary call returns {}). + GH_API_ALERTS_RESPONSE='[]' + export GH_API_ALERTS_RESPONSE + + ORG="test-org" + export ORG + + # Track how many times gh_api has been called so we can return different + # responses for the primary (repo) call vs. the proxy (alerts) call. + GH_API_CALL_COUNT=0 + export GH_API_CALL_COUNT + + gh_api() { + GH_API_CALL_COUNT=$((GH_API_CALL_COUNT + 1)) + local endpoint="$1"; shift + local jq_filter="" + while [ $# -gt 0 ]; do + if [ "$1" = "--jq" ] && [ $# -gt 1 ]; then + jq_filter="$2"; shift 2 + else + shift + fi + done + # First call is always the primary repo endpoint. + # Subsequent calls are the alerts proxy. + local response + if [ "$GH_API_CALL_COUNT" -eq 1 ]; then + response="$GH_API_RESPONSE" + else + response="$GH_API_ALERTS_RESPONSE" + fi + if [ -n "$jq_filter" ]; then + printf '%s' "$response" | jq -r "$jq_filter" 2>/dev/null || echo "{}" + else + printf '%s' "$response" + fi + } + export -f gh_api + + add_finding() { + local repo="$1" category="$2" check="$3" severity="$4" detail="$5" + printf '%s|%s|%s|%s|%s\n' "$repo" "$category" "$check" "$severity" "$detail" \ + >>"$FINDINGS_LOG" + } + export -f add_finding + + # shellcheck source=/dev/null + . "${TT_SCRIPTS_LIB_DIR}/push-protection.sh" +} + +teardown() { + tt_cleanup_tmpdir +} + +findings() { + if [ -f "$FINDINGS_LOG" ]; then + cat "$FINDINGS_LOG" + else + echo "" + fi +} + +finding_count() { + if [ -f "$FINDINGS_LOG" ]; then + wc -l < "$FINDINGS_LOG" | tr -d ' ' + else + echo "0" + fi +} + +# --------------------------------------------------------------------------- +# Fully-compliant repo — no findings expected +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: no findings when all settings are enabled" { + pp_check_security_and_analysis "my-repo" + [ "$(finding_count)" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# warning-severity, null actual → plan does not support feature → no finding +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: secret_scanning_ai_detection absent emits no finding (plan-gated)" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + if grep -q "secret_scanning_ai_detection" "$FINDINGS_LOG" 2>/dev/null; then + echo "unexpected finding for secret_scanning_ai_detection when status is absent" >&2 + false + fi +} + +@test "pp_check_security_and_analysis: secret_scanning_non_provider_patterns absent emits no finding (plan-gated)" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + if grep -q "secret_scanning_non_provider_patterns" "$FINDINGS_LOG" 2>/dev/null; then + echo "unexpected finding for secret_scanning_non_provider_patterns when status is absent" >&2 + false + fi +} + +@test "pp_check_security_and_analysis: dependabot_security_updates absent emits no finding (warning, plan-gated)" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + if grep -q "dependabot_security_updates" "$FINDINGS_LOG" 2>/dev/null; then + echo "unexpected finding for dependabot_security_updates when status is absent" >&2 + false + fi +} + +# --------------------------------------------------------------------------- +# warning-severity, disabled actual → feature exists but off → finding emitted +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: secret_scanning_ai_detection disabled emits a warning finding" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"disabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + grep -q "secret_scanning_ai_detection" "$FINDINGS_LOG" + grep -q "|warning|" "$FINDINGS_LOG" +} + +@test "pp_check_security_and_analysis: secret_scanning_non_provider_patterns disabled emits a warning finding" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"disabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + grep -q "secret_scanning_non_provider_patterns" "$FINDINGS_LOG" +} + +# --------------------------------------------------------------------------- +# error-severity, null actual → always emits a finding (not plan-gated) +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: secret_scanning absent emits an error finding" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + grep -q "secret_scanning" "$FINDINGS_LOG" + grep -q "|error|" "$FINDINGS_LOG" +} + +@test "pp_check_security_and_analysis: secret_scanning_push_protection absent emits an error finding" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + grep -q "secret_scanning_push_protection" "$FINDINGS_LOG" + grep -q "|error|" "$FINDINGS_LOG" +} + +# --------------------------------------------------------------------------- +# error-severity, disabled actual → always emits a finding +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: secret_scanning disabled emits an error finding" { + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"disabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + grep -q "secret_scanning" "$FINDINGS_LOG" + grep -q "|error|" "$FINDINGS_LOG" +} + +# --------------------------------------------------------------------------- +# All plan-gated warning settings absent — no findings from those keys +# --------------------------------------------------------------------------- + +@test "pp_check_security_and_analysis: no findings when only error settings are present and compliant" { + # Only the two required (error-severity) settings are present and enabled. + # All three warning-severity settings are absent (plan-gated). + GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"}}')" + pp_check_security_and_analysis "my-repo" + [ "$(finding_count)" -eq 0 ] +}