feat: implement issue #355 — Compliance: secret_scanning_ai_detection

scripts/lib/push-protection.sh

@@ -42,11 +42,19 @@
 PP_REQUIRED_SA_SETTINGS=(
   "secret_scanning:enabled:error:Secret scanning must be enabled"
   "secret_scanning_push_protection:enabled:error:Secret scanning push protection must be enabled"
-  "secret_scanning_ai_detection:enabled:warning:Secret scanning AI detection should be enabled"
-  "secret_scanning_non_provider_patterns:enabled:warning:Secret scanning non-provider patterns should be enabled"
+  "secret_scanning_ai_detection:enabled:warning:Secret scanning AI detection should be enabled (requires GitHub Copilot subscription; absent means the feature is unavailable for the current org plan)"
+  "secret_scanning_non_provider_patterns:enabled:warning:Secret scanning non-provider patterns should be enabled (requires GitHub Advanced Security)"
   "dependabot_security_updates:enabled:warning:Dependabot security updates should be enabled"
 )
 
+# Plan-gated keys that cannot be configured without a plan upgrade. A null/absent
+# status for these keys is not actionable and should not generate a finding.
+PP_PLAN_GATED_KEYS=(
+  "secret_scanning_ai_detection"
+  "secret_scanning_non_provider_patterns"
+  "dependabot_security_updates"
+)
+
 # Minimum entries that every repo's .gitignore MUST contain. Every repo
 # starting from the org baseline at /.gitignore satisfies these by default.
 PP_REQUIRED_GITIGNORE_PATTERNS=(
@@ -115,7 +123,34 @@ pp_apply_security_and_analysis() {
   full_payload=$(echo "$payload" | jq '{security_and_analysis: .}')
 
   if echo "$full_payload" | gh api -X PATCH "repos/$ORG/$repo" --input - > /dev/null 2>&1; then
-    ok "$ORG/$repo security_and_analysis updated successfully"
+    ok "$ORG/$repo security_and_analysis PATCH accepted"
+
+    # Verify each patched key was actually applied. Some settings (e.g.
+    # secret_scanning_ai_detection, secret_scanning_non_provider_patterns)
+    # require GitHub Advanced Security or a Copilot subscription; the API
+    # accepts the PATCH (HTTP 200) but silently ignores keys that the org
+    # plan does not support. Re-fetch and warn for any key still not applied.
+    local post_patch post_actuals
+    post_patch=$(gh api "repos/$ORG/$repo" --jq '.security_and_analysis // {}' 2>/dev/null || echo "{}")
+    post_actuals=$(echo "$post_patch" | jq -r 'to_entries | .[] | "\(.key):\(.value.status? // "null")"' 2>/dev/null || echo "")
+
+    local -A actuals
+    while IFS=':' read -r k v; do
+      if [ -n "$k" ]; then
+        actuals["$k"]="$v"
+      fi
+    done <<< "$post_actuals"
+
+    local post_entry post_key post_expected post_actual
+    for post_entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do
+      IFS=':' read -r post_key post_expected _ _ <<< "$post_entry"
+      post_actual="${actuals[$post_key]:-null}"
+      if [ "$post_actual" != "$post_expected" ]; then
+        info "  $post_key still not $post_expected after PATCH — the org plan may not support this feature (current: $post_actual)"
+      else
+        ok "  $post_key: $post_actual (verified)"
+      fi
+    done
   else
     err "Failed to PATCH security_and_analysis for $ORG/$repo — the authenticated token must have repository admin permissions (or the org plan may not support these features)"
     return 1
@@ -167,15 +202,29 @@ pp_check_security_and_analysis() {
     return
   fi
 
-  local entry key expected severity detail actual
+  local entry key expected severity detail actual is_plan_gated
   for entry in "${PP_REQUIRED_SA_SETTINGS[@]}"; do
     IFS=':' read -r key expected severity detail <<< "$entry"
     actual=$(echo "$sa" | jq -r ".\"$key\".status // \"null\"")
-    if [ "$actual" != "$expected" ]; then
-      add_finding "$repo" "push-protection" "$key" "$severity" \
-        "$detail (current: \`$actual\`, expected: \`$expected\`)" \
-        "$PP_STANDARD_REF#required-repo-level-settings"
+    if [ "$actual" = "$expected" ]; then
+      continue
+    fi
+    # A null/absent status for a plan-gated key means the feature is unavailable
+    # for the current org plan — skip rather than creating a non-actionable
+    # compliance finding that cannot be remediated without a plan upgrade.
+    is_plan_gated=false
+    for plan_key in "${PP_PLAN_GATED_KEYS[@]}"; do
+      if [ "$key" = "$plan_key" ]; then
+        is_plan_gated=true
+        break
+      fi
+    done
+    if [ "$is_plan_gated" = true ] && [ "$actual" = "null" ]; then
+      continue
     fi
+    add_finding "$repo" "push-protection" "$key" "$severity" \
+      "$detail (current: \`$actual\`, expected: \`$expected\`)" \
+      "$PP_STANDARD_REF#required-repo-level-settings"
   done
 }
 

test/scripts/lib/helpers/setup.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Common test helpers for scripts/lib bats suites.
+
+TT_REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../../.." && pwd)"
+export TT_REPO_ROOT
+
+TT_SCRIPTS_LIB_DIR="${TT_REPO_ROOT}/scripts/lib"
+export TT_SCRIPTS_LIB_DIR
+
+tt_make_tmpdir() {
+  TT_TMP="$(mktemp -d)"
+  export TT_TMP
+}
+
+tt_cleanup_tmpdir() {
+  if [ -n "${TT_TMP:-}" ] && [ -d "${TT_TMP}" ]; then
+    rm -rf "${TT_TMP}"
+  fi
+}

test/scripts/lib/push-protection.bats

@@ -0,0 +1,208 @@
+#!/usr/bin/env bats
+# Tests for scripts/lib/push-protection.sh — pp_check_security_and_analysis()
+#
+# Covers the plan-gated feature handling: warning-severity settings whose API
+# status is null (absent from the response) must NOT generate a finding, since
+# a null status indicates the org plan does not support the feature and there
+# is nothing the operator can do to remediate it.
+
+load 'helpers/setup'
+
+# ---------------------------------------------------------------------------
+# Helpers: build repo-API JSON payloads
+# ---------------------------------------------------------------------------
+
+# Build the full repo JSON object that gh api "repos/ORG/REPO" returns.
+# Accepts an inline security_and_analysis JSON fragment as $1.
+# The mock gh_api then applies ".security_and_analysis // {}" to this object.
+make_repo_json() {
+  local sa_json="$1"
+  printf '{"security_and_analysis":%s}' "$sa_json"
+}
+
+# ---------------------------------------------------------------------------
+# Test setup / teardown
+# ---------------------------------------------------------------------------
+
+setup() {
+  tt_make_tmpdir
+
+  # Log file for add_finding calls — each call appends one line:
+  #   "<repo>|<category>|<check>|<severity>|<detail>"
+  FINDINGS_LOG="${TT_TMP}/findings.log"
+  export FINDINGS_LOG
+
+  # Default: gh_api returns a fully-compliant security_and_analysis object
+  # wrapped in the full repo JSON so the jq filter ".security_and_analysis // {}"
+  # resolves correctly.
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  export GH_API_RESPONSE
+
+  # Second-call response for the secret-scanning alerts proxy check
+  # (used only when the primary call returns {}).
+  GH_API_ALERTS_RESPONSE='[]'
+  export GH_API_ALERTS_RESPONSE
+
+  ORG="test-org"
+  export ORG
+
+  # Track how many times gh_api has been called so we can return different
+  # responses for the primary (repo) call vs. the proxy (alerts) call.
+  GH_API_CALL_COUNT=0
+  export GH_API_CALL_COUNT
+
+  gh_api() {
+    GH_API_CALL_COUNT=$((GH_API_CALL_COUNT + 1))
+    local endpoint="$1"; shift
+    local jq_filter=""
+    while [ $# -gt 0 ]; do
+      if [ "$1" = "--jq" ] && [ $# -gt 1 ]; then
+        jq_filter="$2"; shift 2
+      else
+        shift
+      fi
+    done
+    # First call is always the primary repo endpoint.
+    # Subsequent calls are the alerts proxy.
+    local response
+    if [ "$GH_API_CALL_COUNT" -eq 1 ]; then
+      response="$GH_API_RESPONSE"
+    else
+      response="$GH_API_ALERTS_RESPONSE"
+    fi
+    if [ -n "$jq_filter" ]; then
+      printf '%s' "$response" | jq -r "$jq_filter" 2>/dev/null || echo "{}"
+    else
+      printf '%s' "$response"
+    fi
+  }
+  export -f gh_api
+
+  add_finding() {
+    local repo="$1" category="$2" check="$3" severity="$4" detail="$5"
+    printf '%s|%s|%s|%s|%s\n' "$repo" "$category" "$check" "$severity" "$detail" \
+      >>"$FINDINGS_LOG"
+  }
+  export -f add_finding
+
+  # shellcheck source=/dev/null
+  . "${TT_SCRIPTS_LIB_DIR}/push-protection.sh"
+}
+
+teardown() {
+  tt_cleanup_tmpdir
+}
+
+findings() {
+  if [ -f "$FINDINGS_LOG" ]; then
+    cat "$FINDINGS_LOG"
+  else
+    echo ""
+  fi
+}
+
+finding_count() {
+  if [ -f "$FINDINGS_LOG" ]; then
+    wc -l < "$FINDINGS_LOG" | tr -d ' '
+  else
+    echo "0"
+  fi
+}
+
+# ---------------------------------------------------------------------------
+# Fully-compliant repo — no findings expected
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: no findings when all settings are enabled" {
+  pp_check_security_and_analysis "my-repo"
+  [ "$(finding_count)" -eq 0 ]
+}
+
+# ---------------------------------------------------------------------------
+# warning-severity, null actual → plan does not support feature → no finding
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: secret_scanning_ai_detection absent emits no finding (plan-gated)" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  if grep -q "secret_scanning_ai_detection" "$FINDINGS_LOG" 2>/dev/null; then
+    echo "unexpected finding for secret_scanning_ai_detection when status is absent" >&2
+    false
+  fi
+}
+
+@test "pp_check_security_and_analysis: secret_scanning_non_provider_patterns absent emits no finding (plan-gated)" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  if grep -q "secret_scanning_non_provider_patterns" "$FINDINGS_LOG" 2>/dev/null; then
+    echo "unexpected finding for secret_scanning_non_provider_patterns when status is absent" >&2
+    false
+  fi
+}
+
+@test "pp_check_security_and_analysis: dependabot_security_updates absent emits no finding (warning, plan-gated)" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  if grep -q "dependabot_security_updates" "$FINDINGS_LOG" 2>/dev/null; then
+    echo "unexpected finding for dependabot_security_updates when status is absent" >&2
+    false
+  fi
+}
+
+# ---------------------------------------------------------------------------
+# warning-severity, disabled actual → feature exists but off → finding emitted
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: secret_scanning_ai_detection disabled emits a warning finding" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"disabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  grep -q "secret_scanning_ai_detection" "$FINDINGS_LOG"
+  grep -q "|warning|" "$FINDINGS_LOG"
+}
+
+@test "pp_check_security_and_analysis: secret_scanning_non_provider_patterns disabled emits a warning finding" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"disabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  grep -q "secret_scanning_non_provider_patterns" "$FINDINGS_LOG"
+}
+
+# ---------------------------------------------------------------------------
+# error-severity, null actual → always emits a finding (not plan-gated)
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: secret_scanning absent emits an error finding" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  grep -q "secret_scanning" "$FINDINGS_LOG"
+  grep -q "|error|" "$FINDINGS_LOG"
+}
+
+@test "pp_check_security_and_analysis: secret_scanning_push_protection absent emits an error finding" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  grep -q "secret_scanning_push_protection" "$FINDINGS_LOG"
+  grep -q "|error|" "$FINDINGS_LOG"
+}
+
+# ---------------------------------------------------------------------------
+# error-severity, disabled actual → always emits a finding
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: secret_scanning disabled emits an error finding" {
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"disabled"},"secret_scanning_push_protection":{"status":"enabled"},"secret_scanning_ai_detection":{"status":"enabled"},"secret_scanning_non_provider_patterns":{"status":"enabled"},"dependabot_security_updates":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  grep -q "secret_scanning" "$FINDINGS_LOG"
+  grep -q "|error|" "$FINDINGS_LOG"
+}
+
+# ---------------------------------------------------------------------------
+# All plan-gated warning settings absent — no findings from those keys
+# ---------------------------------------------------------------------------
+
+@test "pp_check_security_and_analysis: no findings when only error settings are present and compliant" {
+  # Only the two required (error-severity) settings are present and enabled.
+  # All three warning-severity settings are absent (plan-gated).
+  GH_API_RESPONSE="$(make_repo_json '{"secret_scanning":{"status":"enabled"},"secret_scanning_push_protection":{"status":"enabled"}}')"
+  pp_check_security_and_analysis "my-repo"
+  [ "$(finding_count)" -eq 0 ]
+}