mirror of
https://github.com/Sea-Haven-Industries/shoc-pr-review-runner.git
synced 2026-10-05 10:21:57 +00:00
Adam Moussa
3c541cc7b4
The first live run reported the reviewed PR's CI as "missing" while it was actually green, and the model cited that as a reason to withhold approval. Reading check-runs needs the App's checks:read permission, which the App did not have, so the call 403'd and the fallback turned an authorization failure into the factual claim "this PR has no CI". That is the exact failure this runner exists to prevent, applied to a governance signal instead of a gate. Distinguish the two: an unreadable signal is now recorded as "unknown", the evidence report says unknown is not evidence of absent or failing CI, and the skill tells the reviewer that a signal the runner could not read is not a finding. Request checks:read at token mint so the signal is readable at all. Also correct the App verification snippet in the README: listing an installation's selected repositories needs the installation token, so the documented /user/installations call does not work for an org admin.
323 lines
13 KiB
YAML
323 lines
13 KiB
YAML
name: Review PR
|
|
|
|
# SHOC PR Review Runner — Phase 1 (spec §8, §26).
|
|
# Manually dispatched. Checks out exact PR heads read-only, runs clean
|
|
# build/test gates, generates a truthful evidence report, invokes the Fireworks
|
|
# review agent, validates its output, and publishes artifacts.
|
|
#
|
|
# This workflow NEVER writes to the product repositories or their PRs: the
|
|
# GITHUB_TOKEN carries contents:read only (listing any permission zeroes every
|
|
# unlisted scope), and product-repo access uses a GitHub App installation token
|
|
# downscoped at mint time to contents/pull-requests/metadata READ.
|
|
#
|
|
# SECURITY ARCHITECTURE — why this is two jobs:
|
|
# Running the product repos' build gates executes code authored in the PR under
|
|
# review (npm lifecycle scripts, eslint/vite/vitest configs, MSBuild targets).
|
|
# That code must never share a job with a secret, because step-level `env:` is
|
|
# not an isolation boundary: PR code can poison $GITHUB_ENV for later steps,
|
|
# read a later step's /proc environ, or overwrite the runner's own scripts.
|
|
# Therefore:
|
|
# job `gates` — executes untrusted PR code. Holds NO Fireworks key, and the
|
|
# App token is revoked before the first build command runs.
|
|
# job `review` — holds the Fireworks key. Executes NO product-repo code; it
|
|
# re-checks out this repo fresh (so tampered scripts from the
|
|
# gates job cannot follow) and consumes only text artifacts.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
review_type:
|
|
description: Review type
|
|
required: true
|
|
type: choice
|
|
options: [frontend, backend, paired]
|
|
frontend_pr:
|
|
description: Frontend PR number (required for frontend/paired)
|
|
required: false
|
|
type: string
|
|
backend_pr:
|
|
description: Backend PR number (required for backend/paired)
|
|
required: false
|
|
type: string
|
|
ticket:
|
|
description: SH ticket identifier or URL
|
|
required: false
|
|
type: string
|
|
review_notes:
|
|
description: "Reviewer context. Sent to the Fireworks model and kept in run artifacts for 30 days — do not paste credentials."
|
|
required: false
|
|
type: string
|
|
run_mocked_e2e:
|
|
description: Run the mocked Playwright suite (frontend/paired)
|
|
required: true
|
|
type: boolean
|
|
default: true
|
|
model:
|
|
description: Fireworks review model
|
|
required: true
|
|
type: choice
|
|
default: deepseek-v4-pro
|
|
options: [deepseek-v4-pro, kimi-k2p6]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: review-pr-${{ github.event.inputs.review_type }}-${{ github.event.inputs.frontend_pr }}-${{ github.event.inputs.backend_pr }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
FRONTEND_REPO: Sea-Haven-Industries/shoc-frontend-new
|
|
BACKEND_REPO: Sea-Haven-Industries/shoc-backend
|
|
COMPANION_BRANCH: dev
|
|
REVIEW_TYPE: ${{ github.event.inputs.review_type }}
|
|
TICKET: ${{ github.event.inputs.ticket }}
|
|
REVIEW_NOTES: ${{ github.event.inputs.review_notes }}
|
|
|
|
jobs:
|
|
gates:
|
|
name: Gates (${{ github.event.inputs.review_type }})
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- name: Checkout runner
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Configure workspace paths
|
|
# The product checkouts live OUTSIDE github.workspace so PR code is
|
|
# never a sibling of this repo's scripts. RUNNER_TEMP is only available
|
|
# as a shell variable, not in a workflow-level env block.
|
|
run: |
|
|
{
|
|
echo "WORKSPACE_DIR=$RUNNER_TEMP/workspace"
|
|
echo "ARTIFACTS_DIR=$RUNNER_TEMP/workspace/artifacts"
|
|
echo "STATE_DIR=$RUNNER_TEMP/runner-state-$(openssl rand -hex 8)"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Resolve and validate inputs
|
|
id: inputs
|
|
env:
|
|
FRONTEND_PR: ${{ github.event.inputs.frontend_pr }}
|
|
BACKEND_PR: ${{ github.event.inputs.backend_pr }}
|
|
run: ./scripts/resolve-inputs.sh
|
|
|
|
- name: Mint read-only App token
|
|
id: app-token
|
|
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
|
with:
|
|
app-id: ${{ secrets.SHOC_REVIEW_APP_ID }}
|
|
private-key: ${{ secrets.SHOC_REVIEW_APP_PRIVATE_KEY }}
|
|
owner: Sea-Haven-Industries
|
|
repositories: shoc-frontend-new,shoc-backend
|
|
# Downscope at mint time so the token stays read-only even if the App
|
|
# installation is later granted broader permissions.
|
|
permission-contents: read
|
|
permission-pull-requests: read
|
|
permission-checks: read
|
|
permission-metadata: read
|
|
|
|
- name: Resolve frontend PR head
|
|
if: steps.inputs.outputs.frontend_pr != ''
|
|
id: frontend-head
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token }}
|
|
PR_NUMBER: ${{ steps.inputs.outputs.frontend_pr }}
|
|
run: ./scripts/resolve-pr-head.sh frontend "$FRONTEND_REPO" "$PR_NUMBER"
|
|
|
|
- name: Resolve backend PR head
|
|
if: steps.inputs.outputs.backend_pr != ''
|
|
id: backend-head
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token }}
|
|
PR_NUMBER: ${{ steps.inputs.outputs.backend_pr }}
|
|
run: ./scripts/resolve-pr-head.sh backend "$BACKEND_REPO" "$PR_NUMBER"
|
|
|
|
- name: Checkout product repositories at exact heads
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token }}
|
|
FRONTEND_SHA: ${{ steps.frontend-head.outputs.frontend_sha }}
|
|
BACKEND_SHA: ${{ steps.backend-head.outputs.backend_sha }}
|
|
run: ./scripts/checkout-repositories.sh
|
|
|
|
- name: Collect review context
|
|
# Runs before any PR-authored code executes, so the collected diff and
|
|
# file contents cannot be tampered with by the build.
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token }}
|
|
run: ./scripts/collect-context.sh
|
|
|
|
- name: Revoke App token before running untrusted code
|
|
env:
|
|
GH_TOKEN: ${{ steps.app-token.outputs.token }}
|
|
run: |
|
|
gh api -X DELETE /installation/token --silent || echo "token revoke returned non-zero (it also expires on its own)"
|
|
|
|
# Everything below this line may execute code authored in the PR.
|
|
# No secret is present in this job from here on.
|
|
|
|
- name: Set up .NET
|
|
if: inputs.review_type == 'backend' || inputs.review_type == 'paired'
|
|
uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
|
|
with:
|
|
dotnet-version: 8.0.x
|
|
|
|
- name: Backend gates
|
|
if: inputs.review_type == 'backend' || inputs.review_type == 'paired'
|
|
continue-on-error: true
|
|
run: ./scripts/run-backend-gates.sh
|
|
|
|
- name: Set up Node
|
|
if: inputs.review_type == 'frontend' || inputs.review_type == 'paired'
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: "24"
|
|
|
|
- name: Frontend gates
|
|
if: inputs.review_type == 'frontend' || inputs.review_type == 'paired'
|
|
continue-on-error: true
|
|
env:
|
|
RUN_MOCKED_E2E: ${{ inputs.run_mocked_e2e }}
|
|
run: ./scripts/run-frontend-gates.sh
|
|
|
|
- name: Stop stray background processes
|
|
if: always()
|
|
run: |
|
|
# PR-authored scripts can background processes that would otherwise
|
|
# keep running and mutate files after the gates finish.
|
|
pkill -u "$(id -u)" -f 'node|dotnet|vite|playwright' 2>/dev/null || true
|
|
sleep 2
|
|
|
|
- name: Stage gate results for the review job
|
|
if: always()
|
|
run: |
|
|
cp "$STATE_DIR/gate-status.tsv" "$ARTIFACTS_DIR/gate-status.tsv" 2>/dev/null || true
|
|
ls -la "$ARTIFACTS_DIR"
|
|
|
|
- name: Upload gates context
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always()
|
|
with:
|
|
name: gates-context-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/workspace/artifacts/
|
|
retention-days: 1
|
|
if-no-files-found: warn
|
|
|
|
review:
|
|
name: Review
|
|
needs: gates
|
|
if: always() && needs.gates.result != 'cancelled'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout runner
|
|
# Fresh checkout: scripts tampered with in the gates job cannot follow.
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Configure workspace paths
|
|
# The product checkouts live OUTSIDE github.workspace so PR code is
|
|
# never a sibling of this repo's scripts. RUNNER_TEMP is only available
|
|
# as a shell variable, not in a workflow-level env block.
|
|
run: |
|
|
{
|
|
echo "WORKSPACE_DIR=$RUNNER_TEMP/workspace"
|
|
echo "ARTIFACTS_DIR=$RUNNER_TEMP/workspace/artifacts"
|
|
echo "STATE_DIR=$RUNNER_TEMP/runner-state-$(openssl rand -hex 8)"
|
|
} >> "$GITHUB_ENV"
|
|
|
|
- name: Download gates context
|
|
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
|
with:
|
|
name: gates-context-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/workspace/artifacts
|
|
|
|
- name: Point the gate table at the downloaded results
|
|
run: |
|
|
mkdir -p "$STATE_DIR"
|
|
cp "$ARTIFACTS_DIR/gate-status.tsv" "$STATE_DIR/gate-status.tsv" 2>/dev/null || true
|
|
|
|
- name: Generate evidence report
|
|
run: ./scripts/generate-evidence.sh
|
|
|
|
- name: Run review agent
|
|
id: agent
|
|
continue-on-error: true
|
|
env:
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
MODEL: ${{ inputs.model }}
|
|
run: ./scripts/run-review-agent.sh
|
|
|
|
- name: Artifact redaction check
|
|
id: redact
|
|
# Runs even when earlier steps failed so nothing is uploaded unscanned.
|
|
if: always()
|
|
env:
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
run: ./scripts/redact-check.sh
|
|
|
|
- name: Job summary
|
|
if: always() && steps.redact.outcome == 'success'
|
|
run: |
|
|
{
|
|
echo "## SHOC PR Review — ${REVIEW_TYPE}"
|
|
echo ""
|
|
echo "### Gate status (machine-recorded, authoritative)"
|
|
echo ""
|
|
echo "| Gate | Status |"
|
|
echo "| --- | --- |"
|
|
if [ -f "$STATE_DIR/gate-status.tsv" ]; then
|
|
awk -F'\t' '{printf "| %s | %s |\n", $1, $2}' "$STATE_DIR/gate-status.tsv"
|
|
fi
|
|
echo ""
|
|
if [ -f "$ARTIFACTS_DIR/review.md" ] && [ "${{ steps.agent.outcome }}" = "success" ]; then
|
|
echo "### Review (model-generated, copy into GitHub manually)"
|
|
echo ""
|
|
echo "The block below is model output influenced by PR content. The"
|
|
echo "gate table above is the authoritative record of what ran."
|
|
echo ""
|
|
echo '```markdown'
|
|
cat "$ARTIFACTS_DIR/review.md"
|
|
echo '```'
|
|
else
|
|
echo "### No valid review produced"
|
|
echo ""
|
|
echo "Agent step outcome: ${{ steps.agent.outcome }} — see validation-errors.txt in the artifacts."
|
|
fi
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload review
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always() && steps.redact.outcome == 'success'
|
|
with:
|
|
name: review-${{ github.run_id }}
|
|
path: |
|
|
${{ runner.temp }}/workspace/artifacts/review.md
|
|
${{ runner.temp }}/workspace/artifacts/review-evidence.md
|
|
${{ runner.temp }}/workspace/artifacts/gate-status.tsv
|
|
${{ runner.temp }}/workspace/artifacts/validation-errors.txt
|
|
${{ runner.temp }}/workspace/artifacts/logs/
|
|
retention-days: 30
|
|
if-no-files-found: warn
|
|
|
|
- name: Upload prompt and diff material
|
|
# Contains full private-repo source; kept briefly for debugging only.
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always() && steps.redact.outcome == 'success'
|
|
with:
|
|
name: review-context-${{ github.run_id }}
|
|
path: |
|
|
${{ runner.temp }}/workspace/artifacts/agent-prompt*.txt
|
|
${{ runner.temp }}/workspace/artifacts/agent-raw-response*
|
|
${{ runner.temp }}/workspace/artifacts/prompt-*.txt
|
|
${{ runner.temp }}/workspace/artifacts/*.diff
|
|
retention-days: 2
|
|
if-no-files-found: warn
|
|
|
|
- name: Fail run if agent or validation failed
|
|
if: steps.agent.outcome != 'success'
|
|
run: |
|
|
echo "Review agent or output validation failed — see artifacts." >&2
|
|
exit 1
|