Post visual diff comment and job summary on visual test failure
When visual tests fail on a PR: - Writes diff images (base64 data URIs) to the job summary for inline viewing - Posts a PR comment listing failing tests with a link to the job summary - Uploads .vitest-attachments/ as an artifact (include-hidden-files: true) The job now has pull-requests: write permission for posting the comment. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
81334e977c
commit
e662212dca
1 changed files with 33 additions and 19 deletions
52
.github/workflows/ci.yml
vendored
52
.github/workflows/ci.yml
vendored
|
|
@ -113,6 +113,9 @@ jobs:
|
||||||
visual-tests:
|
visual-tests:
|
||||||
name: Visual Tests
|
name: Visual Tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pull-requests: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
- uses: pnpm/action-setup@v6
|
- uses: pnpm/action-setup@v6
|
||||||
|
|
@ -141,6 +144,26 @@ jobs:
|
||||||
id: visual-tests
|
id: visual-tests
|
||||||
run: pnpm --filter @trails-cool/planner test:visual
|
run: pnpm --filter @trails-cool/planner test:visual
|
||||||
|
|
||||||
|
- name: Write diff images to job summary
|
||||||
|
if: failure() && steps.visual-tests.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
diffs=$(find apps/planner/.vitest-attachments -name "*-diff-*.png" 2>/dev/null | sort)
|
||||||
|
if [ -z "$diffs" ]; then exit 0; fi
|
||||||
|
|
||||||
|
echo "## Visual regression failures" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
for diff in $diffs; do
|
||||||
|
name=$(basename "$diff" | sed 's/-diff-chromium-[a-z]*\.png//' | sed 's/-/ /g')
|
||||||
|
b64=$(base64 -w 0 "$diff")
|
||||||
|
echo "### \`$name\`" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "<img src=\"data:image/png;base64,$b64\" alt=\"$name diff\" style=\"max-width:100%\">" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "" >> $GITHUB_STEP_SUMMARY
|
||||||
|
done
|
||||||
|
echo "To update snapshots if the change is intentional:" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo "pnpm --filter @trails-cool/planner test:visual:update" >> $GITHUB_STEP_SUMMARY
|
||||||
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||||
|
|
||||||
- name: Post diff comment on PR
|
- name: Post diff comment on PR
|
||||||
if: failure() && steps.visual-tests.outcome == 'failure' && github.event_name == 'pull_request'
|
if: failure() && steps.visual-tests.outcome == 'failure' && github.event_name == 'pull_request'
|
||||||
env:
|
env:
|
||||||
|
|
@ -149,28 +172,18 @@ jobs:
|
||||||
diffs=$(find apps/planner/.vitest-attachments -name "*-diff-*.png" 2>/dev/null | sort)
|
diffs=$(find apps/planner/.vitest-attachments -name "*-diff-*.png" 2>/dev/null | sort)
|
||||||
if [ -z "$diffs" ]; then exit 0; fi
|
if [ -z "$diffs" ]; then exit 0; fi
|
||||||
|
|
||||||
|
run_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||||
body="## Visual regression failures"$'\n\n'
|
body="## Visual regression failures"$'\n\n'
|
||||||
body+="| Test | Diff (reference → actual) |"$'\n'
|
body+="The following tests produced screenshot diffs:"$'\n\n'
|
||||||
body+="|------|--------------------------|"$'\n'
|
|
||||||
|
|
||||||
for diff in $diffs; do
|
for diff in $diffs; do
|
||||||
# Upload image to GitHub's CDN via the issue assets endpoint
|
name=$(basename "$diff" | sed 's/-diff-chromium-[a-z]*\.png//' | sed 's/-/ /g')
|
||||||
response=$(curl -sf -X POST \
|
body+="- \`$name\`"$'\n'
|
||||||
-H "Authorization: Bearer $GH_TOKEN" \
|
|
||||||
-H "Content-Type: image/png" \
|
|
||||||
-H "Accept: application/json" \
|
|
||||||
--data-binary @"$diff" \
|
|
||||||
"https://uploads.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/assets?name=$(basename $diff)")
|
|
||||||
url=$(echo "$response" | jq -r '.url // empty')
|
|
||||||
name=$(basename "$diff" | sed 's/-diff-chromium-linux\.png//' | sed 's/-/ /g')
|
|
||||||
if [ -n "$url" ]; then
|
|
||||||
body+="| \`$name\` |  |"$'\n'
|
|
||||||
else
|
|
||||||
body+="| \`$name\` | *(upload failed)* |"$'\n'
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
body+=$'\n'"**[View diff images in the job summary]($run_url)**"$'\n\n'
|
||||||
body+=$'\n'"Run \`pnpm --filter @trails-cool/planner test:visual:update\` to regenerate snapshots if the change is intentional."
|
body+="To update snapshots if the change is intentional:"$'\n'
|
||||||
|
body+="\`\`\`"$'\n'
|
||||||
|
body+="pnpm --filter @trails-cool/planner test:visual:update"$'\n'
|
||||||
|
body+="\`\`\`"
|
||||||
|
|
||||||
gh pr comment ${{ github.event.pull_request.number }} --body "$body"
|
gh pr comment ${{ github.event.pull_request.number }} --body "$body"
|
||||||
|
|
||||||
|
|
@ -180,6 +193,7 @@ jobs:
|
||||||
with:
|
with:
|
||||||
name: visual-snapshots-diff
|
name: visual-snapshots-diff
|
||||||
path: apps/planner/.vitest-attachments/
|
path: apps/planner/.vitest-attachments/
|
||||||
|
include-hidden-files: true
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
|
|
||||||
e2e:
|
e2e:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue