diff --git a/.github/workflows/ctt.yml b/.github/workflows/ctt.yml new file mode 100644 index 0000000000..fddc0dd925 --- /dev/null +++ b/.github/workflows/ctt.yml @@ -0,0 +1,170 @@ +name: CTT regression + +# Runs the OPC UA Compliance Test Tool (CTT) against the reference server and reports the result here. +# The CTT is licensed to OPC Foundation members only, so it runs in the private repository +# OPCFoundation/UA-.NETStandard-CTT; this workflow only starts that run, waits for it and posts its public +# summary (result counts and the test cases that changed against the baseline, never CTT scripts or logs). +# +# Opt-in: weekly on master, manual, and pull requests labelled 'ctt'. A maintainer adds the label after +# reviewing the PR: the reviewed head commit is built and run next to the CTT; a later push is not run (label +# again). The label is removed when the run finishes. It is not a required check. +# +# Needs Actions read/write access to the private repository through either +# - a GitHub App installed on OPCFoundation/UA-.NETStandard-CTT: variable CTT_APP_ID, secret CTT_APP_PRIVATE_KEY +# - or a fine-grained token for that repository: secret CTT_TOKEN +# pull_request_target is required for the secret; the job never checks out or runs PR code. +# +# The results are a regression signal for this code base, not a certification or a statement of compliance. + +on: + schedule: + - cron: '0 2 * * 0' + workflow_dispatch: + pull_request_target: + types: [labeled] + +permissions: {} + +concurrency: + group: ctt-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +env: + CTT_REPO: OPCFoundation/UA-.NETStandard-CTT + CTT_LABEL: ctt + +jobs: + ctt: + if: >- + github.repository == 'OPCFoundation/UA-.NETStandard' && + (github.event_name != 'pull_request_target' || github.event.label.name == 'ctt') + name: CTT + runs-on: ubuntu-latest + timeout-minutes: 420 + permissions: + pull-requests: write # summary comment, remove the label + statuses: write # 'CTT' commit status on the tested commit + steps: + - name: Resolve target + id: target + env: + PR: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ -n "$PR" ]; then + echo "ref=refs/pull/$PR/head" >> "$GITHUB_OUTPUT" + echo "sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + else + echo "ref=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT" + echo "sha=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + fi + + - name: Set pending status + env: + GH_TOKEN: ${{ github.token }} + SHA: ${{ steps.target.outputs.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + gh api "repos/$GITHUB_REPOSITORY/statuses/$SHA" -f state=pending -f context=CTT \ + -f description='CTT regression run in progress' -f target_url="$RUN_URL" > /dev/null + + - name: Run CTT + id: run + env: + APP_ID: ${{ vars.CTT_APP_ID }} + APP_KEY: ${{ secrets.CTT_APP_PRIVATE_KEY }} + STATIC_TOKEN: ${{ secrets.CTT_TOKEN }} + REF: ${{ steps.target.outputs.ref }} + SHA: ${{ steps.target.outputs.sha }} + run: | + set -euo pipefail + b64url() { openssl base64 -A | tr '+/' '-_' | tr -d '='; } + # App installation tokens expire after an hour and a run takes longer: mint a new one per call. + token() { + if [ -z "$APP_ID" ]; then printf '%s' "$STATIC_TOKEN"; return; fi + local now hdr payload sig jwt inst + now=$(date +%s) + hdr=$(printf '{"alg":"RS256","typ":"JWT"}' | b64url) + payload=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' $((now - 60)) $((now + 540)) "$APP_ID" | b64url) + sig=$(printf '%s.%s' "$hdr" "$payload" | openssl dgst -sha256 -sign "$key_file" -binary | b64url) + jwt="$hdr.$payload.$sig" + inst=$(curl -fsS -H "Authorization: Bearer $jwt" -H 'Accept: application/vnd.github+json' \ + "https://api.github.com/repos/$CTT_REPO/installation" | jq -r .id) + curl -fsS -X POST -H "Authorization: Bearer $jwt" -H 'Accept: application/vnd.github+json' \ + "https://api.github.com/app/installations/$inst/access_tokens" | jq -r .token + } + if [ -z "$APP_ID" ] && [ -z "$STATIC_TOKEN" ]; then + echo '::error::No access to the CTT repository: set CTT_APP_ID + CTT_APP_PRIVATE_KEY or CTT_TOKEN'; exit 1 + fi + key_file="$RUNNER_TEMP/ctt-app.pem" + if [ -n "$APP_ID" ]; then + (umask 077; printf '%s\n' "$APP_KEY" > "$key_file") + trap 'rm -f "$key_file"' EXIT + fi + correlation="ua-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + GH_TOKEN=$(token) gh workflow run ctt.yml -R "$CTT_REPO" -f ua_repository="$GITHUB_REPOSITORY" \ + -f ua_ref="$REF" -f expected_sha="$SHA" -f correlation_id="$correlation" + + id='' + for i in $(seq 1 30); do + sleep 10 + id=$(GH_TOKEN=$(token) gh run list -R "$CTT_REPO" --workflow ctt.yml --event workflow_dispatch --limit 30 \ + --json databaseId,displayTitle --jq "[.[] | select(.displayTitle | endswith(\" $correlation\"))][0].databaseId // empty") + [ -n "$id" ] && break + done + if [ -z "$id" ]; then echo '::error::The CTT run did not show up'; exit 1; fi + echo "CTT run $id" + + # Runs queue behind each other in the CTT repository (one CTT at a time). + while [ "$(GH_TOKEN=$(token) gh run view "$id" -R "$CTT_REPO" --json status --jq .status || echo unknown)" != completed ]; do + sleep 60 + done + conclusion=$(GH_TOKEN=$(token) gh run view "$id" -R "$CTT_REPO" --json conclusion --jq .conclusion) + echo "conclusion=$conclusion" >> "$GITHUB_OUTPUT" + echo "CTT run $id: $conclusion" + + mkdir -p ctt-summary + if ! GH_TOKEN=$(token) gh run download "$id" -R "$CTT_REPO" -n ctt-public-summary -D ctt-summary; then + printf '### CTT\n\nThe CTT run ended before it produced results.\n' > ctt-summary/public-summary.md + fi + cat ctt-summary/public-summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Report + if: always() + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + SHA: ${{ steps.target.outputs.sha }} + CONCLUSION: ${{ steps.run.outputs.conclusion }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + # Green only with a downloaded report that shows no regression and no failed part; a successful run + # without its report is an error, not a pass. + regressions=$(jq -r '.regressions // empty' ctt-summary/report.json 2>/dev/null || true) + part_failures=$(jq -r '.partFailures | length' ctt-summary/report.json 2>/dev/null || true) + if [ "$CONCLUSION" = success ] && [ "$regressions" = 0 ] && [ "$part_failures" = 0 ]; then + state=success; description='No regressions against the CTT baseline' + elif [ -n "$regressions" ] && [ "$regressions" != 0 ]; then + state=failure; description="$regressions regression(s) against the CTT baseline" + else + state=error; description='The CTT run failed' + fi + gh api "repos/$GITHUB_REPOSITORY/statuses/$SHA" -f state="$state" -f context=CTT \ + -f description="$description" -f target_url="$RUN_URL" > /dev/null + if [ -n "$PR" ]; then + { + echo '' + if [ -f ctt-summary/public-summary.md ]; then cat ctt-summary/public-summary.md; else echo "CTT: $description"; fi + echo + echo "Commit $SHA, [workflow run]($RUN_URL)." + } > comment.md + existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" --paginate \ + --jq '.[] | select(.user.login == "github-actions[bot]" and (.body | startswith(""))) | .id' | head -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@comment.md > /dev/null + else + gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@comment.md > /dev/null + fi + gh api -X DELETE "repos/$GITHUB_REPOSITORY/issues/$PR/labels/$CTT_LABEL" > /dev/null 2>&1 || true + fi + [ "$state" = success ] diff --git a/README.md b/README.md index 1f4815c343..e9f8644847 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ [![Connection Stability](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/stability-test.yml/badge.svg?branch=master&event=schedule)](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/stability-test.yml?query=branch%3Amaster+event%3Aschedule) [![Foreign Interop](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/interop-foreign.yml/badge.svg?branch=master&event=schedule)](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/interop-foreign.yml?query=branch%3Amaster+event%3Aschedule) [![HA Long-Haul](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/sample-ha-longhaul.yml/badge.svg?branch=master&event=schedule)](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/sample-ha-longhaul.yml?query=branch%3Amaster+event%3Aschedule) +[![CTT Regression](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/ctt.yml/badge.svg?branch=master&event=schedule)](https://github.com/OPCFoundation/UA-.NETStandard/actions/workflows/ctt.yml?query=branch%3Amaster+event%3Aschedule) > 🆕 **This is version 2.0 of the OPC UA .NET Standard Stack (current `master`).** > diff --git a/plans/ctt-testing.md b/plans/ctt-testing.md index 6590a32bd2..a119163c1b 100644 --- a/plans/ctt-testing.md +++ b/plans/ctt-testing.md @@ -10,6 +10,11 @@ configuration notes. Read that file before a run and treat the failures it lists investigate what is not in it. When you find a new CTT defect, file it in Mantis and add the failing test, a short abstract and the Mantis link there. +**In CI.** `.github/workflows/ctt.yml` ("CTT regression") runs the full CTT weekly on master and on pull +requests a maintainer labels `ctt`. The CTT is member-only software, so the run itself happens in a private +OPC Foundation repository; this repository only receives the result counts and the test cases that changed +against the baseline (commit status `CTT`, PR comment, README badge). + ## Setup Placeholders used below: