UA-.NETStandard/.github/workflows/nightly.yml

name: Nightly full validation

# GitHub-hosted equivalent of the full-scope workload Azure Pipelines used to
# run on a schedule.
on:
  schedule:
    # Weekly full-scope run, Sunday 02:00 UTC.
    - cron: '0 2 * * 0'
  workflow_dispatch:
    inputs:
      include_macos:
        description: 'Include the macOS runners'
        type: boolean
        default: true

permissions:
  contents: read

env:
  # See the identical setting in buildandtest.yml: Nerdbank.GitVersioning
  # appends its cloud-build variables to $GITHUB_ENV once per project, and a
  # parallel solution build tears a line. The `Set cloud version` step already
  # exports the same variables once per job via `nbgv cloud -c -a`.
  NBGV_SetCloudBuildVersionVars: 'false'

# Deliberately separate from the pull-request workflow's group, and never
# cancel-in-progress: a manual full-scope run is expensive and approved, so a
# new push must not discard it.
concurrency:
  group: nightly-${{ github.ref }}
  cancel-in-progress: false

jobs:
  discover:
    name: Expand the full workload
    runs-on: ubuntu-latest
    outputs:
      tests: ${{ steps.matrix.outputs.tests }}
      builds: ${{ steps.matrix.outputs.builds }}
      crash_corpus_tests: ${{ steps.corpus.outputs.tests }}
      aot_matrix: ${{ steps.aot.outputs.matrix }}
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - name: Expand the full test and build matrix
      id: matrix
      shell: pwsh
      env:
        INCLUDE_MACOS: ${{ github.event_name == 'schedule' || inputs.include_macos }}
      run: |
        ./.github/scripts/get-ci-matrix.ps1 `
          -Scope full `
          -ExcludeMacOS:($env:INCLUDE_MACOS -ne 'true') `
          -ManifestPath '${{ runner.temp }}/workload-manifest.json'

    # The crash corpus only extends the encoder fuzz project, so its matrix is
    # the same profile table narrowed to that one project rather than a second
    # hand-maintained profile list. macOS is excluded: the corpus job is about
    # input coverage, not platform coverage.
    - name: Expand the crash-corpus matrix
      id: corpus
      shell: pwsh
      run: |
        ./.github/scripts/get-ci-matrix.ps1 `
          -Scope full `
          -ExcludeMacOS `
          -OnlyProject 'fuzzing/Opc.Ua.Encoders.Fuzz.Tests/Opc.Ua.Encoders.Fuzz.Tests.csproj'

    - name: Build the NativeAOT matrix
      id: aot
      shell: pwsh
      env:
        INCLUDE_MACOS: ${{ github.event_name == 'schedule' || inputs.include_macos }}
      run: |
        # Windows is included here and not in the pull-request workflow, so
        # the full-scope run is where win-x64 Native AoT is covered.
        $matrix = @(
            @{ os = 'ubuntu-latest'; rid = 'linux-x64' }
            @{ os = 'windows-latest'; rid = 'win-x64' }
        )
        if ($env:INCLUDE_MACOS -eq 'true') {
            $matrix += @{ os = 'macos-15-intel'; rid = 'osx-x64' }
            $matrix += @{ os = 'macos-latest'; rid = 'osx-arm64' }
        }
        "matrix=$(ConvertTo-Json -InputObject @($matrix) -Compress)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append

    - name: Upload the workload manifest
      uses: actions/upload-artifact@v7
      with:
        name: workload-manifest
        path: ${{ runner.temp }}/workload-manifest.json
        if-no-files-found: error

  build-solutions:
    name: build-${{ matrix.entry.name }}
    needs: discover
    runs-on: ${{ matrix.entry.runsOn }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        entry: ${{ fromJSON(needs.discover.outputs.builds) }}
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - name: Setup .NET
      uses: actions/setup-dotnet@v6
      with:
        dotnet-version: ${{ matrix.entry.dotnet }}

    - name: Set cloud version
      shell: pwsh
      run: ./.azurepipelines/set-version.ps1

    - name: Build ${{ matrix.entry.solution }} (${{ matrix.entry.customTestTarget }})
      shell: pwsh
      env:
        CI_SOLUTION: ${{ matrix.entry.solution }}
        CI_CUSTOM_TEST_TARGET: ${{ matrix.entry.customTestTarget }}
        CI_CONFIGURATION: ${{ matrix.entry.configuration }}
      run: |
        dotnet build $env:CI_SOLUTION `
          --configuration $env:CI_CONFIGURATION `
          "/p:CustomTestTarget=$($env:CI_CUSTOM_TEST_TARGET)"
        if ($LASTEXITCODE -ne 0) {
            Write-Host "::error title=Solution build::$($env:CI_SOLUTION) failed to build for $($env:CI_CUSTOM_TEST_TARGET) ($($env:CI_CONFIGURATION))."
            exit $LASTEXITCODE
        }

  tests:
    name: test-${{ matrix.entry.name }}
    needs: discover
    runs-on: ${{ matrix.entry.runsOn }}
    timeout-minutes: ${{ matrix.entry.timeoutMinutes }}
    strategy:
      fail-fast: false
      # Bounded so one approved full-scope run cannot take the whole
      # organisation's runner concurrency for itself.
      max-parallel: 40
      matrix:
        entry: ${{ fromJSON(needs.discover.outputs.tests) }}
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - uses: ./.github/actions/run-dotnet-tests
      with:
        projects: ${{ matrix.entry.projects }}
        custom-test-target: ${{ matrix.entry.customTestTarget }}
        framework: ${{ matrix.entry.framework }}
        configuration: ${{ matrix.entry.configuration }}
        dotnet: ${{ matrix.entry.dotnet }}
        filter: ${{ matrix.entry.filter }}
        hang-timeout: ${{ matrix.entry.hangTimeout }}
        per-project-timeout: ${{ matrix.entry.perProjectTimeout }}
        coverage: ${{ matrix.entry.coverage }}
        artifact-name: dotnet-results-${{ matrix.entry.id }}

  aot:
    name: aot-${{ matrix.os }}
    needs: discover
    runs-on: ${{ matrix.os }}
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.discover.outputs.aot_matrix) }}
    env:
      CONFIGURATION: Release
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - name: Setup .NET
      uses: actions/setup-dotnet@v6
      with:
        dotnet-version: '10.0.401'

    - name: Set cloud version
      shell: pwsh
      run: ./.azurepipelines/set-version.ps1

    # The three AOT binaries are separate projects to bound the native linker's
    # input, and every one of them is published and run on every AOT platform.
    - name: Publish and run the NativeAOT tests
      shell: pwsh
      env:
        RID: ${{ matrix.rid }}
      run: |
        $ErrorActionPreference = 'Stop'
        $projects = @(
            'tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj',
            'tests/Opc.Ua.Aot.Tests.Historian/Opc.Ua.Aot.Tests.Historian.csproj',
            'tests/Opc.Ua.Aot.Tests.Mcp/Opc.Ua.Aot.Tests.Mcp.csproj')
        $failed = @()
        foreach ($project in $projects) {
            $stem = [System.IO.Path]::GetFileNameWithoutExtension($project)
            Write-Host "::group::$stem ($($env:RID))"
            try {
                $output = dotnet publish $project --configuration $env:CONFIGURATION |
                    ForEach-Object { Write-Host $_; "$_" }
                if ($LASTEXITCODE -ne 0) {
                    # Surface the MSBuild errors in the annotation, not just
                    # the exit code buried in the collapsed group.
                    $errors = @($output | Where-Object { $_ -match ': error ' } | Select-Object -Unique -First 5)
                    $reason = if ($errors.Count -gt 0) { $errors -join ' | ' } else { 'see the step log' }
                    throw "Publish failed with exit code $LASTEXITCODE`: $reason"
                }

                $publishDir = Join-Path (Split-Path $project -Parent) "bin/$($env:CONFIGURATION)/net10.0/$($env:RID)/publish"
                if (-not (Test-Path -LiteralPath $publishDir -PathType Container)) {
                    throw "Publish succeeded but '$publishDir' does not exist."
                }
                # Match the exact file name: on Linux and macOS the native
                # executable has no extension, so for 'Opc.Ua.Aot.Tests'
                # PowerShell reports BaseName 'Opc.Ua.Aot' and Extension '.Tests'.
                $name = if ($IsWindows) { "$stem.exe" } else { $stem }
                $executable = Get-ChildItem -LiteralPath $publishDir -File |
                    Where-Object { $_.Name -ceq $name } |
                    Select-Object -First 1
                if ($null -eq $executable) {
                    # A managed publish that silently fell back from NativeAOT
                    # would leave a .dll here and nothing to run; that is a
                    # failure, not an empty pass.
                    $found = (Get-ChildItem -LiteralPath $publishDir -File |
                        Where-Object { $_.Name -like "$stem*" } |
                        ForEach-Object Name) -join ', '
                    throw "No NativeAOT executable '$name' was produced in '$publishDir' (found: $found)."
                }

                & $executable.FullName --results-directory "./TestResults/$stem"
                if ($LASTEXITCODE -ne 0) {
                    throw "The NativeAOT test run exited with code $LASTEXITCODE."
                }
            }
            catch {
                $failed += $stem
                Write-Host "::error title=$stem ($($env:RID))::$($_.Exception.Message)"
            }
            finally {
                Write-Host '::endgroup::'
            }
        }
        if ($failed.Count -gt 0) {
            throw "NativeAOT failed for: $($failed -join ', ')."
        }

    - name: Upload NativeAOT results
      if: ${{ always() }}
      uses: actions/upload-artifact@v7
      with:
        name: aot-results-${{ matrix.os }}-${{ matrix.rid }}
        path: |
          **/TestResults/**
          **/*-report.html

  coverage:
    name: coverage
    needs: [discover, tests]
    runs-on: ubuntu-latest
    # always(): the coverage of a run whose tests failed is still worth
    # reporting.
    if: ${{ always() }}
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - name: Setup .NET
      uses: actions/setup-dotnet@v6
      with:
        dotnet-version: '10.0.401'

    - name: Download test results
      uses: actions/download-artifact@v8
      with:
        pattern: dotnet-results-*
        path: coverage-artifacts

    - name: Detect coverage
      id: probe
      shell: pwsh
      env:
        TESTS_RESULT: ${{ needs.tests.result }}
      run: |
        $reports = @(Get-ChildItem -Path coverage-artifacts -Recurse -File -Filter *.cobertura.xml -ErrorAction SilentlyContinue)
        Write-Host "Found $($reports.Count) Cobertura report(s)."
        "has_reports=$(($reports.Count -gt 0).ToString().ToLowerInvariant())" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
        if ($reports.Count -gt 0) {
            exit 0
        }

        # Whether "no coverage" is expected depends on whether the matrix ran. A
        # matrix that completed and published nothing means the collector is
        # broken, and reporting green would hide that indefinitely.
        if ($env:TESTS_RESULT -eq 'success') {
            Write-Host '::error title=Coverage::The test matrix completed but published no coverage at all; the collector is not working.'
            exit 1
        }
        Write-Host "::warning title=Coverage::The test matrix did not complete (result: $($env:TESTS_RESULT)), so there is no coverage to evaluate."

    - name: Merge coverage report
      if: ${{ steps.probe.outputs.has_reports == 'true' }}
      shell: pwsh
      run: |
        $ErrorActionPreference = 'Stop'
        dotnet tool install --tool-path tools dotnet-reportgenerator-globaltool
        ./tools/reportgenerator `
            '-reports:coverage-artifacts/**/*.cobertura.xml' `
            '-targetdir:CodeCoverage' `
            '-reporttypes:Cobertura;HtmlSummary' `
            '-title:UA .Net Standard Full Test Coverage' `
            '-assemblyfilters:-*.Tests'
        if ($LASTEXITCODE -ne 0) {
            Write-Host "::error title=Coverage::ReportGenerator failed with exit code $LASTEXITCODE."
            exit $LASTEXITCODE
        }

    - name: Upload merged coverage report
      if: ${{ steps.probe.outputs.has_reports == 'true' }}
      uses: actions/upload-artifact@v7
      with:
        name: coverage-report
        path: CodeCoverage

    # Project floors only: there is no pull request to diff against, so
    # check-coverage.ps1 skips its changed-lines gate.
    - name: Evaluate coverage
      if: ${{ steps.probe.outputs.has_reports == 'true' }}
      shell: pwsh
      run: |
        ./.azurepipelines/check-coverage.ps1 `
            -CoberturaPath CodeCoverage/Cobertura.xml `
            -ThresholdsPath coverage-thresholds.json `
            -RepoRoot '${{ github.workspace }}' `
            -SummaryPath coverage-summary.md
        $outcome = $LASTEXITCODE
        if (Test-Path coverage-summary.md) {
            Get-Content coverage-summary.md | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append
        }
        if ($outcome -ne 0) {
            Write-Host '::error title=Coverage::Coverage thresholds were not met; see the job summary for the numbers.'
            exit 1
        }

  # Replays the full fuzz crash corpus that Azure DevOps used to supply as the
  # secure file FuzzingArtifacts.zip. The corpus lives on the orphan branch
  # 'fuzz-corpus' of this repository (see fuzzing/CrashCorpus.md) and takes too
  # long to replay on every pull request, so it runs here. The commit is
  # pinned: a push to the branch changes nothing until this SHA is updated in a
  # reviewed pull request.
  crash-corpus:
    name: crash-corpus-${{ matrix.entry.name }}
    needs: discover
    runs-on: ${{ matrix.entry.runsOn }}
    timeout-minutes: 90
    env:
      FUZZ_CORPUS_COMMIT: 843ba944a7b144a7ae6429151846d9e5ab28d29f
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        entry: ${{ fromJSON(needs.discover.outputs.crash_corpus_tests) }}
    steps:
    - uses: actions/checkout@v7
      with:
        fetch-depth: 0

    - name: Check out the crash corpus
      uses: actions/checkout@v7
      with:
        ref: ${{ env.FUZZ_CORPUS_COMMIT }}
        path: .fuzz-crash-corpus
        fetch-depth: 1
        persist-credentials: false

    - name: Overlay the crash corpus
      shell: pwsh
      run: |
        $ErrorActionPreference = 'Stop'
        $source = Join-Path $env:GITHUB_WORKSPACE '.fuzz-crash-corpus'
        $actual = (git -C $source rev-parse HEAD).Trim()
        if ($actual -ne $env:FUZZ_CORPUS_COMMIT) {
            throw "The crash corpus is at $actual, expected $env:FUZZ_CORPUS_COMMIT."
        }

        # Each top-level folder overlays the fuzzing/ project of the same name,
        # the location the Azure template extracted to, so the assets land where
        # FuzzTargetTestsBase enumerates them.
        $count = 0
        foreach ($area in Get-ChildItem -LiteralPath $source -Directory | Where-Object Name -ne '.git') {
            $target = Join-Path 'fuzzing' $area.Name
            if (-not (Test-Path -LiteralPath $target)) {
                throw "Corpus folder '$($area.Name)' has no matching fuzzing/ project."
            }
            Copy-Item -Path (Join-Path $area.FullName '*') -Destination $target -Recurse -Force
            $count += @(Get-ChildItem -LiteralPath $area.FullName -Recurse -File).Count
        }
        if ($count -eq 0) {
            throw 'The crash corpus is empty; a replay of nothing would report coverage that was never performed.'
        }
        Remove-Item -LiteralPath $source -Recurse -Force
        Write-Host "Overlaid $count corpus files from commit $actual."
        "Crash corpus: ``$actual`` ($count files)" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append

    - uses: ./.github/actions/run-dotnet-tests
      with:
        projects: ${{ matrix.entry.projects }}
        custom-test-target: ${{ matrix.entry.customTestTarget }}
        framework: ${{ matrix.entry.framework }}
        configuration: ${{ matrix.entry.configuration }}
        dotnet: ${{ matrix.entry.dotnet }}
        filter: ${{ matrix.entry.filter }}
        hang-timeout: ${{ matrix.entry.hangTimeout }}
        per-project-timeout: ${{ matrix.entry.perProjectTimeout }}
        # Not dotnet-results-*: the coverage job merges those, and this replay
        # measures inputs, not code coverage.
        artifact-name: crash-corpus-results-${{ matrix.entry.id }}

  nightly-summary:
    # Deliberately NOT named 'build-and-test summary': that name belongs to the
    # required pull-request check in buildandtest.yml, and two workflows
    # reporting the same context would make a manual run able to satisfy - or
    # break - branch protection.
    name: nightly summary
    runs-on: ubuntu-latest
    if: ${{ always() }}
    needs: [discover, build-solutions, tests, aot, coverage, crash-corpus]
    steps:
    - name: Evaluate job results
      shell: pwsh
      env:
        DISCOVER_RESULT: ${{ needs.discover.result }}
        BUILD_RESULT: ${{ needs.build-solutions.result }}
        TEST_RESULT: ${{ needs.tests.result }}
        AOT_RESULT: ${{ needs.aot.result }}
        COVERAGE_RESULT: ${{ needs.coverage.result }}
        CORPUS_RESULT: ${{ needs.crash-corpus.result }}
      run: |
        $results = [ordered]@{
          'discover'        = $env:DISCOVER_RESULT
          'build-solutions' = $env:BUILD_RESULT
          'tests'           = $env:TEST_RESULT
          'aot'             = $env:AOT_RESULT
          'coverage'        = $env:COVERAGE_RESULT
          'crash-corpus'    = $env:CORPUS_RESULT
        }

        $bad = @()
        foreach ($job in $results.GetEnumerator()) {
          Write-Host "$($job.Key): $($job.Value)"
          if (@('success', 'skipped') -notcontains $job.Value) {
            $bad += "$($job.Key)=$($job.Value)"
          }
        }

        # The crash-corpus replay is part of the full scope and its matrix is
        # never empty, so unlike the other jobs it may not be skipped.
        if ($env:CORPUS_RESULT -eq 'skipped') {
          $bad += 'crash-corpus=skipped'
        }

        if ($bad.Count -gt 0) {
          Write-Host "::error::Nightly jobs did not succeed: $($bad -join ', ')"
          exit 1
        }
        Write-Host 'All nightly jobs succeeded or were intentionally skipped.'