From 0b64dc03d5a53853254128dcced990069a1238d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 19:30:47 +0000 Subject: [PATCH 1/2] fix(rspec): quarantine fixed-pending examples instead of hard-failing A `pending` example whose body now passes is turned by RSpec into a real failure (PendingExampleFixedError) that reds the build. Example#set_exception short-circuited on metadata[:pending] before reaching handle_quarantine_check, so such an example never consulted the quarantine list: a fixed-pending test the user had quarantined still failed the build and uploaded as a plain, un-quarantined failure (is_quarantined derives from metadata[:quarantined_exception], which only handle_quarantine_check sets). - set_exception: let a PendingExampleFixedError fall through to the quarantine check (ordinary still-failing pending examples stay green). - status_and_exception: when the swallowed fixed-pending leaves RSpec at :pending, use execution_result.pending_fixed? to still report it as a failure carrying the fixed error, so it serializes as failure + quarantined. Coverage (ruby gem upload action), mirroring variant_quarantine_spec.rb: - spec/pending_spec.rb: a fixed-pending example. - with-variant step now runs variant_quarantine_spec.rb + pending_spec.rb and asserts the run passes (both quarantined under smoke-test-variant). - new no-variant step runs pending_spec.rb alone and asserts a non-zero exit, isolating the fixed-pending failure when it is not quarantined. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RHvMvkpGYtQYZ3FQ4j2w65 --- .../actions/test_ruby_gem_uploads/action.yaml | 32 ++++++++++++++++--- .../spec/pending_spec.rb | 11 +++++++ .../lib/trunk_spec_helper.rb | 19 ++++++++--- 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 .github/actions/test_ruby_gem_uploads/spec/pending_spec.rb diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 19ed9964..ea8e6f54 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -174,20 +174,20 @@ runs: TRUNK_TEST_COLLECTION_ID: ${{ inputs.test-collection-id }} TRUNK_VARIANT: "" - - name: Run variant quarantine test with variant (should pass - quarantined) + - name: Run variant and pending quarantine tests with variant (should pass - quarantined) id: run-variant-test-with-variant if: inputs.alpine != 'true' shell: bash working-directory: ${{ github.action_path }} run: | - bundle exec rspec spec/variant_quarantine_spec.rb --format documentation + bundle exec rspec spec/variant_quarantine_spec.rb spec/pending_spec.rb --format documentation EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then - echo "ERROR: Variant quarantine test with variant should have passed (test should be quarantined)" - echo "Note: The variant_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" + echo "ERROR: Quarantine tests with variant should have passed (tests should be quarantined)" + echo "Note: variant_quarantine_test and pending_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" exit 1 else - echo "SUCCESS: Variant quarantine test with variant passed (test was quarantined)" + echo "SUCCESS: Quarantine tests with variant passed (tests were quarantined)" fi env: TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} @@ -196,6 +196,28 @@ runs: TRUNK_TEST_COLLECTION_ID: ${{ inputs.test-collection-id }} TRUNK_VARIANT: smoke-test-variant + - name: Run pending quarantine test without variant (should fail - not quarantined) + id: run-pending-test-no-variant + if: inputs.alpine != 'true' + shell: bash + working-directory: ${{ github.action_path }} + run: | + set +e + bundle exec rspec spec/pending_spec.rb --format documentation + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "ERROR: Pending quarantine test without variant should have failed (fixed-pending is not quarantined)" + exit 1 + else + echo "SUCCESS: Pending quarantine test without variant failed as expected (fixed-pending is not quarantined)" + fi + env: + TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + TRUNK_TEST_COLLECTION_ID: ${{ inputs.test-collection-id }} + TRUNK_VARIANT: "" + - name: Run quarantine lookup failure abort test (should fail fast, skip slow_test) id: run-quarantine-abort-test if: inputs.alpine != 'true' diff --git a/.github/actions/test_ruby_gem_uploads/spec/pending_spec.rb b/.github/actions/test_ruby_gem_uploads/spec/pending_spec.rb new file mode 100644 index 00000000..3269465b --- /dev/null +++ b/.github/actions/test_ruby_gem_uploads/spec/pending_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# A fixed-pending example (body now passes -> PendingExampleFixedError) reds the +# build unless quarantined: it fails without a variant and passes (quarantined) +# when run with the smoke-test-variant variant. +describe 'pending_quarantine_test' do + it 'should be quarantined when run with variant' do + pending('expected to still fail, but the body now passes') + expect(2 + 2).to eq(4) + end +end diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 78e38104..1405f399 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -108,13 +108,19 @@ class Example # decide if we want to fail the test or not # trunk-ignore(rubocop/Naming/AccessorMethodName) def set_exception(exception) - return set_exception_core(exception) if metadata[:pending] + # Pending stays green, except a fixed pending example is a real failure. + return set_exception_core(exception) if metadata[:pending] && !pending_example_fixed?(exception) return set_exception_core(exception) if trunk_disabled return set_exception_core(exception) if metadata[:retry_attempts]&.positive? handle_quarantine_check(exception) end + def pending_example_fixed?(exception) + defined?(RSpec::Core::Pending::PendingExampleFixedError) && + exception.is_a?(RSpec::Core::Pending::PendingExampleFixedError) + end + # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength,rubocop/Metrics/CyclomaticComplexity) def handle_quarantine_check(exception) id = generate_trunk_id @@ -401,10 +407,13 @@ def status_and_exception(example) # the build (the pending expectation was violated), so it is a failure here too. [Status.new('failure'), example.exception || quarantined_exception] when :pending - # Not a skip (handled above), so this is a pending example whose body ran and - # failed as expected: the pending expectation ("this should fail") was met, so - # RSpec keeps the build green -- report it as a success. - [Status.new('success'), nil] + # A quarantined fixed-pending is left :pending but pending_fixed? -- still a + # failure. Otherwise the pending "should fail" expectation was met -> success. + if result.pending_fixed? + [Status.new('failure'), quarantined_exception || example.exception] + else + [Status.new('success'), nil] + end else [Status.new(result.status.to_s), example.exception || quarantined_exception] end From ed94abe9290b3ba3c173820f5bdd5166eab7d1db Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Thu, 9 Jul 2026 22:22:44 +0000 Subject: [PATCH 2/2] fix! --- rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 1405f399..5c5606a1 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -113,7 +113,15 @@ def set_exception(exception) return set_exception_core(exception) if trunk_disabled return set_exception_core(exception) if metadata[:retry_attempts]&.positive? + handle_quarantine_check_safely(exception) + end + + # If the quarantine machinery itself blows up, the failure must stand. + def handle_quarantine_check_safely(exception) handle_quarantine_check(exception) + rescue StandardError => e + puts "Quarantine check errored (#{e.class}: #{e.message}), treating test as not quarantined".yellow + set_exception_core(exception) end def pending_example_fixed?(exception)