From a8ac29c5d896f5f39adce5cf689b3f68eb1dd7ea Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 19 Apr 2026 15:24:27 -0400 Subject: [PATCH 1/2] Pro-rate new_state_pension by reported amount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous formula `eligible * p.amount * WEEKS_IN_YEAR` paid the flat max NSP rate to every NEW-type retiree, regardless of their actual NI record length or pre-2016 SERPS/S2P accrual. This: - overstated retirees with partial NI records (< 35 qualifying years), who under the real NSP rules get pro-rated amounts; and - silently dropped the Protected Payment component — the portion of a retiree's starting amount that exceeds the new flat rate because of pre-2016 SERPS/S2P contributions — because the formula clamped everyone at p.amount. Switch to the same share-of-max pattern `basic_state_pension` already uses: `share = reported / max_new_data_year`, scaled by the current period's max. Share above 1 is preserved so Protected Payment flows through, closing part of the ~£12 bn residual state-pension gap vs the OBR target (tracked in #1632). Five new YAML tests cover under-max (partial NI), exact max (full rate), above-max (Protected Payment), non-NEW type returning zero, and zero-max guard. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../fix-new-state-pension-reported.fixed.md | 1 + .../baseline/gov/dwp/new_state_pension.yaml | 58 +++++++++++++++++++ .../variables/gov/dwp/new_state_pension.py | 34 +++++++++-- uv.lock | 2 +- 4 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 changelog.d/fix-new-state-pension-reported.fixed.md create mode 100644 policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml diff --git a/changelog.d/fix-new-state-pension-reported.fixed.md b/changelog.d/fix-new-state-pension-reported.fixed.md new file mode 100644 index 000000000..ce7645c90 --- /dev/null +++ b/changelog.d/fix-new-state-pension-reported.fixed.md @@ -0,0 +1 @@ +Pro-rate `new_state_pension` by the reported FRS amount instead of paying a flat maximum to every NEW-type retiree. The prior formula returned `eligible * p.amount` regardless of NI record length, which overstated partial-record retirees (< 35 qualifying years) and silently dropped the pre-2016 SERPS/S2P Protected Payment component for retirees whose reported amount exceeds the flat max rate. Matches the pattern already used by `basic_state_pension` and helps close the ~£12 bn residual state-pension gap vs the OBR target tracked in #1632. diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml new file mode 100644 index 000000000..37e8dbdf4 --- /dev/null +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml @@ -0,0 +1,58 @@ +- name: New State Pension uses reported amount, not flat max + # Under 35 NI years — partial rate should be preserved, not paid at max. + period: 2021 + input: + people: + person1: + state_pension_reported: 7000 # Below max (9627.80 = 185.15 * 52) + state_pension_type: NEW + age: 67 + output: + new_state_pension: 7000 + +- name: New State Pension equals max for full-rate retirees + period: 2021 + input: + people: + person1: + state_pension_reported: 9627.80 # Exactly the max + state_pension_type: NEW + age: 67 + output: + new_state_pension: 9627.80 + +- name: New State Pension captures Protected Payment above max rate + # Pre-2016 SERPS/S2P accrual — reported exceeds flat max; the excess is + # the Protected Payment component of NSP and must flow through. + period: 2021 + input: + people: + person1: + state_pension_reported: 12000 # Above max — Protected Payment + state_pension_type: NEW + age: 67 + output: + new_state_pension: 12000 + +- name: Basic type returns zero for new_state_pension + period: 2021 + input: + people: + person1: + state_pension_reported: 5000 + state_pension_type: BASIC + age: 70 + output: + new_state_pension: 0 + +- name: New State Pension is zero when max is zero + period: 2021 + input: + gov.dwp.state_pension.new_state_pension.amount: 0 + people: + person1: + state_pension_reported: 9000 + state_pension_type: NEW + age: 67 + output: + new_state_pension: 0 diff --git a/policyengine_uk/variables/gov/dwp/new_state_pension.py b/policyengine_uk/variables/gov/dwp/new_state_pension.py index 6cfc9be7d..779e4bc5a 100644 --- a/policyengine_uk/variables/gov/dwp/new_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/new_state_pension.py @@ -10,10 +10,32 @@ class new_state_pension(Variable): def formula(person, period, parameters): simulation = person.simulation - if simulation.dataset is None: - return 0 + has_dataset = simulation.dataset is not None - type = person("state_pension_type", period) - eligible = type == type.possible_values.NEW - p = parameters(period).gov.dwp.state_pension.new_state_pension - return eligible * p.amount * WEEKS_IN_YEAR + if has_dataset: + try: + data_year = min(simulation.dataset.years) + except: + data_year = period.start.year + else: + data_year = period.start.year + + pension_type = person("state_pension_type", period) + eligible = pension_type == pension_type.possible_values.NEW + + reported_weekly = person("state_pension_reported", data_year) / WEEKS_IN_YEAR + p = parameters.gov.dwp.state_pension.new_state_pension + max_new_data_year = p.amount(data_year) + max_new_period = p.amount(period) + + # Pro-rate by reported amount rather than flat max: + # share < 1 → under-35-year NI records get their actual partial rate + # share > 1 → pre-2016 SERPS/S2P Protected Payment adds on top + # The previous ``eligible * p.amount`` overstated partial-record + # retirees and dropped the Protected Payment tail entirely. + share = where( + max_new_data_year > 0, + reported_weekly / max_new_data_year, + 0, + ) + return eligible * share * max_new_period * WEEKS_IN_YEAR diff --git a/uv.lock b/uv.lock index 5e4c8dc15..3591c22cb 100644 --- a/uv.lock +++ b/uv.lock @@ -1584,7 +1584,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.88.3" +version = "2.88.5" source = { editable = "." } dependencies = [ { name = "microdf-python", marker = "python_full_version >= '3.11'" }, From e09ca22164ca95c88f3e63acf3ce3e28336a8841 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 19 Apr 2026 15:38:41 -0400 Subject: [PATCH 2/2] Cap NSP at flat max and route Protected Payment through ASP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subagent review of #1634 flagged two issues with the initial pass: 1. The uncapped NSP formula mixed 'headline new rate' with 'Protected Payment' semantics, surprising any downstream consumer who reads the variable name literally. BASIC routes excess through additional_state_pension; NEW should too for symmetry. 2. No test case for `state_pension_reported = 0` on a NEW-type retiree (deferral / FRS non-report) — the single highest-impact aggregate-drift scenario. Previously the formula paid full flat max regardless; under the new formula it correctly returns zero. Changes: - `new_state_pension`: cap share at 1 (use `min_(reported, max)`) like basic_state_pension does. Flat max is now the ceiling. - `additional_state_pension`: extend to NEW-type retirees, using the NSP max as the ceiling (was BASIC-only, using the basic max). This captures Protected Payment as an explicit top-up. - Added deferral/zero-report test case. - Fixed stale 9627.80 comment to the correct 2021 figure 9339.20. Full 978-case policy suite passes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../fix-new-state-pension-reported.fixed.md | 2 +- .../baseline/gov/dwp/new_state_pension.yaml | 42 ++++++++++++++--- .../gov/dwp/additional_state_pension.py | 46 +++++++++++++------ .../variables/gov/dwp/new_state_pension.py | 13 +++--- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/changelog.d/fix-new-state-pension-reported.fixed.md b/changelog.d/fix-new-state-pension-reported.fixed.md index ce7645c90..0c32f3376 100644 --- a/changelog.d/fix-new-state-pension-reported.fixed.md +++ b/changelog.d/fix-new-state-pension-reported.fixed.md @@ -1 +1 @@ -Pro-rate `new_state_pension` by the reported FRS amount instead of paying a flat maximum to every NEW-type retiree. The prior formula returned `eligible * p.amount` regardless of NI record length, which overstated partial-record retirees (< 35 qualifying years) and silently dropped the pre-2016 SERPS/S2P Protected Payment component for retirees whose reported amount exceeds the flat max rate. Matches the pattern already used by `basic_state_pension` and helps close the ~£12 bn residual state-pension gap vs the OBR target tracked in #1632. +Replace `new_state_pension`'s flat-max payout with a `min(reported, max) / max * period_max` formula mirroring `basic_state_pension`, and extend `additional_state_pension` to NEW-type retirees so any pre-2016 SERPS/S2P Protected Payment flows through as an add-on instead of being silently dropped. Partial-NI-record retirees now receive their actual pro-rated rate rather than the full flat max. Closes part of the ~£12 bn residual state-pension gap vs the OBR target tracked in #1632. diff --git a/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml b/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml index 37e8dbdf4..b5a4287ac 100644 --- a/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml +++ b/policyengine_uk/tests/policy/baseline/gov/dwp/new_state_pension.yaml @@ -4,7 +4,7 @@ input: people: person1: - state_pension_reported: 7000 # Below max (9627.80 = 185.15 * 52) + state_pension_reported: 7000 # Below max (9339.20 = 179.60 * 52) state_pension_type: NEW age: 67 output: @@ -15,15 +15,16 @@ input: people: person1: - state_pension_reported: 9627.80 # Exactly the max + state_pension_reported: 9339.20 # Exactly the 2021 max state_pension_type: NEW age: 67 output: - new_state_pension: 9627.80 + new_state_pension: 9339.20 -- name: New State Pension captures Protected Payment above max rate - # Pre-2016 SERPS/S2P accrual — reported exceeds flat max; the excess is - # the Protected Payment component of NSP and must flow through. +- name: New State Pension caps at max; Protected Payment flows to additional_state_pension + # Pre-2016 SERPS/S2P accrual — reported exceeds flat max. The flat-rate + # component stays capped at the max, and the excess is treated as a + # Protected Payment top-up tracked via additional_state_pension. period: 2021 input: people: @@ -32,7 +33,34 @@ state_pension_type: NEW age: 67 output: - new_state_pension: 12000 + new_state_pension: 9339.20 + additional_state_pension: 2660.80 # 12000 − 9339.20 + +- name: Additional State Pension captures SERPS/S2P for BASIC retirees + period: 2021 + input: + people: + person1: + state_pension_reported: 9000 # Above basic max (7155.20) + state_pension_type: BASIC + age: 70 + output: + additional_state_pension: 1844.80 # 9000 − 7155.20 + +- name: NEW-type retiree with zero reported pension gets zero NSP + # Deferral or FRS non-report. Matches reality — a deferred NSP claimant + # genuinely receives nothing until they start drawing. This differs from + # the old formula, which paid the flat max regardless of reported. + period: 2021 + input: + people: + person1: + state_pension_reported: 0 + state_pension_type: NEW + age: 67 + output: + new_state_pension: 0 + additional_state_pension: 0 - name: Basic type returns zero for new_state_pension period: 2021 diff --git a/policyengine_uk/variables/gov/dwp/additional_state_pension.py b/policyengine_uk/variables/gov/dwp/additional_state_pension.py index fcf87f001..2e77b4f77 100644 --- a/policyengine_uk/variables/gov/dwp/additional_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/additional_state_pension.py @@ -10,25 +10,45 @@ class additional_state_pension(Variable): def formula(person, period, parameters): simulation = person.simulation - if simulation.dataset is None: - return 0 - try: - data_year = min(simulation.dataset.years) - except: + has_dataset = simulation.dataset is not None + if has_dataset: + try: + data_year = min(simulation.dataset.years) + except: + data_year = period.start.year + else: data_year = period.start.year reported = person("state_pension_reported", data_year) / WEEKS_IN_YEAR - type = person("state_pension_type", data_year) - sp_amount = parameters.gov.dwp.state_pension.basic_state_pension.amount - max_sp_data_year = sp_amount(data_year) - max_sp_period = sp_amount(period) + pension_type = person("state_pension_type", data_year) + types = pension_type.possible_values + + bsp_amount = parameters.gov.dwp.state_pension.basic_state_pension.amount + nsp_amount = parameters.gov.dwp.state_pension.new_state_pension.amount + + # Each pension type has its own flat-rate ceiling; anything above + # the ceiling is an add-on: + # BASIC → SERPS / S2P (pre-2016 earnings-related top-up) + # NEW → Protected Payment (pre-2016 accrual exceeding the + # new flat rate, folded into NSP under current law) + max_for_type_data = select( + [pension_type == types.BASIC, pension_type == types.NEW], + [bsp_amount(data_year), nsp_amount(data_year)], + default=0, + ) + max_for_type_period = select( + [pension_type == types.BASIC, pension_type == types.NEW], + [bsp_amount(period), nsp_amount(period)], + default=0, + ) + amount_in_data_year = where( - type == type.possible_values.BASIC, - max_(reported - max_sp_data_year, 0), + pension_type != types.NONE, + max_(reported - max_for_type_data, 0), 0, ) uprating = where( - max_sp_data_year > 0, - max_sp_period / max_sp_data_year, + max_for_type_data > 0, + max_for_type_period / max_for_type_data, 1, ) return amount_in_data_year * uprating * WEEKS_IN_YEAR diff --git a/policyengine_uk/variables/gov/dwp/new_state_pension.py b/policyengine_uk/variables/gov/dwp/new_state_pension.py index 779e4bc5a..bd99d7065 100644 --- a/policyengine_uk/variables/gov/dwp/new_state_pension.py +++ b/policyengine_uk/variables/gov/dwp/new_state_pension.py @@ -28,14 +28,15 @@ def formula(person, period, parameters): max_new_data_year = p.amount(data_year) max_new_period = p.amount(period) - # Pro-rate by reported amount rather than flat max: - # share < 1 → under-35-year NI records get their actual partial rate - # share > 1 → pre-2016 SERPS/S2P Protected Payment adds on top - # The previous ``eligible * p.amount`` overstated partial-record - # retirees and dropped the Protected Payment tail entirely. + # Pro-rate by reported amount, capped at the flat max so a partial + # NI record gets the appropriate partial rate. Any reported amount + # above the flat max is Protected Payment (pre-2016 SERPS/S2P) and + # flows through ``additional_state_pension`` — mirrors the + # BASIC / ASP split so ``new_state_pension`` is always the headline + # flat-rate component. share = where( max_new_data_year > 0, - reported_weekly / max_new_data_year, + min_(reported_weekly, max_new_data_year) / max_new_data_year, 0, ) return eligible * share * max_new_period * WEEKS_IN_YEAR