fix(gov): bound EndBlock vote tally work

app/app.go

@@ -581,9 +581,8 @@ func New(
 
 	// register the staking hooks
 	// NOTE: stakingKeeper above is passed by reference, so that it will contain these hooks
-	app.StakingKeeper = *stakingKeeper.SetHooks(
-		stakingtypes.NewMultiStakingHooks(app.DistrKeeper.Hooks(), app.SlashingKeeper.Hooks()),
-	)
+	stakingHooks := stakingtypes.NewMultiStakingHooks(app.DistrKeeper.Hooks(), app.SlashingKeeper.Hooks())
+	app.StakingKeeper = *stakingKeeper.SetHooks(&stakingHooks)
 
 	// ... other modules keepers
 
@@ -801,6 +800,7 @@ func New(
 		appCodec, keys[govtypes.StoreKey], app.GetSubspace(govtypes.ModuleName), app.AccountKeeper, app.BankKeeper,
 		&stakingKeeper, app.ParamsKeeper, govRouter,
 	)
+	stakingHooks.AddHooks(app.GovKeeper.StakingHooks())
 
 	// this line is used by starport scaffolding # stargate/app/keeperDefinition
 

sei-cosmos/proto/cosmos/gov/v1beta1/genesis.proto

@@ -41,4 +41,22 @@ message GenesisState {
     (gogoproto.nullable) = false,
     (gogoproto.moretags) = "yaml:\"tally_params\""
   ];
+  // vote_delegation_snapshots defines the delegation shares maintained for each vote.
+  repeated VoteDelegationSnapshot vote_delegation_snapshots = 8 [(gogoproto.nullable) = false];
+}
+
+// VoteDelegationSnapshot defines the per-validator delegation shares maintained for a vote.
+message VoteDelegationSnapshot {
+  uint64 proposal_id = 1 [(gogoproto.moretags) = "yaml:\"proposal_id\""];
+  string voter = 2;
+  repeated VoteDelegation delegations = 3 [(gogoproto.nullable) = false];
+}
+
+// VoteDelegation defines a voter's shares in one validator.
+message VoteDelegation {
+  string validator = 1;
+  string shares = 2 [
+    (gogoproto.customtype) = "github.com/sei-protocol/sei-chain/sei-cosmos/types.Dec",
+    (gogoproto.nullable) = false
+  ];
 }

sei-cosmos/x/gov/abci.go

@@ -13,7 +13,13 @@ import (
 
 var logger = seilog.NewLogger("cosmos", "x", "gov")
 
-// EndBlocker called every block, process inflation, update validator set.
+// MaxVotesProcessedPerBlock is the governance vote-record budget shared by backfill, tallying, and cleanup.
+const MaxVotesProcessedPerBlock = 1000
+
+// minTallyCleanupVotesPerBlock reserves part of the budget for completed tally archives.
+const minTallyCleanupVotesPerBlock = 100
+
+// EndBlocker expires governance proposals and advances bounded vote tally work.
 func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 	endBlockerStart := time.Now()
 	defer func() {
@@ -50,11 +56,18 @@ func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 		return false
 	})
 
+	remainingVotes := MaxVotesProcessedPerBlock
+	remainingVotes -= keeper.CleanupTallyVotes(ctx, minTallyCleanupVotesPerBlock)
+
 	// fetch active proposals whose voting periods have ended (are passed the block time)
 	keeper.IterateActiveProposalsQueue(ctx, ctx.BlockHeader().Time, func(proposal types.Proposal) bool {
 		var tagValue, logMsg string
 
-		passes, burnDeposits, tallyResults := keeper.Tally(ctx, proposal)
+		complete, processed, passes, burnDeposits, tallyResults := keeper.TallyIncremental(ctx, proposal, remainingVotes)
+		remainingVotes -= processed
+		if !complete {
+			return true
+		}
 
 		// If an expedited proposal fails, we do not want to update
 		// the deposit at this point since the proposal is converted to regular.
@@ -141,6 +154,8 @@ func EndBlocker(ctx sdk.Context, keeper keeper.Keeper) {
 				sdk.NewAttribute(types.AttributeKeyProposalResult, tagValue),
 			),
 		)
-		return false
+		return remainingVotes == 0
 	})
+
+	keeper.CleanupTallyVotes(ctx, remainingVotes)
 }

sei-cosmos/x/gov/abci_test.go

@@ -2,6 +2,7 @@ package gov_test
 
 import (
 	"context"
+	"encoding/binary"
 	"testing"
 	"time"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/sei-protocol/sei-chain/app/legacyabci"
 	sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types"
 	"github.com/sei-protocol/sei-chain/sei-cosmos/x/gov"
+	govkeeper "github.com/sei-protocol/sei-chain/sei-cosmos/x/gov/keeper"
 	"github.com/sei-protocol/sei-chain/sei-cosmos/x/gov/types"
 	"github.com/sei-protocol/sei-chain/sei-cosmos/x/staking"
 )
@@ -432,10 +434,18 @@ func TestExpeditedProposalPassAndConvertToRegular(t *testing.T) {
 			require.NoError(t, err)
 			require.NotNil(t, res)
 
+			proposal, ok := app.GovKeeper.GetProposal(ctx, proposalID)
+			require.True(t, ok)
+			if tc.isExpeditedPasses {
+				// Validator votes YES before the expedited voting period expires.
+				err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, addrs[0], types.NewNonSplitVoteOption(types.OptionYes))
+				require.NoError(t, err)
+			}
+
 			votingParams := app.GovKeeper.GetVotingParams(ctx)
 			newHeader = ctx.BlockHeader()
 
-			newHeader.Time = ctx.BlockHeader().Time.Add(app.GovKeeper.GetDepositParams(ctx).MaxDepositPeriod).Add(votingParams.ExpeditedVotingPeriod)
+			newHeader.Time = proposal.VotingEndTime
 			ctx = ctx.WithBlockHeader(newHeader)
 
 			inactiveQueue = app.GovKeeper.InactiveProposalQueueIterator(ctx, ctx.BlockHeader().Time)
@@ -446,25 +456,20 @@ func TestExpeditedProposalPassAndConvertToRegular(t *testing.T) {
 			require.True(t, activeQueue.Valid())
 
 			activeProposalID := types.GetProposalIDFromBytes(activeQueue.Value())
-			proposal, ok := app.GovKeeper.GetProposal(ctx, activeProposalID)
+			proposal, ok = app.GovKeeper.GetProposal(ctx, activeProposalID)
 			require.True(t, ok)
 			require.Equal(t, types.StatusVotingPeriod, proposal.Status)
 
 			activeQueue.Close()
 
-			if tc.isExpeditedPasses {
-				// Validator votes YES, letting the expedited proposal pass.
-				err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, addrs[0], types.NewNonSplitVoteOption(types.OptionYes))
-				require.NoError(t, err)
-			}
-
 			// Here the expedited proposal is converted to regular after expiry.
 			gov.EndBlocker(ctx, app.GovKeeper)
 
 			activeQueue = app.GovKeeper.ActiveProposalQueueIterator(ctx, ctx.BlockHeader().Time)
 
 			if tc.isExpeditedPasses {
 				require.False(t, activeQueue.Valid())
+				activeQueue.Close()
 
 				proposal, ok = app.GovKeeper.GetProposal(ctx, activeProposalID)
 				require.True(t, ok)
@@ -485,9 +490,7 @@ func TestExpeditedProposalPassAndConvertToRegular(t *testing.T) {
 			}
 
 			// Expedited proposal should be converted to a regular proposal instead.
-			require.True(t, activeQueue.Valid())
-
-			activeProposalID = types.GetProposalIDFromBytes(activeQueue.Value())
+			require.False(t, activeQueue.Valid())
 			activeQueue.Close()
 
 			proposal, ok = app.GovKeeper.GetProposal(ctx, activeProposalID)
@@ -506,8 +509,14 @@ func TestExpeditedProposalPassAndConvertToRegular(t *testing.T) {
 			expectedIntermediateMofuleAccCoings := initialModuleAccCoins.Add(proposalCoins...).Add(proposalCoins...)
 			require.Equal(t, expectedIntermediateMofuleAccCoings, intermediateModuleAccCoins)
 
+			if tc.isRegularEventuallyPassing {
+				// Validator votes YES before the converted regular voting period expires.
+				err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, addrs[0], types.NewNonSplitVoteOption(types.OptionYes))
+				require.NoError(t, err)
+			}
+
 			// block header time at the voting period
-			newHeader.Time = ctx.BlockHeader().Time.Add(app.GovKeeper.GetDepositParams(ctx).MaxDepositPeriod).Add(votingParams.VotingPeriod)
+			newHeader.Time = proposal.VotingEndTime
 			ctx = ctx.WithBlockHeader(newHeader)
 
 			inactiveQueue = app.GovKeeper.InactiveProposalQueueIterator(ctx, ctx.BlockHeader().Time)
@@ -517,12 +526,6 @@ func TestExpeditedProposalPassAndConvertToRegular(t *testing.T) {
 			activeQueue = app.GovKeeper.ActiveProposalQueueIterator(ctx, ctx.BlockHeader().Time)
 			require.True(t, activeQueue.Valid())
 
-			if tc.isRegularEventuallyPassing {
-				// Validator votes YES, letting the converted regular proposal pass.
-				err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, addrs[0], types.NewNonSplitVoteOption(types.OptionYes))
-				require.NoError(t, err)
-			}
-
 			// Here we validate the converted regular proposal
 			gov.EndBlocker(ctx, app.GovKeeper)
 
@@ -606,6 +609,158 @@ func TestEndBlockerProposalHandlerFailed(t *testing.T) {
 	gov.EndBlocker(ctx, app.GovKeeper)
 }
 
+func TestEndBlockerBoundsVoteBackfillTallyAndCleanupWork(t *testing.T) {
+	app := seiapp.Setup(t, false, false, false)
+	ctx := app.BaseApp.NewContext(false, tmproto.Header{})
+
+	cleanupProposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, cleanupProposal)
+	cleanupProposal, found := app.GovKeeper.GetProposal(ctx, cleanupProposal.ProposalId)
+	require.True(t, found)
+	for i := 0; i < 101; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			cleanupProposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+	complete, processed, _, _, _ := app.GovKeeper.TallyIncremental(ctx, cleanupProposal, 101)
+	require.True(t, complete)
+	require.Equal(t, 101, processed)
+	app.GovKeeper.RemoveFromActiveProposalQueue(ctx, cleanupProposal.ProposalId, cleanupProposal.VotingEndTime)
+	cleanupProposal.Status = types.StatusRejected
+	app.GovKeeper.SetProposal(ctx, cleanupProposal)
+
+	proposal, err := app.GovKeeper.SubmitProposal(ctx, TestProposal)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, proposal)
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+
+	for i := 0; i < gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+	store := ctx.KVStore(app.GetKey(types.StoreKey))
+	for i := 0; i < gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		store.Delete(types.VoteDelegationsKey(proposal.ProposalId, addr))
+		store.Delete(types.VoterProposalsKey(addr, proposal.ProposalId))
+	}
+	require.NoError(t, govkeeper.NewMigrator(app.GovKeeper).Migrate3to4(ctx))
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusVotingPeriod, proposal.Status)
+	require.False(t, app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	require.True(t, app.GovKeeper.IsVoteDelegationBackfillInProgress(ctx, proposal.ProposalId))
+	require.Len(t, app.GovKeeper.GetVotes(ctx, proposal.ProposalId), gov.MaxVotesProcessedPerBlock+1)
+	require.Empty(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false))
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, cleanupProposal.ProposalId, false), 1)
+	tracked := 0
+	for i := 0; i < gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		if store.Has(types.VoteDelegationsKey(proposal.ProposalId, addr)) {
+			tracked++
+		}
+	}
+	require.Equal(t, 900, tracked)
+
+	newVoter := make(sdk.AccAddress, 20)
+	binary.BigEndian.PutUint64(newVoter[12:], uint64(gov.MaxVotesProcessedPerBlock+2))
+	err = app.GovKeeper.AddVote(ctx, proposal.ProposalId, newVoter, types.NewNonSplitVoteOption(types.OptionNo))
+	require.ErrorIs(t, err, types.ErrInactiveProposal)
+
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusVotingPeriod, proposal.Status)
+	require.True(t, app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	require.False(t, app.GovKeeper.IsVoteDelegationBackfillInProgress(ctx, proposal.ProposalId))
+	require.Len(t, app.GovKeeper.GetVotes(ctx, proposal.ProposalId), gov.MaxVotesProcessedPerBlock+1)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 898)
+	require.Empty(t, app.GovKeeper.GetArchivedTallyVotes(ctx, cleanupProposal.ProposalId, false))
+
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusRejected, proposal.Status)
+	require.False(t, app.GovKeeper.IsTallying(ctx, proposal.ProposalId))
+	require.Empty(t, app.GovKeeper.GetVotes(ctx, proposal.ProposalId))
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 104)
+
+	gov.EndBlocker(ctx, app.GovKeeper)
+	require.Empty(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false))
+}
+
+func TestEndBlockerKeepsExpeditedAndRegularTallyArchivesSeparate(t *testing.T) {
+	app := seiapp.Setup(t, false, false, false)
+	ctx := app.BaseApp.NewContext(false, tmproto.Header{})
+
+	proposal, err := app.GovKeeper.SubmitProposalWithExpedite(ctx, TestExpeditedProposal, true)
+	require.NoError(t, err)
+	app.GovKeeper.ActivateVotingPeriod(ctx, proposal)
+	proposal, found := app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+
+	for i := 0; i < 2*gov.MaxVotesProcessedPerBlock+1; i++ {
+		addr := make(sdk.AccAddress, 20)
+		binary.BigEndian.PutUint64(addr[12:], uint64(i+1))
+		require.NoError(t, app.GovKeeper.AddVote(
+			ctx,
+			proposal.ProposalId,
+			addr,
+			types.NewNonSplitVoteOption(types.OptionYes),
+		))
+	}
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+	gov.EndBlocker(ctx, app.GovKeeper)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusVotingPeriod, proposal.Status)
+	require.False(t, proposal.IsExpedited)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, true), 1002)
+
+	regularVoter := make(sdk.AccAddress, 20)
+	binary.BigEndian.PutUint64(regularVoter[12:], uint64(2*gov.MaxVotesProcessedPerBlock+2))
+	require.NoError(t, app.GovKeeper.AddVote(
+		ctx,
+		proposal.ProposalId,
+		regularVoter,
+		types.NewNonSplitVoteOption(types.OptionNo),
+	))
+
+	ctx = ctx.WithBlockTime(proposal.VotingEndTime)
+	gov.EndBlocker(ctx, app.GovKeeper)
+
+	proposal, found = app.GovKeeper.GetProposal(ctx, proposal.ProposalId)
+	require.True(t, found)
+	require.Equal(t, types.StatusRejected, proposal.Status)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, true), 3)
+	require.Len(t, app.GovKeeper.GetArchivedTallyVotes(ctx, proposal.ProposalId, false), 1)
+}
+
 // With expedited proposal's minimum deposit set higher than the default deposit, we must
 // initialize and deposit an amount depositMultiplier times larger
 // than the regular min deposit amount.