From 3e3358f33804cf8d0fce5981bd2b580831e237a7 Mon Sep 17 00:00:00 2001 From: Francesco4203 Date: Wed, 29 May 2024 13:41:33 +0700 Subject: [PATCH] pruneSnapshot: prune nSnapshotPrune snapshots from the previous 5 days pruneSnapshot: traverse back using block number instead of hash (which needs to be calculated more costly) --- consensus/consortium/v2/consortium.go | 10 ++--- consensus/consortium/v2/snapshot.go | 58 +++++++++++---------------- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/consensus/consortium/v2/consortium.go b/consensus/consortium/v2/consortium.go index fb7e6f9546..12ff23c94a 100644 --- a/consensus/consortium/v2/consortium.go +++ b/consensus/consortium/v2/consortium.go @@ -391,7 +391,7 @@ func (c *Consortium) verifyCascadingFields(chain consensus.ChainHeaderReader, he return consortiumCommon.ErrExtraValidators } - if isTrippEffective{ + if isTrippEffective { if isEpoch && len(extraData.BlockProducers) == 0 { return consortiumCommon.ErrExtraValidators } @@ -516,7 +516,7 @@ func (c *Consortium) snapshot(chain consensus.ChainHeaderReader, number uint64, // So the final result must be [998: addressN - 1,999: addressN] snap.Recents = consortiumCommon.RemoveOutdatedRecents(snapV1.Recents, number) - if err := snap.store(c.db); err != nil { + if err := snap.store(c.db, chain); err != nil { return nil, err } log.Info("Stored checkpoint snapshot to disk", "number", number, "hash", hash) @@ -575,7 +575,7 @@ func (c *Consortium) snapshot(chain consensus.ChainHeaderReader, number uint64, // If we've generated a new checkpoint snapshot, save to disk if snap.Number%c.config.EpochV2 == 0 && len(headers) > 0 { - if err = snap.store(c.db); err != nil { + if err = snap.store(c.db, chain); err != nil { return nil, err } log.Trace("Stored snapshot to disk", "number", snap.Number, "hash", snap.Hash) @@ -1637,7 +1637,7 @@ func (c *Consortium) IsPeriodBlock(chain consensus.ChainHeaderReader, header *ty if c.testTrippPeriod { return true } - + number := header.Number.Uint64() if number%c.config.EpochV2 != 0 || !chain.Config().IsTripp(header.Number) { return false @@ -1659,7 +1659,7 @@ func (c *Consortium) IsPeriodBlock(chain consensus.ChainHeaderReader, header *ty // IsTrippEffective returns indicator whether the Tripp consensus rule is effective, // which is the first period that is greater than Tripp period, calculated by formula: -// period := timestamp / dayInSeconds. +// period := timestamp / dayInSeconds. func (c *Consortium) IsTrippEffective(chain consensus.ChainHeaderReader, header *types.Header) bool { if c.chainConfig.IsTripp(header.Number) { if c.testTrippEffective { diff --git a/consensus/consortium/v2/snapshot.go b/consensus/consortium/v2/snapshot.go index 7d49de212e..209bc0dcde 100644 --- a/consensus/consortium/v2/snapshot.go +++ b/consensus/consortium/v2/snapshot.go @@ -115,53 +115,43 @@ func loadSnapshot( // snapshot pruning // delete the nSnapshotsPrune oldest snapshots -func pruneSnapshot(db ethdb.Database, nSnapshotsPrune int) error { - // Get all snapshots (hash, block number) from the database - nSnapShots := 0 - snapshots := make(map[common.Hash]uint64) - it := db.NewIterator(rawdb.ConsortiumSnapshotPrefix, nil) - defer it.Release() - for it.Next() { - snap := new(Snapshot) - if err := json.Unmarshal(it.Value(), snap); err != nil { - return err +func (s *Snapshot) pruneSnapshot(db ethdb.Database, nSnapshotPrune int, chain consensus.ChainHeaderReader) error { + log.Info("Pruning snapshots at block", "block", s.Number, "nSnapshotPrune", nSnapshotPrune) + // Get block number to start pruning + curBlockNumber := s.Number + curBlockNumber -= curBlockNumber % 200 // make sure the block number is multiple of 200 + curBlockNumber -= 200 * 144 * 5 // keep the last 5 days of snapshots + + // delete nSnapshotPrune snapshots starting from curBlockNumber to the older ones + batch := db.NewBatch() + for nSnapshotPrune > 0 { + nSnapshotPrune-- + header := chain.GetHeaderByNumber(curBlockNumber) + if header == nil { + // no more snapshots to prune + break } - snapshots[snap.Hash] = snap.Number - nSnapShots++ - } - - // Sort the snapshots by block number - hashes := make([]common.Hash, 0, nSnapShots) - for hash := range snapshots { - hashes = append(hashes, hash) - } - sort.Slice(hashes, func(i, j int) bool { - return snapshots[hashes[i]] < snapshots[hashes[j]] - }) - - // Prune the snapshots - if nSnapshotsPrune > nSnapShots { - nSnapshotsPrune = nSnapShots - 1 - } - for i := 0; i < nSnapshotsPrune; i++ { - if err := db.Delete(append(rawdb.ConsortiumSnapshotPrefix, hashes[i][:]...)); err != nil { + curHash := header.Hash() + if err := batch.Delete(append(rawdb.ConsortiumSnapshotPrefix, curHash[:]...)); err != nil { return err } + curBlockNumber -= 200 } - return nil + log.Info("Pruned snapshots done") + return batch.Write() } // store inserts the snapshot into the database. -func (s *Snapshot) store(db ethdb.Database) error { +func (s *Snapshot) store(db ethdb.Database, chain consensus.ChainHeaderReader) error { blob, err := json.Marshal(s) if err != nil { return err } // prune nSnapshotPrune snapshots every prunePeriod blocks - nSnapshotsPrune := 200 * 144 // 1 day - prunePeriod := 200 * 144 // 1 day + nSnapshotPrune := 144 * 2 // 2 days + prunePeriod := 200 * 144 // 1 day if s.Number%uint64(prunePeriod) == 0 { - if err := pruneSnapshot(db, nSnapshotsPrune); err != nil { + if err := s.pruneSnapshot(db, nSnapshotPrune, chain); err != nil { log.Error("Failed to prune snapshots", "err", err) } }