-
Notifications
You must be signed in to change notification settings - Fork 248
[DNM] feat: delete go-header store cmd #3040
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
julienrbrt
wants to merge
28
commits into
main
Choose a base branch
from
julien/go-header-clean
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+1,102
−0
Open
Changes from all commits
Commits
Show all changes
28 commits
Select commit
Hold shift + click to select a range
83b25d6
refactor(store): replace go-header store by ev-node store
julienrbrt ebae42f
simplify
julienrbrt d56115d
go mod tidy
julienrbrt 1e26328
append in cache instead of duplicate store
julienrbrt 915842e
Merge branch 'main' into julien/simplify-store
julienrbrt bfc0701
clean-up
julienrbrt ca8651b
remove exchanger (partial revert #2855)
julienrbrt 9092ede
add changelog
julienrbrt 9c74889
updates
julienrbrt dee6ed5
use generic and improve tail
julienrbrt 70dc901
lint
julienrbrt 8106374
simplify code
julienrbrt 25cc36b
wait for height
julienrbrt dfe1071
cleanup duplicates
julienrbrt 0124c25
remove unecessary go-header store write
julienrbrt b0fa1e2
Merge branch 'main' into julien/simplify-store
julienrbrt 1300d16
Merge branch 'main' into julien/simplify-store
julienrbrt 9101f11
update test based on new behavior
julienrbrt 0588001
kiss
julienrbrt e717aa2
Merge branch 'main' into julien/simplify-store
julienrbrt 0cc54d0
update sync service
julienrbrt 08d8594
[DNM] feat: delete go-header store cmd
julienrbrt f1460be
updates
julienrbrt 514ba02
Merge branch 'main' into julien/go-header-clean
julienrbrt f74fcda
Merge branch 'main' into julien/go-header-clean
julienrbrt 56b5321
Merge branch 'main' into julien/go-header-clean
julienrbrt 2eca3d1
extract into standalone
julienrbrt 7472bd7
correct prefix
julienrbrt File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,159 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| ds "github.com/ipfs/go-datastore" | ||
| "github.com/ipfs/go-datastore/namespace" | ||
| "github.com/ipfs/go-datastore/query" | ||
| "github.com/spf13/cobra" | ||
|
|
||
| rollcmd "github.com/evstack/ev-node/pkg/cmd" | ||
| "github.com/evstack/ev-node/pkg/config" | ||
| "github.com/evstack/ev-node/pkg/store" | ||
| ) | ||
|
|
||
| const evmDbName = "evm-single" | ||
|
|
||
| const ( | ||
| // go-header store prefixes used prior to the unified store migration | ||
| headerSyncPrefix = "/headerSync" | ||
| dataSyncPrefix = "/dataSync" | ||
| ) | ||
|
|
||
| func main() { | ||
| // Initiate the root command | ||
| rootCmd := &cobra.Command{ | ||
| Use: "db-cleaner", | ||
| } | ||
|
|
||
| config.AddGlobalFlags(rootCmd, "evm") | ||
|
|
||
| rootCmd.AddCommand( | ||
| NewCleanupGoHeaderCmd(), | ||
| ) | ||
|
|
||
| if err := rootCmd.Execute(); err != nil { | ||
| // Print to stderr and exit with error | ||
| fmt.Fprintln(os.Stderr, err) | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| // NewCleanupGoHeaderCmd creates a command to delete the legacy go-header store data. | ||
| // This data is no longer needed after the migration to the unified store approach | ||
| // where HeaderStoreAdapter and DataStoreAdapter read directly from the ev-node store. | ||
| func NewCleanupGoHeaderCmd() *cobra.Command { | ||
| var dryRun bool | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "clean-evm", | ||
| Short: "Delete legacy go-header store data from disk", | ||
| Long: `Delete the legacy go-header store data (headerSync and dataSync prefixes) from the database. | ||
|
|
||
| This command removes data that was previously duplicated by the go-header library | ||
| for P2P sync operations. After the migration to the unified store approach, | ||
| this data is no longer needed as the HeaderStoreAdapter and DataStoreAdapter | ||
| now read directly from the ev-node store. | ||
|
|
||
| WARNING: Make sure the node is stopped before running this command. | ||
| This operation is irreversible.`, | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| nodeConfig, err := rollcmd.ParseConfig(cmd) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| goCtx := cmd.Context() | ||
| if goCtx == nil { | ||
| goCtx = context.Background() | ||
| } | ||
|
|
||
| // Open the database | ||
| rawDB, err := store.NewDefaultKVStore(nodeConfig.RootDir, nodeConfig.DBPath, evmDbName) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open database: %w", err) | ||
| } | ||
| defer func() { | ||
| if closeErr := rawDB.Close(); closeErr != nil { | ||
| cmd.Printf("Warning: failed to close database: %v\n", closeErr) | ||
| } | ||
| }() | ||
|
|
||
| evDB := store.NewEvNodeKVStore(rawDB) | ||
| // Delete headerSync prefix | ||
| headerCount, err := deletePrefix(goCtx, evDB, headerSyncPrefix, dryRun) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to delete headerSync data: %w", err) | ||
| } | ||
|
|
||
| // Delete dataSync prefix | ||
| dataCount, err := deletePrefix(goCtx, evDB, dataSyncPrefix, dryRun) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to delete dataSync data: %w", err) | ||
| } | ||
|
|
||
| totalCount := headerCount + dataCount | ||
|
|
||
| if dryRun { | ||
| cmd.Printf("Dry run: would delete %d keys (%d headerSync, %d dataSync)\n", | ||
| totalCount, headerCount, dataCount) | ||
| } else { | ||
| if totalCount == 0 { | ||
| cmd.Println("No legacy go-header store data found to delete.") | ||
| } else { | ||
| cmd.Printf("Successfully deleted %d keys (%d headerSync, %d dataSync)\n", | ||
| totalCount, headerCount, dataCount) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| }, | ||
| } | ||
|
|
||
| cmd.Flags().BoolVar(&dryRun, "dry-run", false, "show what would be deleted without actually deleting") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| // deletePrefix deletes all keys with the given prefix from the datastore. | ||
| // Returns the number of keys deleted. | ||
| func deletePrefix(ctx context.Context, db ds.Batching, prefix string, dryRun bool) (int, error) { | ||
| count := 0 | ||
|
|
||
| pdb := namespace.Wrap(db, ds.NewKey(prefix)) | ||
| results, err := pdb.Query(ctx, query.Query{ | ||
| KeysOnly: true, | ||
| }) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to query keys with prefix %s: %w", prefix, err) | ||
| } | ||
|
|
||
| batch, err := pdb.Batch(ctx) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to create batch: %w", err) | ||
| } | ||
|
|
||
| for result := range results.Next() { | ||
| if result.Error != nil { | ||
| return count, fmt.Errorf("error iterating results: %w", result.Error) | ||
| } | ||
|
|
||
| if !dryRun { | ||
| if err := batch.Delete(ctx, ds.NewKey(result.Key)); err != nil { | ||
| return count, fmt.Errorf("failed to delete key %s: %w", result.Key, err) | ||
| } | ||
| } | ||
| count++ | ||
| } | ||
|
|
||
| if !dryRun && count > 0 { | ||
| if err := batch.Commit(ctx); err != nil { | ||
| return count, fmt.Errorf("failed to commit batch delete: %w", err) | ||
| } | ||
| } | ||
|
|
||
| return count, nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| module github.com/evstack/ev-node/tools/db-cleaner | ||
|
|
||
| go 1.25.6 | ||
|
|
||
| require ( | ||
| github.com/evstack/ev-node v1.0.0-rc.3 | ||
| github.com/ipfs/go-datastore v0.9.0 | ||
| github.com/spf13/cobra v1.10.2 | ||
| ) | ||
|
|
||
| require ( | ||
| connectrpc.com/connect v1.19.1 // indirect | ||
| connectrpc.com/grpcreflect v1.3.0 // indirect | ||
| github.com/armon/go-metrics v0.4.1 // indirect | ||
| github.com/benbjohnson/clock v1.3.5 // indirect | ||
| github.com/beorn7/perks v1.0.1 // indirect | ||
| github.com/boltdb/bolt v1.3.1 // indirect | ||
| github.com/celestiaorg/go-header v0.8.1 // indirect | ||
| github.com/celestiaorg/go-libp2p-messenger v0.2.2 // indirect | ||
| github.com/celestiaorg/go-square/merkle v0.0.0-20240627094109-7d01436067a3 // indirect | ||
| github.com/celestiaorg/go-square/v3 v3.0.2 // indirect | ||
| github.com/celestiaorg/nmt v0.24.2 // indirect | ||
| github.com/cenkalti/backoff/v5 v5.0.3 // indirect | ||
| github.com/cespare/xxhash/v2 v2.3.0 // indirect | ||
| github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect | ||
| github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect | ||
| github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect | ||
| github.com/dgraph-io/badger/v4 v4.5.1 // indirect | ||
| github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect | ||
| github.com/dunglas/httpsfv v1.1.0 // indirect | ||
| github.com/dustin/go-humanize v1.0.1 // indirect | ||
| github.com/evstack/ev-node/core v1.0.0-rc.1 // indirect | ||
| github.com/fatih/color v1.18.0 // indirect | ||
| github.com/filecoin-project/go-clock v0.1.0 // indirect | ||
| github.com/filecoin-project/go-jsonrpc v0.10.0 // indirect | ||
| github.com/flynn/noise v1.1.0 // indirect | ||
| github.com/fsnotify/fsnotify v1.9.0 // indirect | ||
| github.com/go-kit/kit v0.13.0 // indirect | ||
| github.com/go-logr/logr v1.4.3 // indirect | ||
| github.com/go-logr/stdr v1.2.2 // indirect | ||
| github.com/go-viper/mapstructure/v2 v2.4.0 // indirect | ||
| github.com/goccy/go-yaml v1.19.2 // indirect | ||
| github.com/gogo/protobuf v1.3.2 // indirect | ||
| github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect | ||
| github.com/google/flatbuffers v25.1.24+incompatible // indirect | ||
| github.com/google/gopacket v1.1.19 // indirect | ||
| github.com/google/uuid v1.6.0 // indirect | ||
| github.com/gorilla/websocket v1.5.3 // indirect | ||
| github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect | ||
| github.com/hashicorp/go-hclog v1.6.2 // indirect | ||
| github.com/hashicorp/go-immutable-radix v1.3.1 // indirect | ||
| github.com/hashicorp/go-metrics v0.5.4 // indirect | ||
| github.com/hashicorp/go-msgpack v0.5.5 // indirect | ||
| github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect | ||
| github.com/hashicorp/golang-lru v1.0.2 // indirect | ||
| github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect | ||
| github.com/hashicorp/raft v1.7.3 // indirect | ||
| github.com/hashicorp/raft-boltdb v0.0.0-20251103221153-05f9dd7a5148 // indirect | ||
| github.com/huin/goupnp v1.3.0 // indirect | ||
| github.com/inconshreveable/mousetrap v1.1.0 // indirect | ||
| github.com/ipfs/boxo v0.35.2 // indirect | ||
| github.com/ipfs/go-cid v0.6.0 // indirect | ||
| github.com/ipfs/go-ds-badger4 v0.1.8 // indirect | ||
| github.com/ipfs/go-log/v2 v2.9.1 // indirect | ||
| github.com/ipld/go-ipld-prime v0.21.0 // indirect | ||
| github.com/jackpal/go-nat-pmp v1.0.2 // indirect | ||
| github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect | ||
| github.com/klauspost/compress v1.18.0 // indirect | ||
| github.com/klauspost/cpuid/v2 v2.3.0 // indirect | ||
| github.com/koron/go-ssdp v0.0.6 // indirect | ||
| github.com/libp2p/go-buffer-pool v0.1.0 // indirect | ||
| github.com/libp2p/go-cidranger v1.1.0 // indirect | ||
| github.com/libp2p/go-flow-metrics v0.3.0 // indirect | ||
| github.com/libp2p/go-libp2p v0.47.0 // indirect | ||
| github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect | ||
| github.com/libp2p/go-libp2p-kad-dht v0.37.1 // indirect | ||
| github.com/libp2p/go-libp2p-kbucket v0.8.0 // indirect | ||
| github.com/libp2p/go-libp2p-pubsub v0.15.0 // indirect | ||
| github.com/libp2p/go-libp2p-record v0.3.1 // indirect | ||
| github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect | ||
| github.com/libp2p/go-msgio v0.3.0 // indirect | ||
| github.com/libp2p/go-netroute v0.4.0 // indirect | ||
| github.com/libp2p/go-reuseport v0.4.0 // indirect | ||
| github.com/libp2p/go-yamux/v5 v5.0.1 // indirect | ||
| github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect | ||
| github.com/mattn/go-colorable v0.1.14 // indirect | ||
| github.com/mattn/go-isatty v0.0.20 // indirect | ||
| github.com/miekg/dns v1.1.68 // indirect | ||
| github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect | ||
| github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect | ||
| github.com/minio/sha256-simd v1.0.1 // indirect | ||
| github.com/mitchellh/mapstructure v1.5.0 // indirect | ||
| github.com/mr-tron/base58 v1.2.0 // indirect | ||
| github.com/multiformats/go-base32 v0.1.0 // indirect | ||
| github.com/multiformats/go-base36 v0.2.0 // indirect | ||
| github.com/multiformats/go-multiaddr v0.16.1 // indirect | ||
| github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect | ||
| github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect | ||
| github.com/multiformats/go-multibase v0.2.0 // indirect | ||
| github.com/multiformats/go-multicodec v0.10.0 // indirect | ||
| github.com/multiformats/go-multihash v0.2.3 // indirect | ||
| github.com/multiformats/go-multistream v0.6.1 // indirect | ||
| github.com/multiformats/go-varint v0.1.0 // indirect | ||
| github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect | ||
| github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect | ||
| github.com/pelletier/go-toml/v2 v2.2.4 // indirect | ||
| github.com/pion/datachannel v1.5.10 // indirect | ||
| github.com/pion/dtls/v2 v2.2.12 // indirect | ||
| github.com/pion/dtls/v3 v3.0.6 // indirect | ||
| github.com/pion/ice/v4 v4.0.10 // indirect | ||
| github.com/pion/interceptor v0.1.40 // indirect | ||
| github.com/pion/logging v0.2.3 // indirect | ||
| github.com/pion/mdns/v2 v2.0.7 // indirect | ||
| github.com/pion/randutil v0.1.0 // indirect | ||
| github.com/pion/rtcp v1.2.15 // indirect | ||
| github.com/pion/rtp v1.8.19 // indirect | ||
| github.com/pion/sctp v1.8.39 // indirect | ||
| github.com/pion/sdp/v3 v3.0.13 // indirect | ||
| github.com/pion/srtp/v3 v3.0.6 // indirect | ||
| github.com/pion/stun v0.6.1 // indirect | ||
| github.com/pion/stun/v3 v3.0.0 // indirect | ||
| github.com/pion/transport/v2 v2.2.10 // indirect | ||
| github.com/pion/transport/v3 v3.0.7 // indirect | ||
| github.com/pion/turn/v4 v4.0.2 // indirect | ||
| github.com/pion/webrtc/v4 v4.1.2 // indirect | ||
| github.com/pkg/errors v0.9.1 // indirect | ||
| github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect | ||
| github.com/polydawn/refmt v0.89.0 // indirect | ||
| github.com/prometheus/client_golang v1.23.2 // indirect | ||
| github.com/prometheus/client_model v0.6.2 // indirect | ||
| github.com/prometheus/common v0.66.1 // indirect | ||
| github.com/prometheus/procfs v0.17.0 // indirect | ||
| github.com/quic-go/qpack v0.6.0 // indirect | ||
| github.com/quic-go/quic-go v0.59.0 // indirect | ||
| github.com/quic-go/webtransport-go v0.10.0 // indirect | ||
| github.com/rs/zerolog v1.34.0 // indirect | ||
| github.com/sagikazarmark/locafero v0.11.0 // indirect | ||
| github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect | ||
| github.com/spaolacci/murmur3 v1.1.0 // indirect | ||
| github.com/spf13/afero v1.15.0 // indirect | ||
| github.com/spf13/cast v1.10.0 // indirect | ||
| github.com/spf13/pflag v1.0.10 // indirect | ||
| github.com/spf13/viper v1.21.0 // indirect | ||
| github.com/stretchr/objx v0.5.2 // indirect | ||
| github.com/stretchr/testify v1.11.1 // indirect | ||
| github.com/subosito/gotenv v1.6.0 // indirect | ||
| github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect | ||
| github.com/wlynxg/anet v0.0.5 // indirect | ||
| go.opencensus.io v0.24.0 // indirect | ||
| go.opentelemetry.io/auto/sdk v1.2.1 // indirect | ||
| go.opentelemetry.io/otel v1.40.0 // indirect | ||
| go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect | ||
| go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect | ||
| go.opentelemetry.io/otel/metric v1.40.0 // indirect | ||
| go.opentelemetry.io/otel/sdk v1.40.0 // indirect | ||
| go.opentelemetry.io/otel/trace v1.40.0 // indirect | ||
| go.opentelemetry.io/proto/otlp v1.9.0 // indirect | ||
| go.uber.org/dig v1.19.0 // indirect | ||
| go.uber.org/fx v1.24.0 // indirect | ||
| go.uber.org/mock v0.5.2 // indirect | ||
| go.uber.org/multierr v1.11.0 // indirect | ||
| go.uber.org/zap v1.27.1 // indirect | ||
| go.yaml.in/yaml/v2 v2.4.3 // indirect | ||
| go.yaml.in/yaml/v3 v3.0.4 // indirect | ||
| golang.org/x/crypto v0.47.0 // indirect | ||
| golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect | ||
| golang.org/x/mod v0.32.0 // indirect | ||
| golang.org/x/net v0.49.0 // indirect | ||
| golang.org/x/sync v0.19.0 // indirect | ||
| golang.org/x/sys v0.40.0 // indirect | ||
| golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect | ||
| golang.org/x/text v0.33.0 // indirect | ||
| golang.org/x/time v0.12.0 // indirect | ||
| golang.org/x/tools v0.41.0 // indirect | ||
| golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect | ||
| gonum.org/v1/gonum v0.17.0 // indirect | ||
| google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect | ||
| google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect | ||
| google.golang.org/grpc v1.78.0 // indirect | ||
| google.golang.org/protobuf v1.36.11 // indirect | ||
| gopkg.in/yaml.v3 v3.0.1 // indirect | ||
| lukechampine.com/blake3 v1.4.1 // indirect | ||
| ) |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The variable
evmDbNameis not defined in this file or package, which will cause a compilation error. It should be defined, likely as a constant, representing the name of the EVM database.