Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Export distribution versions #2014

Open
wants to merge 50 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
02f5d61
Export Distribution Only
Kbhat1 Dec 26, 2024
9e47cc5
Migrate distribution tool
Kbhat1 Dec 27, 2024
894b9cb
Check read
Kbhat1 Dec 27, 2024
f13a6d4
More logs
Kbhat1 Dec 27, 2024
d68dd2b
Timestamps
Kbhat1 Dec 27, 2024
b64e9c2
prefix db
Kbhat1 Dec 27, 2024
5dba22a
Fix call
Kbhat1 Dec 27, 2024
705b461
concurrent goroutines
Kbhat1 Dec 27, 2024
cd3bef5
Concurrent export
Kbhat1 Dec 30, 2024
e305148
Update to seidb migration
Kbhat1 Dec 31, 2024
8e51c44
Test
Kbhat1 Dec 31, 2024
a5e47ba
More dense logging
Kbhat1 Dec 31, 2024
36004b5
More dense
Kbhat1 Dec 31, 2024
0dd34fa
Update seidb
Kbhat1 Dec 31, 2024
929e575
More logging
Kbhat1 Dec 31, 2024
9faf21a
Remove log
Kbhat1 Dec 31, 2024
6000bd2
Every million log
Kbhat1 Dec 31, 2024
f47a2af
Update
Kbhat1 Dec 31, 2024
ad12d17
Correct home dir
Kbhat1 Dec 31, 2024
2ecf3e9
Update to write
Kbhat1 Dec 31, 2024
5db3504
Update whole distribution
Kbhat1 Jan 5, 2025
373d424
Verify distribution
Kbhat1 Jan 8, 2025
9c90cb3
Update to set on each mismatch
Kbhat1 Jan 8, 2025
94b196e
Only update specific versions
Kbhat1 Jan 9, 2025
ebe673b
new catchup
Kbhat1 Jan 9, 2025
4b99f79
Raw iterate versions
Kbhat1 Jan 9, 2025
60145e8
update version
Kbhat1 Jan 9, 2025
2d36b02
Update
Kbhat1 Jan 9, 2025
2cbe605
Update seidb
Kbhat1 Jan 9, 2025
e5e4508
more logging
Kbhat1 Jan 9, 2025
73bfe13
Bump seidb
Kbhat1 Jan 9, 2025
b0df739
Update
Kbhat1 Jan 9, 2025
6d0b6c5
Update
Kbhat1 Jan 9, 2025
c9c89da
Update log
Kbhat1 Jan 9, 2025
45c9bbb
Less logs
Kbhat1 Jan 9, 2025
8fbf0de
totalExported
Kbhat1 Jan 9, 2025
49f5e9e
remove unncessary increment
Kbhat1 Jan 9, 2025
80be213
tools
Kbhat1 Jan 10, 2025
ee265c0
Update sei-db
Kbhat1 Jan 10, 2025
19ca4a1
Update verification
Kbhat1 Jan 10, 2025
10df0f2
Remove logs
Kbhat1 Jan 10, 2025
05c6504
Panic
Kbhat1 Jan 10, 2025
fb04412
Skip more
Kbhat1 Jan 10, 2025
e7a8c7e
Sync mode
Kbhat1 Jan 10, 2025
98f2ba7
Update
Kbhat1 Jan 10, 2025
e4433d1
Verify once more
Kbhat1 Jan 12, 2025
d0a884c
Verification
Kbhat1 Jan 12, 2025
35b7421
Set
Kbhat1 Jan 13, 2025
bc56bc7
Update new counter
Kbhat1 Jan 13, 2025
87c2926
Verify
Kbhat1 Jan 13, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions app/test_state_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ func (s *InMemoryStateStore) Has(storeKey string, version int64, key []byte) (bo
return ok, nil
}

func (db *InMemoryStateStore) Set(storeKey string, key, value []byte, version int64) error {
return nil
}

func (s *InMemoryStateStore) Iterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) {
s.mu.RLock()
defer s.mu.RUnlock()
Expand Down
2 changes: 1 addition & 1 deletion cmd/seid/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ func newApp(
homeDir := cast.ToString(appOpts.Get(flags.FlagHome))
stateStore := app.GetStateStore()
migrationHeight := cast.ToInt64(appOpts.Get("migrate-height"))
migrator := ss.NewMigrator(db, stateStore)
migrator := ss.NewMigrator(db, stateStore, stateStore)
if err := migrator.Migrate(migrationHeight, homeDir); err != nil {
panic(err)
}
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ require (
github.com/rs/cors v1.8.2
github.com/rs/zerolog v1.30.0
github.com/sei-protocol/goutils v0.0.2
github.com/sei-protocol/sei-db v0.0.27-0.20240123064153-d6dfa112e760
github.com/sei-protocol/sei-db v0.0.47-0.20241231152822-f02af9285b76
github.com/sirkon/goproxy v1.4.8
github.com/spf13/cast v1.5.0
github.com/spf13/cobra v1.6.1
Expand Down Expand Up @@ -352,7 +352,7 @@ replace (
github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.3
github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-9.0.20241224143343-21ee50facc96
github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.46
github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.47-0.20250110223130-1e7403e5d572
// Latest goleveldb is broken, we have to stick to this version
github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.4.3
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1347,8 +1347,8 @@ github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQp
github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
github.com/sei-protocol/sei-cosmos v0.3.48 h1:kSDweeTaLZ4TByLqAD6/hmtgAhAJHwXU1beyqsVXJkQ=
github.com/sei-protocol/sei-cosmos v0.3.48/go.mod h1:XC417pB6NwxP/cQ2XTSZLzVnP8dMZ//4uCXS3SxFgoM=
github.com/sei-protocol/sei-db v0.0.46 h1:naXfSp1I3UgJJm/iSvXpdFzr9nofEOxp/EekcAVj7wY=
github.com/sei-protocol/sei-db v0.0.46/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA=
github.com/sei-protocol/sei-db v0.0.47-0.20250110223130-1e7403e5d572 h1:99T2xbVAidlaDxQhX+XjQo5ttMciuXUfKeRAMMGShLc=
github.com/sei-protocol/sei-db v0.0.47-0.20250110223130-1e7403e5d572/go.mod h1:m5g7p0QeAS3dNJHIl28zQpzOgxQmvYqPb7t4hwgIOCA=
github.com/sei-protocol/sei-iavl v0.2.0 h1:OisPjXiDT+oe+aeckzDEFgkZCYuUjHgs/PP8DPicN+I=
github.com/sei-protocol/sei-iavl v0.2.0/go.mod h1:qRf8QYUPfrAO7K6VDB2B2l/N7K5L76OorioGBcJBIbw=
github.com/sei-protocol/sei-ibc-go/v3 v3.3.3 h1:0kg1giMHiKMLXOCFLqH/9kqdl5bH+unzI8qRF6qwXzw=
Expand Down
1 change: 1 addition & 0 deletions tools/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ func ToolCmd() *cobra.Command {
}
toolsCmd.AddCommand(scanner.ScanCmd())
toolsCmd.AddCommand(migration.MigrateCmd())
toolsCmd.AddCommand(migration.MigrateSSCmd())
toolsCmd.AddCommand(migration.VerifyMigrationCmd())
toolsCmd.AddCommand(migration.GenerateStats())
return toolsCmd
Expand Down
48 changes: 46 additions & 2 deletions tools/migration/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func MigrateCmd() *cobra.Command {
Short: "A tool to migrate full IAVL data store to SeiDB. Use this tool to migrate IAVL to SeiDB SC and SS database.",
Run: execute,
}
cmd.PersistentFlags().String("home-dir", "/root/.sei", "Sei home directory")
cmd.PersistentFlags().String("home-dir", "/root/.old_sei", "Sei home directory")
return cmd
}

Expand All @@ -48,6 +48,50 @@ func migrateSC(version int64, homeDir string, db dbm.DB) error {
return migrator.Migrate(version)
}

func MigrateSSCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "migrate-ss",
Short: "A tool to migrate full ss distribution module",
Run: executeSS,
}
cmd.PersistentFlags().String("home-dir", "/root/.old_sei", "Sei home directory")
return cmd
}

func executeSS(cmd *cobra.Command, _ []string) {
homeDir, _ := cmd.Flags().GetString("home-dir")
dataDir := filepath.Join(homeDir, "data")
db, err := dbm.NewGoLevelDB("application", dataDir)
if err != nil {
panic(err)
}
latestVersion := rootmulti.GetLatestVersion(db)
fmt.Printf("latest version: %d\n", latestVersion)

if err = migrateSS(latestVersion, homeDir, db); err != nil {
panic(err)
}
}

func migrateSS(version int64, homeDir string, db dbm.DB) error {
ssConfig := config.DefaultStateStoreConfig()
ssConfig.Enable = true
ssConfig.KeepRecent = 0

stateStore, err := sstypes.NewStateStore(log.NewNopLogger(), homeDir, ssConfig)
if err != nil {
return err
}

oldStateStore, err := sstypes.NewStateStore(log.NewNopLogger(), "/root/.sei", ssConfig)
if err != nil {
return err
}

migrator := ss.NewMigrator(db, stateStore, oldStateStore)
return migrator.Migrate(version, homeDir)
}

func VerifyMigrationCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "verify-migration",
Expand Down Expand Up @@ -95,7 +139,7 @@ func verifySS(version int64, homeDir string, db dbm.DB) error {
return err
}

migrator := ss.NewMigrator(db, stateStore)
migrator := ss.NewMigrator(db, stateStore, stateStore)
return migrator.Verify(version)
}

Expand Down
101 changes: 71 additions & 30 deletions tools/migration/ss/migrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,64 +13,105 @@ import (
)

type Migrator struct {
iavlDB dbm.DB
stateStore types.StateStore
iavlDB dbm.DB
stateStore types.StateStore
oldStateStore types.StateStore
}

// TODO: make this configurable?
const (
DefaultCacheSize int = 10000
)

func NewMigrator(db dbm.DB, stateStore types.StateStore) *Migrator {
func NewMigrator(db dbm.DB, stateStore types.StateStore, oldStateStore types.StateStore) *Migrator {
return &Migrator{
iavlDB: db,
stateStore: stateStore,
iavlDB: db,
stateStore: stateStore,
oldStateStore: oldStateStore,
}
}

func (m *Migrator) Migrate(version int64, homeDir string) error {
ch := make(chan types.RawSnapshotNode, 1000)
// Channel to send RawSnapshotNodes to the importer.
ch := make(chan types.RawSnapshotNode, 10000)
// Channel to capture errors from both goroutines below.
errCh := make(chan error, 2)

// Get the latest key, if any, to resume from
latestKey, err := m.stateStore.GetLatestMigratedKey()
if err != nil {
return fmt.Errorf("failed to get latest key: %w", err)
}

latestModule, err := m.stateStore.GetLatestMigratedModule()
if err != nil {
return fmt.Errorf("failed to get latest module: %w", err)
}
fmt.Printf("Starting migration for 'distribution' module from version\n")

fmt.Println("Starting migration...")

// Goroutine to iterate through IAVL and export leaf nodes
// Goroutine #1: Export distribution leaf nodes into ch
go func() {
defer close(ch)
errCh <- ExportLeafNodesFromKey(m.iavlDB, ch, latestKey, latestModule)
errCh <- exportDistributionLeafNodes(m.oldStateStore, m.stateStore, ch)
}()

// Import nodes into PebbleDB
go func() {
errCh <- m.stateStore.RawImport(ch)
}()
// go func() {
// errCh <- m.stateStore.RawImport(ch)
// }()

// Block until both processes complete
for i := 0; i < 2; i++ {
// Wait for both goroutines to complete
for i := 0; i < 1; i++ {
if err := <-errCh; err != nil {
return err
}
}

// Set earliest and latest version in the database
err = m.stateStore.SetEarliestVersion(1, true)
return nil
}

func exportDistributionLeafNodes(
oldStateStore types.StateStore,
newStateStore types.StateStore,
ch chan<- types.RawSnapshotNode,
) error {

var totalExported int
startTime := time.Now()
Fixed Show fixed Hide fixed

Check warning

Code scanning / CodeQL

Calling the system time Warning

Calling the system time may be a possible source of non-determinism

// RawIterate will scan *all* keys in the "distribution" store.
// We'll filter them by version in the callback.
// var misMatch int
stop, err := oldStateStore.RawIterate("distribution", func(key, value []byte, version int64) bool {
totalExported++
valBz, err := newStateStore.Get("distribution", version, key)
if err != nil {
panic(err)
}
if value != nil && valBz != nil && !bytes.Equal(valBz, value) {
// misMatch++
panic(fmt.Errorf("Value mismatch for key %s: expected %s, got %s\n", string(key), string(value), string(valBz)))
}
// ch <- types.RawSnapshotNode{
// StoreKey: "distribution",
// Key: key,
// Value: value,
// Version: version,
// }

// Optional progress logging every 1,000,000 keys:
if totalExported%1_000_000 == 0 {
fmt.Printf("[SingleWorker][%s] Verified %d distribution keys so far\n",
time.Now().Format(time.RFC3339), totalExported,

Check warning

Code scanning / CodeQL

Calling the system time Warning

Calling the system time may be a possible source of non-determinism
)
}
// Return false to continue iterating
return false
})
if err != nil {
return err
return fmt.Errorf("RawIterate error: %w", err)
}
if stop {
fmt.Printf("[SingleWorker][%s] Iteration stopped early; callback returned true at some point.\n",
time.Now().Format(time.RFC3339),

Check warning

Code scanning / CodeQL

Calling the system time Warning

Calling the system time may be a possible source of non-determinism
)
}

return m.stateStore.SetLatestVersion(version)
fmt.Printf(
"[%s] Completed exporting distribution store for versions. Total keys: %d. Duration: %s\n",
time.Now().Format(time.RFC3339), totalExported, time.Since(startTime),
)
fmt.Printf("Finished export at %s\n", time.Now().Format(time.RFC3339))

Check warning

Code scanning / CodeQL

Calling the system time Warning

Calling the system time may be a possible source of non-determinism
return nil
}

func (m *Migrator) Verify(version int64) error {
Expand Down