diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index b4f6a175c7..5b334e4667 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -121,7 +121,7 @@ var freezers = []string{ // state freezer. func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) { if ancientDir == "" { - return NewMemoryFreezer(readOnly, stateFreezerTableConfigs), nil + return NewMemoryFreezer(readOnly, 0, stateFreezerTableConfigs), nil } var name string if verkle { @@ -140,7 +140,7 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset // trienode freezer. func NewTrienodeFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) { if ancientDir == "" { - return NewMemoryFreezer(readOnly, trienodeFreezerTableConfigs), nil + return NewMemoryFreezer(readOnly, 0, trienodeFreezerTableConfigs), nil } var name string if verkle { diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 0ff9c59635..dad20f3ed7 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -69,7 +69,7 @@ func NewChainFreezer(datadir string, namespace string, readonly bool, offset uin func newChainFreezer(datadir string, eraDir string, namespace string, readonly bool, offset uint64) (*chainFreezer, error) { if datadir == "" { return &chainFreezer{ - ancients: NewMemoryFreezer(readonly, chainFreezerTableConfigs), + ancients: NewMemoryFreezer(readonly, offset, chainFreezerTableConfigs), quit: make(chan struct{}), trigger: make(chan chan struct{}), }, nil diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 96740d0c6e..eeaab2df77 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -29,6 +29,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +var errTruncateBelowOffset = errors.New("truncation below offset") + // memoryTable is used to store a list of sequential items in memory. type memoryTable struct { items uint64 // Number of stored items in the table, including the deleted ones @@ -149,7 +151,7 @@ func (b *memoryBatch) reset(freezer *MemoryFreezer) { b.size = make(map[string]int64) for name, table := range freezer.tables { - b.next[name] = table.items + b.next[name] = table.items + freezer.offset.Load() } } @@ -215,16 +217,24 @@ type MemoryFreezer struct { } // NewMemoryFreezer initializes an in-memory freezer instance. -func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer { +func NewMemoryFreezer(readonly bool, offset uint64, tableName map[string]freezerTableConfig) *MemoryFreezer { tables := make(map[string]*memoryTable) for name, cfg := range tableName { tables[name] = newMemoryTable(name, cfg) } - return &MemoryFreezer{ + freezer := &MemoryFreezer{ writeBatch: newMemoryBatch(), readonly: readonly, tables: tables, } + + freezer.offset.Store(offset) + + // Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to + // represent the absolute number of blocks already frozen. + freezer.items += offset + + return freezer } // todo: @anshalshukla || @manav2401 - Check if implementation is required @@ -240,7 +250,11 @@ func (f *MemoryFreezer) ItemAmountInAncient() (uint64, error) { f.lock.RLock() defer f.lock.RUnlock() - return f.items - f.offset.Load(), nil + offset := f.offset.Load() + if f.items < offset { + return 0, nil + } + return f.items - offset, nil } // Ancient retrieves an ancient binary blob from the in-memory freezer. @@ -252,7 +266,11 @@ func (f *MemoryFreezer) Ancient(kind string, number uint64) ([]byte, error) { if t == nil { return nil, errUnknownTable } - data, err := t.retrieve(number, 1, 0) + offset := f.offset.Load() + if number < offset { + return nil, errOutOfBounds + } + data, err := t.retrieve(number-offset, 1, 0) if err != nil { return nil, err } @@ -273,7 +291,11 @@ func (f *MemoryFreezer) AncientRange(kind string, start, count, maxBytes uint64) if t == nil { return nil, errUnknownTable } - return t.retrieve(start, count, maxBytes) + offset := f.offset.Load() + if start < offset { + return nil, errOutOfBounds + } + return t.retrieve(start-offset, count, maxBytes) } // Ancients returns the ancient item numbers in the freezer. @@ -322,18 +344,20 @@ func (f *MemoryFreezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (wri return 0, errReadOnly } // Roll back all tables to the starting position in case of error. + old := f.items + offset := f.offset.Load() defer func(old uint64) { if err == nil { return } // The write operation has failed. Go back to the previous item position. for name, table := range f.tables { - err := table.truncateHead(old) + err := table.truncateHead(old - offset) if err != nil { log.Error("Freezer table roll-back failed", "table", name, "index", old, "err", err) } } - }(f.items) + }(old) // Modify the ancients in batch. f.writeBatch.reset(f) @@ -361,8 +385,12 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) { if old <= items { return old, nil } + offset := f.offset.Load() + if items < offset { + return 0, errTruncateBelowOffset + } for _, table := range f.tables { - if err := table.truncateHead(items); err != nil { + if err := table.truncateHead(items - offset); err != nil { return 0, err } } @@ -384,9 +412,13 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { if old >= tail { return old, nil } + offset := f.offset.Load() + if tail < offset { + return 0, errTruncateBelowOffset + } for _, table := range f.tables { if table.config.prunable { - if err := table.truncateTail(tail); err != nil { + if err := table.truncateTail(tail - offset); err != nil { return 0, err } } @@ -422,7 +454,7 @@ func (f *MemoryFreezer) Reset() error { tables[name] = newMemoryTable(name, table.config) } f.tables = tables - f.items, f.tail = 0, 0 + f.items, f.tail = f.offset.Load(), 0 return nil } diff --git a/core/rawdb/freezer_memory_test.go b/core/rawdb/freezer_memory_test.go index 4bd31d8027..8fb62ca250 100644 --- a/core/rawdb/freezer_memory_test.go +++ b/core/rawdb/freezer_memory_test.go @@ -17,6 +17,8 @@ package rawdb import ( + "bytes" + "errors" "testing" "github.com/ethereum/go-ethereum/core/rawdb/ancienttest" @@ -32,7 +34,7 @@ func TestMemoryFreezer(t *testing.T) { prunable: true, } } - return NewMemoryFreezer(false, tables) + return NewMemoryFreezer(false, 0, tables) }) ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore { tables := make(map[string]freezerTableConfig) @@ -42,6 +44,131 @@ func TestMemoryFreezer(t *testing.T) { prunable: true, } } - return NewMemoryFreezer(false, tables) + return NewMemoryFreezer(false, 0, tables) }) } + +func TestMemoryFreezerOffset(t *testing.T) { + freezer := NewMemoryFreezer(false, 10, map[string]freezerTableConfig{ + "test": {noSnappy: true, prunable: true}, + }) + if got := freezer.AncientOffSet(); got != 10 { + t.Fatalf("AncientOffSet() = %d, want 10", got) + } + if got, err := freezer.Ancients(); err != nil { + t.Fatalf("Ancients() returned error: %v", err) + } else if got != 10 { + t.Fatalf("Ancients() = %d, want 10", got) + } + if got, err := freezer.ItemAmountInAncient(); err != nil { + t.Fatalf("ItemAmountInAncient() returned error: %v", err) + } else if got != 0 { + t.Fatalf("ItemAmountInAncient() = %d, want 0", got) + } + + if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error { + return op.AppendRaw("test", 0, []byte("zero")) + }); !errors.Is(err, errOutOrderInsertion) { + t.Fatalf("AppendRaw before offset error = %v, want %v", err, errOutOrderInsertion) + } + + if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error { + if err := op.AppendRaw("test", 10, []byte("ten")); err != nil { + return err + } + if err := op.AppendRaw("test", 11, []byte("eleven")); err != nil { + return err + } + return op.AppendRaw("test", 12, []byte("twelve")) + }); err != nil { + t.Fatalf("ModifyAncients() returned error: %v", err) + } + + if got, err := freezer.Ancients(); err != nil { + t.Fatalf("Ancients() returned error: %v", err) + } else if got != 13 { + t.Fatalf("Ancients() = %d, want 13", got) + } + if got, err := freezer.ItemAmountInAncient(); err != nil { + t.Fatalf("ItemAmountInAncient() returned error: %v", err) + } else if got != 3 { + t.Fatalf("ItemAmountInAncient() = %d, want 3", got) + } + + if blob, err := freezer.Ancient("test", 10); err != nil { + t.Fatalf("Ancient(10) returned error: %v", err) + } else if !bytes.Equal(blob, []byte("ten")) { + t.Fatalf("Ancient(10) = %q, want %q", blob, []byte("ten")) + } + if _, err := freezer.Ancient("test", 9); !errors.Is(err, errOutOfBounds) { + t.Fatalf("Ancient(9) error = %v, want %v", err, errOutOfBounds) + } + + if batch, err := freezer.AncientRange("test", 10, 2, 0); err != nil { + t.Fatalf("AncientRange(10, 2, 0) returned error: %v", err) + } else if len(batch) != 2 || !bytes.Equal(batch[0], []byte("ten")) || !bytes.Equal(batch[1], []byte("eleven")) { + t.Fatalf("AncientRange(10, 2, 0) = %q, want %q", batch, [][]byte{[]byte("ten"), []byte("eleven")}) + } + if _, err := freezer.AncientRange("test", 9, 1, 0); !errors.Is(err, errOutOfBounds) { + t.Fatalf("AncientRange(9, 1, 0) error = %v, want %v", err, errOutOfBounds) + } + + if _, err := freezer.TruncateHead(9); !errors.Is(err, errTruncateBelowOffset) { + t.Fatalf("TruncateHead(9) error = %v, want %v", err, errTruncateBelowOffset) + } + if _, err := freezer.TruncateTail(9); !errors.Is(err, errTruncateBelowOffset) { + t.Fatalf("TruncateTail(9) error = %v, want %v", err, errTruncateBelowOffset) + } + + if old, err := freezer.TruncateTail(11); err != nil { + t.Fatalf("TruncateTail(11) returned error: %v", err) + } else if old != 0 { + t.Fatalf("TruncateTail(11) old tail = %d, want 0", old) + } + if blob, err := freezer.Ancient("test", 11); err != nil { + t.Fatalf("Ancient(11) returned error: %v", err) + } else if !bytes.Equal(blob, []byte("eleven")) { + t.Fatalf("Ancient(11) = %q, want %q", blob, []byte("eleven")) + } + if _, err := freezer.Ancient("test", 10); !errors.Is(err, errOutOfBounds) { + t.Fatalf("Ancient(10) after tail truncation error = %v, want %v", err, errOutOfBounds) + } + + if old, err := freezer.TruncateHead(12); err != nil { + t.Fatalf("TruncateHead(12) returned error: %v", err) + } else if old != 13 { + t.Fatalf("TruncateHead(12) old head = %d, want 13", old) + } + if got, err := freezer.Ancients(); err != nil { + t.Fatalf("Ancients() after head truncation returned error: %v", err) + } else if got != 12 { + t.Fatalf("Ancients() after head truncation = %d, want 12", got) + } + if _, err := freezer.Ancient("test", 12); !errors.Is(err, errOutOfBounds) { + t.Fatalf("Ancient(12) after head truncation error = %v, want %v", err, errOutOfBounds) + } +} + +func TestMemoryFreezerResetKeepsOffset(t *testing.T) { + freezer := NewMemoryFreezer(false, 7, map[string]freezerTableConfig{ + "test": {noSnappy: true, prunable: true}, + }) + if err := freezer.Reset(); err != nil { + t.Fatalf("Reset() returned error: %v", err) + } + if got, err := freezer.Ancients(); err != nil { + t.Fatalf("Ancients() returned error: %v", err) + } else if got != 7 { + t.Fatalf("Ancients() = %d, want 7", got) + } + if got, err := freezer.ItemAmountInAncient(); err != nil { + t.Fatalf("ItemAmountInAncient() returned error: %v", err) + } else if got != 0 { + t.Fatalf("ItemAmountInAncient() = %d, want 0", got) + } + if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error { + return op.AppendRaw("test", 7, []byte("seven")) + }); err != nil { + t.Fatalf("ModifyAncients() after reset returned error: %v", err) + } +}