Add offset support to MemoryFreezer

core/rawdb/ancient_scheme.go

@@ -121,7 +121,7 @@ var freezers = []string{
 //     state freezer.
 func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
 	if ancientDir == "" {
-		return NewMemoryFreezer(readOnly, stateFreezerTableConfigs), nil
+		return NewMemoryFreezer(readOnly, 0, stateFreezerTableConfigs), nil
 	}
 	var name string
 	if verkle {
@@ -140,7 +140,7 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset
 //     trienode freezer.
 func NewTrienodeFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
 	if ancientDir == "" {
-		return NewMemoryFreezer(readOnly, trienodeFreezerTableConfigs), nil
+		return NewMemoryFreezer(readOnly, 0, trienodeFreezerTableConfigs), nil
 	}
 	var name string
 	if verkle {

core/rawdb/chain_freezer.go

@@ -69,7 +69,7 @@ func NewChainFreezer(datadir string, namespace string, readonly bool, offset uin
 func newChainFreezer(datadir string, eraDir string, namespace string, readonly bool, offset uint64) (*chainFreezer, error) {
 	if datadir == "" {
 		return &chainFreezer{
-			ancients: NewMemoryFreezer(readonly, chainFreezerTableConfigs),
+			ancients: NewMemoryFreezer(readonly, offset, chainFreezerTableConfigs),
 			quit:     make(chan struct{}),
 			trigger:  make(chan chan struct{}),
 		}, nil

core/rawdb/freezer_memory.go

@@ -29,6 +29,8 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
+var errTruncateBelowOffset = errors.New("truncation below offset")
+
 // memoryTable is used to store a list of sequential items in memory.
 type memoryTable struct {
 	items  uint64   // Number of stored items in the table, including the deleted ones
@@ -149,7 +151,7 @@ func (b *memoryBatch) reset(freezer *MemoryFreezer) {
 	b.size = make(map[string]int64)
 
 	for name, table := range freezer.tables {
-		b.next[name] = table.items
+		b.next[name] = table.items + freezer.offset.Load()
 	}
 }
 
@@ -215,16 +217,24 @@ type MemoryFreezer struct {
 }
 
 // NewMemoryFreezer initializes an in-memory freezer instance.
-func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
+func NewMemoryFreezer(readonly bool, offset uint64, tableName map[string]freezerTableConfig) *MemoryFreezer {
 	tables := make(map[string]*memoryTable)
 	for name, cfg := range tableName {
 		tables[name] = newMemoryTable(name, cfg)
 	}
-	return &MemoryFreezer{
+	freezer := &MemoryFreezer{
 		writeBatch: newMemoryBatch(),
 		readonly:   readonly,
 		tables:     tables,
 	}
+
+	freezer.offset.Store(offset)
+
+	// Some blocks in ancientDB may have already been frozen and been pruned, so adding the offset to
+	// represent the absolute number of blocks already frozen.
+	freezer.items += offset
+
+	return freezer
 }
 
 // todo: @anshalshukla || @manav2401 - Check if implementation is required
@@ -240,7 +250,11 @@ func (f *MemoryFreezer) ItemAmountInAncient() (uint64, error) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
-	return f.items - f.offset.Load(), nil
+	offset := f.offset.Load()
+	if f.items < offset {
+		return 0, nil
+	}
+	return f.items - offset, nil
 }
 
 // Ancient retrieves an ancient binary blob from the in-memory freezer.
@@ -252,7 +266,11 @@ func (f *MemoryFreezer) Ancient(kind string, number uint64) ([]byte, error) {
 	if t == nil {
 		return nil, errUnknownTable
 	}
-	data, err := t.retrieve(number, 1, 0)
+	offset := f.offset.Load()
+	if number < offset {
+		return nil, errOutOfBounds
+	}
+	data, err := t.retrieve(number-offset, 1, 0)
 	if err != nil {
 		return nil, err
 	}
@@ -273,7 +291,11 @@ func (f *MemoryFreezer) AncientRange(kind string, start, count, maxBytes uint64)
 	if t == nil {
 		return nil, errUnknownTable
 	}
-	return t.retrieve(start, count, maxBytes)
+	offset := f.offset.Load()
+	if start < offset {
+		return nil, errOutOfBounds
+	}
+	return t.retrieve(start-offset, count, maxBytes)
 }
 
 // Ancients returns the ancient item numbers in the freezer.
@@ -322,18 +344,20 @@ func (f *MemoryFreezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (wri
 		return 0, errReadOnly
 	}
 	// Roll back all tables to the starting position in case of error.
+	old := f.items
+	offset := f.offset.Load()
 	defer func(old uint64) {
 		if err == nil {
 			return
 		}
 		// The write operation has failed. Go back to the previous item position.
 		for name, table := range f.tables {
-			err := table.truncateHead(old)
+			err := table.truncateHead(old - offset)
 			if err != nil {
 				log.Error("Freezer table roll-back failed", "table", name, "index", old, "err", err)
 			}
 		}
-	}(f.items)
+	}(old)
 
 	// Modify the ancients in batch.
 	f.writeBatch.reset(f)
@@ -361,8 +385,12 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) {
 	if old <= items {
 		return old, nil
 	}
+	offset := f.offset.Load()
+	if items < offset {
+		return 0, errTruncateBelowOffset
+	}
 	for _, table := range f.tables {
-		if err := table.truncateHead(items); err != nil {
+		if err := table.truncateHead(items - offset); err != nil {
 			return 0, err
 		}
 	}
@@ -384,9 +412,13 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) {
 	if old >= tail {
 		return old, nil
 	}
+	offset := f.offset.Load()
+	if tail < offset {
+		return 0, errTruncateBelowOffset
+	}
 	for _, table := range f.tables {
 		if table.config.prunable {
-			if err := table.truncateTail(tail); err != nil {
+			if err := table.truncateTail(tail - offset); err != nil {
 				return 0, err
 			}
 		}
@@ -422,7 +454,7 @@ func (f *MemoryFreezer) Reset() error {
 		tables[name] = newMemoryTable(name, table.config)
 	}
 	f.tables = tables
-	f.items, f.tail = 0, 0
+	f.items, f.tail = f.offset.Load(), 0
 	return nil
 }
 

core/rawdb/freezer_memory_test.go

@@ -17,6 +17,8 @@
 package rawdb
 
 import (
+	"bytes"
+	"errors"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/core/rawdb/ancienttest"
@@ -32,7 +34,7 @@
 				prunable: true,
 			}
 		}
-		return NewMemoryFreezer(false, tables)
+		return NewMemoryFreezer(false, 0, tables)
 	})
 	ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore {
 		tables := make(map[string]freezerTableConfig)
@@ -42,6 +44,131 @@
 				prunable: true,
 			}
 		}
-		return NewMemoryFreezer(false, tables)
+		return NewMemoryFreezer(false, 0, tables)
 	})
 }
+
+func TestMemoryFreezerOffset(t *testing.T) {
+	freezer := NewMemoryFreezer(false, 10, map[string]freezerTableConfig{
+		"test": {noSnappy: true, prunable: true},
+	})
+	if got := freezer.AncientOffSet(); got != 10 {
+		t.Fatalf("AncientOffSet() = %d, want 10", got)
+	}
+	if got, err := freezer.Ancients(); err != nil {
+		t.Fatalf("Ancients() returned error: %v", err)
+	} else if got != 10 {
+		t.Fatalf("Ancients() = %d, want 10", got)
+	}
+	if got, err := freezer.ItemAmountInAncient(); err != nil {
+		t.Fatalf("ItemAmountInAncient() returned error: %v", err)
+	} else if got != 0 {
+		t.Fatalf("ItemAmountInAncient() = %d, want 0", got)
+	}
+
+	if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		return op.AppendRaw("test", 0, []byte("zero"))
+	}); !errors.Is(err, errOutOrderInsertion) {
+		t.Fatalf("AppendRaw before offset error = %v, want %v", err, errOutOrderInsertion)
+	}
+
+	if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		if err := op.AppendRaw("test", 10, []byte("ten")); err != nil {
+			return err
+		}
+		if err := op.AppendRaw("test", 11, []byte("eleven")); err != nil {
+			return err
+		}
+		return op.AppendRaw("test", 12, []byte("twelve"))
+	}); err != nil {
+		t.Fatalf("ModifyAncients() returned error: %v", err)
+	}
+
+	if got, err := freezer.Ancients(); err != nil {
+		t.Fatalf("Ancients() returned error: %v", err)
+	} else if got != 13 {
+		t.Fatalf("Ancients() = %d, want 13", got)
+	}
+	if got, err := freezer.ItemAmountInAncient(); err != nil {
+		t.Fatalf("ItemAmountInAncient() returned error: %v", err)
+	} else if got != 3 {
+		t.Fatalf("ItemAmountInAncient() = %d, want 3", got)
+	}
+
+	if blob, err := freezer.Ancient("test", 10); err != nil {
+		t.Fatalf("Ancient(10) returned error: %v", err)
+	} else if !bytes.Equal(blob, []byte("ten")) {
+		t.Fatalf("Ancient(10) = %q, want %q", blob, []byte("ten"))
+	}
+	if _, err := freezer.Ancient("test", 9); !errors.Is(err, errOutOfBounds) {
+		t.Fatalf("Ancient(9) error = %v, want %v", err, errOutOfBounds)
+	}
+
+	if batch, err := freezer.AncientRange("test", 10, 2, 0); err != nil {
+		t.Fatalf("AncientRange(10, 2, 0) returned error: %v", err)
+	} else if len(batch) != 2 || !bytes.Equal(batch[0], []byte("ten")) || !bytes.Equal(batch[1], []byte("eleven")) {
+		t.Fatalf("AncientRange(10, 2, 0) = %q, want %q", batch, [][]byte{[]byte("ten"), []byte("eleven")})
+	}
+	if _, err := freezer.AncientRange("test", 9, 1, 0); !errors.Is(err, errOutOfBounds) {
+		t.Fatalf("AncientRange(9, 1, 0) error = %v, want %v", err, errOutOfBounds)
+	}
+
+	if _, err := freezer.TruncateHead(9); !errors.Is(err, errTruncateBelowOffset) {
+		t.Fatalf("TruncateHead(9) error = %v, want %v", err, errTruncateBelowOffset)
+	}
+	if _, err := freezer.TruncateTail(9); !errors.Is(err, errTruncateBelowOffset) {
+		t.Fatalf("TruncateTail(9) error = %v, want %v", err, errTruncateBelowOffset)
+	}
+
+	if old, err := freezer.TruncateTail(11); err != nil {
+		t.Fatalf("TruncateTail(11) returned error: %v", err)
+	} else if old != 0 {
+		t.Fatalf("TruncateTail(11) old tail = %d, want 0", old)
+	}
+	if blob, err := freezer.Ancient("test", 11); err != nil {
+		t.Fatalf("Ancient(11) returned error: %v", err)
+	} else if !bytes.Equal(blob, []byte("eleven")) {
+		t.Fatalf("Ancient(11) = %q, want %q", blob, []byte("eleven"))
+	}
+	if _, err := freezer.Ancient("test", 10); !errors.Is(err, errOutOfBounds) {
+		t.Fatalf("Ancient(10) after tail truncation error = %v, want %v", err, errOutOfBounds)
+	}
+
+	if old, err := freezer.TruncateHead(12); err != nil {
+		t.Fatalf("TruncateHead(12) returned error: %v", err)
+	} else if old != 13 {
+		t.Fatalf("TruncateHead(12) old head = %d, want 13", old)
+	}
+	if got, err := freezer.Ancients(); err != nil {
+		t.Fatalf("Ancients() after head truncation returned error: %v", err)
+	} else if got != 12 {
+		t.Fatalf("Ancients() after head truncation = %d, want 12", got)
+	}
+	if _, err := freezer.Ancient("test", 12); !errors.Is(err, errOutOfBounds) {
+		t.Fatalf("Ancient(12) after head truncation error = %v, want %v", err, errOutOfBounds)
+	}
+}
+
+func TestMemoryFreezerResetKeepsOffset(t *testing.T) {
+	freezer := NewMemoryFreezer(false, 7, map[string]freezerTableConfig{
+		"test": {noSnappy: true, prunable: true},
+	})
+	if err := freezer.Reset(); err != nil {
+		t.Fatalf("Reset() returned error: %v", err)
+	}
+	if got, err := freezer.Ancients(); err != nil {
+		t.Fatalf("Ancients() returned error: %v", err)
+	} else if got != 7 {
+		t.Fatalf("Ancients() = %d, want 7", got)
+	}
+	if got, err := freezer.ItemAmountInAncient(); err != nil {
+		t.Fatalf("ItemAmountInAncient() returned error: %v", err)
+	} else if got != 0 {
+		t.Fatalf("ItemAmountInAncient() = %d, want 0", got)
+	}
+	if _, err := freezer.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		return op.AppendRaw("test", 7, []byte("seven"))
+	}); err != nil {
+		t.Fatalf("ModifyAncients() after reset returned error: %v", err)
+	}
+}